PR #24445 opened by James Almer (jamrial) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24445 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24445.patch
Too many merge conflicts to attempt individual cherry-picks. >From 96f225f3474ca8012cf6d53a51455f98ccfac609 Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Sun, 14 Jun 2026 10:06:27 -0300 Subject: [PATCH 1/2] avformat/iamf_writer: reject muxing PCM streams Supporting PCM streams requires API changes present in newer releases that can't be backported, so don't create invalid files. Signed-off-by: James Almer <[email protected]> (cherry picked from commit 1034b144ff73aeae201ecbf132b3ca89b321ec16) --- libavformat/iamf_writer.c | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/libavformat/iamf_writer.c b/libavformat/iamf_writer.c index 163d7e1200..8617469aa7 100644 --- a/libavformat/iamf_writer.c +++ b/libavformat/iamf_writer.c @@ -89,12 +89,6 @@ static int populate_audio_roll_distance(IAMFCodecConfig *codec_config) codec_config->audio_roll_distance = -1; break; case AV_CODEC_ID_FLAC: - case AV_CODEC_ID_PCM_S16BE: - case AV_CODEC_ID_PCM_S24BE: - case AV_CODEC_ID_PCM_S32BE: - case AV_CODEC_ID_PCM_S16LE: - case AV_CODEC_ID_PCM_S24LE: - case AV_CODEC_ID_PCM_S32LE: codec_config->audio_roll_distance = 0; break; default: @@ -494,35 +488,12 @@ static int iamf_write_codec_config(const IAMFContext *iamf, avio_write(dyn_bc, codec_config->extradata, codec_config->extradata_size); break; case AV_CODEC_ID_PCM_S16LE: - avio_w8(dyn_bc, 1); - avio_w8(dyn_bc, 16); - avio_wb32(dyn_bc, codec_config->sample_rate); - break; case AV_CODEC_ID_PCM_S24LE: - avio_w8(dyn_bc, 1); - avio_w8(dyn_bc, 24); - avio_wb32(dyn_bc, codec_config->sample_rate); - break; case AV_CODEC_ID_PCM_S32LE: - avio_w8(dyn_bc, 1); - avio_w8(dyn_bc, 32); - avio_wb32(dyn_bc, codec_config->sample_rate); - break; case AV_CODEC_ID_PCM_S16BE: - avio_w8(dyn_bc, 0); - avio_w8(dyn_bc, 16); - avio_wb32(dyn_bc, codec_config->sample_rate); - break; case AV_CODEC_ID_PCM_S24BE: - avio_w8(dyn_bc, 0); - avio_w8(dyn_bc, 24); - avio_wb32(dyn_bc, codec_config->sample_rate); - break; case AV_CODEC_ID_PCM_S32BE: - avio_w8(dyn_bc, 0); - avio_w8(dyn_bc, 32); - avio_wb32(dyn_bc, codec_config->sample_rate); - break; + return AVERROR(ENOSYS); default: break; } -- 2.52.0 >From 17638a3a3cbd2e3f8098ee091669fa854453ef9e Mon Sep 17 00:00:00 2001 From: James Almer <[email protected]> Date: Thu, 10 Sep 2026 16:20:01 -0300 Subject: [PATCH 2/2] avformat/iamf: backport assorted fixes Signed-off-by: James Almer <[email protected]> --- libavformat/iamf_parse.c | 166 +++++++++++++++-- libavformat/iamf_reader.c | 34 +++- libavformat/iamf_writer.c | 167 ++++++++++++++---- libavformat/iamfdec.c | 2 +- tests/ref/fate/iamf-5_1-copy | 2 +- tests/ref/fate/iamf-5_1-demux | 2 +- tests/ref/fate/iamf-5_1_4 | 6 +- tests/ref/fate/iamf-7_1_4 | 6 +- tests/ref/fate/mov-mp4-iamf-5_1_4 | 6 +- tests/ref/fate/mov-mp4-iamf-7_1_4-video-first | 6 +- tests/ref/fate/mov-mp4-iamf-7_1_4-video-last | 6 +- 11 files changed, 332 insertions(+), 71 deletions(-) diff --git a/libavformat/iamf_parse.c b/libavformat/iamf_parse.c index bab266e175..667124dd69 100644 --- a/libavformat/iamf_parse.c +++ b/libavformat/iamf_parse.c @@ -347,6 +347,37 @@ static int update_extradata(AVCodecParameters *codecpar) return 0; } +static int parse_coupled_substream(AVChannelLayout *out, AVChannelLayout *in, int n) +{ + if (in->u.mask & AV_CH_LAYOUT_STEREO) { + out->u.map[n++].id = AV_CHAN_FRONT_LEFT; + out->u.map[n++].id = AV_CHAN_FRONT_RIGHT; + in->u.mask &= ~AV_CH_LAYOUT_STEREO; + } else if (in->u.mask & (AV_CH_FRONT_LEFT_OF_CENTER|AV_CH_FRONT_RIGHT_OF_CENTER)) { + out->u.map[n++].id = AV_CHAN_FRONT_LEFT_OF_CENTER; + out->u.map[n++].id = AV_CHAN_FRONT_RIGHT_OF_CENTER; + in->u.mask &= ~(AV_CH_FRONT_LEFT_OF_CENTER|AV_CH_FRONT_RIGHT_OF_CENTER); + } else if (in->u.mask & (AV_CH_SIDE_LEFT|AV_CH_SIDE_RIGHT)) { + out->u.map[n++].id = AV_CHAN_SIDE_LEFT; + out->u.map[n++].id = AV_CHAN_SIDE_RIGHT; + in->u.mask &= ~(AV_CH_SIDE_LEFT|AV_CH_SIDE_RIGHT); + } else if (in->u.mask & (AV_CH_BACK_LEFT|AV_CH_BACK_RIGHT)) { + out->u.map[n++].id = AV_CHAN_BACK_LEFT; + out->u.map[n++].id = AV_CHAN_BACK_RIGHT; + in->u.mask &= ~(AV_CH_BACK_LEFT|AV_CH_BACK_RIGHT); + } else if (in->u.mask & (AV_CH_TOP_FRONT_LEFT|AV_CH_TOP_FRONT_RIGHT)) { + out->u.map[n++].id = AV_CHAN_TOP_FRONT_LEFT; + out->u.map[n++].id = AV_CHAN_TOP_FRONT_RIGHT; + in->u.mask &= ~(AV_CH_TOP_FRONT_LEFT|AV_CH_TOP_FRONT_RIGHT); + } else if (in->u.mask & (AV_CH_TOP_BACK_LEFT|AV_CH_TOP_BACK_RIGHT)) { + out->u.map[n++].id = AV_CHAN_TOP_BACK_LEFT; + out->u.map[n++].id = AV_CHAN_TOP_BACK_RIGHT; + in->u.mask &= ~(AV_CH_TOP_BACK_LEFT|AV_CH_TOP_BACK_RIGHT); + } + + return n; +} + static int scalable_channel_layout_config(void *s, AVIOContext *pb, IAMFAudioElement *audio_element, const IAMFCodecConfig *codec_config) @@ -364,11 +395,13 @@ static int scalable_channel_layout_config(void *s, AVIOContext *pb, return AVERROR(ENOMEM); audio_element->nb_layers = nb_layers; - for (int i = 0; i < nb_layers; i++) { + for (int i = 0, n = 0; i < nb_layers; i++) { + AVChannelLayout ch_layout = { 0 }; AVIAMFLayer *layer; int loudspeaker_layout, output_gain_is_present_flag; int substream_count, coupled_substream_count; int ret, byte = avio_r8(pb); + int channels; layer = av_iamf_audio_element_add_layer(audio_element->element); if (!layer) @@ -381,7 +414,8 @@ static int scalable_channel_layout_config(void *s, AVIOContext *pb, substream_count = avio_r8(pb); coupled_substream_count = avio_r8(pb); - if (substream_count + k > audio_element->nb_substreams) + if (!substream_count || coupled_substream_count > substream_count || + substream_count + k > audio_element->nb_substreams) return AVERROR_INVALIDDATA; audio_element->layers[i].substream_count = substream_count; @@ -391,14 +425,29 @@ static int scalable_channel_layout_config(void *s, AVIOContext *pb, layer->output_gain = av_make_q(sign_extend(avio_rb16(pb), 16), 1 << 8); } - if (loudspeaker_layout < 10) - av_channel_layout_copy(&layer->ch_layout, &ff_iamf_scalable_ch_layouts[loudspeaker_layout]); - else - layer->ch_layout = (AVChannelLayout){ .order = AV_CHANNEL_ORDER_UNSPEC, - .nb_channels = substream_count + - coupled_substream_count }; + if (loudspeaker_layout < 10) { + av_channel_layout_copy(&ch_layout, &ff_iamf_scalable_ch_layouts[loudspeaker_layout]); + if (i) { + uint64_t mask = av_channel_layout_subset(&audio_element->element->layers[i-1]->ch_layout, UINT64_MAX); + // When the first layer is Mono, the second layer may not have the C channel (e.g. Stereo) + if (audio_element->element->layers[i-1]->ch_layout.nb_channels == 1) + n--; + else if ((ch_layout.u.mask & mask) != mask) + return AVERROR_INVALIDDATA; + ch_layout.u.mask &= ~mask; + } + } else { + av_log(s, AV_LOG_ERROR, "Unsupported loudspeaker_layout %d\n", loudspeaker_layout); + return AVERROR(ENOSYS); + } - if (i && layer->ch_layout.nb_channels <= audio_element->element->layers[i-1]->ch_layout.nb_channels) + channels = ch_layout.nb_channels; + if (i) { + if (ch_layout.nb_channels <= audio_element->element->layers[i-1]->ch_layout.nb_channels) + return AVERROR_INVALIDDATA; + channels -= audio_element->element->layers[i-1]->ch_layout.nb_channels; + } + if (channels != substream_count + coupled_substream_count) return AVERROR_INVALIDDATA; for (int j = 0; j < substream_count; j++) { @@ -412,6 +461,36 @@ static int scalable_channel_layout_config(void *s, AVIOContext *pb, return ret; } + ret = av_channel_layout_custom_init(&layer->ch_layout, ch_layout.nb_channels); + if (ret < 0) + return ret; + for (int j = 0; j < n; j++) + layer->ch_layout.u.map[j].id = av_channel_layout_channel_from_index(&audio_element->element->layers[i-1]->ch_layout, j); + + coupled_substream_count = audio_element->layers[i].coupled_substream_count; + while (coupled_substream_count--) { + n = parse_coupled_substream(&layer->ch_layout, &ch_layout, n); + } + + substream_count -= audio_element->layers[i].coupled_substream_count; + n = parse_coupled_substream(&layer->ch_layout, &ch_layout, n); // In case the first layer is Mono + while (substream_count--) { + if (ch_layout.u.mask & AV_CH_FRONT_CENTER) { + layer->ch_layout.u.map[n++].id = AV_CHAN_FRONT_CENTER; + ch_layout.u.mask &= ~AV_CH_FRONT_CENTER; + } + if (ch_layout.u.mask & AV_CH_LOW_FREQUENCY) { + layer->ch_layout.u.map[n++].id = AV_CHAN_LOW_FREQUENCY; + ch_layout.u.mask &= ~AV_CH_LOW_FREQUENCY; + } + } + + if (n != ch_layout.nb_channels) + return AVERROR_INVALIDDATA; + + ret = av_channel_layout_retype(&layer->ch_layout, AV_CHANNEL_ORDER_NATIVE, 0); + if (ret < 0 && ret != AVERROR(ENOSYS)) + return ret; } if (k != audio_element->nb_substreams) @@ -474,6 +553,10 @@ static int ambisonics_config(void *s, AVIOContext *pb, for (int i = 0; i < output_channel_count; i++) layer->ch_layout.u.map[i].id = avio_r8(pb) + AV_CHAN_AMBISONIC_BASE; + + ret = av_channel_layout_retype(&layer->ch_layout, AV_CHANNEL_ORDER_AMBISONIC, 0); + if (ret < 0 && ret != AVERROR(ENOSYS)) + return ret; } else { int coupled_substream_count = avio_r8(pb); // M int nb_demixing_matrix = substream_count + coupled_substream_count; @@ -508,6 +591,7 @@ static int ambisonics_config(void *s, AVIOContext *pb, static int param_parse(void *s, IAMFContext *c, AVIOContext *pb, unsigned int type, const IAMFAudioElement *audio_element, + const IAMFCodecConfig *codec_config, AVIAMFParamDefinition **out_param_definition) { IAMFParamDefinition *param_definition = NULL; @@ -530,17 +614,29 @@ static int param_parse(void *s, IAMFContext *c, AVIOContext *pb, if (mode == 0) { duration = ffio_read_leb(pb); - if (!duration) + if (!duration || duration > av_rescale(codec_config->nb_samples, + codec_config->sample_rate, parameter_rate)) { + av_log(s, AV_LOG_ERROR, "Invalid block duration in parameter_id %u\n", parameter_id); return AVERROR_INVALIDDATA; + } constant_subblock_duration = ffio_read_leb(pb); if (constant_subblock_duration == 0) nb_subblocks = ffio_read_leb(pb); else { + if (constant_subblock_duration > duration) { + av_log(s, AV_LOG_ERROR, "Invalid block duration in parameter_id %u\n", parameter_id); + return AVERROR_INVALIDDATA; + } nb_subblocks = duration / constant_subblock_duration; total_duration = duration; } } + if (nb_subblocks > duration) { + av_log(s, AV_LOG_ERROR, "Invalid duration or subblock count in parameter_id %u\n", parameter_id); + return AVERROR_INVALIDDATA; + } + param = av_iamf_param_definition_alloc(type, nb_subblocks, ¶m_size); if (!param) return AVERROR(ENOMEM); @@ -551,6 +647,11 @@ static int param_parse(void *s, IAMFContext *c, AVIOContext *pb, if (constant_subblock_duration == 0) { subblock_duration = ffio_read_leb(pb); + if (subblock_duration > duration - total_duration) { + av_log(s, AV_LOG_ERROR, "Invalid subblock durations in parameter_id %u\n", parameter_id); + av_free(param); + return AVERROR_INVALIDDATA; + } total_duration += subblock_duration; } else if (i == nb_subblocks - 1) subblock_duration = duration - i * constant_subblock_duration; @@ -595,7 +696,7 @@ static int param_parse(void *s, IAMFContext *c, AVIOContext *pb, if (param_definition) { if (param_definition->param_size != param_size || memcmp(param_definition->param, param, param_size)) { - av_log(s, AV_LOG_ERROR, "Incosistent parameters for parameter_id %u\n", parameter_id); + av_log(s, AV_LOG_ERROR, "Inconsistent parameters for parameter_id %u\n", parameter_id); av_free(param); return AVERROR_INVALIDDATA; } @@ -669,7 +770,7 @@ static int audio_element_obu(void *s, IAMFContext *c, AVIOContext *pb, int len) codec_config = ff_iamf_get_codec_config(c, codec_config_id); if (!codec_config) { - av_log(s, AV_LOG_ERROR, "Non existant codec config id %d referenced in an audio element\n", codec_config_id); + av_log(s, AV_LOG_ERROR, "Non existent codec config id %d referenced in an audio element\n", codec_config_id); ret = AVERROR_INVALIDDATA; goto fail; } @@ -777,13 +878,17 @@ static int audio_element_obu(void *s, IAMFContext *c, AVIOContext *pb, int len) ret = AVERROR_INVALIDDATA; goto fail; } - ret = param_parse(s, c, pbc, type, audio_element, &element->demixing_info); + ret = param_parse(s, c, pbc, type, + audio_element, codec_config, + &element->demixing_info); } else if (type == AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN) { if (element->recon_gain_info) { ret = AVERROR_INVALIDDATA; goto fail; } - ret = param_parse(s, c, pbc, type, audio_element, &element->recon_gain_info); + ret = param_parse(s, c, pbc, type, + audio_element, codec_config, + &element->recon_gain_info); } else { unsigned param_definition_size = ffio_read_leb(pbc); avio_skip(pbc, param_definition_size); @@ -838,6 +943,7 @@ static int label_string(AVIOContext *pb, char **label) static int mix_presentation_obu(void *s, IAMFContext *c, AVIOContext *pb, int len) { AVIAMFMixPresentation *mix; + const IAMFCodecConfig *codec_config = NULL; IAMFMixPresentation **tmp, *mix_presentation = NULL; FFIOContext b; AVIOContext *pbc; @@ -887,6 +993,11 @@ static int mix_presentation_obu(void *s, IAMFContext *c, AVIOContext *pb, int le mix_presentation->cmix = mix; mix_presentation->count_label = ffio_read_leb(pbc); + if (mix_presentation->count_label > len - avio_tell(pbc)) { + mix_presentation->count_label = 0; + ret = AVERROR_INVALIDDATA; + goto fail; + } mix_presentation->language_label = av_calloc(mix_presentation->count_label, sizeof(*mix_presentation->language_label)); if (!mix_presentation->language_label) { @@ -913,6 +1024,12 @@ static int mix_presentation_obu(void *s, IAMFContext *c, AVIOContext *pb, int le } nb_submixes = ffio_read_leb(pbc); + if (!nb_submixes) { + av_log(s, AV_LOG_ERROR, "Mix presentation %u has no submixes\n", mix_presentation_id); + ret = AVERROR_INVALIDDATA; + goto fail; + } + for (int i = 0; i < nb_submixes; i++) { AVIAMFSubmix *sub_mix; unsigned nb_elements, nb_layouts; @@ -924,9 +1041,17 @@ static int mix_presentation_obu(void *s, IAMFContext *c, AVIOContext *pb, int le } nb_elements = ffio_read_leb(pbc); + if (!nb_elements) { + av_log(s, AV_LOG_ERROR, "Submix %d from Mix presentation %u has no audio elements\n", + i, mix_presentation_id); + ret = AVERROR_INVALIDDATA; + goto fail; + } + for (int j = 0; j < nb_elements; j++) { AVIAMFSubmixElement *submix_element; IAMFAudioElement *audio_element = NULL; + const IAMFCodecConfig *config = NULL; unsigned int rendering_config_extension_size; submix_element = av_iamf_submix_add_element(sub_mix); @@ -949,6 +1074,7 @@ static int mix_presentation_obu(void *s, IAMFContext *c, AVIOContext *pb, int le ret = AVERROR_INVALIDDATA; goto fail; } + config = ff_iamf_get_codec_config(c, audio_element->codec_config_id); for (int k = 0; k < mix_presentation->count_label; k++) { char *annotation = NULL; @@ -967,14 +1093,22 @@ static int mix_presentation_obu(void *s, IAMFContext *c, AVIOContext *pb, int le avio_skip(pbc, rendering_config_extension_size); ret = param_parse(s, c, pbc, AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN, - NULL, + audio_element, config, &submix_element->element_mix_config); if (ret < 0) goto fail; submix_element->default_mix_gain = av_make_q(sign_extend(avio_rb16(pbc), 16), 1 << 8); + + if (!codec_config || (config->nb_samples > + av_rescale(codec_config->nb_samples, + codec_config->sample_rate, + config->sample_rate))) + codec_config = config; } - ret = param_parse(s, c, pbc, AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN, NULL, &sub_mix->output_mix_config); + ret = param_parse(s, c, pbc, AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN, + NULL, codec_config, + &sub_mix->output_mix_config); if (ret < 0) goto fail; sub_mix->default_mix_gain = av_make_q(sign_extend(avio_rb16(pbc), 16), 1 << 8); diff --git a/libavformat/iamf_reader.c b/libavformat/iamf_reader.c index 9a8371d9e3..eab49a8fc0 100644 --- a/libavformat/iamf_reader.c +++ b/libavformat/iamf_reader.c @@ -105,6 +105,7 @@ static int audio_frame_obu(AVFormatContext *s, const IAMFDemuxContext *c, static int parameter_block_obu(AVFormatContext *s, IAMFDemuxContext *c, AVIOContext *pbc, int len) { + const IAMFAudioElement *audio_element; const IAMFParamDefinition *param_definition; const AVIAMFParamDefinition *param; AVIAMFParamDefinition *out_param = NULL; @@ -135,12 +136,13 @@ static int parameter_block_obu(AVFormatContext *s, IAMFDemuxContext *c, parameter_id = ffio_read_leb(pb); param_definition = ff_iamf_get_param_definition(&c->iamf, parameter_id); if (!param_definition) { - av_log(s, AV_LOG_VERBOSE, "Non existant parameter_id %d referenced in a parameter block. Ignoring\n", + av_log(s, AV_LOG_VERBOSE, "Non existent parameter_id %d referenced in a parameter block. Ignoring\n", parameter_id); ret = 0; goto fail; } + audio_element = param_definition->audio_element; param = param_definition->param; if (!param_definition->mode) { duration = ffio_read_leb(pb); @@ -148,10 +150,23 @@ static int parameter_block_obu(AVFormatContext *s, IAMFDemuxContext *c, ret = AVERROR_INVALIDDATA; goto fail; } + if (audio_element) { + const IAMFCodecConfig *codec_config = ff_iamf_get_codec_config(&c->iamf, audio_element->codec_config_id); + if (duration > av_rescale(codec_config->nb_samples, codec_config->sample_rate, param->parameter_rate)) { + av_log(s, AV_LOG_ERROR, "Invalid block duration in parameter_id %u\n", parameter_id); + ret = AVERROR_INVALIDDATA; + goto fail; + } + } constant_subblock_duration = ffio_read_leb(pb); if (constant_subblock_duration == 0) nb_subblocks = ffio_read_leb(pb); else { + if (constant_subblock_duration > duration) { + av_log(s, AV_LOG_ERROR, "Invalid block duration in parameter_id %u\n", parameter_id); + ret = AVERROR_INVALIDDATA; + goto fail; + } nb_subblocks = duration / constant_subblock_duration; total_duration = duration; } @@ -161,6 +176,12 @@ static int parameter_block_obu(AVFormatContext *s, IAMFDemuxContext *c, nb_subblocks = param->nb_subblocks; } + if (nb_subblocks > duration) { + av_log(s, AV_LOG_ERROR, "Invalid duration or subblock count in parameter_id %u\n", parameter_id); + ret = AVERROR_INVALIDDATA; + goto fail; + } + out_param = av_iamf_param_definition_alloc(param->type, nb_subblocks, &out_param_size); if (!out_param) { ret = AVERROR(ENOMEM); @@ -180,6 +201,11 @@ static int parameter_block_obu(AVFormatContext *s, IAMFDemuxContext *c, if (!param_definition->mode && !constant_subblock_duration) { subblock_duration = ffio_read_leb(pb); + if (duration - total_duration > subblock_duration) { + av_log(s, AV_LOG_ERROR, "Invalid subblock durations in parameter_id %u\n", parameter_id); + ret = AVERROR_INVALIDDATA; + goto fail; + } total_duration += subblock_duration; } else if (i == nb_subblocks - 1) subblock_duration = duration - i * constant_subblock_duration; @@ -214,10 +240,10 @@ static int parameter_block_obu(AVFormatContext *s, IAMFDemuxContext *c, } case AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN: { AVIAMFReconGain *recon = subblock; - const IAMFAudioElement *audio_element = param_definition->audio_element; - const AVIAMFAudioElement *element = audio_element->celement; + const AVIAMFAudioElement *element; - av_assert0(audio_element && element); + av_assert0(audio_element && audio_element->celement); + element = audio_element->celement; for (int i = 0; i < element->nb_layers; i++) { const AVIAMFLayer *layer = element->layers[i]; if (layer->flags & AV_IAMF_LAYER_FLAG_RECON_GAIN) { diff --git a/libavformat/iamf_writer.c b/libavformat/iamf_writer.c index 8617469aa7..840c47330f 100644 --- a/libavformat/iamf_writer.c +++ b/libavformat/iamf_writer.c @@ -19,6 +19,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "libavutil/bprint.h" #include "libavutil/channel_layout.h" #include "libavutil/intreadwrite.h" #include "libavutil/iamf.h" @@ -99,12 +100,18 @@ static int populate_audio_roll_distance(IAMFCodecConfig *codec_config) } static int fill_codec_config(IAMFContext *iamf, const AVStreamGroup *stg, - IAMFCodecConfig *codec_config) + IAMFCodecConfig *codec_config, void *log_ctx) { const AVStream *st = stg->streams[0]; IAMFCodecConfig **tmp; int j, ret = 0; + if (!st->codecpar->frame_size) { + av_log(log_ctx, AV_LOG_ERROR, "frame_size is unset for stream id %d\n", + st->id); + return AVERROR(EINVAL); + } + codec_config->codec_id = st->codecpar->codec_id; codec_config->codec_tag = st->codecpar->codec_tag; switch (codec_config->codec_id) { @@ -234,7 +241,7 @@ int ff_iamf_add_audio_element(IAMFContext *iamf, const AVStreamGroup *stg, void return AVERROR(EINVAL); } if (layer->ambisonics_mode >= AV_IAMF_AMBISONICS_MODE_PROJECTION) { - av_log(log_ctx, AV_LOG_ERROR, "Unsuported ambisonics mode %d\n", layer->ambisonics_mode); + av_log(log_ctx, AV_LOG_ERROR, "Unsupported ambisonics mode %d\n", layer->ambisonics_mode); return AVERROR_PATCHWELCOME; } for (int i = 0; i < stg->nb_streams; i++) { @@ -243,18 +250,53 @@ int ff_iamf_add_audio_element(IAMFContext *iamf, const AVStreamGroup *stg, void return AVERROR(EINVAL); } } - } else + } else { + AVBPrint bp; + + if (iamf_audio_element->nb_layers < 1) { + av_log(log_ctx, AV_LOG_ERROR, "Invalid amount of layers for CHANNEL_BASED audio element. Must be >= 1\n"); + return AVERROR(EINVAL); + } + for (int j, i = 0; i < iamf_audio_element->nb_layers; i++) { const AVIAMFLayer *layer = iamf_audio_element->layers[i]; + const AVIAMFLayer *prev_layer; + uint64_t prev_mask; + for (j = 0; j < FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts); j++) - if (!av_channel_layout_compare(&layer->ch_layout, &ff_iamf_scalable_ch_layouts[j])) + if (av_channel_layout_subset(&layer->ch_layout, UINT64_MAX) == + av_channel_layout_subset(&ff_iamf_scalable_ch_layouts[j], UINT64_MAX)) break; if (j >= FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts)) { - av_log(log_ctx, AV_LOG_ERROR, "Unsupported channel layout in stream group #%d\n", i); + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_AUTOMATIC); + av_channel_layout_describe_bprint(&layer->ch_layout, &bp); + av_log(log_ctx, AV_LOG_ERROR, "Unsupported channel layout in Audio Element id %"PRId64 + ", Layer %d: %s\n", + stg->id, i, bp.str); + av_bprint_finalize(&bp, NULL); + return AVERROR(EINVAL); + } + + if (!i) + continue; + + prev_layer = iamf_audio_element->layers[i-1]; + prev_mask = av_channel_layout_subset(&prev_layer->ch_layout, UINT64_MAX); + if (av_channel_layout_subset(&layer->ch_layout, prev_mask) != prev_mask || (layer->ch_layout.nb_channels <= + prev_layer->ch_layout.nb_channels)) { + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_AUTOMATIC); + av_bprintf(&bp, "Channel layout \""); + av_channel_layout_describe_bprint(&layer->ch_layout, &bp); + av_bprintf(&bp, "\" can't follow channel layout \""); + av_channel_layout_describe_bprint(&prev_layer->ch_layout, &bp); + av_bprintf(&bp, "\" in Scalable Audio Element id %"PRId64, stg->id); + av_log(log_ctx, AV_LOG_ERROR, "%s\n", bp.str); + av_bprint_finalize(&bp, NULL); return AVERROR(EINVAL); } } + } for (int i = 0; i < iamf->nb_audio_elements; i++) { if (stg->id == iamf->audio_elements[i]->audio_element_id) { @@ -267,7 +309,7 @@ int ff_iamf_add_audio_element(IAMFContext *iamf, const AVStreamGroup *stg, void if (!codec_config) return AVERROR(ENOMEM); - ret = fill_codec_config(iamf, stg, codec_config); + ret = fill_codec_config(iamf, stg, codec_config, log_ctx); if (ret < 0) { av_free(codec_config); return ret; @@ -517,6 +559,25 @@ static inline int rescale_rational(AVRational q, int b) return av_clip_int16(av_rescale(q.num, b, q.den)); } +static void get_loudspeaker_layout(const AVIAMFLayer *layer, int *playout) +{ + int layout; + + for (layout = 0; layout < FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts); layout++) { + if (!av_channel_layout_compare(&layer->ch_layout, &ff_iamf_scalable_ch_layouts[layout])) + break; + } + if (layout >= FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts)) { + for (layout = 0; layout < FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts); layout++) + if (av_channel_layout_subset(&layer->ch_layout, UINT64_MAX) == + av_channel_layout_subset(&ff_iamf_scalable_ch_layouts[layout], UINT64_MAX)) + break; + } + av_assert0(layout < FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts)); + + *playout = layout; +} + static int scalable_channel_layout_config(const IAMFAudioElement *audio_element, AVIOContext *dyn_bc) { @@ -532,10 +593,8 @@ static int scalable_channel_layout_config(const IAMFAudioElement *audio_element, for (int i = 0; i < element->nb_layers; i++) { const AVIAMFLayer *layer = element->layers[i]; int layout; - for (layout = 0; layout < FF_ARRAY_ELEMS(ff_iamf_scalable_ch_layouts); layout++) { - if (!av_channel_layout_compare(&layer->ch_layout, &ff_iamf_scalable_ch_layouts[layout])) - break; - } + + get_loudspeaker_layout(layer, &layout); init_put_bits(&pb, header, sizeof(header)); put_bits(&pb, 4, layout); put_bits(&pb, 1, !!layer->output_gain_flags); @@ -650,27 +709,56 @@ static int iamf_write_audio_element(const IAMFContext *iamf, for (int i = 0; i < audio_element->nb_substreams; i++) ffio_write_leb(dyn_bc, audio_element->substreams[i].audio_substream_id); - if (element->nb_layers == 1) - param_definition_types &= ~AV_IAMF_PARAMETER_DEFINITION_DEMIXING; - if (element->nb_layers > 1) - param_definition_types |= AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN; - if (codec_config->codec_tag == MKTAG('f','L','a','C') || - codec_config->codec_tag == MKTAG('i','p','c','m')) - param_definition_types &= ~AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN; + /* When audio_element_type = 1, num_parameters SHALL be set to 0 */ + if (element->audio_element_type == AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE) + param_definition_types = 0; + else { + int layout = 0; + get_loudspeaker_layout(element->layers[0], &layout); + /* When the loudspeaker_layout = 15, the type PARAMETER_DEFINITION_DEMIXING SHALL NOT be present. */ + if (layout == 15) { + param_definition_types &= ~AV_IAMF_PARAMETER_DEFINITION_DEMIXING; + /* expanded_loudspeaker_layout SHALL only be present when num_layers = 1 and loudspeaker_layout is set to 15 */ + if (element->nb_layers > 1) { + av_log(log_ctx, AV_LOG_ERROR, "expanded_loudspeaker_layout present when using more than one layer in " + "Stream Group #%u\n", + audio_element->audio_element_id); + return AVERROR(EINVAL); + } + } + /* When the loudspeaker_layout of the (non-)scalable channel audio (i.e., num_layers = 1) is less than or equal to 3.1.2ch, + * (i.e., Mono, Stereo, or 3.1.2ch), the type PARAMETER_DEFINITION_DEMIXING SHALL NOT be present. */ + else if (element->nb_layers == 1 && (layout == 0 || layout == 1 || layout == 8)) + param_definition_types &= ~AV_IAMF_PARAMETER_DEFINITION_DEMIXING; + /* When num_layers > 1, the type PARAMETER_DEFINITION_RECON_GAIN SHALL be present */ + if (element->nb_layers > 1) + param_definition_types |= AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN; + /* When codec_id = fLaC or ipcm, the type PARAMETER_DEFINITION_RECON_GAIN SHALL NOT be present. */ + if (codec_config->codec_tag == MKTAG('f','L','a','C') || + codec_config->codec_tag == MKTAG('i','p','c','m')) + param_definition_types &= ~AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN; + if ((param_definition_types & AV_IAMF_PARAMETER_DEFINITION_DEMIXING) && !element->demixing_info) { + if (element->nb_layers > 1) { + get_loudspeaker_layout(element->layers[element->nb_layers-1], &layout); + /* When the highest loudspeaker_layout of the scalable channel audio (i.e., num_layers > 1) is greater than 3.1.2ch, + * (i.e., 5.1.2ch, 5.1.4ch, 7.1.2ch, or 7.1.4ch), type PARAMETER_DEFINITION_DEMIXING SHALL be present. */ + if (layout == 3 || layout == 4 || layout == 6 || layout == 7) { + av_log(log_ctx, AV_LOG_ERROR, "demixing_info needed but not set in Stream Group #%u\n", + audio_element->audio_element_id); + return AVERROR(EINVAL); + } + } + param_definition_types &= ~AV_IAMF_PARAMETER_DEFINITION_DEMIXING; + } + } ffio_write_leb(dyn_bc, av_popcount(param_definition_types)); // num_parameters - if (param_definition_types & 1) { + if (param_definition_types & AV_IAMF_PARAMETER_DEFINITION_DEMIXING) { const AVIAMFParamDefinition *param = element->demixing_info; const IAMFParamDefinition *param_def; const AVIAMFDemixingInfo *demix; - if (!param) { - av_log(log_ctx, AV_LOG_ERROR, "demixing_info needed but not set in Stream Group #%u\n", - audio_element->audio_element_id); - return AVERROR(EINVAL); - } - demix = av_iamf_param_definition_get_subblock(param, 0); ffio_write_leb(dyn_bc, AV_IAMF_PARAMETER_DEFINITION_DEMIXING); // type @@ -682,7 +770,7 @@ static int iamf_write_audio_element(const IAMFContext *iamf, avio_w8(dyn_bc, demix->dmixp_mode << 5); // dmixp_mode avio_w8(dyn_bc, element->default_w << 4); // default_w } - if (param_definition_types & 2) { + if (param_definition_types & AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN) { const AVIAMFParamDefinition *param = element->recon_gain_info; const IAMFParamDefinition *param_def; @@ -813,6 +901,9 @@ static int iamf_write_mixing_presentation(const IAMFContext *iamf, av_log(log_ctx, AV_LOG_ERROR, "Invalid Sound System value in a submix\n"); return AVERROR(EINVAL); } + } else if (submix_layout->layout_type != AV_IAMF_SUBMIX_LAYOUT_TYPE_BINAURAL) { + av_log(log_ctx, AV_LOG_ERROR, "Unsupported Layout Type value in a submix\n"); + return AVERROR(EINVAL); } init_put_bits(&pbc, header, sizeof(header)); put_bits(&pbc, 2, submix_layout->layout_type); // layout_type @@ -1072,6 +1163,8 @@ int ff_iamf_write_audio_frame(const IAMFContext *iamf, AVIOContext *pb, { uint8_t header[MAX_IAMF_OBU_HEADER_SIZE]; PutBitContext pbc; + const IAMFAudioElement *audio_element; + IAMFCodecConfig *codec_config; AVIOContext *dyn_bc; const uint8_t *side_data; uint8_t *dyn_buf = NULL; @@ -1081,9 +1174,14 @@ int ff_iamf_write_audio_frame(const IAMFContext *iamf, AVIOContext *pb, audio_substream_id + IAMF_OBU_IA_AUDIO_FRAME_ID0 : IAMF_OBU_IA_AUDIO_FRAME; int ret; + audio_element = get_audio_element(iamf, audio_substream_id); + if (!audio_element) + return AVERROR(EINVAL); + codec_config = ff_iamf_get_codec_config(iamf, audio_element->codec_config_id); + if (!codec_config) + return AVERROR(EINVAL); + if (!pkt->size) { - const IAMFAudioElement *audio_element; - IAMFCodecConfig *codec_config; size_t new_extradata_size; const uint8_t *new_extradata = av_packet_get_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, @@ -1091,12 +1189,6 @@ int ff_iamf_write_audio_frame(const IAMFContext *iamf, AVIOContext *pb, if (!new_extradata || new_extradata_size > INT_MAX - AV_INPUT_BUFFER_PADDING_SIZE) return AVERROR_INVALIDDATA; - audio_element = get_audio_element(iamf, audio_substream_id); - if (!audio_element) - return AVERROR(EINVAL); - codec_config = ff_iamf_get_codec_config(iamf, audio_element->codec_config_id); - if (!codec_config) - return AVERROR(EINVAL); av_free(codec_config->extradata); codec_config->extradata = av_malloc(new_extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); @@ -1119,6 +1211,15 @@ int ff_iamf_write_audio_frame(const IAMFContext *iamf, AVIOContext *pb, discard_padding = AV_RL32(side_data + 4); } + if (codec_config->codec_id == AV_CODEC_ID_OPUS) { + // IAMF's num_samples_to_trim_at_start is the same as Opus's pre-skip. + if (!skip_samples) + skip_samples = pkt->dts < 0 + ? av_rescale(-pkt->dts, 48000, pkt->time_base.den) + : 0; + discard_padding = av_rescale(discard_padding, 48000, pkt->time_base.den); + } + ret = avio_open_dyn_buf(&dyn_bc); if (ret < 0) return ret; diff --git a/libavformat/iamfdec.c b/libavformat/iamfdec.c index 8feb06d9e6..02f01abc6c 100644 --- a/libavformat/iamfdec.c +++ b/libavformat/iamfdec.c @@ -109,7 +109,7 @@ static int iamf_read_header(AVFormatContext *s) if (!i && !j && audio_element->layers[0].substream_count == 1) st->disposition |= AV_DISPOSITION_DEFAULT; - else + else if (audio_element->nb_layers > 1 || audio_element->layers[0].substream_count > 1) st->disposition |= AV_DISPOSITION_DEPENDENT; st->id = substream->audio_substream_id; avpriv_set_pts_info(st, 64, 1, st->codecpar->sample_rate); diff --git a/tests/ref/fate/iamf-5_1-copy b/tests/ref/fate/iamf-5_1-copy index ed2046fe3b..62b77d8774 100644 --- a/tests/ref/fate/iamf-5_1-copy +++ b/tests/ref/fate/iamf-5_1-copy @@ -37,7 +37,7 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1 +channel_layout=6 channels (FL+FR+BL+BR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] diff --git a/tests/ref/fate/iamf-5_1-demux b/tests/ref/fate/iamf-5_1-demux index ed2046fe3b..62b77d8774 100644 --- a/tests/ref/fate/iamf-5_1-demux +++ b/tests/ref/fate/iamf-5_1-demux @@ -37,7 +37,7 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1 +channel_layout=6 channels (FL+FR+BL+BR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] diff --git a/tests/ref/fate/iamf-5_1_4 b/tests/ref/fate/iamf-5_1_4 index fdee50b8e0..bb4a2d7a01 100644 --- a/tests/ref/fate/iamf-5_1_4 +++ b/tests/ref/fate/iamf-5_1_4 @@ -111,17 +111,17 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1 +channel_layout=6 channels (FL+FR+BL+BR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1.2 +channel_layout=8 channels (FL+FR+BL+BR+FC+LFE+TFL+TFR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1.4 +channel_layout=10 channels (FL+FR+BL+BR+FC+LFE+TFL+TFR+TBL+TBR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] diff --git a/tests/ref/fate/iamf-7_1_4 b/tests/ref/fate/iamf-7_1_4 index ccce96ea84..4259e40135 100644 --- a/tests/ref/fate/iamf-7_1_4 +++ b/tests/ref/fate/iamf-7_1_4 @@ -127,17 +127,17 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=3.1.2 +channel_layout=6 channels (FL+FR+TFL+TFR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=7.1.2 +channel_layout=10 channels (FL+FR+TFL+TFR+FC+LFE+SL+SR+BL+BR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=7.1.4 +channel_layout=12 channels (FL+FR+TFL+TFR+FC+LFE+SL+SR+BL+BR+TBL+TBR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] diff --git a/tests/ref/fate/mov-mp4-iamf-5_1_4 b/tests/ref/fate/mov-mp4-iamf-5_1_4 index 36106528ab..1890b408be 100644 --- a/tests/ref/fate/mov-mp4-iamf-5_1_4 +++ b/tests/ref/fate/mov-mp4-iamf-5_1_4 @@ -111,17 +111,17 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1 +channel_layout=6 channels (FL+FR+BL+BR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1.2 +channel_layout=8 channels (FL+FR+BL+BR+FC+LFE+TFL+TFR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=5.1.4 +channel_layout=10 channels (FL+FR+BL+BR+FC+LFE+TFL+TFR+TBL+TBR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] diff --git a/tests/ref/fate/mov-mp4-iamf-7_1_4-video-first b/tests/ref/fate/mov-mp4-iamf-7_1_4-video-first index d3b37896b2..d5a1fe1cad 100644 --- a/tests/ref/fate/mov-mp4-iamf-7_1_4-video-first +++ b/tests/ref/fate/mov-mp4-iamf-7_1_4-video-first @@ -158,17 +158,17 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=3.1.2 +channel_layout=6 channels (FL+FR+TFL+TFR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=7.1.2 +channel_layout=10 channels (FL+FR+TFL+TFR+FC+LFE+SL+SR+BL+BR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=7.1.4 +channel_layout=12 channels (FL+FR+TFL+TFR+FC+LFE+SL+SR+BL+BR+TBL+TBR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] diff --git a/tests/ref/fate/mov-mp4-iamf-7_1_4-video-last b/tests/ref/fate/mov-mp4-iamf-7_1_4-video-last index ede4a40025..caf89d41f6 100644 --- a/tests/ref/fate/mov-mp4-iamf-7_1_4-video-last +++ b/tests/ref/fate/mov-mp4-iamf-7_1_4-video-last @@ -158,17 +158,17 @@ output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=3.1.2 +channel_layout=6 channels (FL+FR+TFL+TFR+FC+LFE) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=7.1.2 +channel_layout=10 channels (FL+FR+TFL+TFR+FC+LFE+SL+SR+BL+BR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] [SUBCOMPONENT] -channel_layout=7.1.4 +channel_layout=12 channels (FL+FR+TFL+TFR+FC+LFE+SL+SR+BL+BR+TBL+TBR) output_gain_flags=0 output_gain=0/1 [/SUBCOMPONENT] -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
