Hello, Am Dienstag, 3. August 2021, 17:48:12 CEST schrieb Andreas Rheinhardt: >> Carsten Gross: >> [RDS data on new ARD satellite audio transponder in AAC-LATM, e.g. NDR] > > Using frame side data (or also packet side data -- a matching packet > side data would be needed if one wanted to preserve this data when > reencoding/encoding from scratch; this would also allow to write a > bitstream filter to extract this without decoding) is the right way to > do this.
I've worked on a version to decode AAC-RDS and put it into AVFrameSideData. Perhaps it's a starting point for the other RDS cases as well (there is also a standard for embedding RDS into MPEG-1/2 audio and also separate RDS data-streams in mp2t, this is implemented in my application ts2shout - https://github.com/carsten-gross/ts2shout but not in this patch). As this is my first contribution I'd like to ask you for special attention if everything is done allright by myself and if it is acceptable at all (unfortunatly I forget to do proper branching and therefore did only a diff between "my" and the master version) I also added a simple demonstration application decode_audio_rds.c that is directly derived from decode_audio.c in doc/examples, but expects MP4-LOAS "with" RDS. I've put two extracts from radio transmissions on my web server http://www.siski.de/~carsten/audio/NDR2.mp4 and http://www.siski.de/~carsten/audio/MDR-THUE.mp4 to test it. To download the files you'll need wget/curl to fetch it. The files can be given to the demo application (FLOAT-wav is generated, RDS is written to stderr). The demo application is there for better testing and not strictly necessary, of course. Main use case is currently my own application ts2shout that already implements this method for AAC-LATM radio stations (e.g. ARD NDR, ARD MDR on Astra 19.2E 10891 MHz and 11052 MHz) Here's the patch (Sorry not in git format, file list followed by plain diff): --- configure doc/examples/decode_audio_rds.c doc/examples/Makefile libavcodec/aacdec_template.c libavcodec/defs.h libavutil/frame.c libavutil/frame.h --- ffmpeg/configure 2021-10-18 22:05:21.955807013 +0200 +++ FFmpeg/configure 2021-10-18 22:08:16.225672523 +0200 @@ -1708,6 +1708,7 @@ avio_list_dir_example avio_reading_example decode_audio_example + decode_audio_rds_example decode_video_example demuxing_decoding_example encode_audio_example --- ffmpeg/doc/examples/decode_audio_rds.c 1970-01-01 01:00:00.000000000 +0100 +++ FFmpeg/doc/examples/decode_audio_rds.c 2021-10-18 22:25:40.764894237 +0200 @@ -0,0 +1,371 @@ +/* + * Copyright (c) 2001 Fabrice Bellard + * Copyright (c) 2021 Carsten Gross + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * @file + * audio decoding with RDS FrameSide data libavcodec API example + * + * @example decode_audio.c + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <libavutil/frame.h> +#include <libavutil/mem.h> + +#include <libavcodec/avcodec.h> + +#define AUDIO_INBUF_SIZE 20480 +#define AUDIO_REFILL_THRESH 4096 + +static int get_format_from_sample_fmt(const char **fmt, + enum AVSampleFormat sample_fmt) +{ + int i; + struct sample_fmt_entry { + enum AVSampleFormat sample_fmt; const char *fmt_be, *fmt_le; + } sample_fmt_entries[] = { + { AV_SAMPLE_FMT_U8, "u8", "u8" }, + { AV_SAMPLE_FMT_S16, "s16be", "s16le" }, + { AV_SAMPLE_FMT_S32, "s32be", "s32le" }, + { AV_SAMPLE_FMT_FLT, "f32be", "f32le" }, + { AV_SAMPLE_FMT_DBL, "f64be", "f64le" }, + }; + *fmt = NULL; + + for (i = 0; i < FF_ARRAY_ELEMS(sample_fmt_entries); i++) { + struct sample_fmt_entry *entry = &sample_fmt_entries[i]; + if (sample_fmt == entry->sample_fmt) { + *fmt = AV_NE(entry->fmt_be, entry->fmt_le); + return 0; + } + } + + fprintf(stderr, + "sample format %s is not supported as output format\n", + av_get_sample_fmt_name(sample_fmt)); + return -1; +} + +/* + * RDS - radio data system stuff to present RDS text messages + * + */ + +/* Convert EBU table 1 to latin 1 + * Taken out of http://www.interactive-radio-system.com/docs/EN50067_RDS_Standard.pdf + * non latin1 characters are translated to "." */ + +static uint8_t ebutable1[] = { + /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0xe1,0xe0,0xe9,0xe8,0xed,0xec,0xf3,0xf2,0xfa,0xf9,0xd1,0xc7,0x2e,0xdf,0xa1,0x2e, /* 0x80 - 0x8f */ + 0xe2,0xe4,0xea,0xeb,0xee,0xef,0xf4,0xf6,0xfb,0xfc,0xf1,0xe7,0x2e,0x2e,0x2e,0x2e, /* 0x90 - 0x9f */ + 0xaa,0x2e,0xa9,0x2e,0x2e,0x2e,0x2e,0x2e,0x2e,0x2e,0x2e,0x24,0x2e,0x2e,0x2e,0x2e, /* 0xa0 - 0xaf */ + 0xba,0xb9,0xb2,0xb3,0xb1,0x2e,0x2e,0x2e,0xb5,0xbf,0xf7,0xb0,0xbc,0xbd,0xbe,0xa7, /* 0xb0 - 0xbf */ + 0xc1,0xc0,0xc9,0xc8,0xcd,0xcc,0xd3,0xd2,0xda,0xd9,0x2e,0x2e,0x2e,0x2e,0xd0,0x2d, /* 0xc0 - 0xcf */ + 0xc2,0xc4,0xca,0xcb,0xce,0xcf,0xd4,0xd6,0xdb,0xdc,0x2d,0x2e,0x2e,0x2e,0x2e,0x2e, /* 0xd0 - 0xdf */ + 0xc3,0xc5,0xc6,0x2e,0x2e,0xdd,0xd5,0xd8,0xde,0x2e,0x2e,0x2e,0x2e,0x2e,0x2e,0xf0, /* 0xe0 - 0xef */ + 0xe3,0xe5,0xe6,0x2e,0x2e,0xfd,0xf5,0xf8,0xfe,0x2e,0x2e,0x2e,0x2e,0x2e,0x2e,0x2e, /* 0xf0 - 0xff */ +}; + +static uint8_t ebu2latin1(uint8_t character) { + if (character < 0x20) { + if (character == 0x0d || character == 0x0a) { + return ' '; + } + return '.'; + } + if (character >= 0x80) { + return ebutable1[(character & 0x7f)]; + } + return character; +} + +static char * handle_rt(uint8_t* rds_message, uint8_t size, char * text) { + uint8_t msg_len = rds_message[7]; + uint8_t i; + static char radiotext[128]; + /* Radiotext consists of two message parts with 64 characters each + * indexed by an index being either 0 or 1, we ignore this for this + * demonstration code */ + if (msg_len > 0x41) { + msg_len = 0x41; + } + radiotext[0x41] = 0; + /* Cleanup old message */ + if (msg_len > 0) { + for (i = msg_len - 1; i < 0x40; i++) { + radiotext[i] = ' '; + } + } + /* Check and convert message to latin1 */ + for (i = 9; i < 8 + msg_len; i++) { + /* is some character different? */ + radiotext[i - 9] = ebu2latin1(rds_message[i]); + } + memcpy(text, radiotext, 0x41); + return radiotext; +} + +/* Handle a RDS data chunk. */ +static void rds_handle_message(uint8_t* rds_message, uint8_t size, uint32_t audio_frame) { + uint8_t type = rds_message[4]; + static char text[128]; + switch (type) { + case 0x0a: // RT (Radiotexta) + // DumpHex(rds_message, size); + handle_rt(rds_message, size, text); + fprintf(stderr, "RT (Frame#%d): %s\n", audio_frame, text); + break; + case 0x02: + //handle_ps(rds_message, size); + break; + } +} + +static void rds_convert(uint8_t* buffer, size_t size, uint32_t audio_frame) { + static uint8_t current_pos = 0; + static uint8_t rds_message[255]; + int32_t i = 0; + uint8_t rds_data_size = buffer[2]; + if (rds_data_size == 0) { + return; + } + for (i = 0; i < size; i++) { + uint8_t mychar = buffer[i]; + if (mychar == 0xfd) { + /* special marker: 0xfd 0x01 means 0xfe, 0xfd 0x02 means 0xff */ + i++; + rds_message[current_pos] = mychar + buffer[i]; + current_pos ++; + } else { + rds_message[current_pos] = mychar; + current_pos ++; + } + } + rds_handle_message(rds_message, current_pos, audio_frame); + current_pos = 0; + return; +} + +static void decode(AVCodecContext *dec_ctx, AVPacket *pkt, AVFrame *frame, + FILE *outfile) +{ + int i, ch; + int ret, data_size; + static uint32_t frame_nr = 0; + /* send the packet with the compressed data to the decoder */ + ret = avcodec_send_packet(dec_ctx, pkt); + if (ret < 0) { + fprintf(stderr, "Error submitting the packet to the decoder\n"); + exit(1); + } + + /* read all the output frames (in general there may be any number of them */ + while (ret >= 0) { + AVFrameSideData* sd = NULL; + ret = avcodec_receive_frame(dec_ctx, frame); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) + return; + else if (ret < 0) { + fprintf(stderr, "Error during decoding\n"); + exit(1); + } + frame_nr++; + data_size = av_get_bytes_per_sample(dec_ctx->sample_fmt); + if (data_size < 0) { + /* This should not occur, checking just for paranoia */ + fprintf(stderr, "Failed to calculate data size\n"); + exit(1); + } + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_RDS_DATA_PACKET); + if (sd) { + unsigned char* data = sd->data; + if ( sd->size >= 2) { + int startval = 1; + int used = 0; + int count = 0; // ensure termination, also for broken data + while ( startval < (sd->size - 1) && count < 15 ) { + int i = startval; + for (; i < (sd->size - 1); i++) { + if ( data[i] == 0xff + && data[i+1] == 0xfe) { + rds_convert(data + startval, i - startval, frame_nr); + used += 1; + startval = i + 2; + break; + } + } + count++; + } + if (used > 0) { + rds_convert(data + startval, sd->size - startval - 1, frame_nr); + } else { + rds_convert(sd->data + 1, sd->size - 2, frame_nr); + } + } + } + for (i = 0; i < frame->nb_samples; i++) + for (ch = 0; ch < dec_ctx->channels; ch++) + fwrite(frame->data[ch] + data_size*i, 1, data_size, outfile); + } +} + +int main(int argc, char **argv) +{ + const char *outfilename, *filename; + const AVCodec *codec; + AVCodecContext *c= NULL; + AVCodecParserContext *parser = NULL; + int len, ret; + FILE *f, *outfile; + uint8_t inbuf[AUDIO_INBUF_SIZE + AV_INPUT_BUFFER_PADDING_SIZE]; + uint8_t *data; + size_t data_size; + AVPacket *pkt; + AVFrame *decoded_frame = NULL; + enum AVSampleFormat sfmt; + int n_channels = 0; + const char *fmt; + + if (argc <= 2) { + fprintf(stderr, "Usage: %s <input file> <output file>\n", argv[0]); + exit(0); + } + filename = argv[1]; + outfilename = argv[2]; + + pkt = av_packet_alloc(); + + /* find the MPEG audio decoder */ + codec = avcodec_find_decoder(AV_CODEC_ID_AAC_LATM); + if (!codec) { + fprintf(stderr, "Codec not found\n"); + exit(1); + } + + parser = av_parser_init(codec->id); + if (!parser) { + fprintf(stderr, "Parser not found\n"); + exit(1); + } + + c = avcodec_alloc_context3(codec); + if (!c) { + fprintf(stderr, "Could not allocate audio codec context\n"); + exit(1); + } + + /* open it */ + if (avcodec_open2(c, codec, NULL) < 0) { + fprintf(stderr, "Could not open codec\n"); + exit(1); + } + + f = fopen(filename, "rb"); + if (!f) { + fprintf(stderr, "Could not open %s\n", filename); + exit(1); + } + outfile = fopen(outfilename, "wb"); + if (!outfile) { + av_free(c); + exit(1); + } + + /* decode until eof */ + data = inbuf; + data_size = fread(inbuf, 1, AUDIO_INBUF_SIZE, f); + + if (data_size < 3) { + fprintf(stderr, "File to short\n"); + exit(1); + } + while (data_size > 0) { + if (!decoded_frame) { + if (!(decoded_frame = av_frame_alloc())) { + fprintf(stderr, "Could not allocate audio frame\n"); + exit(1); + } + } + + ret = av_parser_parse2(parser, c, &pkt->data, &pkt->size, + data, data_size, + AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + if (ret < 0) { + fprintf(stderr, "Error while parsing\n"); + exit(1); + } + data += ret; + data_size -= ret; + + if (pkt->size) + decode(c, pkt, decoded_frame, outfile); + + if (data_size < AUDIO_REFILL_THRESH) { + memmove(inbuf, data, data_size); + data = inbuf; + len = fread(data + data_size, 1, + AUDIO_INBUF_SIZE - data_size, f); + if (len > 0) + data_size += len; + } + } + + /* flush the decoder */ + pkt->data = NULL; + pkt->size = 0; + decode(c, pkt, decoded_frame, outfile); + + /* print output pcm infomations, because there have no metadata of pcm */ + sfmt = c->sample_fmt; + + if (av_sample_fmt_is_planar(sfmt)) { + const char *packed = av_get_sample_fmt_name(sfmt); + printf("Warning: the sample format the decoder produced is planar " + "(%s). This example will output the first channel only.\n", + packed ? packed : "?"); + sfmt = av_get_packed_sample_fmt(sfmt); + } + + n_channels = c->channels; + if ((ret = get_format_from_sample_fmt(&fmt, sfmt)) < 0) + goto end; + + printf("Play the output audio file with the command:\n" + "ffplay -f %s -ac %d -ar %d %s\n", + fmt, n_channels, c->sample_rate, + outfilename); +end: + fclose(outfile); + fclose(f); + + avcodec_free_context(&c); + av_parser_close(parser); + av_frame_free(&decoded_frame); + av_packet_free(&pkt); + + return 0; +} --- ffmpeg/doc/examples/Makefile 2021-10-18 22:05:21.959806965 +0200 +++ FFmpeg/doc/examples/Makefile 2021-08-05 22:03:00.405426588 +0200 @@ -1,6 +1,7 @@ EXAMPLES-$(CONFIG_AVIO_LIST_DIR_EXAMPLE) += avio_list_dir EXAMPLES-$(CONFIG_AVIO_READING_EXAMPLE) += avio_reading EXAMPLES-$(CONFIG_DECODE_AUDIO_EXAMPLE) += decode_audio +EXAMPLES-$(CONFIG_DECODE_AUDIO_RDS_EXAMPLE) += decode_audio_rds EXAMPLES-$(CONFIG_DECODE_VIDEO_EXAMPLE) += decode_video EXAMPLES-$(CONFIG_DEMUXING_DECODING_EXAMPLE) += demuxing_decoding EXAMPLES-$(CONFIG_ENCODE_AUDIO_EXAMPLE) += encode_audio --- ffmpeg/libavcodec/aacdec_template.c 2021-10-18 22:05:21.975806769 +0200 +++ FFmpeg/libavcodec/aacdec_template.c 2021-10-19 14:45:17.180614937 +0200 @@ -1351,22 +1351,91 @@ } /** - * Skip data_stream_element; reference: table 4.10. - */ -static int skip_data_stream_element(AACContext *ac, GetBitContext *gb) -{ + * decode data_stream_element; reference: table 4.10. + * A DSE is an arbitrary binary data element referenced to by it's "position" + * One could use the position of the DSE to identify it, but trying to identify the + * data we are interested in seems to be a better approach that fits better + * into the library access model of libavcodec/ffmpeg + */ +static int decode_data_stream_element(AACContext *ac, GetBitContext *gb, int dse_id) +{ + static const int DSE_BUFFER_SIZE=1024; + unsigned char buffer[1024]; // DSE_BUFFER_SIZE + static unsigned char result_buffer[1024]; + static int result_buffer_count = 0; + int byte_count; + int i = 0; + AVFrameSideData *sd = NULL; + AVFrame *frame; int byte_align = get_bits1(gb); int count = get_bits(gb, 8); if (count == 255) count += get_bits(gb, 8); if (byte_align) align_get_bits(gb); - + byte_count = count; if (get_bits_left(gb) < 8 * count) { - av_log(ac->avctx, AV_LOG_ERROR, "skip_data_stream_element: "overread_err); + av_log(ac->avctx, AV_LOG_ERROR, "decode_data_stream_element: "overread_err); return AVERROR_INVALIDDATA; } - skip_bits_long(gb, 8 * count); + if (byte_count >= DSE_BUFFER_SIZE) { + skip_bits_long(gb, 8 * count); + av_log(ac->avctx, AV_LOG_DEBUG, "decode_data_stream_element: " + "byte_count (%d) >= %d", byte_count, DSE_BUFFER_SIZE); + count = 0; /* don't copy buffer into side data element, because DSE is too large */ + } else { + while (byte_count > 0 && byte_count < DSE_BUFFER_SIZE) { + buffer[i++] = get_bits(gb, 8); + byte_count--; + } + } + frame = ac->frame; + if (frame) { + // This is in some programmes, don't know why + if (count == 3 && buffer[0] == 0xbc && buffer[1] == 0xc0 && buffer[2] == 0x00) { + // nothing + } else if (count >=1 ) { + // Try to detect UECP radio data system data, as this is the + // only AAC side data we are currently interested in. + // In some cases there are several UECP messages concated together, the user application + // has to split it up. If UECP messages are distributed over several + // frames we try to do our best to collect it up to 1K. + if (result_buffer_count + count + 1 < DSE_BUFFER_SIZE ) { + memcpy(result_buffer + result_buffer_count, buffer, count); + result_buffer_count += count; + } else { + /* Overflow */ + result_buffer_count = 0; + return 0; + } + // If buffer is not valid, but terminated with 0xff, try to start over + // This happens at the beginning of reception + if (result_buffer_count >= 1 && result_buffer[0] != 0xfe + && result_buffer[result_buffer_count - 1] == 0xff) { + result_buffer_count = 0; + } + /* if the RDS data is complete just put it into side data */ + if (result_buffer_count >= 1 && result_buffer[0] == 0xfe + && result_buffer[result_buffer_count - 1] == 0xff) { + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_RDS_DATA_PACKET); + if (!sd) { + sd = av_frame_new_side_data(frame, AV_FRAME_DATA_RDS_DATA_PACKET, result_buffer_count); + if (!sd) { + return AVERROR(ENOMEM); + } + memcpy(sd->data, result_buffer, result_buffer_count); + } else { + // memory is already reserved + if (!av_realloc(sd->data, result_buffer_count)) { + return AVERROR(ENOMEM); + } + memcpy(sd->data, result_buffer, result_buffer_count); + sd->size = count; + } + result_buffer_count = 0; + } + } + } return 0; } @@ -3229,6 +3298,7 @@ ChannelElement *che = NULL, *che_prev = NULL; enum RawDataBlockType elem_type, che_prev_type = TYPE_END; int err, elem_id; + int dse_count = 0; int samples = 0, multiplier, audio_found = 0, pce_found = 0; int is_dmono, sce_count = 0; int payload_alignment; @@ -3314,7 +3384,8 @@ break; case TYPE_DSE: - err = skip_data_stream_element(ac, gb); + err = decode_data_stream_element(ac, gb, dse_count); + dse_count++; break; case TYPE_PCE: { --- ffmpeg/libavutil/frame.c 2021-10-18 22:05:22.263803241 +0200 +++ FFmpeg/libavutil/frame.c 2021-10-18 22:08:16.357670905 +0200 @@ -728,6 +728,7 @@ case AV_FRAME_DATA_SEI_UNREGISTERED: return "H.26[45] User Data Unregistered SEI message"; case AV_FRAME_DATA_FILM_GRAIN_PARAMS: return "Film grain parameters"; case AV_FRAME_DATA_DETECTION_BBOXES: return "Bounding boxes for object detection and classification"; + case AV_FRAME_DATA_RDS_DATA_PACKET: return "UECP RDS data packet"; } return NULL; } --- ffmpeg/libavutil/frame.h 2021-10-18 22:05:22.263803241 +0200 +++ FFmpeg/libavutil/frame.h 2021-10-19 14:46:16.371891923 +0200 @@ -187,6 +187,14 @@ * as described by AVDetectionBBoxHeader. */ AV_FRAME_DATA_DETECTION_BBOXES, + /** + * This is RDS data in UECP format, possibly stored in data stream + * elements (DSE) of AAC audio frames. Interpretation of DSE normally + * is dependent on context, UECP radio data system data is identified + * by the STA and STP bytes. UECP RDS data is also common in MPEG-1/2 audio, + * but currently not implemented. + */ + AV_FRAME_DATA_RDS_DATA_PACKET }; enum AVActiveFormatDescription { -- Viele Grüße, Carsten Groß -- Carsten Gross | http://www.siski.de/~carsten/ _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".