> On Aug 2, 2018, at 03:50, Ronak <ronak2121-at-yahoo....@ffmpeg.org> wrote:
>
> <0001-libavformat-hlsenc-Fix-HLS-Manifest-Generation-from-.patch>
From e7ff03ed52c709d647a112833427b44c41e3ed12 Mon Sep 17 00:00:00 2001
From: "Ronak Patel (Audible)" <ron...@audible.com>
Date: Tue, 31 Jul 2018 19:05:18 -0400
Subject: [PATCH] libavformat/hlsenc: Fix HLS Manifest Generation from an N^2
algorithm to N.
This fixes the creation of the hls manifest in hlsenc.c by writing the header
once in hls_write_header, and the segments individually in the hls_window
method.
Files that would previously take over a week to fragment now take 1 minute on
the same hardware. This was a 153 hour audio file (2.2GB of audio).
Signed-off-by: Ronak Patel <ronak2...@yahoo.com>
---
libavformat/hlsenc.c | 143 +++++++++++++++++++++++++++------------------------
1 file changed, 76 insertions(+), 67 deletions(-)
diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index b5644f0..b15645d 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -488,7 +488,6 @@ static int hls_delete_old_segments(AVFormatContext *s,
HLSContext *hls,
}
p = (char *)av_basename(dirname);
*p = '\0';
-
}
while (segment) {
@@ -1365,63 +1364,37 @@ fail:
return ret;
}
-static int hls_window(AVFormatContext *s, int last, VariantStream *vs)
+static int hls_write_manifest_segment(AVFormatContext *s, int last,
VariantStream *vs)
{
HLSContext *hls = s->priv_data;
HLSSegment *en;
- int target_duration = 0;
int ret = 0;
- char temp_filename[1024];
- int64_t sequence = FFMAX(hls->start_sequence, vs->sequence -
vs->nb_entries);
- const char *proto = avio_find_protocol_name(s->url);
- int use_rename = proto && !strcmp(proto, "file");
- static unsigned warned_non_file;
+ int byterange_mode;
char *key_uri = NULL;
char *iv_string = NULL;
AVDictionary *options = NULL;
double prog_date_time = vs->initial_prog_date_time;
double *prog_date_time_p = (hls->flags & HLS_PROGRAM_DATE_TIME) ?
&prog_date_time : NULL;
- int byterange_mode = (hls->flags & HLS_SINGLE_FILE) || (hls->max_seg_size
> 0);
-
- hls->version = 3;
- if (byterange_mode) {
- hls->version = 4;
- sequence = 0;
- }
-
- if (hls->flags & HLS_INDEPENDENT_SEGMENTS) {
- hls->version = 6;
- }
- if (hls->segment_type == SEGMENT_TYPE_FMP4) {
- hls->version = 7;
- }
+ if (last) {
- if (!use_rename && !warned_non_file++)
- av_log(s, AV_LOG_ERROR, "Cannot use rename on non file protocol, this
may lead to races and temporary partial files\n");
+ if ((hls->flags & HLS_OMIT_ENDLIST==0)) {
+ ff_hls_write_end_list(hls->m3u8_out);
+ }
- set_http_options(s, &options, hls);
- snprintf(temp_filename, sizeof(temp_filename), use_rename ? "%s.tmp" :
"%s", vs->m3u8_name);
- if ((ret = hlsenc_io_open(s, &hls->m3u8_out, temp_filename, &options)) < 0)
- goto fail;
+ if (vs->vtt_m3u8_name) {
+ ff_hls_write_end_list(hls->sub_m3u8_out);
+ }
+ } else {
- for (en = vs->segments; en; en = en->next) {
- if (target_duration <= en->duration)
- target_duration = lrint(en->duration);
- }
+ byterange_mode = (hls->flags & HLS_SINGLE_FILE) || (hls->max_seg_size
> 0);
- vs->discontinuity_set = 0;
- ff_hls_write_playlist_header(hls->m3u8_out, hls->version, hls->allowcache,
- target_duration, sequence, hls->pl_type);
+ en = vs->last_segment;
+ if (en == NULL) {
+ ret = 1;
+ goto fail;
+ }
- if((hls->flags & HLS_DISCONT_START) && sequence==hls->start_sequence &&
vs->discontinuity_set==0 ){
- avio_printf(hls->m3u8_out, "#EXT-X-DISCONTINUITY\n");
- vs->discontinuity_set = 1;
- }
- if (vs->has_video && (hls->flags & HLS_INDEPENDENT_SEGMENTS)) {
- avio_printf(hls->m3u8_out, "#EXT-X-INDEPENDENT-SEGMENTS\n");
- }
- for (en = vs->segments; en; en = en->next) {
if ((hls->encrypt || hls->key_info_file) && (!key_uri ||
strcmp(en->key_uri, key_uri) ||
av_strcasecmp(en->iv_string, iv_string))) {
avio_printf(hls->m3u8_out, "#EXT-X-KEY:METHOD=AES-128,URI=\"%s\"",
en->key_uri);
@@ -1444,17 +1417,8 @@ static int hls_window(AVFormatContext *s, int last,
VariantStream *vs)
if (ret < 0) {
av_log(s, AV_LOG_WARNING, "ff_hls_write_file_entry get error\n");
}
- }
-
- if (last && (hls->flags & HLS_OMIT_ENDLIST)==0)
- ff_hls_write_end_list(hls->m3u8_out);
- if( vs->vtt_m3u8_name ) {
- if ((ret = hlsenc_io_open(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name,
&options)) < 0)
- goto fail;
- ff_hls_write_playlist_header(hls->sub_m3u8_out, hls->version,
hls->allowcache,
- target_duration, sequence,
PLAYLIST_TYPE_NONE);
- for (en = vs->segments; en; en = en->next) {
+ if( vs->vtt_m3u8_name ) {
ret = ff_hls_write_file_entry(hls->sub_m3u8_out, 0, byterange_mode,
en->duration, 0, en->size, en->pos,
vs->baseurl, en->sub_filename, NULL);
@@ -1462,22 +1426,16 @@ static int hls_window(AVFormatContext *s, int last,
VariantStream *vs)
av_log(s, AV_LOG_WARNING, "ff_hls_write_file_entry get
error\n");
}
}
-
- if (last)
- ff_hls_write_end_list(hls->sub_m3u8_out);
-
}
fail:
av_dict_free(&options);
- hlsenc_io_close(s, &hls->m3u8_out, temp_filename);
- hlsenc_io_close(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name);
- if (ret >= 0 && use_rename)
- ff_rename(temp_filename, vs->m3u8_name, s);
- if (ret >= 0 && hls->master_pl_name)
- if (create_master_playlist(s, vs) < 0)
+ if (ret >= 0 && hls->master_pl_name) {
+ if (create_master_playlist(s, vs) < 0) {
av_log(s, AV_LOG_WARNING, "Master playlist creation failed\n");
+ }
+ }
return ret;
}
@@ -2078,6 +2036,11 @@ static int hls_write_header(AVFormatContext *s)
int ret, i, j;
AVDictionary *options = NULL;
VariantStream *vs = NULL;
+ int target_duration = 0;
+ int byterange_mode = 0;
+
+ target_duration = lrint(hls->time);
+ byterange_mode = (hls->flags & HLS_SINGLE_FILE) || (hls->max_seg_size > 0);
for (i = 0; i < hls->nb_varstreams; i++) {
vs = &hls->var_streams[i];
@@ -2091,7 +2054,7 @@ static int hls_write_header(AVFormatContext *s)
goto fail;
}
av_dict_free(&options);
- //av_assert0(s->nb_streams == hls->avf->nb_streams);
+
for (j = 0; j < vs->nb_streams; j++) {
AVStream *inner_st;
AVStream *outer_st = vs->streams[j];
@@ -2130,6 +2093,47 @@ static int hls_write_header(AVFormatContext *s)
}
}
}
+
+ // write the header for the main hls playlist
+ int64_t sequence = FFMAX(hls->start_sequence, vs->sequence -
vs->nb_entries);
+ hls->version = 3;
+ if (byterange_mode) {
+ hls->version = 4;
+ sequence = 0;
+ }
+
+ if (hls->flags & HLS_INDEPENDENT_SEGMENTS) {
+ hls->version = 6;
+ }
+
+ if (hls->segment_type == SEGMENT_TYPE_FMP4) {
+ hls->version = 7;
+ }
+
+ set_http_options(s, &options, hls);
+ if ((ret = hlsenc_io_open(s, &hls->m3u8_out, vs->m3u8_name, &options)) <
0) {
+ goto fail;
+ }
+
+ vs->discontinuity_set = 0;
+ ff_hls_write_playlist_header(hls->m3u8_out, hls->version, hls->allowcache,
target_duration, sequence, hls->pl_type);
+
+ if((hls->flags & HLS_DISCONT_START) && sequence==hls->start_sequence &&
vs->discontinuity_set==0 ){
+ avio_printf(hls->m3u8_out, "#EXT-X-DISCONTINUITY\n");
+ vs->discontinuity_set = 1;
+ }
+ if (vs->has_video && (hls->flags & HLS_INDEPENDENT_SEGMENTS)) {
+ avio_printf(hls->m3u8_out, "#EXT-X-INDEPENDENT-SEGMENTS\n");
+ }
+
+ if( vs->vtt_m3u8_name ) {
+ if ((ret = hlsenc_io_open(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name,
&options)) < 0) {
+ goto fail;
+ }
+
+ ff_hls_write_playlist_header(hls->sub_m3u8_out, hls->version,
hls->allowcache, target_duration, sequence, PLAYLIST_TYPE_NONE);
+ }
+
fail:
return ret;
@@ -2334,10 +2338,11 @@ static int hls_write_packet(AVFormatContext *s,
AVPacket *pkt)
return ret;
}
- if (!vs->fmp4_init_mode || byterange_mode)
- if ((ret = hls_window(s, 0, vs)) < 0) {
+ if (!vs->fmp4_init_mode || byterange_mode) {
+ if ((ret = hls_write_manifest_segment(s, 0, vs)) < 0) {
return ret;
}
+ }
}
vs->packets_written++;
@@ -2421,7 +2426,7 @@ failed:
avformat_free_context(oc);
vs->avf = NULL;
- hls_window(s, 1, vs);
+ hls_write_manifest_segment(s, 1, vs);
av_freep(&vs->fmp4_init_filename);
if (vtt_oc) {
@@ -2447,6 +2452,9 @@ failed:
av_freep(&ccs->language);
}
+ hlsenc_io_close(s, &hls->m3u8_out, vs->m3u8_name);
+ hlsenc_io_close(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name);
+
ff_format_io_close(s, &hls->m3u8_out);
ff_format_io_close(s, &hls->sub_m3u8_out);
av_freep(&hls->key_basename);
@@ -2884,3 +2892,4 @@ AVOutputFormat ff_hls_muxer = {
.write_trailer = hls_write_trailer,
.priv_class = &hls_class,
};
+
--
2.6.3
I have read this patch some problem for this patch.
1. maybe there will have a problem when duration is not same when every
fragment, for example:
liuqideMacBook-Pro:xxx liuqi$ ./ffmpeg -v quiet -i
~/Movies/Test/bbb_sunflower_1080p_30fps_normal.mp4 -c copy -f hls
-hls_list_size 0 output_test.m3u8
liuqideMacBook-Pro:xxx liuqi$ head -n 10 output_test.m3u8
#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:8
#EXT-X-MEDIA-SEQUENCE:0
#EXTINF:3.866667,
output_test0.ts
#EXTINF:7.300000,
output_test1.ts
#EXTINF:8.333333,
output_test2.ts
the output_test0.ts’s duration is short than output_test1.ts, the
#EXT-X-TARGETDURATION need update to the longest duration.
this operation (check the longest duration) will happen at every fragment write
complete.
it will not update when move the update option to the hls_write_header,
2. the version maybe will update when use hls_segment_type or append_list etc.
when the operation is support from different version m3u8.
3. need update segments vs->segments when hls_list_size option is set.
Thanks
Steven
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
