Control: tags 739936 patch Control: tags 740421 patch As reported in #740421 libav's HTTP audio streaming is really slow on startup, and as reported in #739936 it doesn't support ICY metadata. So I decided to see if I could merge ffmpeg's HTTP thingy in libav... and it turns out that it was actually rather easy.
Basically I copied libavformat/http.c from ffmpeg to libav, and fixed a bunch
of compile issues. The patch also includes the addtion of av_asprintf() and
av_strtok() since they are needed to build.
Again, I don't know what's libav policy for merging stuff from ffmpeg, but this
patch brings a pretty big improvement to libav's audio streaming support.
(also worth noting is that I only tested this patch with libav's package from
experimental).
Cheers
--
perl -E '$_=q;$/= @{[@_]};and s;\S+;<inidehG ordnasselA>;eg;say~~reverse'
From a0bc33f5fa8290f5724252784cabf4f167f18ebf Mon Sep 17 00:00:00 2001 From: Alessandro Ghedini <alessan...@ghedini.me> Date: Wed, 5 Mar 2014 16:53:03 +0100 Subject: [PATCH] http: merge HTTP parser from ffmpeg Bug-Debian: https://bugs.debian.org/740421 Bug-Debian: https://bugs.debian.org/739936 --- libavformat/http.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++--- libavutil/avstring.c | 55 ++++++++++++ libavutil/avstring.h | 34 +++++++ 3 files changed, 324 insertions(+), 13 deletions(-) diff --git a/libavformat/http.c b/libavformat/http.c index 96f56f8..2e529f3 100644 --- a/libavformat/http.c +++ b/libavformat/http.c @@ -50,18 +50,30 @@ typedef struct { int line_count; int http_code; int64_t chunksize; /**< Used if "Transfer-Encoding: chunked" otherwise -1. */ - int64_t off, filesize; + char *content_type; + char *user_agent; + int64_t off, filesize, req_end_offset; + int icy_data_read; ///< how much data was read since last ICY metadata packet + int icy_metaint; ///< after how many bytes of read data a new metadata packet will be found char *location; HTTPAuthState auth_state; HTTPAuthState proxy_auth_state; char *headers; int willclose; /**< Set if the server correctly handles Connection: close and will close the connection after feeding us the content. */ + int seekable; /**< Control seekability, 0 = disable, 1 = enable, -1 = probe. */ int chunked_post; int end_chunked_post; /**< A flag which indicates if the end of chunked encoding has been sent. */ int end_header; /**< A flag which indicates we have finished to read POST reply. */ int multiple_requests; /**< A flag which indicates if we use persistent connections. */ uint8_t *post_data; int post_datalen; + int is_akamai; + int is_mediagateway; + char *mime_type; + char *cookies; ///< holds newline (\n) delimited Set-Cookie header field values (without the "Set-Cookie: " field name) + int icy; + char *icy_metadata_headers; + char *icy_metadata_packet; #if CONFIG_ZLIB int compressed; z_stream inflate_stream; @@ -74,16 +86,27 @@ typedef struct { #define OFFSET(x) offsetof(HTTPContext, x) #define D AV_OPT_FLAG_DECODING_PARAM #define E AV_OPT_FLAG_ENCODING_PARAM +#define DEFAULT_USER_AGENT "Lavf/" AV_STRINGIFY(LIBAVFORMAT_VERSION) static const AVOption options[] = { +{"seekable", "control seekability of connection", OFFSET(seekable), AV_OPT_TYPE_INT, {.i64 = -1}, -1, 1, D }, {"chunked_post", "use chunked transfer-encoding for posts", OFFSET(chunked_post), AV_OPT_TYPE_INT, {.i64 = 1}, 0, 1, E }, -{"headers", "custom HTTP headers, can override built in default headers", OFFSET(headers), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"headers", "set custom HTTP headers, can override built in default headers", OFFSET(headers), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"content_type", "force a content type", OFFSET(content_type), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"user-agent", "override User-Agent header", OFFSET(user_agent), AV_OPT_TYPE_STRING, {.str = DEFAULT_USER_AGENT}, 0, 0, D }, {"multiple_requests", "use persistent connections", OFFSET(multiple_requests), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, D|E }, -{"post_data", "custom HTTP post data", OFFSET(post_data), AV_OPT_TYPE_BINARY, .flags = D|E }, +{"post_data", "set custom HTTP post data", OFFSET(post_data), AV_OPT_TYPE_BINARY, .flags = D|E }, +{"mime_type", "set MIME type", OFFSET(mime_type), AV_OPT_TYPE_STRING, {0}, 0, 0, 0 }, +{"cookies", "set cookies to be sent in applicable future requests, use newline delimited Set-Cookie HTTP field value syntax", OFFSET(cookies), AV_OPT_TYPE_STRING, {0}, 0, 0, D }, +{"icy", "request ICY metadata", OFFSET(icy), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, D }, +{"icy_metadata_headers", "return ICY metadata headers", OFFSET(icy_metadata_headers), AV_OPT_TYPE_STRING, {0}, 0, 0, 0 }, +{"icy_metadata_packet", "return current ICY metadata packet", OFFSET(icy_metadata_packet), AV_OPT_TYPE_STRING, {0}, 0, 0, 0 }, {"auth_type", "HTTP authentication type", OFFSET(auth_state.auth_type), AV_OPT_TYPE_INT, {.i64 = HTTP_AUTH_NONE}, HTTP_AUTH_NONE, HTTP_AUTH_BASIC, D|E, "auth_type" }, {"none", "No auth method set, autodetect", 0, AV_OPT_TYPE_CONST, {.i64 = HTTP_AUTH_NONE}, 0, 0, D|E, "auth_type" }, {"basic", "HTTP basic authentication", 0, AV_OPT_TYPE_CONST, {.i64 = HTTP_AUTH_BASIC}, 0, 0, D|E, "auth_type" }, {"send_expect_100", "Force sending an Expect: 100-continue header for POST", OFFSET(send_expect_100), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, E }, {"location", "The actual location of the data received", OFFSET(location), AV_OPT_TYPE_STRING, { 0 }, 0, 0, D|E }, +{"offset", "initial byte offset", OFFSET(off), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, D }, +{"end_offset", "try to limit the request to bytes preceding this offset", OFFSET(req_end_offset), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, D }, {NULL} }; #define HTTP_CLASS(flavor)\ @@ -219,6 +242,7 @@ int ff_http_do_new_request(URLContext *h, const char *uri) int ret; s->off = 0; + s->icy_data_read = 0; av_free(s->location); s->location = av_strdup(uri); if (!s->location) @@ -236,7 +260,10 @@ static int http_open(URLContext *h, const char *uri, int flags, HTTPContext *s = h->priv_data; int ret; - h->is_streamed = 1; + if( s->seekable == 1 ) + h->is_streamed = 0; + else + h->is_streamed = 1; s->filesize = -1; s->location = av_strdup(uri); @@ -361,8 +388,9 @@ static int process_line(URLContext *h, char *line, int line_count, if ((slash = strchr(p, '/')) && strlen(slash) > 0) s->filesize = strtoll(slash+1, NULL, 10); } - h->is_streamed = 0; /* we _can_ in fact seek */ - } else if (!av_strcasecmp(tag, "Accept-Ranges") && !strncmp(p, "bytes", 5)) { + if (s->seekable == -1 && (!s->is_akamai || s->filesize != 2147483647)) + h->is_streamed = 0; /* we _can_ in fact seek */ + } else if (!av_strcasecmp(tag, "Accept-Ranges") && !strncmp(p, "bytes", 5) && s->seekable == -1) { h->is_streamed = 0; } else if (!av_strcasecmp (tag, "Transfer-Encoding") && !av_strncasecmp(p, "chunked", 7)) { s->filesize = -1; @@ -376,6 +404,38 @@ static int process_line(URLContext *h, char *line, int line_count, } else if (!av_strcasecmp (tag, "Connection")) { if (!strcmp(p, "close")) s->willclose = 1; + } else if (!av_strcasecmp (tag, "Server")) { + if (!av_strcasecmp (p, "AkamaiGHost")) { + s->is_akamai = 1; + } else if (!av_strncasecmp (p, "MediaGateway", 12)) { + s->is_mediagateway = 1; + } + } else if (!av_strcasecmp (tag, "Content-Type")) { + av_free(s->mime_type); s->mime_type = av_strdup(p); + } else if (!av_strcasecmp (tag, "Set-Cookie")) { + if (!s->cookies) { + if (!(s->cookies = av_strdup(p))) + return AVERROR(ENOMEM); + } else { + char *tmp = s->cookies; + size_t str_size = strlen(tmp) + strlen(p) + 2; + if (!(s->cookies = av_malloc(str_size))) { + s->cookies = tmp; + return AVERROR(ENOMEM); + } + snprintf(s->cookies, str_size, "%s\n%s", tmp, p); + av_free(tmp); + } + } else if (!av_strcasecmp (tag, "Icy-MetaInt")) { + s->icy_metaint = strtoll(p, NULL, 10); + } else if (!av_strncasecmp(tag, "Icy-", 4)) { + // Concat all Icy- header lines + char *buf = av_asprintf("%s%s: %s\n", + s->icy_metadata_headers ? s->icy_metadata_headers : "", tag, p); + if (!buf) + return AVERROR(ENOMEM); + av_freep(&s->icy_metadata_headers); + s->icy_metadata_headers = buf; } else if (!av_strcasecmp (tag, "Content-Encoding")) { if (!av_strncasecmp(p, "gzip", 4) || !av_strncasecmp(p, "deflate", 7)) { #if CONFIG_ZLIB @@ -399,13 +459,112 @@ static int process_line(URLContext *h, char *line, int line_count, // the header at all if this is the case). } else { av_log(h, AV_LOG_WARNING, "Unknown content coding: %s\n", p); - return AVERROR(ENOSYS); } } } return 1; } +/** + * Create a string containing cookie values for use as a HTTP cookie header + * field value for a particular path and domain from the cookie values stored in + * the HTTP protocol context. The cookie string is stored in *cookies. + * + * @return a negative value if an error condition occurred, 0 otherwise + */ +static int get_cookies(HTTPContext *s, char **cookies, const char *path, + const char *domain) +{ + // cookie strings will look like Set-Cookie header field values. Multiple + // Set-Cookie fields will result in multiple values delimited by a newline + int ret = 0; + char *next, *cookie, *set_cookies = av_strdup(s->cookies), *cset_cookies = set_cookies; + + if (!set_cookies) return AVERROR(EINVAL); + + *cookies = NULL; + while ((cookie = av_strtok(set_cookies, "\n", &next))) { + int domain_offset = 0; + char *param, *next_param, *cdomain = NULL, *cpath = NULL, *cvalue = NULL; + set_cookies = NULL; + + while ((param = av_strtok(cookie, "; ", &next_param))) { + cookie = NULL; + if (!av_strncasecmp("path=", param, 5)) { + av_free(cpath); + cpath = av_strdup(¶m[5]); + } else if (!av_strncasecmp("domain=", param, 7)) { + // if the cookie specifies a sub-domain, skip the leading dot thereby + // supporting URLs that point to sub-domains and the master domain + int leading_dot = (param[7] == '.'); + av_free(cdomain); + cdomain = av_strdup(¶m[7+leading_dot]); + } else if (!av_strncasecmp("secure", param, 6) || + !av_strncasecmp("comment", param, 7) || + !av_strncasecmp("max-age", param, 7) || + !av_strncasecmp("version", param, 7)) { + // ignore Comment, Max-Age, Secure and Version + } else { + av_free(cvalue); + cvalue = av_strdup(param); + } + } + if (!cdomain) + cdomain = av_strdup(domain); + + // ensure all of the necessary values are valid + if (!cdomain || !cpath || !cvalue) { + av_log(s, AV_LOG_WARNING, + "Invalid cookie found, no value, path or domain specified\n"); + goto done_cookie; + } + + // check if the request path matches the cookie path + if (av_strncasecmp(path, cpath, strlen(cpath))) + goto done_cookie; + + // the domain should be at least the size of our cookie domain + domain_offset = strlen(domain) - strlen(cdomain); + if (domain_offset < 0) + goto done_cookie; + + // match the cookie domain + if (av_strcasecmp(&domain[domain_offset], cdomain)) + goto done_cookie; + + // cookie parameters match, so copy the value + if (!*cookies) { + if (!(*cookies = av_strdup(cvalue))) { + ret = AVERROR(ENOMEM); + goto done_cookie; + } + } else { + char *tmp = *cookies; + size_t str_size = strlen(cvalue) + strlen(*cookies) + 3; + if (!(*cookies = av_malloc(str_size))) { + ret = AVERROR(ENOMEM); + goto done_cookie; + } + snprintf(*cookies, str_size, "%s; %s", tmp, cvalue); + av_free(tmp); + } + + done_cookie: + av_free(cdomain); + av_free(cpath); + av_free(cvalue); + if (ret < 0) { + if (*cookies) av_freep(cookies); + av_free(cset_cookies); + return ret; + } + } + + av_free(cset_cookies); + + return 0; +} + static inline int has_header(const char *str, const char *header) { /* header + 2 to skip over CRLF prefix. (make sure you have one!) */ @@ -436,6 +595,9 @@ static int http_read_header(URLContext *h, int *new_location) s->line_count++; } + if (s->seekable == -1 && s->is_mediagateway && s->filesize == 2000000000) + h->is_streamed = 1; /* we can in fact _not_ seek */ + return err; } @@ -445,7 +607,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, { HTTPContext *s = h->priv_data; int post, err; - char headers[1024] = ""; + char headers[4096] = ""; char *authstr = NULL, *proxyauthstr = NULL; int64_t off = s->off; int len = 0; @@ -482,14 +644,23 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, /* set default headers if needed */ if (!has_header(s->headers, "\r\nUser-Agent: ")) - len += av_strlcatf(headers + len, sizeof(headers) - len, - "User-Agent: %s\r\n", LIBAVFORMAT_IDENT); + len += av_strlcatf(headers + len, sizeof(headers) - len, + "User-Agent: %s\r\n", s->user_agent); if (!has_header(s->headers, "\r\nAccept: ")) len += av_strlcpy(headers + len, "Accept: */*\r\n", sizeof(headers) - len); - if (!has_header(s->headers, "\r\nRange: ") && !post) + // Note: we send this on purpose even when s->off is 0 when we're probing, + // since it allows us to detect more reliably if a (non-conforming) + // server supports seeking by analysing the reply headers. + if (!has_header(s->headers, "\r\nRange: ") && !post && (s->off > 0 || s->req_end_offset || s->seekable == -1)) { len += av_strlcatf(headers + len, sizeof(headers) - len, - "Range: bytes=%"PRId64"-\r\n", s->off); + "Range: bytes=%"PRId64"-", s->off); + if (s->req_end_offset) + len += av_strlcatf(headers + len, sizeof(headers) - len, + "%"PRId64, s->req_end_offset - 1); + len += av_strlcpy(headers + len, "\r\n", + sizeof(headers) - len); + } if (send_expect_100 && !has_header(s->headers, "\r\nExpect: ")) len += av_strlcatf(headers + len, sizeof(headers) - len, "Expect: 100-continue\r\n"); @@ -510,6 +681,21 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, if (!has_header(s->headers, "\r\nContent-Length: ") && s->post_data) len += av_strlcatf(headers + len, sizeof(headers) - len, "Content-Length: %d\r\n", s->post_datalen); + if (!has_header(s->headers, "\r\nContent-Type: ") && s->content_type) + len += av_strlcatf(headers + len, sizeof(headers) - len, + "Content-Type: %s\r\n", s->content_type); + if (!has_header(s->headers, "\r\nCookie: ") && s->cookies) { + char *cookies = NULL; + if (!get_cookies(s, &cookies, path, hoststr)) { + len += av_strlcatf(headers + len, sizeof(headers) - len, + "Cookie: %s\r\n", cookies); + av_free(cookies); + } + } + if (!has_header(s->headers, "\r\nIcy-MetaData: ") && s->icy) { + len += av_strlcatf(headers + len, sizeof(headers) - len, + "Icy-MetaData: %d\r\n", 1); + } /* now add in custom headers */ if (s->headers) @@ -531,6 +717,9 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, av_freep(&authstr); av_freep(&proxyauthstr); + + av_log(h, AV_LOG_DEBUG, "request: %s\n", s->buffer); + if ((err = ffurl_write(s->hd, s->buffer, strlen(s->buffer))) < 0) return err; @@ -543,6 +732,7 @@ static int http_connect(URLContext *h, const char *path, const char *local_path, s->buf_end = s->buffer; s->line_count = 0; s->off = 0; + s->icy_data_read = 0; s->filesize = -1; s->willclose = 0; s->end_chunked_post = 0; @@ -582,6 +772,7 @@ static int http_buf_read(URLContext *h, uint8_t *buf, int size) } if (len > 0) { s->off += len; + s->icy_data_read += len; if (s->chunksize > 0) s->chunksize -= len; } @@ -655,6 +846,32 @@ static int http_read(URLContext *h, uint8_t *buf, int size) } size = FFMIN(size, s->chunksize); } + if (s->icy_metaint > 0) { + int remaining = s->icy_metaint - s->icy_data_read; /* until next metadata packet */ + if (!remaining) { + // The metadata packet is variable sized. It has a 1 byte header + // which sets the length of the packet (divided by 16). If it's 0, + // the metadata doesn't change. After the packet, icy_metaint bytes + // of normal data follow. + int ch = http_getc(s); + if (ch < 0) + return ch; + if (ch > 0) { + char data[255 * 16 + 1]; + int n; + int ret; + ch *= 16; + for (n = 0; n < ch; n++) + data[n] = http_getc(s); + data[ch + 1] = 0; + if ((ret = av_opt_set(s, "icy_metadata_packet", data, 0)) < 0) + return ret; + } + s->icy_data_read = 0; + remaining = s->icy_metaint; + } + size = FFMIN(size, remaining); + } #if CONFIG_ZLIB if (s->compressed) return http_buf_read_compressed(h, buf, size); @@ -737,6 +954,8 @@ static int64_t http_seek(URLContext *h, int64_t off, int whence) if (whence == AVSEEK_SIZE) return s->filesize; + else if ((whence == SEEK_CUR && off == 0) || (whence == SEEK_SET && off == s->off)) + return s->off; else if ((s->filesize == -1 && whence == SEEK_END) || h->is_streamed) return -1; @@ -824,7 +1043,10 @@ static int http_proxy_open(URLContext *h, const char *uri, int flags) char *authstr; int new_loc; - h->is_streamed = 1; + if( s->seekable == 1 ) + h->is_streamed = 0; + else + h->is_streamed = 1; av_url_split(NULL, 0, auth, sizeof(auth), hostname, sizeof(hostname), &port, pathbuf, sizeof(pathbuf), uri); diff --git a/libavutil/avstring.c b/libavutil/avstring.c index 3ea7be0..6690c2f 100644 --- a/libavutil/avstring.c +++ b/libavutil/avstring.c @@ -108,6 +108,32 @@ size_t av_strlcatf(char *dst, size_t size, const char *fmt, ...) return len; } +char *av_asprintf(const char *fmt, ...) +{ + char *p = NULL; + va_list va; + int len; + + va_start(va, fmt); + len = vsnprintf(NULL, 0, fmt, va); + va_end(va); + if (len < 0) + goto end; + + p = av_malloc(len + 1); + if (!p) + goto end; + + va_start(va, fmt); + len = vsnprintf(p, len + 1, fmt, va); + va_end(va); + if (len < 0) + av_freep(&p); + +end: + return p; +} + char *av_d2str(double d) { char *str = av_malloc(16); @@ -153,6 +179,35 @@ char *av_get_token(const char **buf, const char *term) return ret; } +char *av_strtok(char *s, const char *delim, char **saveptr) +{ + char *tok; + + if (!s && !(s = *saveptr)) + return NULL; + + /* skip leading delimiters */ + s += strspn(s, delim); + + /* s now points to the first non delimiter char, or to the end of the string */ + if (!*s) { + *saveptr = NULL; + return NULL; + } + tok = s++; + + /* skip non delimiters */ + s += strcspn(s, delim); + if (*s) { + *s = 0; + *saveptr = s+1; + } else { + *saveptr = NULL; + } + + return tok; +} + int av_strcasecmp(const char *a, const char *b) { uint8_t c1, c2; diff --git a/libavutil/avstring.h b/libavutil/avstring.h index b7d1098..44e19c8 100644 --- a/libavutil/avstring.h +++ b/libavutil/avstring.h @@ -131,6 +131,16 @@ size_t av_strlcat(char *dst, const char *src, size_t size); size_t av_strlcatf(char *dst, size_t size, const char *fmt, ...) av_printf_format(3, 4); /** + * Print arguments following specified format into a large enough auto + * allocated buffer. It is similar to GNU asprintf(). + * @param fmt printf-compatible format string, specifying how the + * following parameters are used. + * @return the allocated string + * @note You have to free the string yourself with av_free(). + */ +char *av_asprintf(const char *fmt, ...) av_printf_format(1, 2); + +/** * Convert a number to a av_malloced string. */ char *av_d2str(double d); @@ -152,6 +162,30 @@ char *av_d2str(double d); char *av_get_token(const char **buf, const char *term); /** + * Split the string into several tokens which can be accessed by + * successive calls to av_strtok(). + * + * A token is defined as a sequence of characters not belonging to the + * set specified in delim. + * + * On the first call to av_strtok(), s should point to the string to + * parse, and the value of saveptr is ignored. In subsequent calls, s + * should be NULL, and saveptr should be unchanged since the previous + * call. + * + * This function is similar to strtok_r() defined in POSIX.1. + * + * @param s the string to parse, may be NULL + * @param delim 0-terminated list of token delimiters, must be non-NULL + * @param saveptr user-provided pointer which points to stored + * information necessary for av_strtok() to continue scanning the same + * string. saveptr is updated to point to the next character after the + * first delimiter found, or to NULL if the string was terminated + * @return the found token, or NULL when no token is found + */ +char *av_strtok(char *s, const char *delim, char **saveptr); + +/** * Locale-independent conversion of ASCII isdigit. */ int av_isdigit(int c); -- 1.9.0
signature.asc
Description: Digital signature
