PR #24151 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24151 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24151.patch
Based on top of https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23476 >From 9b9aefb1d7d4f6fb144567eb2c82b59f393bc4ed Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 14 Jun 2026 11:35:13 +0200 Subject: [PATCH 01/14] avformat/shared: don't acquire more blocks on already errored caches Correctly mirrors the logic used to decide whether to write back to the cache or not. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 5fd01b185e..1a9af58477 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -680,7 +680,7 @@ read_block: av_fallthrough; case BLOCK_NONE: - if (s->read_only) + if (s->read_only || s->write_err) break; /* don't mark block as pending */ if (atomic_compare_exchange_strong_explicit(&block->state, &state, BLOCK_PENDING, -- 2.52.0 >From dd331ffcc1b770216d4b4274b37877f7d6b17632 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 14 Jun 2026 11:21:19 +0200 Subject: [PATCH 02/14] avformat/shared: clean up PENDING state on error/early exit This makes sure threads that fail for reasons other than the underlying I/O failing clean up after their own PENDING state on failure, unless another thread updated the block state in the meantime. Subsumes the existing "is_race" condition, which is inverted to "acquired" that is 1 exactly when we were the thread that set the PENDING state. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 56 +++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 1a9af58477..59ebfa68f0 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -612,7 +612,7 @@ static int shared_read(URLContext *h, unsigned char *buf, int size) Block *const block = &s->spacemap->blocks[block_id]; unsigned state = atomic_load_explicit(&block->state, memory_order_acquire); int64_t pending_since = 0; - int verify_read = 0, is_race = 0; + int verify_read = 0, acquired = 0; retry: switch (state) { @@ -688,6 +688,7 @@ read_block: memory_order_acquire)) { /* Acquired pending state, proceed to fetch the block */ + acquired = 1; state = BLOCK_PENDING; break; } @@ -697,14 +698,11 @@ read_block: case BLOCK_PENDING: /* Another thread is busy fetching this block, wait for it to finish */ if (!s->timeout) { - is_race = 1; break; /* no timeout requested, immediately race to fetch block */ } else if (pending_since) { int64_t new = av_gettime_relative(); - if (new - pending_since >= s->timeout) { - is_race = 1; + if (new - pending_since >= s->timeout) break; /* timeout expired, try to fetch the block ourselves */ - } } else { pending_since = av_gettime_relative(); } @@ -715,6 +713,17 @@ read_block: goto retry; } + /* Release pending state on failure to avoid stalling other threads */ +#define RELEASE_PENDING(block, state) \ + do { \ + if (acquired) { \ + av_assert1(state == BLOCK_PENDING); \ + atomic_compare_exchange_strong_explicit( \ + &block->state, &state, BLOCK_NONE, memory_order_relaxed, \ + memory_order_relaxed); \ + } \ + } while (0) + /* Cache miss, fetch this block from underlying protocol */ s->nb_miss++; @@ -725,15 +734,7 @@ read_block: if (inner_pos < 0) { av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n", av_err2str(inner_pos)); - if (!read_only) { - /* Release pending state to avoid stalling other threads. Don't - * mark this as failed, since the seek error may be unrelated to - * the block and should probably be tried again. */ - atomic_compare_exchange_strong_explicit(&block->state, &state, - BLOCK_NONE, - memory_order_relaxed, - memory_order_relaxed); - } + RELEASE_PENDING(block, state); return inner_pos; } @@ -744,8 +745,10 @@ read_block: if (read_only) { /* Directly defer to the underlying protocol */ ret = ffurl_read(s->inner, buf, size); - if (ret < 0) + if (ret < 0) { + av_assert1(!acquired); return ret; + } /* Verify the read data against the cached data if requested */ if (verify_read && memcmp(buf, tmp, ret)) { @@ -759,7 +762,7 @@ read_block: } int write_back = 1; - if (s->cache_data && !is_race) { + if (s->cache_data && acquired) { /* Read directly into memory mapped cache file */ tmp = s->cache_data + block_pos; write_back = 0; @@ -781,15 +784,16 @@ read_block: else if (ret < 0) { av_log(h, AV_LOG_ERROR, "Failed to read block 0x%"PRIx64": %s\n", block_id, av_err2str(ret)); - int new_state = BLOCK_FAILED; - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT) - new_state = BLOCK_NONE; /* transient error, allow retries */ + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT) { + RELEASE_PENDING(block, state); + return ret; /* transient error, allow retries */ + } /* Try to mark block as failed; ignore errors - any mismatch * here will mean that either another thread already marked it * as failed, or successfully cached it in the meantime */ atomic_compare_exchange_strong_explicit(&block->state, &state, - new_state, + BLOCK_FAILED, memory_order_relaxed, memory_order_relaxed); return ret; @@ -802,8 +806,10 @@ read_block: if (bytes_read < block_size) { /* Learned location of true EOF, update filesize */ ret = set_filesize(h, inner_pos + bytes_read); - if (ret < 0) + if (ret < 0) { + RELEASE_PENDING(block, state); return ret; + } } if (bytes_read > 0) { @@ -812,12 +818,7 @@ read_block: av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n", av_err2str(ret)); s->write_err = 1; - /* Mark as NONE, not FAILED, since the block itself is fine - - * just absent from the cache. */ - atomic_compare_exchange_strong_explicit(&block->state, &state, - BLOCK_NONE, - memory_order_relaxed, - memory_order_relaxed); + RELEASE_PENDING(block, state); } else { uint32_t crc = get_block_crc(tmp, bytes_read); av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at " @@ -826,6 +827,7 @@ read_block: atomic_store_explicit(&block->state, crc, memory_order_release); } } else { + RELEASE_PENDING(block, state); return AVERROR_EOF; } -- 2.52.0 >From 27d997e383e957e698a9c9b487ec022fdad1d4f4 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 14 Jun 2026 11:25:45 +0200 Subject: [PATCH 03/14] avformat/shared: correctly early-exit on read past last block end Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavformat/shared.c b/libavformat/shared.c index 59ebfa68f0..67a0fe4206 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -656,6 +656,8 @@ retry: tmp += (ptrdiff_t) offset; size = FFMIN(size, block_size - offset); + if (size <= 0) + return AVERROR_EOF; if (s->verify) { verify_read = 1; break; /* fall through to the cache miss logic */ -- 2.52.0 >From d78f47f25923e5f0792489d4aaa1cc60480fe812 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 14 Jun 2026 11:43:01 +0200 Subject: [PATCH 04/14] avformat/shared: early exit on prior spacemap I/O failure The code implicitly assumes that the caller won't re-call read()/seek() after observing a prior AVERROR(EIO). However, this is not necessarily a future-proof guarantee, so it's safer to explicitly re-error in this case. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 67a0fe4206..2711d57bcf 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -592,6 +592,8 @@ static int shared_read(URLContext *h, unsigned char *buf, int size) SharedContext *s = h->priv_data; uint8_t *tmp; int ret; + if (!s->spacemap) + return AVERROR(EIO); if (size <= 0) return 0; @@ -845,9 +847,11 @@ read_block: static int64_t shared_seek(URLContext *h, int64_t pos, int whence) { SharedContext *s = h->priv_data; - const int64_t filesize = get_filesize(h); int64_t res; + if (!s->spacemap) + return AVERROR(EIO); + const int64_t filesize = get_filesize(h); switch (whence) { case AVSEEK_SIZE: if (filesize) -- 2.52.0 >From 3745d939fabcc49ed5e21a8163740a6a75691857 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 14 Jun 2026 11:44:33 +0200 Subject: [PATCH 05/14] avformat/shared: hard-error on -cache_verify data mismatch Makes this debug option way more useful. Also matches the existing behavior on CRC mismatches. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 4 ++-- libavformat/shared.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index f7e65b1c9d..9790317d28 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1716,8 +1716,8 @@ be initialized if it does not already exist. @item cache_verify If true, verify any data read from the cache against the underlying input -stream, and report any mismatches. Note that this will make the cache layer -effectively useless. This is purely a debug option. +stream, and error out on any mismatches. Note that this will make the cache +layer effectively useless. This is purely a debug option. @item cache_timeout If set to a nonzero value, specifies the maximum time (in microseconds) to wait diff --git a/libavformat/shared.c b/libavformat/shared.c index 2711d57bcf..189afa7c6d 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -752,6 +752,8 @@ read_block: if (ret < 0) { av_assert1(!acquired); return ret; + } else { + s->pos = s->inner_pos = inner_pos + ret; } /* Verify the read data against the cached data if requested */ @@ -759,9 +761,9 @@ read_block: av_log(h, AV_LOG_ERROR, "Cache verification failed for %d bytes " "in block 0x%"PRIx64" at offset 0x%"PRIx64" + %"PRId64"!\n", ret, block_id, block_pos, offset); + ret = AVERROR(EIO); } - s->pos = s->inner_pos = inner_pos + ret; return ret; } -- 2.52.0 >From 55e087728603d75af3899086d5b71b8b982b522b Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 14 Jun 2026 11:51:22 +0200 Subject: [PATCH 06/14] avformat/shared: allow interruption during BLOCK_PENDING read loop Also allows nonblocking calls. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavformat/shared.c b/libavformat/shared.c index 189afa7c6d..fab7eabeb2 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -711,8 +711,14 @@ read_block: pending_since = av_gettime_relative(); } + if (h->flags & AVIO_FLAG_NONBLOCK) + return AVERROR(EAGAIN); + /* Make sure we try a few times before giving up */ av_usleep(s->timeout >> 4); + if (ff_check_interrupt(&h->interrupt_callback)) + return AVERROR_EXIT; + state = atomic_load_explicit(&block->state, memory_order_acquire); goto retry; } -- 2.52.0 >From 4a4b7e35d9c2acf2bee1963c9bd96946fa2759c0 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 21 Jul 2026 13:32:46 +0200 Subject: [PATCH 07/14] avformat/shared: don't consider EINTR a hard failure Don't trigger `write_err` for this transient failure. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index fab7eabeb2..167006ff2d 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -827,9 +827,11 @@ read_block: if (bytes_read > 0) { ret = write_back ? write_cache(s, tmp, bytes_read, block_pos) : 0; if (ret < 0) { - av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n", - av_err2str(ret)); - s->write_err = 1; + if (ret != AVERROR(EINTR)) { + av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n", + av_err2str(ret)); + s->write_err = 1; + } RELEASE_PENDING(block, state); } else { uint32_t crc = get_block_crc(tmp, bytes_read); -- 2.52.0 >From c4f5db8c8dbef9b84de3fce72aa11c4e6fcf8a5b Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 15:47:27 +0200 Subject: [PATCH 08/14] avformat/shared: don't re-read spacemap block size per call This would otherwise trigger OOB/UB if the spacemap header is corrupted. Instead, read it once during init, verify it there, and then cache that file for the remainder of the process. Sponsored-by: nxtedition AB Reported-by: Mateusz Gierblinski <[email protected]> Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 167006ff2d..5c80701283 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -155,7 +155,7 @@ typedef struct SharedContext { /* options */ char *cache_dir; - int block_shift; ///< requested shift; may disagree with actual + int block_shift; ///< requested shift; updated on init if it disagrees int read_only; int64_t timeout; int retry_errors; @@ -295,7 +295,8 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * if (ret < 0) goto fail; - s->block_size = 1 << atomic_load(&s->spacemap->block_shift); + /* s->block_shift is fully settled after spacemap_init() */ + s->block_size = 1 << s->block_shift; int64_t filesize = get_filesize(h); if (!filesize) { @@ -313,7 +314,7 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * if (filesize > 0) { int64_t last_pos = filesize - 1; - int64_t last_block = last_pos >> atomic_load(&s->spacemap->block_shift); + int64_t last_block = last_pos >> s->block_shift; ret = spacemap_grow(h, last_block); if (ret < 0) goto fail; @@ -513,6 +514,7 @@ static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]) av_log(h, AV_LOG_ERROR, "Invalid block shift %d in cache file!\n", shift); return AVERROR(EINVAL); } + s->block_shift = shift; } for (int i = 0; i < HASH_SIZE; i++) { @@ -602,8 +604,7 @@ static int shared_read(URLContext *h, unsigned char *buf, int size) if (size <= 0) return AVERROR_EOF; - const int shift = atomic_load_explicit(&s->spacemap->block_shift, memory_order_relaxed); - const int64_t block_id = s->pos >> shift; + const int64_t block_id = s->pos >> s->block_shift; const int64_t offset = s->pos & (s->block_size - 1); const int64_t block_pos = block_id * s->block_size; int block_size = clamp_size(h, s->block_size, block_pos); -- 2.52.0 >From 22ff835500160a2ec11ed958c1e8983892dd82e8 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 15:55:59 +0200 Subject: [PATCH 09/14] avformat/shared: don't re-read spacemap filesize per call See previous commit for justification. This commit also addresses another edge case where we neglected to bounds check the stored filesize against the int64_t limits. Sponsored-by: nxtedition AB Reported-by: Mateusz Gierblinski <[email protected]> Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 5c80701283..701b541bfb 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -168,6 +168,7 @@ typedef struct SharedContext { int block_size; int write_err; ///< write error occurred int num_corrupt; + int64_t filesize; ///< once known /* cache file */ uint8_t *cache_data; ///< optional mmap of the cache file @@ -215,7 +216,15 @@ static int spacemap_grow(URLContext *h, int64_t block); static int64_t get_filesize(URLContext *h) { SharedContext *s = h->priv_data; - return atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed); + if (!s->filesize) { + uint64_t size = atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed); + if (size > INT64_MAX) + return AVERROR(EINVAL); + else if (size) + s->filesize = size; + } + + return s->filesize; } static int set_filesize(URLContext *h, int64_t new_size) @@ -299,7 +308,10 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * s->block_size = 1 << s->block_shift; int64_t filesize = get_filesize(h); - if (!filesize) { + if (filesize < 0) { + ret = (int) filesize; + goto fail; + } else if (!filesize) { /* Filesize is not yet known, try to get it from the underlying URL */ filesize = ffurl_size(s->inner); if (filesize < 0 && filesize != AVERROR(ENOSYS)) { @@ -459,7 +471,10 @@ static int spacemap_grow(URLContext *h, int64_t block) /* When streaming files without known size, round up the number of blocks * to the nearest multiple of the block size to reduce the rate of resizes */ - if (!get_filesize(h)) { + int64_t filesize = get_filesize(h); + if (filesize < 0) + return (int) filesize; + else if (filesize) { av_assert0(s->block_size > 0); map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size); } @@ -863,6 +878,9 @@ static int64_t shared_seek(URLContext *h, int64_t pos, int whence) return AVERROR(EIO); const int64_t filesize = get_filesize(h); + if (filesize < 0) + return filesize; + switch (whence) { case AVSEEK_SIZE: if (filesize) -- 2.52.0 >From 4250c950d8d875ce9f3071dae8eca05f5c578dcd Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 16:17:29 +0200 Subject: [PATCH 10/14] avformat/shared: don't try opening spacemap if cache file failed This avoids potentially clobbering `errno` if the s->mapfd fails for a different reason, and is just cleaner in general. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 701b541bfb..b09f423f01 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -292,7 +292,7 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * s->cache_path, s->inner->filename); s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT, 0660); - s->mapfd = avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660); + s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660) : -1; if (s->fd < 0 || s->mapfd < 0) { ret = AVERROR(errno); av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n", -- 2.52.0 >From 134371de842c0da3be23a4f762844360d9c1f23e Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 16:46:42 +0200 Subject: [PATCH 11/14] avformat/shared: add -ignore_errors option This allows the shared: cache protocol to ignore errors in the underlying URL (e.g. HTTP server went down) and continue serving the cached data purely from the cache file. I'm not sure what a good name would be here, since -ignore_errors overlaps somewhat with -retry_errors, but I think the distinction is clear enough. (-ignore_errors keeps going after the inner protocol fails, -retry_errors will retry blocks that previously failed in the inner protocol) Default off because it's a possibly surprising change in behavior. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 6 ++++++ libavformat/shared.c | 46 +++++++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/doc/protocols.texi b/doc/protocols.texi index 9790317d28..83c6451f3a 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1728,6 +1728,12 @@ assumed that the other process may have gotten stuck or died in the meantime. If set to zero, no waiting is done and all processes will immediately race to try and fetch the same missing blocks themselves. Defaults to 10000 (10 ms). +@item ignore_errors +If true, failures of the underlying input stream are not treated as fatal, and +playback continues using only the data already present in the cache file. +Default is false. Any attempt to read into uncached data will result in an +unrecoverable IO error. + @item retry_errors If true (the default), transient read errors from the underlying input stream are ignored and retried again. If false, any blocks that previously failed diff --git a/libavformat/shared.c b/libavformat/shared.c index b09f423f01..8c49864468 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -158,6 +158,7 @@ typedef struct SharedContext { int block_shift; ///< requested shift; updated on init if it disagrees int read_only; int64_t timeout; + int ignore_errors; int retry_errors; int retry_corrupt; int verify; @@ -267,8 +268,10 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * av_strstart(arg, "shared:", &arg); ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback, options, h->protocol_whitelist, h->protocol_blacklist, h); - - if (ret < 0) + if (ret < 0 && s->ignore_errors) { + av_log(h, AV_LOG_WARNING, "Underlying URL failed to open: %s. " + "Continuing with cache file only.\n", av_err2str(ret)); + } else if (ret < 0) goto fail; uint8_t hash[HASH_SIZE]; @@ -289,10 +292,11 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * } av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n", - s->cache_path, s->inner->filename); + s->cache_path, s->inner ? s->inner->filename : arg); - s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT, 0660); - s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660) : -1; + const int mode = O_RDWR | (s->inner ? O_CREAT : 0); + s->fd = avpriv_open(s->cache_path, mode, 0660); + s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, mode, 0660) : -1; if (s->fd < 0 || s->mapfd < 0) { ret = AVERROR(errno); av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n", @@ -313,7 +317,7 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * goto fail; } else if (!filesize) { /* Filesize is not yet known, try to get it from the underlying URL */ - filesize = ffurl_size(s->inner); + filesize = s->inner ? ffurl_size(s->inner) : 0; if (filesize < 0 && filesize != AVERROR(ENOSYS)) { ret = (int) filesize; goto fail; @@ -753,6 +757,13 @@ read_block: /* Cache miss, fetch this block from underlying protocol */ s->nb_miss++; + if (!s->inner) { + av_log(h, AV_LOG_ERROR, "Cache miss for block 0x%"PRIx64" at offset " + "0x%"PRIx64", but underlying protocol is not available!\n", + block_id, block_pos); + return AVERROR(EIO); + } + const int read_only = s->read_only || s->write_err || verify_read; int64_t inner_pos = read_only ? s->pos : block_pos; if (s->inner_pos != inner_pos) { @@ -885,10 +896,15 @@ static int64_t shared_seek(URLContext *h, int64_t pos, int whence) case AVSEEK_SIZE: if (filesize) return filesize; - res = ffurl_seek(s->inner, pos, whence); + res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS); if (res > 0) { if (set_filesize(h, res) < 0) return AVERROR(EINVAL); + } else if (res < 0 && res != AVERROR(ENOSYS) && s->ignore_errors) { + av_log(h, AV_LOG_WARNING, "Underlying URL failed to get size: %s. " + "Continuing with cache file only.\n", av_err2str(res)); + ffurl_closep(&s->inner); + res = AVERROR(ENOSYS); } return res; case SEEK_SET: @@ -901,10 +917,17 @@ static int64_t shared_seek(URLContext *h, int64_t pos, int whence) pos += filesize; break; } + /* Defer to underlying protocol if filesize is unknown */ - res = ffurl_seek(s->inner, pos, whence); - if (res < 0) + res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS); + if (res < 0 && res != AVERROR(ENOSYS) && s->ignore_errors) { + av_log(h, AV_LOG_WARNING, "Underlying URL failed to get seek: %s. " + "Continuing with cache file only.\n", av_err2str(res)); + ffurl_closep(&s->inner); + return AVERROR(ENOSYS); + } else if (res < 0) return res; + /* Opportunistically update known filesize */ if (set_filesize(h, res - pos) < 0) return AVERROR(EINVAL); @@ -924,13 +947,13 @@ static int64_t shared_seek(URLContext *h, int64_t pos, int whence) static int shared_get_file_handle(URLContext *h) { SharedContext *s = h->priv_data; - return ffurl_get_file_handle(s->inner); + return s->inner ? ffurl_get_file_handle(s->inner) : -1; } static int shared_get_short_seek(URLContext *h) { SharedContext *s = h->priv_data; - int ret = ffurl_get_short_seek(s->inner); + int ret = s->inner ? ffurl_get_short_seek(s->inner) : 0; return ret > 0 ? FFMAX(ret, s->block_size) : s->block_size; } @@ -943,6 +966,7 @@ static const AVOption options[] = { { "read_only", "Don't write data to the cache, only read from it", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D }, { "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D }, { "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D }, + { "ignore_errors", "Continue even if the inner URL failed", OFFSET(ignore_errors), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D }, { "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D }, { "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D }, {0}, -- 2.52.0 >From 934b3710cfab29f211e26db2d6dfd1bce901233e Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 16:58:08 +0200 Subject: [PATCH 12/14] avformat/shared: add Spacemap header size assertion Ensures the padding is correctly updated after additions. Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavformat/shared.c b/libavformat/shared.c index 8c49864468..08b538c287 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -34,6 +34,7 @@ #include "url.h" +#include <assert.h> #include <errno.h> #include <fcntl.h> #include <inttypes.h> @@ -128,6 +129,8 @@ typedef struct Spacemap { Block blocks[]; } Spacemap; +static_assert(offsetof(Spacemap, blocks) == 128, "Spacemap header layout mismatch"); + /* Set to value iff the current value is unset (zero) */ #define DEF_SET_ONCE(ctype, atype) \ static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \ -- 2.52.0 >From 8a219f44800fd60b42e6078aa91987438429f34e Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 17:09:17 +0200 Subject: [PATCH 13/14] avformat/shared: add spacemap metadata for the number of cached blocks This can only ever be a lower bound in theory, because a process could crash exactly in between committing the cached block and updating the count, though that edge case is incredibly rare. This is a backwards-compatible addition so it doesn't require a cache version update. New clients will simply see a gross underestimate of the number of cached blocks, which means the upcoming option does not work retroactively, but I consider that a better trade-off than invalidating all existing cache objects for no reason. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- libavformat/shared.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavformat/shared.c b/libavformat/shared.c index 08b538c287..8fd1a08b40 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -124,7 +124,8 @@ typedef struct Spacemap { atomic_ushort block_shift; atomic_ullong filesize; /* byte offset of true EOF, or 0 if unknown */ atomic_uchar hash[HASH_SIZE]; /* hash of resource URI / filename */ - char reserved[80]; + atomic_ullong blocks_cached; /* (lower bound on) the number of blocks cached */ + char reserved[72]; Block blocks[]; } Spacemap; @@ -869,6 +870,7 @@ read_block: "offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id, block_pos, crc); atomic_store_explicit(&block->state, crc, memory_order_release); + atomic_fetch_add_explicit(&s->spacemap->blocks_cached, 1, memory_order_release); } } else { RELEASE_PENDING(block, state); -- 2.52.0 >From 99b1b4146f0ed84e4853839b4cbc7f8d2d92d658 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Fri, 14 Aug 2026 17:24:53 +0200 Subject: [PATCH 14/14] avformat/shared: add -cache_size_max option This allows users to limit the amount of cached data. Note that this is *not* a limit on the size of the file cached. For example, a 10 GB file can work with a 1 GB cache size limit if the user only ever plays some parts of the file (e.g. 1 minute from a 100 minute stream). After this point, s->read_only is simply turned on, giving us the expected fallback behavior for uncached blocks. Sponsored-by: nxtedition AB Signed-off-by: Niklas Haas <[email protected]> --- doc/protocols.texi | 8 ++++++++ libavformat/shared.c | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/doc/protocols.texi b/doc/protocols.texi index 83c6451f3a..75d7812334 100644 --- a/doc/protocols.texi +++ b/doc/protocols.texi @@ -1744,6 +1744,14 @@ If true (the default), blocks whose contents fail the CRC integrity check are re-fetched from the underlying input stream, overwriting the corrupt cached data. If false, cache corruption is treated as a fatal read error. +@item cache_size_max +Maximum amount of data to cache, in bytes. Accepts the usual size suffixes, +e.g. @code{10G} for 10*10^9 bytes, or @code{10Gi} for 10*2^30 bytes. Defaults +to 0, meaning no limit. + +Once this limit is reached, the protocol switches to read-only mode (see +@option{read_only}). Already cached data is never evicted, so once this limit +is reached, no new data will ever be cached until the cache file is deleted. @end table URL Syntax is diff --git a/libavformat/shared.c b/libavformat/shared.c index 8fd1a08b40..bec2a14098 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -166,6 +166,7 @@ typedef struct SharedContext { int retry_errors; int retry_corrupt; int verify; + int64_t cache_size_max; /* misc state */ int64_t pos; ///< current logical position @@ -174,6 +175,7 @@ typedef struct SharedContext { int write_err; ///< write error occurred int num_corrupt; int64_t filesize; ///< once known + int64_t blocks_max; ///< maximum number of blocks to cache /* cache file */ uint8_t *cache_data; ///< optional mmap of the cache file @@ -314,6 +316,7 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * /* s->block_shift is fully settled after spacemap_init() */ s->block_size = 1 << s->block_shift; + s->blocks_max = s->cache_size_max >> s->block_shift; int64_t filesize = get_filesize(h); if (filesize < 0) { @@ -768,6 +771,16 @@ read_block: return AVERROR(EIO); } + if (!s->read_only && s->blocks_max) { + int64_t cached = atomic_load_explicit(&s->spacemap->blocks_cached, memory_order_relaxed); + if (cached >= s->blocks_max) { + av_log(h, AV_LOG_WARNING, "Cache size limit reached (%"PRId64" blocks " + " = %"PRId64" bytes), switching to read-only mode.\n", + s->blocks_max, s->blocks_max << s->block_shift); + s->read_only = 1; + } + } + const int read_only = s->read_only || s->write_err || verify_read; int64_t inner_pos = read_only ? s->pos : block_pos; if (s->inner_pos != inner_pos) { @@ -974,6 +987,7 @@ static const AVOption options[] = { { "ignore_errors", "Continue even if the inner URL failed", OFFSET(ignore_errors), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D }, { "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D }, { "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D }, + { "cache_size_max", "Limit the maximum amount of data cached", OFFSET(cache_size_max), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D }, {0}, }; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
