PR #24466 opened by Kacper Michajłow (kasper93) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24466 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24466.patch
From 951f9901d1c44d150f32bae93933ebc5676ca4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <[email protected]> Date: Sun, 13 Sep 2026 03:29:04 +0200 Subject: [PATCH 1/3] avutil/mem: keep poisoned memory undefined for MSan and Valgrind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONFIG_MEMORY_POISONING fills fresh allocations with a pattern. Memory checkers that track initialization take that fill for real data and stop reporting reads of it, which defeats the purpose of the fill. Signed-off-by: Kacper Michajłow <[email protected]> --- configure | 2 ++ libavutil/mem.c | 21 +++++++++++++-------- libavutil/sanitizer.h | 44 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 libavutil/sanitizer.h diff --git a/configure b/configure index 0b428c15d0..178112c810 100755 --- a/configure +++ b/configure @@ -2477,6 +2477,7 @@ HEADERS_LIST=" termios_h udplite_h unistd_h + valgrind_memcheck_h valgrind_valgrind_h windows_h winsock2_h @@ -6940,6 +6941,7 @@ check_headers sys/time.h check_headers sys/un.h check_headers termios.h check_headers unistd.h +check_headers valgrind/memcheck.h check_headers valgrind/valgrind.h check_func_headers VideoToolbox/VTCompressionSession.h VTCompressionSessionPrepareToEncodeFrames -framework VideoToolbox check_func_headers VideoToolbox/VideoToolbox.h VTDecompressionSessionDecodeFrame -framework VideoToolbox diff --git a/libavutil/mem.c b/libavutil/mem.c index e8e5f5f2e2..a796d4dde1 100644 --- a/libavutil/mem.c +++ b/libavutil/mem.c @@ -45,6 +45,7 @@ #include "intreadwrite.h" #include "macros.h" #include "mem.h" +#include "sanitizer.h" #ifdef MALLOC_PREFIX @@ -66,6 +67,14 @@ void free(void *ptr); #define FF_MEMORY_POISON 0x2a +static void poison_memory(void *ptr, size_t size) +{ +#if CONFIG_MEMORY_POISONING + memset(ptr, FF_MEMORY_POISON, size); +#endif + FF_MEM_UNDEFINED(ptr, size); +} + /* NOTE: if you want to override these functions with your own * implementations (not recommended) you have to link libav* as * dynamic libraries and remove -Wl,-Bsymbolic from the linker flags. @@ -145,10 +154,8 @@ void *av_malloc(size_t size) size = 1; ptr= av_malloc(1); } -#if CONFIG_MEMORY_POISONING if (ptr) - memset(ptr, FF_MEMORY_POISON, size); -#endif + poison_memory(ptr, size); return ptr; } @@ -163,10 +170,8 @@ void *av_realloc(void *ptr, size_t size) #else ret = realloc(ptr, size + !size); #endif -#if CONFIG_MEMORY_POISONING if (ret && !ptr) - memset(ret, FF_MEMORY_POISON, size); -#endif + poison_memory(ret, size); return ret; } @@ -347,8 +352,8 @@ void *av_dynarray2_add(void **tab_ptr, int *nb_ptr, size_t elem_size, tab_elem_data = (uint8_t *)*tab_ptr + (*nb_ptr) * elem_size; if (elem_data) memcpy(tab_elem_data, elem_data, elem_size); - else if (CONFIG_MEMORY_POISONING) - memset(tab_elem_data, FF_MEMORY_POISON, elem_size); + else + poison_memory(tab_elem_data, elem_size); }, { av_freep(tab_ptr); *nb_ptr = 0; diff --git a/libavutil/sanitizer.h b/libavutil/sanitizer.h new file mode 100644 index 0000000000..5c3a8728f2 --- /dev/null +++ b/libavutil/sanitizer.h @@ -0,0 +1,44 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_SANITIZER_H +#define AVUTIL_SANITIZER_H + +#include "config.h" + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define HAVE_MSAN 1 +#endif +#endif +#ifndef HAVE_MSAN +#define HAVE_MSAN 0 +#endif + +/* Mark memory as uninitialized. */ +#if HAVE_MSAN +#include <sanitizer/msan_interface.h> +#define FF_MEM_UNDEFINED(ptr, size) __msan_allocated_memory(ptr, size) +#elif HAVE_VALGRIND_MEMCHECK_H +#include <valgrind/memcheck.h> +#define FF_MEM_UNDEFINED(ptr, size) VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) +#else +#define FF_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size)) +#endif + +#endif /* AVUTIL_SANITIZER_H */ -- 2.52.0 From 86337af67efc396f939711b7ccf94caac8b293fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <[email protected]> Date: Sun, 13 Sep 2026 04:17:18 +0200 Subject: [PATCH 2/3] avutil/{buffer,refstruct}: annotate pooled memory for ASan and MSan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Poison the memory when it enters the pool, unpoison it when it is handed out again or freed, and mark it undefined on reuse. Only buffers with av_buffer_default_free are touched, since custom pool allocators may not be compatible with it. Signed-off-by: Kacper Michajłow <[email protected]> --- libavutil/buffer.c | 10 ++++++++++ libavutil/refstruct.c | 5 +++++ libavutil/sanitizer.h | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/libavutil/buffer.c b/libavutil/buffer.c index a8101d83f0..074fc458e8 100644 --- a/libavutil/buffer.c +++ b/libavutil/buffer.c @@ -24,6 +24,7 @@ #include "buffer_internal.h" #include "common.h" #include "mem.h" +#include "sanitizer.h" #include "thread.h" static AVBufferRef *buffer_create(AVBuffer *buf, uint8_t *data, size_t size, @@ -305,6 +306,8 @@ static void buffer_pool_flush(AVBufferPool *pool) BufferPoolEntry *buf = pool->pool; pool->pool = buf->next; + if (buf->free == av_buffer_default_free) + FF_ASAN_UNPOISON(buf->data, pool->size); buf->free(buf->opaque, buf->data); av_freep(&buf); } @@ -347,6 +350,9 @@ static void pool_release_buffer(void *opaque, uint8_t *data) BufferPoolEntry *buf = opaque; AVBufferPool *pool = buf->pool; + if (buf->free == av_buffer_default_free) + FF_ASAN_POISON(buf->data, pool->size); + ff_mutex_lock(&pool->mutex); buf->next = pool->pool; pool->pool = buf; @@ -402,6 +408,10 @@ AVBufferRef *av_buffer_pool_get(AVBufferPool *pool) pool->pool = buf->next; buf->next = NULL; buf->buffer.flags_internal |= BUFFER_FLAG_NO_FREE; + if (buf->free == av_buffer_default_free) { + FF_ASAN_UNPOISON(buf->data, pool->size); + FF_MEM_UNDEFINED(buf->data, pool->size); + } } } else { ret = pool_alloc_buffer(pool); diff --git a/libavutil/refstruct.c b/libavutil/refstruct.c index 7a5eb24826..f7b4059c65 100644 --- a/libavutil/refstruct.c +++ b/libavutil/refstruct.c @@ -21,6 +21,7 @@ #include <string.h> #include "refstruct.h" +#include "sanitizer.h" #include "avassert.h" #include "error.h" @@ -215,6 +216,7 @@ static void pool_free(AVRefStructPool *pool) static void pool_free_entry(AVRefStructPool *pool, RefCount *ref) { + FF_ASAN_UNPOISON(get_userdata(ref), pool->size); if (pool->free_entry_cb) pool->free_entry_cb(pool->opaque, get_userdata(ref)); av_free(ref); @@ -227,6 +229,7 @@ static void pool_return_entry(void *ref_) ff_mutex_lock(&pool->mutex); if (!pool->uninited) { + FF_ASAN_POISON(get_userdata(ref), pool->size); ref->opaque.nc = pool->available_entries; pool->available_entries = ref; ref = NULL; @@ -258,6 +261,8 @@ static int refstruct_pool_get_ext(void *datap, AVRefStructPool *pool) if (pool->available_entries) { RefCount *ref = pool->available_entries; ret = get_userdata(ref); + FF_ASAN_UNPOISON(ret, pool->size); + FF_MEM_UNDEFINED(ret, pool->size); pool->available_entries = ref->opaque.nc; ref->opaque.nc = pool; atomic_init(&ref->refcount, 1); diff --git a/libavutil/sanitizer.h b/libavutil/sanitizer.h index 5c3a8728f2..dedf705729 100644 --- a/libavutil/sanitizer.h +++ b/libavutil/sanitizer.h @@ -22,14 +22,34 @@ #include "config.h" #if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define HAVE_ASAN 1 +#endif #if __has_feature(memory_sanitizer) #define HAVE_MSAN 1 #endif #endif +#if defined(__SANITIZE_ADDRESS__) +#undef HAVE_ASAN +#define HAVE_ASAN 1 +#endif +#ifndef HAVE_ASAN +#define HAVE_ASAN 0 +#endif #ifndef HAVE_MSAN #define HAVE_MSAN 0 #endif +/* Mark allocated memory that nothing may touch until it is unpoisoned. */ +#if HAVE_ASAN +#include <sanitizer/asan_interface.h> +#define FF_ASAN_POISON(ptr, size) __asan_poison_memory_region(ptr, size) +#define FF_ASAN_UNPOISON(ptr, size) __asan_unpoison_memory_region(ptr, size) +#else +#define FF_ASAN_POISON(ptr, size) ((void)(ptr), (void)(size)) +#define FF_ASAN_UNPOISON(ptr, size) ((void)(ptr), (void)(size)) +#endif + /* Mark memory as uninitialized. */ #if HAVE_MSAN #include <sanitizer/msan_interface.h> -- 2.52.0 From 7fccf5d0b531f58c1ce4f0f38d8ff004bdb4db49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <[email protected]> Date: Sun, 13 Sep 2026 03:30:17 +0200 Subject: [PATCH 3/3] avutil/mem: give av_malloc exact redzones under ASan on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LLVM ASan runtime for Windows has no interceptor for the _aligned_malloc family. The UCRT builds it on top of the intercepted malloc, so blocks are tracked and use after free is caught, but the alignment slack on both sides of the returned pointer stays addressable and small overflows and underflows go unnoticed. The MSVC runtime intercepts the family and does not have this problem. This probably could be fixed in compiler-rt upstream, but it's not much code to support it, so let's have it. Signed-off-by: Kacper Michajłow <[email protected]> --- libavutil/mem.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/libavutil/mem.c b/libavutil/mem.c index a796d4dde1..a0e2eb3501 100644 --- a/libavutil/mem.c +++ b/libavutil/mem.c @@ -75,6 +75,69 @@ static void poison_memory(void *ptr, size_t size) FF_MEM_UNDEFINED(ptr, size); } +/* LLVM ASan on Windows does not intercept _aligned_malloc(), leaving the + * alignment unpoisoned. Align manually and poison it. */ +#if HAVE_ASAN && HAVE_ALIGNED_MALLOC && !HAVE_POSIX_MEMALIGN && !HAVE_MEMALIGN +#define ASAN_ALIGNED_ALLOC 1 + +typedef struct AsanAlignedHeader { + void *base; + size_t size; +} AsanAlignedHeader; + +static void *asan_aligned_malloc(size_t size) +{ + AsanAlignedHeader *hdr; + uint8_t *base, *ptr; + size_t total; + + if (size > SIZE_MAX - ALIGN - sizeof(*hdr)) + return NULL; + total = size + ALIGN + sizeof(*hdr); + base = malloc(total); + if (!base) + return NULL; + ptr = (uint8_t *)FFALIGN((uintptr_t)base + sizeof(*hdr), ALIGN); + hdr = (AsanAlignedHeader *)ptr - 1; + hdr->base = base; + hdr->size = size; + FF_ASAN_POISON(base, ptr - base); + FF_ASAN_POISON(ptr + size, base + total - (ptr + size)); + return ptr; +} + +static AsanAlignedHeader *asan_aligned_header(void *ptr) +{ + AsanAlignedHeader *hdr = (AsanAlignedHeader *)ptr - 1; + FF_ASAN_UNPOISON(hdr, sizeof(*hdr)); + return hdr; +} + +static void asan_aligned_free(void *ptr) +{ + if (ptr) + free(asan_aligned_header(ptr)->base); +} + +static void *asan_aligned_realloc(void *ptr, size_t size) +{ + AsanAlignedHeader *hdr; + void *ret; + + if (!ptr) + return asan_aligned_malloc(size); + ret = asan_aligned_malloc(size); + if (!ret) + return NULL; + hdr = asan_aligned_header(ptr); + memcpy(ret, ptr, FFMIN(size, hdr->size)); + free(hdr->base); + return ret; +} +#else +#define ASAN_ALIGNED_ALLOC 0 +#endif + /* NOTE: if you want to override these functions with your own * implementations (not recommended) you have to link libav* as * dynamic libraries and remove -Wl,-Bsymbolic from the linker flags. @@ -111,7 +174,9 @@ void *av_malloc(size_t size) if (size > atomic_load_explicit(&max_alloc_size, memory_order_relaxed)) return NULL; -#if HAVE_POSIX_MEMALIGN +#if ASAN_ALIGNED_ALLOC + ptr = asan_aligned_malloc(size); +#elif HAVE_POSIX_MEMALIGN if (size) //OS X on SDK 10.6 has a broken posix_memalign implementation if (posix_memalign(&ptr, ALIGN, size)) ptr = NULL; @@ -165,7 +230,9 @@ void *av_realloc(void *ptr, size_t size) if (size > atomic_load_explicit(&max_alloc_size, memory_order_relaxed)) return NULL; -#if HAVE_ALIGNED_MALLOC +#if ASAN_ALIGNED_ALLOC + ret = asan_aligned_realloc(ptr, size + !size); +#elif HAVE_ALIGNED_MALLOC ret = _aligned_realloc(ptr, size + !size, ALIGN); #else ret = realloc(ptr, size + !size); @@ -242,7 +309,9 @@ int av_reallocp_array(void *ptr, size_t nmemb, size_t size) void av_free(void *ptr) { -#if HAVE_ALIGNED_MALLOC +#if ASAN_ALIGNED_ALLOC + asan_aligned_free(ptr); +#elif HAVE_ALIGNED_MALLOC _aligned_free(ptr); #else free(ptr); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
