To avoid draining the mempools, KASAN shouldn't put the mempool elements
into the quarantine upon mempool_free(). It shouldn't store
allocation/deallocation stacks upon mempool_alloc()/mempool_free() either.
Therefore make kasan_[un]poison_element() just change the shadow memory,
not the metadata.
Signed-off-by: Alexander Potapenko <gli...@google.com>
Reported-by: Kuthonuzo Luruo <kuthonuzo.lu...@hpe.com>
---
include/linux/kasan.h | 8 ++++++--
mm/kasan/kasan.c | 48 +++++++++++++++++++++++++++++++++++++++++++++---
mm/mempool.c | 5 +++--
mm/slab.c | 4 ++--
mm/slab.h | 2 +-
5 files changed, 57 insertions(+), 10 deletions(-)
diff --git a/include/linux/kasan.h b/include/linux/kasan.h
index 611927f..bafc13a 100644
--- a/include/linux/kasan.h
+++ b/include/linux/kasan.h
@@ -63,8 +63,10 @@ void kasan_kfree(void *ptr);
void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size,
gfp_t flags);
void kasan_krealloc(const void *object, size_t new_size, gfp_t flags);
+void kasan_unpoison_kmalloc(const void *object, size_t size, gfp_t flags);
-void kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
+void kasan_slab_alloc(struct kmem_cache *s, void *object, bool just_unpoison,
+ gfp_t flags);
bool kasan_slab_free(struct kmem_cache *s, void *object);
void kasan_poison_slab_free(struct kmem_cache *s, void *object);
@@ -107,9 +109,11 @@ static inline void kasan_kmalloc(struct kmem_cache *s,
const void *object,
size_t size, gfp_t flags) {}
static inline void kasan_krealloc(const void *object, size_t new_size,
gfp_t flags) {}
+static inline void kasan_unpoison_kmalloc(const void *object, size_t size,
+ gfp_t flags) {}
static inline void kasan_slab_alloc(struct kmem_cache *s, void *object,
- gfp_t flags) {}
+ bool just_unpoison, gfp_t flags) {}
static inline bool kasan_slab_free(struct kmem_cache *s, void *object)
{
return false;
diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c
index 18b6a2b..8820f22 100644
--- a/mm/kasan/kasan.c
+++ b/mm/kasan/kasan.c
@@ -503,9 +503,13 @@ struct kasan_free_meta *get_free_info(struct kmem_cache
*cache,
}
#endif
-void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags)
+void kasan_slab_alloc(struct kmem_cache *cache, void *object,
+ bool just_unpoison, gfp_t flags)
{
- kasan_kmalloc(cache, object, cache->object_size, flags);
+ if (just_unpoison)
+ kasan_unpoison_shadow(object, cache->object_size);
+ else
+ kasan_kmalloc(cache, object, cache->object_size, flags);
}
void kasan_poison_slab_free(struct kmem_cache *cache, void *object)
@@ -611,6 +615,31 @@ void kasan_kmalloc_large(const void *ptr, size_t size,
gfp_t flags)
KASAN_PAGE_REDZONE);
}
+void kasan_unpoison_kmalloc(const void *object, size_t size, gfp_t flags)
+{
+ struct page *page;
+ unsigned long redzone_start;
+ unsigned long redzone_end;
+
+ if (unlikely(object == ZERO_SIZE_PTR) || (object == NULL))
+ return;
+
+ page = virt_to_head_page(object);
+ redzone_start = round_up((unsigned long)(object + size),
+ KASAN_SHADOW_SCALE_SIZE);
+
+ if (unlikely(!PageSlab(page)))
+ redzone_end = (unsigned long)object +
+ (PAGE_SIZE << compound_order(page));
+ else
+ redzone_end = round_up(
+ (unsigned long)object + page->slab_cache->object_size,
+ KASAN_SHADOW_SCALE_SIZE);
+ kasan_unpoison_shadow(object, size);
+ kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
+ KASAN_KMALLOC_REDZONE);
+}
+
void kasan_krealloc(const void *object, size_t size, gfp_t flags)
{
struct page *page;
@@ -636,7 +665,20 @@ void kasan_kfree(void *ptr)
kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
KASAN_FREE_PAGE);
else
- kasan_slab_free(page->slab_cache, ptr);
+ kasan_poison_slab_free(page->slab_cache, ptr);
+}
+
+void kasan_poison_kfree(void *ptr)
+{
+ struct page *page;
+
+ page = virt_to_head_page(ptr);
+
+ if (unlikely(!PageSlab(page)))
+ kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
+ KASAN_FREE_PAGE);
+ else
+ kasan_poison_slab_free(page->slab_cache, ptr);
}
void kasan_kfree_large(const void *ptr)
diff --git a/mm/mempool.c b/mm/mempool.c
index 9e075f8..bcd48c6 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -115,9 +115,10 @@ static void kasan_poison_element(mempool_t *pool, void
*element)
static void kasan_unpoison_element(mempool_t *pool, void *element, gfp_t flags)
{
if (pool->alloc == mempool_alloc_slab)
- kasan_slab_alloc(pool->pool_data, element, flags);
+ kasan_slab_alloc(pool->pool_data, element,
+ /*just_unpoison*/ false, flags);
if (pool->alloc == mempool_kmalloc)
- kasan_krealloc(element, (size_t)pool->pool_data, flags);
+ kasan_unpoison_kmalloc(element, (size_t)pool->pool_data, flags);
if (pool->alloc == mempool_alloc_pages)
kasan_alloc_pages(element, (unsigned long)pool->pool_data);
}
diff --git a/mm/slab.c b/mm/slab.c
index cc8bbc1..b42ae23 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3606,7 +3606,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t
flags)
{
void *ret = slab_alloc(cachep, flags, _RET_IP_);
- kasan_slab_alloc(cachep, ret, flags);
+ kasan_slab_alloc(cachep, ret, /*just_unpoison*/ false, flags);
trace_kmem_cache_alloc(_RET_IP_, ret,
cachep->object_size, cachep->size, flags);
@@ -3696,7 +3696,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep,
gfp_t flags, int nodeid)
{
void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
- kasan_slab_alloc(cachep, ret, flags);
+ kasan_slab_alloc(cachep, ret, /*just_unpoison*/ false, flags);
trace_kmem_cache_alloc_node(_RET_IP_, ret,
cachep->object_size, cachep->size,
flags, nodeid);
diff --git a/mm/slab.h b/mm/slab.h
index dedb1a9..d56c042 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -405,7 +405,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache
*s, gfp_t flags,
kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
kmemleak_alloc_recursive(object, s->object_size, 1,
s->flags, flags);
- kasan_slab_alloc(s, object, flags);
+ kasan_slab_alloc(s, object, /*just_unpoison*/false, flags);
}
memcg_kmem_put_cache(s);
}
--
2.8.0.rc3.226.g39d4020
