Module Name: src Committed By: maxv Date: Mon Jul 27 09:24:28 UTC 2015
Modified Files: src/sys/kern: subr_kmem.c src/sys/uvm: files.uvm Removed Files: src/sys/uvm: uvm_kmguard.c uvm_kmguard.h Log Message: Several changes and improvements in KMEM_GUARD: - merge uvm_kmguard.{c,h} into subr_kmem.c. It is only user there, and makes it more consistent. Also, it allows us to enable KMEM_GUARD without enabling DEBUG. - rename uvm_kmguard_XXX to kmem_guard_XXX, for consistency - improve kmem_guard_alloc() so that it supports allocations bigger than PAGE_SIZE - remove the canary value, and use directly the kmem header as underflow pattern. - fix some comments (The UAF fifo is disabled for the moment; we actually need to register the va and its size, and add a weight support not to consume too much memory.) To generate a diff of this commit: cvs rdiff -u -r1.60 -r1.61 src/sys/kern/subr_kmem.c cvs rdiff -u -r1.24 -r1.25 src/sys/uvm/files.uvm cvs rdiff -u -r1.11 -r0 src/sys/uvm/uvm_kmguard.c cvs rdiff -u -r1.2 -r0 src/sys/uvm/uvm_kmguard.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/kern/subr_kmem.c diff -u src/sys/kern/subr_kmem.c:1.60 src/sys/kern/subr_kmem.c:1.61 --- src/sys/kern/subr_kmem.c:1.60 Tue Jul 22 07:38:41 2014 +++ src/sys/kern/subr_kmem.c Mon Jul 27 09:24:28 2015 @@ -1,11 +1,11 @@ -/* $NetBSD: subr_kmem.c,v 1.60 2014/07/22 07:38:41 maxv Exp $ */ +/* $NetBSD: subr_kmem.c,v 1.61 2015/07/27 09:24:28 maxv Exp $ */ /*- - * Copyright (c) 2009 The NetBSD Foundation, Inc. + * Copyright (c) 2009-2015 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation - * by Andrew Doran. + * by Andrew Doran and Maxime Villard. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -87,10 +87,10 @@ * Check the pattern on allocation. * * KMEM_GUARD - * A kernel with "option DEBUG" has "kmguard" debugging feature compiled - * in. See the comment in uvm/uvm_kmguard.c for what kind of bugs it tries - * to detect. Even if compiled in, it's disabled by default because it's - * very expensive. You can enable it on boot by: + * A kernel with "option DEBUG" has "kmem_guard" debugging feature compiled + * in. See the comment below for what kind of bugs it tries to detect. Even + * if compiled in, it's disabled by default because it's very expensive. + * You can enable it on boot by: * boot -d * db> w kmem_guard_depth 0t30000 * db> c @@ -100,7 +100,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.60 2014/07/22 07:38:41 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.61 2015/07/27 09:24:28 maxv Exp $"); #include <sys/param.h> #include <sys/callback.h> @@ -112,7 +112,6 @@ __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c, #include <uvm/uvm_extern.h> #include <uvm/uvm_map.h> -#include <uvm/uvm_kmguard.h> #include <lib/libkern/libkern.h> @@ -182,8 +181,10 @@ static size_t kmem_cache_big_maxidx __re #endif /* defined(DIAGNOSTIC) */ #if defined(DEBUG) && defined(_HARDKERNEL) +#define KMEM_SIZE #define KMEM_POISON #define KMEM_GUARD +static void *kmem_freecheck; #endif /* defined(DEBUG) */ #if defined(KMEM_POISON) @@ -222,10 +223,20 @@ static void kmem_size_check(void *, size #ifndef KMEM_GUARD_DEPTH #define KMEM_GUARD_DEPTH 0 #endif +struct kmem_guard { + u_int kg_depth; + intptr_t * kg_fifo; + u_int kg_rotor; + vmem_t * kg_vmem; +}; + +static bool kmem_guard_init(struct kmem_guard *, u_int, vmem_t *); +static void *kmem_guard_alloc(struct kmem_guard *, size_t, bool); +static void kmem_guard_free(struct kmem_guard *, size_t, void *); + int kmem_guard_depth = KMEM_GUARD_DEPTH; -size_t kmem_guard_size; -static struct uvm_kmguard kmem_guard; -static void *kmem_freecheck; +static bool kmem_guard_enabled; +static struct kmem_guard kmem_guard; #endif /* defined(KMEM_GUARD) */ CTASSERT(KM_SLEEP == PR_WAITOK); @@ -246,8 +257,8 @@ kmem_intr_alloc(size_t requested_size, k KASSERT(requested_size > 0); #ifdef KMEM_GUARD - if (requested_size <= kmem_guard_size) { - return uvm_kmguard_alloc(&kmem_guard, requested_size, + if (kmem_guard_enabled) { + return kmem_guard_alloc(&kmem_guard, requested_size, (kmflags & KM_SLEEP) != 0); } #endif @@ -324,8 +335,8 @@ kmem_intr_free(void *p, size_t requested KASSERT(requested_size > 0); #ifdef KMEM_GUARD - if (requested_size <= kmem_guard_size) { - uvm_kmguard_free(&kmem_guard, requested_size, p); + if (kmem_guard_enabled) { + kmem_guard_free(&kmem_guard, requested_size, p); return; } #endif @@ -372,7 +383,6 @@ kmem_intr_free(void *p, size_t requested void * kmem_alloc(size_t size, km_flag_t kmflags) { - KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()), "kmem(9) should not be used from the interrupt context"); return kmem_intr_alloc(size, kmflags); @@ -386,7 +396,6 @@ kmem_alloc(size_t size, km_flag_t kmflag void * kmem_zalloc(size_t size, km_flag_t kmflags) { - KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()), "kmem(9) should not be used from the interrupt context"); return kmem_intr_zalloc(size, kmflags); @@ -400,7 +409,6 @@ kmem_zalloc(size_t size, km_flag_t kmfla void kmem_free(void *p, size_t size) { - KASSERT(!cpu_intr_p()); KASSERT(!cpu_softintr_p()); kmem_intr_free(p, size); @@ -466,9 +474,8 @@ kmem_create_caches(const struct kmem_cac void kmem_init(void) { - #ifdef KMEM_GUARD - uvm_kmguard_init(&kmem_guard, &kmem_guard_depth, &kmem_guard_size, + kmem_guard_enabled = kmem_guard_init(&kmem_guard, kmem_guard_depth, kmem_va_arena); #endif kmem_cache_maxidx = kmem_create_caches(kmem_cache_sizes, @@ -480,10 +487,34 @@ kmem_init(void) size_t kmem_roundup_size(size_t size) { - return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1); } +/* + * Used to dynamically allocate string with kmem accordingly to format. + */ +char * +kmem_asprintf(const char *fmt, ...) +{ + int size __diagused, len; + va_list va; + char *str; + + va_start(va, fmt); + len = vsnprintf(NULL, 0, fmt, va); + va_end(va); + + str = kmem_alloc(len + 1, KM_SLEEP); + + va_start(va, fmt); + size = vsnprintf(str, len + 1, fmt, va); + va_end(va); + + KASSERT(size == len); + + return str; +} + /* ------------------ DEBUG / DIAGNOSTIC ------------------ */ #if defined(KMEM_POISON) || defined(KMEM_REDZONE) @@ -626,27 +657,162 @@ kmem_redzone_check(void *p, size_t sz) #endif /* defined(KMEM_REDZONE) */ +#if defined(KMEM_GUARD) /* - * Used to dynamically allocate string with kmem accordingly to format. - */ -char * -kmem_asprintf(const char *fmt, ...) + * The ultimate memory allocator for debugging, baby. It tries to catch: + * + * 1. Overflow, in realtime. A guard page sits immediately after the + * requested area; a read/write overflow therefore triggers a page + * fault. + * 2. Invalid pointer/size passed, at free. A kmem_header structure sits + * just before the requested area, and holds the allocated size. Any + * difference with what is given at free triggers a panic. + * 3. Underflow, at free. If an underflow occurs, the kmem header will be + * modified, and 2. will trigger a panic. + * 4. Use-after-free. When freeing, the memory is unmapped, and depending + * on the value of kmem_guard_depth, the kernel will more or less delay + * the recycling of that memory. Which means that any ulterior read/write + * access to the memory will trigger a page fault, given it hasn't been + * recycled yet. + */ + +#include <sys/atomic.h> +#include <uvm/uvm.h> + +static bool +kmem_guard_init(struct kmem_guard *kg, u_int depth, vmem_t *vm) +{ + vaddr_t va; + + /* If not enabled, we have nothing to do. */ + if (depth == 0) { + return false; + } + depth = roundup(depth, PAGE_SIZE / sizeof(void *)); + KASSERT(depth != 0); + + /* + * Allocate fifo. + */ + va = uvm_km_alloc(kernel_map, depth * sizeof(void *), PAGE_SIZE, + UVM_KMF_WIRED | UVM_KMF_ZERO); + if (va == 0) { + return false; + } + + /* + * Init object. + */ + kg->kg_vmem = vm; + kg->kg_fifo = (void *)va; + kg->kg_depth = depth; + kg->kg_rotor = 0; + + printf("kmem_guard(%p): depth %d\n", kg, depth); + return true; +} + +static void * +kmem_guard_alloc(struct kmem_guard *kg, size_t requested_size, bool waitok) +{ + struct vm_page *pg; + vm_flag_t flags; + vmem_addr_t va; + vaddr_t loopva; + vsize_t loopsize; + size_t size; + void **p; + + /* + * Compute the size: take the kmem header into account, and add a guard + * page at the end. + */ + size = round_page(requested_size + SIZE_SIZE) + PAGE_SIZE; + + /* Allocate pages of kernel VA, but do not map anything in yet. */ + flags = VM_BESTFIT | (waitok ? VM_SLEEP : VM_NOSLEEP); + if (vmem_alloc(kg->kg_vmem, size, flags, &va) != 0) { + return NULL; + } + + loopva = va; + loopsize = size - PAGE_SIZE; + + while (loopsize) { + pg = uvm_pagealloc(NULL, loopva, NULL, 0); + if (__predict_false(pg == NULL)) { + if (waitok) { + uvm_wait("kmem_guard"); + continue; + } else { + uvm_km_pgremove_intrsafe(kernel_map, va, + va + size); + vmem_free(kg->kg_vmem, va, size); + return NULL; + } + } + + pg->flags &= ~PG_BUSY; /* new page */ + UVM_PAGE_OWN(pg, NULL); + pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg), + VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE); + + loopva += PAGE_SIZE; + loopsize -= PAGE_SIZE; + } + + pmap_update(pmap_kernel()); + + /* + * Offset the returned pointer so that the unmapped guard page sits + * immediately after the returned object. + */ + p = (void **)((va + (size - PAGE_SIZE) - requested_size) & ~(uintptr_t)ALIGNBYTES); + kmem_size_set((uint8_t *)p - SIZE_SIZE, requested_size); + return (void *)p; +} + +static void +kmem_guard_free(struct kmem_guard *kg, size_t requested_size, void *p) { - int size __diagused, len; - va_list va; - char *str; + vaddr_t va; + u_int rotor; + size_t size; + uint8_t *ptr; - va_start(va, fmt); - len = vsnprintf(NULL, 0, fmt, va); - va_end(va); + ptr = (uint8_t *)p - SIZE_SIZE; + kmem_size_check(ptr, requested_size); + va = trunc_page((vaddr_t)ptr); + size = round_page(requested_size + SIZE_SIZE) + PAGE_SIZE; - str = kmem_alloc(len + 1, KM_SLEEP); + KASSERT(pmap_extract(pmap_kernel(), va, NULL)); + KASSERT(!pmap_extract(pmap_kernel(), va + (size - PAGE_SIZE), NULL)); - va_start(va, fmt); - size = vsnprintf(str, len + 1, fmt, va); - va_end(va); + /* + * Unmap and free the pages. The last one is never allocated. + */ + uvm_km_pgremove_intrsafe(kernel_map, va, va + size); + pmap_update(pmap_kernel()); - KASSERT(size == len); +#if 0 + /* + * XXX: Here, we need to atomically register the va and its size in the + * fifo. + */ - return str; + /* + * Put the VA allocation into the list and swap an old one out to free. + * This behaves mostly like a fifo. + */ + rotor = atomic_inc_uint_nv(&kg->kg_rotor) % kg->kg_depth; + va = (vaddr_t)atomic_swap_ptr(&kg->kg_fifo[rotor], (void *)va); + if (va != 0) { + vmem_free(kg->kg_vmem, va, size); + } +#else + (void)rotor; + vmem_free(kg->kg_vmem, va, size); +#endif } + +#endif /* defined(KMEM_GUARD) */ Index: src/sys/uvm/files.uvm diff -u src/sys/uvm/files.uvm:1.24 src/sys/uvm/files.uvm:1.25 --- src/sys/uvm/files.uvm:1.24 Sun Apr 12 12:44:13 2015 +++ src/sys/uvm/files.uvm Mon Jul 27 09:24:28 2015 @@ -1,4 +1,4 @@ -# $NetBSD: files.uvm,v 1.24 2015/04/12 12:44:13 joerg Exp $ +# $NetBSD: files.uvm,v 1.25 2015/07/27 09:24:28 maxv Exp $ # # UVM options @@ -28,7 +28,6 @@ file uvm/uvm_glue.c uvm file uvm/uvm_init.c uvm file uvm/uvm_io.c uvm file uvm/uvm_km.c uvm -file uvm/uvm_kmguard.c debug file uvm/uvm_loan.c uvm file uvm/uvm_map.c uvm file uvm/uvm_meter.c uvm