Re: malloc pages map to user space
On Thursday, March 22, 2012 3:57:11 pm Eric Saint-Etienne wrote: > > If your kernel module creates a device in /dev that implements the > > mmap method, then you don't need to worry about mucking around with > > vm_maps and objects and whatnot. Your mmap method just needs to be > > able to convert offsets into the device into physical memory > > addresses, > > Yes I'm aware of this facility, thank you. > > > and the vm infrastructure will do the rest for you. > > Since this mapping is on the main path of the driver, I'm worried that > the overhead on each access of a page fault and a function call (the > pager associated with a cdev mmap) is too much to bear. It only does this on the first page fault though, not every access to the page. This can be a bit of a downfall as well as you can't easily invalidate a mapping once you've established it. -- John Baldwin ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
Re: malloc pages map to user space
Here is some code which fails with malloc < 1 page
and sometimes succeeds with large mallocs (> 16 pages)
What's wrong?
#include
#include
#include
#include
#include
#include
#include
#include
#include
// Copyright: skeleton took from an older post on the freebsd list
#include
#include
#include
#include
#include
#include
#include
#include
MALLOC_DEFINE(M_FIVEG_SYSC, "fiveg_sysc", "fiveg_sysc test");
struct args {
unsigned char **p;
};
/* String to be located in maped buffer */
#define SIZE PAGE_SIZE // 1 page always fail
static void initialize_array(char *p) {
int i;
for (i = 0; i < 26; i++)
p[i] = i+'a';
p[26] = '!';
p[27] = '\0';
}
static vm_offset_t addr; // allocated/freed at module load/unload
/* Syscall func */
static int syscf(struct thread *td, void *sa) {
vm_offset_t user_addr; /* User space address */
struct args *uap = (struct args*) sa;
struct proc *procp = (struct proc *)td->td_proc;
struct vmspace *vms = procp->p_vmspace;
vm_map_t map;
int result;
vm_object_t object;
vm_ooffset_tobjoffset;
vm_map_entry_t entry;
vm_pindex_t pindex;
vm_prot_t prot;
boolean_t wired;
map = kernel_map; // it always return data within kmeme anyway
uprintf("KERNEL string is '%s' (%p)\n", (char*) addr, (void*) addr);
result = vm_map_lookup(&map, addr, VM_PROT_ALL, &entry, &object,
&pindex, &prot, &wired);
if (result != KERN_SUCCESS) {
uprintf("KERNEL vm_map_lookup failed (%d)\n", result);
return ENOMEM;
}
vm_map_lookup_done(map, entry);
if (object == kernel_object) uprintf("object is kernel_object\n");
else if (object == kmem_object) uprintf("object is kmem_object\n");
else uprintf("object=%p (not kmem, not kernel)\n", object);
uprintf("entry=%p\n", entry);
/* Offset in vm_object */
objoffset = addr - entry->start + entry->offset;
user_addr = 0;
result = vm_map_find(&vms->vm_map, object, objoffset, (vm_offset_t
*) &user_addr, SIZE, VMFS_ANY_SPACE, VM_PROT_RW, VM_PROT_RW, 0);
if (result != KERN_SUCCESS)
uprintf("vm_map_find failed: %d\n", result);
else {
*uap->p = (char*) user_addr;
uprintf("KERNEL ---> Syscall: user_addr for allocating space =
0x%lx\n", user_addr);
}
return result;
}
/* Sysent entity for syscall */
static struct sysent sc_sysent = {
1, /* Number of arguments */
syscf /* Syscall function*/
};
//static struct sysent *old_sysent;
/* Offset in sysent[] */
static int offset = NO_SYSCALL;
/* Loader */
static int
load (struct module *m, int cmd, void *something)
{
int error = 0;
switch(cmd){
case MOD_LOAD:
//MALLOC(addr, vm_offset_t, SIZE, M_FIVEG_SYSC, M_WAITOK | M_ZERO);
addr = (vm_offset_t) malloc(SIZE, M_FIVEG_SYSC, M_WAITOK);
initialize_array((char*) addr);
uprintf("KERNEL Module with sysc loaded. Offset = %d \n", offset);
break;
case MOD_UNLOAD:
free((void*) addr, M_FIVEG_SYSC);
uprintf("KERNEL Module with sysc unloaded. Offset = %d \n", offset);
break;
default:
error = EOPNOTSUPP;
break;
}
return (error);
/* Syscall macro*/
SYSCALL_MODULE(fiveg_sysc, &offset, &sc_sysent, load, NULL);
/* eof */
-- USERLAND PROGRAM
#include
#include
#include
#include
#include
#include
int main(int argc, char *argv[])
{
int sysc_num, error;
struct module_stat mstat;
/* This Variable will save the addres of remapped buffer */
unsigned char *some_var = NULL;
/* Pointer to pointer to remapped buffer */
unsigned char **p = &some_var;
/* Search module with system call */
mstat.version = sizeof(mstat);
if (!(modstat(modfind("sys/fiveg_sysc"), &mstat))){
/* Our module */
sysc_num = mstat.data.intval;
printf("USER: Module found, Syscall number = %d \n", sysc_num);
/* make system call */
error = syscall(sysc_num, p);
if (error != 0) {
printf("USER: an error occured: %d\n", error);
return -1;
}
/* Read the string from remapped buffer */
printf("USER: p = %p\n", p);
printf("USER: *p = %p\n", *p);
printf("USER: String = %s\n", *p);
} else
printf("USER: Module seems to be not loaded! \n");
return 0;
}
/* eof */
___
freebsd-hackers@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-hackers
To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
Re: malloc pages map to user space
> If your kernel module creates a device in /dev that implements the > mmap method, then you don't need to worry about mucking around with > vm_maps and objects and whatnot. Your mmap method just needs to be > able to convert offsets into the device into physical memory > addresses, Yes I'm aware of this facility, thank you. > and the vm infrastructure will do the rest for you. Since this mapping is on the main path of the driver, I'm worried that the overhead on each access of a page fault and a function call (the pager associated with a cdev mmap) is too much to bear. So I'd like to do it the "hard" way which I feel is the most optimized. Thanks! ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
Re: malloc pages map to user space
On Thu, Mar 22, 2012 at 10:42 AM, Eric Saint-Etienne wrote: > Actually when using kernel_map, the object returned is NULL! However the > the vm_entry_t it returns seems a valid address, its 'object' field is NULL > too (that's consistent) > That's the reason why I didn't find it in any existing 'puclic' map (such as > kernel_map, buffers_map, kmem_map, exec_map or pipe_map) > > But a NULL object isn't good at anything and I'm not sure what to do with > a vm_entry_t only... Any idea how to insert it in the process map? If your kernel module creates a device in /dev that implements the mmap method, then you don't need to worry about mucking around with vm_maps and objects and whatnot. Your mmap method just needs to be able to convert offsets into the device into physical memory addresses, and the vm infrastructure will do the rest for you. ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
Re: malloc pages map to user space
> By using kernel_map instead of kmem_map, vm_map_lookup() now always > return a vm_object. That's a big progress. > As expected, when this object is kmem_object, the user mapping works > fine (for smaller or larger mallocs.) > > Otherwise that object doesn't match kernel_object. It's an anonymous > object to me. > Using that "anonymous" vm_object for mapping into user map (using > vm_map_find()) doesn't directly fail, > it does provide a virtual address in the user map. However I read > zeros at that address, from within the user process. Actually when using kernel_map, the object returned is NULL! However the the vm_entry_t it returns seems a valid address, its 'object' field is NULL too (that's consistent) That's the reason why I didn't find it in any existing 'puclic' map (such as kernel_map, buffers_map, kmem_map, exec_map or pipe_map) But a NULL object isn't good at anything and I'm not sure what to do with a vm_entry_t only... Any idea how to insert it in the process map? ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
Re: malloc pages map to user space
On Wednesday, March 21, 2012 7:27:58 pm Eric Saint-Etienne wrote: > Hi, > > >From within the freeBSD kernel, not all malloc are made equal: > * malloc() smaller than KMEM_ZMAX (set to one page size) end up in > UMA SLABs, themselves laid out in powers of 2 (from 16 bytes, 32... to > 4096 bytes) > * bigger malloc() are done through uma_large_malloc() which uses the > kmem wired space > > In my driver, I need to map some malloc-ed memory, obtained from > another module, into userspace. > > The problem: on the smaller mallocs, as well as on some bigeer ones > (8k seems fine, 64k fails): vm_map_lookup() fails finding the > underlying vm object. > > Do somebody know how (or better, have a piece of code!) to retrieve > the vm_object associated with malloc-ed memory? (small and big ones) > > As far as I can see in the vm code, there isn't any object associated > with the slabs (the smaller mallocs), it seems that a huge chunk of > virtual space is used "as is", so I presume the virtual addresses > where the SLABs are have some remarkable property, with respect to > physical addresses, that could allow creating an object from scratch? > > The usual answer is: use mmap(). It seems mmap() is the solution to > everything. But what I dislike with mmap() is the following cost *for > each page*: > 1/ a page fault > 2/ a call to a pager function that will do the "on demand" mapping. You can prefault each page in userland after you call mmap() if you want to pay the cost up front vs. later during runtime. However, there is another option you can use (though it might require you to rework your interfaces a bit, esp. depending on how you get the malloc'd memory in the first place). Specifically, I recently added an extension to the 'shm_open()' API in the kernel that lets you map a shared memory object into the kernel (shm_map() and shm_unmap(), the change should be easy to MFC to 8 and 9 (I use it in 8 and will MFC it soonish to those stable branches)). The way the workflow works in this case is that userland creates a shared memory object (usually an anonymous one, so using SHM_ANON), and then passes the fd in as part of an ioctl request to something in the kernel. The kernel code uses fget() to convert the fd to a reference to a 'struct file *'. You can then pass that file to shm_map(). Once the kernel code is done with the mapping it should call shm_unmap() to release it as well as using fdrop() to release the reference to the 'fp' obtained from fget(). shm_map() will wire down pages for the mapped region of the shm (and alloc them if needed). As a bonus then, if you mmap() the shm in userland after the ioctl, you can pass MAP_PREFAULT_READ to mmap() and it should remove all the page faults in userland. -- John Baldwin ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
Re: malloc pages map to user space
I've refined the behaviour I observe, which isn't consistent depending on the size one mallocates. (see interleaved comments) > In my driver, I need to map some malloc-ed memory, obtained from > another module, into userspace. > > The problem: on the smaller mallocs, as well as on some bigeer ones > (8k seems fine, 64k fails): vm_map_lookup() fails finding the > underlying vm object. In the current implementation I'm calling vm_map_lookup() against the kmem_map. As a result it either return the kmem_object, or fails at all (for smaller or larger mallocs.) > Do somebody know how (or better, have a piece of code!) to retrieve > the vm_object associated with malloc-ed memory? (small and big ones) > > As far as I can see in the vm code, there isn't any object associated > with the slabs (the smaller mallocs), it seems that a huge chunk of > virtual space is used "as is", so I presume the virtual addresses > where the SLABs are have some remarkable property, with respect to > physical addresses, that could allow creating an object from scratch? By using kernel_map instead of kmem_map, vm_map_lookup() now always return a vm_object. That's a big progress. As expected, when this object is kmem_object, the user mapping works fine (for smaller or larger mallocs.) Otherwise that object doesn't match kernel_object. It's an anonymous object to me. Using that "anonymous" vm_object for mapping into user map (using vm_map_find()) doesn't directly fail, it does provide a virtual address in the user map. However I read zeros at that address, from within the user process. Any help would be highly appreciated. Thanks! Eric ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
malloc pages map to user space
Hi, >From within the freeBSD kernel, not all malloc are made equal: * malloc() smaller than KMEM_ZMAX (set to one page size) end up in UMA SLABs, themselves laid out in powers of 2 (from 16 bytes, 32... to 4096 bytes) * bigger malloc() are done through uma_large_malloc() which uses the kmem wired space In my driver, I need to map some malloc-ed memory, obtained from another module, into userspace. The problem: on the smaller mallocs, as well as on some bigeer ones (8k seems fine, 64k fails): vm_map_lookup() fails finding the underlying vm object. Do somebody know how (or better, have a piece of code!) to retrieve the vm_object associated with malloc-ed memory? (small and big ones) As far as I can see in the vm code, there isn't any object associated with the slabs (the smaller mallocs), it seems that a huge chunk of virtual space is used "as is", so I presume the virtual addresses where the SLABs are have some remarkable property, with respect to physical addresses, that could allow creating an object from scratch? The usual answer is: use mmap(). It seems mmap() is the solution to everything. But what I dislike with mmap() is the following cost *for each page*: 1/ a page fault 2/ a call to a pager function that will do the "on demand" mapping. Thank you. Eric ___ freebsd-hackers@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/freebsd-hackers To unsubscribe, send any mail to "freebsd-hackers-unsubscr...@freebsd.org"
