The first event page is always big enough to handle all events. Handling of multiple events pages is not supported by user mode, and not necessary.
Signed-off-by: Yong Zhao <yong.z...@amd.com> Signed-off-by: Felix Kuehling <felix.kuehl...@amd.com> --- drivers/gpu/drm/amd/amdkfd/kfd_events.c | 197 +++++++++++--------------------- drivers/gpu/drm/amd/amdkfd/kfd_events.h | 1 - drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 +- 3 files changed, 70 insertions(+), 132 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index f178248..f800e48 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -41,6 +41,9 @@ struct kfd_event_waiter { bool activated; /* Becomes true when event is signaled */ }; +#define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT +#define SLOT_BITMAP_LONGS BITS_TO_LONGS(SLOTS_PER_PAGE) + /* * Over-complicated pooled allocator for event notification slots. * @@ -51,132 +54,98 @@ struct kfd_event_waiter { * Individual signal events are then allocated a slot in a page. */ -struct signal_page { - struct list_head event_pages; /* kfd_process.signal_event_pages */ +struct kfd_signal_page { uint64_t *kernel_address; uint64_t __user *user_address; - uint32_t page_index; /* Index into the mmap aperture. */ unsigned int free_slots; - unsigned long used_slot_bitmap[0]; + unsigned long used_slot_bitmap[SLOT_BITMAP_LONGS]; }; -#define SLOTS_PER_PAGE KFD_SIGNAL_EVENT_LIMIT -#define SLOT_BITMAP_SIZE BITS_TO_LONGS(SLOTS_PER_PAGE) -#define BITS_PER_PAGE (ilog2(SLOTS_PER_PAGE)+1) -#define SIGNAL_PAGE_SIZE (sizeof(struct signal_page) + \ - SLOT_BITMAP_SIZE * sizeof(long)) - /* * For signal events, the event ID is used as the interrupt user data. * For SQ s_sendmsg interrupts, this is limited to 8 bits. */ #define INTERRUPT_DATA_BITS 12 -#define SIGNAL_EVENT_ID_SLOT_SHIFT 0 -static uint64_t *page_slots(struct signal_page *page) +static uint64_t *page_slots(struct kfd_signal_page *page) { return page->kernel_address; } static bool allocate_free_slot(struct kfd_process *process, - struct signal_page **out_page, - unsigned int *out_slot_index) + unsigned int *out_slot_index) { - struct signal_page *page; + struct kfd_signal_page *page = process->signal_page; + unsigned int slot; - list_for_each_entry(page, &process->signal_event_pages, event_pages) { - if (page->free_slots > 0) { - unsigned int slot = - find_first_zero_bit(page->used_slot_bitmap, - SLOTS_PER_PAGE); + if (!page || page->free_slots == 0) { + pr_debug("No free event signal slots were found for process %p\n", + process); - __set_bit(slot, page->used_slot_bitmap); - page->free_slots--; + return false; + } - page_slots(page)[slot] = UNSIGNALED_EVENT_SLOT; + slot = find_first_zero_bit(page->used_slot_bitmap, SLOTS_PER_PAGE); - *out_page = page; - *out_slot_index = slot; + __set_bit(slot, page->used_slot_bitmap); + page->free_slots--; - pr_debug("Allocated event signal slot in page %p, slot %d\n", - page, slot); + page_slots(page)[slot] = UNSIGNALED_EVENT_SLOT; - return true; - } - } + *out_slot_index = slot; - pr_debug("No free event signal slots were found for process %p\n", - process); + pr_debug("Allocated event signal slot in page %p, slot %d\n", + page, slot); - return false; + return true; } -#define list_tail_entry(head, type, member) \ - list_entry((head)->prev, type, member) - -static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p) +static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p) { void *backing_store; - struct signal_page *page; + struct kfd_signal_page *page; - page = kzalloc(SIGNAL_PAGE_SIZE, GFP_KERNEL); + page = kzalloc(sizeof(*page), GFP_KERNEL); if (!page) - goto fail_alloc_signal_page; + return NULL; page->free_slots = SLOTS_PER_PAGE; - backing_store = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO, + backing_store = (void *) __get_free_pages(GFP_KERNEL, get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); if (!backing_store) goto fail_alloc_signal_store; - /* prevent user-mode info leaks */ + /* Initialize all events to unsignaled */ memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, - KFD_SIGNAL_EVENT_LIMIT * 8); + KFD_SIGNAL_EVENT_LIMIT * 8); page->kernel_address = backing_store; - - if (list_empty(&p->signal_event_pages)) - page->page_index = 0; - else - page->page_index = list_tail_entry(&p->signal_event_pages, - struct signal_page, - event_pages)->page_index + 1; - pr_debug("Allocated new event signal page at %p, for process %p\n", page, p); - pr_debug("Page index is %d\n", page->page_index); - - list_add(&page->event_pages, &p->signal_event_pages); - return true; + return page; fail_alloc_signal_store: kfree(page); -fail_alloc_signal_page: - return false; + return NULL; } -static bool allocate_event_notification_slot(struct file *devkfd, - struct kfd_process *p, - struct signal_page **page, - unsigned int *signal_slot_index) +static bool allocate_event_notification_slot(struct kfd_process *p, + unsigned int *signal_slot_index) { - bool ret; - - ret = allocate_free_slot(p, page, signal_slot_index); - if (!ret) { - ret = allocate_signal_page(devkfd, p); - if (ret) - ret = allocate_free_slot(p, page, signal_slot_index); + if (!p->signal_page) { + p->signal_page = allocate_signal_page(p); + if (!p->signal_page) + return false; } - return ret; + return allocate_free_slot(p, signal_slot_index); } /* Assumes that the process's event_mutex is locked. */ -static void release_event_notification_slot(struct signal_page *page, +static void release_event_notification_slot(struct kfd_signal_page *page, size_t slot_index) { __clear_bit(slot_index, page->used_slot_bitmap); @@ -187,22 +156,6 @@ static void release_event_notification_slot(struct signal_page *page, */ } -static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p, - unsigned int page_index) -{ - struct signal_page *page; - - /* - * This is safe because we don't delete signal pages until the - * process exits. - */ - list_for_each_entry(page, &p->signal_event_pages, event_pages) - if (page->page_index == page_index) - return page; - - return NULL; -} - /* * Assumes that p->event_mutex is held and of course that p is not going * away (current or locked). @@ -218,13 +171,6 @@ static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id) return NULL; } -static u32 make_signal_event_id(struct signal_page *page, - unsigned int signal_slot_index) -{ - return page->page_index | - (signal_slot_index << SIGNAL_EVENT_ID_SLOT_SHIFT); -} - /* * Produce a kfd event id for a nonsignal event. * These are arbitrary numbers, so we do a sequential search through @@ -270,10 +216,9 @@ static u32 make_nonsignal_event_id(struct kfd_process *p) } static struct kfd_event *lookup_event_by_page_slot(struct kfd_process *p, - struct signal_page *page, unsigned int signal_slot) { - return lookup_event_by_id(p, make_signal_event_id(page, signal_slot)); + return lookup_event_by_id(p, signal_slot); } static int create_signal_event(struct file *devkfd, @@ -288,8 +233,7 @@ static int create_signal_event(struct file *devkfd, return -ENOMEM; } - if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page, - &ev->signal_slot_index)) { + if (!allocate_event_notification_slot(p, &ev->signal_slot_index)) { pr_warn("Signal event wasn't created because out of kernel memory\n"); return -ENOMEM; } @@ -297,10 +241,9 @@ static int create_signal_event(struct file *devkfd, p->signal_event_count++; ev->user_signal_address = - &ev->signal_page->user_address[ev->signal_slot_index]; + &p->signal_page->user_address[ev->signal_slot_index]; - ev->event_id = make_signal_event_id(ev->signal_page, - ev->signal_slot_index); + ev->event_id = ev->signal_slot_index; pr_debug("Signal event number %zu created with id %d, address %p\n", p->signal_event_count, ev->event_id, @@ -327,7 +270,7 @@ void kfd_event_init_process(struct kfd_process *p) { mutex_init(&p->event_mutex); hash_init(p->events); - INIT_LIST_HEAD(&p->signal_event_pages); + p->signal_page = NULL; p->next_nonsignal_event_id = KFD_FIRST_NONSIGNAL_EVENT_ID; p->signal_event_count = 0; } @@ -341,8 +284,9 @@ static void destroy_event(struct kfd_process *p, struct kfd_event *ev) waiter->event = NULL; wake_up_all(&ev->wq); - if (ev->signal_page) { - release_event_notification_slot(ev->signal_page, + if ((ev->type == KFD_EVENT_TYPE_SIGNAL || + ev->type == KFD_EVENT_TYPE_DEBUG) && p->signal_page) { + release_event_notification_slot(p->signal_page, ev->signal_slot_index); p->signal_event_count--; } @@ -365,12 +309,11 @@ static void destroy_events(struct kfd_process *p) * We assume that the process is being destroyed and there is no need to * unmap the pages or keep bookkeeping data in order. */ -static void shutdown_signal_pages(struct kfd_process *p) +static void shutdown_signal_page(struct kfd_process *p) { - struct signal_page *page, *tmp; + struct kfd_signal_page *page = p->signal_page; - list_for_each_entry_safe(page, tmp, &p->signal_event_pages, - event_pages) { + if (page) { free_pages((unsigned long)page->kernel_address, get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); kfree(page); @@ -380,7 +323,7 @@ static void shutdown_signal_pages(struct kfd_process *p) void kfd_event_free_process(struct kfd_process *p) { destroy_events(p); - shutdown_signal_pages(p); + shutdown_signal_page(p); } static bool event_can_be_gpu_signaled(const struct kfd_event *ev) @@ -420,8 +363,7 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p, case KFD_EVENT_TYPE_DEBUG: ret = create_signal_event(devkfd, p, ev); if (!ret) { - *event_page_offset = (ev->signal_page->page_index | - KFD_MMAP_EVENTS_MASK); + *event_page_offset = KFD_MMAP_EVENTS_MASK; *event_page_offset <<= PAGE_SHIFT; *event_slot_index = ev->signal_slot_index; } @@ -523,13 +465,17 @@ int kfd_reset_event(struct kfd_process *p, uint32_t event_id) static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) { - page_slots(ev->signal_page)[ev->signal_slot_index] = + page_slots(p->signal_page)[ev->signal_slot_index] = UNSIGNALED_EVENT_SLOT; } -static bool is_slot_signaled(struct signal_page *page, unsigned int index) +static bool is_slot_signaled(struct kfd_process *p, unsigned int index) { - return page_slots(page)[index] != UNSIGNALED_EVENT_SLOT; + if (!p->signal_page) + return false; + else + return page_slots(p->signal_page)[index] != + UNSIGNALED_EVENT_SLOT; } static void set_event_from_interrupt(struct kfd_process *p, @@ -562,22 +508,19 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id, /* Partial ID is a full ID. */ ev = lookup_event_by_id(p, partial_id); set_event_from_interrupt(p, ev); - } else { + } else if (p->signal_page) { /* * Partial ID is in fact partial. For now we completely * ignore it, but we could use any bits we did receive to * search faster. */ - struct signal_page *page; unsigned int i; - list_for_each_entry(page, &p->signal_event_pages, event_pages) - for (i = 0; i < SLOTS_PER_PAGE; i++) - if (is_slot_signaled(page, i)) { - ev = lookup_event_by_page_slot(p, - page, i); - set_event_from_interrupt(p, ev); - } + for (i = 0; i < SLOTS_PER_PAGE; i++) + if (is_slot_signaled(p, i)) { + ev = lookup_event_by_page_slot(p, i); + set_event_from_interrupt(p, ev); + } } mutex_unlock(&p->event_mutex); @@ -842,9 +785,8 @@ int kfd_wait_on_events(struct kfd_process *p, int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) { - unsigned int page_index; unsigned long pfn; - struct signal_page *page; + struct kfd_signal_page *page; /* check required size is logical */ if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) != @@ -853,13 +795,10 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) return -EINVAL; } - page_index = vma->vm_pgoff; - - page = lookup_signal_page_by_index(p, page_index); + page = p->signal_page; if (!page) { /* Probably KFD bug, but mmap is user-accessible. */ - pr_debug("Signal page could not be found for page_index %u\n", - page_index); + pr_debug("Signal page could not be found\n"); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h index 96f9122..f85fcee 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h @@ -60,7 +60,6 @@ struct kfd_event { wait_queue_head_t wq; /* List of event waiters. */ /* Only for signal events. */ - struct signal_page *signal_page; unsigned int signal_slot_index; uint64_t __user *user_signal_address; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index d3cf53a..c1b3ee2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -540,8 +540,8 @@ struct kfd_process { struct mutex event_mutex; /* All events in process hashed by ID, linked on kfd_event.events. */ DECLARE_HASHTABLE(events, 4); - /* struct slot_page_header.event_pages */ - struct list_head signal_event_pages; + /* Event page */ + struct kfd_signal_page *signal_page; u32 next_nonsignal_event_id; size_t signal_event_count; bool signal_event_limit_reached; -- 2.7.4 _______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx