Module Name: xsrc Committed By: christos Date: Wed Nov 5 17:58:59 UTC 2014
Modified Files: xsrc/external/mit/xf86-video-intel/dist/src: backlight.c xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810: i810_dri.c xsrc/external/mit/xf86-video-intel/dist/src/sna: kgem.c xsrc/external/mit/xf86-video-intel/dist/src/uxa: intel.h Removed Files: xsrc/external/mit/xf86-video-intel/dist/src/sna: sna_dri.c Log Message: merge conflicts To generate a diff of this commit: cvs rdiff -u -r1.1.1.1 -r1.2 \ xsrc/external/mit/xf86-video-intel/dist/src/backlight.c cvs rdiff -u -r1.2 -r1.3 \ xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810/i810_dri.c cvs rdiff -u -r1.2 -r1.3 \ xsrc/external/mit/xf86-video-intel/dist/src/sna/kgem.c cvs rdiff -u -r1.3 -r0 \ xsrc/external/mit/xf86-video-intel/dist/src/sna/sna_dri.c cvs rdiff -u -r1.1.1.2 -r1.2 \ xsrc/external/mit/xf86-video-intel/dist/src/uxa/intel.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: xsrc/external/mit/xf86-video-intel/dist/src/backlight.c diff -u xsrc/external/mit/xf86-video-intel/dist/src/backlight.c:1.1.1.1 xsrc/external/mit/xf86-video-intel/dist/src/backlight.c:1.2 --- xsrc/external/mit/xf86-video-intel/dist/src/backlight.c:1.1.1.1 Wed Nov 5 12:56:20 2014 +++ xsrc/external/mit/xf86-video-intel/dist/src/backlight.c Wed Nov 5 12:58:59 2014 @@ -83,7 +83,7 @@ void backlight_init(struct backlight *b) b->max = -1; } -#ifdef __OpenBSD__ +#if defined(__OpenBSD__) || defined(__NetBSD__) #include <dev/wscons/wsconsio.h> #include <xf86Priv.h> Index: xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810/i810_dri.c diff -u xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810/i810_dri.c:1.2 xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810/i810_dri.c:1.3 --- xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810/i810_dri.c:1.2 Wed Apr 2 22:50:29 2014 +++ xsrc/external/mit/xf86-video-intel/dist/src/legacy/i810/i810_dri.c Wed Nov 5 12:58:59 2014 @@ -7,6 +7,7 @@ #include <string.h> #include <assert.h> +#include "xorg-server.h" #include "xf86.h" #include "xf86_OSproc.h" Index: xsrc/external/mit/xf86-video-intel/dist/src/sna/kgem.c diff -u xsrc/external/mit/xf86-video-intel/dist/src/sna/kgem.c:1.2 xsrc/external/mit/xf86-video-intel/dist/src/sna/kgem.c:1.3 --- xsrc/external/mit/xf86-video-intel/dist/src/sna/kgem.c:1.2 Wed Apr 2 23:59:19 2014 +++ xsrc/external/mit/xf86-video-intel/dist/src/sna/kgem.c Wed Nov 5 12:58:59 2014 @@ -35,7 +35,9 @@ #include <unistd.h> #include <sys/ioctl.h> #include <sys/mman.h> +#include <sys/stat.h> #include <time.h> +#include <sched.h> #include <errno.h> #include <fcntl.h> @@ -59,8 +61,10 @@ static struct kgem_bo * search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags); #define DBG_NO_HW 0 +#define DBG_NO_EXEC 0 #define DBG_NO_TILING 0 #define DBG_NO_CACHE 0 +#define DBG_NO_SNOOP_CACHE 0 #define DBG_NO_CACHE_LEVEL 0 #define DBG_NO_CPU 0 #define DBG_NO_CREATE2 0 @@ -79,6 +83,7 @@ search_snoop_cache(struct kgem *kgem, un #define DBG_NO_HANDLE_LUT 0 #define DBG_NO_WT 0 #define DBG_DUMP 0 +#define DBG_NO_MALLOC_CACHE 0 #define FORCE_MMAP_SYNC 0 /* ((1 << DOMAIN_CPU) | (1 << DOMAIN_GTT)) */ @@ -86,7 +91,8 @@ search_snoop_cache(struct kgem *kgem, un #define DEBUG_SYNC 0 #endif -#define SHOW_BATCH 0 +#define SHOW_BATCH_BEFORE 0 +#define SHOW_BATCH_AFTER 0 #if 0 #define ASSERT_IDLE(kgem__, handle__) assert(!__kgem_busy(kgem__, handle__)) @@ -100,8 +106,12 @@ search_snoop_cache(struct kgem *kgem, un * is being simultaneously being read by the GPU, or within the sampler * prefetch. In general, the chipsets seem to have a requirement that sampler * offsets be aligned to a cacheline (64 bytes). + * + * Actually, it turns out the BLT color pattern (BR15) has the most severe + * alignment restrictions, 64 bytes for 8-bpp, 128 bytes for 16-bpp and 256 + * bytes for 32-bpp. */ -#define UPLOAD_ALIGNMENT 128 +#define UPLOAD_ALIGNMENT 256 #define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) #define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE) @@ -110,12 +120,8 @@ search_snoop_cache(struct kgem *kgem, un #define MAX_CPU_VMA_CACHE INT16_MAX #define MAP_PRESERVE_TIME 10 -#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) | 1)) -#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) | 3)) -#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 2) -#define __MAP_TYPE(ptr) ((uintptr_t)(ptr) & 3) - -#define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) | (ring))) +#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) | 1)) +#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 1) #define LOCAL_I915_PARAM_HAS_BLT 11 #define LOCAL_I915_PARAM_HAS_RELAXED_FENCING 12 @@ -131,7 +137,7 @@ search_snoop_cache(struct kgem *kgem, un #define LOCAL_I915_EXEC_NO_RELOC (1<<11) #define LOCAL_I915_EXEC_HANDLE_LUT (1<<12) -#define LOCAL_I915_GEM_CREATE2 0x32 +#define LOCAL_I915_GEM_CREATE2 0x34 #define LOCAL_IOCTL_I915_GEM_CREATE2 DRM_IOWR (DRM_COMMAND_BASE + LOCAL_I915_GEM_CREATE2, struct local_i915_gem_create2) struct local_i915_gem_create2 { uint64_t size; @@ -153,8 +159,8 @@ struct local_i915_gem_userptr { uint64_t user_ptr; uint64_t user_size; uint32_t flags; -#define I915_USERPTR_READ_ONLY (1<<0) -#define I915_USERPTR_UNSYNCHRONIZED (1<<31) +#define I915_USERPTR_READ_ONLY 0x1 +#define I915_USERPTR_UNSYNCHRONIZED 0x80000000 uint32_t handle; }; @@ -168,7 +174,9 @@ struct local_i915_gem_caching { }; #define LOCAL_I915_GEM_SET_CACHING 0x2f +#define LOCAL_I915_GEM_GET_CACHING 0x30 #define LOCAL_IOCTL_I915_GEM_SET_CACHING DRM_IOW(DRM_COMMAND_BASE + LOCAL_I915_GEM_SET_CACHING, struct local_i915_gem_caching) +#define LOCAL_IOCTL_I915_GEM_GET_CACHING DRM_IOW(DRM_COMMAND_BASE + LOCAL_I915_GEM_GET_CACHING, struct local_i915_gem_caching) struct kgem_buffer { struct kgem_bo base; @@ -176,7 +184,12 @@ struct kgem_buffer { uint32_t used; uint32_t need_io : 1; uint32_t write : 2; - uint32_t mmapped : 1; + uint32_t mmapped : 2; +}; +enum { + MMAPPED_NONE, + MMAPPED_GTT, + MMAPPED_CPU }; static struct kgem_bo *__kgem_freed_bo; @@ -191,6 +204,27 @@ static inline int bytes(struct kgem_bo * #define bucket(B) (B)->size.pages.bucket #define num_pages(B) (B)->size.pages.count +static int do_ioctl(int fd, unsigned long req, void *arg) +{ + int err; + +restart: + if (ioctl(fd, req, arg) == 0) + return 0; + + err = errno; + + if (err == EINTR) + goto restart; + + if (err == EAGAIN) { + sched_yield(); + goto restart; + } + + return -err; +} + #ifdef DEBUG_MEMORY static void debug_alloc(struct kgem *kgem, size_t size) { @@ -202,7 +236,6 @@ static void debug_alloc__bo(struct kgem debug_alloc(kgem, bytes(bo)); } #else -#define debug_alloc(k, b) #define debug_alloc__bo(k, b) #endif @@ -215,12 +248,38 @@ static void assert_tiling(struct kgem *k VG_CLEAR(tiling); tiling.handle = bo->handle; - tiling.tiling_mode = -1; - (void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling); + tiling.tiling_mode = bo->tiling; + (void)do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling); assert(tiling.tiling_mode == bo->tiling); } + +static void assert_cacheing(struct kgem *kgem, struct kgem_bo *bo) +{ + struct local_i915_gem_caching arg; + int expect = kgem->has_llc ? SNOOPED : UNCACHED; + + VG_CLEAR(arg); + arg.handle = bo->handle; + arg.caching = expect; + + (void)do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_GET_CACHING, &arg); + + assert(arg.caching == expect); +} + +static void assert_bo_retired(struct kgem_bo *bo) +{ + DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__, + bo->handle, bo->domain, bo->exec != NULL, bo->rq != NULL)); + assert(bo->refcnt); + assert(bo->rq == NULL); + assert(bo->exec == NULL); + assert(list_is_empty(&bo->request)); +} #else #define assert_tiling(kgem, bo) +#define assert_cacheing(kgem, bo) +#define assert_bo_retired(bo) #endif static void kgem_sna_reset(struct kgem *kgem) @@ -244,20 +303,30 @@ static void kgem_sna_flush(struct kgem * static bool gem_set_tiling(int fd, uint32_t handle, int tiling, int stride) { struct drm_i915_gem_set_tiling set_tiling; - int ret; + int err; if (DBG_NO_TILING) return false; VG_CLEAR(set_tiling); - do { - set_tiling.handle = handle; - set_tiling.tiling_mode = tiling; - set_tiling.stride = stride; - - ret = ioctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling); - } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); - return ret == 0; +restart: + set_tiling.handle = handle; + set_tiling.tiling_mode = tiling; + set_tiling.stride = stride; + + if (ioctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) == 0) + return true; + + err = errno; + if (err == EINTR) + goto restart; + + if (err == EAGAIN) { + sched_yield(); + goto restart; + } + + return false; } static bool gem_set_caching(int fd, uint32_t handle, int caching) @@ -267,7 +336,7 @@ static bool gem_set_caching(int fd, uint VG_CLEAR(arg); arg.handle = handle; arg.caching = caching; - return drmIoctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0; + return do_ioctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0; } static uint32_t gem_userptr(int fd, void *ptr, int size, int read_only) @@ -282,9 +351,9 @@ static uint32_t gem_userptr(int fd, void arg.flags |= I915_USERPTR_READ_ONLY; if (DBG_NO_UNSYNCHRONIZED_USERPTR || - drmIoctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &arg)) { + do_ioctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &arg)) { arg.flags &= ~I915_USERPTR_UNSYNCHRONIZED; - if (drmIoctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &arg)) { + if (do_ioctl(fd, LOCAL_IOCTL_I915_GEM_USERPTR, &arg)) { DBG(("%s: failed to map %p + %d bytes: %d\n", __FUNCTION__, ptr, size, errno)); return 0; @@ -294,15 +363,29 @@ static uint32_t gem_userptr(int fd, void return arg.handle; } -static bool __kgem_throttle_retire(struct kgem *kgem, unsigned flags) +static bool __kgem_throttle(struct kgem *kgem, bool harder) { - if (flags & CREATE_NO_RETIRE) { - DBG(("%s: not retiring per-request\n", __FUNCTION__)); - return false; - } + /* Let this be woken up by sigtimer so that we don't block here + * too much and completely starve X. We will sleep again shortly, + * and so catch up or detect the hang. + */ + do { + if (ioctl(kgem->fd, DRM_IOCTL_I915_GEM_THROTTLE) == 0) { + kgem->need_throttle = 0; + return false; + } + + if (errno == EIO) + return true; + } while (harder); - if (!kgem->need_retire) { - DBG(("%s: nothing to retire\n", __FUNCTION__)); + return false; +} + +static bool __kgem_throttle_retire(struct kgem *kgem, unsigned flags) +{ + if (flags & CREATE_NO_RETIRE || !kgem->need_retire) { + DBG(("%s: not retiring\n", __FUNCTION__)); return false; } @@ -314,7 +397,7 @@ static bool __kgem_throttle_retire(struc return false; } - kgem_throttle(kgem); + __kgem_throttle(kgem, false); return kgem_retire(kgem); } @@ -328,54 +411,60 @@ static void *__kgem_bo_map__gtt(struct k bo->handle, bytes(bo))); assert(bo->proxy == NULL); assert(!bo->snoop); - assert(kgem_bo_can_map(kgem, bo)); + assert(num_pages(bo) <= kgem->aperture_mappable / 4); retry_gtt: VG_CLEAR(mmap_arg); mmap_arg.handle = bo->handle; +#ifdef __NetBSD__ if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg)) { err = errno; - +#else + if ((err = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg))) { +#endif assert(err != EINVAL); (void)__kgem_throttle_retire(kgem, 0); if (kgem_expire_cache(kgem)) goto retry_gtt; - if (kgem->need_expire) { - kgem_cleanup_cache(kgem); + if (kgem_cleanup_cache(kgem)) goto retry_gtt; - } - ErrorF("%s: failed to retrieve GTT offset for handle=%d: %d\n", - __FUNCTION__, bo->handle, err); + ERR(("%s: failed to retrieve GTT offset for handle=%d: %d\n", + __FUNCTION__, bo->handle, -err)); return NULL; } retry_mmap: +#ifdef __NetBSD__ err = -drmMap(kgem->fd, mmap_arg.offset, bytes(bo), &ptr); if (err) { +#else + ptr = mmap(0, bytes(bo), PROT_READ | PROT_WRITE, MAP_SHARED, + kgem->fd, mmap_arg.offset); + if (ptr == MAP_FAILED) { + err = errno; +#endif assert(err != EINVAL); if (__kgem_throttle_retire(kgem, 0)) goto retry_mmap; - if (kgem->need_expire) { - kgem_cleanup_cache(kgem); + if (kgem_cleanup_cache(kgem)) goto retry_mmap; - } - ErrorF("%s: failed to mmap handle=%d, %d bytes, into GTT domain: %d\n", - __FUNCTION__, bo->handle, bytes(bo), err); + ERR(("%s: failed to mmap handle=%d, %d bytes, into GTT domain: %d\n", + __FUNCTION__, bo->handle, bytes(bo), err)); ptr = NULL; } return ptr; } -static int __gem_write(int fd, uint32_t handle, - int offset, int length, - const void *src) +static int gem_write(int fd, uint32_t handle, + int offset, int length, + const void *src) { struct drm_i915_gem_pwrite pwrite; @@ -387,12 +476,12 @@ static int __gem_write(int fd, uint32_t pwrite.offset = offset; pwrite.size = length; pwrite.data_ptr = (uintptr_t)src; - return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite); + return do_ioctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite); } -static int gem_write(int fd, uint32_t handle, - int offset, int length, - const void *src) +static int gem_write__cachealigned(int fd, uint32_t handle, + int offset, int length, + const void *src) { struct drm_i915_gem_pwrite pwrite; @@ -411,7 +500,7 @@ static int gem_write(int fd, uint32_t ha pwrite.size = length; pwrite.data_ptr = (uintptr_t)src; } - return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite); + return do_ioctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite); } static int gem_read(int fd, uint32_t handle, const void *dst, @@ -428,9 +517,9 @@ static int gem_read(int fd, uint32_t han pread.offset = offset; pread.size = length; pread.data_ptr = (uintptr_t)dst; - ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_PREAD, &pread); + ret = do_ioctl(fd, DRM_IOCTL_I915_GEM_PREAD, &pread); if (ret) { - DBG(("%s: failed, errno=%d\n", __FUNCTION__, errno)); + DBG(("%s: failed, errno=%d\n", __FUNCTION__, -ret)); return ret; } @@ -445,7 +534,7 @@ bool __kgem_busy(struct kgem *kgem, int VG_CLEAR(busy); busy.handle = handle; busy.busy = !kgem->wedged; - (void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy); + (void)do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy); DBG(("%s: handle=%d, busy=%d, wedged=%d\n", __FUNCTION__, handle, busy.busy, kgem->wedged)); @@ -461,6 +550,26 @@ static void kgem_bo_retire(struct kgem * assert(list_is_empty(&bo->vma)); if (bo->rq) { + __kgem_bo_clear_busy(bo); + kgem_retire(kgem); + assert_bo_retired(bo); + } else { + assert(bo->exec == NULL); + assert(list_is_empty(&bo->request)); + assert(!bo->needs_flush); + ASSERT_IDLE(kgem, bo->handle); + } +} + +static void kgem_bo_maybe_retire(struct kgem *kgem, struct kgem_bo *bo) +{ + DBG(("%s: retiring bo handle=%d (needed flush? %d), rq? %d [busy?=%d]\n", + __FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL, + __kgem_busy(kgem, bo->handle))); + assert(bo->exec == NULL); + assert(list_is_empty(&bo->vma)); + + if (bo->rq) { if (!__kgem_busy(kgem, bo->handle)) { __kgem_bo_clear_busy(bo); kgem_retire(kgem); @@ -474,20 +583,33 @@ static void kgem_bo_retire(struct kgem * bool kgem_bo_write(struct kgem *kgem, struct kgem_bo *bo, const void *data, int length) { + int err; + assert(bo->refcnt); - assert(!bo->purged); assert(bo->proxy == NULL); ASSERT_IDLE(kgem, bo->handle); assert(length <= bytes(bo)); - if (gem_write(kgem->fd, bo->handle, 0, length, data)) +retry: + if ((err = gem_write(kgem->fd, bo->handle, 0, length, data))) { + assert(err != EINVAL); + + (void)__kgem_throttle_retire(kgem, 0); + if (kgem_expire_cache(kgem)) + goto retry; + + if (kgem_cleanup_cache(kgem)) + goto retry; + + ERR(("%s: failed to write %d bytes into BO handle=%d: %d\n", + __FUNCTION__, length, bo->handle, -err)); return false; + } DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain)); - if (bo->exec == NULL) { - kgem_bo_retire(kgem, bo); - bo->domain = DOMAIN_NONE; - } + if (bo->exec == NULL) + kgem_bo_maybe_retire(kgem, bo); + bo->domain = DOMAIN_NONE; bo->gtt_dirty = true; return true; } @@ -499,7 +621,7 @@ static uint32_t gem_create(int fd, int n VG_CLEAR(create); create.handle = 0; create.size = PAGE_SIZE * num_pages; - (void)drmIoctl(fd, DRM_IOCTL_I915_GEM_CREATE, &create); + (void)do_ioctl(fd, DRM_IOCTL_I915_GEM_CREATE, &create); return create.handle; } @@ -518,7 +640,7 @@ kgem_bo_set_purgeable(struct kgem *kgem, VG_CLEAR(madv); madv.handle = bo->handle; madv.madv = I915_MADV_DONTNEED; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) { + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) { bo->purged = 1; kgem->need_purge |= !madv.retained && bo->domain == DOMAIN_GPU; return madv.retained; @@ -542,7 +664,7 @@ kgem_bo_is_retained(struct kgem *kgem, s VG_CLEAR(madv); madv.handle = bo->handle; madv.madv = I915_MADV_DONTNEED; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) return madv.retained; return false; @@ -562,7 +684,7 @@ kgem_bo_clear_purgeable(struct kgem *kge VG_CLEAR(madv); madv.handle = bo->handle; madv.madv = I915_MADV_WILLNEED; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) { + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) { bo->purged = !madv.retained; kgem->need_purge |= !madv.retained && bo->domain == DOMAIN_GPU; return madv.retained; @@ -578,7 +700,7 @@ static void gem_close(int fd, uint32_t h VG_CLEAR(close); close.handle = handle; - (void)drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, &close); + (void)do_ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close); } constant inline static unsigned long __fls(unsigned long word) @@ -606,6 +728,8 @@ constant inline static int cache_bucket( static struct kgem_bo *__kgem_bo_init(struct kgem_bo *bo, int handle, int num_pages) { + DBG(("%s(handle=%d, num_pages=%d)\n", __FUNCTION__, handle, num_pages)); + assert(num_pages); memset(bo, 0, sizeof(*bo)); @@ -662,8 +786,12 @@ static struct kgem_request *__kgem_reque static void __kgem_request_free(struct kgem_request *rq) { _list_del(&rq->list); - *(struct kgem_request **)rq = __kgem_freed_request; - __kgem_freed_request = rq; + if (DBG_NO_MALLOC_CACHE) { + free(rq); + } else { + *(struct kgem_request **)rq = __kgem_freed_request; + __kgem_freed_request = rq; + } } static struct list *inactive(struct kgem *kgem, int num_pages) @@ -708,7 +836,7 @@ total_ram_size(void) static unsigned cpu_cache_size__cpuid4(void) { - /* Deterministic Cache Parmaeters (Function 04h)": + /* Deterministic Cache Parameters (Function 04h)": * When EAX is initialized to a value of 4, the CPUID instruction * returns deterministic cache information in the EAX, EBX, ECX * and EDX registers. This function requires ECX be initialized @@ -724,11 +852,12 @@ cpu_cache_size__cpuid4(void) unsigned int eax, ebx, ecx, edx; unsigned int llc_size = 0; - int cnt = 0; + int cnt; if (__get_cpuid_max(BASIC_CPUID, NULL) < 4) return 0; + cnt = 0; do { unsigned associativity, line_partitions, line_size, sets; @@ -789,7 +918,7 @@ static int gem_param(struct kgem *kgem, VG_CLEAR(gp); gp.param = name; gp.value = &v; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GETPARAM, &gp)) + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GETPARAM, &gp)) return -1; VG(VALGRIND_MAKE_MEM_DEFINED(&v, sizeof(v))); @@ -803,10 +932,9 @@ static bool test_has_execbuffer2(struct memset(&execbuf, 0, sizeof(execbuf)); execbuf.buffer_count = 1; - return (drmIoctl(kgem->fd, + return do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, - &execbuf) == -1 && - errno == EFAULT); + &execbuf) == -EFAULT; } static bool test_has_no_reloc(struct kgem *kgem) @@ -827,14 +955,10 @@ static bool test_has_handle_lut(struct k static bool test_has_wt(struct kgem *kgem) { -#if defined(USE_WT) if (DBG_NO_WT) return false; return gem_param(kgem, LOCAL_I915_PARAM_HAS_WT) > 0; -#else - return false; -#endif } static bool test_has_semaphores_enabled(struct kgem *kgem) @@ -861,18 +985,6 @@ static bool test_has_semaphores_enabled( return detected; } -static bool __kgem_throttle(struct kgem *kgem) -{ - /* Let this be woken up by sigtimer so that we don't block here - * too much and completely starve X. We will sleep again shortly, - * and so catch up or detect the hang. - */ - if (ioctl(kgem->fd, DRM_IOCTL_I915_GEM_THROTTLE) == 0) - return false; - - return errno == EIO; -} - static bool is_hw_supported(struct kgem *kgem, struct pci_device *dev) { @@ -890,7 +1002,7 @@ static bool is_hw_supported(struct kgem * hw acceleration. */ - if (kgem->gen == 060 && dev->revision < 8) { + if (kgem->gen == 060 && dev && dev->revision < 8) { /* pre-production SNB with dysfunctional BLT */ return false; } @@ -953,7 +1065,6 @@ static bool test_has_caching(struct kgem static bool test_has_userptr(struct kgem *kgem) { -#if defined(USE_USERPTR) uint32_t handle; void *ptr; @@ -964,6 +1075,9 @@ static bool test_has_userptr(struct kgem if (kgem->gen == 040) return false; + if (kgem->gen >= 0100) + return false; /* FIXME https://bugs.freedesktop.org/show_bug.cgi?id=79053 */ + if (posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE)) return false; @@ -972,9 +1086,6 @@ static bool test_has_userptr(struct kgem free(ptr); return handle != 0; -#else - return false; -#endif } static bool test_has_create2(struct kgem *kgem) @@ -988,7 +1099,7 @@ static bool test_has_create2(struct kgem memset(&args, 0, sizeof(args)); args.size = PAGE_SIZE; args.caching = DISPLAY; - if (drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args) == 0) + if (do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args) == 0) gem_close(kgem->fd, args.handle); return args.handle != 0; @@ -1019,6 +1130,38 @@ static int kgem_get_screen_index(struct return sna->scrn->scrnIndex; } +static int __find_debugfs(struct kgem *kgem) +{ + int i; + + for (i = 0; i < DRM_MAX_MINOR; i++) { + char path[80]; + + sprintf(path, "/sys/kernel/debug/dri/%d/i915_wedged", i); + if (access(path, R_OK) == 0) + return i; + + sprintf(path, "/debug/dri/%d/i915_wedged", i); + if (access(path, R_OK) == 0) + return i; + } + + return -1; +} + +static int kgem_get_minor(struct kgem *kgem) +{ + struct stat st; + + if (fstat(kgem->fd, &st)) + return __find_debugfs(kgem); + + if (!S_ISCHR(st.st_mode)) + return __find_debugfs(kgem); + + return st.st_rdev & 0x63; +} + static bool kgem_init_pinned_batches(struct kgem *kgem) { int count[2] = { 16, 4 }; @@ -1049,8 +1192,9 @@ static bool kgem_init_pinned_batches(str } pin.alignment = 0; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) { + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) { gem_close(kgem->fd, pin.handle); + free(bo); goto err; } bo->presumed_offset = pin.offset; @@ -1106,7 +1250,7 @@ static void kgem_init_swizzling(struct k if (!gem_set_tiling(kgem->fd, tiling.handle, I915_TILING_X, 512)) goto out; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling)) + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling)) goto out; choose_memcpy_tiled_x(kgem, tiling.swizzle_mode); @@ -1202,6 +1346,10 @@ void kgem_init(struct kgem *kgem, int fd DBG(("%s: can blt to cpu? %d\n", __FUNCTION__, kgem->can_blt_cpu)); + kgem->can_render_y = gen != 021 && (gen >> 3) != 4; + DBG(("%s: can render to Y-tiled surfaces? %d\n", __FUNCTION__, + kgem->can_render_y)); + kgem->has_secure_batches = test_has_secure_batches(kgem); DBG(("%s: can use privileged batchbuffers? %d\n", __FUNCTION__, kgem->has_secure_batches)); @@ -1214,7 +1362,7 @@ void kgem_init(struct kgem *kgem, int fd xf86DrvMsg(kgem_get_screen_index(kgem), X_WARNING, "Detected unsupported/dysfunctional hardware, disabling acceleration.\n"); kgem->wedged = 1; - } else if (__kgem_throttle(kgem)) { + } else if (__kgem_throttle(kgem, false)) { xf86DrvMsg(kgem_get_screen_index(kgem), X_WARNING, "Detected a hung GPU, disabling acceleration.\n"); kgem->wedged = 1; @@ -1227,7 +1375,7 @@ void kgem_init(struct kgem *kgem, int fd if (gen == 022) /* 865g cannot handle a batch spanning multiple pages */ kgem->batch_size = PAGE_SIZE / sizeof(uint32_t); - if ((gen >> 3) == 7) + if (gen >= 070) kgem->batch_size = 16*1024; if (!kgem->has_relaxed_delta && kgem->batch_size > 4*1024) kgem->batch_size = 4*1024; @@ -1241,10 +1389,6 @@ void kgem_init(struct kgem *kgem, int fd DBG(("%s: maximum batch size? %d\n", __FUNCTION__, kgem->batch_size)); - kgem->min_alignment = 4; - if (gen < 040) - kgem->min_alignment = 64; - kgem->half_cpu_cache_pages = cpu_cache_size() >> 13; DBG(("%s: last-level cache size: %d bytes, threshold in pages: %d\n", __FUNCTION__, cpu_cache_size(), kgem->half_cpu_cache_pages)); @@ -1257,7 +1401,7 @@ void kgem_init(struct kgem *kgem, int fd VG_CLEAR(aperture); aperture.aper_size = 0; - (void)drmIoctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture); + (void)do_ioctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture); if (aperture.aper_size == 0) aperture.aper_size = 64*1024*1024; @@ -1278,13 +1422,19 @@ void kgem_init(struct kgem *kgem, int fd kgem->aperture_low, kgem->aperture_low / (1024*1024), kgem->aperture_high, kgem->aperture_high / (1024*1024))); - kgem->aperture_mappable = agp_aperture_size(dev, gen); + kgem->aperture_mappable = 256 * 1024 * 1024; + if (dev != NULL) + kgem->aperture_mappable = agp_aperture_size(dev, gen); if (kgem->aperture_mappable == 0 || kgem->aperture_mappable > aperture.aper_size) kgem->aperture_mappable = aperture.aper_size; DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__, kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024))); + kgem->aperture_fenceable = MIN(256*1024*1024, kgem->aperture_mappable); + DBG(("%s: aperture fenceable=%d [%d MiB]\n", __FUNCTION__, + kgem->aperture_fenceable, kgem->aperture_fenceable / (1024*1024))); + kgem->buffer_size = 64 * 1024; while (kgem->buffer_size < kgem->aperture_mappable >> 10) kgem->buffer_size *= 2; @@ -1312,6 +1462,14 @@ void kgem_init(struct kgem *kgem, int fd if (kgem->max_gpu_size > totalram / 4) kgem->max_gpu_size = totalram / 4; + if (kgem->aperture_high > totalram / 2) { + kgem->aperture_high = totalram / 2; + kgem->aperture_low = kgem->aperture_high / 4; + DBG(("%s: reduced aperture watermaks to fit into ram; low=%d [%d], high=%d [%d]\n", __FUNCTION__, + kgem->aperture_low, kgem->aperture_low / (1024*1024), + kgem->aperture_high, kgem->aperture_high / (1024*1024))); + } + kgem->max_cpu_size = kgem->max_object_size; half_gpu_max = kgem->max_gpu_size / 2; @@ -1322,7 +1480,7 @@ void kgem_init(struct kgem *kgem, int fd if (kgem->has_llc) kgem->max_upload_tile_size = kgem->max_copy_tile_size; else - kgem->max_upload_tile_size = kgem->aperture_mappable / 4; + kgem->max_upload_tile_size = kgem->aperture_fenceable / 4; if (kgem->max_upload_tile_size > half_gpu_max) kgem->max_upload_tile_size = half_gpu_max; if (kgem->max_upload_tile_size > kgem->aperture_high/2) @@ -1360,8 +1518,11 @@ void kgem_init(struct kgem *kgem, int fd kgem->max_upload_tile_size, kgem->max_copy_tile_size)); /* Convert the aperture thresholds to pages */ + kgem->aperture_mappable /= PAGE_SIZE; + kgem->aperture_fenceable /= PAGE_SIZE; kgem->aperture_low /= PAGE_SIZE; kgem->aperture_high /= PAGE_SIZE; + kgem->aperture_total /= PAGE_SIZE; kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2; if ((int)kgem->fence_max < 0) @@ -1380,7 +1541,7 @@ void kgem_init(struct kgem *kgem, int fd } /* XXX hopefully a good approximation */ -uint32_t kgem_get_unique_id(struct kgem *kgem) +static uint32_t kgem_get_unique_id(struct kgem *kgem) { uint32_t id; id = ++kgem->unique_id; @@ -1395,10 +1556,10 @@ inline static uint32_t kgem_pitch_alignm return 256; if (flags & CREATE_SCANOUT) return 64; - return kgem->min_alignment; + return 8; } -void kgem_get_tile_size(struct kgem *kgem, int tiling, +void kgem_get_tile_size(struct kgem *kgem, int tiling, int pitch, int *tile_width, int *tile_height, int *tile_size) { if (kgem->gen <= 030) { @@ -1435,6 +1596,10 @@ void kgem_get_tile_size(struct kgem *kge *tile_size = 4096; break; } + + /* Force offset alignment to tile-row */ + if (tiling && kgem->gen < 033) + *tile_width = pitch; } static uint32_t kgem_surface_size(struct kgem *kgem, @@ -1457,16 +1622,16 @@ static uint32_t kgem_surface_size(struct if (tiling) { if (kgem->gen < 030) { tile_width = 128; - tile_height = 32; + tile_height = 16; } else { tile_width = 512; - tile_height = 16; + tile_height = 8; } } else { tile_width = 2 * bpp >> 3; tile_width = ALIGN(tile_width, kgem_pitch_alignment(kgem, flags)); - tile_height = 2; + tile_height = 1; } } else switch (tiling) { default: @@ -1474,43 +1639,54 @@ static uint32_t kgem_surface_size(struct tile_width = 2 * bpp >> 3; tile_width = ALIGN(tile_width, kgem_pitch_alignment(kgem, flags)); - tile_height = 2; + tile_height = 1; break; - /* XXX align to an even tile row */ case I915_TILING_X: tile_width = 512; - tile_height = 16; + tile_height = 8; break; case I915_TILING_Y: tile_width = 128; - tile_height = 64; + tile_height = 32; break; } + /* XXX align to an even tile row */ + if (!kgem->has_relaxed_fencing) + tile_height *= 2; *pitch = ALIGN(width * bpp / 8, tile_width); height = ALIGN(height, tile_height); + DBG(("%s: tile_width=%d, tile_height=%d => aligned pitch=%d, height=%d\n", + __FUNCTION__, tile_width, tile_height, *pitch, height)); + if (kgem->gen >= 040) return PAGE_ALIGN(*pitch * height); /* If it is too wide for the blitter, don't even bother. */ if (tiling != I915_TILING_NONE) { - if (*pitch > 8192) + if (*pitch > 8192) { + DBG(("%s: too wide for tiled surface (pitch=%d, limit=%d)\n", + __FUNCTION__, *pitch, 8192)); return 0; + } for (size = tile_width; size < *pitch; size <<= 1) ; *pitch = size; } else { - if (*pitch >= 32768) + if (*pitch >= 32768) { + DBG(("%s: too wide for linear surface (pitch=%d, limit=%d)\n", + __FUNCTION__, *pitch, 32767)); return 0; + } } size = *pitch * height; if (relaxed_fencing || tiling == I915_TILING_NONE) return PAGE_ALIGN(size); - /* We need to allocate a pot fence region for a tiled buffer. */ + /* We need to allocate a pot fence region for a tiled buffer. */ if (kgem->gen < 030) tile_width = 512 * 1024; else @@ -1520,13 +1696,54 @@ static uint32_t kgem_surface_size(struct return tile_width; } +bool kgem_check_surface_size(struct kgem *kgem, + uint32_t width, + uint32_t height, + uint32_t bpp, + uint32_t tiling, + uint32_t pitch, + uint32_t size) +{ + uint32_t min_size, min_pitch; + int tile_width, tile_height, tile_size; + + DBG(("%s(width=%d, height=%d, bpp=%d, tiling=%d, pitch=%d, size=%d)\n", + __FUNCTION__, width, height, bpp, tiling, pitch, size)); + + if (pitch & 3) + return false; + + min_size = kgem_surface_size(kgem, kgem->has_relaxed_fencing, 0, + width, height, bpp, tiling, + &min_pitch); + + DBG(("%s: min_pitch=%d, min_size=%d\n", __FUNCTION__, min_pitch, min_size)); + + if (size < min_size) + return false; + + if (pitch < min_pitch) + return false; + + kgem_get_tile_size(kgem, tiling, min_pitch, + &tile_width, &tile_height, &tile_size); + + DBG(("%s: tile_width=%d, tile_size=%d\n", __FUNCTION__, tile_width, tile_size)); + if (pitch & (tile_width - 1)) + return false; + if (size & (tile_size - 1)) + return false; + + return true; +} + static uint32_t kgem_aligned_height(struct kgem *kgem, uint32_t height, uint32_t tiling) { uint32_t tile_height; if (kgem->gen <= 030) { - tile_height = tiling ? kgem->gen < 030 ? 32 : 16 : 1; + tile_height = tiling ? kgem->gen < 030 ? 16 : 8 : 1; } else switch (tiling) { /* XXX align to an even tile row */ default: @@ -1534,13 +1751,17 @@ static uint32_t kgem_aligned_height(stru tile_height = 1; break; case I915_TILING_X: - tile_height = 16; + tile_height = 8; break; case I915_TILING_Y: - tile_height = 64; + tile_height = 32; break; } + /* XXX align to an even tile row */ + if (!kgem->has_relaxed_fencing) + tile_height *= 2; + return ALIGN(height, tile_height); } @@ -1565,10 +1786,15 @@ kgem_add_handle(struct kgem *kgem, struc static void kgem_add_bo(struct kgem *kgem, struct kgem_bo *bo) { + assert(bo->refcnt); + assert(bo->proxy == NULL); + bo->exec = kgem_add_handle(kgem, bo); bo->rq = MAKE_REQUEST(kgem->next_request, kgem->ring); list_move_tail(&bo->request, &kgem->next_request->buffers); + if (bo->io && !list_is_empty(&bo->list)) + list_move(&bo->list, &kgem->batch_buffers); /* XXX is it worth working around gcc here? */ kgem->flush |= bo->flush; @@ -1611,7 +1837,6 @@ static void kgem_fixup_self_relocs(struc } } - } static void kgem_bo_binding_free(struct kgem *kgem, struct kgem_bo *bo) @@ -1621,34 +1846,25 @@ static void kgem_bo_binding_free(struct b = bo->binding.next; while (b) { struct kgem_bo_binding *next = b->next; - free (b); + free(b); b = next; } } -static void kgem_bo_release_map(struct kgem *kgem, struct kgem_bo *bo) +static void kgem_bo_rmfb(struct kgem *kgem, struct kgem_bo *bo) { - int type = IS_CPU_MAP(bo->map); - - assert(!IS_USER_MAP(bo->map)); - - DBG(("%s: releasing %s vma for handle=%d, count=%d\n", - __FUNCTION__, type ? "CPU" : "GTT", - bo->handle, kgem->vma[type].count)); - - VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo))); - munmap(MAP(bo->map), bytes(bo)); - bo->map = NULL; - - if (!list_is_empty(&bo->vma)) { - list_del(&bo->vma); - kgem->vma[type].count--; + if (bo->scanout && bo->delta) { + DBG(("%s: releasing fb=%d for handle=%d\n", + __FUNCTION__, bo->delta, bo->handle)); + /* XXX will leak if we are not DRM_MASTER. *shrug* */ + do_ioctl(kgem->fd, DRM_IOCTL_MODE_RMFB, &bo->delta); + bo->delta = 0; } } static void kgem_bo_free(struct kgem *kgem, struct kgem_bo *bo) { - DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); + DBG(("%s: handle=%d, size=%d\n", __FUNCTION__, bo->handle, bytes(bo))); assert(bo->refcnt == 0); assert(bo->proxy == NULL); assert(bo->exec == NULL); @@ -1660,28 +1876,39 @@ static void kgem_bo_free(struct kgem *kg #endif kgem_bo_binding_free(kgem, bo); + kgem_bo_rmfb(kgem, bo); - if (IS_USER_MAP(bo->map)) { + if (IS_USER_MAP(bo->map__cpu)) { assert(bo->rq == NULL); assert(!__kgem_busy(kgem, bo->handle)); - assert(MAP(bo->map) != bo || bo->io || bo->flush); + assert(MAP(bo->map__cpu) != bo || bo->io || bo->flush); if (!(bo->io || bo->flush)) { DBG(("%s: freeing snooped base\n", __FUNCTION__)); - assert(bo != MAP(bo->map)); - free(MAP(bo->map)); + assert(bo != MAP(bo->map__cpu)); + free(MAP(bo->map__cpu)); } - bo->map = NULL; + bo->map__cpu = NULL; } - if (bo->map) - kgem_bo_release_map(kgem, bo); - assert(list_is_empty(&bo->vma)); - assert(bo->map == NULL); + + DBG(("%s: releasing %p:%p vma for handle=%d, count=%d\n", + __FUNCTION__, bo->map__gtt, bo->map__cpu, + bo->handle, list_is_empty(&bo->vma) ? 0 : kgem->vma[bo->map__gtt == NULL].count)); + + if (!list_is_empty(&bo->vma)) { + _list_del(&bo->vma); + kgem->vma[bo->map__gtt == NULL].count--; + } + + if (bo->map__gtt) + munmap(MAP(bo->map__gtt), bytes(bo)); + if (bo->map__cpu) + munmap(MAP(bo->map__cpu), bytes(bo)); _list_del(&bo->list); _list_del(&bo->request); gem_close(kgem->fd, bo->handle); - if (!bo->io) { + if (!bo->io && !DBG_NO_MALLOC_CACHE) { *(struct kgem_bo **)bo = __kgem_freed_bo; __kgem_freed_bo = bo; } else @@ -1706,29 +1933,36 @@ inline static void kgem_bo_move_to_inact assert(!bo->needs_flush); assert(list_is_empty(&bo->vma)); assert_tiling(kgem, bo); + assert_cacheing(kgem, bo); ASSERT_IDLE(kgem, bo->handle); - kgem->need_expire = true; - if (bucket(bo) >= NUM_CACHE_BUCKETS) { - list_move(&bo->list, &kgem->large_inactive); - return; - } + if (bo->map__gtt) { + munmap(MAP(bo->map__gtt), bytes(bo)); + bo->map__gtt = NULL; + } - assert(bo->flush == false); - list_move(&bo->list, &kgem->inactive[bucket(bo)]); - if (bo->map) { - int type = IS_CPU_MAP(bo->map); - if (bucket(bo) >= NUM_CACHE_BUCKETS || - (!type && !__kgem_bo_is_mappable(kgem, bo))) { - munmap(MAP(bo->map), bytes(bo)); - bo->map = NULL; - } - if (bo->map) { - list_add(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]); - kgem->vma[type].count++; + list_move(&bo->list, &kgem->large_inactive); + } else { + assert(bo->flush == false); + list_move(&bo->list, &kgem->inactive[bucket(bo)]); + if (bo->map__gtt) { + if (!kgem_bo_can_map(kgem, bo)) { + munmap(MAP(bo->map__gtt), bytes(bo)); + bo->map__gtt = NULL; + } + if (bo->map__gtt) { + list_add(&bo->vma, &kgem->vma[0].inactive[bucket(bo)]); + kgem->vma[0].count++; + } + } + if (bo->map__cpu && !bo->map__gtt) { + list_add(&bo->vma, &kgem->vma[1].inactive[bucket(bo)]); + kgem->vma[1].count++; } } + + kgem->need_expire = true; } static struct kgem_bo *kgem_bo_replace_io(struct kgem_bo *bo) @@ -1739,7 +1973,11 @@ static struct kgem_bo *kgem_bo_replace_i return bo; assert(!bo->snoop); - base = malloc(sizeof(*base)); + if (__kgem_freed_bo) { + base = __kgem_freed_bo; + __kgem_freed_bo = *(struct kgem_bo **)base; + } else + base = malloc(sizeof(*base)); if (base) { DBG(("%s: transferring io handle=%d to bo\n", __FUNCTION__, bo->handle)); @@ -1765,10 +2003,10 @@ inline static void kgem_bo_remove_from_i list_del(&bo->list); assert(bo->rq == NULL); assert(bo->exec == NULL); - if (bo->map) { - assert(!list_is_empty(&bo->vma)); + if (!list_is_empty(&bo->vma)) { + assert(bo->map__gtt || bo->map__cpu); list_del(&bo->vma); - kgem->vma[IS_CPU_MAP(bo->map)].count--; + kgem->vma[bo->map__gtt == NULL].count--; } } @@ -1779,8 +2017,10 @@ inline static void kgem_bo_remove_from_a list_del(&bo->list); assert(bo->rq != NULL); - if (bo->rq == (void *)kgem) + if (RQ(bo->rq) == (void *)kgem) { + assert(bo->exec == NULL); list_del(&bo->request); + } assert(list_is_empty(&bo->vma)); } @@ -1806,7 +2046,7 @@ static bool check_scanout_size(struct kg VG_CLEAR(info); info.fb_id = bo->delta; - if (drmIoctl(kgem->fd, DRM_IOCTL_MODE_GETFB, &info)) + if (do_ioctl(kgem->fd, DRM_IOCTL_MODE_GETFB, &info)) return false; gem_close(kgem->fd, info.handle); @@ -1826,15 +2066,16 @@ static void kgem_bo_move_to_scanout(stru { assert(bo->refcnt == 0); assert(bo->scanout); - assert(bo->delta); assert(!bo->flush); assert(!bo->snoop); assert(!bo->io); - if (bo->purged) { - DBG(("%s: discarding purged scanout - external name?\n", - __FUNCTION__)); - kgem_bo_free(kgem, bo); + if (bo->purged) { /* for stolen fb */ + if (!bo->exec) { + DBG(("%s: discarding purged scanout - stolen?\n", + __FUNCTION__)); + kgem_bo_free(kgem, bo); + } return; } @@ -1844,16 +2085,25 @@ static void kgem_bo_move_to_scanout(stru list_move_tail(&bo->list, &kgem->scanout); else list_move(&bo->list, &kgem->scanout); + + kgem->need_expire = true; + } static void kgem_bo_move_to_snoop(struct kgem *kgem, struct kgem_bo *bo) { assert(bo->reusable); + assert(!bo->scanout); assert(!bo->flush); assert(!bo->needs_flush); assert(bo->refcnt == 0); assert(bo->exec == NULL); + if (DBG_NO_SNOOP_CACHE) { + kgem_bo_free(kgem, bo); + return; + } + if (num_pages(bo) > kgem->max_cpu_size >> 13) { DBG(("%s handle=%d discarding large CPU buffer (%d >%d pages)\n", __FUNCTION__, bo->handle, num_pages(bo), kgem->max_cpu_size >> 13)); @@ -1866,6 +2116,31 @@ static void kgem_bo_move_to_snoop(struct DBG(("%s: moving %d to snoop cachee\n", __FUNCTION__, bo->handle)); list_add(&bo->list, &kgem->snoop); + kgem->need_expire = true; +} + +static bool kgem_bo_move_to_cache(struct kgem *kgem, struct kgem_bo *bo) +{ + bool retired = false; + + DBG(("%s: release handle=%d\n", __FUNCTION__, bo->handle)); + + if (bo->prime) { + DBG(("%s: discarding imported prime handle=%d\n", + __FUNCTION__, bo->handle)); + kgem_bo_free(kgem, bo); + } else if (bo->snoop) { + kgem_bo_move_to_snoop(kgem, bo); + } else if (bo->scanout) { + kgem_bo_move_to_scanout(kgem, bo); + } else if ((bo = kgem_bo_replace_io(bo))->reusable && + kgem_bo_set_purgeable(kgem, bo)) { + kgem_bo_move_to_inactive(kgem, bo); + retired = true; + } else + kgem_bo_free(kgem, bo); + + return retired; } static struct kgem_bo * @@ -1932,6 +2207,7 @@ void kgem_bo_undo(struct kgem *kgem, str if (kgem->nexec != 1 || bo->exec == NULL) return; + assert(bo); DBG(("%s: only handle in batch, discarding last operations for handle=%d\n", __FUNCTION__, bo->handle)); @@ -1942,23 +2218,70 @@ void kgem_bo_undo(struct kgem *kgem, str bo->refcnt++; kgem_reset(kgem); bo->refcnt--; + + assert(kgem->nreloc == 0); + assert(kgem->nexec == 0); + assert(bo->exec == NULL); } -static void __kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo) +void kgem_bo_pair_undo(struct kgem *kgem, struct kgem_bo *a, struct kgem_bo *b) { - DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); + if (kgem->nexec > 2) + return; - assert(list_is_empty(&bo->list)); - assert(bo->refcnt == 0); - assert(!bo->purged || !bo->reusable); - assert(bo->proxy == NULL); - assert_tiling(kgem, bo); + if (kgem->nexec == 1) { + if (a) + kgem_bo_undo(kgem, a); + if (b) + kgem_bo_undo(kgem, b); + return; + } - bo->binding.offset = 0; + if (a == NULL || b == NULL) + return; + if (a->exec == NULL || b->exec == NULL) + return; + + DBG(("%s: only handles in batch, discarding last operations for handle=%d and handle=%d\n", + __FUNCTION__, a->handle, b->handle)); + + assert(a->exec == &kgem->exec[0] || a->exec == &kgem->exec[1]); + assert(a->handle == kgem->exec[0].handle || a->handle == kgem->exec[1].handle); + assert(RQ(a->rq) == kgem->next_request); + assert(b->exec == &kgem->exec[0] || b->exec == &kgem->exec[1]); + assert(b->handle == kgem->exec[0].handle || b->handle == kgem->exec[1].handle); + assert(RQ(b->rq) == kgem->next_request); + + a->refcnt++; + b->refcnt++; + kgem_reset(kgem); + b->refcnt--; + a->refcnt--; + + assert(kgem->nreloc == 0); + assert(kgem->nexec == 0); + assert(a->exec == NULL); + assert(b->exec == NULL); +} + +static void __kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo) +{ + DBG(("%s: handle=%d, size=%d\n", __FUNCTION__, bo->handle, bytes(bo))); + + assert(list_is_empty(&bo->list)); + assert(bo->refcnt == 0); + assert(bo->proxy == NULL); + assert(bo->active_scanout == 0); + assert_tiling(kgem, bo); + + bo->binding.offset = 0; if (DBG_NO_CACHE) goto destroy; + if (bo->prime) + goto destroy; + if (bo->snoop && !bo->flush) { DBG(("%s: handle=%d is snooped\n", __FUNCTION__, bo->handle)); assert(bo->reusable); @@ -1969,7 +2292,7 @@ static void __kgem_bo_destroy(struct kge kgem_bo_move_to_snoop(kgem, bo); return; } - if (!IS_USER_MAP(bo->map)) + if (!IS_USER_MAP(bo->map__cpu)) bo->flush = false; if (bo->scanout) { @@ -1985,15 +2308,13 @@ static void __kgem_bo_destroy(struct kge goto destroy; } - if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU) - kgem_bo_release_map(kgem, bo); - assert(list_is_empty(&bo->vma)); assert(list_is_empty(&bo->list)); assert(bo->flush == false); assert(bo->snoop == false); assert(bo->io == false); assert(bo->scanout == false); + assert_cacheing(kgem, bo); kgem_bo_undo(kgem, bo); assert(bo->refcnt == 0); @@ -2016,7 +2337,7 @@ static void __kgem_bo_destroy(struct kge assert(bo->exec == NULL); assert(list_is_empty(&bo->request)); - if (!IS_CPU_MAP(bo->map)) { + if (bo->map__cpu == NULL || bucket(bo) >= NUM_CACHE_BUCKETS) { if (!kgem_bo_set_purgeable(kgem, bo)) goto destroy; @@ -2044,31 +2365,35 @@ static void kgem_bo_unref(struct kgem *k static void kgem_buffer_release(struct kgem *kgem, struct kgem_buffer *bo) { + assert(bo->base.io); while (!list_is_empty(&bo->base.vma)) { struct kgem_bo *cached; cached = list_first_entry(&bo->base.vma, struct kgem_bo, vma); assert(cached->proxy == &bo->base); + assert(cached != &bo->base); list_del(&cached->vma); - assert(*(struct kgem_bo **)cached->map == cached); - *(struct kgem_bo **)cached->map = NULL; - cached->map = NULL; + assert(*(struct kgem_bo **)cached->map__gtt == cached); + *(struct kgem_bo **)cached->map__gtt = NULL; + cached->map__gtt = NULL; kgem_bo_destroy(kgem, cached); } } -static bool kgem_retire__buffers(struct kgem *kgem) +void kgem_retire__buffers(struct kgem *kgem) { - bool retired = false; - while (!list_is_empty(&kgem->active_buffers)) { struct kgem_buffer *bo = list_last_entry(&kgem->active_buffers, struct kgem_buffer, base.list); + DBG(("%s: handle=%d, busy? %d [%d]\n", + __FUNCTION__, bo->base.handle, bo->base.rq != NULL, bo->base.exec != NULL)); + + assert(bo->base.exec == NULL || RQ(bo->base.rq) == kgem->next_request); if (bo->base.rq) break; @@ -2077,10 +2402,7 @@ static bool kgem_retire__buffers(struct list_del(&bo->base.list); kgem_buffer_release(kgem, bo); kgem_bo_unref(kgem, &bo->base); - retired = true; } - - return retired; } static bool kgem_retire__flushing(struct kgem *kgem) @@ -2089,7 +2411,7 @@ static bool kgem_retire__flushing(struct bool retired = false; list_for_each_entry_safe(bo, next, &kgem->flushing, request) { - assert(bo->rq == (void *)kgem); + assert(RQ(bo->rq) == (void *)kgem); assert(bo->exec == NULL); if (__kgem_busy(kgem, bo->handle)) @@ -2100,23 +2422,14 @@ static bool kgem_retire__flushing(struct if (bo->refcnt) continue; - if (bo->snoop) { - kgem_bo_move_to_snoop(kgem, bo); - } else if (bo->scanout) { - kgem_bo_move_to_scanout(kgem, bo); - } else if ((bo = kgem_bo_replace_io(bo))->reusable && - kgem_bo_set_purgeable(kgem, bo)) { - kgem_bo_move_to_inactive(kgem, bo); - retired = true; - } else - kgem_bo_free(kgem, bo); + retired |= kgem_bo_move_to_cache(kgem, bo); } #if HAS_DEBUG_FULL { int count = 0; list_for_each_entry(bo, &kgem->flushing, request) count++; - ErrorF("%s: %d bo on flushing list\n", __FUNCTION__, count); + DBG(("%s: %d bo on flushing list\n", __FUNCTION__, count)); } #endif @@ -2125,13 +2438,13 @@ static bool kgem_retire__flushing(struct return retired; } - static bool __kgem_retire_rq(struct kgem *kgem, struct kgem_request *rq) { bool retired = false; DBG(("%s: request %d complete\n", __FUNCTION__, rq->bo->handle)); + assert(RQ(rq->bo->rq) == rq); while (!list_is_empty(&rq->buffers)) { struct kgem_bo *bo; @@ -2152,7 +2465,8 @@ static bool __kgem_retire_rq(struct kgem DBG(("%s: moving %d to flushing\n", __FUNCTION__, bo->handle)); list_add(&bo->request, &kgem->flushing); - bo->rq = (void *)kgem; + bo->rq = MAKE_REQUEST(kgem, RQ_RING(bo->rq)); + kgem->need_retire = true; continue; } @@ -2161,23 +2475,13 @@ static bool __kgem_retire_rq(struct kgem if (bo->refcnt) continue; - if (bo->snoop) { - kgem_bo_move_to_snoop(kgem, bo); - } else if (bo->scanout) { - kgem_bo_move_to_scanout(kgem, bo); - } else if ((bo = kgem_bo_replace_io(bo))->reusable && - kgem_bo_set_purgeable(kgem, bo)) { - kgem_bo_move_to_inactive(kgem, bo); - retired = true; - } else { - DBG(("%s: closing %d\n", - __FUNCTION__, bo->handle)); - kgem_bo_free(kgem, bo); - } + retired |= kgem_bo_move_to_cache(kgem, bo); } assert(rq->bo->rq == NULL); + assert(rq->bo->exec == NULL); assert(list_is_empty(&rq->bo->request)); + assert(rq->bo->refcnt > 0); if (--rq->bo->refcnt == 0) { if (kgem_bo_set_purgeable(kgem, rq->bo)) { @@ -2204,6 +2508,7 @@ static bool kgem_retire__requests_ring(s rq = list_first_entry(&kgem->requests[ring], struct kgem_request, list); + assert(rq->ring == ring); if (__kgem_busy(kgem, rq->bo->handle)) break; @@ -2224,8 +2529,8 @@ static bool kgem_retire__requests_ring(s struct kgem_request, list)->bo; - ErrorF("%s: ring=%d, %d outstanding requests, oldest=%d\n", - __FUNCTION__, ring, count, bo ? bo->handle : 0); + DBG(("%s: ring=%d, %d outstanding requests, oldest=%d\n", + __FUNCTION__, ring, count, bo ? bo->handle : 0)); } #endif @@ -2249,13 +2554,12 @@ bool kgem_retire(struct kgem *kgem) { bool retired = false; - DBG(("%s\n", __FUNCTION__)); + DBG(("%s, need_retire?=%d\n", __FUNCTION__, kgem->need_retire)); kgem->need_retire = false; retired |= kgem_retire__flushing(kgem); retired |= kgem_retire__requests(kgem); - retired |= kgem_retire__buffers(kgem); DBG(("%s -- retired=%d, need_retire=%d\n", __FUNCTION__, retired, kgem->need_retire)); @@ -2269,10 +2573,12 @@ bool __kgem_ring_is_idle(struct kgem *kg { struct kgem_request *rq; + assert(ring < ARRAY_SIZE(kgem->requests)); assert(!list_is_empty(&kgem->requests[ring])); rq = list_last_entry(&kgem->requests[ring], struct kgem_request, list); + assert(rq->ring == ring); if (__kgem_busy(kgem, rq->bo->handle)) { DBG(("%s: last requests handle=%d still busy\n", __FUNCTION__, rq->bo->handle)); @@ -2283,15 +2589,53 @@ bool __kgem_ring_is_idle(struct kgem *kg __FUNCTION__, ring, rq->bo->handle)); kgem_retire__requests_ring(kgem, ring); + assert(list_is_empty(&kgem->requests[ring])); return true; } +#if 0 +static void kgem_commit__check_reloc(struct kgem *kgem) +{ + struct kgem_request *rq = kgem->next_request; + struct kgem_bo *bo; + bool has_64bit = kgem->gen >= 0100; + int i; + + for (i = 0; i < kgem->nreloc; i++) { + list_for_each_entry(bo, &rq->buffers, request) { + if (bo->target_handle == kgem->reloc[i].target_handle) { + uint64_t value = 0; + gem_read(kgem->fd, rq->bo->handle, &value, kgem->reloc[i].offset, has_64bit ? 8 : 4); + assert(bo->exec->offset == -1 || value == bo->exec->offset + (int)kgem->reloc[i].delta); + break; + } + } + } +} +#else +#define kgem_commit__check_reloc(kgem) +#endif + +#ifndef NDEBUG +static void kgem_commit__check_buffers(struct kgem *kgem) +{ + struct kgem_buffer *bo; + + list_for_each_entry(bo, &kgem->active_buffers, base.list) + assert(bo->base.exec == NULL); +} +#else +#define kgem_commit__check_buffers(kgem) +#endif + static void kgem_commit(struct kgem *kgem) { struct kgem_request *rq = kgem->next_request; struct kgem_bo *bo, *next; + kgem_commit__check_reloc(kgem); + list_for_each_entry_safe(bo, next, &rq->buffers, request) { assert(next->request.prev == &bo->request); @@ -2310,6 +2654,7 @@ static void kgem_commit(struct kgem *kge if (!bo->refcnt && !bo->reusable) { assert(!bo->snoop); + assert(!bo->proxy); kgem_bo_free(kgem, bo); continue; } @@ -2320,11 +2665,10 @@ static void kgem_commit(struct kgem *kge if (bo->proxy) { /* proxies are not used for domain tracking */ - bo->exec = NULL; __kgem_bo_clear_busy(bo); } - kgem->scanout_busy |= bo->scanout; + kgem->scanout_busy |= bo->scanout && bo->needs_flush; } if (rq == &kgem->static_request) { @@ -2336,7 +2680,7 @@ static void kgem_commit(struct kgem *kge set_domain.handle = rq->bo->handle; set_domain.read_domains = I915_GEM_DOMAIN_GTT; set_domain.write_domain = I915_GEM_DOMAIN_GTT; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); kgem_throttle(kgem); } @@ -2344,15 +2688,19 @@ static void kgem_commit(struct kgem *kge kgem_retire(kgem); assert(list_is_empty(&rq->buffers)); - assert(rq->bo->map == NULL); + assert(rq->bo->map__gtt == NULL); + assert(rq->bo->map__cpu == NULL); gem_close(kgem->fd, rq->bo->handle); kgem_cleanup_cache(kgem); } else { + assert(rq->ring < ARRAY_SIZE(kgem->requests)); list_add_tail(&rq->list, &kgem->requests[rq->ring]); kgem->need_throttle = kgem->need_retire = 1; } kgem->next_request = NULL; + + kgem_commit__check_buffers(kgem); } static void kgem_close_list(struct kgem *kgem, struct list *head) @@ -2374,17 +2722,18 @@ static void kgem_finish_buffers(struct k struct kgem_buffer *bo, *next; list_for_each_entry_safe(bo, next, &kgem->batch_buffers, base.list) { - DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s\n", + DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s, refcnt=%d\n", __FUNCTION__, bo->base.handle, bo->used, bo->base.exec!=NULL, - bo->write, bo->mmapped ? IS_CPU_MAP(bo->base.map) ? "cpu" : "gtt" : "no")); + bo->write, bo->mmapped == MMAPPED_CPU ? "cpu" : bo->mmapped == MMAPPED_GTT ? "gtt" : "no", + bo->base.refcnt)); assert(next->base.list.prev == &bo->base.list); assert(bo->base.io); assert(bo->base.refcnt >= 1); - if (!bo->base.exec) { - DBG(("%s: skipping unattached handle=%d, used=%d\n", - __FUNCTION__, bo->base.handle, bo->used)); + if (bo->base.refcnt > 1 && !bo->base.exec) { + DBG(("%s: skipping unattached handle=%d, used=%d, refcnt=%d\n", + __FUNCTION__, bo->base.handle, bo->used, bo->base.refcnt)); continue; } @@ -2394,27 +2743,28 @@ static void kgem_finish_buffers(struct k } if (bo->mmapped) { - int used; + uint32_t used; assert(!bo->need_io); used = ALIGN(bo->used, PAGE_SIZE); if (!DBG_NO_UPLOAD_ACTIVE && used + PAGE_SIZE <= bytes(&bo->base) && - (kgem->has_llc || !IS_CPU_MAP(bo->base.map) || bo->base.snoop)) { - DBG(("%s: retaining upload buffer (%d/%d)\n", - __FUNCTION__, bo->used, bytes(&bo->base))); + (kgem->has_llc || bo->mmapped == MMAPPED_GTT || bo->base.snoop)) { + DBG(("%s: retaining upload buffer (%d/%d): used=%d, refcnt=%d\n", + __FUNCTION__, bo->used, bytes(&bo->base), used, bo->base.refcnt)); bo->used = used; list_move(&bo->base.list, &kgem->active_buffers); + kgem->need_retire = true; continue; } DBG(("%s: discarding mmapped buffer, used=%d, map type=%d\n", - __FUNCTION__, bo->used, (int)__MAP_TYPE(bo->base.map))); + __FUNCTION__, bo->used, bo->mmapped)); goto decouple; } - if (!bo->used) { + if (!bo->used || !bo->base.exec) { /* Unless we replace the handle in the execbuffer, * then this bo will become active. So decouple it * from the buffer list and track it in the normal @@ -2493,8 +2843,8 @@ static void kgem_finish_buffers(struct k bo->base.handle, shrink->handle)); assert(bo->used <= bytes(shrink)); - if (gem_write(kgem->fd, shrink->handle, - 0, bo->used, bo->mem) == 0) { + if (gem_write__cachealigned(kgem->fd, shrink->handle, + 0, bo->used, bo->mem) == 0) { shrink->target_handle = kgem->has_handle_lut ? bo->base.target_handle : shrink->handle; for (n = 0; n < kgem->nreloc; n++) { @@ -2532,8 +2882,8 @@ static void kgem_finish_buffers(struct k __FUNCTION__, bo->base.handle, bo->used, bytes(&bo->base))); ASSERT_IDLE(kgem, bo->base.handle); assert(bo->used <= bytes(&bo->base)); - gem_write(kgem->fd, bo->base.handle, - 0, bo->used, bo->mem); + gem_write__cachealigned(kgem->fd, bo->base.handle, + 0, bo->used, bo->mem); bo->need_io = 0; decouple: @@ -2555,6 +2905,7 @@ static void kgem_cleanup(struct kgem *kg rq = list_first_entry(&kgem->requests[n], struct kgem_request, list); + assert(rq->ring == n); while (!list_is_empty(&rq->buffers)) { struct kgem_bo *bo; @@ -2582,33 +2933,65 @@ static int kgem_batch_write(struct kgem ASSERT_IDLE(kgem, handle); +#if DBG_NO_EXEC + { + uint32_t batch[] = { MI_BATCH_BUFFER_END, 0}; + return gem_write(kgem->fd, handle, 0, sizeof(batch), batch); + } +#endif + + +retry: /* If there is no surface data, just upload the batch */ - if (kgem->surface == kgem->batch_size) - return gem_write(kgem->fd, handle, - 0, sizeof(uint32_t)*kgem->nbatch, - kgem->batch); + if (kgem->surface == kgem->batch_size) { + if ((ret = gem_write__cachealigned(kgem->fd, handle, + 0, sizeof(uint32_t)*kgem->nbatch, + kgem->batch)) == 0) + return 0; + + goto expire; + } /* Are the batch pages conjoint with the surface pages? */ if (kgem->surface < kgem->nbatch + PAGE_SIZE/sizeof(uint32_t)) { assert(size == PAGE_ALIGN(kgem->batch_size*sizeof(uint32_t))); - return gem_write(kgem->fd, handle, - 0, kgem->batch_size*sizeof(uint32_t), - kgem->batch); + if ((ret = gem_write__cachealigned(kgem->fd, handle, + 0, kgem->batch_size*sizeof(uint32_t), + kgem->batch)) == 0) + return 0; + + goto expire; } /* Disjoint surface/batch, upload separately */ - ret = gem_write(kgem->fd, handle, - 0, sizeof(uint32_t)*kgem->nbatch, - kgem->batch); - if (ret) - return ret; + if ((ret = gem_write__cachealigned(kgem->fd, handle, + 0, sizeof(uint32_t)*kgem->nbatch, + kgem->batch))) + goto expire; ret = PAGE_ALIGN(sizeof(uint32_t) * kgem->batch_size); ret -= sizeof(uint32_t) * kgem->surface; assert(size-ret >= kgem->nbatch*sizeof(uint32_t)); - return __gem_write(kgem->fd, handle, - size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t), - kgem->batch + kgem->surface); + if (gem_write(kgem->fd, handle, + size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t), + kgem->batch + kgem->surface)) + goto expire; + + return 0; + +expire: + assert(ret != EINVAL); + + (void)__kgem_throttle_retire(kgem, 0); + if (kgem_expire_cache(kgem)) + goto retry; + + if (kgem_cleanup_cache(kgem)) + goto retry; + + ERR(("%s: failed to write batch (handle=%d): %d\n", + __FUNCTION__, handle, -ret)); + return ret; } void kgem_reset(struct kgem *kgem) @@ -2634,24 +3017,14 @@ void kgem_reset(struct kgem *kgem) assert(bo->domain == DOMAIN_GPU || bo->domain == DOMAIN_NONE); list_add(&bo->request, &kgem->flushing); bo->rq = (void *)kgem; + kgem->need_retire = true; } else __kgem_bo_clear_busy(bo); if (bo->refcnt || bo->rq) continue; - if (bo->snoop) { - kgem_bo_move_to_snoop(kgem, bo); - } else if (bo->scanout) { - kgem_bo_move_to_scanout(kgem, bo); - } else if ((bo = kgem_bo_replace_io(bo))->reusable && - kgem_bo_set_purgeable(kgem, bo)) { - kgem_bo_move_to_inactive(kgem, bo); - } else { - DBG(("%s: closing %d\n", - __FUNCTION__, bo->handle)); - kgem_bo_free(kgem, bo); - } + kgem_bo_move_to_cache(kgem, bo); } if (rq != &kgem->static_request) { @@ -2666,9 +3039,12 @@ void kgem_reset(struct kgem *kgem) kgem->nreloc__self = 0; kgem->aperture = 0; kgem->aperture_fenced = 0; + kgem->aperture_max_fence = 0; kgem->nbatch = 0; kgem->surface = kgem->batch_size; kgem->mode = KGEM_NONE; + kgem->needs_semaphore = false; + kgem->needs_reservation = false; kgem->flush = 0; kgem->batch_flags = kgem->batch_flags_base; @@ -2682,7 +3058,7 @@ static int compact_batch_surface(struct int size, shrink, n; if (!kgem->has_relaxed_delta) - return kgem->batch_size; + return kgem->batch_size * sizeof(uint32_t); /* See if we can pack the contents into one or two pages */ n = ALIGN(kgem->batch_size, 1024); @@ -2719,12 +3095,12 @@ kgem_create_batch(struct kgem *kgem, int list); if (!bo->rq) { out_4096: + assert(bo->refcnt > 0); list_move_tail(&bo->list, &kgem->pinned_batches[0]); return kgem_bo_reference(bo); } if (!__kgem_busy(kgem, bo->handle)) { - assert(RQ(bo->rq)->bo == bo); __kgem_retire_rq(kgem, RQ(bo->rq)); goto out_4096; } @@ -2736,45 +3112,126 @@ out_4096: list); if (!bo->rq) { out_16384: + assert(bo->refcnt > 0); list_move_tail(&bo->list, &kgem->pinned_batches[1]); return kgem_bo_reference(bo); } if (!__kgem_busy(kgem, bo->handle)) { - assert(RQ(bo->rq)->bo == bo); __kgem_retire_rq(kgem, RQ(bo->rq)); goto out_16384; } } - if (kgem->gen == 020 && !kgem->has_pinned_batches) { - assert(size <= 16384); + if (kgem->gen == 020) { + bo = kgem_create_linear(kgem, size, CREATE_CACHED | CREATE_TEMPORARY); + if (bo) + return bo; - bo = list_first_entry(&kgem->pinned_batches[size > 4096], - struct kgem_bo, - list); - list_move_tail(&bo->list, &kgem->pinned_batches[size > 4096]); + /* Nothing available for reuse, rely on the kernel wa */ + if (kgem->has_pinned_batches) { + bo = kgem_create_linear(kgem, size, CREATE_CACHED | CREATE_TEMPORARY); + if (bo) { + kgem->batch_flags &= ~LOCAL_I915_EXEC_IS_PINNED; + return bo; + } + } - DBG(("%s: syncing due to busy batches\n", __FUNCTION__)); + if (size < 16384) { + bo = list_first_entry(&kgem->pinned_batches[size > 4096], + struct kgem_bo, + list); + list_move_tail(&bo->list, &kgem->pinned_batches[size > 4096]); - VG_CLEAR(set_domain); - set_domain.handle = bo->handle; - set_domain.read_domains = I915_GEM_DOMAIN_GTT; - set_domain.write_domain = I915_GEM_DOMAIN_GTT; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { - DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); - kgem_throttle(kgem); - return NULL; - } + DBG(("%s: syncing due to busy batches\n", __FUNCTION__)); - kgem_retire(kgem); - assert(bo->rq == NULL); - return kgem_bo_reference(bo); + VG_CLEAR(set_domain); + set_domain.handle = bo->handle; + set_domain.read_domains = I915_GEM_DOMAIN_GTT; + set_domain.write_domain = I915_GEM_DOMAIN_GTT; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); + kgem_throttle(kgem); + return NULL; + } + + kgem_retire(kgem); + assert(bo->rq == NULL); + return kgem_bo_reference(bo); + } } return kgem_create_linear(kgem, size, CREATE_NO_THROTTLE); } +#if !NDEBUG +static bool dump_file(const char *path) +{ + FILE *file; + size_t len = 0; + char *line = NULL; + + file = fopen(path, "r"); + if (file == NULL) + return false; + + while (getline(&line, &len, file) != -1) + ErrorF("%s", line); + + free(line); + fclose(file); + return true; +} + +static void dump_debugfs(struct kgem *kgem, const char *name) +{ + char path[80]; + int minor = kgem_get_minor(kgem); + + if (minor < 0) + return; + + sprintf(path, "/sys/kernel/debug/dri/%d/%s", minor, name); + if (dump_file(path)) + return; + + sprintf(path, "/debug/dri/%d/%s", minor, name); + if (dump_file(path)) + return; +} + +static void dump_gtt_info(struct kgem *kgem) +{ + dump_debugfs(kgem, "i915_gem_gtt"); +} + +static void dump_fence_regs(struct kgem *kgem) +{ + dump_debugfs(kgem, "i915_gem_fence_regs"); +} +#endif + +static int do_execbuf(struct kgem *kgem, struct drm_i915_gem_execbuffer2 *execbuf) +{ + int ret; + +retry: + ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, execbuf); + if (ret == 0) + return 0; + + DBG(("%s: failed ret=%d, throttling and discarding cache\n", __FUNCTION__, ret)); + (void)__kgem_throttle_retire(kgem, 0); + if (kgem_expire_cache(kgem)) + goto retry; + + if (kgem_cleanup_cache(kgem)) + goto retry; + + /* last gasp */ + return do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_EXECBUFFER2, execbuf); +} + void _kgem_submit(struct kgem *kgem) { struct kgem_request *rq; @@ -2791,9 +3248,10 @@ void _kgem_submit(struct kgem *kgem) batch_end = kgem_end_batch(kgem); kgem_sna_flush(kgem); - DBG(("batch[%d/%d]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n", - kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface, kgem->batch_size, - kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture)); + DBG(("batch[%d/%d, flags=%x]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d [fenced=%d]\n", + kgem->mode, kgem->ring, kgem->batch_flags, + batch_end, kgem->nbatch, kgem->surface, kgem->batch_size, + kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced)); assert(kgem->nbatch <= kgem->batch_size); assert(kgem->nbatch <= kgem->surface); @@ -2803,7 +3261,7 @@ void _kgem_submit(struct kgem *kgem) kgem_finish_buffers(kgem); -#if SHOW_BATCH +#if SHOW_BATCH_BEFORE __kgem_batch_debug(kgem, batch_end); #endif @@ -2839,7 +3297,7 @@ void _kgem_submit(struct kgem *kgem) if (kgem_batch_write(kgem, handle, size) == 0) { struct drm_i915_gem_execbuffer2 execbuf; - int ret, retry = 3; + int ret; memset(&execbuf, 0, sizeof(execbuf)); execbuf.buffers_ptr = (uintptr_t)kgem->exec; @@ -2857,15 +3315,7 @@ void _kgem_submit(struct kgem *kgem) } } - ret = drmIoctl(kgem->fd, - DRM_IOCTL_I915_GEM_EXECBUFFER2, - &execbuf); - while (ret == -1 && errno == EBUSY && retry--) { - __kgem_throttle(kgem); - ret = drmIoctl(kgem->fd, - DRM_IOCTL_I915_GEM_EXECBUFFER2, - &execbuf); - } + ret = do_execbuf(kgem, &execbuf); if (DEBUG_SYNC && ret == 0) { struct drm_i915_gem_set_domain set_domain; @@ -2874,19 +3324,20 @@ void _kgem_submit(struct kgem *kgem) set_domain.read_domains = I915_GEM_DOMAIN_GTT; set_domain.write_domain = I915_GEM_DOMAIN_GTT; - ret = drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain); + ret = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain); } - if (ret == -1) { - DBG(("%s: GPU hang detected [%d]\n", - __FUNCTION__, errno)); + if (ret < 0) { kgem_throttle(kgem); - kgem->wedged = true; + if (!kgem->wedged) { + xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR, + "Failed to submit rendering commands, disabling acceleration.\n"); + kgem->wedged = true; + } #if !NDEBUG - ret = errno; - ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d: errno=%d\n", + ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d, fenced=%d, high=%d,%d: errno=%d\n", kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface, - kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, errno); + kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced, kgem->aperture_high, kgem->aperture_total, -ret); for (i = 0; i < kgem->nexec; i++) { struct kgem_bo *bo, *found = NULL; @@ -2918,21 +3369,38 @@ void _kgem_submit(struct kgem *kgem) (int)kgem->reloc[i].presumed_offset); } + { + struct drm_i915_gem_get_aperture aperture; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture) == 0) + ErrorF("Aperture size %lld, available %lld\n", + (long long)aperture.aper_size, + (long long)aperture.aper_available_size); + } + + if (ret == -ENOSPC) + dump_gtt_info(kgem); + if (ret == -EDEADLK) + dump_fence_regs(kgem); + if (DEBUG_SYNC) { int fd = open("/tmp/batchbuffer", O_WRONLY | O_CREAT | O_APPEND, 0666); if (fd != -1) { - write(fd, kgem->batch, batch_end*sizeof(uint32_t)); + int ignored = write(fd, kgem->batch, batch_end*sizeof(uint32_t)); + assert(ignored == batch_end*sizeof(uint32_t)); close(fd); } - FatalError("SNA: failed to submit batchbuffer, errno=%d\n", ret); + FatalError("SNA: failed to submit batchbuffer, errno=%d\n", -ret); } #endif } } - - kgem_commit(kgem); } +#if SHOW_BATCH_AFTER + if (gem_read(kgem->fd, rq->bo->handle, kgem->batch, 0, batch_end*sizeof(uint32_t) == 0)) + __kgem_batch_debug(kgem, batch_end); +#endif + kgem_commit(kgem); if (kgem->wedged) kgem_cleanup(kgem); @@ -2941,54 +3409,60 @@ void _kgem_submit(struct kgem *kgem) assert(kgem->next_request != NULL); } -static void find_hang_state(struct kgem *kgem, char *path, int maxlen) +static bool find_hang_state(struct kgem *kgem, char *path, int maxlen) { - int i; + int minor = kgem_get_minor(kgem); /* Search for our hang state in a few canonical locations. * In the unlikely event of having multiple devices, we * will need to check which minor actually corresponds to ours. */ - for (i = 0; i < DRM_MAX_MINOR; i++) { - snprintf(path, maxlen, "/sys/class/drm/card%d/error", i); - if (access(path, R_OK) == 0) - return; + snprintf(path, maxlen, "/sys/class/drm/card%d/error", minor); + if (access(path, R_OK) == 0) + return true; - snprintf(path, maxlen, "/sys/kernel/debug/dri%d/i915_error_state", i); - if (access(path, R_OK) == 0) - return; + snprintf(path, maxlen, "/sys/kernel/debug/dri/%d/i915_error_state", minor); + if (access(path, R_OK) == 0) + return true; - snprintf(path, maxlen, "/debug/dri%d/i915_error_state", i); - if (access(path, R_OK) == 0) - return; - } + snprintf(path, maxlen, "/debug/dri/%d/i915_error_state", minor); + if (access(path, R_OK) == 0) + return true; path[0] = '\0'; + return false; } void kgem_throttle(struct kgem *kgem) { - kgem->need_throttle = 0; if (kgem->wedged) return; - kgem->wedged = __kgem_throttle(kgem); + kgem->wedged = __kgem_throttle(kgem, true); if (kgem->wedged) { + static int once; char path[128]; - find_hang_state(kgem, path, sizeof(path)); - xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR, "Detected a hung GPU, disabling acceleration.\n"); - if (*path != '\0') + if (!once && find_hang_state(kgem, path, sizeof(path))) { xf86DrvMsg(kgem_get_screen_index(kgem), X_ERROR, "When reporting this, please include %s and the full dmesg.\n", path); + once = 1; + } + + kgem->need_throttle = false; } } -void kgem_purge_cache(struct kgem *kgem) +int kgem_is_wedged(struct kgem *kgem) +{ + return __kgem_throttle(kgem, true); +} + +static void kgem_purge_cache(struct kgem *kgem) { struct kgem_bo *bo, *next; int i; @@ -3014,9 +3488,8 @@ void kgem_clean_scanout_cache(struct kge bo = list_first_entry(&kgem->scanout, struct kgem_bo, list); assert(bo->scanout); - assert(bo->delta); assert(!bo->refcnt); - assert(bo->exec == NULL); + assert(!bo->prime); assert(bo->proxy == NULL); if (bo->exec || __kgem_busy(kgem, bo->handle)) @@ -3026,9 +3499,7 @@ void kgem_clean_scanout_cache(struct kge __FUNCTION__, bo->handle, bo->delta, bo->reusable)); list_del(&bo->list); - /* XXX will leak if we are not DRM_MASTER. *shrug* */ - drmIoctl(kgem->fd, DRM_IOCTL_MODE_RMFB, &bo->delta); - bo->delta = 0; + kgem_bo_rmfb(kgem, bo); bo->scanout = false; if (!bo->purged) { @@ -3104,8 +3575,8 @@ bool kgem_expire_cache(struct kgem *kgem int snoop_count = 0; list_for_each_entry(bo, &kgem->snoop, list) snoop_count++, snoop_size += bytes(bo); - ErrorF("%s: still allocated %d bo, %ld bytes, in snoop cache\n", - __FUNCTION__, snoop_count, snoop_size); + DBG(("%s: still allocated %d bo, %ld bytes, in snoop cache\n", + __FUNCTION__, snoop_count, snoop_size)); } #endif @@ -3118,9 +3589,11 @@ bool kgem_expire_cache(struct kgem *kgem if (kgem->need_purge) kgem_purge_cache(kgem); - expire = 0; + if (kgem->need_retire) + kgem_retire(kgem); - idle = !kgem->need_retire; + expire = 0; + idle = true; for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) { idle &= list_is_empty(&kgem->inactive[i]); list_for_each_entry(bo, &kgem->inactive[i], list) { @@ -3132,15 +3605,13 @@ bool kgem_expire_cache(struct kgem *kgem bo->delta = now; } } - if (idle) { - DBG(("%s: idle\n", __FUNCTION__)); - kgem->need_expire = false; + if (expire == 0) { + DBG(("%s: idle? %d\n", __FUNCTION__, idle)); + kgem->need_expire = !idle; return false; } - if (expire == 0) - return true; - idle = !kgem->need_retire; + idle = true; for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) { struct list preserve; @@ -3154,7 +3625,7 @@ bool kgem_expire_cache(struct kgem *kgem break; } - if (bo->map && bo->delta + MAP_PRESERVE_TIME > expire) { + if (bo->map__cpu && bo->delta + MAP_PRESERVE_TIME > expire) { idle = false; list_move_tail(&bo->list, &preserve); } else { @@ -3180,8 +3651,8 @@ bool kgem_expire_cache(struct kgem *kgem for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) list_for_each_entry(bo, &kgem->inactive[i], list) inactive_count++, inactive_size += bytes(bo); - ErrorF("%s: still allocated %d bo, %ld bytes, in inactive cache\n", - __FUNCTION__, inactive_count, inactive_size); + DBG(("%s: still allocated %d bo, %ld bytes, in inactive cache\n", + __FUNCTION__, inactive_count, inactive_size)); } #endif @@ -3189,12 +3660,12 @@ bool kgem_expire_cache(struct kgem *kgem __FUNCTION__, count, size, idle)); kgem->need_expire = !idle; - return !idle; + return count; (void)count; (void)size; } -void kgem_cleanup_cache(struct kgem *kgem) +bool kgem_cleanup_cache(struct kgem *kgem) { unsigned int i; int n; @@ -3215,7 +3686,7 @@ void kgem_cleanup_cache(struct kgem *kge set_domain.handle = rq->bo->handle; set_domain.read_domains = I915_GEM_DOMAIN_GTT; set_domain.write_domain = I915_GEM_DOMAIN_GTT; - (void)drmIoctl(kgem->fd, + (void)do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain); } @@ -3224,6 +3695,9 @@ void kgem_cleanup_cache(struct kgem *kge kgem_retire(kgem); kgem_cleanup(kgem); + if (!kgem->need_expire) + return false; + for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) { while (!list_is_empty(&kgem->inactive[i])) kgem_bo_free(kgem, @@ -3247,6 +3721,7 @@ void kgem_cleanup_cache(struct kgem *kge kgem->need_purge = false; kgem->need_expire = false; + return true; } static struct kgem_bo * @@ -3291,8 +3766,10 @@ retry_large: goto discard; list_del(&bo->list); - if (bo->rq == (void *)kgem) + if (RQ(bo->rq) == (void *)kgem) { + assert(bo->exec == NULL); list_del(&bo->request); + } bo->delta = 0; assert_tiling(kgem, bo); @@ -3346,7 +3823,7 @@ discard: __FUNCTION__, for_cpu ? "cpu" : "gtt")); cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)]; list_for_each_entry(bo, cache, vma) { - assert(IS_CPU_MAP(bo->map) == for_cpu); + assert(for_cpu ? bo->map__cpu : bo->map__gtt); assert(bucket(bo) == cache_bucket(num_pages)); assert(bo->proxy == NULL); assert(bo->rq == NULL); @@ -3370,6 +3847,8 @@ discard: continue; kgem_bo_remove_from_inactive(kgem, bo); + assert(list_is_empty(&bo->vma)); + assert(list_is_empty(&bo->list)); bo->tiling = I915_TILING_NONE; bo->pitch = 0; @@ -3426,10 +3905,10 @@ discard: bo->pitch = 0; } - if (bo->map) { + if (bo->map__gtt || bo->map__cpu) { if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) { int for_cpu = !!(flags & CREATE_CPU_MAP); - if (IS_CPU_MAP(bo->map) != for_cpu) { + if (for_cpu ? bo->map__cpu : bo->map__gtt){ if (first != NULL) break; @@ -3444,6 +3923,9 @@ discard: continue; } } else { + if (flags & CREATE_GTT_MAP && !kgem_bo_can_map(kgem, bo)) + continue; + if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) { if (first != NULL) break; @@ -3465,6 +3947,7 @@ discard: __FUNCTION__, bo->handle, num_pages(bo), use_active ? "active" : "inactive")); assert(list_is_empty(&bo->list)); + assert(list_is_empty(&bo->vma)); assert(use_active || bo->domain != DOMAIN_GPU); assert(!bo->needs_flush || use_active); assert_tiling(kgem, bo); @@ -3486,6 +3969,7 @@ discard: __FUNCTION__, first->handle, num_pages(first), use_active ? "active" : "inactive")); assert(list_is_empty(&first->list)); + assert(list_is_empty(&first->vma)); assert(use_active || first->domain != DOMAIN_GPU); assert(!first->needs_flush || use_active); ASSERT_MAYBE_IDLE(kgem, first->handle, !use_active); @@ -3498,25 +3982,39 @@ discard: struct kgem_bo *kgem_create_for_name(struct kgem *kgem, uint32_t name) { struct drm_gem_open open_arg; + struct drm_i915_gem_get_tiling tiling; struct kgem_bo *bo; DBG(("%s(name=%d)\n", __FUNCTION__, name)); VG_CLEAR(open_arg); open_arg.name = name; - if (drmIoctl(kgem->fd, DRM_IOCTL_GEM_OPEN, &open_arg)) + if (do_ioctl(kgem->fd, DRM_IOCTL_GEM_OPEN, &open_arg)) return NULL; DBG(("%s: new handle=%d\n", __FUNCTION__, open_arg.handle)); + + VG_CLEAR(tiling); + tiling.handle = open_arg.handle; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling)) { + DBG(("%s(name=%d) get-tiling failed, ret=%d\n", __FUNCTION__, name, errno)); + gem_close(kgem->fd, open_arg.handle); + return NULL; + } + + DBG(("%s: handle=%d, tiling=%d\n", __FUNCTION__, tiling.handle, tiling.tiling_mode)); + bo = __kgem_bo_alloc(open_arg.handle, open_arg.size / PAGE_SIZE); if (bo == NULL) { gem_close(kgem->fd, open_arg.handle); return NULL; } + bo->unique_id = kgem_get_unique_id(kgem); + bo->tiling = tiling.tiling_mode; bo->reusable = false; - bo->flush = true; - bo->purged = true; /* no coherency guarrantees */ + bo->prime = true; + bo->purged = true; /* no coherency guarantees */ debug_alloc__bo(kgem, bo); return bo; @@ -3527,23 +4025,41 @@ struct kgem_bo *kgem_create_for_prime(st #ifdef DRM_IOCTL_PRIME_FD_TO_HANDLE struct drm_prime_handle args; struct drm_i915_gem_get_tiling tiling; + struct local_i915_gem_caching caching; struct kgem_bo *bo; + off_t seek; DBG(("%s(name=%d)\n", __FUNCTION__, name)); VG_CLEAR(args); args.fd = name; args.flags = 0; - if (drmIoctl(kgem->fd, DRM_IOCTL_PRIME_FD_TO_HANDLE, &args)) + if (do_ioctl(kgem->fd, DRM_IOCTL_PRIME_FD_TO_HANDLE, &args)) { + DBG(("%s(name=%d) fd-to-handle failed, ret=%d\n", __FUNCTION__, name, errno)); return NULL; + } VG_CLEAR(tiling); tiling.handle = args.handle; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling)) { + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling)) { + DBG(("%s(name=%d) get-tiling failed, ret=%d\n", __FUNCTION__, name, errno)); gem_close(kgem->fd, args.handle); return NULL; } + /* Query actual size, overriding specified if available */ + seek = lseek(args.fd, 0, SEEK_END); + DBG(("%s: estimated size=%ld, actual=%lld\n", + __FUNCTION__, (long)size, (long long)seek)); + if (seek != -1) { + if (size > seek) { + DBG(("%s(name=%d) estimated required size [%d] is larger than actual [%ld]\n", __FUNCTION__, name, size, (long)seek)); + gem_close(kgem->fd, args.handle); + return NULL; + } + size = seek; + } + DBG(("%s: new handle=%d, tiling=%d\n", __FUNCTION__, args.handle, tiling.tiling_mode)); bo = __kgem_bo_alloc(args.handle, NUM_PAGES(size)); @@ -3552,8 +4068,45 @@ struct kgem_bo *kgem_create_for_prime(st return NULL; } + bo->unique_id = kgem_get_unique_id(kgem); bo->tiling = tiling.tiling_mode; bo->reusable = false; + bo->prime = true; + bo->domain = DOMAIN_NONE; + + /* is this a special bo (e.g. scanout or CPU coherent)? */ + + VG_CLEAR(caching); + caching.handle = args.handle; + caching.caching = kgem->has_llc; + (void)drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_GET_CACHING, &caching); + DBG(("%s: imported handle=%d has caching %d\n", __FUNCTION__, args.handle, caching.caching)); + switch (caching.caching) { + case 0: + if (kgem->has_llc) { + DBG(("%s: interpreting handle=%d as a foreign scanout\n", + __FUNCTION__, args.handle)); + bo->scanout = true; + } + break; + case 1: + if (!kgem->has_llc) { + DBG(("%s: interpreting handle=%d as a foreign snooped buffer\n", + __FUNCTION__, args.handle)); + bo->snoop = true; + if (bo->tiling) { + DBG(("%s: illegal snooped tiled buffer\n", __FUNCTION__)); + kgem_bo_free(kgem, bo); + return NULL; + } + } + break; + case 2: + DBG(("%s: interpreting handle=%d as a foreign scanout\n", + __FUNCTION__, args.handle)); + bo->scanout = true; + break; + } debug_alloc__bo(kgem, bo); return bo; @@ -3571,7 +4124,7 @@ int kgem_bo_export_to_prime(struct kgem args.handle = bo->handle; args.flags = O_CLOEXEC; - if (drmIoctl(kgem->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args)) + if (do_ioctl(kgem->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args)) return -1; bo->reusable = false; @@ -3654,6 +4207,9 @@ int kgem_choose_tiling(struct kgem *kgem if (tiling < 0) return tiling; + if (tiling == I915_TILING_Y && !kgem->can_render_y) + tiling = I915_TILING_X; + if (tiling && (height == 1 || width == 1)) { DBG(("%s: disabling tiling [%dx%d] for single row/col\n", __FUNCTION__,width, height)); @@ -3679,13 +4235,13 @@ int kgem_choose_tiling(struct kgem *kgem goto done; } - if (tiling == I915_TILING_X && width * bpp <= 8*512/2) { + if (tiling == I915_TILING_X && width * bpp <= 8*512) { DBG(("%s: too thin [width %d, %d bpp] for TILING_X\n", __FUNCTION__, width, bpp)); tiling = I915_TILING_NONE; goto done; } - if (tiling == I915_TILING_Y && width * bpp <= 8*128/2) { + if (tiling == I915_TILING_Y && width * bpp < 8*128) { DBG(("%s: too thin [%d] for TILING_Y\n", __FUNCTION__, width)); tiling = I915_TILING_NONE; @@ -3755,9 +4311,9 @@ unsigned kgem_can_create_2d(struct kgem if (size > 0) { if (size <= kgem->max_cpu_size) flags |= KGEM_CAN_CREATE_CPU; - if (size <= kgem->max_gpu_size) + if (size > 4096 && size <= kgem->max_gpu_size) flags |= KGEM_CAN_CREATE_GPU; - if (size <= kgem->aperture_mappable/4) + if (size <= PAGE_SIZE*kgem->aperture_mappable/4) flags |= KGEM_CAN_CREATE_GTT; if (size > kgem->large_object_size) flags |= KGEM_CAN_CREATE_LARGE; @@ -3776,9 +4332,11 @@ unsigned kgem_can_create_2d(struct kgem &pitch); DBG(("%s: tiled[%d] size=%d\n", __FUNCTION__, tiling, size)); if (size > 0 && size <= kgem->max_gpu_size) - flags |= KGEM_CAN_CREATE_GPU; - if (size > 0 && size <= kgem->aperture_mappable/4) + flags |= KGEM_CAN_CREATE_GPU | KGEM_CAN_CREATE_TILED; + if (size > 0 && size <= PAGE_SIZE*kgem->aperture_mappable/4) flags |= KGEM_CAN_CREATE_GTT; + if (size > PAGE_SIZE*kgem->aperture_mappable/4) + flags &= ~KGEM_CAN_CREATE_GTT; if (size > kgem->large_object_size) flags |= KGEM_CAN_CREATE_LARGE; if (size > kgem->max_object_size) { @@ -3786,6 +4344,15 @@ unsigned kgem_can_create_2d(struct kgem __FUNCTION__, size, kgem->max_object_size)); return 0; } + if (kgem->gen < 040) { + int fence_size = 1024 * 1024; + while (fence_size < size) + fence_size <<= 1; + if (fence_size > kgem->max_gpu_size) + flags &= ~KGEM_CAN_CREATE_GPU | KGEM_CAN_CREATE_TILED; + if (fence_size > PAGE_SIZE*kgem->aperture_fenceable/4) + flags &= ~KGEM_CAN_CREATE_GTT; + } } return flags; @@ -3800,11 +4367,11 @@ inline int kgem_bo_fenced_size(struct kg assert(kgem->gen < 040); if (kgem->gen < 030) - size = 512 * 1024; + size = 512 * 1024 / PAGE_SIZE; else - size = 1024 * 1024; - while (size < bytes(bo)) - size *= 2; + size = 1024 * 1024 / PAGE_SIZE; + while (size < num_pages(bo)) + size <<= 1; return size; } @@ -3825,9 +4392,9 @@ __kgem_bo_create_as_display(struct kgem args.tiling_mode = tiling; args.stride = pitch; - if (drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args)) { + if (do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args)) { args.placement = LOCAL_I915_CREATE_PLACEMENT_SYSTEM; - if (drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args)) + if (do_ioctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args)) return NULL; } @@ -3847,8 +4414,10 @@ __kgem_bo_create_as_display(struct kgem bo->domain = DOMAIN_NONE; if (__kgem_busy(kgem, bo->handle)) { + assert(bo->exec == NULL); list_add(&bo->request, &kgem->flushing); bo->rq = (void *)kgem; + kgem->need_retire = true; } assert_tiling(kgem, bo); @@ -3857,6 +4426,56 @@ __kgem_bo_create_as_display(struct kgem return bo; } +static void __kgem_bo_make_scanout(struct kgem *kgem, + struct kgem_bo *bo, + int width, int height) +{ + ScrnInfoPtr scrn = + container_of(kgem, struct sna, kgem)->scrn; + struct drm_mode_fb_cmd arg; + + assert(bo->proxy == NULL); + + if (!scrn->vtSema) + return; + + DBG(("%s: create fb %dx%d@%d/%d\n", + __FUNCTION__, width, height, scrn->depth, scrn->bitsPerPixel)); + + VG_CLEAR(arg); + arg.width = width; + arg.height = height; + arg.pitch = bo->pitch; + arg.bpp = scrn->bitsPerPixel; + arg.depth = scrn->depth; + arg.handle = bo->handle; + + /* First move the scanout out of cached memory */ + if (kgem->has_llc) { + if (!gem_set_caching(kgem->fd, bo->handle, DISPLAY) && + !gem_set_caching(kgem->fd, bo->handle, UNCACHED)) + return; + } + + bo->scanout = true; + + /* Then pre-emptively move the object into the mappable + * portion to avoid rebinding later when busy. + */ + if (bo->map__gtt == NULL) + bo->map__gtt = __kgem_bo_map__gtt(kgem, bo); + if (bo->map__gtt) { + *(uint32_t *)bo->map__gtt = 0; + bo->domain = DOMAIN_GTT; + } + + if (do_ioctl(kgem->fd, DRM_IOCTL_MODE_ADDFB, &arg) == 0) { + DBG(("%s: attached fb=%d to handle=%d\n", + __FUNCTION__, arg.fb_id, arg.handle)); + bo->delta = arg.fb_id; + } +} + struct kgem_bo *kgem_create_2d(struct kgem *kgem, int width, int height, @@ -3874,7 +4493,6 @@ struct kgem_bo *kgem_create_2d(struct kg if (tiling < 0) exact = true, tiling = -tiling; - DBG(("%s(%dx%d, bpp=%d, tiling=%d, exact=%d, inactive=%d, cpu-mapping=%d, gtt-mapping=%d, scanout?=%d, prime?=%d, temp?=%d)\n", __FUNCTION__, width, height, bpp, tiling, exact, !!(flags & CREATE_INACTIVE), @@ -3886,7 +4504,11 @@ struct kgem_bo *kgem_create_2d(struct kg size = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags, width, height, bpp, tiling, &pitch); - assert(size && size <= kgem->max_object_size); + if (size == 0) { + DBG(("%s: invalid surface size (too large?)\n", __FUNCTION__)); + return NULL; + } + size /= PAGE_SIZE; bucket = cache_bucket(size); @@ -3895,25 +4517,19 @@ struct kgem_bo *kgem_create_2d(struct kg list_for_each_entry_reverse(bo, &kgem->scanout, list) { assert(bo->scanout); - assert(bo->delta); assert(!bo->flush); + assert(!bo->refcnt); assert_tiling(kgem, bo); if (size > num_pages(bo) || num_pages(bo) > 2*size) continue; - if (!check_scanout_size(kgem, bo, width, height)) + if (bo->tiling != tiling || bo->pitch != pitch) + /* No tiling/pitch without recreating fb */ continue; - if (bo->tiling != tiling || - (tiling != I915_TILING_NONE && bo->pitch != pitch)) { - if (!gem_set_tiling(kgem->fd, bo->handle, - tiling, pitch)) - continue; - - bo->tiling = tiling; - bo->pitch = pitch; - } + if (bo->delta && !check_scanout_size(kgem, bo, width, height)) + continue; if (flags & CREATE_INACTIVE && bo->rq) { last = bo; @@ -3943,9 +4559,69 @@ struct kgem_bo *kgem_create_2d(struct kg return last; } - bo = __kgem_bo_create_as_display(kgem, size, tiling, pitch); + if (container_of(kgem, struct sna, kgem)->scrn->vtSema) { + ScrnInfoPtr scrn = container_of(kgem, struct sna, kgem)->scrn; + + list_for_each_entry_reverse(bo, &kgem->scanout, list) { + struct drm_mode_fb_cmd arg; + + assert(bo->scanout); + assert(!bo->refcnt); + + if (size > num_pages(bo) || num_pages(bo) > 2*size) + continue; + + if (flags & CREATE_INACTIVE && bo->rq) + continue; + + list_del(&bo->list); + + if (bo->tiling != tiling || bo->pitch != pitch) { + if (bo->delta) { + kgem_bo_rmfb(kgem, bo); + bo->delta = 0; + } + + if (gem_set_tiling(kgem->fd, bo->handle, + tiling, pitch)) { + bo->tiling = tiling; + bo->pitch = pitch; + } else { + kgem_bo_free(kgem, bo); + break; + } + } + + VG_CLEAR(arg); + arg.width = width; + arg.height = height; + arg.pitch = bo->pitch; + arg.bpp = scrn->bitsPerPixel; + arg.depth = scrn->depth; + arg.handle = bo->handle; + + if (do_ioctl(kgem->fd, DRM_IOCTL_MODE_ADDFB, &arg)) { + kgem_bo_free(kgem, bo); + break; + } + + bo->delta = arg.fb_id; + bo->unique_id = kgem_get_unique_id(kgem); + + DBG((" 2:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n", + bo->pitch, bo->tiling, bo->handle, bo->unique_id)); + assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); + assert_tiling(kgem, bo); + bo->refcnt = 1; + return bo; + } + } + + bo = __kgem_bo_create_as_display(kgem, size, tiling, pitch); if (bo) return bo; + + flags |= CREATE_INACTIVE; } if (bucket >= NUM_CACHE_BUCKETS) { @@ -3997,7 +4673,6 @@ struct kgem_bo *kgem_create_2d(struct kg assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); assert_tiling(kgem, bo); bo->refcnt = 1; - bo->flush = true; return bo; } @@ -4038,6 +4713,10 @@ large_inactive: assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); assert_tiling(kgem, bo); bo->refcnt = 1; + + if (flags & CREATE_SCANOUT) + __kgem_bo_make_scanout(kgem, bo, width, height); + return bo; } @@ -4057,9 +4736,9 @@ large_inactive: assert(bucket(bo) == bucket); assert(bo->refcnt == 0); assert(!bo->scanout); - assert(bo->map); - assert(IS_CPU_MAP(bo->map) == for_cpu); + assert(for_cpu ? bo->map__cpu : bo->map__gtt); assert(bo->rq == NULL); + assert(bo->exec == NULL); assert(list_is_empty(&bo->request)); assert(bo->flush == false); assert_tiling(kgem, bo); @@ -4070,11 +4749,20 @@ large_inactive: continue; } + if (flags & UNCACHED && !kgem->has_llc && bo->domain != DOMAIN_CPU) + continue; + if (bo->tiling != tiling || (tiling != I915_TILING_NONE && bo->pitch != pitch)) { - DBG(("inactive vma with wrong tiling: %d < %d\n", - bo->tiling, tiling)); - continue; + if (bo->map__gtt || + !gem_set_tiling(kgem->fd, bo->handle, + tiling, pitch)) { + DBG(("inactive GTT vma with wrong tiling: %d < %d\n", + bo->tiling, tiling)); + continue; + } + bo->tiling = tiling; + bo->pitch = pitch; } if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) { @@ -4086,9 +4774,10 @@ large_inactive: bo->pitch = pitch; bo->delta = 0; bo->unique_id = kgem_get_unique_id(kgem); - bo->domain = DOMAIN_NONE; kgem_bo_remove_from_inactive(kgem, bo); + assert(list_is_empty(&bo->list)); + assert(list_is_empty(&bo->vma)); DBG((" from inactive vma: pitch=%d, tiling=%d: handle=%d, id=%d\n", bo->pitch, bo->tiling, bo->handle, bo->unique_id)); @@ -4119,7 +4808,7 @@ large_inactive: retry = NUM_CACHE_BUCKETS - bucket; if (retry > 3 && (flags & CREATE_TEMPORARY) == 0) retry = 3; -search_again: +search_active: assert(bucket < NUM_CACHE_BUCKETS); cache = &kgem->active[bucket][tiling]; if (tiling) { @@ -4197,52 +4886,44 @@ search_again: } } - if (--retry && exact) { - if (kgem->gen >= 040) { - for (i = I915_TILING_NONE; i <= I915_TILING_Y; i++) { - if (i == tiling) - continue; - - cache = &kgem->active[bucket][i]; - list_for_each_entry(bo, cache, list) { - assert(!bo->purged); - assert(bo->refcnt == 0); - assert(bo->reusable); - assert(!bo->scanout); - assert(bo->flush == false); - assert_tiling(kgem, bo); + if (kgem->gen >= 040) { + for (i = I915_TILING_Y; i >= I915_TILING_NONE; i--) { + cache = &kgem->active[bucket][i]; + list_for_each_entry(bo, cache, list) { + assert(!bo->purged); + assert(bo->refcnt == 0); + assert(bo->reusable); + assert(!bo->scanout); + assert(bo->flush == false); + assert_tiling(kgem, bo); - if (num_pages(bo) < size) - continue; + if (num_pages(bo) < size) + continue; + if (bo->tiling != tiling || + (tiling != I915_TILING_NONE && bo->pitch != pitch)) { if (!gem_set_tiling(kgem->fd, bo->handle, tiling, pitch)) continue; + } - kgem_bo_remove_from_active(kgem, bo); + kgem_bo_remove_from_active(kgem, bo); - bo->unique_id = kgem_get_unique_id(kgem); - bo->pitch = pitch; - bo->tiling = tiling; - bo->delta = 0; - DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n", - bo->pitch, bo->tiling, bo->handle, bo->unique_id)); - assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); - assert_tiling(kgem, bo); - bo->refcnt = 1; - return bo; - } + bo->unique_id = kgem_get_unique_id(kgem); + bo->pitch = pitch; + bo->tiling = tiling; + bo->delta = 0; + DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n", + bo->pitch, bo->tiling, bo->handle, bo->unique_id)); + assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); + assert_tiling(kgem, bo); + bo->refcnt = 1; + return bo; } } - - bucket++; - goto search_again; - } - - if (!exact) { /* allow an active near-miss? */ - i = tiling; - while (--i >= 0) { + } else if (!exact) { /* allow an active near-miss? */ + for (i = tiling; i >= I915_TILING_NONE; i--) { tiled_height = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags, width, height, bpp, tiling, &pitch); cache = active(kgem, tiled_height / PAGE_SIZE, i); @@ -4282,6 +4963,11 @@ search_again: } } + if (--retry) { + bucket++; + goto search_active; + } + skip_active_search: bucket = cache_bucket(size); retry = NUM_CACHE_BUCKETS - bucket; @@ -4309,9 +4995,6 @@ search_inactive: if (!gem_set_tiling(kgem->fd, bo->handle, tiling, pitch)) continue; - - if (bo->map) - kgem_bo_release_map(kgem, bo); } if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) { @@ -4320,6 +5003,8 @@ search_inactive: } kgem_bo_remove_from_inactive(kgem, bo); + assert(list_is_empty(&bo->list)); + assert(list_is_empty(&bo->vma)); bo->pitch = pitch; bo->tiling = tiling; @@ -4336,55 +5021,104 @@ search_inactive: assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); assert_tiling(kgem, bo); bo->refcnt = 1; + + if (flags & CREATE_SCANOUT) + __kgem_bo_make_scanout(kgem, bo, width, height); + return bo; } - if (flags & CREATE_INACTIVE && - !list_is_empty(&kgem->active[bucket][tiling]) && - __kgem_throttle_retire(kgem, flags)) { - flags &= ~CREATE_INACTIVE; - goto search_inactive; + if ((flags & CREATE_NO_RETIRE) == 0) { + list_for_each_entry_reverse(bo, &kgem->active[bucket][tiling], list) { + if (bo->exec) + break; + + if (size > num_pages(bo)) + continue; + + if (__kgem_busy(kgem, bo->handle)) { + if (flags & CREATE_NO_THROTTLE) + goto no_retire; + + do { + if (!kgem->need_throttle) { + DBG(("%s: not throttling for active handle=%d\n", __FUNCTION__, bo->handle)); + goto no_retire; + } + + __kgem_throttle(kgem, false); + } while (__kgem_busy(kgem, bo->handle)); + } + + DBG(("%s: flushed active handle=%d\n", __FUNCTION__, bo->handle)); + + kgem_bo_remove_from_active(kgem, bo); + __kgem_bo_clear_busy(bo); + + if (tiling != I915_TILING_NONE && bo->pitch != pitch) { + if (!gem_set_tiling(kgem->fd, bo->handle, tiling, pitch)) { + kgem_bo_free(kgem, bo); + goto no_retire; + } + } + + bo->pitch = pitch; + bo->unique_id = kgem_get_unique_id(kgem); + bo->delta = 0; + DBG((" 2:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n", + bo->pitch, bo->tiling, bo->handle, bo->unique_id)); + assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo)); + assert_tiling(kgem, bo); + bo->refcnt = 1; + + if (flags & CREATE_SCANOUT) + __kgem_bo_make_scanout(kgem, bo, width, height); + + return bo; + } +no_retire: + flags |= CREATE_NO_RETIRE; } if (--retry) { bucket++; - flags &= ~CREATE_INACTIVE; goto search_inactive; } create: - if (flags & CREATE_CACHED) + if (flags & CREATE_CACHED) { + DBG(("%s: no cached bo found, requested not to create a new bo\n", __FUNCTION__)); return NULL; + } if (bucket >= NUM_CACHE_BUCKETS) size = ALIGN(size, 1024); handle = gem_create(kgem->fd, size); - if (handle == 0) + if (handle == 0) { + DBG(("%s: kernel allocation (gem_create) failure\n", __FUNCTION__)); return NULL; + } bo = __kgem_bo_alloc(handle, size); if (!bo) { + DBG(("%s: malloc failed\n", __FUNCTION__)); gem_close(kgem->fd, handle); return NULL; } - if (bucket >= NUM_CACHE_BUCKETS) { - DBG(("%s: marking large bo for automatic flushing\n", - __FUNCTION__)); - bo->flush = true; - } - bo->unique_id = kgem_get_unique_id(kgem); if (tiling == I915_TILING_NONE || gem_set_tiling(kgem->fd, handle, tiling, pitch)) { bo->tiling = tiling; bo->pitch = pitch; + if (flags & CREATE_SCANOUT) + __kgem_bo_make_scanout(kgem, bo, width, height); } else { if (flags & CREATE_EXACT) { - if (bo->pitch != pitch || bo->tiling != tiling) { - kgem_bo_free(kgem, bo); - return NULL; - } + DBG(("%s: failed to set exact tiling (gem_set_tiling)\n", __FUNCTION__)); + gem_close(kgem->fd, handle); + free(bo); + return NULL; } } @@ -4502,17 +5236,26 @@ void _kgem_bo_destroy(struct kgem *kgem, __FUNCTION__, bo->handle, bo->proxy != NULL)); if (bo->proxy) { + assert(!bo->reusable); + kgem_bo_binding_free(kgem, bo); + + assert(list_is_empty(&bo->list)); _list_del(&bo->vma); _list_del(&bo->request); - if (bo->io && bo->exec == NULL) + + if (bo->io && bo->domain == DOMAIN_CPU) _kgem_bo_delete_buffer(kgem, bo); + kgem_bo_unref(kgem, bo->proxy); - kgem_bo_binding_free(kgem, bo); - free(bo); - return; - } - __kgem_bo_destroy(kgem, bo); + if (DBG_NO_MALLOC_CACHE) { + free(bo); + } else { + *(struct kgem_bo **)bo = __kgem_freed_bo; + __kgem_freed_bo = bo; + } + } else + __kgem_bo_destroy(kgem, bo); } static void __kgem_flush(struct kgem *kgem, struct kgem_bo *bo) @@ -4531,10 +5274,11 @@ static void __kgem_flush(struct kgem *kg void kgem_scanout_flush(struct kgem *kgem, struct kgem_bo *bo) { - kgem_bo_submit(kgem, bo); if (!bo->needs_flush) return; + kgem_bo_submit(kgem, bo); + /* If the kernel fails to emit the flush, then it will be forced when * we assume direct access. And as the usual failure is EIO, we do * not actually care. @@ -4551,9 +5295,106 @@ void kgem_scanout_flush(struct kgem *kge bo->domain = DOMAIN_NONE; } +inline static bool nearly_idle(struct kgem *kgem) +{ + int ring = kgem->ring == KGEM_BLT; + + if (list_is_singular(&kgem->requests[ring])) + return true; + + return __kgem_ring_is_idle(kgem, ring); +} + inline static bool needs_semaphore(struct kgem *kgem, struct kgem_bo *bo) { - return kgem->nreloc && bo->rq && RQ_RING(bo->rq) != kgem->ring; + if (kgem->needs_semaphore) + return false; + + if (bo->rq == NULL || RQ_RING(bo->rq) == kgem->ring) + return false; + + kgem->needs_semaphore = true; + return true; +} + +inline static bool needs_reservation(struct kgem *kgem, struct kgem_bo *bo) +{ + if (kgem->needs_reservation) + return false; + + if (bo->presumed_offset) + return false; + + kgem->needs_reservation = true; + return nearly_idle(kgem); +} + +inline static bool needs_batch_flush(struct kgem *kgem, struct kgem_bo *bo) +{ + bool flush = false; + + if (needs_semaphore(kgem, bo)) { + DBG(("%s: flushing before handle=%d for required semaphore\n", __FUNCTION__, bo->handle)); + flush = true; + } + + if (needs_reservation(kgem, bo)) { + DBG(("%s: flushing before handle=%d for new reservation\n", __FUNCTION__, bo->handle)); + flush = true; + } + + return kgem->nreloc ? flush : false; +} + +static bool aperture_check(struct kgem *kgem, unsigned num_pages) +{ + struct drm_i915_gem_get_aperture aperture; + int reserve; + + if (kgem->aperture) + return false; + + /* Leave some space in case of alignment issues */ + reserve = kgem->aperture_mappable / 2; + if (kgem->gen < 033 && reserve < kgem->aperture_max_fence) + reserve = kgem->aperture_max_fence; + if (!kgem->has_llc) + reserve += kgem->nexec * PAGE_SIZE * 2; + + DBG(("%s: num_pages=%d, holding %d pages in reserve, total aperture %d\n", + __FUNCTION__, num_pages, reserve, kgem->aperture_total)); + num_pages += reserve; + + VG_CLEAR(aperture); + aperture.aper_available_size = kgem->aperture_total; + aperture.aper_available_size *= PAGE_SIZE; + (void)do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture); + + DBG(("%s: aperture required %ld bytes, available %ld bytes\n", + __FUNCTION__, + (long)num_pages * PAGE_SIZE, + (long)aperture.aper_available_size)); + + return num_pages <= aperture.aper_available_size / PAGE_SIZE; +} + +static inline bool kgem_flush(struct kgem *kgem, bool flush) +{ + if (unlikely(kgem->wedged)) + return false; + + if (kgem->nreloc == 0) + return true; + + if (container_of(kgem, struct sna, kgem)->flags & SNA_POWERSAVE) + return true; + + if (kgem->flush == flush && kgem->aperture < kgem->aperture_low) + return true; + + DBG(("%s: opportunistic flushing? flush=%d,%d, aperture=%d/%d, idle?=%d\n", + __FUNCTION__, kgem->flush, flush, kgem->aperture, kgem->aperture_low, kgem_ring_is_idle(kgem, kgem->ring))); + return !kgem_ring_is_idle(kgem, kgem->ring); } bool kgem_check_bo(struct kgem *kgem, ...) @@ -4563,6 +5404,7 @@ bool kgem_check_bo(struct kgem *kgem, .. int num_exec = 0; int num_pages = 0; bool flush = false; + bool busy = true; va_start(ap, kgem); while ((bo = va_arg(ap, struct kgem_bo *))) { @@ -4571,13 +5413,16 @@ bool kgem_check_bo(struct kgem *kgem, .. if (bo->exec) continue; - if (needs_semaphore(kgem, bo)) + if (needs_batch_flush(kgem, bo)) { + va_end(ap); return false; + } num_pages += num_pages(bo); num_exec++; flush |= bo->flush; + busy &= bo->rq != NULL; } va_end(ap); @@ -4587,35 +5432,26 @@ bool kgem_check_bo(struct kgem *kgem, .. if (!num_pages) return true; - if (kgem_flush(kgem, flush)) - return false; - - if (kgem->aperture > kgem->aperture_low && - kgem_ring_is_idle(kgem, kgem->ring)) { - DBG(("%s: current aperture usage (%d) is greater than low water mark (%d)\n", - __FUNCTION__, kgem->aperture, kgem->aperture_low)); + if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) { + DBG(("%s: out of exec slots (%d + %d / %d)\n", __FUNCTION__, + kgem->nexec, num_exec, KGEM_EXEC_SIZE(kgem))); return false; } if (num_pages + kgem->aperture > kgem->aperture_high) { - DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n", - __FUNCTION__, num_pages + kgem->aperture, kgem->aperture_high)); - return false; + DBG(("%s: final aperture usage (%d + %d) is greater than high water mark (%d)\n", + __FUNCTION__, kgem->aperture, num_pages, kgem->aperture_high)); + return aperture_check(kgem, num_pages); } - if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) { - DBG(("%s: out of exec slots (%d + %d / %d)\n", __FUNCTION__, - kgem->nexec, num_exec, KGEM_EXEC_SIZE(kgem))); - return false; - } + if (busy) + return true; - return true; + return kgem_flush(kgem, flush); } bool kgem_check_bo_fenced(struct kgem *kgem, struct kgem_bo *bo) { - uint32_t size; - assert(bo->refcnt); while (bo->proxy) bo = bo->proxy; @@ -4625,54 +5461,93 @@ bool kgem_check_bo_fenced(struct kgem *k if (kgem->gen < 040 && bo->tiling != I915_TILING_NONE && (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) { + uint32_t size; + + assert(bo->tiling == I915_TILING_X); + if (kgem->nfence >= kgem->fence_max) return false; - if (3*kgem->aperture_fenced > kgem->aperture_mappable && - kgem_ring_is_idle(kgem, kgem->ring)) - return false; + if (kgem->aperture_fenced) { + size = 3*kgem->aperture_fenced; + if (kgem->aperture_total == kgem->aperture_mappable) + size += kgem->aperture; + if (size > kgem->aperture_fenceable && + kgem_ring_is_idle(kgem, kgem->ring)) { + DBG(("%s: opportunistic fence flush\n", __FUNCTION__)); + return false; + } + } - size = kgem->aperture_fenced; - size += kgem_bo_fenced_size(kgem, bo); - if (3*size > 2*kgem->aperture_mappable) + size = kgem_bo_fenced_size(kgem, bo); + if (size > kgem->aperture_max_fence) + kgem->aperture_max_fence = size; + size += kgem->aperture_fenced; + if (kgem->gen < 033 && size < 2 * kgem->aperture_max_fence) + size = 2 * kgem->aperture_max_fence; + if (kgem->aperture_total == kgem->aperture_mappable) + size += kgem->aperture; + if (size > kgem->aperture_fenceable) { + DBG(("%s: estimated fence space required %d (fenced=%d, max_fence=%d, aperture=%d) exceeds fenceable aperture %d\n", + __FUNCTION__, size, kgem->aperture_fenced, kgem->aperture_max_fence, kgem->aperture, kgem->aperture_fenceable)); return false; + } } return true; } - if (needs_semaphore(kgem, bo)) - return false; - - if (kgem_flush(kgem, bo->flush)) - return false; - if (kgem->nexec >= KGEM_EXEC_SIZE(kgem) - 1) return false; - if (kgem->aperture > kgem->aperture_low && - kgem_ring_is_idle(kgem, kgem->ring)) - return false; - - if (kgem->aperture + num_pages(bo) > kgem->aperture_high) + if (needs_batch_flush(kgem, bo)) return false; assert_tiling(kgem, bo); if (kgem->gen < 040 && bo->tiling != I915_TILING_NONE) { + uint32_t size; + + assert(bo->tiling == I915_TILING_X); + if (kgem->nfence >= kgem->fence_max) return false; - if (3*kgem->aperture_fenced > kgem->aperture_mappable && - kgem_ring_is_idle(kgem, kgem->ring)) - return false; + if (kgem->aperture_fenced) { + size = 3*kgem->aperture_fenced; + if (kgem->aperture_total == kgem->aperture_mappable) + size += kgem->aperture; + if (size > kgem->aperture_fenceable && + kgem_ring_is_idle(kgem, kgem->ring)) { + DBG(("%s: opportunistic fence flush\n", __FUNCTION__)); + return false; + } + } - size = kgem->aperture_fenced; - size += kgem_bo_fenced_size(kgem, bo); - if (3*size > 2*kgem->aperture_mappable) + size = kgem_bo_fenced_size(kgem, bo); + if (size > kgem->aperture_max_fence) + kgem->aperture_max_fence = size; + size += kgem->aperture_fenced; + if (kgem->gen < 033 && size < 2 * kgem->aperture_max_fence) + size = 2 * kgem->aperture_max_fence; + if (kgem->aperture_total == kgem->aperture_mappable) + size += kgem->aperture; + if (size > kgem->aperture_fenceable) { + DBG(("%s: estimated fence space required %d (fenced=%d, max_fence=%d, aperture=%d) exceeds fenceable aperture %d\n", + __FUNCTION__, size, kgem->aperture_fenced, kgem->aperture_max_fence, kgem->aperture, kgem->aperture_fenceable)); return false; + } } - return true; + if (kgem->aperture + kgem->aperture_fenced + num_pages(bo) > kgem->aperture_high) { + DBG(("%s: final aperture usage (%d + %d) is greater than high water mark (%d)\n", + __FUNCTION__, kgem->aperture, num_pages(bo), kgem->aperture_high)); + return aperture_check(kgem, num_pages(bo)); + } + + if (bo->rq) + return true; + + return kgem_flush(kgem, bo->flush); } bool kgem_check_many_bo_fenced(struct kgem *kgem, ...) @@ -4684,6 +5559,7 @@ bool kgem_check_many_bo_fenced(struct kg int num_pages = 0; int fenced_size = 0; bool flush = false; + bool busy = true; va_start(ap, kgem); while ((bo = va_arg(ap, struct kgem_bo *))) { @@ -4703,49 +5579,73 @@ bool kgem_check_many_bo_fenced(struct kg continue; } - if (needs_semaphore(kgem, bo)) + if (needs_batch_flush(kgem, bo)) { + va_end(ap); return false; + } assert_tiling(kgem, bo); num_pages += num_pages(bo); num_exec++; if (kgem->gen < 040 && bo->tiling) { - fenced_size += kgem_bo_fenced_size(kgem, bo); + uint32_t size = kgem_bo_fenced_size(kgem, bo); + if (size > kgem->aperture_max_fence) + kgem->aperture_max_fence = size; + fenced_size += size; num_fence++; } flush |= bo->flush; + busy &= bo->rq != NULL; } va_end(ap); if (num_fence) { + uint32_t size; + if (kgem->nfence + num_fence > kgem->fence_max) return false; - if (3*kgem->aperture_fenced > kgem->aperture_mappable && - kgem_ring_is_idle(kgem, kgem->ring)) - return false; + if (kgem->aperture_fenced) { + size = 3*kgem->aperture_fenced; + if (kgem->aperture_total == kgem->aperture_mappable) + size += kgem->aperture; + if (size > kgem->aperture_fenceable && + kgem_ring_is_idle(kgem, kgem->ring)) { + DBG(("%s: opportunistic fence flush\n", __FUNCTION__)); + return false; + } + } - if (3*(fenced_size + kgem->aperture_fenced) > 2*kgem->aperture_mappable) + size = kgem->aperture_fenced; + size += fenced_size; + if (kgem->gen < 033 && size < 2 * kgem->aperture_max_fence) + size = 2 * kgem->aperture_max_fence; + if (kgem->aperture_total == kgem->aperture_mappable) + size += kgem->aperture; + if (size > kgem->aperture_fenceable) { + DBG(("%s: estimated fence space required %d (fenced=%d, max_fence=%d, aperture=%d) exceeds fenceable aperture %d\n", + __FUNCTION__, size, kgem->aperture_fenced, kgem->aperture_max_fence, kgem->aperture, kgem->aperture_fenceable)); return false; + } } - if (num_pages) { - if (kgem_flush(kgem, flush)) - return false; - - if (kgem->aperture > kgem->aperture_low && - kgem_ring_is_idle(kgem, kgem->ring)) - return false; + if (num_pages == 0) + return true; - if (num_pages + kgem->aperture > kgem->aperture_high) - return false; + if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) + return false; - if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) - return false; + if (num_pages + kgem->aperture > kgem->aperture_high - kgem->aperture_fenced) { + DBG(("%s: final aperture usage (%d + %d + %d) is greater than high water mark (%d)\n", + __FUNCTION__, kgem->aperture, kgem->aperture_fenced, num_pages, kgem->aperture_high)); + return aperture_check(kgem, num_pages); } - return true; + if (busy) + return true; + + return kgem_flush(kgem, flush); } uint32_t kgem_add_reloc(struct kgem *kgem, @@ -4759,12 +5659,14 @@ uint32_t kgem_add_reloc(struct kgem *kge DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n", __FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain)); + assert(kgem->gen < 0100); assert((read_write_domain & 0x7fff) == 0 || bo != NULL); index = kgem->nreloc++; assert(index < ARRAY_SIZE(kgem->reloc)); kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]); if (bo) { + assert(kgem->mode != KGEM_NONE); assert(bo->refcnt); while (bo->proxy) { DBG(("%s: adding proxy [delta=%d] for handle=%d\n", @@ -4778,6 +5680,7 @@ uint32_t kgem_add_reloc(struct kgem *kge bo->rq = MAKE_REQUEST(kgem->next_request, kgem->ring); bo->exec = &_kgem_dummy_exec; + bo->domain = DOMAIN_GPU; } if (read_write_domain & 0x7fff && !bo->gpu_dirty) @@ -4796,6 +5699,7 @@ uint32_t kgem_add_reloc(struct kgem *kge if (kgem->gen < 040 && read_write_domain & KGEM_RELOC_FENCED) { if (bo->tiling && (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) { + assert(bo->tiling == I915_TILING_X); assert(kgem->nfence < kgem->fence_max); kgem->aperture_fenced += kgem_bo_fenced_size(kgem, bo); @@ -4827,6 +5731,81 @@ uint32_t kgem_add_reloc(struct kgem *kge return delta; } +uint64_t kgem_add_reloc64(struct kgem *kgem, + uint32_t pos, + struct kgem_bo *bo, + uint32_t read_write_domain, + uint64_t delta) +{ + int index; + + DBG(("%s: handle=%d, pos=%d, delta=%ld, domains=%08x\n", + __FUNCTION__, bo ? bo->handle : 0, pos, (long)delta, read_write_domain)); + + assert(kgem->gen >= 0100); + assert((read_write_domain & 0x7fff) == 0 || bo != NULL); + + index = kgem->nreloc++; + assert(index < ARRAY_SIZE(kgem->reloc)); + kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]); + if (bo) { + assert(kgem->mode != KGEM_NONE); + assert(bo->refcnt); + while (bo->proxy) { + DBG(("%s: adding proxy [delta=%ld] for handle=%d\n", + __FUNCTION__, (long)bo->delta, bo->handle)); + delta += bo->delta; + assert(bo->handle == bo->proxy->handle); + /* need to release the cache upon batch submit */ + if (bo->exec == NULL) { + list_move_tail(&bo->request, + &kgem->next_request->buffers); + bo->rq = MAKE_REQUEST(kgem->next_request, + kgem->ring); + bo->exec = &_kgem_dummy_exec; + bo->domain = DOMAIN_GPU; + } + + if (read_write_domain & 0x7fff && !bo->gpu_dirty) + __kgem_bo_mark_dirty(bo); + + bo = bo->proxy; + assert(bo->refcnt); + } + assert(bo->refcnt); + + if (bo->exec == NULL) + kgem_add_bo(kgem, bo); + assert(bo->rq == MAKE_REQUEST(kgem->next_request, kgem->ring)); + assert(RQ_RING(bo->rq) == kgem->ring); + + DBG(("%s[%d] = (delta=%d, target handle=%d, presumed=%llx)\n", + __FUNCTION__, index, delta, bo->target_handle, (long long)bo->presumed_offset)); + kgem->reloc[index].delta = delta; + kgem->reloc[index].target_handle = bo->target_handle; + kgem->reloc[index].presumed_offset = bo->presumed_offset; + + if (read_write_domain & 0x7fff && !bo->gpu_dirty) { + assert(!bo->snoop || kgem->can_blt_cpu); + __kgem_bo_mark_dirty(bo); + } + + delta += bo->presumed_offset; + } else { + DBG(("%s[%d] = (delta=%d, target handle=batch)\n", + __FUNCTION__, index, delta)); + kgem->reloc[index].delta = delta; + kgem->reloc[index].target_handle = ~0U; + kgem->reloc[index].presumed_offset = 0; + if (kgem->nreloc__self < 256) + kgem->reloc__self[kgem->nreloc__self++] = index; + } + kgem->reloc[index].read_domains = read_write_domain >> 16; + kgem->reloc[index].write_domain = read_write_domain & 0x7fff; + + return delta; +} + static void kgem_trim_vma_cache(struct kgem *kgem, int type, int bucket) { int i, j; @@ -4849,6 +5828,7 @@ static void kgem_trim_vma_cache(struct k i = 0; while (kgem->vma[type].count > 0) { struct kgem_bo *bo = NULL; + void **ptr; for (j = 0; bo == NULL && j < ARRAY_SIZE(kgem->vma[type].inactive); @@ -4861,15 +5841,14 @@ static void kgem_trim_vma_cache(struct k break; DBG(("%s: discarding inactive %s vma cache for %d\n", - __FUNCTION__, - IS_CPU_MAP(bo->map) ? "CPU" : "GTT", bo->handle)); - assert(IS_CPU_MAP(bo->map) == type); - assert(bo->map); + __FUNCTION__, type ? "CPU" : "GTT", bo->handle)); + + ptr = type ? &bo->map__cpu : &bo->map__gtt; assert(bo->rq == NULL); - VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo))); - munmap(MAP(bo->map), bytes(bo)); - bo->map = NULL; + VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(*ptr), bytes(bo))); + munmap(MAP(*ptr), bytes(bo)); + *ptr = NULL; list_del(&bo->vma); kgem->vma[type].count--; @@ -4885,13 +5864,13 @@ void *kgem_bo_map__async(struct kgem *kg { void *ptr; - DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__, - bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain)); + DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__, + bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain)); assert(bo->proxy == NULL); assert(list_is_empty(&bo->list)); - assert(!IS_USER_MAP(bo->map)); assert_tiling(kgem, bo); + assert(!bo->purged || bo->reusable); if (bo->tiling == I915_TILING_NONE && !bo->scanout && kgem->has_llc) { DBG(("%s: converting request for GTT map into CPU map\n", @@ -4899,12 +5878,9 @@ void *kgem_bo_map__async(struct kgem *kg return kgem_bo_map__cpu(kgem, bo); } - if (IS_CPU_MAP(bo->map)) - kgem_bo_release_map(kgem, bo); - - ptr = bo->map; + ptr = MAP(bo->map__gtt); if (ptr == NULL) { - assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2); + assert(num_pages(bo) <= kgem->aperture_mappable / 2); kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo)); @@ -4917,7 +5893,7 @@ void *kgem_bo_map__async(struct kgem *kg * issue with compositing managers which need to frequently * flush CPU damage to their GPU bo. */ - bo->map = ptr; + bo->map__gtt = ptr; DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle)); } @@ -4928,14 +5904,14 @@ void *kgem_bo_map(struct kgem *kgem, str { void *ptr; - DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__, - bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain)); + DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__, + bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain)); assert(bo->proxy == NULL); assert(list_is_empty(&bo->list)); - assert(!IS_USER_MAP(bo->map)); assert(bo->exec == NULL); assert_tiling(kgem, bo); + assert(!bo->purged || bo->reusable); if (bo->tiling == I915_TILING_NONE && !bo->scanout && (kgem->has_llc || bo->domain == DOMAIN_CPU)) { @@ -4947,12 +5923,9 @@ void *kgem_bo_map(struct kgem *kgem, str return ptr; } - if (IS_CPU_MAP(bo->map)) - kgem_bo_release_map(kgem, bo); - - ptr = bo->map; + ptr = MAP(bo->map__gtt); if (ptr == NULL) { - assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2); + assert(num_pages(bo) <= kgem->aperture_mappable / 2); assert(kgem->gen != 021 || bo->tiling != I915_TILING_Y); kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo)); @@ -4966,7 +5939,7 @@ void *kgem_bo_map(struct kgem *kgem, str * issue with compositing managers which need to frequently * flush CPU damage to their GPU bo. */ - bo->map = ptr; + bo->map__gtt = ptr; DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle)); } @@ -4982,11 +5955,13 @@ void *kgem_bo_map(struct kgem *kgem, str set_domain.handle = bo->handle; set_domain.read_domains = I915_GEM_DOMAIN_GTT; set_domain.write_domain = I915_GEM_DOMAIN_GTT; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) { - kgem_bo_retire(kgem, bo); - bo->domain = DOMAIN_GTT; - bo->gtt_dirty = true; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); + kgem_throttle(kgem); } + kgem_bo_retire(kgem, bo); + bo->domain = DOMAIN_GTT; + bo->gtt_dirty = true; } return ptr; @@ -4996,20 +5971,18 @@ void *kgem_bo_map__gtt(struct kgem *kgem { void *ptr; - DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__, - bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain)); + DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__, + bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain)); + assert(bo->proxy == NULL); assert(bo->exec == NULL); assert(list_is_empty(&bo->list)); - assert(!IS_USER_MAP(bo->map)); assert_tiling(kgem, bo); + assert(!bo->purged || bo->reusable); - if (IS_CPU_MAP(bo->map)) - kgem_bo_release_map(kgem, bo); - - ptr = bo->map; + ptr = MAP(bo->map__gtt); if (ptr == NULL) { - assert(bytes(bo) <= kgem->aperture_mappable / 4); + assert(num_pages(bo) <= kgem->aperture_mappable / 4); kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo)); @@ -5022,7 +5995,7 @@ void *kgem_bo_map__gtt(struct kgem *kgem * issue with compositing managers which need to frequently * flush CPU damage to their GPU bo. */ - bo->map = ptr; + bo->map__gtt = ptr; DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle)); } @@ -5031,28 +6004,23 @@ void *kgem_bo_map__gtt(struct kgem *kgem void *kgem_bo_map__debug(struct kgem *kgem, struct kgem_bo *bo) { - if (bo->map) - return MAP(bo->map); - - kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo)); - return bo->map = __kgem_bo_map__gtt(kgem, bo); + return kgem_bo_map__async(kgem, bo); } void *kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo) { struct drm_i915_gem_mmap mmap_arg; + int err; - DBG(("%s(handle=%d, size=%d, mapped? %d)\n", - __FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map))); + DBG(("%s(handle=%d, size=%d, map=%p:%p)\n", + __FUNCTION__, bo->handle, bytes(bo), bo->map__gtt, bo->map__cpu)); assert(!bo->purged); assert(list_is_empty(&bo->list)); assert(bo->proxy == NULL); + assert_tiling(kgem, bo); - if (IS_CPU_MAP(bo->map)) - return MAP(bo->map); - - if (bo->map) - kgem_bo_release_map(kgem, bo); + if (bo->map__cpu) + return MAP(bo->map__cpu); kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo)); @@ -5061,92 +6029,24 @@ retry: mmap_arg.handle = bo->handle; mmap_arg.offset = 0; mmap_arg.size = bytes(bo); - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) { - int err = errno; - + if ((err = do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg))) { assert(err != EINVAL); if (__kgem_throttle_retire(kgem, 0)) goto retry; - if (kgem->need_expire) { - kgem_cleanup_cache(kgem); + if (kgem_cleanup_cache(kgem)) goto retry; - } - ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n", - __FUNCTION__, bo->handle, bytes(bo), err); + ERR(("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n", + __FUNCTION__, bo->handle, bytes(bo), -err)); return NULL; } VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo))); DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle)); - bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr); - return (void *)(uintptr_t)mmap_arg.addr_ptr; -} - -void *__kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo) -{ - struct drm_i915_gem_mmap mmap_arg; - - DBG(("%s(handle=%d, size=%d, mapped? %d)\n", - __FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map))); - assert(bo->refcnt); - assert(!bo->purged); - assert(list_is_empty(&bo->list)); - assert(bo->proxy == NULL); - - if (IS_CPU_MAP(bo->map)) - return MAP(bo->map); - -retry: - VG_CLEAR(mmap_arg); - mmap_arg.handle = bo->handle; - mmap_arg.offset = 0; - mmap_arg.size = bytes(bo); - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) { - int err = errno; - - assert(err != EINVAL); - - if (__kgem_throttle_retire(kgem, 0)) - goto retry; - - if (kgem->need_expire) { - kgem_cleanup_cache(kgem); - goto retry; - } - - ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n", - __FUNCTION__, bo->handle, bytes(bo), err); - return NULL; - } - - VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo))); - if (bo->map && bo->domain == DOMAIN_CPU) { - DBG(("%s: discarding GTT vma for %d\n", __FUNCTION__, bo->handle)); - kgem_bo_release_map(kgem, bo); - } - if (bo->map == NULL) { - DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle)); - bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr); - } - return (void *)(uintptr_t)mmap_arg.addr_ptr; -} - -void __kgem_bo_unmap__cpu(struct kgem *kgem, struct kgem_bo *bo, void *ptr) -{ - DBG(("%s(handle=%d, size=%d)\n", - __FUNCTION__, bo->handle, bytes(bo))); - assert(bo->refcnt); - - if (IS_CPU_MAP(bo->map)) { - assert(ptr == MAP(bo->map)); - return; - } - - munmap(ptr, bytes(bo)); + return bo->map__cpu = (void *)(uintptr_t)mmap_arg.addr_ptr; } uint32_t kgem_bo_flink(struct kgem *kgem, struct kgem_bo *bo) @@ -5155,7 +6055,7 @@ uint32_t kgem_bo_flink(struct kgem *kgem VG_CLEAR(flink); flink.handle = bo->handle; - if (drmIoctl(kgem->fd, DRM_IOCTL_GEM_FLINK, &flink)) + if (do_ioctl(kgem->fd, DRM_IOCTL_GEM_FLINK, &flink)) return 0; DBG(("%s: flinked handle=%d to name=%d, marking non-reusable\n", @@ -5170,13 +6070,6 @@ uint32_t kgem_bo_flink(struct kgem *kgem kgem_bo_unclean(kgem, bo); - /* Henceforth, we need to broadcast all updates to clients and - * flush our rendering before doing so. - */ - bo->flush = true; - if (bo->exec) - kgem->flush = 1; - return flink.name; } @@ -5190,6 +6083,8 @@ struct kgem_bo *kgem_create_map(struct k assert(MAP(ptr) == ptr); + DBG(("%s(%p size=%d, read-only?=%d) - has_userptr?=%d\n", __FUNCTION__, + ptr, size, read_only, kgem->has_userptr)); if (!kgem->has_userptr) return NULL; @@ -5203,8 +6098,10 @@ struct kgem_bo *kgem_create_map(struct k handle = gem_userptr(kgem->fd, (void *)first_page, last_page-first_page, read_only); - if (handle == 0) + if (handle == 0) { + DBG(("%s: import failed, errno=%d\n", __FUNCTION__, errno)); return NULL; + } bo = __kgem_bo_alloc(handle, (last_page - first_page) / PAGE_SIZE); if (bo == NULL) { @@ -5212,6 +6109,7 @@ struct kgem_bo *kgem_create_map(struct k return NULL; } + bo->unique_id = kgem_get_unique_id(kgem); bo->snoop = !kgem->has_llc; debug_alloc__bo(kgem, bo); @@ -5227,7 +6125,7 @@ struct kgem_bo *kgem_create_map(struct k bo = proxy; } - bo->map = MAKE_USER_MAP(ptr); + bo->map__cpu = MAKE_USER_MAP(ptr); DBG(("%s(ptr=%p, size=%d, pages=%d, read_only=%d) => handle=%d (proxy? %d)\n", __FUNCTION__, ptr, size, NUM_PAGES(size), read_only, handle, bo->proxy != NULL)); @@ -5238,6 +6136,8 @@ void kgem_bo_sync__cpu(struct kgem *kgem { DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); assert(!bo->scanout); + assert_tiling(kgem, bo); + kgem_bo_submit(kgem, bo); /* SHM pixmaps use proxies for subpage offsets */ @@ -5259,10 +6159,12 @@ void kgem_bo_sync__cpu(struct kgem *kgem set_domain.read_domains = I915_GEM_DOMAIN_CPU; set_domain.write_domain = I915_GEM_DOMAIN_CPU; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) { - kgem_bo_retire(kgem, bo); - bo->domain = DOMAIN_CPU; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); + kgem_throttle(kgem); } + kgem_bo_retire(kgem, bo); + bo->domain = DOMAIN_CPU; } } @@ -5270,6 +6172,7 @@ void kgem_bo_sync__cpu_full(struct kgem { DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); assert(!bo->scanout || !write); + assert_tiling(kgem, bo); if (write || bo->needs_flush) kgem_bo_submit(kgem, bo); @@ -5295,10 +6198,17 @@ void kgem_bo_sync__cpu_full(struct kgem set_domain.read_domains = I915_GEM_DOMAIN_CPU; set_domain.write_domain = write ? I915_GEM_DOMAIN_CPU : 0; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) { + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); + kgem_throttle(kgem); + } + if (write) { + kgem_bo_retire(kgem, bo); + bo->domain = DOMAIN_CPU; + } else { if (bo->exec == NULL) - kgem_bo_retire(kgem, bo); - bo->domain = write ? DOMAIN_CPU : DOMAIN_NONE; + kgem_bo_maybe_retire(kgem, bo); + bo->domain = DOMAIN_NONE; } } } @@ -5308,6 +6218,7 @@ void kgem_bo_sync__gtt(struct kgem *kgem DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); assert(bo->refcnt); assert(bo->proxy == NULL); + assert_tiling(kgem, bo); kgem_bo_submit(kgem, bo); @@ -5324,11 +6235,13 @@ void kgem_bo_sync__gtt(struct kgem *kgem set_domain.read_domains = I915_GEM_DOMAIN_GTT; set_domain.write_domain = I915_GEM_DOMAIN_GTT; - if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) { - kgem_bo_retire(kgem, bo); - bo->domain = DOMAIN_GTT; - bo->gtt_dirty = true; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); + kgem_throttle(kgem); } + kgem_bo_retire(kgem, bo); + bo->domain = DOMAIN_GTT; + bo->gtt_dirty = true; } } @@ -5374,11 +6287,13 @@ struct kgem_bo *kgem_create_proxy(struct bo->proxy = kgem_bo_reference(target); bo->delta = offset; - if (target->exec) { + /* Proxies are only tracked for busyness on the current rq */ + if (target->exec && !bo->io) { + assert(RQ(target->rq) == kgem->next_request); list_move_tail(&bo->request, &kgem->next_request->buffers); bo->exec = &_kgem_dummy_exec; + bo->rq = target->rq; } - bo->rq = target->rq; return bo; } @@ -5394,7 +6309,7 @@ buffer_alloc(void) bo->mem = NULL; bo->need_io = false; - bo->mmapped = true; + bo->mmapped = MMAPPED_CPU; return bo; } @@ -5469,7 +6384,7 @@ search_snoopable_buffer(struct kgem *kge assert(bo->base.snoop); assert(bo->base.tiling == I915_TILING_NONE); assert(num_pages(&bo->base) >= alloc); - assert(bo->mmapped == true); + assert(bo->mmapped == MMAPPED_CPU); assert(bo->need_io == false); bo->mem = kgem_bo_map__cpu(kgem, &bo->base); @@ -5509,14 +6424,14 @@ create_snoopable_buffer(struct kgem *kge return NULL; } - debug_alloc(kgem, alloc); __kgem_bo_init(&bo->base, handle, alloc); + debug_alloc__bo(kgem, &bo->base); DBG(("%s: created CPU (LLC) handle=%d for buffer, size %d\n", __FUNCTION__, bo->base.handle, alloc)); } assert(bo->base.refcnt == 1); - assert(bo->mmapped == true); + assert(bo->mmapped == MMAPPED_CPU); assert(bo->need_io == false); bo->mem = kgem_bo_map__cpu(kgem, &bo->base); @@ -5545,14 +6460,14 @@ create_snoopable_buffer(struct kgem *kge return NULL; } - debug_alloc(kgem, alloc); __kgem_bo_init(&bo->base, handle, alloc); + debug_alloc__bo(kgem, &bo->base); DBG(("%s: created CPU handle=%d for buffer, size %d\n", __FUNCTION__, bo->base.handle, alloc)); } assert(bo->base.refcnt == 1); - assert(bo->mmapped == true); + assert(bo->mmapped == MMAPPED_CPU); assert(bo->need_io == false); if (!gem_set_caching(kgem->fd, bo->base.handle, SNOOPED)) @@ -5589,17 +6504,17 @@ free_caching: return NULL; } - debug_alloc(kgem, alloc); __kgem_bo_init(&bo->base, handle, alloc); + debug_alloc__bo(kgem, &bo->base); DBG(("%s: created snoop handle=%d for buffer\n", __FUNCTION__, bo->base.handle)); - assert(bo->mmapped == true); + assert(bo->mmapped == MMAPPED_CPU); assert(bo->need_io == false); bo->base.refcnt = 1; bo->base.snoop = true; - bo->base.map = MAKE_USER_MAP(bo->mem); + bo->base.map__cpu = MAKE_USER_MAP(bo->mem); return bo; } @@ -5632,13 +6547,14 @@ struct kgem_bo *kgem_create_buffer(struc /* We can reuse any write buffer which we can fit */ if (flags == KGEM_BUFFER_LAST && bo->write == KGEM_BUFFER_WRITE && - bo->base.refcnt == 1 && !bo->mmapped && + bo->base.refcnt == 1 && + bo->mmapped == MMAPPED_NONE && size <= bytes(&bo->base)) { DBG(("%s: reusing write buffer for read of %d bytes? used=%d, total=%d\n", __FUNCTION__, size, bo->used, bytes(&bo->base))); - gem_write(kgem->fd, bo->base.handle, - 0, bo->used, bo->mem); - kgem_buffer_release(kgem, bo); + gem_write__cachealigned(kgem->fd, bo->base.handle, + 0, bo->used, bo->mem); + assert(list_is_empty(&bo->base.vma)); bo->need_io = 0; bo->write = 0; offset = 0; @@ -5676,10 +6592,11 @@ struct kgem_bo *kgem_create_buffer(struc list_for_each_entry(bo, &kgem->active_buffers, base.list) { assert(bo->base.io); assert(bo->base.refcnt >= 1); + assert(bo->base.exec == NULL); assert(bo->mmapped); - assert(!IS_CPU_MAP(bo->base.map) || kgem->has_llc || bo->base.snoop); + assert(bo->mmapped == MMAPPED_GTT || kgem->has_llc || bo->base.snoop); - if (!kgem->has_llc && (bo->write & ~flags) & KGEM_BUFFER_INPLACE) { + if ((bo->write & ~flags) & KGEM_BUFFER_INPLACE && !bo->base.snoop) { DBG(("%s: skip write %x buffer, need %x\n", __FUNCTION__, bo->write, flags)); continue; @@ -5693,6 +6610,30 @@ struct kgem_bo *kgem_create_buffer(struc list_move(&bo->base.list, &kgem->batch_buffers); goto done; } + + if (bo->base.refcnt == 1 && + size <= bytes(&bo->base) && + (bo->base.rq == NULL || + !__kgem_busy(kgem, bo->base.handle))) { + DBG(("%s: reusing whole buffer? size=%d, total=%d\n", + __FUNCTION__, size, bytes(&bo->base))); + __kgem_bo_clear_busy(&bo->base); + assert(list_is_empty(&bo->base.vma)); + + switch (bo->mmapped) { + case MMAPPED_CPU: + kgem_bo_sync__cpu(kgem, &bo->base); + break; + case MMAPPED_GTT: + kgem_bo_sync__gtt(kgem, &bo->base); + break; + } + + offset = 0; + bo->used = size; + list_move(&bo->base.list, &kgem->batch_buffers); + goto done; + } } } #endif @@ -5706,9 +6647,9 @@ struct kgem_bo *kgem_create_buffer(struc alloc = PAGE_ALIGN(size); assert(alloc); + alloc /= PAGE_SIZE; if (alloc > kgem->aperture_mappable / 4) flags &= ~KGEM_BUFFER_INPLACE; - alloc /= PAGE_SIZE; if (kgem->has_llc && (flags & KGEM_BUFFER_WRITE_INPLACE) != KGEM_BUFFER_WRITE_INPLACE) { @@ -5735,10 +6676,9 @@ struct kgem_bo *kgem_create_buffer(struc goto skip_llc; } __kgem_bo_init(&bo->base, handle, alloc); + debug_alloc__bo(kgem, &bo->base); DBG(("%s: created LLC handle=%d for buffer\n", __FUNCTION__, bo->base.handle)); - - debug_alloc(kgem, alloc); } assert(bo->mmapped); @@ -5778,7 +6718,7 @@ skip_llc: VG_CLEAR(info); info.handle = handle; - if (drmIoctl(kgem->fd, + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_BUFFER_INFO, &fino) == 0) { old->presumed_offset = info.addr; @@ -5794,7 +6734,7 @@ skip_llc: CREATE_EXACT | CREATE_INACTIVE | CREATE_GTT_MAP); if (old == NULL) { old = search_linear_cache(kgem, alloc, CREATE_INACTIVE); - if (old && !__kgem_bo_is_mappable(kgem, old)) { + if (old && !kgem_bo_can_map(kgem, old)) { _kgem_bo_destroy(kgem, old); old = NULL; } @@ -5802,7 +6742,7 @@ skip_llc: if (old) { DBG(("%s: reusing handle=%d for buffer\n", __FUNCTION__, old->handle)); - assert(__kgem_bo_is_mappable(kgem, old)); + assert(kgem_bo_can_map(kgem, old)); assert(!old->snoop); assert(old->rq == NULL); @@ -5818,8 +6758,10 @@ skip_llc: bo->mem = kgem_bo_map(kgem, &bo->base); if (bo->mem) { - if (IS_CPU_MAP(bo->base.map)) + if (bo->mem == MAP(bo->base.map__cpu)) flags &= ~KGEM_BUFFER_INPLACE; + else + bo->mmapped = MMAPPED_GTT; goto init; } else { bo->base.refcnt = 0; @@ -5899,7 +6841,7 @@ skip_llc: __FUNCTION__, handle)); __kgem_bo_init(&bo->base, handle, alloc); - debug_alloc(kgem, alloc * PAGE_SIZE); + debug_alloc__bo(kgem, &bo->base); } assert(bo->mmapped); @@ -5938,7 +6880,8 @@ init: assert(!bo->need_io || !bo->base.needs_flush); assert(!bo->need_io || bo->base.domain != DOMAIN_GPU); assert(bo->mem); - assert(!bo->mmapped || bo->base.map != NULL); + assert(bo->mmapped != MMAPPED_GTT || MAP(bo->base.map__gtt) == bo->mem); + assert(bo->mmapped != MMAPPED_CPU || MAP(bo->base.map__cpu) == bo->mem); bo->used = size; bo->write = flags & KGEM_BUFFER_WRITE_INPLACE; @@ -5952,6 +6895,7 @@ init: done: bo->used = ALIGN(bo->used, UPLOAD_ALIGNMENT); + assert(bo->used && bo->used <= bytes(&bo->base)); assert(bo->mem); *ret = (char *)bo->mem + offset; return kgem_create_proxy(kgem, &bo->base, offset, size); @@ -6008,7 +6952,7 @@ struct kgem_bo *kgem_create_buffer_2d(st bo->size.bytes -= stride; } - bo->map = MAKE_CPU_MAP(*ret); + bo->map__cpu = *ret; bo->pitch = stride; bo->unique_id = kgem_get_unique_id(kgem); return bo; @@ -6039,13 +6983,21 @@ struct kgem_bo *kgem_upload_source_image bo = kgem_create_buffer_2d(kgem, width, height, bpp, KGEM_BUFFER_WRITE_INPLACE, &dst); - if (bo) - memcpy_blt(data, dst, bpp, - stride, bo->pitch, - box->x1, box->y1, - 0, 0, - width, height); + if (bo == NULL) + return NULL; + + if (sigtrap_get()) { + kgem_bo_destroy(kgem, bo); + return NULL; + } + + memcpy_blt(data, dst, bpp, + stride, bo->pitch, + box->x1, box->y1, + 0, 0, + width, height); + sigtrap_put(); return bo; } @@ -6053,10 +7005,10 @@ void kgem_proxy_bo_attach(struct kgem_bo struct kgem_bo **ptr) { DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); - assert(bo->map == NULL || IS_CPU_MAP(bo->map)); + assert(bo->map__gtt == NULL); assert(bo->proxy); list_add(&bo->vma, &bo->proxy->vma); - bo->map = ptr; + bo->map__gtt = ptr; *ptr = kgem_bo_reference(bo); } @@ -6089,24 +7041,25 @@ void kgem_buffer_read_sync(struct kgem * bo->base.domain, __kgem_busy(kgem, bo->base.handle))); - assert(!IS_CPU_MAP(bo->base.map) || bo->base.snoop || kgem->has_llc); + assert(bo->mmapped == MMAPPED_GTT || bo->base.snoop || kgem->has_llc); VG_CLEAR(set_domain); set_domain.handle = bo->base.handle; set_domain.write_domain = 0; set_domain.read_domains = - IS_CPU_MAP(bo->base.map) ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT; + bo->mmapped == MMAPPED_CPU ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT; - if (drmIoctl(kgem->fd, - DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) - return; + if (do_ioctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) { + DBG(("%s: sync: GPU hang detected\n", __FUNCTION__)); + kgem_throttle(kgem); + } } else { if (gem_read(kgem->fd, bo->base.handle, (char *)bo->mem+offset, offset, length)) return; } - kgem_bo_retire(kgem, &bo->base); + kgem_bo_maybe_retire(kgem, &bo->base); bo->base.domain = DOMAIN_NONE; } @@ -6114,6 +7067,8 @@ uint32_t kgem_bo_get_binding(struct kgem { struct kgem_bo_binding *b; + assert(bo->refcnt); + for (b = &bo->binding; b && b->offset; b = b->next) if (format == b->format) return b->offset; @@ -6125,6 +7080,8 @@ void kgem_bo_set_binding(struct kgem_bo { struct kgem_bo_binding *b; + assert(bo->refcnt); + for (b = &bo->binding; b; b = b->next) { if (b->offset) continue; @@ -6168,6 +7125,7 @@ kgem_replace_bo(struct kgem *kgem, * we only attempt to allocate a linear bo. */ assert(src->tiling == I915_TILING_NONE); + assert(kgem_bo_can_blt(kgem, src)); size = height * pitch; size = NUM_PAGES(size); @@ -6191,9 +7149,11 @@ kgem_replace_bo(struct kgem *kgem, dst->pitch = pitch; dst->unique_id = kgem_get_unique_id(kgem); dst->refcnt = 1; + assert(dst->tiling == I915_TILING_NONE); + assert(kgem_bo_can_blt(kgem, dst)); kgem_set_mode(kgem, KGEM_BLT, dst); - if (!kgem_check_batch(kgem, 8) || + if (!kgem_check_batch(kgem, 10) || !kgem_check_reloc(kgem, 2) || !kgem_check_many_bo_fenced(kgem, src, dst, NULL)) { kgem_submit(kgem); @@ -6222,22 +7182,67 @@ kgem_replace_bo(struct kgem *kgem, } b = kgem->batch + kgem->nbatch; - b[0] = br00; - b[1] = br13; - b[2] = 0; - b[3] = height << 16 | width; - b[4] = kgem_add_reloc(kgem, kgem->nbatch + 4, dst, - I915_GEM_DOMAIN_RENDER << 16 | - I915_GEM_DOMAIN_RENDER | - KGEM_RELOC_FENCED, - 0); - b[5] = 0; - b[6] = pitch; - b[7] = kgem_add_reloc(kgem, kgem->nbatch + 7, src, - I915_GEM_DOMAIN_RENDER << 16 | - KGEM_RELOC_FENCED, - 0); - kgem->nbatch += 8; + if (kgem->gen >= 0100) { + b[0] = br00 | 8; + b[1] = br13; + b[2] = 0; + b[3] = height << 16 | width; + *(uint64_t *)(b+4) = + kgem_add_reloc64(kgem, kgem->nbatch + 4, dst, + I915_GEM_DOMAIN_RENDER << 16 | + I915_GEM_DOMAIN_RENDER | + KGEM_RELOC_FENCED, + 0); + b[6] = 0; + b[7] = pitch; + *(uint64_t *)(b+8) = + kgem_add_reloc64(kgem, kgem->nbatch + 8, src, + I915_GEM_DOMAIN_RENDER << 16 | + KGEM_RELOC_FENCED, + 0); + kgem->nbatch += 10; + } else { + b[0] = br00 | 6; + b[1] = br13; + b[2] = 0; + b[3] = height << 16 | width; + b[4] = kgem_add_reloc(kgem, kgem->nbatch + 4, dst, + I915_GEM_DOMAIN_RENDER << 16 | + I915_GEM_DOMAIN_RENDER | + KGEM_RELOC_FENCED, + 0); + b[5] = 0; + b[6] = pitch; + b[7] = kgem_add_reloc(kgem, kgem->nbatch + 7, src, + I915_GEM_DOMAIN_RENDER << 16 | + KGEM_RELOC_FENCED, + 0); + kgem->nbatch += 8; + } return dst; } + +bool kgem_bo_convert_to_gpu(struct kgem *kgem, + struct kgem_bo *bo, + unsigned flags) +{ + DBG(("%s: converting handle=%d from CPU to GPU, flags=%x\n", __FUNCTION__, bo->handle)); + assert(bo->tiling == I915_TILING_NONE); + + if (kgem->has_llc) + return true; + + if (flags & MOVE_ASYNC_HINT && __kgem_bo_is_busy(kgem, bo)) + return false; + + assert(bo->snoop); + + kgem_bo_submit(kgem, bo); + + if (!gem_set_caching(kgem->fd, bo->handle, UNCACHED)) + return false; + + bo->snoop = false; + return true; +} Index: xsrc/external/mit/xf86-video-intel/dist/src/uxa/intel.h diff -u xsrc/external/mit/xf86-video-intel/dist/src/uxa/intel.h:1.1.1.2 xsrc/external/mit/xf86-video-intel/dist/src/uxa/intel.h:1.2 --- xsrc/external/mit/xf86-video-intel/dist/src/uxa/intel.h:1.1.1.2 Wed Nov 5 12:56:22 2014 +++ xsrc/external/mit/xf86-video-intel/dist/src/uxa/intel.h Wed Nov 5 12:58:59 2014 @@ -743,7 +743,7 @@ Bool intel_sync_init(ScreenPtr screen); void intel_sync_close(ScreenPtr screen); #else static inline Bool intel_sync_init(ScreenPtr screen) { return 0; } -void intel_sync_close(ScreenPtr screen); +static inline void intel_sync_close(ScreenPtr screen) { } #endif /*