WebSVN – Kolibri OS – Diff – /contrib/sdk/sources/Intel-2D/sna/kgem.c

 #include
 #endif
-#include "sna_cpuid.h"
 #include "sna_cpuid.h"
 static struct kgem_bo *
 search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
 #define DBG_NO_TILING 0
 #define DBG_NO_CACHE 0
 #define DBG_NO_CACHE_LEVEL 0
 #define DBG_NO_CPU 0
 #define DBG_NO_CREATE2 1
-#define DBG_NO_USERPTR 0
+#define DBG_NO_USERPTR 1
 #define DBG_NO_UNSYNCHRONIZED_USERPTR 0
 #define DBG_NO_LLC 0
 #define DBG_NO_SEMAPHORES 0
 #define DBG_NO_MADV 1
 #define DBG_NO_UPLOAD_CACHE 0
 #define DBG_NO_MAP_UPLOAD 0
 #define DBG_NO_RELAXED_FENCING 0
 #define DBG_NO_SECURE_BATCHES 0
 #define DBG_NO_PINNED_BATCHES 0
 #define DBG_NO_FAST_RELOC 0
-#define DBG_NO_HANDLE_LUT 1
+#define DBG_NO_HANDLE_LUT 0
 #define DBG_NO_WT 0
 #define DBG_DUMP 0
 #define MAX_GTT_VMA_CACHE 512
 #define MAX_CPU_VMA_CACHE INT16_MAX
-#define MAP_PRESERVE_TIME 10
+#define MAP_PRESERVE_TIME 10
-#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) | 1))
-#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) | 3))
-#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 2)
+#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) | 1))
-#define __MAP_TYPE(ptr) ((uintptr_t)(ptr) & 3)
+#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 1)
 	struct kgem_bo base;
 	void *mem;
 	uint32_t used;
 	uint32_t need_io : 1;
 	uint32_t write : 2;
-	uint32_t mmapped : 1;
+	uint32_t mmapped : 2;
+};
+enum {
+	MMAPPED_NONE,
+	MMAPPED_GTT,
+	MMAPPED_CPU
 };
 static struct kgem_bo *__kgem_freed_bo;
 static struct kgem_request *__kgem_freed_request;
 	arg.handle = handle;
 	arg.caching = caching;
 	return drmIoctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0;
+}
+static uint32_t gem_userptr(int fd, void *ptr, int size, int read_only)
+{
+    return 0;
+}
 static bool __kgem_throttle_retire(struct kgem *kgem, unsigned flags)
+{
 	DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__,
 	     bo->handle, bytes(bo)));
 	assert(bo->proxy == NULL);
 	assert(!bo->snoop);
-	assert(kgem_bo_can_map(kgem, bo));
+	assert(num_pages(bo) <= kgem->aperture_mappable / 4);
 retry_gtt:
 	VG_CLEAR(mmap_arg);
+	mmap_arg.handle = bo->handle;
-	mmap_arg.handle = bo->handle;
+	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg)) {
-	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg)) {
+		int err = 0;
-		(void)__kgem_throttle_retire(kgem, 0);
-		if (kgem_expire_cache(kgem))
+		(void)__kgem_throttle_retire(kgem, 0);
-			goto retry_gtt;
+		if (kgem_expire_cache(kgem))
-		if (kgem->need_expire) {
+			goto retry_gtt;
-			kgem_cleanup_cache(kgem);
-			goto retry_gtt;
+		if (kgem_cleanup_cache(kgem))
 			goto retry_gtt;
-		printf("%s: failed to retrieve GTT offset for handle=%d\n",
+		ErrorF("%s: failed to retrieve GTT offset for handle=%d: %d\n",
+	}
 	return ptr;
 }
-static int __gem_write(int fd, uint32_t handle,
+static int gem_write(int fd, uint32_t handle,
 		       int offset, int length,
 		       const void *src)
 	pwrite.size = length;
 	pwrite.data_ptr = (uintptr_t)src;
 	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
+}
-static int gem_write(int fd, uint32_t handle,
+static int gem_write__cachealigned(int fd, uint32_t handle,
 		     int offset, int length,
 		     const void *src)
+{
+}
 static unsigned
 cpu_cache_size__cpuid4(void)
 {
-	/* Deterministic Cache Parmaeters (Function 04h)":
+	/* Deterministic Cache Parameters (Function 04h)":
 	 *    When EAX is initialized to a value of 4, the CPUID instruction
 	 *    returns deterministic cache information in the EAX, EBX, ECX
 	 *    and EDX registers.  This function requires ECX be initialized
 	 *    with an index which indicates which cache to return information
 	/* Although pre-855gm the GMCH is fubar, it works mostly. So
 	 * let the user decide through "NoAccel" whether or not to risk
 	 * hw acceleration.
 	 */
-	if (kgem->gen == 060 && dev->revision < 8) {
+	if (kgem->gen == 060 && dev && dev->revision < 8) {
 		/* pre-production SNB with dysfunctional BLT */
 		return false;
+	}
+}
 static bool kgem_init_pinned_batches(struct kgem *kgem)
+{
-	int count[2] = { 2, 2 };
+	int count[2] = { 4, 4 };
 	int size[2] = { 1, 2 };
 	int n, i;
+			}
 			pin.alignment = 0;
 			if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) {
+				gem_close(kgem->fd, pin.handle);
-				gem_close(kgem->fd, pin.handle);
+				free(bo);
 				goto err;
+			}
 			bo->presumed_offset = pin.offset;
 			debug_alloc__bo(kgem, bo);
     kgem->has_no_reloc = test_has_no_reloc(kgem);
     DBG(("%s: has no-reloc? %d\n", __FUNCTION__,
          kgem->has_no_reloc));
-    kgem->has_handle_lut = test_has_handle_lut(kgem);
-    kgem->has_handle_lut = 0;
+    kgem->has_handle_lut = test_has_handle_lut(kgem);
     DBG(("%s: has handle-lut? %d\n", __FUNCTION__,
          kgem->has_handle_lut));
     kgem->can_blt_cpu = gen >= 030;
     DBG(("%s: can blt to cpu? %d\n", __FUNCTION__,
+         kgem->can_blt_cpu));
+	kgem->can_render_y = gen != 021 && (gen >> 3) != 4;
+	DBG(("%s: can render to Y-tiled surfaces? %d\n", __FUNCTION__,
-         kgem->can_blt_cpu));
+	     kgem->can_render_y));
     kgem->has_secure_batches = test_has_secure_batches(kgem);
     DBG(("%s: can use privileged batchbuffers? %d\n", __FUNCTION__,
+    }
     DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
          kgem->aperture_low, kgem->aperture_low / (1024*1024),
          kgem->aperture_high, kgem->aperture_high / (1024*1024)));
+	kgem->aperture_mappable = 256 * 1024 * 1024;
 	if (dev != NULL)
     kgem->aperture_mappable = agp_aperture_size(dev, gen);
     if (kgem->aperture_mappable == 0 ||
         kgem->aperture_mappable > aperture.aper_size)
         kgem->aperture_mappable = aperture.aper_size;
     if (kgem->max_object_size > totalram / 2)
         kgem->max_object_size = totalram / 2;
     if (kgem->max_gpu_size > totalram / 4)
         kgem->max_gpu_size = totalram / 4;
+	if (kgem->aperture_high > totalram / 2) {
+		kgem->aperture_high = totalram / 2;
+		kgem->aperture_low = kgem->aperture_high / 4;
+		DBG(("%s: reduced aperture watermaks to fit into ram; low=%d [%d], high=%d [%d]\n", __FUNCTION__,
+		     kgem->aperture_low, kgem->aperture_low / (1024*1024),
+		     kgem->aperture_high, kgem->aperture_high / (1024*1024)));
+	}
     kgem->max_cpu_size = kgem->max_object_size;
     half_gpu_max = kgem->max_gpu_size / 2;
          __FUNCTION__,
          kgem->max_gpu_size, kgem->max_cpu_size,
          kgem->max_upload_tile_size, kgem->max_copy_tile_size));
+    /* Convert the aperture thresholds to pages */
-    /* Convert the aperture thresholds to pages */
+	kgem->aperture_mappable /= PAGE_SIZE;
     kgem->aperture_low /= PAGE_SIZE;
+    kgem->aperture_high /= PAGE_SIZE;
-    kgem->aperture_high /= PAGE_SIZE;
+	kgem->aperture_total /= PAGE_SIZE;
     kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
     if ((int)kgem->fence_max < 0)
 	if (flags & CREATE_SCANOUT)
 		return 64;
 	return kgem->min_alignment;
+}
-void kgem_get_tile_size(struct kgem *kgem, int tiling,
+void kgem_get_tile_size(struct kgem *kgem, int tiling, int pitch,
 			int *tile_width, int *tile_height, int *tile_size)
+{
 	if (kgem->gen <= 030) {
 		if (tiling) {
 		*tile_width = 128;
 		*tile_height = 32;
 		*tile_size = 4096;
 		break;
+	}
+	/* Force offset alignment to tile-row */
+	if (tiling && kgem->gen < 033)
+		*tile_width = pitch;
+}
 uint32_t kgem_surface_size(struct kgem *kgem,
 				  bool relaxed_fencing,
 	return exec;
+}
 static void kgem_add_bo(struct kgem *kgem, struct kgem_bo *bo)
+{
+	assert(bo->refcnt);
+	assert(bo->proxy == NULL);
 	bo->exec = kgem_add_handle(kgem, bo);
 	bo->rq = MAKE_REQUEST(kgem->next_request, kgem->ring);
+	list_move_tail(&bo->request, &kgem->next_request->buffers);
 	if (bo->io && !list_is_empty(&bo->list))
-	list_move_tail(&bo->request, &kgem->next_request->buffers);
+		list_move(&bo->list, &kgem->batch_buffers);
-		free (b);
+		free(b);
 		b = next;
+	}
+}
-static void kgem_bo_release_map(struct kgem *kgem, struct kgem_bo *bo)
-{
-	int type = IS_CPU_MAP(bo->map);
-	assert(!IS_USER_MAP(bo->map));
-	DBG(("%s: releasing %s vma for handle=%d, count=%d\n",
-	     __FUNCTION__, type ? "CPU" : "GTT",
-	     bo->handle, kgem->vma[type].count));
-	VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo)));
-	user_free(MAP(bo->map));
-	bo->map = NULL;
-	if (!list_is_empty(&bo->vma)) {
-		list_del(&bo->vma);
-		kgem->vma[type].count--;
-	}
-}
 static void kgem_bo_free(struct kgem *kgem, struct kgem_bo *bo)
+{
 	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
 	assert(bo->refcnt == 0);
 	kgem->debug_memory.bo_bytes -= bytes(bo);
 #endif
 	kgem_bo_binding_free(kgem, bo);
-	if (IS_USER_MAP(bo->map)) {
+	if (IS_USER_MAP(bo->map__cpu)) {
 		assert(bo->rq == NULL);
 		assert(!__kgem_busy(kgem, bo->handle));
-		assert(MAP(bo->map) != bo || bo->io || bo->flush);
+		assert(MAP(bo->map__cpu) != bo || bo->io || bo->flush);
 		if (!(bo->io || bo->flush)) {
 			DBG(("%s: freeing snooped base\n", __FUNCTION__));
-			assert(bo != MAP(bo->map));
+			assert(bo != MAP(bo->map__cpu));
-			free(MAP(bo->map));
+			free(MAP(bo->map__cpu));
+		}
+		bo->map__cpu = NULL;
+	}
+	DBG(("%s: releasing %p:%p vma for handle=%d, count=%d\n",
+	     __FUNCTION__, bo->map__gtt, bo->map__cpu,
+	     bo->handle, list_is_empty(&bo->vma) ? 0 : kgem->vma[bo->map__gtt == NULL].count));
-		bo->map = NULL;
-	}
+	if (!list_is_empty(&bo->vma)) {
+		_list_del(&bo->vma);
+		kgem->vma[bo->map__gtt == NULL].count--;
+	}
-	if (bo->map)
-		kgem_bo_release_map(kgem, bo);
+//   if (bo->map__gtt)
+//       munmap(MAP(bo->map__gtt), bytes(bo));
-	assert(list_is_empty(&bo->vma));
+//   if (bo->map__cpu)
-	assert(bo->map == NULL);
+//       munmap(MAP(bo->map__cpu), bytes(bo));
 	ASSERT_IDLE(kgem, bo->handle);
 	kgem->need_expire = true;
+	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
-	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
+		if (bo->map__gtt) {
-		list_move(&bo->list, &kgem->large_inactive);
+//           munmap(MAP(bo->map__gtt), bytes(bo));
+			bo->map__gtt = NULL;
+	}
 		return;
 		list_move(&bo->list, &kgem->large_inactive);
+	} else {
-	assert(bo->flush == false);
-	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
-	if (bo->map) {
+	assert(bo->flush == false);
-		int type = IS_CPU_MAP(bo->map);
+	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
-		if (bucket(bo) >= NUM_CACHE_BUCKETS ||
+		if (bo->map__gtt) {
-		    (!type && !__kgem_bo_is_mappable(kgem, bo))) {
+			if (!kgem_bo_can_map(kgem, bo)) {
-//			munmap(MAP(bo->map), bytes(bo));
+//				munmap(MAP(bo->map__gtt), bytes(bo));
+				bo->map__gtt = NULL;
+			}
+			if (bo->map__gtt) {
+				list_add(&bo->vma, &kgem->vma[0].inactive[bucket(bo)]);
+				kgem->vma[0].count++;
-			bo->map = NULL;
+			}
 		}
-		if (bo->map) {
+		if (bo->map__cpu && !bo->map__gtt) {
-			list_add(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]);
+			list_add(&bo->vma, &kgem->vma[1].inactive[bucket(bo)]);
-			kgem->vma[type].count++;
+			kgem->vma[1].count++;
+		}
 	if (!bo->io)
 		return bo;
+	assert(!bo->snoop);
+	if (__kgem_freed_bo) {
+		base = __kgem_freed_bo;
 		__kgem_freed_bo = *(struct kgem_bo **)base;
-	assert(!bo->snoop);
+	} else
 	base = malloc(sizeof(*base));
 	if (base) {
 		DBG(("%s: transferring io handle=%d to bo\n",
 	DBG(("%s: removing handle=%d from inactive\n", __FUNCTION__, bo->handle));
 	list_del(&bo->list);
 	assert(bo->rq == NULL);
 	assert(bo->exec == NULL);
-	if (bo->map) {
+	if (!list_is_empty(&bo->vma)) {
-		assert(!list_is_empty(&bo->vma));
+		assert(bo->map__gtt || bo->map__cpu);
 		list_del(&bo->vma);
-		kgem->vma[IS_CPU_MAP(bo->map)].count--;
+		kgem->vma[bo->map__gtt == NULL].count--;
+	}
+}
 inline static void kgem_bo_remove_from_active(struct kgem *kgem,
 					      struct kgem_bo *bo)
+{
 	DBG(("%s: removing handle=%d from active\n", __FUNCTION__, bo->handle));
+	list_del(&bo->list);
-	list_del(&bo->list);
+	assert(bo->rq != NULL);
+	if (RQ(bo->rq) == (void *)kgem) {
-	assert(bo->rq != NULL);
+		assert(bo->exec == NULL);
-	if (bo->rq == (void *)kgem)
+		list_del(&bo->request);
 		list_del(&bo->request);
 	assert(list_is_empty(&bo->vma));
 void kgem_bo_undo(struct kgem *kgem, struct kgem_bo *bo)
+{
 	if (kgem->nexec != 1 || bo->exec == NULL)
 		return;
 	assert(bo);
 	DBG(("%s: only handle in batch, discarding last operations for handle=%d\n",
 	     __FUNCTION__, bo->handle));
 	assert(bo->exec == &kgem->exec[0]);
 	assert(kgem->exec[0].handle == bo->handle);
 	assert(RQ(bo->rq) == kgem->next_request);
+	bo->refcnt++;
+	kgem_reset(kgem);
+	bo->refcnt--;
-	bo->refcnt++;
+	assert(kgem->nreloc == 0);
-	kgem_reset(kgem);
+	assert(kgem->nexec == 0);
-	bo->refcnt--;
+	assert(bo->exec == NULL);
+}
 			__kgem_bo_clear_busy(bo);
 		if (bo->rq == NULL)
 			kgem_bo_move_to_snoop(kgem, bo);
 		return;
+	}
-	if (!IS_USER_MAP(bo->map))
+	if (!IS_USER_MAP(bo->map__cpu))
 		bo->flush = false;
 	if (bo->scanout) {
 		kgem_bo_move_to_scanout(kgem, bo);
 		DBG(("%s: handle=%d, not reusable\n",
 		     __FUNCTION__, bo->handle));
 		goto destroy;
+	}
-	if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU)
-		kgem_bo_release_map(kgem, bo);
 	assert(list_is_empty(&bo->vma));
 	assert(list_is_empty(&bo->list));
 	assert(bo->flush == false);
 	assert(bo->snoop == false);
+	}
 	assert(bo->exec == NULL);
 	assert(list_is_empty(&bo->request));
-	if (!IS_CPU_MAP(bo->map)) {
+	if (bo->map__cpu == NULL || bucket(bo) >= NUM_CACHE_BUCKETS) {
 		if (!kgem_bo_set_purgeable(kgem, bo))
 			goto destroy;
 		__kgem_bo_destroy(kgem, bo);
+}
 static void kgem_buffer_release(struct kgem *kgem, struct kgem_buffer *bo)
+{
 	assert(bo->base.io);
 	while (!list_is_empty(&bo->base.vma)) {
 		struct kgem_bo *cached;
+		cached = list_first_entry(&bo->base.vma, struct kgem_bo, vma);
-		cached = list_first_entry(&bo->base.vma, struct kgem_bo, vma);
+		assert(cached->proxy == &bo->base);
-		assert(cached->proxy == &bo->base);
+		assert(cached != &bo->base);
 		list_del(&cached->vma);
-		assert(*(struct kgem_bo **)cached->map == cached);
+		assert(*(struct kgem_bo **)cached->map__gtt == cached);
-		*(struct kgem_bo **)cached->map = NULL;
+		*(struct kgem_bo **)cached->map__gtt = NULL;
-		cached->map = NULL;
+		cached->map__gtt = NULL;
 		struct kgem_buffer *bo =
 			list_last_entry(&kgem->active_buffers,
 					struct kgem_buffer,
 					base.list);
+		DBG(("%s: handle=%d, busy? %d [%d]\n",
+		     __FUNCTION__, bo->base.handle, bo->base.rq != NULL, bo->base.exec != NULL));
 		assert(bo->base.exec == NULL || RQ(bo->base.rq) == kgem->next_request);
 		if (bo->base.rq)
 			break;
+{
 	struct kgem_bo *bo, *next;
 	bool retired = false;
 	list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
-		assert(bo->rq == (void *)kgem);
+		assert(RQ(bo->rq) == (void *)kgem);
 		assert(bo->exec == NULL);
 			bo->needs_flush = __kgem_busy(kgem, bo->handle);
 		if (bo->needs_flush) {
 			DBG(("%s: moving %d to flushing\n",
 			     __FUNCTION__, bo->handle));
 			list_add(&bo->request, &kgem->flushing);
+			bo->rq = MAKE_REQUEST(kgem, RQ_RING(bo->rq));
-			bo->rq = (void *)kgem;
+			kgem->need_retire = true;
 			continue;
+		}
 		bo->domain = DOMAIN_NONE;
 			kgem_bo_free(kgem, bo);
+		}
+	}
+	assert(rq->bo->rq == NULL);
-	assert(rq->bo->rq == NULL);
+	assert(rq->bo->exec == NULL);
 	assert(list_is_empty(&rq->bo->request));
 	if (--rq->bo->refcnt == 0) {
 bool kgem_retire(struct kgem *kgem)
+{
 	bool retired = false;
-	DBG(("%s\n", __FUNCTION__));
+	DBG(("%s, need_retire?=%d\n", __FUNCTION__, kgem->need_retire));
 bool __kgem_ring_is_idle(struct kgem *kgem, int ring)
+{
+	struct kgem_request *rq;
 	struct kgem_request *rq;
 	assert(ring < ARRAY_SIZE(kgem->requests));
 	assert(!list_is_empty(&kgem->requests[ring]));
 	DBG(("%s: ring=%d idle (handle=%d)\n",
 	     __FUNCTION__, ring, rq->bo->handle));
+	kgem_retire__requests_ring(kgem, ring);
 	kgem_retire__buffers(kgem);
 	kgem_retire__requests_ring(kgem, ring);
 	assert(list_is_empty(&kgem->requests[ring]));
+	return true;
+}
+#ifndef NDEBUG
+static void kgem_commit__check_buffers(struct kgem *kgem)
+{
+	struct kgem_buffer *bo;
+	list_for_each_entry(bo, &kgem->active_buffers, base.list)
+		assert(bo->base.exec == NULL);
+}
+#else
-	return true;
+#define kgem_commit__check_buffers(kgem)
 #endif
 static void kgem_commit(struct kgem *kgem)
 		bo->exec = NULL;
 		bo->target_handle = -1;
 		if (!bo->refcnt && !bo->reusable) {
+			assert(!bo->snoop);
-			assert(!bo->snoop);
+			assert(!bo->proxy);
 			kgem_bo_free(kgem, bo);
 			continue;
+		}
 		bo->binding.offset = 0;
 		bo->domain = DOMAIN_GPU;
 		bo->gpu_dirty = false;
 		if (bo->proxy) {
-			/* proxies are not used for domain tracking */
+		if (bo->proxy) {
-			bo->exec = NULL;
+			/* proxies are not used for domain tracking */
 			__kgem_bo_clear_busy(bo);
+		}
 		kgem_retire(kgem);
+		assert(list_is_empty(&rq->buffers));
-		assert(list_is_empty(&rq->buffers));
 		assert(rq->bo->map__gtt == NULL);
-		assert(rq->bo->map == NULL);
+		assert(rq->bo->map__cpu == NULL);
 		gem_close(kgem->fd, rq->bo->handle);
 		kgem_cleanup_cache(kgem);
 	} else {
 		list_add_tail(&rq->list, &kgem->requests[rq->ring]);
 		kgem->need_throttle = kgem->need_retire = 1;
+	}
 	kgem->next_request = NULL;
-	kgem->next_request = NULL;
+	kgem_commit__check_buffers(kgem);
+}
 static void kgem_finish_buffers(struct kgem *kgem)
+{
 	struct kgem_buffer *bo, *next;
 	list_for_each_entry_safe(bo, next, &kgem->batch_buffers, base.list) {
-		DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s\n",
+		DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s, refcnt=%d\n",
 		     __FUNCTION__, bo->base.handle, bo->used, bo->base.exec!=NULL,
+		     bo->write, bo->mmapped == MMAPPED_CPU ? "cpu" : bo->mmapped == MMAPPED_GTT ? "gtt" : "no",
-		     bo->write, bo->mmapped ? IS_CPU_MAP(bo->base.map) ? "cpu" : "gtt" : "no"));
+		     bo->base.refcnt));
 		assert(next->base.list.prev == &bo->base.list);
 		assert(bo->base.io);
 		assert(bo->base.refcnt >= 1);
-		if (!bo->base.exec) {
+		if (bo->base.refcnt > 1 && !bo->base.exec) {
-			DBG(("%s: skipping unattached handle=%d, used=%d\n",
+			DBG(("%s: skipping unattached handle=%d, used=%d, refcnt=%d\n",
-			     __FUNCTION__, bo->base.handle, bo->used));
+			     __FUNCTION__, bo->base.handle, bo->used, bo->base.refcnt));
 			continue;
+		}
 		if (!bo->write) {
 			assert(bo->base.exec || bo->base.refcnt > 1);
 			goto decouple;
+		}
 		if (bo->mmapped) {
-			int used;
+			uint32_t used;
 			assert(!bo->need_io);
 			used = ALIGN(bo->used, PAGE_SIZE);
 			if (!DBG_NO_UPLOAD_ACTIVE &&
+			    used + PAGE_SIZE <= bytes(&bo->base) &&
-			    used + PAGE_SIZE <= bytes(&bo->base) &&
+			    (kgem->has_llc || bo->mmapped == MMAPPED_GTT || bo->base.snoop)) {
-			    (kgem->has_llc || !IS_CPU_MAP(bo->base.map) || bo->base.snoop)) {
+				DBG(("%s: retaining upload buffer (%d/%d): used=%d, refcnt=%d\n",
-				DBG(("%s: retaining upload buffer (%d/%d)\n",
+				     __FUNCTION__, bo->used, bytes(&bo->base), used, bo->base.refcnt));
-				     __FUNCTION__, bo->used, bytes(&bo->base)));
+				bo->used = used;
-				bo->used = used;
+				list_move(&bo->base.list,
-				list_move(&bo->base.list,
+					  &kgem->active_buffers);
-					  &kgem->active_buffers);
+				kgem->need_retire = true;
 				continue;
+			}
 			DBG(("%s: discarding mmapped buffer, used=%d, map type=%d\n",
-			     __FUNCTION__, bo->used, (int)__MAP_TYPE(bo->base.map)));
+			     __FUNCTION__, bo->used, bo->mmapped));
 			goto decouple;
 				     __FUNCTION__,
 				     bo->used, bytes(&bo->base), bytes(shrink),
 				     bo->base.handle, shrink->handle));
 				assert(bo->used <= bytes(shrink));
-				if (gem_write(kgem->fd, shrink->handle,
+				if (gem_write__cachealigned(kgem->fd, shrink->handle,
 , bo->used, bo->mem) == 0) {
 					shrink->target_handle =
 						kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
 					for (n = 0; n < kgem->nreloc; n++) {
 		DBG(("%s: handle=%d, uploading %d/%d\n",
 		     __FUNCTION__, bo->base.handle, bo->used, bytes(&bo->base)));
 		ASSERT_IDLE(kgem, bo->base.handle);
 		assert(bo->used <= bytes(&bo->base));
-		gem_write(kgem->fd, bo->base.handle,
+		gem_write__cachealigned(kgem->fd, bo->base.handle,
 , bo->used, bo->mem);
 		bo->need_io = 0;
+{
 	int ret;
+	ASSERT_IDLE(kgem, handle);
 	ASSERT_IDLE(kgem, handle);
+retry:
 	/* If there is no surface data, just upload the batch */
-	if (kgem->surface == kgem->batch_size)
+	if (kgem->surface == kgem->batch_size) {
-		return gem_write(kgem->fd, handle,
+		if (gem_write__cachealigned(kgem->fd, handle,
+, sizeof(uint32_t)*kgem->nbatch,
+					    kgem->batch) == 0)
+			return 0;
-, sizeof(uint32_t)*kgem->nbatch,
+		goto expire;
 				 kgem->batch);
 	/* Are the batch pages conjoint with the surface pages? */
 	if (kgem->surface < kgem->nbatch + PAGE_SIZE/sizeof(uint32_t)) {
 		assert(size == PAGE_ALIGN(kgem->batch_size*sizeof(uint32_t)));
+		if (gem_write__cachealigned(kgem->fd, handle,
+, kgem->batch_size*sizeof(uint32_t),
+					    kgem->batch) == 0)
-		return gem_write(kgem->fd, handle,
+			return 0;
 , kgem->batch_size*sizeof(uint32_t),
-				 kgem->batch);
+		goto expire;
+	}
-	/* Disjoint surface/batch, upload separately */
-	ret = gem_write(kgem->fd, handle,
+	/* Disjoint surface/batch, upload separately */
-, sizeof(uint32_t)*kgem->nbatch,
+	if (gem_write__cachealigned(kgem->fd, handle,
-			kgem->batch);
+, sizeof(uint32_t)*kgem->nbatch,
-	if (ret)
+				    kgem->batch))
-		return ret;
+		goto expire;
 	ret = PAGE_ALIGN(sizeof(uint32_t) * kgem->batch_size);
+	ret -= sizeof(uint32_t) * kgem->surface;
+	assert(size-ret >= kgem->nbatch*sizeof(uint32_t));
+	if (gem_write(kgem->fd, handle,
+			size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t),
+		      kgem->batch + kgem->surface))
+		goto expire;
+	return 0;
+expire:
+	ret = errno;
+	assert(ret != EINVAL);
+	(void)__kgem_throttle_retire(kgem, 0);
+	if (kgem_expire_cache(kgem))
+		goto retry;
+	if (kgem_cleanup_cache(kgem))
-	ret -= sizeof(uint32_t) * kgem->surface;
+		goto retry;
 	assert(size-ret >= kgem->nbatch*sizeof(uint32_t));
-	return __gem_write(kgem->fd, handle,
+	ErrorF("%s: failed to write batch (handle=%d): %d\n",
-			size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t),
+	       __FUNCTION__, handle, ret);
 			if (bo->needs_flush && __kgem_busy(kgem, bo->handle)) {
 				assert(bo->domain == DOMAIN_GPU || bo->domain == DOMAIN_NONE);
 				list_add(&bo->request, &kgem->flushing);
+				bo->rq = (void *)kgem;
-				bo->rq = (void *)kgem;
+				kgem->need_retire = true;
 			} else
 				__kgem_bo_clear_busy(bo);
 	kgem->nexec = 0;
 	kgem->nreloc = 0;
 	kgem->nreloc__self = 0;
 	kgem->aperture = 0;
 	kgem->aperture_fenced = 0;
+	kgem->aperture_max_fence = 0;
 	kgem->nbatch = 0;
 	kgem->surface = kgem->batch_size;
 	kgem->mode = KGEM_NONE;
 	kgem->flush = 0;
 	kgem->batch_flags = kgem->batch_flags_base;
 	assert(kgem->nbatch <= kgem->surface);
 	batch_end = kgem_end_batch(kgem);
 	kgem_sna_flush(kgem);
-	DBG(("batch[%d/%d, flags=%x]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n",
+	DBG(("batch[%d/%d, flags=%x]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d [fenced=%d]\n",
 	     kgem->mode, kgem->ring, kgem->batch_flags,
 	     batch_end, kgem->nbatch, kgem->surface, kgem->batch_size,
-	     kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture));
+	     kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced));
 	assert(kgem->nbatch <= kgem->batch_size);
    		    if (DEBUG_DUMP)
+            {
                 int fd = open("/tmp1/1/batchbuffer.bin", O_CREAT|O_WRONLY|O_BINARY);
 				if (fd != -1) {
-                    write(fd, kgem->batch, size);
+					ret = write(fd, kgem->batch, batch_end*sizeof(uint32_t));
-					close(fd);
+					fd = close(fd);
+				}
                 else printf("SNA: failed to write batchbuffer\n");
                 asm volatile("int3");
 				kgem_throttle(kgem);
 				kgem->wedged = true;
 #if 0
 				ret = errno;
-				ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d: errno=%d\n",
+				ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d, fenced=%d, high=%d,%d: errno=%d\n",
 				       kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
-				       kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, errno);
+				       kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced, kgem->aperture_high, kgem->aperture_total, errno);
 				for (i = 0; i < kgem->nexec; i++) {
 		printf("Detected a hung GPU, disabling acceleration.\n");
 		printf("When reporting this, please include i915_error_state from debugfs and the full dmesg.\n");
+	}
+}
-void kgem_purge_cache(struct kgem *kgem)
+static void kgem_purge_cache(struct kgem *kgem)
+{
 	struct kgem_bo *bo, *next;
 	int i;
 			if (bo->delta > expire) {
 				idle = false;
 				break;
+			}
-			if (bo->map && bo->delta + MAP_PRESERVE_TIME > expire) {
+			if (bo->map__cpu && bo->delta + MAP_PRESERVE_TIME > expire) {
 				idle = false;
 				list_move_tail(&bo->list, &preserve);
 			} else {
 				count++;
 	return !idle;
 	(void)count;
 	(void)size;
+}
-void kgem_cleanup_cache(struct kgem *kgem)
+bool kgem_cleanup_cache(struct kgem *kgem)
+{
 	unsigned int i;
 	int n;
+	}
 	kgem_retire(kgem);
+	kgem_cleanup(kgem);
+	if (!kgem->need_expire)
-	kgem_cleanup(kgem);
+		return false;
 	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
 		while (!list_is_empty(&kgem->inactive[i]))
 			kgem_bo_free(kgem,
 		free(bo);
+	}
 	kgem->need_purge = false;
+	kgem->need_expire = false;
-	kgem->need_expire = false;
+	return true;
+}
 static struct kgem_bo *
 			if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo))
 				goto discard;
+			list_del(&bo->list);
-			list_del(&bo->list);
+			if (RQ(bo->rq) == (void *)kgem) {
+				assert(bo->exec == NULL);
-			if (bo->rq == (void *)kgem)
+				list_del(&bo->request);
 				list_del(&bo->request);
 		int for_cpu = !!(flags & CREATE_CPU_MAP);
 		DBG(("%s: searching for inactive %s map\n",
 		     __FUNCTION__, for_cpu ? "cpu" : "gtt"));
 		cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
 		list_for_each_entry(bo, cache, vma) {
-			assert(IS_CPU_MAP(bo->map) == for_cpu);
+			assert(for_cpu ? bo->map__cpu : bo->map__gtt);
 			assert(bucket(bo) == cache_bucket(num_pages));
 			assert(bo->proxy == NULL);
 			assert(bo->rq == NULL);
 			assert(bo->exec == NULL);
 			assert(!bo->scanout);
 			    !gem_set_tiling(kgem->fd, bo->handle,
 					    I915_TILING_NONE, 0))
 				continue;
+			kgem_bo_remove_from_inactive(kgem, bo);
+			assert(list_is_empty(&bo->vma));
-			kgem_bo_remove_from_inactive(kgem, bo);
+			assert(list_is_empty(&bo->list));
 			bo->tiling = I915_TILING_NONE;
 			bo->pitch = 0;
 			bo->tiling = I915_TILING_NONE;
 			bo->pitch = 0;
 		}
-		if (bo->map) {
+		if (bo->map__gtt || bo->map__cpu) {
 			if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) {
 				int for_cpu = !!(flags & CREATE_CPU_MAP);
-				if (IS_CPU_MAP(bo->map) != for_cpu) {
+				if (for_cpu ? bo->map__cpu : bo->map__gtt){
 					if (first != NULL)
 						break;
 				first = bo;
 				continue;
+			}
+		} else {
+			if (flags & CREATE_GTT_MAP && !kgem_bo_can_map(kgem, bo))
+				continue;
 		} else {
 			if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) {
 				if (first != NULL)
 					break;
 		bo->delta = 0;
 		DBG(("  %s: found handle=%d (num_pages=%d) in linear %s cache\n",
 		     __FUNCTION__, bo->handle, num_pages(bo),
 		     use_active ? "active" : "inactive"));
 		assert(list_is_empty(&bo->list));
+		assert(list_is_empty(&bo->vma));
 		assert(use_active || bo->domain != DOMAIN_GPU);
 		assert(!bo->needs_flush || use_active);
 		assert_tiling(kgem, bo);
 		ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
 		return bo;
 		first->delta = 0;
 		DBG(("  %s: found handle=%d (near-miss) (num_pages=%d) in linear %s cache\n",
 		     __FUNCTION__, first->handle, num_pages(first),
 		     use_active ? "active" : "inactive"));
 		assert(list_is_empty(&first->list));
+		assert(list_is_empty(&first->vma));
 		assert(use_active || first->domain != DOMAIN_GPU);
 		assert(!first->needs_flush || use_active);
 		ASSERT_MAYBE_IDLE(kgem, first->handle, !use_active);
 		return first;
+	}
 	assert(bo->tiling);
 	assert_tiling(kgem, bo);
 	assert(kgem->gen < 040);
 	if (kgem->gen < 030)
-		size = 512 * 1024;
+		size = 512 * 1024 / PAGE_SIZE;
 	else
-		size = 1024 * 1024;
+		size = 1024 * 1024 / PAGE_SIZE;
-	while (size < bytes(bo))
+	while (size < num_pages(bo))
-		size *= 2;
+		size <<= 1;
 	return size;
 	bool exact = flags & (CREATE_EXACT | CREATE_SCANOUT);
 	if (tiling < 0)
-		exact = true, tiling = -tiling;
 		exact = true, tiling = -tiling;
 	DBG(("%s(%dx%d, bpp=%d, tiling=%d, exact=%d, inactive=%d, cpu-mapping=%d, gtt-mapping=%d, scanout?=%d, prime?=%d, temp?=%d)\n", __FUNCTION__,
 	     width, height, bpp, tiling, exact,
 	     !!(flags & CREATE_INACTIVE),
 				 width, height, bpp, tiling, &pitch);
 	assert(size && size <= kgem->max_object_size);
 	size /= PAGE_SIZE;
 	bucket = cache_bucket(size);
-	if (flags & CREATE_SCANOUT) {
-		struct kgem_bo *last = NULL;
-		list_for_each_entry_reverse(bo, &kgem->scanout, list) {
-			assert(bo->scanout);
-			assert(bo->delta);
-			assert(!bo->flush);
-			assert_tiling(kgem, bo);
-			if (size > num_pages(bo) || num_pages(bo) > 2*size)
-				continue;
-			if (bo->tiling != tiling ||
-			    (tiling != I915_TILING_NONE && bo->pitch != pitch)) {
-				if (!gem_set_tiling(kgem->fd, bo->handle,
-						    tiling, pitch))
-					continue;
-				bo->tiling = tiling;
-				bo->pitch = pitch;
-			}
-			if (flags & CREATE_INACTIVE && bo->rq) {
-				last = bo;
-				continue;
-			}
-			list_del(&bo->list);
-			bo->unique_id = kgem_get_unique_id(kgem);
-			DBG(("  1:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n",
-			     bo->pitch, bo->tiling, bo->handle, bo->unique_id));
-			assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
-			assert_tiling(kgem, bo);
-			bo->refcnt = 1;
-			return bo;
-		}
-		if (last) {
-			list_del(&last->list);
-			last->unique_id = kgem_get_unique_id(kgem);
-			DBG(("  1:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n",
-			     last->pitch, last->tiling, last->handle, last->unique_id));
-			assert(last->pitch*kgem_aligned_height(kgem, height, last->tiling) <= kgem_bo_size(last));
-			assert_tiling(kgem, last);
-			last->refcnt = 1;
-			return last;
-		}
-		bo = NULL; //__kgem_bo_create_as_display(kgem, size, tiling, pitch);
-		if (bo)
-			return bo;
-	}
 	if (bucket >= NUM_CACHE_BUCKETS) {
 		DBG(("%s: large bo num pages=%d, bucket=%d\n",
 		     __FUNCTION__, size, bucket));
 			DBG(("  1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
 			     bo->pitch, bo->tiling, bo->handle, bo->unique_id));
 			assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
 			assert_tiling(kgem, bo);
 			bo->refcnt = 1;
-			bo->flush = true;
 			return bo;
+		}
 large_inactive:
 		do {
 			list_for_each_entry(bo, cache, vma) {
 				assert(bucket(bo) == bucket);
 				assert(bo->refcnt == 0);
 				assert(!bo->scanout);
-				assert(bo->map);
-				assert(IS_CPU_MAP(bo->map) == for_cpu);
+				assert(for_cpu ? bo->map__cpu : bo->map__gtt);
 				assert(bo->rq == NULL);
+				assert(bo->exec == NULL);
 				assert(list_is_empty(&bo->request));
 				assert(bo->flush == false);
 				assert_tiling(kgem, bo);
 				bo->delta = 0;
 				bo->unique_id = kgem_get_unique_id(kgem);
 				bo->domain = DOMAIN_NONE;
+				kgem_bo_remove_from_inactive(kgem, bo);
+				assert(list_is_empty(&bo->list));
-				kgem_bo_remove_from_inactive(kgem, bo);
+				assert(list_is_empty(&bo->vma));
 				DBG(("  from inactive vma: pitch=%d, tiling=%d: handle=%d, id=%d\n",
 				     bo->pitch, bo->tiling, bo->handle, bo->unique_id));
 		if (bo->tiling != tiling ||
 		    (tiling != I915_TILING_NONE && bo->pitch != pitch)) {
 			if (!gem_set_tiling(kgem->fd, bo->handle,
 					    tiling, pitch))
 				continue;
-			if (bo->map)
-				kgem_bo_release_map(kgem, bo);
+		}
 		if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
 			kgem_bo_free(kgem, bo);
 			break;
+		}
+		kgem_bo_remove_from_inactive(kgem, bo);
 		assert(list_is_empty(&bo->list));
-		kgem_bo_remove_from_inactive(kgem, bo);
+		assert(list_is_empty(&bo->vma));
 	if (!bo) {
 		gem_close(kgem->fd, handle);
 		return NULL;
+	}
-	if (bucket >= NUM_CACHE_BUCKETS) {
-		DBG(("%s: marking large bo for automatic flushing\n",
-		     __FUNCTION__));
-		bo->flush = true;
-	}
 	bo->unique_id = kgem_get_unique_id(kgem);
 	if (tiling == I915_TILING_NONE ||
 	    gem_set_tiling(kgem->fd, handle, tiling, pitch)) {
 		bo->tiling = tiling;
+{
 	DBG(("%s: handle=%d, proxy? %d\n",
 	     __FUNCTION__, bo->handle, bo->proxy != NULL));
+	if (bo->proxy) {
+		assert(!bo->reusable);
+		kgem_bo_binding_free(kgem, bo);
-	if (bo->proxy) {
+		assert(list_is_empty(&bo->list));
 		_list_del(&bo->vma);
+		_list_del(&bo->request);
-		_list_del(&bo->request);
-		if (bo->io && bo->exec == NULL)
+		if (bo->io && bo->domain == DOMAIN_CPU)
+			_kgem_bo_delete_buffer(kgem, bo);
 			_kgem_bo_delete_buffer(kgem, bo);
-		kgem_bo_unref(kgem, bo->proxy);
-		kgem_bo_binding_free(kgem, bo);
-		free(bo);
-		return;
+		kgem_bo_unref(kgem, bo->proxy);
+		*(struct kgem_bo **)bo = __kgem_freed_bo;
 		__kgem_freed_bo = bo;
 	} else
 	__kgem_bo_destroy(kgem, bo);
+}
 inline static bool needs_semaphore(struct kgem *kgem, struct kgem_bo *bo)
+{
 	return kgem->nreloc && bo->rq && RQ_RING(bo->rq) != kgem->ring;
+}
+static bool aperture_check(struct kgem *kgem, unsigned num_pages)
+{
+	if (kgem->aperture) {
+		struct drm_i915_gem_get_aperture aperture;
+		VG_CLEAR(aperture);
+		aperture.aper_available_size = kgem->aperture_high;
+		aperture.aper_available_size *= PAGE_SIZE;
+		(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
+		DBG(("%s: aperture required %ld bytes, available %ld bytes\n",
+		     __FUNCTION__,
+		     (long)num_pages * PAGE_SIZE,
+		     (long)aperture.aper_available_size));
+		/* Leave some space in case of alignment issues */
+		aperture.aper_available_size -= 1024 * 1024;
+		aperture.aper_available_size -= kgem->aperture_mappable * PAGE_SIZE / 2;
+		if (kgem->gen < 033)
+			aperture.aper_available_size -= kgem->aperture_max_fence * PAGE_SIZE;
+		if (!kgem->has_llc)
+			aperture.aper_available_size -= 2 * kgem->nexec * PAGE_SIZE;
+		DBG(("%s: num_pages=%d, estimated max usable=%ld\n",
+		     __FUNCTION__, num_pages, (long)(aperture.aper_available_size/PAGE_SIZE)));
+		if (num_pages <= aperture.aper_available_size / PAGE_SIZE)
+			return true;
+	}
+	return false;
+}
+static inline bool kgem_flush(struct kgem *kgem, bool flush)
+{
+	if (unlikely(kgem->wedged))
+		return false;
+	if (kgem->nreloc == 0)
+		return true;
+	if (container_of(kgem, struct sna, kgem)->flags & SNA_POWERSAVE)
+		return true;
+	if (kgem->flush == flush && kgem->aperture < kgem->aperture_low)
+		return true;
+	DBG(("%s: opportunistic flushing? flush=%d,%d, aperture=%d/%d, idle?=%d\n",
+	     __FUNCTION__, kgem->flush, flush, kgem->aperture, kgem->aperture_low, kgem_ring_is_idle(kgem, kgem->ring)));
+	return !kgem_ring_is_idle(kgem, kgem->ring);
+}
 bool kgem_check_bo(struct kgem *kgem, ...)
+{
 	va_list ap;
 	struct kgem_bo *bo;
 	int num_exec = 0;
 	int num_pages = 0;
+	bool flush = false;
-	bool flush = false;
+	bool busy = true;
 	va_start(ap, kgem);
 	while ((bo = va_arg(ap, struct kgem_bo *))) {
 		while (bo->proxy)
 			bo = bo->proxy;
 		if (bo->exec)
+			continue;
 			continue;
+		if (needs_semaphore(kgem, bo)) {
 			DBG(("%s: flushing for required semaphore\n", __FUNCTION__));
-		if (needs_semaphore(kgem, bo))
+			return false;
 			return false;
 		num_pages += num_pages(bo);
-		num_pages += num_pages(bo);
+		num_exec++;
 		num_exec++;
 		flush |= bo->flush;
-		flush |= bo->flush;
+		busy &= bo->rq != NULL;
+	}
-	va_end(ap);
-	DBG(("%s: num_pages=+%d, num_exec=+%d\n",
-	     __FUNCTION__, num_pages, num_exec));
+	va_end(ap);
-	if (!num_pages)
-		return true;
+	DBG(("%s: num_pages=+%d, num_exec=+%d\n",
 	     __FUNCTION__, num_pages, num_exec));
 	if (kgem_flush(kgem, flush))
-		return false;
+	if (!num_pages)
 		return true;
 	if (kgem->aperture > kgem->aperture_low &&
+	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) {
-	    kgem_ring_is_idle(kgem, kgem->ring)) {
+		DBG(("%s: out of exec slots (%d + %d / %d)\n", __FUNCTION__,
-		DBG(("%s: current aperture usage (%d) is greater than low water mark (%d)\n",
+		     kgem->nexec, num_exec, KGEM_EXEC_SIZE(kgem)));
-		     __FUNCTION__, kgem->aperture, kgem->aperture_low));
-		return false;
-	}
+		return false;
-	if (num_pages + kgem->aperture > kgem->aperture_high) {
-		DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n",
 		     __FUNCTION__, num_pages + kgem->aperture, kgem->aperture_high));
+	if (num_pages + kgem->aperture > kgem->aperture_high) {
-		return false;
+		DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n",
+		     __FUNCTION__, num_pages + kgem->aperture, kgem->aperture_high));
+		if (!aperture_check(kgem, num_pages + kgem->aperture))
+		return false;
+	}
+	if (busy)
+		return true;
+	return kgem_flush(kgem, flush);
+}
+#if 0
+bool kgem_check_bo_fenced(struct kgem *kgem, struct kgem_bo *bo)
+{
+	assert(bo->refcnt);
+	while (bo->proxy)
+		bo = bo->proxy;
+	assert(bo->refcnt);
+	if (bo->exec) {
+		if (kgem->gen < 040 &&
+		    bo->tiling != I915_TILING_NONE &&
+		    (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
+			uint32_t size;
+			assert(bo->tiling == I915_TILING_X);
+			if (kgem->nfence >= kgem->fence_max)
+				return false;
+			if (kgem->aperture_fenced) {
+				size = 3*kgem->aperture_fenced;
+				if (kgem->aperture_total == kgem->aperture_mappable)
+					size += kgem->aperture;
+				if (size > kgem->aperture_mappable &&
+				    kgem_ring_is_idle(kgem, kgem->ring)) {
+					DBG(("%s: opportunistic fence flush\n", __FUNCTION__));
+					return false;
+				}
+			}
+			size = kgem_bo_fenced_size(kgem, bo);
+			if (size > kgem->aperture_max_fence)
+				kgem->aperture_max_fence = size;
+			size += kgem->aperture_fenced;
+			if (kgem->gen < 033)
+				size += kgem->aperture_max_fence;
+			if (kgem->aperture_total == kgem->aperture_mappable)
+				size += kgem->aperture;
+			if (size > kgem->aperture_mappable) {
+				DBG(("%s: estimated fence space required [%d] exceed aperture [%d]\n",
+				     __FUNCTION__, size, kgem->aperture_mappable));
+				return false;
+			}
+		}
+		return true;
+	}
+	if (kgem->nexec >= KGEM_EXEC_SIZE(kgem) - 1)
+		return false;
+	if (needs_semaphore(kgem, bo)) {
+		DBG(("%s: flushing for required semaphore\n", __FUNCTION__));
+		return false;
+	}
+	assert_tiling(kgem, bo);
+	if (kgem->gen < 040 && bo->tiling != I915_TILING_NONE) {
+		uint32_t size;
+		assert(bo->tiling == I915_TILING_X);
+		if (kgem->nfence >= kgem->fence_max)
+			return false;
+		if (kgem->aperture_fenced) {
+			size = 3*kgem->aperture_fenced;
+			if (kgem->aperture_total == kgem->aperture_mappable)
+				size += kgem->aperture;
+			if (size > kgem->aperture_mappable &&
+			    kgem_ring_is_idle(kgem, kgem->ring)) {
+				DBG(("%s: opportunistic fence flush\n", __FUNCTION__));
+				return false;
+			}
+		}
+		size = kgem_bo_fenced_size(kgem, bo);
+		if (size > kgem->aperture_max_fence)
+			kgem->aperture_max_fence = size;
+		size += kgem->aperture_fenced;
+		if (kgem->gen < 033)
 	int index;
 	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
+	     __FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
 	     __FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
-	assert((read_write_domain & 0x7fff) == 0 || bo != NULL);
-    if( bo != NULL && bo->handle == -2)
-    {
-   		if (bo->exec == NULL)
-			kgem_add_bo(kgem, bo);
-		if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
-			__kgem_bo_mark_dirty(bo);
-		}
-        return 0;
+	assert(kgem->gen < 0100);
-    };
+	assert((read_write_domain & 0x7fff) == 0 || bo != NULL);
 	index = kgem->nreloc++;
+	assert(index < ARRAY_SIZE(kgem->reloc));
-	assert(index < ARRAY_SIZE(kgem->reloc));
+	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
-	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
+	if (bo) {
-	if (bo) {
+		assert(kgem->mode != KGEM_NONE);
 		assert(bo->refcnt);
 		while (bo->proxy) {
 				list_move_tail(&bo->request,
 					       &kgem->next_request->buffers);
 				bo->rq = MAKE_REQUEST(kgem->next_request,
 						      kgem->ring);
 				bo->exec = &_kgem_dummy_exec;
+				bo->domain = DOMAIN_GPU;
+		}
 			if (read_write_domain & 0x7fff && !bo->gpu_dirty)
 		assert(RQ_RING(bo->rq) == kgem->ring);
 		if (kgem->gen < 040 && read_write_domain & KGEM_RELOC_FENCED) {
 			if (bo->tiling &&
+			    (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
-			    (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
+				assert(bo->tiling == I915_TILING_X);
 				assert(kgem->nfence < kgem->fence_max);
 				kgem->aperture_fenced +=
 					kgem_bo_fenced_size(kgem, bo);
 				kgem->nfence++;
 	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
 	return delta;
+}
+uint64_t kgem_add_reloc64(struct kgem *kgem,
+			  uint32_t pos,
+			  struct kgem_bo *bo,
+			  uint32_t read_write_domain,
+			  uint64_t delta)
+{
+	int index;
+	DBG(("%s: handle=%d, pos=%d, delta=%ld, domains=%08x\n",
+	     __FUNCTION__, bo ? bo->handle : 0, pos, (long)delta, read_write_domain));
+	assert(kgem->gen >= 0100);
+	assert((read_write_domain & 0x7fff) == 0 || bo != NULL);
+	index = kgem->nreloc++;
+	assert(index < ARRAY_SIZE(kgem->reloc));
+	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
+	if (bo) {
+		assert(kgem->mode != KGEM_NONE);
+		assert(bo->refcnt);
+		while (bo->proxy) {
+			DBG(("%s: adding proxy [delta=%ld] for handle=%d\n",
+			     __FUNCTION__, (long)bo->delta, bo->handle));
+			delta += bo->delta;
+			assert(bo->handle == bo->proxy->handle);
+			/* need to release the cache upon batch submit */
+			if (bo->exec == NULL) {
+				list_move_tail(&bo->request,
+					       &kgem->next_request->buffers);
+				bo->rq = MAKE_REQUEST(kgem->next_request,
+						      kgem->ring);
+				bo->exec = &_kgem_dummy_exec;
+				bo->domain = DOMAIN_GPU;
+			}
+			if (read_write_domain & 0x7fff && !bo->gpu_dirty)
+				__kgem_bo_mark_dirty(bo);
+			bo = bo->proxy;
+			assert(bo->refcnt);
+		}
+		assert(bo->refcnt);
+		if (bo->exec == NULL)
+			kgem_add_bo(kgem, bo);
+		assert(bo->rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
+		assert(RQ_RING(bo->rq) == kgem->ring);
+		kgem->reloc[index].delta = delta;
+		kgem->reloc[index].target_handle = bo->target_handle;
+		kgem->reloc[index].presumed_offset = bo->presumed_offset;
+		if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
+			assert(!bo->snoop || kgem->can_blt_cpu);
+			__kgem_bo_mark_dirty(bo);
+		}
+		delta += bo->presumed_offset;
+	} else {
+		kgem->reloc[index].delta = delta;
+		kgem->reloc[index].target_handle = ~0U;
+		kgem->reloc[index].presumed_offset = 0;
+		if (kgem->nreloc__self < 256)
+			kgem->reloc__self[kgem->nreloc__self++] = index;
+	}
+	kgem->reloc[index].read_domains = read_write_domain >> 16;
+	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
+	return delta;
+}
 static void kgem_trim_vma_cache(struct kgem *kgem, int type, int bucket)
+{
 	 * vma to within a conservative value.
 	 */
 	i = 0;
 	while (kgem->vma[type].count > 0) {
 		struct kgem_bo *bo = NULL;
+		void **ptr;
 		for (j = 0;
 		     bo == NULL && j < ARRAY_SIZE(kgem->vma[type].inactive);
 		     j++) {
+	}
 		if (bo == NULL)
 			break;
-		DBG(("%s: discarding inactive %s vma cache for %d\n",
-		     __FUNCTION__,
+		DBG(("%s: discarding inactive %s vma cache for %d\n",
+		     __FUNCTION__, type ? "CPU" : "GTT", bo->handle));
-		     IS_CPU_MAP(bo->map) ? "CPU" : "GTT", bo->handle));
-		assert(IS_CPU_MAP(bo->map) == type);
-		assert(bo->map);
+		ptr = type ? &bo->map__cpu : &bo->map__gtt;
 			assert(bo->rq == NULL);
-		VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo)));
+		VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(*ptr), bytes(bo)));
-//		munmap(MAP(bo->map), bytes(bo));
+//		munmap(MAP(*ptr), bytes(bo));
-		bo->map = NULL;
+		*ptr = NULL;
 		list_del(&bo->vma);
 		kgem->vma[type].count--;
 void *kgem_bo_map__async(struct kgem *kgem, struct kgem_bo *bo)
+{
 	void *ptr;
-	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
+	DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__,
-	     bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
+	     bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
 	assert(bo->proxy == NULL);
-	assert(list_is_empty(&bo->list));
+	assert(bo->proxy == NULL);
-	assert(!IS_USER_MAP(bo->map));
+	assert(list_is_empty(&bo->list));
 	assert_tiling(kgem, bo);
 	if (bo->tiling == I915_TILING_NONE && !bo->scanout && kgem->has_llc) {
-		DBG(("%s: converting request for GTT map into CPU map\n",
-		     __FUNCTION__));
-		return kgem_bo_map__cpu(kgem, bo);
-	}
+		DBG(("%s: converting request for GTT map into CPU map\n",
 		     __FUNCTION__));
-	if (IS_CPU_MAP(bo->map))
+		return kgem_bo_map__cpu(kgem, bo);
 		kgem_bo_release_map(kgem, bo);
-	ptr = bo->map;
+	ptr = MAP(bo->map__gtt);
 		/* Cache this mapping to avoid the overhead of an
 		 * excruciatingly slow GTT pagefault. This is more an
 		 * issue with compositing managers which need to frequently
 		 * flush CPU damage to their GPU bo.
 		 */
-		bo->map = ptr;
+		bo->map__gtt = ptr;
 		DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
+	}
 	return ptr;
+}
 void *kgem_bo_map(struct kgem *kgem, struct kgem_bo *bo)
 {
 	void *ptr;
-	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
+	DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__,
-	     bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
 	     bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
 	assert(bo->proxy == NULL);
-	assert(list_is_empty(&bo->list));
+	assert(bo->proxy == NULL);
-	assert(!IS_USER_MAP(bo->map));
+	assert(list_is_empty(&bo->list));
 		if (ptr)
 			kgem_bo_sync__cpu(kgem, bo);
 		return ptr;
+	}
-	if (IS_CPU_MAP(bo->map))
-		kgem_bo_release_map(kgem, bo);
-	ptr = bo->map;
+	ptr = MAP(bo->map__gtt);
 	if (ptr == NULL) {
-		assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2);
+		assert(num_pages(bo) <= kgem->aperture_mappable / 2);
 		assert(kgem->gen != 021 || bo->tiling != I915_TILING_Y);
 		/* Cache this mapping to avoid the overhead of an
 		 * excruciatingly slow GTT pagefault. This is more an
 		 * issue with compositing managers which need to frequently
 		 * flush CPU damage to their GPU bo.
 		 */
-		bo->map = ptr;
+		bo->map__gtt = ptr;
 		DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
+		}
 	if (bo->domain != DOMAIN_GTT || FORCE_MMAP_SYNC & (1 << DOMAIN_GTT)) {
 void *kgem_bo_map__gtt(struct kgem *kgem, struct kgem_bo *bo)
+{
 	void *ptr;
-	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
+	DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__,
-	     bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
+	     bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
 	assert(bo->exec == NULL);
-	assert(list_is_empty(&bo->list));
-	assert(!IS_USER_MAP(bo->map));
-	assert_tiling(kgem, bo);
+	assert(bo->exec == NULL);
-	if (IS_CPU_MAP(bo->map))
+	assert(list_is_empty(&bo->list));
-		kgem_bo_release_map(kgem, bo);
+	assert_tiling(kgem, bo);
-	ptr = bo->map;
+	ptr = MAP(bo->map__gtt);
 	if (ptr == NULL) {
 		/* Cache this mapping to avoid the overhead of an
 		 * excruciatingly slow GTT pagefault. This is more an
 		 * issue with compositing managers which need to frequently
 		 * flush CPU damage to their GPU bo.
 		 */
-		bo->map = ptr;
+		bo->map__gtt = ptr;
 		DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
+	}
 	return ptr;
+}
-void *kgem_bo_map__debug(struct kgem *kgem, struct kgem_bo *bo)
-{
-	if (bo->map)
-		return MAP(bo->map);
+void *kgem_bo_map__debug(struct kgem *kgem, struct kgem_bo *bo)
 	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
-	return bo->map = __kgem_bo_map__gtt(kgem, bo);
+	return kgem_bo_map__async(kgem, bo);
+}
 void *kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo)
 {
 	struct drm_i915_gem_mmap mmap_arg;
-	DBG(("%s(handle=%d, size=%d, mapped? %d)\n",
+	DBG(("%s(handle=%d, size=%d, map=%p:%p)\n",
-	     __FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map)));
+	     __FUNCTION__, bo->handle, bytes(bo), bo->map__gtt, bo->map__cpu));
 	assert(!bo->purged);
-	assert(list_is_empty(&bo->list));
-	assert(bo->proxy == NULL);
-	if (IS_CPU_MAP(bo->map))
+	assert(list_is_empty(&bo->list));
-		return MAP(bo->map);
+	assert(bo->proxy == NULL);
-	if (bo->map)
+	if (bo->map__cpu)
-		kgem_bo_release_map(kgem, bo);
+		return MAP(bo->map__cpu);
 	kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));
+retry:
 	VG_CLEAR(mmap_arg);
-retry:
+	mmap_arg.handle = bo->handle;
-	VG_CLEAR(mmap_arg);
-	mmap_arg.handle = bo->handle;
+	mmap_arg.offset = 0;
-	mmap_arg.offset = 0;
+	mmap_arg.size = bytes(bo);
-	mmap_arg.size = bytes(bo);
 	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
+		int err = 0;
 		if (__kgem_throttle_retire(kgem, 0))
 			goto retry;
 		if (__kgem_throttle_retire(kgem, 0))
-		if (kgem->need_expire) {
+			goto retry;
-			kgem_cleanup_cache(kgem);
-			goto retry;
 		if (kgem_cleanup_cache(kgem))
+			goto retry;
+		ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n",
+		       __FUNCTION__, bo->handle, bytes(bo), err);
+		return NULL;
 		ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain\n",
-		       __FUNCTION__, bo->handle, bytes(bo));
+	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
-		return NULL;
-	}
-	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
 	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
-	bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr);
+	return bo->map__cpu = (void *)(uintptr_t)mmap_arg.addr_ptr;
-	return (void *)(uintptr_t)mmap_arg.addr_ptr;
+}
-}
-void *__kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo)
-{
-	struct drm_i915_gem_mmap mmap_arg;
-	DBG(("%s(handle=%d, size=%d, mapped? %d)\n",
+/*
+struct kgem_bo *kgem_create_map(struct kgem *kgem,
-	     __FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map)));
+				void *ptr, uint32_t size,
-        assert(bo->refcnt);
+				bool read_only)
+{
-	assert(!bo->purged);
+	struct kgem_bo *bo;
+	uintptr_t first_page, last_page;
+	uint32_t handle;
-	assert(list_is_empty(&bo->list));
+	assert(MAP(ptr) == ptr);
-	assert(bo->proxy == NULL);
+	if (!kgem->has_userptr)
-	if (IS_CPU_MAP(bo->map))
+		return NULL;
+	first_page = (uintptr_t)ptr;
+	last_page = first_page + size + PAGE_SIZE - 1;
+	first_page &= ~(PAGE_SIZE-1);
+	last_page &= ~(PAGE_SIZE-1);
-		return MAP(bo->map);
+	assert(last_page > first_page);
+	handle = gem_userptr(kgem->fd,
+			     (void *)first_page, last_page-first_page,
-retry:
+			     read_only);
-	VG_CLEAR(mmap_arg);
-	mmap_arg.handle = bo->handle;
-	mmap_arg.offset = 0;
-	mmap_arg.size = bytes(bo);
-	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
+	if (handle == 0)
-		int err = errno;
+		return NULL;
-		assert(err != EINVAL);
+	bo = __kgem_bo_alloc(handle, (last_page - first_page) / PAGE_SIZE);
-		if (__kgem_throttle_retire(kgem, 0))
+	if (bo == NULL) {
-			goto retry;
+		gem_close(kgem->fd, handle);
+		return NULL;
+	}
+	bo->snoop = !kgem->has_llc;
+	debug_alloc__bo(kgem, bo);
 		if (kgem->need_expire) {
-			kgem_cleanup_cache(kgem);
+	if (first_page != (uintptr_t)ptr) {
-			goto retry;
+		struct kgem_bo *proxy;
 		proxy = kgem_create_proxy(kgem, bo,
 			bo->domain = DOMAIN_CPU;
+		}
+	}
+}
+void kgem_bo_sync__cpu_full(struct kgem *kgem, struct kgem_bo *bo, bool write)
+{
+	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
+	assert(!bo->scanout || !write);
+	if (write || bo->needs_flush)
+		kgem_bo_submit(kgem, bo);
+	/* SHM pixmaps use proxies for subpage offsets */
+	assert(!bo->purged);
+	assert(bo->refcnt);
+	while (bo->proxy)
+		bo = bo->proxy;
+	assert(bo->refcnt);
+	assert(!bo->purged);
+	if (bo->domain != DOMAIN_CPU || FORCE_MMAP_SYNC & (1 << DOMAIN_CPU)) {
+		struct drm_i915_gem_set_domain set_domain;
+		DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
+		     __FUNCTION__, bo->handle,
+		     bo->needs_flush, bo->domain,
+		     __kgem_busy(kgem, bo->handle)));
+		VG_CLEAR(set_domain);
+		set_domain.handle = bo->handle;
+		set_domain.read_domains = I915_GEM_DOMAIN_CPU;
+		set_domain.write_domain = write ? I915_GEM_DOMAIN_CPU : 0;
+		if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
+			if (bo->exec == NULL)
+				kgem_bo_retire(kgem, bo);
+			bo->domain = write ? DOMAIN_CPU : DOMAIN_NONE;
+		}
+	}
+}
+void kgem_bo_sync__gtt(struct kgem *kgem, struct kgem_bo *bo)
+{
+	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
+	assert(bo->refcnt);
+	assert(bo->proxy == NULL);
+	kgem_bo_submit(kgem, bo);
+	if (bo->domain != DOMAIN_GTT || FORCE_MMAP_SYNC & (1 << DOMAIN_GTT)) {
+		struct drm_i915_gem_set_domain set_domain;
+		DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
+		     __FUNCTION__, bo->handle,
+		     bo->needs_flush, bo->domain,
+		     __kgem_busy(kgem, bo->handle)));
+		VG_CLEAR(set_domain);
+		set_domain.handle = bo->handle;
+		set_domain.read_domains = I915_GEM_DOMAIN_GTT;
+		set_domain.write_domain = I915_GEM_DOMAIN_GTT;
+		if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
+			kgem_bo_retire(kgem, bo);
+			bo->domain = DOMAIN_GTT;
+			bo->gtt_dirty = true;
+		}
+	}
+}
 void kgem_clear_dirty(struct kgem *kgem)
+{
 	struct list * const buffers = &kgem->next_request->buffers;
 	assert(!bo->scanout);
 	bo->proxy = kgem_bo_reference(target);
 	bo->delta = offset;
-	if (target->exec) {
+	if (target->exec && !bo->io) {
 		list_move_tail(&bo->request, &kgem->next_request->buffers);
 		bo->exec = &_kgem_dummy_exec;
 	if (bo == NULL)
 		return NULL;
 	bo->mem = NULL;
 	bo->need_io = false;
-	bo->mmapped = true;
+	bo->mmapped = MMAPPED_CPU;
 	return bo;
 		     __FUNCTION__, bo->base.handle, num_pages(&bo->base)));
 		assert(bo->base.snoop);
 		assert(bo->base.tiling == I915_TILING_NONE);
 		assert(num_pages(&bo->base) >= alloc);
-		assert(bo->mmapped == true);
+		assert(bo->mmapped == MMAPPED_CPU);
 		assert(bo->need_io == false);
 		bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
 			DBG(("%s: created CPU (LLC) handle=%d for buffer, size %d\n",
 			     __FUNCTION__, bo->base.handle, alloc));
+		}
 		assert(bo->base.refcnt == 1);
-		assert(bo->mmapped == true);
+		assert(bo->mmapped == MMAPPED_CPU);
 		assert(bo->need_io == false);
 		bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
 			DBG(("%s: created CPU handle=%d for buffer, size %d\n",
 			     __FUNCTION__, bo->base.handle, alloc));
+		}
 		assert(bo->base.refcnt == 1);
-		assert(bo->mmapped == true);
+		assert(bo->mmapped == MMAPPED_CPU);
 		assert(bo->need_io == false);
 		debug_alloc(kgem, alloc);
 		__kgem_bo_init(&bo->base, handle, alloc);
 		DBG(("%s: created snoop handle=%d for buffer\n",
 		     __FUNCTION__, bo->base.handle));
-		assert(bo->mmapped == true);
+		assert(bo->mmapped == MMAPPED_CPU);
 		assert(bo->need_io == false);
 		bo->base.refcnt = 1;
 		bo->base.snoop = true;
-		bo->base.map = MAKE_USER_MAP(bo->mem);
+		bo->base.map__cpu = MAKE_USER_MAP(bo->mem);
 		assert(bo->base.refcnt >= 1);
 		/* We can reuse any write buffer which we can fit */
 		if (flags == KGEM_BUFFER_LAST &&
 		    bo->write == KGEM_BUFFER_WRITE &&
+		    bo->base.refcnt == 1 &&
-		    bo->base.refcnt == 1 && !bo->mmapped &&
+		    bo->mmapped == MMAPPED_NONE &&
 		    size <= bytes(&bo->base)) {
 			DBG(("%s: reusing write buffer for read of %d bytes? used=%d, total=%d\n",
 			     __FUNCTION__, size, bo->used, bytes(&bo->base)));
-			gem_write(kgem->fd, bo->base.handle,
+			gem_write__cachealigned(kgem->fd, bo->base.handle,
 , bo->used, bo->mem);
 			kgem_buffer_release(kgem, bo);
 			bo->need_io = 0;
 			bo->write = 0;
 	if (flags & KGEM_BUFFER_WRITE) {
 		list_for_each_entry(bo, &kgem->active_buffers, base.list) {
 			assert(bo->base.io);
+			assert(bo->base.refcnt >= 1);
-			assert(bo->base.refcnt >= 1);
+			assert(bo->base.exec == NULL);
 			assert(bo->mmapped);
-			assert(!IS_CPU_MAP(bo->base.map) || kgem->has_llc || bo->base.snoop);
+			assert(bo->mmapped == MMAPPED_GTT || kgem->has_llc || bo->base.snoop);
-			if (!kgem->has_llc && (bo->write & ~flags) & KGEM_BUFFER_INPLACE) {
+			if ((bo->write & ~flags) & KGEM_BUFFER_INPLACE && !bo->base.snoop) {
 				DBG(("%s: skip write %x buffer, need %x\n",
 				     __FUNCTION__, bo->write, flags));
 				offset = bo->used;
 				bo->used += size;
 				list_move(&bo->base.list, &kgem->batch_buffers);
 				goto done;
+			}
+			if (size <= bytes(&bo->base) &&
+			    (bo->base.rq == NULL ||
+			     !__kgem_busy(kgem, bo->base.handle))) {
+				DBG(("%s: reusing whole buffer? size=%d, total=%d\n",
+				     __FUNCTION__, size, bytes(&bo->base)));
+				__kgem_bo_clear_busy(&bo->base);
+				kgem_buffer_release(kgem, bo);
+				switch (bo->mmapped) {
+				case MMAPPED_CPU:
+					kgem_bo_sync__cpu(kgem, &bo->base);
+					break;
+				case MMAPPED_GTT:
+					kgem_bo_sync__gtt(kgem, &bo->base);
+					break;
+				}
+				offset = 0;
+				bo->used = size;
+				list_move(&bo->base.list, &kgem->batch_buffers);
+				goto done;
+			}
+		}
+	}
 #endif
 		alloc = ALIGN(size, kgem->buffer_size);
 	if (alloc > MAX_CACHE_SIZE)
 		alloc = PAGE_ALIGN(size);
 	assert(alloc);
 	alloc /= PAGE_SIZE;
 	if (alloc > kgem->aperture_mappable / 4)
-		flags &= ~KGEM_BUFFER_INPLACE;
-	alloc /= PAGE_SIZE;
+		flags &= ~KGEM_BUFFER_INPLACE;
 	if (kgem->has_llc &&
 	    (flags & KGEM_BUFFER_WRITE_INPLACE) != KGEM_BUFFER_WRITE_INPLACE) {
 		if (old == NULL)
 			old = search_linear_cache(kgem, NUM_PAGES(size),
 						  CREATE_EXACT | CREATE_INACTIVE | CREATE_GTT_MAP);
 		if (old == NULL) {
 			old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
-			if (old && !__kgem_bo_is_mappable(kgem, old)) {
+			if (old && !kgem_bo_can_map(kgem, old)) {
 				_kgem_bo_destroy(kgem, old);
 				old = NULL;
+			}
+		}
 		if (old) {
 			DBG(("%s: reusing handle=%d for buffer\n",
 			     __FUNCTION__, old->handle));
-			assert(__kgem_bo_is_mappable(kgem, old));
+			assert(kgem_bo_can_map(kgem, old));
 			assert(!old->snoop);
 			assert(old->rq == NULL);
 			bo = buffer_alloc();
 			assert(bo->mmapped);
 			assert(bo->base.refcnt == 1);
 			bo->mem = kgem_bo_map(kgem, &bo->base);
 			if (bo->mem) {
-				if (IS_CPU_MAP(bo->base.map))
+				if (bo->mem == MAP(bo->base.map__cpu))
+					flags &= ~KGEM_BUFFER_INPLACE;
+				else
-					flags &= ~KGEM_BUFFER_INPLACE;
+					bo->mmapped = MMAPPED_GTT;
 				goto init;
 			} else {
 				bo->base.refcnt = 0;
 				kgem_bo_free(kgem, &bo->base);
 	assert(bo->base.refcnt == 1);
 	assert(num_pages(&bo->base) >= NUM_PAGES(size));
 	assert(!bo->need_io || !bo->base.needs_flush);
 	assert(!bo->need_io || bo->base.domain != DOMAIN_GPU);
 	assert(bo->mem);
+	assert(bo->mmapped != MMAPPED_GTT || MAP(bo->base.map__gtt) == bo->mem);
-	assert(!bo->mmapped || bo->base.map != NULL);
+	assert(bo->mmapped != MMAPPED_CPU || MAP(bo->base.map__cpu) == bo->mem);
 	bo->used = size;
 	bo->write = flags & KGEM_BUFFER_WRITE_INPLACE;
 	DBG(("%s(pages=%d [%d]) new handle=%d, used=%d, write=%d\n",
 	     __FUNCTION__, num_pages(&bo->base), alloc, bo->base.handle, bo->used, bo->write));
 done:
+	bo->used = ALIGN(bo->used, UPLOAD_ALIGNMENT);
-	bo->used = ALIGN(bo->used, UPLOAD_ALIGNMENT);
+	assert(bo->used && bo->used <= bytes(&bo->base));
 	assert(bo->mem);
 	*ret = (char *)bo->mem + offset;
 	return kgem_create_proxy(kgem, &bo->base, offset, size);
 			io->used = min;
+		}
 		bo->size.bytes -= stride;
+	}
-	bo->map = MAKE_CPU_MAP(*ret);
+	bo->map__cpu = *ret;
 	bo->pitch = stride;
 	bo->unique_id = kgem_get_unique_id(kgem);
 	return bo;
 void kgem_proxy_bo_attach(struct kgem_bo *bo,
 			  struct kgem_bo **ptr)
+{
 	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
-	assert(bo->map == NULL || IS_CPU_MAP(bo->map));
+	assert(bo->map__gtt == NULL);
 	assert(bo->proxy);
 	list_add(&bo->vma, &bo->proxy->vma);
-	bo->map = ptr;
+	bo->map__gtt = ptr;
 	*ptr = kgem_bo_reference(bo);
+}
 		     __FUNCTION__,
 		     bo->base.needs_flush,
 		     bo->base.domain,
 		     __kgem_busy(kgem, bo->base.handle)));
-		assert(!IS_CPU_MAP(bo->base.map) || bo->base.snoop || kgem->has_llc);
+		assert(bo->mmapped == MMAPPED_GTT || bo->base.snoop || kgem->has_llc);
 		VG_CLEAR(set_domain);
 		set_domain.handle = bo->base.handle;
 		set_domain.write_domain = 0;
 		set_domain.read_domains =
-			IS_CPU_MAP(bo->base.map) ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT;
+			bo->mmapped == MMAPPED_CPU ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT;
 		if (drmIoctl(kgem->fd,

Subversion Repositories Kolibri OS

(root)/contrib/sdk/sources/Intel-2D/sna/kgem.c – Rev 4403 → 4501