WebSVN – Kolibri OS – Diff – /drivers/video/drm/i915/i915_gem.c

 //#include
 #include
 //#include
 #include
+extern int x86_clflush_size;
+#undef mb
+#undef rmb
+#undef wmb
+#define mb() asm volatile("mfence")
+#define rmb() asm volatile ("lfence")
+#define wmb() asm volatile ("sfence")
+static inline void clflush(volatile void *__p)
+{
+    asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
+}
 static inline long PTR_ERR(const void *ptr)
+{
     return (long) ptr;
+}
+void
+drm_gem_object_free(struct kref *kref)
+{
+    struct drm_gem_object *obj = (struct drm_gem_object *) kref;
+    struct drm_device *dev = obj->dev;
+    BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+    i915_gem_free_object(obj);
+}
 /**
  * Initialize an already allocated GEM object of the specified size with
             struct drm_gem_object *obj, size_t size)
+{
     BUG_ON((size & (PAGE_SIZE - 1)) != 0);
+    obj->dev = dev;
-    obj->dev = dev;
+    kref_init(&obj->refcount);
     atomic_set(&obj->handle_count, 0);
     obj->size = size;
+    return 0;
+}
-    return 0;
 void
 drm_gem_object_release(struct drm_gem_object *obj)
 { }
 		return ret;
+	}
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference(&obj->base);
-//   trace_i915_gem_object_create(obj);
+	drm_gem_object_unreference(&obj->base);
 	*handle_p = handle;
 	return 0;
 err_pages:
-//   while (i--)
+    while (i--)
-//       page_cache_release(obj->pages[i]);
+        FreePage(obj->pages[i]);
 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
+{
 	int page_count = obj->base.size / PAGE_SIZE;
 	int i;
+    ENTER();
 	BUG_ON(obj->madv == __I915_MADV_PURGED);
 //   if (obj->tiling_mode != I915_TILING_NONE)
 //       i915_gem_object_save_bit_17_swizzle(obj);
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
-/*                                           It's a swap!!!
-	for (i = 0; i < page_count; i++) {
-		if (obj->dirty)
-			set_page_dirty(obj->pages[i]);
+	if (obj->madv == I915_MADV_DONTNEED)
-		if (obj->madv == I915_MADV_WILLNEED)
+		obj->dirty = 0;
 			mark_page_accessed(obj->pages[i]);
-        //page_cache_release(obj->pages[i]);
+	for (i = 0; i < page_count; i++) {
         FreePage(obj->pages[i]);
+	}
+	obj->dirty = 0;
 	obj->dirty = 0;
-*/
+    free(obj->pages);
 	obj->pages = NULL;
     free(obj->pages);
 	BUG_ON(ring == NULL);
 	obj->ring = ring;
 	/* Add a reference if we're newly entering the active list. */
 	if (!obj->active) {
-//       drm_gem_object_reference(&obj->base);
+		drm_gem_object_reference(&obj->base);
 		obj->active = 1;
+	}
 		reg = &dev_priv->fence_regs[obj->fence_reg];
 		list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
+	}
+}
+static void
+i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
+{
+	list_del_init(&obj->ring_list);
+	obj->last_rendering_seqno = 0;
+}
+static void
+i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
+{
+	struct drm_device *dev = obj->base.dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	BUG_ON(!obj->active);
+	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
+	i915_gem_object_move_off_active(obj);
+}
+/* Immediately discard the backing storage */
+static void
+i915_gem_object_truncate(struct drm_i915_gem_object *obj)
+{
+	struct inode *inode;
+	/* Our goal here is to return as much of the memory as
+	 * is possible back to the system as we are called from OOM.
+	 * To do this we must instruct the shmfs to drop all of its
+	 * backing pages, *now*.
+	 */
+	obj->madv = __I915_MADV_PURGED;
+}
+static inline int
+i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
+{
 	return obj->madv == I915_MADV_DONTNEED;
+}
 static void
 			obj->base.write_domain = 0;
 			list_del_init(&obj->gpu_write_list);
 			i915_gem_object_move_to_active(obj, ring,
 						       i915_gem_next_request_seqno(ring));
-//			trace_i915_gem_object_change_domain(obj,
-//							    obj->base.read_domains,
 //							    old_write_domain);
+		}
+	}
+/**
+ * Ensures that all rendering to the object has completed and the object is
+ * safe to unbind from the GTT or access from the CPU.
+ */
+int
+i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
+{
+	int ret;
+	/* This function only exists to support waiting for existing rendering,
+	 * not for emitting required flushes.
+	 */
+	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
+	/* If there is rendering queued on the buffer being evicted, wait for
+	 * it.
+	 */
+	if (obj->active) {
+//		ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
+//		if (ret)
+//			return ret;
+	}
+	return 0;
+}
+static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
+{
+	u32 old_write_domain, old_read_domains;
+	/* Act a barrier for all accesses through the GTT */
+	mb();
+	/* Force a pagefault for domain tracking on next user access */
+//	i915_gem_release_mmap(obj);
+	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
+		return;
+	old_read_domains = obj->base.read_domains;
+	old_write_domain = obj->base.write_domain;
+	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
+	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
+}
+/**
+ * Unbinds an object from the GTT aperture.
+ */
+int
+i915_gem_object_unbind(struct drm_i915_gem_object *obj)
+{
+	int ret = 0;
+    ENTER();
+	if (obj->gtt_space == NULL)
+		return 0;
+	if (obj->pin_count != 0) {
+		DRM_ERROR("Attempting to unbind pinned buffer\n");
+		return -EINVAL;
+	}
+	ret = i915_gem_object_finish_gpu(obj);
+	if (ret == -ERESTARTSYS)
+		return ret;
+	/* Continue on if we fail due to EIO, the GPU is hung so we
+	 * should be safe and we need to cleanup or else we might
+	 * cause memory corruption through use-after-free.
+	 */
+	i915_gem_object_finish_gtt(obj);
+	/* Move the object to the CPU domain to ensure that
+	 * any possible CPU writes while it's not in the GTT
+	 * are flushed when we go to remap it.
+	 */
+	if (ret == 0)
+		ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+	if (ret == -ERESTARTSYS)
+		return ret;
+	if (ret) {
+		/* In the event of a disaster, abandon all caches and
+		 * hope for the best.
+		 */
+		i915_gem_clflush_object(obj);
+		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	}
+	/* release the fence reg _after_ flushing */
+	ret = i915_gem_object_put_fence(obj);
+	if (ret == -ERESTARTSYS)
+		return ret;
+	i915_gem_gtt_unbind_object(obj);
+	i915_gem_object_put_pages_gtt(obj);
+	list_del_init(&obj->gtt_list);
+	list_del_init(&obj->mm_list);
+	/* Avoid an unnecessary call to unbind on rebind. */
+	obj->map_and_fenceable = true;
+	drm_mm_put_block(obj->gtt_space);
+	obj->gtt_space = NULL;
+	obj->gtt_offset = 0;
+	if (i915_gem_object_is_purgeable(obj))
+		i915_gem_object_truncate(obj);
+    LEAVE();
+	return ret;
+}
+int
+i915_gem_flush_ring(struct intel_ring_buffer *ring,
+		    uint32_t invalidate_domains,
+		    uint32_t flush_domains)
+{
+	int ret;
+	if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
+		return 0;
+	ret = ring->flush(ring, invalidate_domains, flush_domains);
+	if (ret)
+		return ret;
+	if (flush_domains & I915_GEM_GPU_DOMAINS)
+		i915_gem_process_flushing_list(ring, flush_domains);
+static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
+{
-/**
+	return i915_seqno_passed(ring->get_seqno(ring), seqno);
  * Ensures that all rendering to the object has completed and the object is
+static int
- * safe to unbind from the GTT or access from the CPU.
+i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
- */
+			    struct intel_ring_buffer *pipelined)
+{
+	int ret;
-int
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
-{
+	if (obj->fenced_gpu_access) {
-	int ret;
+		if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+			ret = i915_gem_flush_ring(obj->last_fenced_ring,
+, obj->base.write_domain);
+			if (ret)
-	/* This function only exists to support waiting for existing rendering,
+				return ret;
-	 * not for emitting required flushes.
+		}
-	 */
+		obj->fenced_gpu_access = false;
 	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
-	/* If there is rendering queued on the buffer being evicted, wait for
+	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
+		if (!ring_passed_seqno(obj->last_fenced_ring,
-	 * it.
+				       obj->last_fenced_seqno)) {
+//           ret = i915_wait_request(obj->last_fenced_ring,
+//                       obj->last_fenced_seqno);
+//           if (ret)
+//               return ret;
+		}
+		obj->last_fenced_seqno = 0;
+		obj->last_fenced_ring = NULL;
 	 */
-//	if (obj->active) {
-//		ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
-//		if (ret)
-//			return ret;
+	/* Ensure that all CPU reads are completed before installing a fence
-//	}
+	 * and all writes before removing the fence.
+	 */
-	return 0;
+	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
-}
+		mb();
 int
-i915_gem_flush_ring(struct intel_ring_buffer *ring,
+	return 0;
-		    uint32_t invalidate_domains,
+}
-		    uint32_t flush_domains)
+int
+i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
+{
+	int ret;
 //   if (obj->tiling_mode)
-	int ret;
+//       i915_gem_release_mmap(obj);
 		return ret;
+	}
 	ret = i915_gem_gtt_bind_object(obj);
 	if (ret) {
-//       i915_gem_object_put_pages_gtt(obj);
+        i915_gem_object_put_pages_gtt(obj);
 		drm_mm_put_block(obj->gtt_space);
 		obj->gtt_space = NULL;
 	mappable =
 		obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
-	obj->map_and_fenceable = mappable && fenceable;
 	obj->map_and_fenceable = mappable && fenceable;
 //   trace_i915_gem_object_bind(obj, map_and_fenceable);
 	return 0;
+}
 	 * tracking.
 	 */
 	if (obj->cache_level != I915_CACHE_NONE)
 		return;
+     if(obj->mapped != NULL)
+     {
+        uint8_t *page_virtual;
+        unsigned int i;
+        page_virtual = obj->mapped;
+        asm volatile("mfence");
+        for (i = 0; i < obj->base.size; i += x86_clflush_size)
+            clflush(page_virtual + i);
+        asm volatile("mfence");
+     }
+     else
+     {
-//   trace_i915_gem_object_clflush(obj);
+        uint8_t *page_virtual;
+        unsigned int i;
+        page_virtual = AllocKernelSpace(obj->base.size);
+        if(page_virtual != NULL)
+        {
+            u32_t *src, *dst;
+            u32 count;
+#define page_tabs  0xFDC00000      /* really dirty hack */
+            src =  (u32_t*)obj->pages;
+            dst =  &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12];
+            count = obj->base.size/4096;
+            while(count--)
+            {
+                *dst++ = (0xFFFFF000 & *src++) | 0x001 ;
+            };
+            asm volatile("mfence");
+            for (i = 0; i < obj->base.size; i += x86_clflush_size)
+                clflush(page_virtual + i);
+            asm volatile("mfence");
+            FreeKernelSpace(page_virtual);
+        }
+        else
+        {
-//   drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
+            asm volatile (
+            "mfence         \n"
-     mb();
+            "wbinvd         \n"                 /* this is really ugly  */
+            "mfence");
      __asm__ ("wbinvd");   // this is really ugly
      mb();
+}
 	/* Queue the GPU write cache flushing we need. */
 	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
+}
+/** Flushes the GTT write domain for the object if it's dirty. */
+static void
+i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
+{
+	uint32_t old_write_domain;
+	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
+		return;
+	/* No actual flushing is required for the GTT write domain.  Writes
+	 * to it immediately go to main memory as far as we know, so there's
+	 * no chipset flush.  It also doesn't land in render cache.
+	 *
+	 * However, we do have to enforce the order so that all writes through
+	 * the GTT land before any writes to the device, such as updates to
+	 * the GATT itself.
+	 */
 	wmb();
 	old_write_domain = obj->base.write_domain;
 	obj->base.write_domain = 0;
 	i915_gem_clflush_object(obj);
 	intel_gtt_chipset_flush();
 	old_write_domain = obj->base.write_domain;
 	obj->base.write_domain = 0;
-//	trace_i915_gem_object_change_domain(obj,
-//					    obj->base.read_domains,
 //					    old_write_domain);
+}
 /**
 		old_write_domain = obj->base.write_domain;
 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-		trace_i915_gem_object_change_domain(obj,
-						    old_read_domains,
+		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 						    old_write_domain);
+    }
 	 * the domain values for our changes.
 	 */
 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
-//   trace_i915_gem_object_change_domain(obj,
-//                       old_read_domains,
 //                       old_write_domain);
+	return 0;
+}
+int
+i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
+{
+	int ret;
+	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
+		return 0;
+	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+		ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
+		if (ret)
+			return ret;
+	}
+	/* Ensure that we invalidate the GPU's caches and TLBs. */
+	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
+	return i915_gem_object_wait_rendering(obj);
+}
+/**
+ * Moves a single object to the CPU read, and possibly write domain.
+ *
+ * This function returns when the move is complete, including waiting on
+ * flushes to occur.
+ */
+static int
+i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
+{
+	uint32_t old_write_domain, old_read_domains;
+	int ret;
+	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
+		return 0;
+	ret = i915_gem_object_flush_gpu_write_domain(obj);
+	if (ret)
+		return ret;
+	ret = i915_gem_object_wait_rendering(obj);
+	if (ret)
+		return ret;
+	i915_gem_object_flush_gtt_write_domain(obj);
+	/* If we have a partially-valid cache of the object in the CPU,
+	 * finish invalidating it and free the per-page flags.
+	 */
+	i915_gem_object_set_to_full_cpu_read_domain(obj);
+	old_write_domain = obj->base.write_domain;
+	old_read_domains = obj->base.read_domains;
+	/* Flush the CPU cache if it's still invalid. */
+	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
+		i915_gem_clflush_object(obj);
+		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
+	}
+	/* It should now be out of any other write domains, and we can update
+	 * the domain values for our changes.
+	 */
+	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+	/* If we're writing through the CPU, then the GPU read domains will
+	 * need to be invalidated at next use.
+	 */
+	if (write) {
+		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	}
+	return 0;
+}
-	return 0;
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
-	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
-//   WARN_ON(i915_verify_lists(dev));
+	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
 #if 0
 	if (obj->gtt_space != NULL) {
 			list_move_tail(&obj->mm_list,
 				       &dev_priv->mm.pinned_list);
+	}
 	obj->pin_mappable |= map_and_fenceable;
 //   WARN_ON(i915_verify_lists(dev));
 	return 0;
+}
+void
+i915_gem_object_unpin(struct drm_i915_gem_object *obj)
+{
+	struct drm_device *dev = obj->base.dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	BUG_ON(obj->pin_count == 0);
+	BUG_ON(obj->gtt_space == NULL);
+	if (--obj->pin_count == 0) {
+		if (!obj->active)
+			list_move_tail(&obj->mm_list,
 	obj->map_and_fenceable = true;
 	return obj;
+}
+int i915_gem_init_object(struct drm_gem_object *obj)
+{
+	BUG();
+	return 0;
+}
+static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
+{
+	struct drm_device *dev = obj->base.dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret;
+    ENTER();
+	ret = i915_gem_object_unbind(obj);
+	if (ret == -ERESTARTSYS) {
+		list_move(&obj->mm_list,
+			  &dev_priv->mm.deferred_free_list);
+		return;
+	}
+//	if (obj->base.map_list.map)
+//		drm_gem_free_mmap_offset(&obj->base);
+	drm_gem_object_release(&obj->base);
+	i915_gem_info_remove_obj(dev_priv, obj->base.size);
+	kfree(obj->page_cpu_valid);
+	kfree(obj->bit_17);
+	kfree(obj);
+    LEAVE();
+}
+void i915_gem_free_object(struct drm_gem_object *gem_obj)
+{
     INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
     for (i = 0; i < I915_NUM_RINGS; i++)
         init_ring_lists(&dev_priv->ring[i]);
 	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
         INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
-//    INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
-//              i915_gem_retire_work_handler);
-//    init_completion(&dev_priv->error_completion);
     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
     if (IS_GEN3(dev)) {
         u32 tmp = I915_READ(MI_ARB_STATE);
     for (i = 0; i < dev_priv->num_fence_regs; i++) {
         i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
+    }
-    i915_gem_detect_bit_6_swizzle(dev);
-//    init_waitqueue_head(&dev_priv->pending_flip_queue);
+    i915_gem_detect_bit_6_swizzle(dev);
     dev_priv->mm.interruptible = true;

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 2342 → 2344