WebSVN – Kolibri OS – Diff – /drivers/video/drm/i915/i915_gem.c

 unsigned long vm_mmap(struct file *file, unsigned long addr,
          unsigned long len, unsigned long prot,
-         unsigned long flag, unsigned long offset);
-static inline void clflush(volatile void *__p)
-{
-    asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
+         unsigned long flag, unsigned long offset);
 				 struct drm_i915_gem_object *obj);
 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
 					 struct drm_i915_fence_reg *fence,
 					 bool enable);
 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
 static bool cpu_cache_is_coherent(struct drm_device *dev,
 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
+{
 	return i915_gem_obj_bound_any(obj) && !obj->active;
+}
-#if 0
-int
-i915_gem_init_ioctl(struct drm_device *dev, void *data,
-		    struct drm_file *file)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_i915_gem_init *args = data;
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
-		return -ENODEV;
-	if (args->gtt_start >= args->gtt_end ||
-	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
-		return -EINVAL;
-	/* GEM with user mode setting was never supported on ilk and later. */
-	if (INTEL_INFO(dev)->gen >= 5)
-		return -ENODEV;
-	mutex_lock(&dev->struct_mutex);
-	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
-				  args->gtt_end);
-	dev_priv->gtt.mappable_end = args->gtt_end;
-	mutex_unlock(&dev->struct_mutex);
-	return 0;
-}
-#endif
 int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file)
+{
 		  bool page_do_bit17_swizzling,
 		  bool needs_clflush_before,
 		  bool needs_clflush_after)
+{
 	char *vaddr;
-	int ret = 0;
+	int ret;
 	if (unlikely(page_do_bit17_swizzling))
 		return -EINVAL;
-    vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
+	vaddr = kmap_atomic(page);
 	if (needs_clflush_before)
 		drm_clflush_virt_range(vaddr + shmem_page_offset,
 				       page_length);
 	memcpy(vaddr + shmem_page_offset,
 						user_data,
 						page_length);
 	if (needs_clflush_after)
 		drm_clflush_virt_range(vaddr + shmem_page_offset,
 				       page_length);
-	FreeKernelSpace(vaddr);
+	kunmap_atomic(vaddr);
 //					partial_cacheline_write,
 //					needs_clflush_after);
-		mutex_lock(&dev->struct_mutex);
-next_page:
+		mutex_lock(&dev->struct_mutex);
+		if (ret)
-		if (ret)
+			goto out;
 			goto out;
 next_page:
 		remain -= page_length;
 	 * it would end up going through the fenced access, and we'll get
 	 * different detiling behavior between reading and writing.
 	 * pread/pwrite currently are reading and writing from the CPU
 	 * perspective, requiring manual detiling by the client.
 	 */
-//   if (obj->phys_obj) {
-//       ret = i915_gem_phys_pwrite(dev, obj, args, file);
-//       goto out;
-//   }
 	if (obj->tiling_mode == I915_TILING_NONE &&
 	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
 	    cpu_write_needs_clflush(obj)) {
 		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 		/* Note that the gtt paths might fail with non-page-backed user
 	return !atomic_xchg(&file_priv->rps_wait_boost, true);
+}
 /**
- * __wait_seqno - wait until execution of seqno has finished
+ * __i915_wait_seqno - wait until execution of seqno has finished
  * @ring: the ring expected to report seqno
  * @seqno: duh!
  * @reset_counter: reset sequence associated with the given seqno
  * inserted.
+ *
  * Returns 0 if the seqno was found within the alloted time. Else returns the
  * errno with remaining time filled in timeout argument.
  */
-static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
+int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
 			unsigned reset_counter,
 			bool interruptible,
 			s64 *timeout,
 			struct drm_i915_file_private *file_priv)
+{
 	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
 	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
+		return 0;
-		return 0;
 	timeout_expire = timeout ?
-	timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0;
+		jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
 	if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
 i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
+{
 	struct drm_device *dev = ring->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	bool interruptible = dev_priv->mm.interruptible;
+	unsigned reset_counter;
 	int ret;
 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
 	ret = i915_gem_check_olr(ring, seqno);
 	if (ret)
 		return ret;
-	return __wait_seqno(ring, seqno,
+	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
-			    atomic_read(&dev_priv->gpu_error.reset_counter),
+	return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
-			    interruptible, NULL, NULL);
+				 NULL, NULL);
 }
 static int
-i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
+static int
-				     struct intel_engine_cs *ring)
+i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
+{
 	ret = i915_wait_seqno(ring, seqno);
     if (ret)
         return ret;
-	return i915_gem_object_wait_rendering__tail(obj, ring);
+	return i915_gem_object_wait_rendering__tail(obj);
+}
 	if (ret)
 		return ret;
 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
 	mutex_unlock(&dev->struct_mutex);
+	ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
-	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
+				file_priv);
 	mutex_lock(&dev->struct_mutex);
 	if (ret)
 		return ret;
-	return i915_gem_object_wait_rendering__tail(obj, ring);
+	return i915_gem_object_wait_rendering__tail(obj);
+}
  * Maps the contents of an object, returning the address it is mapped
  * into.
+ *
  * While the mapping holds a reference on the contents of the object, it doesn't
  * imply a ref on the object itself.
+ *
+ * IMPORTANT:
+ *
+ * DRM driver writers who look a this function as an example for how to do GEM
+ * mmap support, please don't implement mmap support like here. The modern way
+ * to implement DRM mmap support is with an mmap offset ioctl (like
+ * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
+ * That way debug tooling like valgrind will understand what's going on, hiding
+ * the mmap call in a driver private ioctl will break that. The i915 driver only
+ * does cpu mmaps this way because we didn't know better.
  */
 int
 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		    struct drm_file *file)
+{
 		if (IS_ERR(page)) {
             dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
 			goto err_pages;
+		}
+#ifdef CONFIG_SWIOTLB
+		if (swiotlb_nr_tbl()) {
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+			sg = sg_next(sg);
-		}
+			continue;
+		}
 #endif
 		if (!i || page_to_pfn(page) != last_pfn + 1) {
 			if (i)
 				sg = sg_next(sg);
 			st->nents++;
 		sg_set_page(sg, page, PAGE_SIZE, 0);
 		} else {
 			sg->length += PAGE_SIZE;
+		}
 		last_pfn = page_to_pfn(page);
+	}
+#ifdef CONFIG_SWIOTLB
-	}
+	if (!swiotlb_nr_tbl())
 #endif
 		sg_mark_end(sg);
 	obj->pages = st;
 static void
 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 			       struct intel_engine_cs *ring)
-{
-	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 seqno = intel_ring_get_seqno(ring);
 	BUG_ON(ring == NULL);
+	}
 	list_move_tail(&obj->ring_list, &ring->active_list);
-	obj->last_read_seqno = seqno;
-	if (obj->fenced_gpu_access) {
-		obj->last_fenced_seqno = seqno;
-		/* Bump MRU to take account of the delayed flush */
-		if (obj->fence_reg != I915_FENCE_REG_NONE) {
-		struct drm_i915_fence_reg *reg;
-		reg = &dev_priv->fence_regs[obj->fence_reg];
-			list_move_tail(®->lru_list,
-				       &dev_priv->mm.fence_list);
 	obj->last_read_seqno = seqno;
+}
 		vma = i915_gem_obj_to_vma(obj, vm);
 		if (vma && !list_empty(&vma->mm_list))
 			list_move_tail(&vma->mm_list, &vm->inactive_list);
+	}
+	intel_fb_obj_flush(obj, true);
 	list_del_init(&obj->ring_list);
 	obj->ring = NULL;
 	obj->last_read_seqno = 0;
 	obj->last_write_seqno = 0;
-	obj->base.write_domain = 0;
 	obj->base.write_domain = 0;
 	obj->last_fenced_seqno = 0;
-	obj->fenced_gpu_access = false;
+	obj->last_fenced_seqno = 0;
 		       struct drm_i915_gem_object *obj,
 		 u32 *out_seqno)
+{
 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
 	struct drm_i915_gem_request *request;
+	struct intel_ringbuffer *ringbuf;
 	u32 request_ring_position, request_start;
 	int ret;
+	request = ring->preallocated_lazy_request;
+	if (WARN_ON(request == NULL))
+		return -ENOMEM;
+	if (i915.enable_execlists) {
+		struct intel_context *ctx = request->ctx;
+		ringbuf = ctx->engine[ring->id].ringbuf;
+	} else
+		ringbuf = ring->buffer;
-	request_start = intel_ring_get_tail(ring->buffer);
+	request_start = intel_ring_get_tail(ringbuf);
 	/*
 	 * Emit any outstanding flushes - execbuf can fail to emit the flush
 	 * after having emitted the batchbuffer command. Hence we need to fix
 	 * things up similar to emitting the lazy request. The difference here
 	 * is that the flush _must_ happen before the next request, no matter
 	 * what.
+	 */
+	if (i915.enable_execlists) {
+		ret = logical_ring_flush_all_caches(ringbuf);
+		if (ret)
+			return ret;
-	 */
+	} else {
    ret = intel_ring_flush_all_caches(ring);
    if (ret)
        return ret;
-	request = ring->preallocated_lazy_request;
-	if (WARN_ON(request == NULL))
 		return -ENOMEM;
 	/* Record the position of the start of the request so that
 	 * should we detect the updated seqno part-way through the
     * GPU processing the request, we never over-estimate the
 	 * position of the head.
+	 */
+	request_ring_position = intel_ring_get_tail(ringbuf);
+	if (i915.enable_execlists) {
+		ret = ring->emit_request(ringbuf);
-	 */
+		if (ret)
-	request_ring_position = intel_ring_get_tail(ring->buffer);
+			return ret;
 	} else {
+	ret = ring->add_request(ring);
-	ret = ring->add_request(ring);
+	if (ret)
-	if (ret)
+		return ret;
 		return ret;
 	 * inactive_list and lose its active reference. Hence we do not need
 	 * to explicitly hold another reference here.
 	 */
 	request->batch_obj = obj;
 	if (!i915.enable_execlists) {
 	/* Hold a reference to the current context so that we can inspect
 	 * it later in case a hangcheck error event fires.
 	 */
 	request->ctx = ring->last_context;
 	if (request->ctx)
+		i915_gem_context_reference(request->ctx);
 		i915_gem_context_reference(request->ctx);
 	request->emitted_jiffies = jiffies;
 	trace_i915_gem_request_add(ring, request->seqno);
 	ring->outstanding_lazy_seqno = 0;
-	ring->preallocated_lazy_request = NULL;
 	ring->preallocated_lazy_request = NULL;
 	if (!dev_priv->ums.mm_suspended) {
 //		i915_queue_hangcheck(ring->dev);
            queue_delayed_work(dev_priv->wq,
-					   &dev_priv->mm.retire_work,
-					   round_jiffies_up_relative(HZ));
+					   &dev_priv->mm.retire_work,
-           intel_mark_busy(dev_priv->dev);
+					   round_jiffies_up_relative(HZ));
            intel_mark_busy(dev_priv->dev);
+	}
+}
 static void i915_gem_free_request(struct drm_i915_gem_request *request)
+{
+	struct intel_context *ctx = request->ctx;
 	list_del(&request->list);
 	i915_gem_request_remove_from_client(request);
+	if (ctx) {
+		if (i915.enable_execlists) {
+			struct intel_engine_cs *ring = request->ring;
+			if (ctx != ring->default_context)
+				intel_lr_context_unpin(ring, ctx);
 	if (request->ctx)
-		i915_gem_context_unreference(request->ctx);
+		i915_gem_context_unreference(ctx);
+	}
 	kfree(request);
 		i915_gem_object_move_to_inactive(obj);
+	}
+	/*
+	 * Clear the execlists queue up before freeing the requests, as those
+	 * are the ones that keep the context and ringbuffer backing objects
+	 * pinned in place.
+	 */
+	while (!list_empty(&ring->execlist_queue)) {
+		struct intel_ctx_submit_request *submit_req;
+		submit_req = list_first_entry(&ring->execlist_queue,
+				struct intel_ctx_submit_request,
+				execlist_link);
+		list_del(&submit_req->execlist_link);
+		intel_runtime_pm_put(dev_priv);
+		i915_gem_context_unreference(submit_req->ctx);
+		kfree(submit_req);
+	}
 	/*
 	 * We must free the requests after all the corresponding objects have
 	 * been moved off active lists. Which is the same order as the normal
 	 * retire_requests function does. This is important if object hold
+	}
+	while (!list_empty(&ring->request_list)) {
-	while (!list_empty(&ring->request_list)) {
+		struct drm_i915_gem_request *request;
-		struct drm_i915_gem_request *request;
+		struct intel_ringbuffer *ringbuf;
 		request = list_first_entry(&ring->request_list,
 					   struct drm_i915_gem_request,
 					   list);
+		if (!i915_seqno_passed(seqno, request->seqno))
+			break;
+		trace_i915_gem_request_retire(ring, request->seqno);
+		/* This is one of the few common intersection points
+		 * between legacy ringbuffer submission and execlists:
+		 * we need to tell them apart in order to find the correct
+		 * ringbuffer to which the request belongs to.
+		 */
+		if (i915.enable_execlists) {
 			struct intel_context *ctx = request->ctx;
-		if (!i915_seqno_passed(seqno, request->seqno))
+			ringbuf = ctx->engine[ring->id].ringbuf;
-			break;
+		} else
 			ringbuf = ring->buffer;
 		trace_i915_gem_request_retire(ring, request->seqno);
 		/* We know the GPU must have read the request to have
 		 * sent us the seqno + interrupt, so use the position
 		 * of tail of the request to update the last known position
 		 * of the GPU head.
 	int i;
 	for_each_ring(ring, dev_priv, i) {
 		i915_gem_retire_requests_ring(ring);
+		idle &= list_empty(&ring->request_list);
+		if (i915.enable_execlists) {
+			unsigned long flags;
+			spin_lock_irqsave(&ring->execlist_lock, flags);
+			idle &= list_empty(&ring->execlist_queue);
+			spin_unlock_irqrestore(&ring->execlist_lock, flags);
+			intel_execlists_retire_requests(ring);
 		idle &= list_empty(&ring->request_list);
+	}
 	if (idle)
 	struct intel_engine_cs *ring = NULL;
 	unsigned reset_counter;
 	u32 seqno = 0;
 	int ret = 0;
+	if (args->flags != 0)
+		return -EINVAL;
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
 		return ret;
 	drm_gem_object_unreference(&obj->base);
 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
 	mutex_unlock(&dev->struct_mutex);
-	return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns,
+	return __i915_wait_seqno(ring, seqno, reset_counter, true,
-			    file->driver_priv);
+				 &args->timeout_ns, file->driver_priv);
 out:
 	/* Continue on if we fail due to EIO, the GPU is hung so we
 	 * should be safe and we need to cleanup or else we might
 	 * cause memory corruption through use-after-free.
 	 */
+	/* Throw away the active reference before moving to the unbound list */
+	i915_gem_object_retire(obj);
 	if (i915_is_ggtt(vma->vm)) {
 	i915_gem_object_finish_gtt(obj);
 	trace_i915_vma_unbind(vma);
 	vma->unbind_vma(vma);
 	list_del_init(&vma->mm_list);
-	/* Avoid an unnecessary call to unbind on rebind. */
+	list_del_init(&vma->mm_list);
 	if (i915_is_ggtt(vma->vm))
-	obj->map_and_fenceable = true;
+		obj->map_and_fenceable = false;
 	struct intel_engine_cs *ring;
 	int ret, i;
 	/* Flush everything onto the inactive list. */
+	for_each_ring(ring, dev_priv, i) {
-	for_each_ring(ring, dev_priv, i) {
+		if (!i915.enable_execlists) {
 		ret = i915_switch_context(ring, ring->default_context);
 		if (ret)
+			return ret;
 			return ret;
 		ret = intel_ring_idle(ring);
 		if (ret)
 	WARN(obj && (!obj->stride || !obj->tiling_mode),
 	     "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
 	     obj->stride, obj->tiling_mode);
+	switch (INTEL_INFO(dev)->gen) {
-	switch (INTEL_INFO(dev)->gen) {
+	case 9:
 	case 8:
 	case 7:
 	case 6:
 	case 5:
 				return ret;
 		obj->last_fenced_seqno = 0;
-	}
+	}
 	obj->fenced_gpu_access = false;
 	return 0;
+}
 			list_move_tail(®->lru_list,
 				       &dev_priv->mm.fence_list);
 			return 0;
+		}
 	} else if (enable) {
+		if (WARN_ON(!obj->map_and_fenceable))
+			return -EINVAL;
 		reg = i915_find_fence_reg(dev);
 		if (IS_ERR(reg))
 			return PTR_ERR(reg);
 	i915_gem_object_update_fence(obj, reg, enable);
 	return 0;
 }
 static bool i915_gem_valid_gtt_space(struct drm_device *dev,
-				     struct drm_mm_node *gtt_space,
+static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
+				     unsigned long cache_level)
 				     unsigned long cache_level)
+	struct drm_mm_node *gtt_space = &vma->node;
-{
+	struct drm_mm_node *other;
-	struct drm_mm_node *other;
+	/*
+	 * On some machines we have to be careful when putting differing types
+	 * of snoopable memory together to avoid the prefetcher crossing memory
-	/* On non-LLC machines we have to be careful when putting differing
+	 * domains and dying. During vm initialisation, we decide whether or not
-	 * types of snoopable memory together to avoid the prefetcher
+	 * these constraints apply and set the drm_mm.color_adjust
-	 * crossing memory domains and dying.
+	 * appropriately.
 	 */
-	if (HAS_LLC(dev))
+	if (vma->vm->mm.color_adjust == NULL)
 		return false;
 	return true;
-}
-static void i915_gem_verify_gtt(struct drm_device *dev)
-{
-#if WATCH_GTT
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_i915_gem_object *obj;
-	int err = 0;
-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
-		if (obj->gtt_space == NULL) {
-			printk(KERN_ERR "object found on GTT list with no space reserved\n");
-			err++;
-			continue;
-		}
-		if (obj->cache_level != obj->gtt_space->color) {
-			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
-			       i915_gem_obj_ggtt_offset(obj),
-			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
-			       obj->cache_level,
-			       obj->gtt_space->color);
-			err++;
-			continue;
-		}
-		if (!i915_gem_valid_gtt_space(dev,
-					      obj->gtt_space,
-					      obj->cache_level)) {
-			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
-			       i915_gem_obj_ggtt_offset(obj),
-			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
-			       obj->cache_level);
-			err++;
-			continue;
-		}
-	}
-	WARN_ON(err);
-#endif
+}
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
 						  DRM_MM_CREATE_DEFAULT);
 	if (ret) {
 		goto err_free_vma;
 	}
-	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
-					      obj->cache_level))) {
+	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
 		ret = -EINVAL;
 		goto err_remove_node;
+	}
 		goto err_remove_node;
 	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	list_add_tail(&vma->mm_list, &vm->inactive_list);
-	if (i915_is_ggtt(vm)) {
-		bool mappable, fenceable;
-		fenceable = (vma->node.size == fence_size &&
-			     (vma->node.start & (fence_alignment - 1)) == 0);
-		mappable = (vma->node.start + obj->base.size <=
-			    dev_priv->gtt.mappable_end);
-	obj->map_and_fenceable = mappable && fenceable;
-	}
-	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
+	list_add_tail(&vma->mm_list, &vm->inactive_list);
 	trace_i915_vma_bind(vma, flags);
-	vma->bind_vma(vma, obj->cache_level,
-		      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
+	vma->bind_vma(vma, obj->cache_level,
 		      flags & PIN_GLOBAL ? GLOBAL_BIND : 0);
 	i915_gem_verify_gtt(dev);
 	return vma;
 	/*
 	 * Stolen memory is always coherent with the GPU as it is explicitly
 	 * marked as wc by the system, or the system is cache-coherent.
 	 */
-	if (obj->stolen)
+	if (obj->stolen || obj->phys_handle)
 		return false;
 	/* If the GPU is snooping the contents of the CPU cache,
 	wmb();
 	old_write_domain = obj->base.write_domain;
+	obj->base.write_domain = 0;
-	obj->base.write_domain = 0;
+	intel_fb_obj_flush(obj, false);
 	trace_i915_gem_object_change_domain(obj,
 					    obj->base.read_domains,
 	i915_gem_chipset_flush(obj->base.dev);
 	old_write_domain = obj->base.write_domain;
+	obj->base.write_domain = 0;
-	obj->base.write_domain = 0;
+	intel_fb_obj_flush(obj, false);
 	trace_i915_gem_object_change_domain(obj,
 					    obj->base.read_domains,
  */
 int
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
+{
 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
 	uint32_t old_write_domain, old_read_domains;
 	int ret;
 	/* Not valid to be called on unbound objects. */
-	if (!i915_gem_obj_bound_any(obj))
+	if (vma == NULL)
 		return -EINVAL;
 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
 		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
 		obj->dirty = 1;
+	}
+	if (write)
+		intel_fb_obj_invalidate(obj, NULL);
 	trace_i915_gem_object_change_domain(obj,
 					    old_read_domains,
 					    old_write_domain);
-	/* And bump the LRU for this access */
-	if (i915_gem_object_is_inactive(obj)) {
-		struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
+	/* And bump the LRU for this access */
-		if (vma)
+	if (i915_gem_object_is_inactive(obj))
-			list_move_tail(&vma->mm_list,
-				       &dev_priv->gtt.base.inactive_list);
 			list_move_tail(&vma->mm_list,
 				       &dev_priv->gtt.base.inactive_list);
 	return 0;
 		DRM_DEBUG("can not change the cache level of pinned objects\n");
 		return -EBUSY;
+	}
 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
-		if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
+		if (!i915_gem_valid_gtt_space(vma, cache_level)) {
 			ret = i915_vma_unbind(vma);
 		if (ret)
 			return ret;
+		}
+            }
 		list_for_each_entry(vma, &obj->vma_list, vma_link)
 			if (drm_mm_node_allocated(&vma->node))
 				vma->bind_vma(vma, cache_level,
-					      obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
+						vma->bound & GLOBAL_BIND);
+	}
 	list_for_each_entry(vma, &obj->vma_list, vma_link)
 		trace_i915_gem_object_change_domain(obj,
 						    old_read_domains,
 						    old_write_domain);
+    }
 	i915_gem_verify_gtt(dev);
 	return 0;
+}
 static bool is_pin_display(struct drm_i915_gem_object *obj)
+{
-	struct i915_vma *vma;
-	if (list_empty(&obj->vma_list))
-		return false;
+	struct i915_vma *vma;
 	vma = i915_gem_obj_to_ggtt(obj);
 	if (!vma)
 	if (write) {
 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	}
+	if (write)
+		intel_fb_obj_invalidate(obj, NULL);
 	trace_i915_gem_object_change_domain(obj,
 					    old_read_domains,
 					    old_write_domain);
 	spin_unlock(&file_priv->mm.lock);
 	if (seqno == 0)
 		return 0;
-	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
+	ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
 	if (ret == 0)
 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 		    uint32_t alignment,
 		    uint64_t flags)
+{
 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 	struct i915_vma *vma;
+	unsigned bound;
 	int ret;
 	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
 		return -ENODEV;
+	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
+		return -EINVAL;
-	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
+	if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
 		return -EINVAL;
 	vma = i915_gem_obj_to_vma(obj, vm);
 			vma = NULL;
+		}
+	}
 	bound = vma ? vma->bound : 0;
 	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
 		vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
 		if (IS_ERR(vma))
 			return PTR_ERR(vma);
+	}
+	if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND))
+		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
+	if ((bound ^ vma->bound) & GLOBAL_BIND) {
+		bool mappable, fenceable;
+		u32 fence_size, fence_alignment;
+		fence_size = i915_gem_get_gtt_size(obj->base.dev,
+						   obj->base.size,
+						   obj->tiling_mode);
+		fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
+							     obj->base.size,
+							     obj->tiling_mode,
+							     true);
+		fenceable = (vma->node.size == fence_size &&
+			     (vma->node.start & (fence_alignment - 1)) == 0);
+		mappable = (vma->node.start + obj->base.size <=
+			    dev_priv->gtt.mappable_end);
+		obj->map_and_fenceable = mappable && fenceable;
+	}
 	if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
-		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
+	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
+{
 	struct drm_i915_gem_pin *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
-	if (INTEL_INFO(dev)->gen >= 6)
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
 		return -ENODEV;
 	ret = i915_mutex_lock_interruptible(dev);
+{
 	struct drm_i915_gem_pin *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
 		return ret;
 int
 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv)
+{
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_gem_madvise *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
 	if (i915_gem_obj_is_pinned(obj)) {
 		ret = -EINVAL;
 		goto out;
+	}
+	if (obj->pages &&
+	    obj->tiling_mode != I915_TILING_NONE &&
+	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
+		if (obj->madv == I915_MADV_WILLNEED)
+			i915_gem_object_unpin_pages(obj);
+		if (args->madv == I915_MADV_WILLNEED)
+			i915_gem_object_pin_pages(obj);
+	}
 	if (obj->madv != __I915_MADV_PURGED)
 		obj->madv = args->madv;
 	obj->ops = ops;
-	obj->fence_reg = I915_FENCE_REG_NONE;
-	obj->madv = I915_MADV_WILLNEED;
-	/* Avoid an unnecessary call to unbind on the first bind. */
+	obj->fence_reg = I915_FENCE_REG_NONE;
-	obj->map_and_fenceable = true;
+	obj->madv = I915_MADV_WILLNEED;
 	if (obj->stolen)
 		i915_gem_object_unpin_pages(obj);
+	WARN_ON(obj->frontbuffer_bits);
+	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
+	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
+	    obj->tiling_mode != I915_TILING_NONE)
-	WARN_ON(obj->frontbuffer_bits);
+		i915_gem_object_unpin_pages(obj);
 	if (WARN_ON(obj->pages_pin_count))
 	obj->pages_pin_count = 0;
 	return NULL;
+}
 void i915_gem_vma_destroy(struct i915_vma *vma)
+{
 	struct i915_address_space *vm = NULL;
 	WARN_ON(vma->node.allocated);
 	/* Keep the vma as a placeholder in the execbuffer reservation lists */
+	if (!list_empty(&vma->exec_list))
+		return;
+	vm = vma->vm;
-	if (!list_empty(&vma->exec_list))
+	if (!i915_is_ggtt(vm))
-		return;
+		i915_ppgtt_put(i915_vm_to_ppgtt(vm));
+{
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret = 0;
-	mutex_lock(&dev->struct_mutex);
-	if (dev_priv->ums.mm_suspended)
-		goto err;
 	mutex_lock(&dev->struct_mutex);
 	ret = i915_gpu_idle(dev);
 	if (ret)
 		goto err;
 	i915_gem_retire_requests(dev);
-	/* Under UMS, be paranoid and evict. */
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+	/* Under UMS, be paranoid and evict. */
-		i915_gem_evict_everything(dev);
-	i915_kernel_lost_context(dev);
-	i915_gem_stop_ringbuffers(dev);
-	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
-	 * We need to replace this with a semaphore, or something.
-	 * And not confound ums.mm_suspended!
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-	 */
+		i915_gem_evict_everything(dev);
 	dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
-							     DRIVER_MODESET);
+	i915_gem_stop_ringbuffers(dev);
+	}
 	return true;
+}
+static void init_unused_ring(struct drm_device *dev, u32 base)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	I915_WRITE(RING_CTL(base), 0);
+	I915_WRITE(RING_HEAD(base), 0);
+	I915_WRITE(RING_TAIL(base), 0);
+	I915_WRITE(RING_START(base), 0);
+}
+static void init_unused_rings(struct drm_device *dev)
+{
+	if (IS_I830(dev)) {
+		init_unused_ring(dev, PRB1_BASE);
+		init_unused_ring(dev, SRB0_BASE);
+		init_unused_ring(dev, SRB1_BASE);
+		init_unused_ring(dev, SRB2_BASE);
+		init_unused_ring(dev, SRB3_BASE);
+	} else if (IS_GEN2(dev)) {
+		init_unused_ring(dev, SRB0_BASE);
+		init_unused_ring(dev, SRB1_BASE);
+	} else if (IS_GEN3(dev)) {
+		init_unused_ring(dev, PRB1_BASE);
+		init_unused_ring(dev, PRB2_BASE);
+	}
 }
-static int i915_gem_init_rings(struct drm_device *dev)
+int i915_gem_init_rings(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int ret;
+	/*
+	 * At least 830 can leave some of the unused rings
+	 * "active" (ie. head != tail) after resume which
+	 * will prevent c3 entry. Makes sure all unused rings
+	 * are totally idle.
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	 */
-	int ret;
+	init_unused_rings(dev);
 	ret = intel_init_render_ring_buffer(dev);
+		}
+	}
 	i915_gem_init_swizzling(dev);
-	ret = i915_gem_init_rings(dev);
+	ret = dev_priv->gt.init_rings(dev);
 	if (ret)
 		return ret;
 	 */
 	ret = i915_gem_context_enable(dev_priv);
 	if (ret && ret != -EIO) {
 		DRM_ERROR("Context enable failed %d\n", ret);
 		i915_gem_cleanup_ringbuffer(dev);
+		return ret;
+	}
+	ret = i915_ppgtt_init_hw(dev);
+	if (ret && ret != -EIO) {
+		DRM_ERROR("PPGTT enable failed %d\n", ret);
+		i915_gem_cleanup_ringbuffer(dev);
+	}
 	return ret;
+}
 int i915_gem_init(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int ret;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	i915.enable_execlists = intel_sanitize_enable_execlists(dev,
-	int ret;
+			i915.enable_execlists);
 	mutex_lock(&dev->struct_mutex);
 	if (IS_VALLEYVIEW(dev)) {
 		/* VLVA0 (potential hack), BIOS isn't actually waking us */
 		I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
+		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
+			      VLV_GTLC_ALLOWWAKEACK), 10))
+			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
+	}
+	if (!i915.enable_execlists) {
+		dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission;
+		dev_priv->gt.init_rings = i915_gem_init_rings;
+		dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
+		dev_priv->gt.stop_ring = intel_stop_ring_buffer;
+	} else {
+		dev_priv->gt.do_execbuf = intel_execlists_submission;
+		dev_priv->gt.init_rings = intel_logical_rings_init;
+		dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
+		dev_priv->gt.stop_ring = intel_logical_ring_stop;
+	}
+//   ret = i915_gem_init_userptr(dev);
-		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
+//   if (ret) {
-			      VLV_GTLC_ALLOWWAKEACK), 10))
+//       mutex_unlock(&dev->struct_mutex);
-			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
+//       return ret;
 //   }
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_engine_cs *ring;
 	int i;
 	for_each_ring(ring, dev_priv, i)
-		intel_cleanup_ring_buffer(ring);
-}
-#if 0
-int
-i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
-		       struct drm_file *file_priv)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int ret;
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
-		return 0;
-	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
-		DRM_ERROR("Reenabling wedged hardware, good luck\n");
-		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
-	}
-	mutex_lock(&dev->struct_mutex);
-	dev_priv->ums.mm_suspended = 0;
-	ret = i915_gem_init_hw(dev);
-	if (ret != 0) {
-		mutex_unlock(&dev->struct_mutex);
-		return ret;
-	}
-	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
-	ret = drm_irq_install(dev, dev->pdev->irq);
-	if (ret)
-		goto cleanup_ringbuffer;
-	mutex_unlock(&dev->struct_mutex);
-	return 0;
-cleanup_ringbuffer:
-	i915_gem_cleanup_ringbuffer(dev);
-	dev_priv->ums.mm_suspended = 1;
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
+		dev_priv->gt.cleanup_ring(ring);
-}
-int
-i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
-		       struct drm_file *file_priv)
-{
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
-		return 0;
-	mutex_lock(&dev->struct_mutex);
-	drm_irq_uninstall(dev);
-	mutex_unlock(&dev->struct_mutex);
-	return i915_gem_suspend(dev);
-}
-void
-i915_gem_lastclose(struct drm_device *dev)
-{
-	int ret;
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
-		return;
-	ret = i915_gem_suspend(dev);
-	if (ret)
-		DRM_ERROR("failed to idle hardware: %d\n", ret);
-}
 #endif
 static void
 init_ring_lists(struct intel_engine_cs *ring)
+{
 	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
 			  i915_gem_idle_work_handler);
 	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
     /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
-    if (IS_GEN3(dev)) {
+	if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) {
 		I915_WRITE(MI_ARB_STATE,
 			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
+    }
+    dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
+	/* Old X drivers will take 0-2 for front, back, depth buffers */
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-    dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
+		dev_priv->fence_reg_start = 3;
 	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
 		dev_priv->num_fence_regs = 32;
 		kfree(file_priv);
 	return ret;
+}
+/**
+ * i915_gem_track_fb - update frontbuffer tracking
+ * old: current GEM buffer for the frontbuffer slots
+ * new: new GEM buffer for the frontbuffer slots
+ * frontbuffer_bits: bitmask of frontbuffer slots
+ *
+ * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
  * from @old and setting them in @new. Both @old and @new can be NULL.
  */
 void i915_gem_track_fb(struct drm_i915_gem_object *old,
 		       struct drm_i915_gem_object *new,
 		       unsigned frontbuffer_bits)
 				  struct i915_address_space *vm)
+{
 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
 	struct i915_vma *vma;
-	if (!dev_priv->mm.aliasing_ppgtt ||
-	    vm == &dev_priv->mm.aliasing_ppgtt->base)
-		vm = &dev_priv->gtt.base;
+	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
 	list_for_each_entry(vma, &o->vma_list, vma_link) {
 				struct i915_address_space *vm)
+{
 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
 	struct i915_vma *vma;
-	if (!dev_priv->mm.aliasing_ppgtt ||
-	    vm == &dev_priv->mm.aliasing_ppgtt->base)
-		vm = &dev_priv->gtt.base;
+	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
 	BUG_ON(list_empty(&o->vma_list));
 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
+{
-	struct i915_vma *vma;
-	/* This WARN has probably outlived its usefulness (callers already
-	 * WARN if they don't find the GGTT vma they expect). When removing,
-	 * remember to remove the pre-check in is_pin_display() as well */
-	if (WARN_ON(list_empty(&obj->vma_list)))
-		return NULL;
+	struct i915_vma *vma;
 	vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
-	if (vma->vm != obj_to_ggtt(obj))
+	if (vma->vm != i915_obj_to_ggtt(obj))
 		return NULL;

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 5060 → 5354