WebSVN – Kolibri OS – Diff – /drivers/video/drm/ttm/ttm_bo.c

 #include
 #include
 #include
 #include
 #include
+#include
 #include
-#define pr_err(fmt, ...) \
+#include
-        printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
+#include
 #define TTM_ASSERT_LOCKED(param)
 #define TTM_DEBUG(fmt, arg...)
 #define TTM_BO_HASH_ORDER 13
 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
-static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
+					  uint32_t *mem_type)
+{
 	int i;
 	for (i = 0; i <= TTM_PL_PRIV5; i++)
-		if (flags & (1 << i)) {
+		if (place->flags & (1 << i)) {
 	size_t acc_size = bo->acc_size;
 	BUG_ON(atomic_read(&bo->list_kref.refcount));
 	BUG_ON(atomic_read(&bo->kref.refcount));
-	BUG_ON(atomic_read(&bo->cpu_writers));
-	BUG_ON(bo->sync_obj != NULL);
+	BUG_ON(atomic_read(&bo->cpu_writers));
 	BUG_ON(bo->mem.mm_node != NULL);
 	BUG_ON(!list_empty(&bo->lru));
 	BUG_ON(!list_empty(&bo->ddestroy));
 	ttm_bo_mem_put(bo, &bo->mem);
 	ww_mutex_unlock (&bo->resv->lock);
+}
+static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
+{
+	struct reservation_object_list *fobj;
+	struct fence *fence;
+	int i;
+	fobj = reservation_object_get_list(bo->resv);
+	fence = reservation_object_get_excl(bo->resv);
+	if (fence && !fence->ops->signaled)
+		fence_enable_sw_signaling(fence);
+	for (i = 0; fobj && i < fobj->shared_count; ++i) {
+		fence = rcu_dereference_protected(fobj->shared[i],
+					reservation_object_held(bo->resv));
+		if (!fence->ops->signaled)
+			fence_enable_sw_signaling(fence);
+	}
+}
 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
+{
-	struct ttm_bo_device *bdev = bo->bdev;
-	struct ttm_bo_global *glob = bo->glob;
-	struct ttm_bo_driver *driver = bdev->driver;
+	struct ttm_bo_device *bdev = bo->bdev;
-	void *sync_obj = NULL;
+	struct ttm_bo_global *glob = bo->glob;
 	int put_count;
 	int ret;
 	spin_lock(&glob->lru_lock);
-	ret = __ttm_bo_reserve(bo, false, true, false, NULL);
-	spin_lock(&bdev->fence_lock);
+	ret = __ttm_bo_reserve(bo, false, true, false, NULL);
 	(void) ttm_bo_wait(bo, false, false, true);
-	if (!ret && !bo->sync_obj) {
+	if (!ret) {
-		spin_unlock(&bdev->fence_lock);
+		if (!ttm_bo_wait(bo, false, false, true)) {
 		put_count = ttm_bo_del_from_lru(bo);
-		spin_unlock(&glob->lru_lock);
-		ttm_bo_cleanup_memtype_use(bo);
+		spin_unlock(&glob->lru_lock);
-		ttm_bo_list_ref_sub(bo, put_count, true);
-		return;
+		ttm_bo_cleanup_memtype_use(bo);
-	if (bo->sync_obj)
+		ttm_bo_list_ref_sub(bo, put_count, true);
 		sync_obj = driver->sync_obj_ref(bo->sync_obj);
 	kref_get(&bo->list_kref);
 	list_add_tail(&bo->ddestroy, &bdev->ddestroy);
-	spin_unlock(&glob->lru_lock);
-	if (sync_obj) {
-		driver->sync_obj_flush(sync_obj);
-		driver->sync_obj_unref(&sync_obj);
+	spin_unlock(&glob->lru_lock);
 //	schedule_delayed_work(&bdev->wq,
 //			      ((HZ / 100) < 1) ? 1 : HZ / 100);
 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 					  bool interruptible,
 					  bool no_wait_gpu)
-{
-	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_bo_driver *driver = bdev->driver;
 	struct ttm_bo_global *glob = bo->glob;
 	int put_count;
-	int ret;
 	int ret;
 	spin_lock(&bdev->fence_lock);
 	ret = ttm_bo_wait(bo, false, false, true);
-	if (ret && !no_wait_gpu) {
-		void *sync_obj;
-		/*
-		 * Take a reference to the fence and unreserve,
-		 * at this point the buffer should be dead, so
-		 * no new sync objects can be attached.
-		 */
-		sync_obj = driver->sync_obj_ref(bo->sync_obj);
-		spin_unlock(&bdev->fence_lock);
+	if (ret && !no_wait_gpu) {
+		long lret;
-		__ttm_bo_unreserve(bo);
+		ww_mutex_unlock(&bo->resv->lock);
 		spin_unlock(&glob->lru_lock);
-		ret = driver->sync_obj_wait(sync_obj, false, interruptible);
+		lret = reservation_object_wait_timeout_rcu(bo->resv,
-		driver->sync_obj_unref(&sync_obj);
-		if (ret)
-			return ret;
-		/*
+							   true,
-		 * remove sync_obj with ttm_bo_wait, the wait should be
-		 * finished, and no new wait object should have been added.
+							   interruptible,
-		 */
-		spin_lock(&bdev->fence_lock);
+* HZ);
-		ret = ttm_bo_wait(bo, false, false, true);
-		WARN_ON(ret);
+		if (lret < 0)
-		spin_unlock(&bdev->fence_lock);
+			return lret;
-		if (ret)
+		else if (lret == 0)
 		 */
 		if (ret) {
 			spin_unlock(&glob->lru_lock);
 			return 0;
+		}
+		/*
+		 * remove sync_obj with ttm_bo_wait, the wait should be
+		 * finished, and no new wait object should have been added.
-	} else
+		 */
-		spin_unlock(&bdev->fence_lock);
+		ret = ttm_bo_wait(bo, false, false, true);
+		WARN_ON(ret);
+	}
 	if (ret || unlikely(list_empty(&bo->ddestroy))) {
 		__ttm_bo_unreserve(bo);
 		spin_unlock(&glob->lru_lock);
  * Repeatedly evict memory from the LRU for @mem_type until we create enough
  * space, or we've evicted everything and there isn't enough space.
  */
 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 					uint32_t mem_type,
-					struct ttm_placement *placement,
+					const struct ttm_place *place,
 					struct ttm_mem_reg *mem,
 					bool interruptible,
 					bool no_wait_gpu)
+{
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 	int ret;
 	do {
-		ret = (*man->func->get_node)(man, bo, placement, 0, mem);
+		ret = (*man->func->get_node)(man, bo, place, mem);
 		if (unlikely(ret != 0))
 			return ret;
 		if (mem->mm_node)
 			break;
 	return result;
+}
 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 				 uint32_t mem_type,
-				 uint32_t proposed_placement,
+				 const struct ttm_place *place,
 				 uint32_t *masked_placement)
+{
 	uint32_t cur_flags = ttm_bo_type_flags(mem_type);
-	if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
+	if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 		return false;
-	if ((proposed_placement & man->available_caching) == 0)
+	if ((place->flags & man->available_caching) == 0)
 		return false;
 	bool has_erestartsys = false;
 	int i, ret;
 	mem->mm_node = NULL;
 	for (i = 0; i < placement->num_placement; ++i) {
+		const struct ttm_place *place = &placement->placement[i];
-		ret = ttm_mem_type_from_flags(placement->placement[i],
-						&mem_type);
+		ret = ttm_mem_type_from_place(place, &mem_type);
 		if (ret)
 			return ret;
 		man = &bdev->man[mem_type];
-		type_ok = ttm_bo_mt_compatible(man,
 						mem_type,
-						placement->placement[i],
+		type_ok = ttm_bo_mt_compatible(man, mem_type, place,
 						&cur_flags);
 						  cur_flags);
 		/*
 		 * Use the access and other non-mapping-related flag bits from
 		 * the memory placement flags to the current flags
 		 */
-		ttm_flag_masked(&cur_flags, placement->placement[i],
+		ttm_flag_masked(&cur_flags, place->flags,
 				~TTM_PL_MASK_MEMTYPE);
 		if (mem_type == TTM_PL_SYSTEM)
 			break;
 		if (man->has_type && man->use_type) {
-			type_found = true;
-			ret = (*man->func->get_node)(man, bo, placement,
+			type_found = true;
-						     cur_flags, mem);
+			ret = (*man->func->get_node)(man, bo, place, mem);
 			if (unlikely(ret))
 				return ret;
+		}
 	if (!type_found)
 		return -EINVAL;
+	for (i = 0; i < placement->num_busy_placement; ++i) {
-	for (i = 0; i < placement->num_busy_placement; ++i) {
+		const struct ttm_place *place = &placement->busy_placement[i];
 		ret = ttm_mem_type_from_flags(placement->busy_placement[i],
-						&mem_type);
+		ret = ttm_mem_type_from_place(place, &mem_type);
 		if (ret)
 			return ret;
 		man = &bdev->man[mem_type];
 		if (!man->has_type)
-			continue;
-		if (!ttm_bo_mt_compatible(man,
-						mem_type,
-						placement->busy_placement[i],
+			continue;
-						&cur_flags))
+		if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
 			continue;
 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 						  cur_flags);
 		/*
 		 * Use the access and other non-mapping-related flag bits from
 		 * the memory placement flags to the current flags
 		 */
-		ttm_flag_masked(&cur_flags, placement->busy_placement[i],
+		ttm_flag_masked(&cur_flags, place->flags,
 				~TTM_PL_MASK_MEMTYPE);
 		if (mem_type == TTM_PL_SYSTEM) {
 			mem->mem_type = mem_type;
 			mem->placement = cur_flags;
 			mem->mm_node = NULL;
 			return 0;
+		}
-		ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
+		ret = ttm_bo_mem_force_space(bo, mem_type, place, mem,
 			bool interruptible,
 			bool no_wait_gpu)
+{
 	int ret = 0;
 	struct ttm_mem_reg mem;
-	struct ttm_bo_device *bdev = bo->bdev;
 	lockdep_assert_held(&bo->resv->lock.base);
 	/*
 	 * FIXME: It's possible to pipeline buffer moves.
 	 * Have the driver move function wait for idle when necessary,
-	 * instead of doing it here.
-	 */
+	 * instead of doing it here.
-	spin_lock(&bdev->fence_lock);
-	ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
+	 */
-	spin_unlock(&bdev->fence_lock);
+	ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
 	if (ret)
 		return ret;
 	mem.num_pages = bo->num_pages;
 			      struct ttm_mem_reg *mem,
 			      uint32_t *new_flags)
+{
 	int i;
-	if (mem->mm_node && placement->lpfn != 0 &&
-	    (mem->start < placement->fpfn ||
-	     mem->start + mem->num_pages > placement->lpfn))
-		return false;
 	for (i = 0; i < placement->num_placement; i++) {
+		const struct ttm_place *heap = &placement->placement[i];
+		if (mem->mm_node &&
+		    (mem->start < heap->fpfn ||
+		     (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
+			continue;
-		*new_flags = placement->placement[i];
+		*new_flags = heap->flags;
 		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
 		    (*new_flags & mem->placement & TTM_PL_MASK_MEM))
 			return true;
+	}
+	for (i = 0; i < placement->num_busy_placement; i++) {
+		const struct ttm_place *heap = &placement->busy_placement[i];
+		if (mem->mm_node &&
+		    (mem->start < heap->fpfn ||
+		     (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
+			continue;
 	for (i = 0; i < placement->num_busy_placement; i++) {
-		*new_flags = placement->busy_placement[i];
+		*new_flags = heap->flags;
 		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
 		    (*new_flags & mem->placement & TTM_PL_MASK_MEM))
+{
 	int ret;
 	uint32_t new_flags;
-	lockdep_assert_held(&bo->resv->lock.base);
-	/* Check that range is valid */
-	if (placement->lpfn || placement->fpfn)
-		if (placement->fpfn > placement->lpfn ||
-			(placement->lpfn - placement->fpfn) < bo->num_pages)
-			return -EINVAL;
+	lockdep_assert_held(&bo->resv->lock.base);
 	/*
 	 * Check whether we need to move buffer.
 	 */
 	if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
+	}
 	return 0;
+}
 EXPORT_SYMBOL(ttm_bo_validate);
-int ttm_bo_check_placement(struct ttm_buffer_object *bo,
-				struct ttm_placement *placement)
-{
-	BUG_ON((placement->fpfn || placement->lpfn) &&
-	       (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
-	return 0;
-}
 int ttm_bo_init(struct ttm_bo_device *bdev,
-		struct ttm_buffer_object *bo,
+        struct ttm_buffer_object *bo,
-		unsigned long size,
+        unsigned long size,
-		enum ttm_bo_type type,
+        enum ttm_bo_type type,
-		struct ttm_placement *placement,
+        struct ttm_placement *placement,
-		uint32_t page_alignment,
+        uint32_t page_alignment,
-		bool interruptible,
+        bool interruptible,
-		struct file *persistent_swap_storage,
+        struct file *persistent_swap_storage,
-		size_t acc_size,
+        size_t acc_size,
+        struct sg_table *sg,
-		struct sg_table *sg,
+		struct reservation_object *resv,
-		void (*destroy) (struct ttm_buffer_object *))
+        void (*destroy) (struct ttm_buffer_object *))
+{
-	int ret = 0;
+    int ret = 0;
-	unsigned long num_pages;
+    unsigned long num_pages;
-	bo->priv_flags = 0;
+    bo->priv_flags = 0;
-	bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
+    bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
-	bo->persistent_swap_storage = persistent_swap_storage;
+    bo->persistent_swap_storage = persistent_swap_storage;
-	bo->acc_size = acc_size;
+    bo->acc_size = acc_size;
-	bo->sg = sg;
+    bo->sg = sg;
+	if (resv) {
+		bo->resv = resv;
+		lockdep_assert_held(&bo->resv->lock.base);
+	} else {
-	bo->resv = &bo->ttm_resv;
+		bo->resv = &bo->ttm_resv;
-	reservation_object_init(bo->resv);
+		reservation_object_init(&bo->ttm_resv);
+	}
 	atomic_inc(&bo->glob->bo_count);
-	drm_vma_node_reset(&bo->vma_node);
+    drm_vma_node_reset(&bo->vma_node);
-	ret = ttm_bo_check_placement(bo, placement);
-	/*
+    /*
-	 * For ttm_bo_type_device buffers, allocate
+     * For ttm_bo_type_device buffers, allocate
-	 * address space from the device.
+     * address space from the device.
-	 */
-	if (likely(!ret) &&
+     */
-	    (bo->type == ttm_bo_type_device ||
+	if (bo->type == ttm_bo_type_device ||
-	     bo->type == ttm_bo_type_sg))
+	    bo->type == ttm_bo_type_sg)
 		ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
+					 bo->mem.num_pages);
+	/* passed reservation objects should already be locked,
+	 * since otherwise lockdep will be angered in radeon.
-					 bo->mem.num_pages);
+	 */
 	if (!resv) {
+		locked = ww_mutex_trylock(&bo->resv->lock);
-	locked = ww_mutex_trylock(&bo->resv->lock);
+		WARN_ON(!locked);
     WARN_ON(!locked);
 	if (likely(!ret))
-	if (likely(!ret))
+		ret = ttm_bo_validate(bo, placement, interruptible, false);
 		ret = ttm_bo_validate(bo, placement, interruptible, false);
 	INIT_LIST_HEAD(&bdev->ddestroy);
 	bdev->dev_mapping = mapping;
 	bdev->glob = glob;
 	bdev->need_dma32 = need_dma32;
 	bdev->val_seq = 0;
-	spin_lock_init(&bdev->fence_lock);
 	mutex_lock(&glob->device_list_mutex);
 	list_add_tail(&bdev->device_list, &glob->device_list);
 	mutex_unlock(&glob->device_list_mutex);
+}
 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 int ttm_bo_wait(struct ttm_buffer_object *bo,
 		bool lazy, bool interruptible, bool no_wait)
 {
-	struct ttm_bo_driver *driver = bo->bdev->driver;
+	struct reservation_object_list *fobj;
-	struct ttm_bo_device *bdev = bo->bdev;
-	void *sync_obj;
-	int ret = 0;
+	struct reservation_object *resv;
+	struct fence *excl;
+	long timeout = 15 * HZ;
+	int i;
+	resv = bo->resv;
+	fobj = reservation_object_get_list(resv);
+	excl = reservation_object_get_excl(resv);
-	if (likely(bo->sync_obj == NULL))
-		return 0;
-	while (bo->sync_obj) {
-		if (driver->sync_obj_signaled(bo->sync_obj)) {
+	if (excl) {
-			void *tmp_obj = bo->sync_obj;
+		if (!fence_is_signaled(excl)) {
-			bo->sync_obj = NULL;
+			if (no_wait)
-			clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
+				return -EBUSY;
+			timeout = fence_wait_timeout(excl,
+						     interruptible, timeout);
+		}
+	}
-			spin_unlock(&bdev->fence_lock);
+	for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) {
-			driver->sync_obj_unref(&tmp_obj);
+		struct fence *fence;
-			spin_lock(&bdev->fence_lock);
-			continue;
-		}
+		fence = rcu_dereference_protected(fobj->shared[i],
+						reservation_object_held(resv));
-		if (no_wait)
-			return -EBUSY;
-		sync_obj = driver->sync_obj_ref(bo->sync_obj);
-		spin_unlock(&bdev->fence_lock);
-		ret = driver->sync_obj_wait(sync_obj,
-					    lazy, interruptible);
-		if (unlikely(ret != 0)) {
-			driver->sync_obj_unref(&sync_obj);
-			spin_lock(&bdev->fence_lock);
-	return ret;
-		}
-		spin_lock(&bdev->fence_lock);
-		if (likely(bo->sync_obj == sync_obj)) {
-			void *tmp_obj = bo->sync_obj;
-			bo->sync_obj = NULL;
-			clear_bit(TTM_BO_PRIV_FLAG_MOVING,
-				  &bo->priv_flags);
-			spin_unlock(&bdev->fence_lock);
 			driver->sync_obj_unref(&sync_obj);
-			driver->sync_obj_unref(&tmp_obj);
+		if (!fence_is_signaled(fence)) {
+			if (no_wait)
+				return -EBUSY;
+			timeout = fence_wait_timeout(fence,
+						     interruptible, timeout);
+		}
+		}
+	if (timeout < 0)
-			spin_lock(&bdev->fence_lock);
+		return timeout;
 		} else {
-			spin_unlock(&bdev->fence_lock);
+	if (timeout == 0)
+			return -EBUSY;

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/ttm/ttm_bo.c – Rev 5078 → 5271