WebSVN – Kolibri OS – Diff – /drivers/video/drm/radeon/radeon_vm.c

  * @head: head of validation list
+ *
  * Add the page directory to the list of BOs to
  * validate for command submission (cayman+).
  */
-struct radeon_cs_reloc *radeon_vm_get_bos(struct radeon_device *rdev,
+struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,
 					  struct radeon_vm *vm,
 					  struct list_head *head)
+{
-	struct radeon_cs_reloc *list;
+	struct radeon_bo_list *list;
 	unsigned i, idx;
 	list = kmalloc_array(vm->max_pde_used + 2,
-			     sizeof(struct radeon_cs_reloc), GFP_KERNEL);
+			     sizeof(struct radeon_bo_list), GFP_KERNEL);
 	if (!list)
 		return NULL;
 	/* add the vm page table to the list */
-	list[0].gobj = NULL;
+	/* add the vm page table to the list */
 	list[0].robj = vm->page_directory;
 	list[0].prefered_domains = RADEON_GEM_DOMAIN_VRAM;
+	list[0].allowed_domains = RADEON_GEM_DOMAIN_VRAM;
-	list[0].allowed_domains = RADEON_GEM_DOMAIN_VRAM;
+	list[0].tv.bo = &vm->page_directory->tbo;
-	list[0].tv.bo = &vm->page_directory->tbo;
-	list[0].tiling_flags = 0;
+	list[0].tv.shared = true;
-	list[0].handle = 0;
+	list[0].tiling_flags = 0;
 	list_add(&list[0].tv.head, head);
-	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {
-		if (!vm->page_tables[i].bo)
+	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {
-			continue;
+		if (!vm->page_tables[i].bo)
 			continue;
 		list[idx].gobj = NULL;
+		list[idx].robj = vm->page_tables[i].bo;
-		list[idx].robj = vm->page_tables[i].bo;
+		list[idx].prefered_domains = RADEON_GEM_DOMAIN_VRAM;
-		list[idx].prefered_domains = RADEON_GEM_DOMAIN_VRAM;
 		list[idx].allowed_domains = RADEON_GEM_DOMAIN_VRAM;
 		list[idx].tv.bo = &list[idx].robj->tbo;
-		list[idx].tiling_flags = 0;
+		list[idx].tv.shared = true;
-		list[idx].handle = 0;
+		list[idx].tiling_flags = 0;
  */
 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
 				       struct radeon_vm *vm, int ring)
+{
 	struct radeon_fence *best[RADEON_NUM_RINGS] = {};
+	struct radeon_vm_id *vm_id = &vm->ids[ring];
 	unsigned choices[2] = {};
 	unsigned i;
+	/* check if the id is still valid */
-	/* check if the id is still valid */
+	if (vm_id->id && vm_id->last_id_use &&
-	if (vm->last_id_use && vm->last_id_use == rdev->vm_manager.active[vm->id])
+	    vm_id->last_id_use == rdev->vm_manager.active[vm_id->id])
 		return NULL;
 	/* we definately need to flush */
-	radeon_fence_unref(&vm->last_flush);
+	vm_id->pd_gpu_addr = ~0ll;
 	/* skip over VMID 0, since it is the system VM */
 	for (i = 1; i < rdev->vm_manager.nvm; ++i) {
 		struct radeon_fence *fence = rdev->vm_manager.active[i];
 		if (fence == NULL) {
 			/* found a free one */
-			vm->id = i;
+			vm_id->id = i;
-			trace_radeon_vm_grab_id(vm->id, ring);
+			trace_radeon_vm_grab_id(i, ring);
+		}
+	}
 	for (i = 0; i < 2; ++i) {
 		if (choices[i]) {
-			vm->id = choices[i];
+			vm_id->id = choices[i];
-			trace_radeon_vm_grab_id(vm->id, ring);
+			trace_radeon_vm_grab_id(choices[i], ring);
 			return rdev->vm_manager.active[choices[i]];
+		}
+	}
  * radeon_vm_flush - hardware flush the vm
+ *
  * @rdev: radeon_device pointer
  * @vm: vm we want to flush
  * @ring: ring to use for flush
+ * @updates: last vm update that is waited for
+ *
  * Flush the vm (cayman+).
+ *
  * Global and local mutex must be locked!
  */
 void radeon_vm_flush(struct radeon_device *rdev,
 		     struct radeon_vm *vm,
-		     int ring)
+		     int ring, struct radeon_fence *updates)
+{
 	uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory);
+	struct radeon_vm_id *vm_id = &vm->ids[ring];
+	if (pd_addr != vm_id->pd_gpu_addr || !vm_id->flushed_updates ||
+	    radeon_fence_is_earlier(vm_id->flushed_updates, updates)) {
+		trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id);
+		radeon_fence_unref(&vm_id->flushed_updates);
+		vm_id->flushed_updates = radeon_fence_ref(updates);
+		vm_id->pd_gpu_addr = pd_addr;
+		radeon_ring_vm_flush(rdev, &rdev->ring[ring],
+				     vm_id->id, vm_id->pd_gpu_addr);
-	/* if we can't remember our last VM flush then flush now! */
-	if (!vm->last_flush || pd_addr != vm->pd_gpu_addr) {
-		trace_radeon_vm_flush(pd_addr, ring, vm->id);
-		vm->pd_gpu_addr = pd_addr;
 		radeon_ring_vm_flush(rdev, ring, vm);
+	}
+}
  */
 void radeon_vm_fence(struct radeon_device *rdev,
 		     struct radeon_vm *vm,
 		     struct radeon_fence *fence)
+{
-	radeon_fence_unref(&vm->fence);
-	vm->fence = radeon_fence_ref(fence);
+	unsigned vm_id = vm->ids[fence->ring].id;
-	radeon_fence_unref(&rdev->vm_manager.active[vm->id]);
-	rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence);
-	radeon_fence_unref(&vm->last_id_use);
+	radeon_fence_unref(&rdev->vm_manager.active[vm_id]);
-	vm->last_id_use = radeon_fence_ref(fence);
+	rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence);
-        /* we just flushed the VM, remember that */
-        if (!vm->last_flush)
+	radeon_fence_unref(&vm->ids[fence->ring].last_id_use);
-                vm->last_flush = radeon_fence_ref(fence);
+	vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence);
+}
  * @bo: bo to clear
  */
 static int radeon_vm_clear_bo(struct radeon_device *rdev,
 			      struct radeon_bo *bo)
+{
-        struct ttm_validate_buffer tv;
-        struct ww_acquire_ctx ticket;
-        struct list_head head;
 	struct radeon_ib ib;
 	unsigned entries;
 	uint64_t addr;
 	int r;
-        memset(&tv, 0, sizeof(tv));
-        tv.bo = &bo->tbo;
-        INIT_LIST_HEAD(&head);
-        list_add(&tv.head, &head);
-        r = ttm_eu_reserve_buffers(&ticket, &head);
+	r = radeon_bo_reserve(bo, false);
         if (r)
 		return r;
         r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
         if (r)
-                goto error;
+		goto error_unreserve;
 	addr = radeon_bo_gpu_offset(bo);
 	entries = radeon_bo_size(bo) / 8;
 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, 256);
 	if (r)
-                goto error;
+		goto error_unreserve;
 	ib.length_dw = 0;
 	radeon_vm_set_pages(rdev, &ib, addr, 0, entries, 0, 0);
 	radeon_asic_vm_pad_ib(rdev, &ib);
 	WARN_ON(ib.length_dw > 64);
+	r = radeon_ib_schedule(rdev, &ib, NULL, false);
-	r = radeon_ib_schedule(rdev, &ib, NULL, false);
+	if (r)
-	if (r)
+		goto error_free;
                 goto error;
 	ib.fence->is_vm_update = true;
-	ttm_eu_fence_buffer_objects(&ticket, &head, ib.fence);
+	radeon_bo_fence(bo, ib.fence, false);
+ *
  * Set offset of @bo_va (cayman+).
  * Validate and set the offset requested within the vm address space.
  * Returns 0 for success, error for failure.
+ *
- * Object has to be reserved!
+ * Object has to be reserved and gets unreserved by this function!
  */
 int radeon_vm_bo_set_addr(struct radeon_device *rdev,
 			  struct radeon_bo_va *bo_va,
 			  uint64_t soffset,
 			  uint32_t flags)
 			tmp->it.start = bo_va->it.start;
 			tmp->it.last = bo_va->it.last;
 			tmp->vm = vm;
 			tmp->addr = bo_va->addr;
 			tmp->bo = radeon_bo_ref(bo_va->bo);
+			spin_lock(&vm->status_lock);
 			list_add(&tmp->vm_status, &vm->freed);
+			spin_unlock(&vm->status_lock);
+		}
 		interval_tree_remove(&bo_va->it, &vm->va);
 		bo_va->it.start = 0;
 		/* drop mutex to allocate and clear page table */
 		mutex_unlock(&vm->mutex);
 		r = radeon_bo_create(rdev, RADEON_VM_PTE_COUNT * 8,
 				     RADEON_GPU_PAGE_SIZE, true,
+				     RADEON_GEM_DOMAIN_VRAM, 0,
-				     RADEON_GEM_DOMAIN_VRAM, 0, NULL, &pt);
+				     NULL, NULL, &pt);
 		if (r)
 			return r;
 		vm->page_tables[pt_idx].addr = 0;
 		vm->page_tables[pt_idx].bo = pt;
+	}
 	mutex_unlock(&vm->mutex);
-	return radeon_bo_reserve(bo_va->bo, false);
+	return 0;
+}
 /**
 		radeon_vm_set_pages(rdev, &ib, last_pde, last_pt, count,
 					incr, R600_PTE_VALID);
 	if (ib.length_dw != 0) {
+		radeon_asic_vm_pad_ib(rdev, &ib);
-		radeon_asic_vm_pad_ib(rdev, &ib);
-		radeon_semaphore_sync_to(ib.semaphore, pd->tbo.sync_obj);
-		radeon_semaphore_sync_to(ib.semaphore, vm->last_id_use);
+		radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, true);
 		WARN_ON(ib.length_dw > ndw);
 		r = radeon_ib_schedule(rdev, &ib, NULL, false);
 		if (r) {
 			radeon_ib_free(rdev, &ib);
 			return r;
-		}
-		radeon_fence_unref(&vm->fence);
+		}
-		vm->fence = radeon_fence_ref(ib.fence);
+		ib.fence->is_vm_update = true;
-		radeon_fence_unref(&vm->last_flush);
+		radeon_bo_fence(pd, ib.fence, false);
+	}
 	radeon_ib_free(rdev, &ib);
+ *
  * Update the page tables in the range @start - @end (cayman+).
+ *
  * Global and local mutex must be locked!
  */
-static void radeon_vm_update_ptes(struct radeon_device *rdev,
+static int radeon_vm_update_ptes(struct radeon_device *rdev,
 				  struct radeon_vm *vm,
 				  struct radeon_ib *ib,
 				  uint64_t start, uint64_t end,
 				  uint64_t dst, uint32_t flags)
+{
 	for (addr = start; addr < end; ) {
 		uint64_t pt_idx = addr >> radeon_vm_block_size;
 		struct radeon_bo *pt = vm->page_tables[pt_idx].bo;
 		unsigned nptes;
 		uint64_t pte;
+		int r;
+		radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, true);
+		r = reservation_object_reserve_shared(pt->tbo.resv);
+		if (r)
-		radeon_semaphore_sync_to(ib->semaphore, pt->tbo.sync_obj);
+			return r;
 		if ((addr & ~mask) == (end & ~mask))
 			nptes = end - addr;
 	if (count) {
 		radeon_vm_frag_ptes(rdev, ib, last_pte,
 				    last_pte + 8 * count,
 				    last_dst, flags);
+	}
+	return 0;
+}
+/**
+ * radeon_vm_fence_pts - fence page tables after an update
+ *
+ * @vm: requested vm
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ * @fence: fence to use
+ *
+ * Fence the page tables in the range @start - @end (cayman+).
+ *
+ * Global and local mutex must be locked!
+ */
+static void radeon_vm_fence_pts(struct radeon_vm *vm,
+				uint64_t start, uint64_t end,
+				struct radeon_fence *fence)
+{
+	unsigned i;
+	start >>= radeon_vm_block_size;
+	end >>= radeon_vm_block_size;
+	for (i = start; i <= end; ++i)
+		radeon_bo_fence(vm->page_tables[i].bo, fence, true);
+}
 /**
  * radeon_vm_bo_update - map a bo into the vm page table
 		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
 			bo_va->bo, vm);
 		return -EINVAL;
+	}
 	spin_lock(&vm->status_lock);
+	list_del_init(&bo_va->vm_status);
-	list_del_init(&bo_va->vm_status);
+	spin_unlock(&vm->status_lock);
 	bo_va->flags &= ~RADEON_VM_PAGE_VALID;
+	bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
+	bo_va->flags &= ~RADEON_VM_PAGE_SNOOPED;
+//   if (bo_va->bo && radeon_ttm_tt_is_readonly(bo_va->bo->tbo.ttm))
-	bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
+//       bo_va->flags &= ~RADEON_VM_PAGE_WRITEABLE;
 	bo_va->flags &= ~RADEON_VM_PAGE_SNOOPED;
 	if (mem) {
 		addr = mem->start << PAGE_SHIFT;
 		if (mem->mem_type != TTM_PL_SYSTEM) {
 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
 	if (r)
 		return r;
 	ib.length_dw = 0;
+	if (!(bo_va->flags & RADEON_VM_PAGE_VALID)) {
+		unsigned i;
+		for (i = 0; i < RADEON_NUM_RINGS; ++i)
+			radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use);
+	}
-	radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,
+	r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,
 			      bo_va->it.last + 1, addr,
+			      radeon_vm_page_flags(bo_va->flags));
+	if (r) {
+		radeon_ib_free(rdev, &ib);
+		return r;
 			      radeon_vm_page_flags(bo_va->flags));
-	radeon_asic_vm_pad_ib(rdev, &ib);
-	WARN_ON(ib.length_dw > ndw);
+	radeon_asic_vm_pad_ib(rdev, &ib);
 	WARN_ON(ib.length_dw > ndw);
 	radeon_semaphore_sync_to(ib.semaphore, vm->fence);
 	r = radeon_ib_schedule(rdev, &ib, NULL, false);
 	if (r) {
+		radeon_ib_free(rdev, &ib);
+		return r;
-		radeon_ib_free(rdev, &ib);
+	}
-		return r;
+	ib.fence->is_vm_update = true;
 	radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence);
-	radeon_fence_unref(&vm->fence);
-	vm->fence = radeon_fence_ref(ib.fence);
+	radeon_fence_unref(&bo_va->last_pt_update);
-	radeon_ib_free(rdev, &ib);
+	bo_va->last_pt_update = radeon_fence_ref(ib.fence);
-	radeon_fence_unref(&vm->last_flush);
+	radeon_ib_free(rdev, &ib);
  * PTs have to be reserved and mutex must be locked!
  */
 int radeon_vm_clear_freed(struct radeon_device *rdev,
 			  struct radeon_vm *vm)
+{
-	struct radeon_bo_va *bo_va, *tmp;
+	struct radeon_bo_va *bo_va;
 	int r;
+	spin_lock(&vm->status_lock);
+	while (!list_empty(&vm->freed)) {
+		bo_va = list_first_entry(&vm->freed,
+			struct radeon_bo_va, vm_status);
+		spin_unlock(&vm->status_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) {
 		r = radeon_vm_bo_update(rdev, bo_va, NULL);
+		radeon_bo_unref(&bo_va->bo);
-		radeon_bo_unref(&bo_va->bo);
+		radeon_fence_unref(&bo_va->last_pt_update);
 		kfree(bo_va);
 		if (r)
+			return r;
-			return r;
+		spin_lock(&vm->status_lock);
+	}
 	spin_unlock(&vm->status_lock);
 	return 0;
  * PTs have to be reserved and mutex must be locked!
  */
 int radeon_vm_clear_invalids(struct radeon_device *rdev,
 			     struct radeon_vm *vm)
+{
-	struct radeon_bo_va *bo_va, *tmp;
+	struct radeon_bo_va *bo_va;
 	int r;
+	spin_lock(&vm->status_lock);
+	while (!list_empty(&vm->invalidated)) {
+		bo_va = list_first_entry(&vm->invalidated,
+			struct radeon_bo_va, vm_status);
+		spin_unlock(&vm->status_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, vm_status) {
 		r = radeon_vm_bo_update(rdev, bo_va, NULL);
 		if (r)
+			return r;
-			return r;
+		spin_lock(&vm->status_lock);
+	}
+	spin_unlock(&vm->status_lock);
 	return 0;
+}
 	list_del(&bo_va->bo_list);
+	mutex_lock(&vm->mutex);
-	mutex_lock(&vm->mutex);
+	interval_tree_remove(&bo_va->it, &vm->va);
-	interval_tree_remove(&bo_va->it, &vm->va);
+	spin_lock(&vm->status_lock);
 	list_del(&bo_va->vm_status);
 	if (bo_va->addr) {
+		bo_va->bo = radeon_bo_ref(bo_va->bo);
-		bo_va->bo = radeon_bo_ref(bo_va->bo);
+		list_add(&bo_va->vm_status, &vm->freed);
-		list_add(&bo_va->vm_status, &vm->freed);
+	} else {
+		radeon_fence_unref(&bo_va->last_pt_update);
-	} else {
+	kfree(bo_va);
 	kfree(bo_va);
 	spin_unlock(&vm->status_lock);
+{
 	struct radeon_bo_va *bo_va;
 	list_for_each_entry(bo_va, &bo->va, bo_list) {
 		if (bo_va->addr) {
-			mutex_lock(&bo_va->vm->mutex);
+			spin_lock(&bo_va->vm->status_lock);
 			list_del(&bo_va->vm_status);
 			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
-			mutex_unlock(&bo_va->vm->mutex);
+			spin_unlock(&bo_va->vm->status_lock);
+		}
+	}
+}
 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
+{
 	const unsigned align = min(RADEON_VM_PTB_ALIGN_SIZE,
 		RADEON_VM_PTE_COUNT * 8);
 	unsigned pd_size, pd_entries, pts_size;
-	int r;
+	int i, r;
 	vm->id = 0;
+	vm->ib_bo_va = NULL;
-	vm->ib_bo_va = NULL;
+	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-	vm->fence = NULL;
+		vm->ids[i].id = 0;
-	vm->last_flush = NULL;
+		vm->ids[i].flushed_updates = NULL;
+		vm->ids[i].last_id_use = NULL;
 	vm->last_id_use = NULL;
 	mutex_init(&vm->mutex);
+	vm->va = RB_ROOT;
-	vm->va = RB_ROOT;
+	spin_lock_init(&vm->status_lock);
 	INIT_LIST_HEAD(&vm->invalidated);
 	INIT_LIST_HEAD(&vm->freed);
 		return -ENOMEM;
+	}
 	r = radeon_bo_create(rdev, pd_size, align, true,
 			     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
-			     &vm->page_directory);
+			     NULL, &vm->page_directory);
 	if (r)
 		return r;
 		interval_tree_remove(&bo_va->it, &vm->va);
 		r = radeon_bo_reserve(bo_va->bo, false);
 		if (!r) {
 			list_del_init(&bo_va->bo_list);
 			radeon_bo_unreserve(bo_va->bo);
+			radeon_fence_unref(&bo_va->last_pt_update);
 			kfree(bo_va);
+		}
+	}
 	list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) {
 		radeon_bo_unref(&bo_va->bo);
+		radeon_fence_unref(&bo_va->last_pt_update);
 		kfree(bo_va);
+	}
 	for (i = 0; i < radeon_vm_num_pdes(rdev); i++)
 		radeon_bo_unref(&vm->page_tables[i].bo);
 	kfree(vm->page_tables);
 	radeon_bo_unref(&vm->page_directory);
+	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-	radeon_fence_unref(&vm->fence);
+		radeon_fence_unref(&vm->ids[i].flushed_updates);
-	radeon_fence_unref(&vm->last_flush);
+		radeon_fence_unref(&vm->ids[i].last_id_use);

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/radeon_vm.c – Rev 5078 → 5271