331,7 → 331,6 |
bo_va->it.start = 0; |
bo_va->it.last = 0; |
bo_va->flags = 0; |
bo_va->addr = 0; |
bo_va->ref_count = 1; |
INIT_LIST_HEAD(&bo_va->bo_list); |
INIT_LIST_HEAD(&bo_va->vm_status); |
458,7 → 457,8 |
/* make sure object fit at this offset */ |
eoffset = soffset + size; |
if (soffset >= eoffset) { |
return -EINVAL; |
r = -EINVAL; |
goto error_unreserve; |
} |
|
last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; |
465,7 → 465,8 |
if (last_pfn > rdev->vm_manager.max_pfn) { |
dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", |
last_pfn, rdev->vm_manager.max_pfn); |
return -EINVAL; |
r = -EINVAL; |
goto error_unreserve; |
} |
|
} else { |
473,52 → 474,57 |
} |
|
mutex_lock(&vm->mutex); |
soffset /= RADEON_GPU_PAGE_SIZE; |
eoffset /= RADEON_GPU_PAGE_SIZE; |
if (soffset || eoffset) { |
struct interval_tree_node *it; |
it = interval_tree_iter_first(&vm->va, soffset, eoffset - 1); |
if (it && it != &bo_va->it) { |
struct radeon_bo_va *tmp; |
tmp = container_of(it, struct radeon_bo_va, it); |
/* bo and tmp overlap, invalid offset */ |
dev_err(rdev->dev, "bo %p va 0x%010Lx conflict with " |
"(bo %p 0x%010lx 0x%010lx)\n", bo_va->bo, |
soffset, tmp->bo, tmp->it.start, tmp->it.last); |
mutex_unlock(&vm->mutex); |
r = -EINVAL; |
goto error_unreserve; |
} |
} |
|
if (bo_va->it.start || bo_va->it.last) { |
if (bo_va->addr) { |
/* add a clone of the bo_va to clear the old address */ |
struct radeon_bo_va *tmp; |
tmp = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); |
if (!tmp) { |
mutex_unlock(&vm->mutex); |
return -ENOMEM; |
r = -ENOMEM; |
goto error_unreserve; |
} |
tmp->it.start = bo_va->it.start; |
tmp->it.last = bo_va->it.last; |
tmp->vm = vm; |
tmp->addr = bo_va->addr; |
tmp->bo = radeon_bo_ref(bo_va->bo); |
spin_lock(&vm->status_lock); |
list_add(&tmp->vm_status, &vm->freed); |
spin_unlock(&vm->status_lock); |
} |
|
interval_tree_remove(&bo_va->it, &vm->va); |
spin_lock(&vm->status_lock); |
bo_va->it.start = 0; |
bo_va->it.last = 0; |
list_del_init(&bo_va->vm_status); |
list_add(&tmp->vm_status, &vm->freed); |
spin_unlock(&vm->status_lock); |
} |
|
soffset /= RADEON_GPU_PAGE_SIZE; |
eoffset /= RADEON_GPU_PAGE_SIZE; |
if (soffset || eoffset) { |
struct interval_tree_node *it; |
it = interval_tree_iter_first(&vm->va, soffset, eoffset - 1); |
if (it) { |
struct radeon_bo_va *tmp; |
tmp = container_of(it, struct radeon_bo_va, it); |
/* bo and tmp overlap, invalid offset */ |
dev_err(rdev->dev, "bo %p va 0x%010Lx conflict with " |
"(bo %p 0x%010lx 0x%010lx)\n", bo_va->bo, |
soffset, tmp->bo, tmp->it.start, tmp->it.last); |
mutex_unlock(&vm->mutex); |
return -EINVAL; |
} |
spin_lock(&vm->status_lock); |
bo_va->it.start = soffset; |
bo_va->it.last = eoffset - 1; |
list_add(&bo_va->vm_status, &vm->cleared); |
spin_unlock(&vm->status_lock); |
interval_tree_insert(&bo_va->it, &vm->va); |
} |
|
bo_va->flags = flags; |
bo_va->addr = 0; |
|
soffset >>= radeon_vm_block_size; |
eoffset >>= radeon_vm_block_size; |
550,7 → 556,6 |
r = radeon_vm_clear_bo(rdev, pt); |
if (r) { |
radeon_bo_unref(&pt); |
radeon_bo_reserve(bo_va->bo, false); |
return r; |
} |
|
570,6 → 575,10 |
|
mutex_unlock(&vm->mutex); |
return 0; |
|
error_unreserve: |
radeon_bo_unreserve(bo_va->bo); |
return r; |
} |
|
/** |
587,11 → 596,9 |
uint64_t result; |
|
/* page table offset */ |
result = rdev->gart.pages_addr[addr >> PAGE_SHIFT]; |
result = rdev->gart.pages_entry[addr >> RADEON_GPU_PAGE_SHIFT]; |
result &= ~RADEON_GPU_PAGE_MASK; |
|
/* in case cpu page size != gpu page size*/ |
result |= addr & (~PAGE_MASK); |
|
return result; |
} |
|
745,9 → 752,11 |
*/ |
|
/* NI is optimized for 256KB fragments, SI and newer for 64KB */ |
uint64_t frag_flags = rdev->family == CHIP_CAYMAN ? |
uint64_t frag_flags = ((rdev->family == CHIP_CAYMAN) || |
(rdev->family == CHIP_ARUBA)) ? |
R600_PTE_FRAG_256KB : R600_PTE_FRAG_64KB; |
uint64_t frag_align = rdev->family == CHIP_CAYMAN ? 0x200 : 0x80; |
uint64_t frag_align = ((rdev->family == CHIP_CAYMAN) || |
(rdev->family == CHIP_ARUBA)) ? 0x200 : 0x80; |
|
uint64_t frag_start = ALIGN(pe_start, frag_align); |
uint64_t frag_end = pe_end & ~(frag_align - 1); |
916,7 → 925,16 |
} |
|
spin_lock(&vm->status_lock); |
if (mem) { |
if (list_empty(&bo_va->vm_status)) { |
spin_unlock(&vm->status_lock); |
return 0; |
} |
list_del_init(&bo_va->vm_status); |
} else { |
list_del(&bo_va->vm_status); |
list_add(&bo_va->vm_status, &vm->cleared); |
} |
spin_unlock(&vm->status_lock); |
|
bo_va->flags &= ~RADEON_VM_PAGE_VALID; |
942,10 → 960,6 |
addr = 0; |
} |
|
if (addr == bo_va->addr) |
return 0; |
bo_va->addr = addr; |
|
trace_radeon_vm_bo_update(bo_va); |
|
nptes = bo_va->it.last - bo_va->it.start + 1; |
1033,7 → 1047,7 |
struct radeon_vm *vm) |
{ |
struct radeon_bo_va *bo_va; |
int r; |
int r = 0; |
|
spin_lock(&vm->status_lock); |
while (!list_empty(&vm->freed)) { |
1044,14 → 1058,15 |
r = radeon_vm_bo_update(rdev, bo_va, NULL); |
radeon_bo_unref(&bo_va->bo); |
radeon_fence_unref(&bo_va->last_pt_update); |
spin_lock(&vm->status_lock); |
list_del(&bo_va->vm_status); |
kfree(bo_va); |
if (r) |
return r; |
break; |
|
spin_lock(&vm->status_lock); |
} |
spin_unlock(&vm->status_lock); |
return 0; |
return r; |
|
} |
|
1107,11 → 1122,12 |
list_del(&bo_va->bo_list); |
|
mutex_lock(&vm->mutex); |
if (bo_va->it.start || bo_va->it.last) |
interval_tree_remove(&bo_va->it, &vm->va); |
|
spin_lock(&vm->status_lock); |
list_del(&bo_va->vm_status); |
|
if (bo_va->addr) { |
if (bo_va->it.start || bo_va->it.last) { |
bo_va->bo = radeon_bo_ref(bo_va->bo); |
list_add(&bo_va->vm_status, &vm->freed); |
} else { |
1138,14 → 1154,13 |
struct radeon_bo_va *bo_va; |
|
list_for_each_entry(bo_va, &bo->va, bo_list) { |
if (bo_va->addr) { |
spin_lock(&bo_va->vm->status_lock); |
list_del(&bo_va->vm_status); |
if (list_empty(&bo_va->vm_status) && |
(bo_va->it.start || bo_va->it.last)) |
list_add(&bo_va->vm_status, &bo_va->vm->invalidated); |
spin_unlock(&bo_va->vm->status_lock); |
} |
} |
} |
|
/** |
* radeon_vm_init - initialize a vm instance |
1173,6 → 1188,7 |
spin_lock_init(&vm->status_lock); |
INIT_LIST_HEAD(&vm->invalidated); |
INIT_LIST_HEAD(&vm->freed); |
INIT_LIST_HEAD(&vm->cleared); |
|
pd_size = radeon_vm_directory_size(rdev); |
pd_entries = radeon_vm_num_pdes(rdev); |