33,6 → 33,8 |
#include "intel_drv.h" |
//#include <linux/dma_remapping.h> |
|
#define __EXEC_OBJECT_HAS_PIN (1<<31) |
#define __EXEC_OBJECT_HAS_FENCE (1<<30) |
|
static unsigned long |
copy_to_user(void __user *to, const void *from, unsigned long n) |
48,35 → 50,35 |
return 0; |
} |
|
struct eb_objects { |
struct list_head objects; |
struct eb_vmas { |
struct list_head vmas; |
int and; |
union { |
struct drm_i915_gem_object *lut[0]; |
struct i915_vma *lut[0]; |
struct hlist_head buckets[0]; |
}; |
}; |
|
static struct eb_objects * |
static struct eb_vmas * |
eb_create(struct drm_i915_gem_execbuffer2 *args) |
{ |
struct eb_objects *eb = NULL; |
struct eb_vmas *eb = NULL; |
|
if (args->flags & I915_EXEC_HANDLE_LUT) { |
int size = args->buffer_count; |
size *= sizeof(struct drm_i915_gem_object *); |
size += sizeof(struct eb_objects); |
unsigned size = args->buffer_count; |
size *= sizeof(struct i915_vma *); |
size += sizeof(struct eb_vmas); |
eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); |
} |
|
if (eb == NULL) { |
int size = args->buffer_count; |
int count = PAGE_SIZE / sizeof(struct hlist_head) / 2; |
unsigned size = args->buffer_count; |
unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2; |
BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head)); |
while (count > 2*size) |
count >>= 1; |
eb = kzalloc(count*sizeof(struct hlist_head) + |
sizeof(struct eb_objects), |
sizeof(struct eb_vmas), |
GFP_TEMPORARY); |
if (eb == NULL) |
return eb; |
85,12 → 87,12 |
} else |
eb->and = -args->buffer_count; |
|
INIT_LIST_HEAD(&eb->objects); |
INIT_LIST_HEAD(&eb->vmas); |
return eb; |
} |
|
static void |
eb_reset(struct eb_objects *eb) |
eb_reset(struct eb_vmas *eb) |
{ |
if (eb->and >= 0) |
memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head)); |
97,52 → 99,102 |
} |
|
static int |
eb_lookup_objects(struct eb_objects *eb, |
eb_lookup_vmas(struct eb_vmas *eb, |
struct drm_i915_gem_exec_object2 *exec, |
const struct drm_i915_gem_execbuffer2 *args, |
struct i915_address_space *vm, |
struct drm_file *file) |
{ |
int i; |
struct drm_i915_gem_object *obj; |
struct list_head objects; |
int i, ret; |
|
INIT_LIST_HEAD(&objects); |
spin_lock(&file->table_lock); |
/* Grab a reference to the object and release the lock so we can lookup |
* or create the VMA without using GFP_ATOMIC */ |
for (i = 0; i < args->buffer_count; i++) { |
struct drm_i915_gem_object *obj; |
|
obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle)); |
if (obj == NULL) { |
spin_unlock(&file->table_lock); |
DRM_DEBUG("Invalid object handle %d at index %d\n", |
exec[i].handle, i); |
return -ENOENT; |
ret = -ENOENT; |
goto err; |
} |
|
if (!list_empty(&obj->exec_list)) { |
if (!list_empty(&obj->obj_exec_link)) { |
spin_unlock(&file->table_lock); |
DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n", |
obj, exec[i].handle, i); |
return -EINVAL; |
ret = -EINVAL; |
goto err; |
} |
|
drm_gem_object_reference(&obj->base); |
list_add_tail(&obj->exec_list, &eb->objects); |
list_add_tail(&obj->obj_exec_link, &objects); |
} |
spin_unlock(&file->table_lock); |
|
obj->exec_entry = &exec[i]; |
i = 0; |
while (!list_empty(&objects)) { |
struct i915_vma *vma; |
|
obj = list_first_entry(&objects, |
struct drm_i915_gem_object, |
obj_exec_link); |
|
/* |
* NOTE: We can leak any vmas created here when something fails |
* later on. But that's no issue since vma_unbind can deal with |
* vmas which are not actually bound. And since only |
* lookup_or_create exists as an interface to get at the vma |
* from the (obj, vm) we don't run the risk of creating |
* duplicated vmas for the same vm. |
*/ |
vma = i915_gem_obj_lookup_or_create_vma(obj, vm); |
if (IS_ERR(vma)) { |
DRM_DEBUG("Failed to lookup VMA\n"); |
ret = PTR_ERR(vma); |
goto err; |
} |
|
/* Transfer ownership from the objects list to the vmas list. */ |
list_add_tail(&vma->exec_list, &eb->vmas); |
list_del_init(&obj->obj_exec_link); |
|
vma->exec_entry = &exec[i]; |
if (eb->and < 0) { |
eb->lut[i] = obj; |
eb->lut[i] = vma; |
} else { |
uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle; |
obj->exec_handle = handle; |
hlist_add_head(&obj->exec_node, |
vma->exec_handle = handle; |
hlist_add_head(&vma->exec_node, |
&eb->buckets[handle & eb->and]); |
} |
++i; |
} |
spin_unlock(&file->table_lock); |
|
return 0; |
|
|
err: |
while (!list_empty(&objects)) { |
obj = list_first_entry(&objects, |
struct drm_i915_gem_object, |
obj_exec_link); |
list_del_init(&obj->obj_exec_link); |
drm_gem_object_unreference(&obj->base); |
} |
/* |
* Objects already transfered to the vmas list will be unreferenced by |
* eb_destroy. |
*/ |
|
static struct drm_i915_gem_object * |
eb_get_object(struct eb_objects *eb, unsigned long handle) |
return ret; |
} |
|
static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle) |
{ |
if (eb->and < 0) { |
if (handle >= -eb->and) |
154,11 → 206,11 |
|
head = &eb->buckets[handle & eb->and]; |
hlist_for_each(node, head) { |
struct drm_i915_gem_object *obj; |
struct i915_vma *vma; |
|
obj = hlist_entry(node, struct drm_i915_gem_object, exec_node); |
if (obj->exec_handle == handle) |
return obj; |
vma = hlist_entry(node, struct i915_vma, exec_node); |
if (vma->exec_handle == handle) |
return vma; |
} |
return NULL; |
} |
165,16 → 217,36 |
} |
|
static void |
eb_destroy(struct eb_objects *eb) |
i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma) |
{ |
while (!list_empty(&eb->objects)) { |
struct drm_i915_gem_object *obj; |
struct drm_i915_gem_exec_object2 *entry; |
struct drm_i915_gem_object *obj = vma->obj; |
|
obj = list_first_entry(&eb->objects, |
struct drm_i915_gem_object, |
if (!drm_mm_node_allocated(&vma->node)) |
return; |
|
entry = vma->exec_entry; |
|
if (entry->flags & __EXEC_OBJECT_HAS_FENCE) |
i915_gem_object_unpin_fence(obj); |
|
if (entry->flags & __EXEC_OBJECT_HAS_PIN) |
i915_gem_object_unpin(obj); |
|
entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN); |
} |
|
static void eb_destroy(struct eb_vmas *eb) |
{ |
while (!list_empty(&eb->vmas)) { |
struct i915_vma *vma; |
|
vma = list_first_entry(&eb->vmas, |
struct i915_vma, |
exec_list); |
list_del_init(&obj->exec_list); |
drm_gem_object_unreference(&obj->base); |
list_del_init(&vma->exec_list); |
i915_gem_execbuffer_unreserve_vma(vma); |
drm_gem_object_unreference(&vma->obj->base); |
} |
kfree(eb); |
} |
181,7 → 253,8 |
|
static inline int use_cpu_reloc(struct drm_i915_gem_object *obj) |
{ |
return (obj->base.write_domain == I915_GEM_DOMAIN_CPU || |
return (HAS_LLC(obj->base.dev) || |
obj->base.write_domain == I915_GEM_DOMAIN_CPU || |
!obj->map_and_fenceable || |
obj->cache_level != I915_CACHE_NONE); |
} |
194,9 → 267,9 |
struct drm_i915_private *dev_priv = dev->dev_private; |
uint32_t page_offset = offset_in_page(reloc->offset); |
char *vaddr; |
int ret = -EINVAL; |
int ret; |
|
ret = i915_gem_object_set_to_cpu_domain(obj, 1); |
ret = i915_gem_object_set_to_cpu_domain(obj, true); |
if (ret) |
return ret; |
|
215,7 → 288,7 |
struct drm_i915_private *dev_priv = dev->dev_private; |
uint32_t __iomem *reloc_entry; |
void __iomem *reloc_page; |
int ret = -EINVAL; |
int ret; |
|
ret = i915_gem_object_set_to_gtt_domain(obj, true); |
if (ret) |
239,7 → 312,7 |
|
static int |
i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, |
struct eb_objects *eb, |
struct eb_vmas *eb, |
struct drm_i915_gem_relocation_entry *reloc, |
struct i915_address_space *vm) |
{ |
246,16 → 319,18 |
struct drm_device *dev = obj->base.dev; |
struct drm_gem_object *target_obj; |
struct drm_i915_gem_object *target_i915_obj; |
struct i915_vma *target_vma; |
uint32_t target_offset; |
int ret = -EINVAL; |
int ret; |
|
/* we've already hold a reference to all valid objects */ |
target_obj = &eb_get_object(eb, reloc->target_handle)->base; |
if (unlikely(target_obj == NULL)) |
target_vma = eb_get_vma(eb, reloc->target_handle); |
if (unlikely(target_vma == NULL)) |
return -ENOENT; |
target_i915_obj = target_vma->obj; |
target_obj = &target_vma->obj->base; |
|
target_i915_obj = to_intel_bo(target_obj); |
target_offset = i915_gem_obj_ggtt_offset(target_i915_obj); |
target_offset = target_vma->node.start; |
|
/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and |
* pipe_control writes because the gpu doesn't properly redirect them |
276,7 → 351,7 |
(int) reloc->offset, |
reloc->read_domains, |
reloc->write_domain); |
return ret; |
return -EINVAL; |
} |
if (unlikely((reloc->write_domain | reloc->read_domains) |
& ~I915_GEM_GPU_DOMAINS)) { |
287,7 → 362,7 |
(int) reloc->offset, |
reloc->read_domains, |
reloc->write_domain); |
return ret; |
return -EINVAL; |
} |
|
target_obj->pending_read_domains |= reloc->read_domains; |
300,13 → 375,14 |
return 0; |
|
/* Check that the relocation address is valid... */ |
if (unlikely(reloc->offset > obj->base.size - 4)) { |
if (unlikely(reloc->offset > |
obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) { |
DRM_DEBUG("Relocation beyond object bounds: " |
"obj %p target %d offset %d size %d.\n", |
obj, reloc->target_handle, |
(int) reloc->offset, |
(int) obj->base.size); |
return ret; |
return -EINVAL; |
} |
if (unlikely(reloc->offset & 3)) { |
DRM_DEBUG("Relocation not 4-byte aligned: " |
313,7 → 389,7 |
"obj %p target %d offset %d.\n", |
obj, reloc->target_handle, |
(int) reloc->offset); |
return ret; |
return -EINVAL; |
} |
|
/* We can't wait for rendering with pagefaults disabled */ |
334,14 → 410,13 |
} |
|
static int |
i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, |
struct eb_objects *eb, |
struct i915_address_space *vm) |
i915_gem_execbuffer_relocate_vma(struct i915_vma *vma, |
struct eb_vmas *eb) |
{ |
#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) |
struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(64)]; |
struct drm_i915_gem_relocation_entry __user *user_relocs; |
struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; |
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; |
int remain, ret; |
|
user_relocs = to_user_ptr(entry->relocs_ptr); |
359,8 → 434,8 |
do { |
u64 offset = r->presumed_offset; |
|
ret = i915_gem_execbuffer_relocate_entry(obj, eb, r, |
vm); |
ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r, |
vma->vm); |
if (ret) |
return ret; |
|
381,17 → 456,16 |
} |
|
static int |
i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, |
struct eb_objects *eb, |
struct drm_i915_gem_relocation_entry *relocs, |
struct i915_address_space *vm) |
i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma, |
struct eb_vmas *eb, |
struct drm_i915_gem_relocation_entry *relocs) |
{ |
const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; |
const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; |
int i, ret; |
|
for (i = 0; i < entry->relocation_count; i++) { |
ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i], |
vm); |
ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i], |
vma->vm); |
if (ret) |
return ret; |
} |
400,10 → 474,9 |
} |
|
static int |
i915_gem_execbuffer_relocate(struct eb_objects *eb, |
struct i915_address_space *vm) |
i915_gem_execbuffer_relocate(struct eb_vmas *eb) |
{ |
struct drm_i915_gem_object *obj; |
struct i915_vma *vma; |
int ret = 0; |
|
/* This is the fast path and we cannot handle a pagefault whilst |
414,8 → 487,8 |
* lockdep complains vehemently. |
*/ |
// pagefault_disable(); |
list_for_each_entry(obj, &eb->objects, exec_list) { |
ret = i915_gem_execbuffer_relocate_object(obj, eb, vm); |
list_for_each_entry(vma, &eb->vmas, exec_list) { |
ret = i915_gem_execbuffer_relocate_vma(vma, eb); |
if (ret) |
break; |
} |
424,26 → 497,24 |
return ret; |
} |
|
#define __EXEC_OBJECT_HAS_PIN (1<<31) |
#define __EXEC_OBJECT_HAS_FENCE (1<<30) |
|
static int |
need_reloc_mappable(struct drm_i915_gem_object *obj) |
need_reloc_mappable(struct i915_vma *vma) |
{ |
struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; |
return entry->relocation_count && !use_cpu_reloc(obj); |
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; |
return entry->relocation_count && !use_cpu_reloc(vma->obj) && |
i915_is_ggtt(vma->vm); |
} |
|
static int |
i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj, |
i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, |
struct intel_ring_buffer *ring, |
struct i915_address_space *vm, |
bool *need_reloc) |
{ |
struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; |
struct drm_i915_private *dev_priv = ring->dev->dev_private; |
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; |
bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; |
bool need_fence, need_mappable; |
struct drm_i915_gem_object *obj = vma->obj; |
int ret; |
|
need_fence = |
450,9 → 521,9 |
has_fenced_gpu_access && |
entry->flags & EXEC_OBJECT_NEEDS_FENCE && |
obj->tiling_mode != I915_TILING_NONE; |
need_mappable = need_fence || need_reloc_mappable(obj); |
need_mappable = need_fence || need_reloc_mappable(vma); |
|
ret = i915_gem_object_pin(obj, vm, entry->alignment, need_mappable, |
ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, need_mappable, |
false); |
if (ret) |
return ret; |
480,8 → 551,8 |
obj->has_aliasing_ppgtt_mapping = 1; |
} |
|
if (entry->offset != i915_gem_obj_offset(obj, vm)) { |
entry->offset = i915_gem_obj_offset(obj, vm); |
if (entry->offset != vma->node.start) { |
entry->offset = vma->node.start; |
*need_reloc = true; |
} |
|
497,62 → 568,48 |
return 0; |
} |
|
static void |
i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj) |
{ |
struct drm_i915_gem_exec_object2 *entry; |
|
if (!i915_gem_obj_bound_any(obj)) |
return; |
|
entry = obj->exec_entry; |
|
if (entry->flags & __EXEC_OBJECT_HAS_FENCE) |
i915_gem_object_unpin_fence(obj); |
|
if (entry->flags & __EXEC_OBJECT_HAS_PIN) |
i915_gem_object_unpin(obj); |
|
entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN); |
} |
|
static int |
i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, |
struct list_head *objects, |
struct i915_address_space *vm, |
struct list_head *vmas, |
bool *need_relocs) |
{ |
struct drm_i915_gem_object *obj; |
struct list_head ordered_objects; |
struct i915_vma *vma; |
struct i915_address_space *vm; |
struct list_head ordered_vmas; |
bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; |
int retry; |
|
INIT_LIST_HEAD(&ordered_objects); |
while (!list_empty(objects)) { |
if (list_empty(vmas)) |
return 0; |
|
vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm; |
|
INIT_LIST_HEAD(&ordered_vmas); |
while (!list_empty(vmas)) { |
struct drm_i915_gem_exec_object2 *entry; |
bool need_fence, need_mappable; |
|
obj = list_first_entry(objects, |
struct drm_i915_gem_object, |
exec_list); |
entry = obj->exec_entry; |
vma = list_first_entry(vmas, struct i915_vma, exec_list); |
obj = vma->obj; |
entry = vma->exec_entry; |
|
need_fence = |
has_fenced_gpu_access && |
entry->flags & EXEC_OBJECT_NEEDS_FENCE && |
obj->tiling_mode != I915_TILING_NONE; |
need_mappable = need_fence || need_reloc_mappable(obj); |
need_mappable = need_fence || need_reloc_mappable(vma); |
|
if (need_mappable) |
list_move(&obj->exec_list, &ordered_objects); |
list_move(&vma->exec_list, &ordered_vmas); |
else |
list_move_tail(&obj->exec_list, &ordered_objects); |
list_move_tail(&vma->exec_list, &ordered_vmas); |
|
obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND; |
obj->base.pending_write_domain = 0; |
obj->pending_fenced_gpu_access = false; |
} |
list_splice(&ordered_objects, objects); |
list_splice(&ordered_vmas, vmas); |
|
/* Attempt to pin all of the buffers into the GTT. |
* This is done in 3 phases: |
571,52 → 628,53 |
int ret = 0; |
|
/* Unbind any ill-fitting objects or pin. */ |
list_for_each_entry(obj, objects, exec_list) { |
struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; |
list_for_each_entry(vma, vmas, exec_list) { |
struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; |
bool need_fence, need_mappable; |
u32 obj_offset; |
|
if (!i915_gem_obj_bound(obj, vm)) |
obj = vma->obj; |
|
if (!drm_mm_node_allocated(&vma->node)) |
continue; |
|
obj_offset = i915_gem_obj_offset(obj, vm); |
need_fence = |
has_fenced_gpu_access && |
entry->flags & EXEC_OBJECT_NEEDS_FENCE && |
obj->tiling_mode != I915_TILING_NONE; |
need_mappable = need_fence || need_reloc_mappable(obj); |
need_mappable = need_fence || need_reloc_mappable(vma); |
|
WARN_ON((need_mappable || need_fence) && |
!i915_is_ggtt(vm)); |
!i915_is_ggtt(vma->vm)); |
|
if ((entry->alignment && |
obj_offset & (entry->alignment - 1)) || |
vma->node.start & (entry->alignment - 1)) || |
(need_mappable && !obj->map_and_fenceable)) |
ret = i915_vma_unbind(i915_gem_obj_to_vma(obj, vm)); |
ret = i915_vma_unbind(vma); |
else |
ret = i915_gem_execbuffer_reserve_object(obj, ring, vm, need_relocs); |
ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs); |
if (ret) |
goto err; |
} |
|
/* Bind fresh objects */ |
list_for_each_entry(obj, objects, exec_list) { |
if (i915_gem_obj_bound(obj, vm)) |
list_for_each_entry(vma, vmas, exec_list) { |
if (drm_mm_node_allocated(&vma->node)) |
continue; |
|
ret = i915_gem_execbuffer_reserve_object(obj, ring, vm, need_relocs); |
ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs); |
if (ret) |
goto err; |
} |
|
err: /* Decrement pin count for bound objects */ |
list_for_each_entry(obj, objects, exec_list) |
i915_gem_execbuffer_unreserve_object(obj); |
|
err: |
if (ret != -ENOSPC || retry++) |
return ret; |
|
// ret = i915_gem_evict_everything(ring->dev); |
/* Decrement pin count for bound objects */ |
list_for_each_entry(vma, vmas, exec_list) |
i915_gem_execbuffer_unreserve_vma(vma); |
|
// ret = i915_gem_evict_vm(vm, true); |
if (ret) |
return ret; |
} while (1); |
627,24 → 685,28 |
struct drm_i915_gem_execbuffer2 *args, |
struct drm_file *file, |
struct intel_ring_buffer *ring, |
struct eb_objects *eb, |
struct drm_i915_gem_exec_object2 *exec, |
struct i915_address_space *vm) |
struct eb_vmas *eb, |
struct drm_i915_gem_exec_object2 *exec) |
{ |
struct drm_i915_gem_relocation_entry *reloc; |
struct drm_i915_gem_object *obj; |
struct i915_address_space *vm; |
struct i915_vma *vma; |
bool need_relocs; |
int *reloc_offset; |
int i, total, ret; |
int count = args->buffer_count; |
unsigned count = args->buffer_count; |
|
if (WARN_ON(list_empty(&eb->vmas))) |
return 0; |
|
vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm; |
|
/* We may process another execbuffer during the unlock... */ |
while (!list_empty(&eb->objects)) { |
obj = list_first_entry(&eb->objects, |
struct drm_i915_gem_object, |
exec_list); |
list_del_init(&obj->exec_list); |
drm_gem_object_unreference(&obj->base); |
while (!list_empty(&eb->vmas)) { |
vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list); |
list_del_init(&vma->exec_list); |
i915_gem_execbuffer_unreserve_vma(vma); |
drm_gem_object_unreference(&vma->obj->base); |
} |
|
mutex_unlock(&dev->struct_mutex); |
708,20 → 770,19 |
|
/* reacquire the objects */ |
eb_reset(eb); |
ret = eb_lookup_objects(eb, exec, args, file); |
ret = eb_lookup_vmas(eb, exec, args, vm, file); |
if (ret) |
goto err; |
|
need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; |
ret = i915_gem_execbuffer_reserve(ring, &eb->objects, vm, &need_relocs); |
ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs); |
if (ret) |
goto err; |
|
list_for_each_entry(obj, &eb->objects, exec_list) { |
int offset = obj->exec_entry - exec; |
ret = i915_gem_execbuffer_relocate_object_slow(obj, eb, |
reloc + reloc_offset[offset], |
vm); |
list_for_each_entry(vma, &eb->vmas, exec_list) { |
int offset = vma->exec_entry - exec; |
ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb, |
reloc + reloc_offset[offset]); |
if (ret) |
goto err; |
} |
740,14 → 801,15 |
|
static int |
i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, |
struct list_head *objects) |
struct list_head *vmas) |
{ |
struct drm_i915_gem_object *obj; |
struct i915_vma *vma; |
uint32_t flush_domains = 0; |
bool flush_chipset = false; |
int ret; |
|
list_for_each_entry(obj, objects, exec_list) { |
list_for_each_entry(vma, vmas, exec_list) { |
struct drm_i915_gem_object *obj = vma->obj; |
ret = i915_gem_object_sync(obj, ring); |
if (ret) |
return ret; |
784,8 → 846,8 |
int count) |
{ |
int i; |
int relocs_total = 0; |
int relocs_max = INT_MAX / sizeof(struct drm_i915_gem_relocation_entry); |
unsigned relocs_total = 0; |
unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry); |
|
for (i = 0; i < count; i++) { |
char __user *ptr = to_user_ptr(exec[i].relocs_ptr); |
809,20 → 871,37 |
* to read, but since we may need to update the presumed |
* offsets during execution, check for full write access. |
*/ |
} |
|
return 0; |
} |
|
static int |
i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, |
const u32 ctx_id) |
{ |
struct i915_ctx_hang_stats *hs; |
|
hs = i915_gem_context_get_hang_stats(dev, file, ctx_id); |
if (IS_ERR(hs)) |
return PTR_ERR(hs); |
|
if (hs->banned) { |
DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id); |
return -EIO; |
} |
|
return 0; |
} |
|
static void |
i915_gem_execbuffer_move_to_active(struct list_head *objects, |
struct i915_address_space *vm, |
i915_gem_execbuffer_move_to_active(struct list_head *vmas, |
struct intel_ring_buffer *ring) |
{ |
struct drm_i915_gem_object *obj; |
struct i915_vma *vma; |
|
list_for_each_entry(obj, objects, exec_list) { |
list_for_each_entry(vma, vmas, exec_list) { |
struct drm_i915_gem_object *obj = vma->obj; |
u32 old_read = obj->base.read_domains; |
u32 old_write = obj->base.write_domain; |
|
832,9 → 911,7 |
obj->base.read_domains = obj->base.pending_read_domains; |
obj->fenced_gpu_access = obj->pending_fenced_gpu_access; |
|
/* FIXME: This lookup gets fixed later <-- danvet */ |
list_move_tail(&i915_gem_obj_to_vma(obj, vm)->mm_list, &vm->active_list); |
i915_gem_object_move_to_active(obj, ring); |
i915_vma_move_to_active(vma, ring); |
if (obj->base.write_domain) { |
obj->dirty = 1; |
obj->last_write_seqno = intel_ring_get_seqno(ring); |
892,11 → 969,11 |
struct i915_address_space *vm) |
{ |
drm_i915_private_t *dev_priv = dev->dev_private; |
struct eb_objects *eb; |
struct eb_vmas *eb; |
struct drm_i915_gem_object *batch_obj; |
struct drm_clip_rect *cliprects = NULL; |
struct intel_ring_buffer *ring; |
u32 ctx_id = i915_execbuffer2_get_context_id(*args); |
const u32 ctx_id = i915_execbuffer2_get_context_id(*args); |
u32 exec_start, exec_len; |
u32 mask, flags; |
int ret, mode, i; |
1005,7 → 1082,8 |
return -EINVAL; |
} |
|
cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects), |
cliprects = kcalloc(args->num_cliprects, |
sizeof(*cliprects), |
GFP_KERNEL); |
if (cliprects == NULL) { |
ret = -ENOMEM; |
1020,6 → 1098,8 |
} |
} |
|
intel_runtime_pm_get(dev_priv); |
|
ret = i915_mutex_lock_interruptible(dev); |
if (ret) |
goto pre_mutex_err; |
1030,6 → 1110,12 |
goto pre_mutex_err; |
} |
|
ret = i915_gem_validate_context(dev, file, ctx_id); |
if (ret) { |
mutex_unlock(&dev->struct_mutex); |
goto pre_mutex_err; |
} |
|
eb = eb_create(args); |
if (eb == NULL) { |
mutex_unlock(&dev->struct_mutex); |
1038,28 → 1124,26 |
} |
|
/* Look up object handles */ |
ret = eb_lookup_objects(eb, exec, args, file); |
ret = eb_lookup_vmas(eb, exec, args, vm, file); |
if (ret) |
goto err; |
|
/* take note of the batch buffer before we might reorder the lists */ |
batch_obj = list_entry(eb->objects.prev, |
struct drm_i915_gem_object, |
exec_list); |
batch_obj = list_entry(eb->vmas.prev, struct i915_vma, exec_list)->obj; |
|
/* Move the objects en-masse into the GTT, evicting if necessary. */ |
need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; |
ret = i915_gem_execbuffer_reserve(ring, &eb->objects, vm, &need_relocs); |
ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs); |
if (ret) |
goto err; |
|
/* The objects are in their final locations, apply the relocations. */ |
if (need_relocs) |
ret = i915_gem_execbuffer_relocate(eb, vm); |
ret = i915_gem_execbuffer_relocate(eb); |
if (ret) { |
if (ret == -EFAULT) { |
ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring, |
eb, exec, vm); |
eb, exec); |
BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
} |
if (ret) |
1076,12 → 1160,11 |
|
/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure |
* batch" bit. Hence we need to pin secure batches into the global gtt. |
* hsw should have this fixed, but let's be paranoid and do it |
* unconditionally for now. */ |
* hsw should have this fixed, but bdw mucks it up again. */ |
if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping) |
i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level); |
|
ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->objects); |
ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->vmas); |
if (ret) |
goto err; |
|
1136,7 → 1219,7 |
|
trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags); |
|
i915_gem_execbuffer_move_to_active(&eb->objects, vm, ring); |
i915_gem_execbuffer_move_to_active(&eb->vmas, ring); |
i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); |
|
err: |
1146,6 → 1229,10 |
|
pre_mutex_err: |
kfree(cliprects); |
|
/* intel_gpu_busy should also get a ref, so it will free when the device |
* is really idle. */ |
intel_runtime_pm_put(dev_priv); |
return ret; |
} |
|