Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 5353 → Rev 5354

/drivers/video/drm/i915/i915_gem.c
73,10 → 73,6
unsigned long len, unsigned long prot,
unsigned long flag, unsigned long offset);
 
static inline void clflush(volatile void *__p)
{
asm volatile("clflush %0" : "+m" (*(volatile char*)__p));
}
 
#define MAX_ERRNO 4095
 
98,7 → 94,7
struct drm_i915_fence_reg *fence,
bool enable);
 
static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
 
static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
 
static bool cpu_cache_is_coherent(struct drm_device *dev,
199,38 → 195,7
return i915_gem_obj_bound_any(obj) && !obj->active;
}
 
 
#if 0
 
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
struct drm_file *file)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_init *args = data;
 
if (drm_core_check_feature(dev, DRIVER_MODESET))
return -ENODEV;
 
if (args->gtt_start >= args->gtt_end ||
(args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
return -EINVAL;
 
/* GEM with user mode setting was never supported on ilk and later. */
if (INTEL_INFO(dev)->gen >= 5)
return -ENODEV;
 
mutex_lock(&dev->struct_mutex);
i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
args->gtt_end);
dev_priv->gtt.mappable_end = args->gtt_end;
mutex_unlock(&dev->struct_mutex);
 
return 0;
}
#endif
 
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
struct drm_file *file)
{
690,12 → 655,12
bool needs_clflush_after)
{
char *vaddr;
int ret = 0;
int ret;
 
if (unlikely(page_do_bit17_swizzling))
return -EINVAL;
 
vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
vaddr = kmap_atomic(page);
if (needs_clflush_before)
drm_clflush_virt_range(vaddr + shmem_page_offset,
page_length);
705,7 → 670,7
if (needs_clflush_after)
drm_clflush_virt_range(vaddr + shmem_page_offset,
page_length);
FreeKernelSpace(vaddr);
kunmap_atomic(vaddr);
 
return ret ? -EFAULT : 0;
}
842,11 → 807,10
 
mutex_lock(&dev->struct_mutex);
 
next_page:
 
if (ret)
goto out;
 
next_page:
remain -= page_length;
user_data += page_length;
offset += page_length;
925,11 → 889,6
* pread/pwrite currently are reading and writing from the CPU
* perspective, requiring manual detiling by the client.
*/
// if (obj->phys_obj) {
// ret = i915_gem_phys_pwrite(dev, obj, args, file);
// goto out;
// }
 
if (obj->tiling_mode == I915_TILING_NONE &&
obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
cpu_write_needs_clflush(obj)) {
1007,7 → 966,7
}
 
/**
* __wait_seqno - wait until execution of seqno has finished
* __i915_wait_seqno - wait until execution of seqno has finished
* @ring: the ring expected to report seqno
* @seqno: duh!
* @reset_counter: reset sequence associated with the given seqno
1024,7 → 983,7
* Returns 0 if the seqno was found within the alloted time. Else returns the
* errno with remaining time filled in timeout argument.
*/
static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
unsigned reset_counter,
bool interruptible,
s64 *timeout,
1045,7 → 1004,8
if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
return 0;
 
timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0;
timeout_expire = timeout ?
jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
 
if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
gen6_rps_boost(dev_priv);
1122,6 → 1082,7
struct drm_device *dev = ring->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
bool interruptible = dev_priv->mm.interruptible;
unsigned reset_counter;
int ret;
 
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1135,14 → 1096,13
if (ret)
return ret;
 
return __wait_seqno(ring, seqno,
atomic_read(&dev_priv->gpu_error.reset_counter),
interruptible, NULL, NULL);
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
NULL, NULL);
}
 
static int
i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
struct intel_engine_cs *ring)
i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
{
if (!obj->active)
return 0;
1179,7 → 1139,7
if (ret)
return ret;
 
return i915_gem_object_wait_rendering__tail(obj, ring);
return i915_gem_object_wait_rendering__tail(obj);
}
 
/* A nonblocking variant of the above wait. This is a highly dangerous routine
1214,12 → 1174,13
 
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
mutex_unlock(&dev->struct_mutex);
ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
file_priv);
mutex_lock(&dev->struct_mutex);
if (ret)
return ret;
 
return i915_gem_object_wait_rendering__tail(obj, ring);
return i915_gem_object_wait_rendering__tail(obj);
}
 
/**
1326,6 → 1287,16
*
* While the mapping holds a reference on the contents of the object, it doesn't
* imply a ref on the object itself.
*
* IMPORTANT:
*
* DRM driver writers who look a this function as an example for how to do GEM
* mmap support, please don't implement mmap support like here. The modern way
* to implement DRM mmap support is with an mmap offset ioctl (like
* i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
* That way debug tooling like valgrind will understand what's going on, hiding
* the mmap call in a driver private ioctl will break that. The i915 driver only
* does cpu mmaps this way because we didn't know better.
*/
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1693,7 → 1664,14
goto err_pages;
 
}
 
#ifdef CONFIG_SWIOTLB
if (swiotlb_nr_tbl()) {
st->nents++;
sg_set_page(sg, page, PAGE_SIZE, 0);
sg = sg_next(sg);
continue;
}
#endif
if (!i || page_to_pfn(page) != last_pfn + 1) {
if (i)
sg = sg_next(sg);
1704,7 → 1682,9
}
last_pfn = page_to_pfn(page);
}
 
#ifdef CONFIG_SWIOTLB
if (!swiotlb_nr_tbl())
#endif
sg_mark_end(sg);
obj->pages = st;
 
1756,8 → 1736,6
i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
struct intel_engine_cs *ring)
{
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
u32 seqno = intel_ring_get_seqno(ring);
 
BUG_ON(ring == NULL);
1776,20 → 1754,7
list_move_tail(&obj->ring_list, &ring->active_list);
 
obj->last_read_seqno = seqno;
 
if (obj->fenced_gpu_access) {
obj->last_fenced_seqno = seqno;
 
/* Bump MRU to take account of the delayed flush */
if (obj->fence_reg != I915_FENCE_REG_NONE) {
struct drm_i915_fence_reg *reg;
 
reg = &dev_priv->fence_regs[obj->fence_reg];
list_move_tail(&reg->lru_list,
&dev_priv->mm.fence_list);
}
}
}
 
void i915_vma_move_to_active(struct i915_vma *vma,
struct intel_engine_cs *ring)
1814,6 → 1779,8
list_move_tail(&vma->mm_list, &vm->inactive_list);
}
 
intel_fb_obj_flush(obj, true);
 
list_del_init(&obj->ring_list);
obj->ring = NULL;
 
1822,7 → 1789,6
obj->base.write_domain = 0;
 
obj->last_fenced_seqno = 0;
obj->fenced_gpu_access = false;
 
obj->active = 0;
drm_gem_object_unreference(&obj->base);
1920,10 → 1886,21
{
struct drm_i915_private *dev_priv = ring->dev->dev_private;
struct drm_i915_gem_request *request;
struct intel_ringbuffer *ringbuf;
u32 request_ring_position, request_start;
int ret;
 
request_start = intel_ring_get_tail(ring->buffer);
request = ring->preallocated_lazy_request;
if (WARN_ON(request == NULL))
return -ENOMEM;
 
if (i915.enable_execlists) {
struct intel_context *ctx = request->ctx;
ringbuf = ctx->engine[ring->id].ringbuf;
} else
ringbuf = ring->buffer;
 
request_start = intel_ring_get_tail(ringbuf);
/*
* Emit any outstanding flushes - execbuf can fail to emit the flush
* after having emitted the batchbuffer command. Hence we need to fix
1931,24 → 1908,32
* is that the flush _must_ happen before the next request, no matter
* what.
*/
if (i915.enable_execlists) {
ret = logical_ring_flush_all_caches(ringbuf);
if (ret)
return ret;
} else {
ret = intel_ring_flush_all_caches(ring);
if (ret)
return ret;
}
 
request = ring->preallocated_lazy_request;
if (WARN_ON(request == NULL))
return -ENOMEM;
 
/* Record the position of the start of the request so that
* should we detect the updated seqno part-way through the
* GPU processing the request, we never over-estimate the
* position of the head.
*/
request_ring_position = intel_ring_get_tail(ring->buffer);
request_ring_position = intel_ring_get_tail(ringbuf);
 
if (i915.enable_execlists) {
ret = ring->emit_request(ringbuf);
if (ret)
return ret;
} else {
ret = ring->add_request(ring);
if (ret)
return ret;
}
 
request->seqno = intel_ring_get_seqno(ring);
request->ring = ring;
1963,6 → 1948,7
*/
request->batch_obj = obj;
 
if (!i915.enable_execlists) {
/* Hold a reference to the current context so that we can inspect
* it later in case a hangcheck error event fires.
*/
1969,6 → 1955,7
request->ctx = ring->last_context;
if (request->ctx)
i915_gem_context_reference(request->ctx);
}
 
request->emitted_jiffies = jiffies;
list_add_tail(&request->list, &ring->request_list);
1988,7 → 1975,6
ring->outstanding_lazy_seqno = 0;
ring->preallocated_lazy_request = NULL;
 
if (!dev_priv->ums.mm_suspended) {
// i915_queue_hangcheck(ring->dev);
 
queue_delayed_work(dev_priv->wq,
1995,7 → 1981,6
&dev_priv->mm.retire_work,
round_jiffies_up_relative(HZ));
intel_mark_busy(dev_priv->dev);
}
 
if (out_seqno)
*out_seqno = request->seqno;
2062,12 → 2047,20
 
static void i915_gem_free_request(struct drm_i915_gem_request *request)
{
struct intel_context *ctx = request->ctx;
 
list_del(&request->list);
i915_gem_request_remove_from_client(request);
 
if (request->ctx)
i915_gem_context_unreference(request->ctx);
if (ctx) {
if (i915.enable_execlists) {
struct intel_engine_cs *ring = request->ring;
 
if (ctx != ring->default_context)
intel_lr_context_unpin(ring, ctx);
}
i915_gem_context_unreference(ctx);
}
kfree(request);
}
 
2122,6 → 2115,23
}
 
/*
* Clear the execlists queue up before freeing the requests, as those
* are the ones that keep the context and ringbuffer backing objects
* pinned in place.
*/
while (!list_empty(&ring->execlist_queue)) {
struct intel_ctx_submit_request *submit_req;
 
submit_req = list_first_entry(&ring->execlist_queue,
struct intel_ctx_submit_request,
execlist_link);
list_del(&submit_req->execlist_link);
intel_runtime_pm_put(dev_priv);
i915_gem_context_unreference(submit_req->ctx);
kfree(submit_req);
}
 
/*
* We must free the requests after all the corresponding objects have
* been moved off active lists. Which is the same order as the normal
* retire_requests function does. This is important if object hold
2222,6 → 2232,7
 
while (!list_empty(&ring->request_list)) {
struct drm_i915_gem_request *request;
struct intel_ringbuffer *ringbuf;
 
request = list_first_entry(&ring->request_list,
struct drm_i915_gem_request,
2231,12 → 2242,24
break;
 
trace_i915_gem_request_retire(ring, request->seqno);
 
/* This is one of the few common intersection points
* between legacy ringbuffer submission and execlists:
* we need to tell them apart in order to find the correct
* ringbuffer to which the request belongs to.
*/
if (i915.enable_execlists) {
struct intel_context *ctx = request->ctx;
ringbuf = ctx->engine[ring->id].ringbuf;
} else
ringbuf = ring->buffer;
 
/* We know the GPU must have read the request to have
* sent us the seqno + interrupt, so use the position
* of tail of the request to update the last known position
* of the GPU head.
*/
ring->buffer->last_retired_head = request->tail;
ringbuf->last_retired_head = request->tail;
 
i915_gem_free_request(request);
}
2261,7 → 2284,16
for_each_ring(ring, dev_priv, i) {
i915_gem_retire_requests_ring(ring);
idle &= list_empty(&ring->request_list);
if (i915.enable_execlists) {
unsigned long flags;
 
spin_lock_irqsave(&ring->execlist_lock, flags);
idle &= list_empty(&ring->execlist_queue);
spin_unlock_irqrestore(&ring->execlist_lock, flags);
 
intel_execlists_retire_requests(ring);
}
}
 
if (idle)
mod_delayed_work(dev_priv->wq,
2353,6 → 2385,9
u32 seqno = 0;
int ret = 0;
 
if (args->flags != 0)
return -EINVAL;
 
ret = i915_mutex_lock_interruptible(dev);
if (ret)
return ret;
2388,8 → 2423,8
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
mutex_unlock(&dev->struct_mutex);
 
return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns,
file->driver_priv);
return __i915_wait_seqno(ring, seqno, reset_counter, true,
&args->timeout_ns, file->driver_priv);
 
out:
drm_gem_object_unreference(&obj->base);
2501,6 → 2536,9
* cause memory corruption through use-after-free.
*/
 
/* Throw away the active reference before moving to the unbound list */
i915_gem_object_retire(obj);
 
if (i915_is_ggtt(vma->vm)) {
i915_gem_object_finish_gtt(obj);
 
2515,9 → 2553,8
vma->unbind_vma(vma);
 
list_del_init(&vma->mm_list);
/* Avoid an unnecessary call to unbind on rebind. */
if (i915_is_ggtt(vma->vm))
obj->map_and_fenceable = true;
obj->map_and_fenceable = false;
 
drm_mm_remove_node(&vma->node);
i915_gem_vma_destroy(vma);
2546,9 → 2583,11
 
/* Flush everything onto the inactive list. */
for_each_ring(ring, dev_priv, i) {
if (!i915.enable_execlists) {
ret = i915_switch_context(ring, ring->default_context);
if (ret)
return ret;
}
 
ret = intel_ring_idle(ring);
if (ret)
2707,6 → 2746,7
obj->stride, obj->tiling_mode);
 
switch (INTEL_INFO(dev)->gen) {
case 9:
case 8:
case 7:
case 6:
2762,7 → 2802,6
obj->last_fenced_seqno = 0;
}
 
obj->fenced_gpu_access = false;
return 0;
}
 
2869,6 → 2908,9
return 0;
}
} else if (enable) {
if (WARN_ON(!obj->map_and_fenceable))
return -EINVAL;
 
reg = i915_find_fence_reg(dev);
if (IS_ERR(reg))
return PTR_ERR(reg);
2890,17 → 2932,20
return 0;
}
 
static bool i915_gem_valid_gtt_space(struct drm_device *dev,
struct drm_mm_node *gtt_space,
static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
unsigned long cache_level)
{
struct drm_mm_node *gtt_space = &vma->node;
struct drm_mm_node *other;
 
/* On non-LLC machines we have to be careful when putting differing
* types of snoopable memory together to avoid the prefetcher
* crossing memory domains and dying.
/*
* On some machines we have to be careful when putting differing types
* of snoopable memory together to avoid the prefetcher crossing memory
* domains and dying. During vm initialisation, we decide whether or not
* these constraints apply and set the drm_mm.color_adjust
* appropriately.
*/
if (HAS_LLC(dev))
if (vma->vm->mm.color_adjust == NULL)
return true;
 
if (!drm_mm_node_allocated(gtt_space))
2920,46 → 2965,6
return true;
}
 
static void i915_gem_verify_gtt(struct drm_device *dev)
{
#if WATCH_GTT
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj;
int err = 0;
 
list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
if (obj->gtt_space == NULL) {
printk(KERN_ERR "object found on GTT list with no space reserved\n");
err++;
continue;
}
 
if (obj->cache_level != obj->gtt_space->color) {
printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
i915_gem_obj_ggtt_offset(obj),
i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
obj->cache_level,
obj->gtt_space->color);
err++;
continue;
}
 
if (!i915_gem_valid_gtt_space(dev,
obj->gtt_space,
obj->cache_level)) {
printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
i915_gem_obj_ggtt_offset(obj),
i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
obj->cache_level);
err++;
continue;
}
}
 
WARN_ON(err);
#endif
}
 
/**
* Finds free space in the GTT aperture and binds the object there.
*/
3032,8 → 3037,7
 
goto err_free_vma;
}
if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
obj->cache_level))) {
if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
ret = -EINVAL;
goto err_remove_node;
}
3045,25 → 3049,10
list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
list_add_tail(&vma->mm_list, &vm->inactive_list);
 
if (i915_is_ggtt(vm)) {
bool mappable, fenceable;
 
fenceable = (vma->node.size == fence_size &&
(vma->node.start & (fence_alignment - 1)) == 0);
 
mappable = (vma->node.start + obj->base.size <=
dev_priv->gtt.mappable_end);
 
obj->map_and_fenceable = mappable && fenceable;
}
 
WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
 
trace_i915_vma_bind(vma, flags);
vma->bind_vma(vma, obj->cache_level,
flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
flags & PIN_GLOBAL ? GLOBAL_BIND : 0);
 
i915_gem_verify_gtt(dev);
return vma;
 
err_remove_node:
3091,7 → 3080,7
* Stolen memory is always coherent with the GPU as it is explicitly
* marked as wc by the system, or the system is cache-coherent.
*/
if (obj->stolen)
if (obj->stolen || obj->phys_handle)
return false;
 
/* If the GPU is snooping the contents of the CPU cache,
3133,6 → 3122,8
old_write_domain = obj->base.write_domain;
obj->base.write_domain = 0;
 
intel_fb_obj_flush(obj, false);
 
trace_i915_gem_object_change_domain(obj,
obj->base.read_domains,
old_write_domain);
3154,6 → 3145,8
old_write_domain = obj->base.write_domain;
obj->base.write_domain = 0;
 
intel_fb_obj_flush(obj, false);
 
trace_i915_gem_object_change_domain(obj,
obj->base.read_domains,
old_write_domain);
3169,11 → 3162,12
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
{
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
uint32_t old_write_domain, old_read_domains;
int ret;
 
/* Not valid to be called on unbound objects. */
if (!i915_gem_obj_bound_any(obj))
if (vma == NULL)
return -EINVAL;
 
if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3207,19 → 3201,18
obj->dirty = 1;
}
 
if (write)
intel_fb_obj_invalidate(obj, NULL);
 
trace_i915_gem_object_change_domain(obj,
old_read_domains,
old_write_domain);
 
/* And bump the LRU for this access */
if (i915_gem_object_is_inactive(obj)) {
struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
if (vma)
if (i915_gem_object_is_inactive(obj))
list_move_tail(&vma->mm_list,
&dev_priv->gtt.base.inactive_list);
 
}
 
return 0;
}
 
3239,7 → 3232,7
}
 
list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
if (!i915_gem_valid_gtt_space(vma, cache_level)) {
ret = i915_vma_unbind(vma);
if (ret)
return ret;
3266,7 → 3259,7
list_for_each_entry(vma, &obj->vma_list, vma_link)
if (drm_mm_node_allocated(&vma->node))
vma->bind_vma(vma, cache_level,
obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
vma->bound & GLOBAL_BIND);
}
 
list_for_each_entry(vma, &obj->vma_list, vma_link)
3296,7 → 3289,6
old_write_domain);
}
 
i915_gem_verify_gtt(dev);
return 0;
}
 
3382,9 → 3374,6
{
struct i915_vma *vma;
 
if (list_empty(&obj->vma_list))
return false;
 
vma = i915_gem_obj_to_ggtt(obj);
if (!vma)
return false;
3543,6 → 3532,9
obj->base.write_domain = I915_GEM_DOMAIN_CPU;
}
 
if (write)
intel_fb_obj_invalidate(obj, NULL);
 
trace_i915_gem_object_change_domain(obj,
old_read_domains,
old_write_domain);
3594,7 → 3586,7
if (seqno == 0)
return 0;
 
ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
if (ret == 0)
queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 
3628,6 → 3620,7
{
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
struct i915_vma *vma;
unsigned bound;
int ret;
 
if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
3636,6 → 3629,9
if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
return -EINVAL;
 
if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
return -EINVAL;
 
vma = i915_gem_obj_to_vma(obj, vm);
if (vma) {
if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3657,6 → 3653,7
}
}
 
bound = vma ? vma->bound : 0;
if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
if (IS_ERR(vma))
3663,9 → 3660,32
return PTR_ERR(vma);
}
 
if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND))
vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
 
if ((bound ^ vma->bound) & GLOBAL_BIND) {
bool mappable, fenceable;
u32 fence_size, fence_alignment;
 
fence_size = i915_gem_get_gtt_size(obj->base.dev,
obj->base.size,
obj->tiling_mode);
fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
obj->base.size,
obj->tiling_mode,
true);
 
fenceable = (vma->node.size == fence_size &&
(vma->node.start & (fence_alignment - 1)) == 0);
 
mappable = (vma->node.start + obj->base.size <=
dev_priv->gtt.mappable_end);
 
obj->map_and_fenceable = mappable && fenceable;
}
 
WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
 
vma->pin_count++;
if (flags & PIN_MAPPABLE)
obj->pin_mappable |= true;
3720,7 → 3740,7
struct drm_i915_gem_object *obj;
int ret;
 
if (INTEL_INFO(dev)->gen >= 6)
if (drm_core_check_feature(dev, DRIVER_MODESET))
return -ENODEV;
 
ret = i915_mutex_lock_interruptible(dev);
3776,6 → 3796,9
struct drm_i915_gem_object *obj;
int ret;
 
if (drm_core_check_feature(dev, DRIVER_MODESET))
return -ENODEV;
 
ret = i915_mutex_lock_interruptible(dev);
if (ret)
return ret;
3855,6 → 3878,7
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_madvise *args = data;
struct drm_i915_gem_object *obj;
int ret;
3882,6 → 3906,15
goto out;
}
 
if (obj->pages &&
obj->tiling_mode != I915_TILING_NONE &&
dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
if (obj->madv == I915_MADV_WILLNEED)
i915_gem_object_unpin_pages(obj);
if (args->madv == I915_MADV_WILLNEED)
i915_gem_object_pin_pages(obj);
}
 
if (obj->madv != __I915_MADV_PURGED)
obj->madv = args->madv;
 
3911,8 → 3944,6
 
obj->fence_reg = I915_FENCE_REG_NONE;
obj->madv = I915_MADV_WILLNEED;
/* Avoid an unnecessary call to unbind on the first bind. */
obj->map_and_fenceable = true;
 
i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
}
4001,6 → 4032,11
 
WARN_ON(obj->frontbuffer_bits);
 
if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
obj->tiling_mode != I915_TILING_NONE)
i915_gem_object_unpin_pages(obj);
 
if (WARN_ON(obj->pages_pin_count))
obj->pages_pin_count = 0;
i915_gem_object_put_pages(obj);
4037,6 → 4073,7
 
void i915_gem_vma_destroy(struct i915_vma *vma)
{
struct i915_address_space *vm = NULL;
WARN_ON(vma->node.allocated);
 
/* Keep the vma as a placeholder in the execbuffer reservation lists */
4043,6 → 4080,11
if (!list_empty(&vma->exec_list))
return;
 
vm = vma->vm;
 
if (!i915_is_ggtt(vm))
i915_ppgtt_put(i915_vm_to_ppgtt(vm));
 
list_del(&vma->vma_link);
 
kfree(vma);
4056,9 → 4098,6
int ret = 0;
 
mutex_lock(&dev->struct_mutex);
if (dev_priv->ums.mm_suspended)
goto err;
 
ret = i915_gpu_idle(dev);
if (ret)
goto err;
4069,15 → 4108,7
if (!drm_core_check_feature(dev, DRIVER_MODESET))
i915_gem_evict_everything(dev);
 
i915_kernel_lost_context(dev);
i915_gem_stop_ringbuffers(dev);
 
/* Hack! Don't let anybody do execbuf while we don't control the chip.
* We need to replace this with a semaphore, or something.
* And not confound ums.mm_suspended!
*/
dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
DRIVER_MODESET);
mutex_unlock(&dev->struct_mutex);
 
del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4164,11 → 4195,46
return true;
}
 
static int i915_gem_init_rings(struct drm_device *dev)
static void init_unused_ring(struct drm_device *dev, u32 base)
{
struct drm_i915_private *dev_priv = dev->dev_private;
 
I915_WRITE(RING_CTL(base), 0);
I915_WRITE(RING_HEAD(base), 0);
I915_WRITE(RING_TAIL(base), 0);
I915_WRITE(RING_START(base), 0);
}
 
static void init_unused_rings(struct drm_device *dev)
{
if (IS_I830(dev)) {
init_unused_ring(dev, PRB1_BASE);
init_unused_ring(dev, SRB0_BASE);
init_unused_ring(dev, SRB1_BASE);
init_unused_ring(dev, SRB2_BASE);
init_unused_ring(dev, SRB3_BASE);
} else if (IS_GEN2(dev)) {
init_unused_ring(dev, SRB0_BASE);
init_unused_ring(dev, SRB1_BASE);
} else if (IS_GEN3(dev)) {
init_unused_ring(dev, PRB1_BASE);
init_unused_ring(dev, PRB2_BASE);
}
}
 
int i915_gem_init_rings(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int ret;
 
/*
* At least 830 can leave some of the unused rings
* "active" (ie. head != tail) after resume which
* will prevent c3 entry. Makes sure all unused rings
* are totally idle.
*/
init_unused_rings(dev);
 
ret = intel_init_render_ring_buffer(dev);
if (ret)
return ret;
4247,7 → 4313,7
 
i915_gem_init_swizzling(dev);
 
ret = i915_gem_init_rings(dev);
ret = dev_priv->gt.init_rings(dev);
if (ret)
return ret;
 
4265,8 → 4331,16
if (ret && ret != -EIO) {
DRM_ERROR("Context enable failed %d\n", ret);
i915_gem_cleanup_ringbuffer(dev);
 
return ret;
}
 
ret = i915_ppgtt_init_hw(dev);
if (ret && ret != -EIO) {
DRM_ERROR("PPGTT enable failed %d\n", ret);
i915_gem_cleanup_ringbuffer(dev);
}
 
return ret;
}
 
4275,6 → 4349,9
struct drm_i915_private *dev_priv = dev->dev_private;
int ret;
 
i915.enable_execlists = intel_sanitize_enable_execlists(dev,
i915.enable_execlists);
 
mutex_lock(&dev->struct_mutex);
 
if (IS_VALLEYVIEW(dev)) {
4285,6 → 4362,24
DRM_DEBUG_DRIVER("allow wake ack timed out\n");
}
 
if (!i915.enable_execlists) {
dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission;
dev_priv->gt.init_rings = i915_gem_init_rings;
dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
dev_priv->gt.stop_ring = intel_stop_ring_buffer;
} else {
dev_priv->gt.do_execbuf = intel_execlists_submission;
dev_priv->gt.init_rings = intel_logical_rings_init;
dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
dev_priv->gt.stop_ring = intel_logical_ring_stop;
}
 
// ret = i915_gem_init_userptr(dev);
// if (ret) {
// mutex_unlock(&dev->struct_mutex);
// return ret;
// }
 
i915_gem_init_global_gtt(dev);
 
ret = i915_gem_context_init(dev);
4316,80 → 4411,9
int i;
 
for_each_ring(ring, dev_priv, i)
intel_cleanup_ring_buffer(ring);
dev_priv->gt.cleanup_ring(ring);
}
 
#if 0
 
int
i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int ret;
 
if (drm_core_check_feature(dev, DRIVER_MODESET))
return 0;
 
if (i915_reset_in_progress(&dev_priv->gpu_error)) {
DRM_ERROR("Reenabling wedged hardware, good luck\n");
atomic_set(&dev_priv->gpu_error.reset_counter, 0);
}
 
mutex_lock(&dev->struct_mutex);
dev_priv->ums.mm_suspended = 0;
 
ret = i915_gem_init_hw(dev);
if (ret != 0) {
mutex_unlock(&dev->struct_mutex);
return ret;
}
 
BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
 
ret = drm_irq_install(dev, dev->pdev->irq);
if (ret)
goto cleanup_ringbuffer;
mutex_unlock(&dev->struct_mutex);
 
return 0;
 
cleanup_ringbuffer:
i915_gem_cleanup_ringbuffer(dev);
dev_priv->ums.mm_suspended = 1;
mutex_unlock(&dev->struct_mutex);
 
return ret;
}
 
int
i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
if (drm_core_check_feature(dev, DRIVER_MODESET))
return 0;
 
mutex_lock(&dev->struct_mutex);
drm_irq_uninstall(dev);
mutex_unlock(&dev->struct_mutex);
 
return i915_gem_suspend(dev);
}
 
void
i915_gem_lastclose(struct drm_device *dev)
{
int ret;
 
if (drm_core_check_feature(dev, DRIVER_MODESET))
return;
 
ret = i915_gem_suspend(dev);
if (ret)
DRM_ERROR("failed to idle hardware: %d\n", ret);
}
#endif
 
static void
init_ring_lists(struct intel_engine_cs *ring)
{
4433,7 → 4457,7
init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
 
/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
if (IS_GEN3(dev)) {
if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) {
I915_WRITE(MI_ARB_STATE,
_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
}
4440,6 → 4464,10
 
dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
 
/* Old X drivers will take 0-2 for front, back, depth buffers */
if (!drm_core_check_feature(dev, DRIVER_MODESET))
dev_priv->fence_reg_start = 3;
 
if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
dev_priv->num_fence_regs = 32;
else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4485,6 → 4513,15
return ret;
}
 
/**
* i915_gem_track_fb - update frontbuffer tracking
* old: current GEM buffer for the frontbuffer slots
* new: new GEM buffer for the frontbuffer slots
* frontbuffer_bits: bitmask of frontbuffer slots
*
* This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
* from @old and setting them in @new. Both @old and @new can be NULL.
*/
void i915_gem_track_fb(struct drm_i915_gem_object *old,
struct drm_i915_gem_object *new,
unsigned frontbuffer_bits)
4522,9 → 4559,7
struct drm_i915_private *dev_priv = o->base.dev->dev_private;
struct i915_vma *vma;
 
if (!dev_priv->mm.aliasing_ppgtt ||
vm == &dev_priv->mm.aliasing_ppgtt->base)
vm = &dev_priv->gtt.base;
WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
 
list_for_each_entry(vma, &o->vma_list, vma_link) {
if (vma->vm == vm)
4565,9 → 4600,7
struct drm_i915_private *dev_priv = o->base.dev->dev_private;
struct i915_vma *vma;
 
if (!dev_priv->mm.aliasing_ppgtt ||
vm == &dev_priv->mm.aliasing_ppgtt->base)
vm = &dev_priv->gtt.base;
WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
 
BUG_ON(list_empty(&o->vma_list));
 
4584,14 → 4617,8
{
struct i915_vma *vma;
 
/* This WARN has probably outlived its usefulness (callers already
* WARN if they don't find the GGTT vma they expect). When removing,
* remember to remove the pre-check in is_pin_display() as well */
if (WARN_ON(list_empty(&obj->vma_list)))
return NULL;
 
vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
if (vma->vm != obj_to_ggtt(obj))
if (vma->vm != i915_obj_to_ggtt(obj))
return NULL;
 
return vma;