Subversion Repositories Kolibri OS

/*

 * Copyright © 2008,2010 Intel Corporation

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the "Software"),

 * to deal in the Software without restriction, including without limitation

 * the rights to use, copy, modify, merge, publish, distribute, sublicense,

 * and/or sell copies of the Software, and to permit persons to whom the

 * Software is furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice (including the next

 * paragraph) shall be included in all copies or substantial portions of the

 * Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING

 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS

 * IN THE SOFTWARE.

 *

 * Authors:

 *    Eric Anholt 

 *    Chris Wilson 

 *

 */

#include 

#include 

#include "i915_drv.h"

#include "i915_trace.h"

#include "intel_drv.h"

#include 

#define  __EXEC_OBJECT_HAS_PIN (1<<31)

#define  __EXEC_OBJECT_HAS_FENCE (1<<30)

#define  __EXEC_OBJECT_NEEDS_MAP (1<<29)

#define  __EXEC_OBJECT_NEEDS_BIAS (1<<28)

#define BATCH_OFFSET_BIAS (256*1024)

static unsigned long

copy_from_user(void *to, const void __user *from, unsigned long n)

{

    memcpy(to, from, n);

    return 0;

}

struct eb_vmas {

	struct list_head vmas;

	int and;

	union {

		struct i915_vma *lut[0];

		struct hlist_head buckets[0];

	};

};

static struct eb_vmas *

eb_create(struct drm_i915_gem_execbuffer2 *args)

{

	struct eb_vmas *eb = NULL;

	if (args->flags & I915_EXEC_HANDLE_LUT) {

		unsigned size = args->buffer_count;

		size *= sizeof(struct i915_vma *);

		size += sizeof(struct eb_vmas);

		eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);

	}

	if (eb == NULL) {

		unsigned size = args->buffer_count;

		unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;

		BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));

		while (count > 2*size)

		count >>= 1;

	eb = kzalloc(count*sizeof(struct hlist_head) +

			     sizeof(struct eb_vmas),

			     GFP_TEMPORARY);

	if (eb == NULL)

		return eb;

	eb->and = count - 1;

	} else

		eb->and = -args->buffer_count;

	INIT_LIST_HEAD(&eb->vmas);

	return eb;

}

static void

eb_reset(struct eb_vmas *eb)

{

	if (eb->and >= 0)

	memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));

}

static int

eb_lookup_vmas(struct eb_vmas *eb,

		  struct drm_i915_gem_exec_object2 *exec,

		  const struct drm_i915_gem_execbuffer2 *args,

	       struct i915_address_space *vm,

		  struct drm_file *file)

{

	struct drm_i915_gem_object *obj;

	struct list_head objects;

	int i, ret;

	INIT_LIST_HEAD(&objects);

	spin_lock(&file->table_lock);

	/* Grab a reference to the object and release the lock so we can lookup

	 * or create the VMA without using GFP_ATOMIC */

	for (i = 0; i < args->buffer_count; i++) {

        obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));

		if (obj == NULL) {

			spin_unlock(&file->table_lock);

			DRM_DEBUG("Invalid object handle %d at index %d\n",

				   exec[i].handle, i);

			ret = -ENOENT;

			goto err;

		}

		if (!list_empty(&obj->obj_exec_link)) {

			spin_unlock(&file->table_lock);

			DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",

				   obj, exec[i].handle, i);

			ret = -EINVAL;

			goto err;

		}

		drm_gem_object_reference(&obj->base);

		list_add_tail(&obj->obj_exec_link, &objects);

	}

	spin_unlock(&file->table_lock);

	i = 0;

	while (!list_empty(&objects)) {

		struct i915_vma *vma;

		obj = list_first_entry(&objects,

				       struct drm_i915_gem_object,

				       obj_exec_link);

		/*

		 * NOTE: We can leak any vmas created here when something fails

		 * later on. But that's no issue since vma_unbind can deal with

		 * vmas which are not actually bound. And since only

		 * lookup_or_create exists as an interface to get at the vma

		 * from the (obj, vm) we don't run the risk of creating

		 * duplicated vmas for the same vm.

		 */

		vma = i915_gem_obj_lookup_or_create_vma(obj, vm);

		if (IS_ERR(vma)) {

			DRM_DEBUG("Failed to lookup VMA\n");

			ret = PTR_ERR(vma);

			goto err;

		}

		/* Transfer ownership from the objects list to the vmas list. */

		list_add_tail(&vma->exec_list, &eb->vmas);

		list_del_init(&obj->obj_exec_link);

		vma->exec_entry = &exec[i];

		if (eb->and < 0) {

			eb->lut[i] = vma;

		} else {

			uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;

			vma->exec_handle = handle;

			hlist_add_head(&vma->exec_node,

				       &eb->buckets[handle & eb->and]);

		}

		++i;

	}

	return 0;

err:

	while (!list_empty(&objects)) {

		obj = list_first_entry(&objects,

				       struct drm_i915_gem_object,

				       obj_exec_link);

		list_del_init(&obj->obj_exec_link);

		drm_gem_object_unreference(&obj->base);

	}

	/*

	 * Objects already transfered to the vmas list will be unreferenced by

	 * eb_destroy.

	 */

	return ret;

}

static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)

{

	if (eb->and < 0) {

		if (handle >= -eb->and)

			return NULL;

		return eb->lut[handle];

	} else {

	struct hlist_head *head;

	struct hlist_node *node;

	head = &eb->buckets[handle & eb->and];

	hlist_for_each(node, head) {

			struct i915_vma *vma;

			vma = hlist_entry(node, struct i915_vma, exec_node);

			if (vma->exec_handle == handle)

				return vma;

	}

	return NULL;

	}

}

static void

i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)

{

	struct drm_i915_gem_exec_object2 *entry;

	struct drm_i915_gem_object *obj = vma->obj;

	if (!drm_mm_node_allocated(&vma->node))

		return;

	entry = vma->exec_entry;

	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)

		i915_gem_object_unpin_fence(obj);

	if (entry->flags & __EXEC_OBJECT_HAS_PIN)

		vma->pin_count--;

	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);

}

static void eb_destroy(struct eb_vmas *eb)

{

	while (!list_empty(&eb->vmas)) {

		struct i915_vma *vma;

		vma = list_first_entry(&eb->vmas,

				       struct i915_vma,

				       exec_list);

		list_del_init(&vma->exec_list);

		i915_gem_execbuffer_unreserve_vma(vma);

		drm_gem_object_unreference(&vma->obj->base);

	}

	kfree(eb);

}

static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)

{

	return (HAS_LLC(obj->base.dev) ||

		obj->base.write_domain == I915_GEM_DOMAIN_CPU ||

		!obj->map_and_fenceable ||

		obj->cache_level != I915_CACHE_NONE);

}

static int

relocate_entry_cpu(struct drm_i915_gem_object *obj,

		   struct drm_i915_gem_relocation_entry *reloc,

		   uint64_t target_offset)

{

    struct drm_device *dev = obj->base.dev;

    uint32_t page_offset = offset_in_page(reloc->offset);

	uint64_t delta = reloc->delta + target_offset;

	char *vaddr;

	int ret;

	ret = i915_gem_object_set_to_cpu_domain(obj, true);

	if (ret)

		return ret;

	vaddr = kmap_atomic(i915_gem_object_get_page(obj,

				reloc->offset >> PAGE_SHIFT));

	*(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);

	if (INTEL_INFO(dev)->gen >= 8) {

		page_offset = offset_in_page(page_offset + sizeof(uint32_t));

		if (page_offset == 0) {

			kunmap_atomic(vaddr);

			vaddr = kmap_atomic(i915_gem_object_get_page(obj,

			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));

		}

		*(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);

	}

	kunmap_atomic(vaddr);

	return 0;

}

static int

relocate_entry_gtt(struct drm_i915_gem_object *obj,

		   struct drm_i915_gem_relocation_entry *reloc,

		   uint64_t target_offset)

{

	struct drm_device *dev = obj->base.dev;

	struct drm_i915_private *dev_priv = dev->dev_private;

	uint64_t delta = reloc->delta + target_offset;

	uint64_t offset;

	void __iomem *reloc_page;

	int ret;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);

	if (ret)

		return ret;

	ret = i915_gem_object_put_fence(obj);

	if (ret)

		return ret;

	/* Map the page containing the relocation we're going to perform.  */

	offset = i915_gem_obj_ggtt_offset(obj);

	offset += reloc->offset;

    MapPage(dev_priv->gtt.mappable,dev_priv->gtt.mappable_base +

                                 (offset & PAGE_MASK), PG_SW);

	reloc_page = dev_priv->gtt.mappable;

	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));

	return 0;

}

static int

i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,

				   struct eb_vmas *eb,

				   struct drm_i915_gem_relocation_entry *reloc)

{

	struct drm_device *dev = obj->base.dev;

	struct drm_gem_object *target_obj;

	struct drm_i915_gem_object *target_i915_obj;

	struct i915_vma *target_vma;

	uint64_t target_offset;

	int ret;

	/* we've already hold a reference to all valid objects */

	target_vma = eb_get_vma(eb, reloc->target_handle);

	if (unlikely(target_vma == NULL))

		return -ENOENT;

	target_i915_obj = target_vma->obj;

	target_obj = &target_vma->obj->base;

	target_offset = target_vma->node.start;

	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and

	 * pipe_control writes because the gpu doesn't properly redirect them

	 * through the ppgtt for non_secure batchbuffers. */

	if (unlikely(IS_GEN6(dev) &&

	    reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&

	    !(target_vma->bound & GLOBAL_BIND)))

		target_vma->bind_vma(target_vma, target_i915_obj->cache_level,

				GLOBAL_BIND);

	/* Validate that the target is in a valid r/w GPU domain */

	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {

		DRM_DEBUG("reloc with multiple write domains: "

			  "obj %p target %d offset %d "

			  "read %08x write %08x",

			  obj, reloc->target_handle,

			  (int) reloc->offset,

			  reloc->read_domains,

			  reloc->write_domain);

		return -EINVAL;

	}

	if (unlikely((reloc->write_domain | reloc->read_domains)

		     & ~I915_GEM_GPU_DOMAINS)) {

		DRM_DEBUG("reloc with read/write non-GPU domains: "

			  "obj %p target %d offset %d "

			  "read %08x write %08x",

			  obj, reloc->target_handle,

			  (int) reloc->offset,

			  reloc->read_domains,

			  reloc->write_domain);

		return -EINVAL;

	}

	target_obj->pending_read_domains |= reloc->read_domains;

	target_obj->pending_write_domain |= reloc->write_domain;

	/* If the relocation already has the right value in it, no

	 * more work needs to be done.

	 */

	if (target_offset == reloc->presumed_offset)

		return 0;

	/* Check that the relocation address is valid... */

	if (unlikely(reloc->offset >

		obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {

		DRM_DEBUG("Relocation beyond object bounds: "

			  "obj %p target %d offset %d size %d.\n",

			  obj, reloc->target_handle,

			  (int) reloc->offset,

			  (int) obj->base.size);

		return -EINVAL;

	}

	if (unlikely(reloc->offset & 3)) {

		DRM_DEBUG("Relocation not 4-byte aligned: "

			  "obj %p target %d offset %d.\n",

			  obj, reloc->target_handle,

			  (int) reloc->offset);

		return -EINVAL;

	}

	/* We can't wait for rendering with pagefaults disabled */

	if (use_cpu_reloc(obj))

		ret = relocate_entry_cpu(obj, reloc, target_offset);

	else

		ret = relocate_entry_gtt(obj, reloc, target_offset);

		if (ret)

			return ret;

	/* and update the user's relocation entry */

	reloc->presumed_offset = target_offset;

	return 0;

}

static int

i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,

				 struct eb_vmas *eb)

{

#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))

	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(64)];

	struct drm_i915_gem_relocation_entry __user *user_relocs;

	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

	int remain, ret;

	user_relocs = to_user_ptr(entry->relocs_ptr);

	remain = entry->relocation_count;

	while (remain) {

		struct drm_i915_gem_relocation_entry *r = stack_reloc;

		int count = remain;

		if (count > ARRAY_SIZE(stack_reloc))

			count = ARRAY_SIZE(stack_reloc);

		remain -= count;

        memcpy(r, user_relocs, count*sizeof(r[0]));

		do {

			u64 offset = r->presumed_offset;

			ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);

			if (ret)

				return ret;

		if (r->presumed_offset != offset)

		{

            memcpy(&user_relocs->presumed_offset,

                   &r->presumed_offset,

                   sizeof(r->presumed_offset));

		}

			user_relocs++;

			r++;

		} while (--count);

	}

	return 0;

#undef N_RELOC

}

static int

i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,

				      struct eb_vmas *eb,

				      struct drm_i915_gem_relocation_entry *relocs)

{

	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

	int i, ret;

	for (i = 0; i < entry->relocation_count; i++) {

		ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);

		if (ret)

			return ret;

	}

	return 0;

}

static int

i915_gem_execbuffer_relocate(struct eb_vmas *eb)

{

	struct i915_vma *vma;

	int ret = 0;

	/* This is the fast path and we cannot handle a pagefault whilst

	 * holding the struct mutex lest the user pass in the relocations

	 * contained within a mmaped bo. For in such a case we, the page

	 * fault handler would call i915_gem_fault() and we would try to

	 * acquire the struct mutex again. Obviously this is bad and so

	 * lockdep complains vehemently.

	 */

//	pagefault_disable();

	list_for_each_entry(vma, &eb->vmas, exec_list) {

		ret = i915_gem_execbuffer_relocate_vma(vma, eb);

		if (ret)

			break;

	}

//   pagefault_enable();

	return ret;

}

static int

i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,

				struct intel_engine_cs *ring,

				   bool *need_reloc)

{

	struct drm_i915_gem_object *obj = vma->obj;

	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

	uint64_t flags;

	int ret;

	flags = 0;

	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)

		flags |= PIN_GLOBAL | PIN_MAPPABLE;

	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)

		flags |= PIN_GLOBAL;

	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)

		flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;

	ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);

	if (ret)

		return ret;

	entry->flags |= __EXEC_OBJECT_HAS_PIN;

		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {

			ret = i915_gem_object_get_fence(obj);

			if (ret)

				return ret;

			if (i915_gem_object_pin_fence(obj))

				entry->flags |= __EXEC_OBJECT_HAS_FENCE;

	}

	if (entry->offset != vma->node.start) {

		entry->offset = vma->node.start;

		*need_reloc = true;

	}

	if (entry->flags & EXEC_OBJECT_WRITE) {

		obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;

		obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;

	}

	return 0;

}

static bool

need_reloc_mappable(struct i915_vma *vma)

{

	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

	if (entry->relocation_count == 0)

		return false;

	if (!i915_is_ggtt(vma->vm))

		return false;

	/* See also use_cpu_reloc() */

	if (HAS_LLC(vma->obj->base.dev))

		return false;

	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)

		return false;

	return true;

}

static bool

eb_vma_misplaced(struct i915_vma *vma)

{

	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

	struct drm_i915_gem_object *obj = vma->obj;

	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&

	       !i915_is_ggtt(vma->vm));

	if (entry->alignment &&

	    vma->node.start & (entry->alignment - 1))

		return true;

	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)

		return true;

	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&

	    vma->node.start < BATCH_OFFSET_BIAS)

		return true;

	return false;

}

static int

i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,

			    struct list_head *vmas,

			    bool *need_relocs)

{

	struct drm_i915_gem_object *obj;

	struct i915_vma *vma;

	struct i915_address_space *vm;

	struct list_head ordered_vmas;

	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;

	int retry;

	i915_gem_retire_requests_ring(ring);

	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;

	INIT_LIST_HEAD(&ordered_vmas);

	while (!list_empty(vmas)) {

		struct drm_i915_gem_exec_object2 *entry;

		bool need_fence, need_mappable;

		vma = list_first_entry(vmas, struct i915_vma, exec_list);

		obj = vma->obj;

		entry = vma->exec_entry;

		if (!has_fenced_gpu_access)

			entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;

		need_fence =

			entry->flags & EXEC_OBJECT_NEEDS_FENCE &&

			obj->tiling_mode != I915_TILING_NONE;

		need_mappable = need_fence || need_reloc_mappable(vma);

		if (need_mappable) {

			entry->flags |= __EXEC_OBJECT_NEEDS_MAP;

			list_move(&vma->exec_list, &ordered_vmas);

		} else

			list_move_tail(&vma->exec_list, &ordered_vmas);

		obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;

		obj->base.pending_write_domain = 0;

	}

	list_splice(&ordered_vmas, vmas);

	/* Attempt to pin all of the buffers into the GTT.

	 * This is done in 3 phases:

	 *

	 * 1a. Unbind all objects that do not match the GTT constraints for

	 *     the execbuffer (fenceable, mappable, alignment etc).

	 * 1b. Increment pin count for already bound objects.

	 * 2.  Bind new objects.

	 * 3.  Decrement pin count.

	 *

	 * This avoid unnecessary unbinding of later objects in order to make

	 * room for the earlier objects *unless* we need to defragment.

	 */

	retry = 0;

	do {

		int ret = 0;

		/* Unbind any ill-fitting objects or pin. */

		list_for_each_entry(vma, vmas, exec_list) {

			if (!drm_mm_node_allocated(&vma->node))

				continue;

			if (eb_vma_misplaced(vma))

				ret = i915_vma_unbind(vma);

			else

				ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);

			if (ret)

				goto err;

		}

		/* Bind fresh objects */

		list_for_each_entry(vma, vmas, exec_list) {

			if (drm_mm_node_allocated(&vma->node))

				continue;

			ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);

			if (ret)

				goto err;

		}

err:

		if (ret != -ENOSPC || retry++)

			return ret;

		/* Decrement pin count for bound objects */

		list_for_each_entry(vma, vmas, exec_list)

			i915_gem_execbuffer_unreserve_vma(vma);

		ret = i915_gem_evict_vm(vm, true);

		if (ret)

			return ret;

	} while (1);

}

static int

i915_gem_execbuffer_relocate_slow(struct drm_device *dev,

				  struct drm_i915_gem_execbuffer2 *args,

				  struct drm_file *file,

				  struct intel_engine_cs *ring,

				  struct eb_vmas *eb,

				  struct drm_i915_gem_exec_object2 *exec)

{

	struct drm_i915_gem_relocation_entry *reloc;

	struct i915_address_space *vm;

	struct i915_vma *vma;

	bool need_relocs;

	int *reloc_offset;

	int i, total, ret;

	unsigned count = args->buffer_count;

	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;

	/* We may process another execbuffer during the unlock... */

	while (!list_empty(&eb->vmas)) {

		vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);

		list_del_init(&vma->exec_list);

		i915_gem_execbuffer_unreserve_vma(vma);

		drm_gem_object_unreference(&vma->obj->base);

	}

	mutex_unlock(&dev->struct_mutex);

	total = 0;

	for (i = 0; i < count; i++)

		total += exec[i].relocation_count;

    reloc_offset = malloc(count * sizeof(*reloc_offset));

    reloc = malloc(total * sizeof(*reloc));

	if (reloc == NULL || reloc_offset == NULL) {

        kfree(reloc);

        kfree(reloc_offset);

		mutex_lock(&dev->struct_mutex);

		return -ENOMEM;

	}

	total = 0;

	for (i = 0; i < count; i++) {

		struct drm_i915_gem_relocation_entry __user *user_relocs;

		u64 invalid_offset = (u64)-1;

		int j;

		user_relocs = to_user_ptr(exec[i].relocs_ptr);

		if (copy_from_user(reloc+total, user_relocs,

				   exec[i].relocation_count * sizeof(*reloc))) {

			ret = -EFAULT;

			mutex_lock(&dev->struct_mutex);

			goto err;

		}

		/* As we do not update the known relocation offsets after

		 * relocating (due to the complexities in lock handling),

		 * we need to mark them as invalid now so that we force the

		 * relocation processing next time. Just in case the target

		 * object is evicted and then rebound into its old

		 * presumed_offset before the next execbuffer - if that

		 * happened we would make the mistake of assuming that the

		 * relocations were valid.

		 */

		for (j = 0; j < exec[i].relocation_count; j++) {

			if (__copy_to_user(&user_relocs[j].presumed_offset,

					 &invalid_offset,

					 sizeof(invalid_offset))) {

				ret = -EFAULT;

				mutex_lock(&dev->struct_mutex);

				goto err;

			}

		}

		reloc_offset[i] = total;

		total += exec[i].relocation_count;

	}

	ret = i915_mutex_lock_interruptible(dev);

	if (ret) {

		mutex_lock(&dev->struct_mutex);

		goto err;

	}

	/* reacquire the objects */

	eb_reset(eb);

	ret = eb_lookup_vmas(eb, exec, args, vm, file);

	if (ret)

			goto err;

	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;

	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);

	if (ret)

		goto err;

	list_for_each_entry(vma, &eb->vmas, exec_list) {

		int offset = vma->exec_entry - exec;

		ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,

							    reloc + reloc_offset[offset]);

		if (ret)

			goto err;

	}

	/* Leave the user relocations as are, this is the painfully slow path,

	 * and we want to avoid the complication of dropping the lock whilst

	 * having buffers reserved in the aperture and so causing spurious

	 * ENOSPC for random operations.

	 */

err:

    kfree(reloc);

    kfree(reloc_offset);

	return ret;

}

static int

i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring,

				struct list_head *vmas)

{

	struct i915_vma *vma;

	uint32_t flush_domains = 0;

	bool flush_chipset = false;

	int ret;

	list_for_each_entry(vma, vmas, exec_list) {

		struct drm_i915_gem_object *obj = vma->obj;

		ret = i915_gem_object_sync(obj, ring);

		if (ret)

			return ret;

		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)

			flush_chipset |= i915_gem_clflush_object(obj, false);

		flush_domains |= obj->base.write_domain;

	}

	if (flush_chipset)

		i915_gem_chipset_flush(ring->dev);

	if (flush_domains & I915_GEM_DOMAIN_GTT)

		wmb();

	/* Unconditionally invalidate gpu caches and ensure that we do flush

	 * any residual writes from the previous batch.

	 */

	return intel_ring_invalidate_all_caches(ring);

}

static bool

i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)

{

	if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)

		return false;

	return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;

}

static int

validate_exec_list(struct drm_device *dev,

		   struct drm_i915_gem_exec_object2 *exec,

		   int count)

{

	unsigned relocs_total = 0;

	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);

	unsigned invalid_flags;

	int i;

	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;

	if (USES_FULL_PPGTT(dev))

		invalid_flags |= EXEC_OBJECT_NEEDS_GTT;

	for (i = 0; i < count; i++) {

		char __user *ptr = to_user_ptr(exec[i].relocs_ptr);

		int length; /* limited by fault_in_pages_readable() */

		if (exec[i].flags & invalid_flags)

			return -EINVAL;

		/* First check for malicious input causing overflow in

		 * the worst case where we need to allocate the entire

		 * relocation tree as a single array.

		 */

		if (exec[i].relocation_count > relocs_max - relocs_total)

			return -EINVAL;

		relocs_total += exec[i].relocation_count;

		length = exec[i].relocation_count *

			sizeof(struct drm_i915_gem_relocation_entry);

		/*

		 * We must check that the entire relocation array is safe

		 * to read, but since we may need to update the presumed

		 * offsets during execution, check for full write access.

		 */

	}

	return 0;

}

static struct intel_context *

i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,

			  struct intel_engine_cs *ring, const u32 ctx_id)

{

	struct intel_context *ctx = NULL;

	struct i915_ctx_hang_stats *hs;

	if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)

		return ERR_PTR(-EINVAL);

	ctx = i915_gem_context_get(file->driver_priv, ctx_id);

	if (IS_ERR(ctx))

		return ctx;

	hs = &ctx->hang_stats;

	if (hs->banned) {

		DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);

		return ERR_PTR(-EIO);

	}

	if (i915.enable_execlists && !ctx->engine[ring->id].state) {

		int ret = intel_lr_context_deferred_create(ctx, ring);

		if (ret) {

			DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);

			return ERR_PTR(ret);

		}

	}

	return ctx;

}

void

i915_gem_execbuffer_move_to_active(struct list_head *vmas,

				   struct intel_engine_cs *ring)

{

	u32 seqno = intel_ring_get_seqno(ring);

	struct i915_vma *vma;

	list_for_each_entry(vma, vmas, exec_list) {

		struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

		struct drm_i915_gem_object *obj = vma->obj;

		u32 old_read = obj->base.read_domains;

		u32 old_write = obj->base.write_domain;

		obj->base.write_domain = obj->base.pending_write_domain;

		if (obj->base.write_domain == 0)

			obj->base.pending_read_domains |= obj->base.read_domains;

		obj->base.read_domains = obj->base.pending_read_domains;

		i915_vma_move_to_active(vma, ring);

		if (obj->base.write_domain) {

			obj->dirty = 1;

			obj->last_write_seqno = seqno;

			intel_fb_obj_invalidate(obj, ring);

			/* update for the implicit flush after a batch */

			obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;

		}

		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {

			obj->last_fenced_seqno = seqno;

			if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {

				struct drm_i915_private *dev_priv = to_i915(ring->dev);

				list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,

					       &dev_priv->mm.fence_list);

			}

		}

		trace_i915_gem_object_change_domain(obj, old_read, old_write);

	}

}

void

i915_gem_execbuffer_retire_commands(struct drm_device *dev,

				    struct drm_file *file,

				    struct intel_engine_cs *ring,

				    struct drm_i915_gem_object *obj)

{

	/* Unconditionally force add_request to emit a full flush. */

	ring->gpu_caches_dirty = true;

	/* Add a breadcrumb for the completion of the batch buffer */

	(void)__i915_add_request(ring, file, obj, NULL);

}

static int

i915_reset_gen7_sol_offsets(struct drm_device *dev,

			    struct intel_engine_cs *ring)

{

	struct drm_i915_private *dev_priv = dev->dev_private;

	int ret, i;

	if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) {

		DRM_DEBUG("sol reset is gen7/rcs only\n");

		return -EINVAL;

	}

	ret = intel_ring_begin(ring, 4 * 3);

	if (ret)

		return ret;

	for (i = 0; i < 4; i++) {

		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));

		intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));

		intel_ring_emit(ring, 0);

	}

	intel_ring_advance(ring);

	return 0;

}

static int

i915_emit_box(struct intel_engine_cs *ring,

	      struct drm_clip_rect *box,

	      int DR1, int DR4)

{

	int ret;

	if (box->y2 <= box->y1 || box->x2 <= box->x1 ||

	    box->y2 <= 0 || box->x2 <= 0) {

		DRM_ERROR("Bad box %d,%d..%d,%d\n",

			  box->x1, box->y1, box->x2, box->y2);

		return -EINVAL;

	}

	if (INTEL_INFO(ring->dev)->gen >= 4) {

		ret = intel_ring_begin(ring, 4);

		if (ret)

			return ret;

		intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO_I965);

		intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16);

		intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16);

		intel_ring_emit(ring, DR4);

	} else {

		ret = intel_ring_begin(ring, 6);

		if (ret)

			return ret;

		intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO);

		intel_ring_emit(ring, DR1);

		intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16);

		intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16);

		intel_ring_emit(ring, DR4);

		intel_ring_emit(ring, 0);

	}

	intel_ring_advance(ring);

	return 0;

}

int

i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,

			     struct intel_engine_cs *ring,

			     struct intel_context *ctx,

			     struct drm_i915_gem_execbuffer2 *args,

			     struct list_head *vmas,

			     struct drm_i915_gem_object *batch_obj,

			     u64 exec_start, u32 flags)

{

	struct drm_clip_rect *cliprects = NULL;

	struct drm_i915_private *dev_priv = dev->dev_private;

	u64 exec_len;

	int instp_mode;

	u32 instp_mask;

	int i, ret = 0;

	if (args->num_cliprects != 0) {

		if (ring != &dev_priv->ring[RCS]) {

			DRM_DEBUG("clip rectangles are only valid with the render ring\n");

			return -EINVAL;

		}

		if (INTEL_INFO(dev)->gen >= 5) {

			DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");

			return -EINVAL;

		}

		if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {

			DRM_DEBUG("execbuf with %u cliprects\n",