WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem_execbuffer.c

Rev	Author	Line No.	Line
3263	Serge	1	/*
		2	* Copyright © 2008,2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Chris Wilson
		26	*
		27	*/
		28
		29	#include
		30	#include
		31	#include "i915_drv.h"
		32	#include "i915_trace.h"
		33	#include "intel_drv.h"
5060	serge	34	#include
6935	serge	35	#include
3263	Serge	36
4560	Serge	37	#define __EXEC_OBJECT_HAS_PIN (1<<31)
		38	#define __EXEC_OBJECT_HAS_FENCE (1<<30)
5354	serge	39	#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
5060	serge	40	#define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
3263	Serge	41
5060	serge	42	#define BATCH_OFFSET_BIAS (256*1024)
3263	Serge	43
4560	Serge	44	struct eb_vmas {
		45	struct list_head vmas;
3263	Serge	46	int and;
3480	Serge	47	union {
4560	Serge	48	struct i915_vma *lut[0];
5060	serge	49	struct hlist_head buckets[0];
3480	Serge	50	};
3263	Serge	51	};
		52
4560	Serge	53	static struct eb_vmas *
3480	Serge	54	eb_create(struct drm_i915_gem_execbuffer2 *args)
3263	Serge	55	{
4560	Serge	56	struct eb_vmas *eb = NULL;
3480	Serge	57
		58	if (args->flags & I915_EXEC_HANDLE_LUT) {
4560	Serge	59	unsigned size = args->buffer_count;
		60	size = sizeof(struct i915_vma );
		61	size += sizeof(struct eb_vmas);
3480	Serge	62	eb = kmalloc(size, GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
		63	}
		64
		65	if (eb == NULL) {
4560	Serge	66	unsigned size = args->buffer_count;
		67	unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
3480	Serge	68	BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
		69	while (count > 2*size)
6084	serge	70	count >>= 1;
		71	eb = kzalloc(count*sizeof(struct hlist_head) +
4560	Serge	72	sizeof(struct eb_vmas),
3480	Serge	73	GFP_TEMPORARY);
6084	serge	74	if (eb == NULL)
		75	return eb;
3263	Serge	76
6084	serge	77	eb->and = count - 1;
3480	Serge	78	} else
		79	eb->and = -args->buffer_count;
		80
4560	Serge	81	INIT_LIST_HEAD(&eb->vmas);
3263	Serge	82	return eb;
		83	}
		84
		85	static void
4560	Serge	86	eb_reset(struct eb_vmas *eb)
3263	Serge	87	{
3480	Serge	88	if (eb->and >= 0)
6084	serge	89	memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
3263	Serge	90	}
		91
3480	Serge	92	static int
4560	Serge	93	eb_lookup_vmas(struct eb_vmas *eb,
6084	serge	94	struct drm_i915_gem_exec_object2 *exec,
		95	const struct drm_i915_gem_execbuffer2 *args,
4560	Serge	96	struct i915_address_space *vm,
6084	serge	97	struct drm_file *file)
3263	Serge	98	{
4560	Serge	99	struct drm_i915_gem_object *obj;
		100	struct list_head objects;
		101	int i, ret;
3480	Serge	102
4560	Serge	103	INIT_LIST_HEAD(&objects);
3480	Serge	104	spin_lock(&file->table_lock);
4560	Serge	105	/* Grab a reference to the object and release the lock so we can lookup
		106	* or create the VMA without using GFP_ATOMIC */
3480	Serge	107	for (i = 0; i < args->buffer_count; i++) {
6084	serge	108	obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
3480	Serge	109	if (obj == NULL) {
		110	spin_unlock(&file->table_lock);
		111	DRM_DEBUG("Invalid object handle %d at index %d\n",
		112	exec[i].handle, i);
4560	Serge	113	ret = -ENOENT;
		114	goto err;
3480	Serge	115	}
		116
4560	Serge	117	if (!list_empty(&obj->obj_exec_link)) {
3480	Serge	118	spin_unlock(&file->table_lock);
		119	DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
		120	obj, exec[i].handle, i);
4560	Serge	121	ret = -EINVAL;
		122	goto err;
3480	Serge	123	}
		124
		125	drm_gem_object_reference(&obj->base);
4560	Serge	126	list_add_tail(&obj->obj_exec_link, &objects);
		127	}
		128	spin_unlock(&file->table_lock);
3480	Serge	129
4560	Serge	130	i = 0;
		131	while (!list_empty(&objects)) {
		132	struct i915_vma *vma;
		133
		134	obj = list_first_entry(&objects,
		135	struct drm_i915_gem_object,
		136	obj_exec_link);
		137
		138	/*
		139	* NOTE: We can leak any vmas created here when something fails
		140	* later on. But that's no issue since vma_unbind can deal with
		141	* vmas which are not actually bound. And since only
		142	* lookup_or_create exists as an interface to get at the vma
		143	* from the (obj, vm) we don't run the risk of creating
		144	* duplicated vmas for the same vm.
		145	*/
5354	serge	146	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
4560	Serge	147	if (IS_ERR(vma)) {
		148	DRM_DEBUG("Failed to lookup VMA\n");
		149	ret = PTR_ERR(vma);
		150	goto err;
		151	}
		152
		153	/* Transfer ownership from the objects list to the vmas list. */
		154	list_add_tail(&vma->exec_list, &eb->vmas);
		155	list_del_init(&obj->obj_exec_link);
		156
		157	vma->exec_entry = &exec[i];
3480	Serge	158	if (eb->and < 0) {
4560	Serge	159	eb->lut[i] = vma;
3480	Serge	160	} else {
		161	uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
4560	Serge	162	vma->exec_handle = handle;
		163	hlist_add_head(&vma->exec_node,
3480	Serge	164	&eb->buckets[handle & eb->and]);
		165	}
4560	Serge	166	++i;
3480	Serge	167	}
		168
		169	return 0;
4560	Serge	170
		171
		172	err:
		173	while (!list_empty(&objects)) {
		174	obj = list_first_entry(&objects,
		175	struct drm_i915_gem_object,
		176	obj_exec_link);
		177	list_del_init(&obj->obj_exec_link);
		178	drm_gem_object_unreference(&obj->base);
		179	}
		180	/*
		181	* Objects already transfered to the vmas list will be unreferenced by
		182	* eb_destroy.
		183	*/
		184
		185	return ret;
3263	Serge	186	}
		187
4560	Serge	188	static struct i915_vma eb_get_vma(struct eb_vmas eb, unsigned long handle)
3263	Serge	189	{
3480	Serge	190	if (eb->and < 0) {
		191	if (handle >= -eb->and)
		192	return NULL;
		193	return eb->lut[handle];
		194	} else {
6084	serge	195	struct hlist_head *head;
		196	struct hlist_node *node;
3263	Serge	197
6084	serge	198	head = &eb->buckets[handle & eb->and];
		199	hlist_for_each(node, head) {
4560	Serge	200	struct i915_vma *vma;
3480	Serge	201
4560	Serge	202	vma = hlist_entry(node, struct i915_vma, exec_node);
		203	if (vma->exec_handle == handle)
		204	return vma;
6084	serge	205	}
		206	return NULL;
3263	Serge	207	}
		208	}
		209
		210	static void
4560	Serge	211	i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
3263	Serge	212	{
4560	Serge	213	struct drm_i915_gem_exec_object2 *entry;
		214	struct drm_i915_gem_object *obj = vma->obj;
3480	Serge	215
4560	Serge	216	if (!drm_mm_node_allocated(&vma->node))
		217	return;
		218
		219	entry = vma->exec_entry;
		220
		221	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
		222	i915_gem_object_unpin_fence(obj);
		223
		224	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
5060	serge	225	vma->pin_count--;
4560	Serge	226
		227	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE \| __EXEC_OBJECT_HAS_PIN);
		228	}
		229
		230	static void eb_destroy(struct eb_vmas *eb)
		231	{
		232	while (!list_empty(&eb->vmas)) {
		233	struct i915_vma *vma;
		234
		235	vma = list_first_entry(&eb->vmas,
		236	struct i915_vma,
3480	Serge	237	exec_list);
4560	Serge	238	list_del_init(&vma->exec_list);
		239	i915_gem_execbuffer_unreserve_vma(vma);
		240	drm_gem_object_unreference(&vma->obj->base);
3480	Serge	241	}
3263	Serge	242	kfree(eb);
		243	}
		244
		245	static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
		246	{
4560	Serge	247	return (HAS_LLC(obj->base.dev) \|\|
		248	obj->base.write_domain == I915_GEM_DOMAIN_CPU \|\|
3263	Serge	249	obj->cache_level != I915_CACHE_NONE);
		250	}
		251
		252	static int
4371	Serge	253	relocate_entry_cpu(struct drm_i915_gem_object *obj,
5060	serge	254	struct drm_i915_gem_relocation_entry *reloc,
		255	uint64_t target_offset)
4371	Serge	256	{
6084	serge	257	struct drm_device *dev = obj->base.dev;
		258	uint32_t page_offset = offset_in_page(reloc->offset);
5060	serge	259	uint64_t delta = reloc->delta + target_offset;
4371	Serge	260	char *vaddr;
4560	Serge	261	int ret;
4371	Serge	262
4560	Serge	263	ret = i915_gem_object_set_to_cpu_domain(obj, true);
4371	Serge	264	if (ret)
		265	return ret;
		266
5354	serge	267	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		268	reloc->offset >> PAGE_SHIFT));
5060	serge	269	(uint32_t )(vaddr + page_offset) = lower_32_bits(delta);
4371	Serge	270
5060	serge	271	if (INTEL_INFO(dev)->gen >= 8) {
		272	page_offset = offset_in_page(page_offset + sizeof(uint32_t));
		273
		274	if (page_offset == 0) {
5354	serge	275	kunmap_atomic(vaddr);
		276	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		277	(reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
5060	serge	278	}
		279
		280	(uint32_t )(vaddr + page_offset) = upper_32_bits(delta);
		281	}
		282
5354	serge	283	kunmap_atomic(vaddr);
		284
4371	Serge	285	return 0;
		286	}
		287
		288	static int
		289	relocate_entry_gtt(struct drm_i915_gem_object *obj,
5060	serge	290	struct drm_i915_gem_relocation_entry *reloc,
		291	uint64_t target_offset)
4371	Serge	292	{
		293	struct drm_device *dev = obj->base.dev;
		294	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	295	uint64_t delta = reloc->delta + target_offset;
5354	serge	296	uint64_t offset;
4371	Serge	297	void __iomem *reloc_page;
4560	Serge	298	int ret;
4371	Serge	299
		300	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		301	if (ret)
		302	return ret;
		303
		304	ret = i915_gem_object_put_fence(obj);
		305	if (ret)
		306	return ret;
		307
		308	/* Map the page containing the relocation we're going to perform. */
5354	serge	309	offset = i915_gem_obj_ggtt_offset(obj);
		310	offset += reloc->offset;
4539	Serge	311	MapPage(dev_priv->gtt.mappable,dev_priv->gtt.mappable_base +
5354	serge	312	(offset & PAGE_MASK), PG_SW);
4539	Serge	313	reloc_page = dev_priv->gtt.mappable;
5354	serge	314	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
4371	Serge	315
6084	serge	316	// io_mapping_unmap_atomic(reloc_page);
5060	serge	317
6084	serge	318	return 0;
		319	}
5354	serge	320
6084	serge	321	static void
		322	clflush_write32(void *addr, uint32_t value)
		323	{
		324	/* This is not a fast path, so KISS. */
		325	drm_clflush_virt_range(addr, sizeof(uint32_t));
		326	(uint32_t )addr = value;
		327	drm_clflush_virt_range(addr, sizeof(uint32_t));
		328	}
		329
		330	static int
		331	relocate_entry_clflush(struct drm_i915_gem_object *obj,
		332	struct drm_i915_gem_relocation_entry *reloc,
		333	uint64_t target_offset)
		334	{
		335	struct drm_device *dev = obj->base.dev;
		336	uint32_t page_offset = offset_in_page(reloc->offset);
		337	uint64_t delta = (int)reloc->delta + target_offset;
		338	char *vaddr;
		339	int ret;
		340
		341	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		342	if (ret)
		343	return ret;
		344
		345	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		346	reloc->offset >> PAGE_SHIFT));
		347	clflush_write32(vaddr + page_offset, lower_32_bits(delta));
		348
		349	if (INTEL_INFO(dev)->gen >= 8) {
		350	page_offset = offset_in_page(page_offset + sizeof(uint32_t));
		351
		352	if (page_offset == 0) {
		353	kunmap_atomic(vaddr);
		354	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		355	(reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
		356	}
		357
		358	clflush_write32(vaddr + page_offset, upper_32_bits(delta));
		359	}
		360
		361	kunmap_atomic(vaddr);
		362
4371	Serge	363	return 0;
		364	}
		365
		366	static int
3263	Serge	367	i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
4560	Serge	368	struct eb_vmas *eb,
5060	serge	369	struct drm_i915_gem_relocation_entry *reloc)
3263	Serge	370	{
		371	struct drm_device *dev = obj->base.dev;
		372	struct drm_gem_object *target_obj;
		373	struct drm_i915_gem_object *target_i915_obj;
4560	Serge	374	struct i915_vma *target_vma;
5060	serge	375	uint64_t target_offset;
4560	Serge	376	int ret;
3263	Serge	377
		378	/* we've already hold a reference to all valid objects */
4560	Serge	379	target_vma = eb_get_vma(eb, reloc->target_handle);
		380	if (unlikely(target_vma == NULL))
3263	Serge	381	return -ENOENT;
4560	Serge	382	target_i915_obj = target_vma->obj;
		383	target_obj = &target_vma->obj->base;
3263	Serge	384
4560	Serge	385	target_offset = target_vma->node.start;
3263	Serge	386
		387	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
		388	* pipe_control writes because the gpu doesn't properly redirect them
		389	* through the ppgtt for non_secure batchbuffers. */
		390	if (unlikely(IS_GEN6(dev) &&
6084	serge	391	reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
		392	ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
		393	PIN_GLOBAL);
		394	if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
		395	return ret;
		396	}
3263	Serge	397
		398	/* Validate that the target is in a valid r/w GPU domain */
		399	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
		400	DRM_DEBUG("reloc with multiple write domains: "
		401	"obj %p target %d offset %d "
		402	"read %08x write %08x",
		403	obj, reloc->target_handle,
		404	(int) reloc->offset,
		405	reloc->read_domains,
		406	reloc->write_domain);
4560	Serge	407	return -EINVAL;
3263	Serge	408	}
		409	if (unlikely((reloc->write_domain \| reloc->read_domains)
		410	& ~I915_GEM_GPU_DOMAINS)) {
		411	DRM_DEBUG("reloc with read/write non-GPU domains: "
		412	"obj %p target %d offset %d "
		413	"read %08x write %08x",
		414	obj, reloc->target_handle,
		415	(int) reloc->offset,
		416	reloc->read_domains,
		417	reloc->write_domain);
4560	Serge	418	return -EINVAL;
3263	Serge	419	}
		420
		421	target_obj->pending_read_domains \|= reloc->read_domains;
		422	target_obj->pending_write_domain \|= reloc->write_domain;
		423
		424	/* If the relocation already has the right value in it, no
		425	* more work needs to be done.
		426	*/
		427	if (target_offset == reloc->presumed_offset)
		428	return 0;
		429
		430	/* Check that the relocation address is valid... */
4560	Serge	431	if (unlikely(reloc->offset >
		432	obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
3263	Serge	433	DRM_DEBUG("Relocation beyond object bounds: "
		434	"obj %p target %d offset %d size %d.\n",
		435	obj, reloc->target_handle,
		436	(int) reloc->offset,
		437	(int) obj->base.size);
4560	Serge	438	return -EINVAL;
3263	Serge	439	}
		440	if (unlikely(reloc->offset & 3)) {
		441	DRM_DEBUG("Relocation not 4-byte aligned: "
		442	"obj %p target %d offset %d.\n",
		443	obj, reloc->target_handle,
		444	(int) reloc->offset);
4560	Serge	445	return -EINVAL;
3263	Serge	446	}
		447
		448	/* We can't wait for rendering with pagefaults disabled */
		449
4371	Serge	450	if (use_cpu_reloc(obj))
5060	serge	451	ret = relocate_entry_cpu(obj, reloc, target_offset);
6084	serge	452	else if (obj->map_and_fenceable)
5060	serge	453	ret = relocate_entry_gtt(obj, reloc, target_offset);
6084	serge	454	else if (1)
		455	ret = relocate_entry_clflush(obj, reloc, target_offset);
		456	else {
		457	WARN_ONCE(1, "Impossible case in relocation handling\n");
		458	ret = -ENODEV;
		459	}
3263	Serge	460
6084	serge	461	if (ret)
		462	return ret;
3263	Serge	463
		464	/* and update the user's relocation entry */
		465	reloc->presumed_offset = target_offset;
		466
		467	return 0;
		468	}
		469
		470	static int
4560	Serge	471	i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
		472	struct eb_vmas *eb)
3263	Serge	473	{
		474	#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
3266	Serge	475	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(64)];
3263	Serge	476	struct drm_i915_gem_relocation_entry __user *user_relocs;
4560	Serge	477	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
3263	Serge	478	int remain, ret;
		479
4539	Serge	480	user_relocs = to_user_ptr(entry->relocs_ptr);
3263	Serge	481
		482	remain = entry->relocation_count;
		483	while (remain) {
		484	struct drm_i915_gem_relocation_entry *r = stack_reloc;
		485	int count = remain;
		486	if (count > ARRAY_SIZE(stack_reloc))
		487	count = ARRAY_SIZE(stack_reloc);
		488	remain -= count;
		489
		490	memcpy(r, user_relocs, count*sizeof(r[0]));
		491
		492	do {
		493	u64 offset = r->presumed_offset;
		494
5060	serge	495	ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
3263	Serge	496	if (ret)
		497	return ret;
		498
4392	Serge	499	if (r->presumed_offset != offset)
		500	{
3263	Serge	501	memcpy(&user_relocs->presumed_offset,
		502	&r->presumed_offset,
		503	sizeof(r->presumed_offset));
4392	Serge	504	}
3263	Serge	505
		506	user_relocs++;
		507	r++;
		508	} while (--count);
		509	}
		510
		511	return 0;
		512	#undef N_RELOC
		513	}
		514
		515	static int
4560	Serge	516	i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
		517	struct eb_vmas *eb,
		518	struct drm_i915_gem_relocation_entry *relocs)
3263	Serge	519	{
4560	Serge	520	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
3263	Serge	521	int i, ret;
		522
		523	for (i = 0; i < entry->relocation_count; i++) {
5060	serge	524	ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
3263	Serge	525	if (ret)
		526	return ret;
		527	}
		528
		529	return 0;
		530	}
		531
		532	static int
4560	Serge	533	i915_gem_execbuffer_relocate(struct eb_vmas *eb)
3263	Serge	534	{
4560	Serge	535	struct i915_vma *vma;
3263	Serge	536	int ret = 0;
		537
		538	/* This is the fast path and we cannot handle a pagefault whilst
		539	* holding the struct mutex lest the user pass in the relocations
		540	* contained within a mmaped bo. For in such a case we, the page
		541	* fault handler would call i915_gem_fault() and we would try to
		542	* acquire the struct mutex again. Obviously this is bad and so
		543	* lockdep complains vehemently.
		544	*/
6935	serge	545	pagefault_disable();
4560	Serge	546	list_for_each_entry(vma, &eb->vmas, exec_list) {
		547	ret = i915_gem_execbuffer_relocate_vma(vma, eb);
3263	Serge	548	if (ret)
		549	break;
		550	}
6935	serge	551	pagefault_enable();
3263	Serge	552
		553	return ret;
		554	}
		555
6084	serge	556	static bool only_mappable_for_reloc(unsigned int flags)
		557	{
		558	return (flags & (EXEC_OBJECT_NEEDS_FENCE \| __EXEC_OBJECT_NEEDS_MAP)) ==
		559	__EXEC_OBJECT_NEEDS_MAP;
		560	}
		561
3263	Serge	562	static int
4560	Serge	563	i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
5060	serge	564	struct intel_engine_cs *ring,
6084	serge	565	bool *need_reloc)
3263	Serge	566	{
5060	serge	567	struct drm_i915_gem_object *obj = vma->obj;
4560	Serge	568	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5060	serge	569	uint64_t flags;
3263	Serge	570	int ret;
		571
6084	serge	572	flags = PIN_USER;
5060	serge	573	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
		574	flags \|= PIN_GLOBAL;
		575
6084	serge	576	if (!drm_mm_node_allocated(&vma->node)) {
		577	/* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
		578	* limit address to the first 4GBs for unflagged objects.
		579	*/
		580	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
		581	flags \|= PIN_ZONE_4G;
		582	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
		583	flags \|= PIN_GLOBAL \| PIN_MAPPABLE;
		584	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
		585	flags \|= BATCH_OFFSET_BIAS \| PIN_OFFSET_BIAS;
		586	if ((flags & PIN_MAPPABLE) == 0)
		587	flags \|= PIN_HIGH;
		588	}
		589
5060	serge	590	ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
6084	serge	591	if ((ret == -ENOSPC \|\| ret == -E2BIG) &&
		592	only_mappable_for_reloc(entry->flags))
		593	ret = i915_gem_object_pin(obj, vma->vm,
		594	entry->alignment,
		595	flags & ~PIN_MAPPABLE);
3263	Serge	596	if (ret)
		597	return ret;
		598
		599	entry->flags \|= __EXEC_OBJECT_HAS_PIN;
		600
6084	serge	601	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
		602	ret = i915_gem_object_get_fence(obj);
		603	if (ret)
		604	return ret;
3263	Serge	605
6084	serge	606	if (i915_gem_object_pin_fence(obj))
		607	entry->flags \|= __EXEC_OBJECT_HAS_FENCE;
3263	Serge	608	}
		609
4560	Serge	610	if (entry->offset != vma->node.start) {
		611	entry->offset = vma->node.start;
3480	Serge	612	*need_reloc = true;
		613	}
3266	Serge	614
3480	Serge	615	if (entry->flags & EXEC_OBJECT_WRITE) {
		616	obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
		617	obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
		618	}
		619
3263	Serge	620	return 0;
		621	}
		622
5060	serge	623	static bool
5354	serge	624	need_reloc_mappable(struct i915_vma *vma)
5060	serge	625	{
		626	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5354	serge	627
		628	if (entry->relocation_count == 0)
		629	return false;
		630
		631	if (!i915_is_ggtt(vma->vm))
		632	return false;
		633
		634	/* See also use_cpu_reloc() */
		635	if (HAS_LLC(vma->obj->base.dev))
		636	return false;
		637
		638	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		639	return false;
		640
		641	return true;
		642	}
		643
		644	static bool
		645	eb_vma_misplaced(struct i915_vma *vma)
		646	{
		647	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5060	serge	648	struct drm_i915_gem_object *obj = vma->obj;
		649
5354	serge	650	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
5060	serge	651	!i915_is_ggtt(vma->vm));
		652
		653	if (entry->alignment &&
		654	vma->node.start & (entry->alignment - 1))
		655	return true;
		656
		657	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
		658	vma->node.start < BATCH_OFFSET_BIAS)
		659	return true;
		660
6084	serge	661	/* avoid costly ping-pong once a batch bo ended up non-mappable */
		662	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
		663	return !only_mappable_for_reloc(entry->flags);
		664
		665	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
		666	(vma->node.start + vma->node.size - 1) >> 32)
		667	return true;
		668
5060	serge	669	return false;
		670	}
		671
3263	Serge	672	static int
5060	serge	673	i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
4560	Serge	674	struct list_head *vmas,
6084	serge	675	struct intel_context *ctx,
3480	Serge	676	bool *need_relocs)
3263	Serge	677	{
		678	struct drm_i915_gem_object *obj;
4560	Serge	679	struct i915_vma *vma;
		680	struct i915_address_space *vm;
		681	struct list_head ordered_vmas;
3263	Serge	682	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
		683	int retry;
		684
5060	serge	685	i915_gem_retire_requests_ring(ring);
		686
4560	Serge	687	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
		688
		689	INIT_LIST_HEAD(&ordered_vmas);
		690	while (!list_empty(vmas)) {
3263	Serge	691	struct drm_i915_gem_exec_object2 *entry;
		692	bool need_fence, need_mappable;
		693
4560	Serge	694	vma = list_first_entry(vmas, struct i915_vma, exec_list);
		695	obj = vma->obj;
		696	entry = vma->exec_entry;
3263	Serge	697
6084	serge	698	if (ctx->flags & CONTEXT_NO_ZEROMAP)
		699	entry->flags \|= __EXEC_OBJECT_NEEDS_BIAS;
		700
5354	serge	701	if (!has_fenced_gpu_access)
		702	entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
3263	Serge	703	need_fence =
		704	entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
		705	obj->tiling_mode != I915_TILING_NONE;
4560	Serge	706	need_mappable = need_fence \|\| need_reloc_mappable(vma);
3263	Serge	707
5354	serge	708	if (need_mappable) {
		709	entry->flags \|= __EXEC_OBJECT_NEEDS_MAP;
4560	Serge	710	list_move(&vma->exec_list, &ordered_vmas);
5354	serge	711	} else
4560	Serge	712	list_move_tail(&vma->exec_list, &ordered_vmas);
3263	Serge	713
3480	Serge	714	obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
3263	Serge	715	obj->base.pending_write_domain = 0;
		716	}
4560	Serge	717	list_splice(&ordered_vmas, vmas);
3263	Serge	718
		719	/* Attempt to pin all of the buffers into the GTT.
		720	* This is done in 3 phases:
		721	*
		722	* 1a. Unbind all objects that do not match the GTT constraints for
		723	* the execbuffer (fenceable, mappable, alignment etc).
		724	* 1b. Increment pin count for already bound objects.
		725	* 2. Bind new objects.
		726	* 3. Decrement pin count.
		727	*
		728	* This avoid unnecessary unbinding of later objects in order to make
		729	* room for the earlier objects unless we need to defragment.
		730	*/
		731	retry = 0;
		732	do {
		733	int ret = 0;
		734
		735	/* Unbind any ill-fitting objects or pin. */
4560	Serge	736	list_for_each_entry(vma, vmas, exec_list) {
		737	if (!drm_mm_node_allocated(&vma->node))
3263	Serge	738	continue;
		739
5354	serge	740	if (eb_vma_misplaced(vma))
4560	Serge	741	ret = i915_vma_unbind(vma);
3263	Serge	742	else
4560	Serge	743	ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
3263	Serge	744	if (ret)
		745	goto err;
		746	}
		747
		748	/* Bind fresh objects */
4560	Serge	749	list_for_each_entry(vma, vmas, exec_list) {
		750	if (drm_mm_node_allocated(&vma->node))
3263	Serge	751	continue;
		752
4560	Serge	753	ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
3263	Serge	754	if (ret)
		755	goto err;
		756	}
		757
4560	Serge	758	err:
3263	Serge	759	if (ret != -ENOSPC \|\| retry++)
		760	return ret;
		761
4560	Serge	762	/* Decrement pin count for bound objects */
		763	list_for_each_entry(vma, vmas, exec_list)
		764	i915_gem_execbuffer_unreserve_vma(vma);
		765
5060	serge	766	ret = i915_gem_evict_vm(vm, true);
3263	Serge	767	if (ret)
		768	return ret;
		769	} while (1);
		770	}
		771
		772	static int
		773	i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3480	Serge	774	struct drm_i915_gem_execbuffer2 *args,
3263	Serge	775	struct drm_file *file,
5060	serge	776	struct intel_engine_cs *ring,
4560	Serge	777	struct eb_vmas *eb,
6084	serge	778	struct drm_i915_gem_exec_object2 *exec,
		779	struct intel_context *ctx)
3263	Serge	780	{
		781	struct drm_i915_gem_relocation_entry *reloc;
4560	Serge	782	struct i915_address_space *vm;
		783	struct i915_vma *vma;
3480	Serge	784	bool need_relocs;
3263	Serge	785	int *reloc_offset;
		786	int i, total, ret;
4560	Serge	787	unsigned count = args->buffer_count;
3263	Serge	788
4560	Serge	789	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
		790
3263	Serge	791	/* We may process another execbuffer during the unlock... */
4560	Serge	792	while (!list_empty(&eb->vmas)) {
		793	vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
		794	list_del_init(&vma->exec_list);
		795	i915_gem_execbuffer_unreserve_vma(vma);
		796	drm_gem_object_unreference(&vma->obj->base);
3263	Serge	797	}
		798
		799	mutex_unlock(&dev->struct_mutex);
		800
		801	total = 0;
		802	for (i = 0; i < count; i++)
		803	total += exec[i].relocation_count;
		804
6084	serge	805	reloc_offset = __builtin_malloc(count * sizeof(*reloc_offset));
		806	reloc = __builtin_malloc(total * sizeof(*reloc));
3263	Serge	807	if (reloc == NULL \|\| reloc_offset == NULL) {
3266	Serge	808	kfree(reloc);
		809	kfree(reloc_offset);
3263	Serge	810	mutex_lock(&dev->struct_mutex);
		811	return -ENOMEM;
		812	}
		813
		814	total = 0;
		815	for (i = 0; i < count; i++) {
		816	struct drm_i915_gem_relocation_entry __user *user_relocs;
		817	u64 invalid_offset = (u64)-1;
		818	int j;
		819
4539	Serge	820	user_relocs = to_user_ptr(exec[i].relocs_ptr);
3263	Serge	821
		822	if (copy_from_user(reloc+total, user_relocs,
		823	exec[i].relocation_count * sizeof(*reloc))) {
		824	ret = -EFAULT;
		825	mutex_lock(&dev->struct_mutex);
		826	goto err;
		827	}
		828
		829	/* As we do not update the known relocation offsets after
		830	* relocating (due to the complexities in lock handling),
		831	* we need to mark them as invalid now so that we force the
		832	* relocation processing next time. Just in case the target
		833	* object is evicted and then rebound into its old
		834	* presumed_offset before the next execbuffer - if that
		835	* happened we would make the mistake of assuming that the
		836	* relocations were valid.
		837	*/
		838	for (j = 0; j < exec[i].relocation_count; j++) {
5060	serge	839	if (__copy_to_user(&user_relocs[j].presumed_offset,
6084	serge	840	&invalid_offset,
		841	sizeof(invalid_offset))) {
3263	Serge	842	ret = -EFAULT;
		843	mutex_lock(&dev->struct_mutex);
		844	goto err;
		845	}
		846	}
		847
		848	reloc_offset[i] = total;
		849	total += exec[i].relocation_count;
		850	}
		851
		852	ret = i915_mutex_lock_interruptible(dev);
		853	if (ret) {
		854	mutex_lock(&dev->struct_mutex);
		855	goto err;
		856	}
		857
		858	/* reacquire the objects */
		859	eb_reset(eb);
4560	Serge	860	ret = eb_lookup_vmas(eb, exec, args, vm, file);
3480	Serge	861	if (ret)
6084	serge	862	goto err;
3263	Serge	863
3480	Serge	864	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
6084	serge	865	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
3263	Serge	866	if (ret)
		867	goto err;
		868
4560	Serge	869	list_for_each_entry(vma, &eb->vmas, exec_list) {
		870	int offset = vma->exec_entry - exec;
		871	ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
		872	reloc + reloc_offset[offset]);
3263	Serge	873	if (ret)
		874	goto err;
		875	}
		876
		877	/* Leave the user relocations as are, this is the painfully slow path,
		878	* and we want to avoid the complication of dropping the lock whilst
		879	* having buffers reserved in the aperture and so causing spurious
		880	* ENOSPC for random operations.
		881	*/
		882
		883	err:
3266	Serge	884	kfree(reloc);
		885	kfree(reloc_offset);
3263	Serge	886	return ret;
		887	}
		888
		889	static int
6084	serge	890	i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
4560	Serge	891	struct list_head *vmas)
3263	Serge	892	{
6084	serge	893	const unsigned other_rings = ~intel_ring_flag(req->ring);
4560	Serge	894	struct i915_vma *vma;
3263	Serge	895	uint32_t flush_domains = 0;
4104	Serge	896	bool flush_chipset = false;
3263	Serge	897	int ret;
		898
4560	Serge	899	list_for_each_entry(vma, vmas, exec_list) {
		900	struct drm_i915_gem_object *obj = vma->obj;
3263	Serge	901
6084	serge	902	if (obj->active & other_rings) {
		903	ret = i915_gem_object_sync(obj, req->ring, &req);
		904	if (ret)
		905	return ret;
		906	}
		907
3263	Serge	908	if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
4104	Serge	909	flush_chipset \|= i915_gem_clflush_object(obj, false);
3263	Serge	910
		911	flush_domains \|= obj->base.write_domain;
		912	}
		913
4104	Serge	914	if (flush_chipset)
6084	serge	915	i915_gem_chipset_flush(req->ring->dev);
3263	Serge	916
		917	if (flush_domains & I915_GEM_DOMAIN_GTT)
		918	wmb();
		919
		920	/* Unconditionally invalidate gpu caches and ensure that we do flush
		921	* any residual writes from the previous batch.
		922	*/
6084	serge	923	return intel_ring_invalidate_all_caches(req);
3263	Serge	924	}
		925
		926	static bool
		927	i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
		928	{
3480	Serge	929	if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
		930	return false;
		931
6084	serge	932	/* Kernel clipping was a DRI1 misfeature */
		933	if (exec->num_cliprects \|\| exec->cliprects_ptr)
		934	return false;
		935
		936	if (exec->DR4 == 0xffffffff) {
		937	DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
		938	exec->DR4 = 0;
		939	}
		940	if (exec->DR1 \|\| exec->DR4)
		941	return false;
		942
		943	if ((exec->batch_start_offset \| exec->batch_len) & 0x7)
		944	return false;
		945
		946	return true;
3263	Serge	947	}
		948
		949	static int
5354	serge	950	validate_exec_list(struct drm_device *dev,
		951	struct drm_i915_gem_exec_object2 *exec,
3263	Serge	952	int count)
		953	{
4560	Serge	954	unsigned relocs_total = 0;
		955	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
5354	serge	956	unsigned invalid_flags;
		957	int i;
3263	Serge	958
5354	serge	959	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
		960	if (USES_FULL_PPGTT(dev))
		961	invalid_flags \|= EXEC_OBJECT_NEEDS_GTT;
		962
3263	Serge	963	for (i = 0; i < count; i++) {
3746	Serge	964	char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
3263	Serge	965	int length; /* limited by fault_in_pages_readable() */
		966
5354	serge	967	if (exec[i].flags & invalid_flags)
3263	Serge	968	return -EINVAL;
		969
6084	serge	970	if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
		971	return -EINVAL;
		972
3480	Serge	973	/* First check for malicious input causing overflow in
		974	* the worst case where we need to allocate the entire
		975	* relocation tree as a single array.
		976	*/
		977	if (exec[i].relocation_count > relocs_max - relocs_total)
		978	return -EINVAL;
		979	relocs_total += exec[i].relocation_count;
		980
3263	Serge	981	length = exec[i].relocation_count *
		982	sizeof(struct drm_i915_gem_relocation_entry);
3746	Serge	983	/*
		984	* We must check that the entire relocation array is safe
		985	* to read, but since we may need to update the presumed
		986	* offsets during execution, check for full write access.
		987	*/
4560	Serge	988	}
3263	Serge	989
4560	Serge	990	return 0;
		991	}
		992
5060	serge	993	static struct intel_context *
4560	Serge	994	i915_gem_validate_context(struct drm_device dev, struct drm_file file,
5060	serge	995	struct intel_engine_cs *ring, const u32 ctx_id)
4560	Serge	996	{
5060	serge	997	struct intel_context *ctx = NULL;
4560	Serge	998	struct i915_ctx_hang_stats *hs;
		999
5060	serge	1000	if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
		1001	return ERR_PTR(-EINVAL);
4560	Serge	1002
5060	serge	1003	ctx = i915_gem_context_get(file->driver_priv, ctx_id);
		1004	if (IS_ERR(ctx))
		1005	return ctx;
		1006
		1007	hs = &ctx->hang_stats;
4560	Serge	1008	if (hs->banned) {
		1009	DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
5060	serge	1010	return ERR_PTR(-EIO);
3263	Serge	1011	}
		1012
5354	serge	1013	if (i915.enable_execlists && !ctx->engine[ring->id].state) {
6084	serge	1014	int ret = intel_lr_context_deferred_alloc(ctx, ring);
5354	serge	1015	if (ret) {
		1016	DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
		1017	return ERR_PTR(ret);
		1018	}
		1019	}
		1020
5060	serge	1021	return ctx;
3263	Serge	1022	}
		1023
5354	serge	1024	void
4560	Serge	1025	i915_gem_execbuffer_move_to_active(struct list_head *vmas,
6084	serge	1026	struct drm_i915_gem_request *req)
3263	Serge	1027	{
6084	serge	1028	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
4560	Serge	1029	struct i915_vma *vma;
3263	Serge	1030
4560	Serge	1031	list_for_each_entry(vma, vmas, exec_list) {
5354	serge	1032	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
4560	Serge	1033	struct drm_i915_gem_object *obj = vma->obj;
3263	Serge	1034	u32 old_read = obj->base.read_domains;
		1035	u32 old_write = obj->base.write_domain;
		1036
6084	serge	1037	obj->dirty = 1; /* be paranoid */
3480	Serge	1038	obj->base.write_domain = obj->base.pending_write_domain;
		1039	if (obj->base.write_domain == 0)
		1040	obj->base.pending_read_domains \|= obj->base.read_domains;
3263	Serge	1041	obj->base.read_domains = obj->base.pending_read_domains;
		1042
6084	serge	1043	i915_vma_move_to_active(vma, req);
3263	Serge	1044	if (obj->base.write_domain) {
6084	serge	1045	i915_gem_request_assign(&obj->last_write_req, req);
5060	serge	1046
6084	serge	1047	intel_fb_obj_invalidate(obj, ORIGIN_CS);
5060	serge	1048
		1049	/* update for the implicit flush after a batch */
		1050	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
3263	Serge	1051	}
5354	serge	1052	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
6084	serge	1053	i915_gem_request_assign(&obj->last_fenced_req, req);
5354	serge	1054	if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
		1055	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		1056	list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
		1057	&dev_priv->mm.fence_list);
		1058	}
		1059	}
3263	Serge	1060
		1061	trace_i915_gem_object_change_domain(obj, old_read, old_write);
		1062	}
		1063	}
		1064
5354	serge	1065	void
6084	serge	1066	i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
3263	Serge	1067	{
		1068	/* Unconditionally force add_request to emit a full flush. */
6084	serge	1069	params->ring->gpu_caches_dirty = true;
3263	Serge	1070
		1071	/* Add a breadcrumb for the completion of the batch buffer */
6084	serge	1072	__i915_add_request(params->request, params->batch_obj, true);
3263	Serge	1073	}
		1074
		1075	static int
		1076	i915_reset_gen7_sol_offsets(struct drm_device *dev,
6084	serge	1077	struct drm_i915_gem_request *req)
3263	Serge	1078	{
6084	serge	1079	struct intel_engine_cs *ring = req->ring;
5060	serge	1080	struct drm_i915_private *dev_priv = dev->dev_private;
3263	Serge	1081	int ret, i;
		1082
5060	serge	1083	if (!IS_GEN7(dev) \|\| ring != &dev_priv->ring[RCS]) {
		1084	DRM_DEBUG("sol reset is gen7/rcs only\n");
		1085	return -EINVAL;
		1086	}
3263	Serge	1087
6084	serge	1088	ret = intel_ring_begin(req, 4 * 3);
3263	Serge	1089	if (ret)
		1090	return ret;
		1091
		1092	for (i = 0; i < 4; i++) {
		1093	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		1094	intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
		1095	intel_ring_emit(ring, 0);
		1096	}
		1097
		1098	intel_ring_advance(ring);
		1099
		1100	return 0;
		1101	}
		1102
6084	serge	1103	static struct drm_i915_gem_object*
		1104	i915_gem_execbuffer_parse(struct intel_engine_cs *ring,
		1105	struct drm_i915_gem_exec_object2 *shadow_exec_entry,
		1106	struct eb_vmas *eb,
		1107	struct drm_i915_gem_object *batch_obj,
		1108	u32 batch_start_offset,
		1109	u32 batch_len,
		1110	bool is_master)
5354	serge	1111	{
6084	serge	1112	struct drm_i915_gem_object *shadow_batch_obj;
		1113	struct i915_vma *vma;
5354	serge	1114	int ret;
		1115
6084	serge	1116	shadow_batch_obj = i915_gem_batch_pool_get(&ring->batch_pool,
		1117	PAGE_ALIGN(batch_len));
		1118	if (IS_ERR(shadow_batch_obj))
		1119	return shadow_batch_obj;
5354	serge	1120
6084	serge	1121	ret = i915_parse_cmds(ring,
		1122	batch_obj,
		1123	shadow_batch_obj,
		1124	batch_start_offset,
		1125	batch_len,
		1126	is_master);
		1127	if (ret)
		1128	goto err;
5354	serge	1129
6084	serge	1130	ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
		1131	if (ret)
		1132	goto err;
5354	serge	1133
6084	serge	1134	i915_gem_object_unpin_pages(shadow_batch_obj);
5354	serge	1135
6084	serge	1136	memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
		1137
		1138	vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
		1139	vma->exec_entry = shadow_exec_entry;
		1140	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
		1141	drm_gem_object_reference(&shadow_batch_obj->base);
		1142	list_add_tail(&vma->exec_list, &eb->vmas);
		1143
		1144	shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
		1145
		1146	return shadow_batch_obj;
		1147
		1148	err:
		1149	i915_gem_object_unpin_pages(shadow_batch_obj);
		1150	if (ret == -EACCES) /* unhandled chained batch */
		1151	return batch_obj;
		1152	else
		1153	return ERR_PTR(ret);
5354	serge	1154	}
		1155
		1156	int
6084	serge	1157	i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
		1158	struct drm_i915_gem_execbuffer2 *args,
		1159	struct list_head *vmas)
5060	serge	1160	{
6084	serge	1161	struct drm_device *dev = params->dev;
		1162	struct intel_engine_cs *ring = params->ring;
5060	serge	1163	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1164	u64 exec_start, exec_len;
5060	serge	1165	int instp_mode;
		1166	u32 instp_mask;
6084	serge	1167	int ret;
5060	serge	1168
6084	serge	1169	ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
5060	serge	1170	if (ret)
6084	serge	1171	return ret;
5060	serge	1172
6084	serge	1173	ret = i915_switch_context(params->request);
5060	serge	1174	if (ret)
6084	serge	1175	return ret;
5060	serge	1176
6084	serge	1177	WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<id),
		1178	"%s didn't clear reload\n", ring->name);
		1179
5060	serge	1180	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
		1181	instp_mask = I915_EXEC_CONSTANTS_MASK;
		1182	switch (instp_mode) {
		1183	case I915_EXEC_CONSTANTS_REL_GENERAL:
		1184	case I915_EXEC_CONSTANTS_ABSOLUTE:
		1185	case I915_EXEC_CONSTANTS_REL_SURFACE:
		1186	if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
		1187	DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
6084	serge	1188	return -EINVAL;
5060	serge	1189	}
		1190
		1191	if (instp_mode != dev_priv->relative_constants_mode) {
		1192	if (INTEL_INFO(dev)->gen < 4) {
		1193	DRM_DEBUG("no rel constants on pre-gen4\n");
6084	serge	1194	return -EINVAL;
5060	serge	1195	}
		1196
		1197	if (INTEL_INFO(dev)->gen > 5 &&
		1198	instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
		1199	DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
6084	serge	1200	return -EINVAL;
5060	serge	1201	}
		1202
		1203	/* The HW changed the meaning on this bit on gen6 */
		1204	if (INTEL_INFO(dev)->gen >= 6)
		1205	instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
		1206	}
		1207	break;
		1208	default:
		1209	DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
6084	serge	1210	return -EINVAL;
5060	serge	1211	}
		1212
		1213	if (ring == &dev_priv->ring[RCS] &&
6084	serge	1214	instp_mode != dev_priv->relative_constants_mode) {
		1215	ret = intel_ring_begin(params->request, 4);
5060	serge	1216	if (ret)
6084	serge	1217	return ret;
5060	serge	1218
		1219	intel_ring_emit(ring, MI_NOOP);
		1220	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		1221	intel_ring_emit(ring, INSTPM);
		1222	intel_ring_emit(ring, instp_mask << 16 \| instp_mode);
		1223	intel_ring_advance(ring);
		1224
		1225	dev_priv->relative_constants_mode = instp_mode;
		1226	}
		1227
		1228	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
6084	serge	1229	ret = i915_reset_gen7_sol_offsets(dev, params->request);
5060	serge	1230	if (ret)
6084	serge	1231	return ret;
5060	serge	1232	}
		1233
6084	serge	1234	exec_len = args->batch_len;
		1235	exec_start = params->batch_obj_vm_offset +
		1236	params->args_batch_start_offset;
5060	serge	1237
6084	serge	1238	ret = ring->dispatch_execbuffer(params->request,
		1239	exec_start, exec_len,
		1240	params->dispatch_flags);
		1241	if (ret)
		1242	return ret;
5060	serge	1243
6084	serge	1244	trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
5060	serge	1245
6084	serge	1246	i915_gem_execbuffer_move_to_active(vmas, params->request);
		1247	i915_gem_execbuffer_retire_commands(params);
5060	serge	1248
6084	serge	1249	return 0;
5060	serge	1250	}
		1251
		1252	/**
		1253	* Find one BSD ring to dispatch the corresponding BSD command.
		1254	* The Ring ID is returned.
		1255	*/
		1256	static int gen8_dispatch_bsd_ring(struct drm_device *dev,
		1257	struct drm_file *file)
		1258	{
		1259	struct drm_i915_private *dev_priv = dev->dev_private;
		1260	struct drm_i915_file_private *file_priv = file->driver_priv;
		1261
		1262	/* Check whether the file_priv is using one ring */
		1263	if (file_priv->bsd_ring)
		1264	return file_priv->bsd_ring->id;
		1265	else {
		1266	/* If no, use the ping-pong mechanism to select one ring */
		1267	int ring_id;
		1268
		1269	mutex_lock(&dev->struct_mutex);
		1270	if (dev_priv->mm.bsd_ring_dispatch_index == 0) {
		1271	ring_id = VCS;
		1272	dev_priv->mm.bsd_ring_dispatch_index = 1;
		1273	} else {
		1274	ring_id = VCS2;
		1275	dev_priv->mm.bsd_ring_dispatch_index = 0;
		1276	}
		1277	file_priv->bsd_ring = &dev_priv->ring[ring_id];
		1278	mutex_unlock(&dev->struct_mutex);
		1279	return ring_id;
		1280	}
		1281	}
		1282
		1283	static struct drm_i915_gem_object *
		1284	eb_get_batch(struct eb_vmas *eb)
		1285	{
		1286	struct i915_vma vma = list_entry(eb->vmas.prev, typeof(vma), exec_list);
		1287
		1288	/*
		1289	* SNA is doing fancy tricks with compressing batch buffers, which leads
		1290	* to negative relocation deltas. Usually that works out ok since the
		1291	* relocate address is still positive, except when the batch is placed
		1292	* very low in the GTT. Ensure this doesn't happen.
		1293	*
		1294	* Note that actual hangs have only been observed on gen7, but for
		1295	* paranoia do it everywhere.
		1296	*/
		1297	vma->exec_entry->flags \|= __EXEC_OBJECT_NEEDS_BIAS;
		1298
		1299	return vma->obj;
		1300	}
		1301
		1302	static int
3263	Serge	1303	i915_gem_do_execbuffer(struct drm_device dev, void data,
		1304	struct drm_file *file,
		1305	struct drm_i915_gem_execbuffer2 *args,
5060	serge	1306	struct drm_i915_gem_exec_object2 *exec)
3263	Serge	1307	{
5060	serge	1308	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1309	struct eb_vmas *eb;
3263	Serge	1310	struct drm_i915_gem_object *batch_obj;
6084	serge	1311	struct drm_i915_gem_exec_object2 shadow_exec_entry;
5060	serge	1312	struct intel_engine_cs *ring;
		1313	struct intel_context *ctx;
		1314	struct i915_address_space *vm;
6084	serge	1315	struct i915_execbuffer_params params_master; /* XXX: will be removed later */
		1316	struct i915_execbuffer_params *params = ¶ms_master;
4560	Serge	1317	const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
6084	serge	1318	u32 dispatch_flags;
5060	serge	1319	int ret;
3480	Serge	1320	bool need_relocs;
3263	Serge	1321
3480	Serge	1322	if (!i915_gem_check_execbuffer(args))
3263	Serge	1323	return -EINVAL;
		1324
5354	serge	1325	ret = validate_exec_list(dev, exec, args->buffer_count);
3263	Serge	1326	if (ret)
		1327	return ret;
		1328
6084	serge	1329	dispatch_flags = 0;
3263	Serge	1330	if (args->flags & I915_EXEC_SECURE) {
		1331
6084	serge	1332	dispatch_flags \|= I915_DISPATCH_SECURE;
3263	Serge	1333	}
		1334	if (args->flags & I915_EXEC_IS_PINNED)
6084	serge	1335	dispatch_flags \|= I915_DISPATCH_PINNED;
3263	Serge	1336
5060	serge	1337	if ((args->flags & I915_EXEC_RING_MASK) > LAST_USER_RING) {
3263	Serge	1338	DRM_DEBUG("execbuf with unknown ring: %d\n",
		1339	(int)(args->flags & I915_EXEC_RING_MASK));
		1340	return -EINVAL;
		1341	}
5060	serge	1342
6084	serge	1343	if (((args->flags & I915_EXEC_RING_MASK) != I915_EXEC_BSD) &&
		1344	((args->flags & I915_EXEC_BSD_MASK) != 0)) {
		1345	DRM_DEBUG("execbuf with non bsd ring but with invalid "
		1346	"bsd dispatch flags: %d\n", (int)(args->flags));
		1347	return -EINVAL;
		1348	}
		1349
5060	serge	1350	if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
		1351	ring = &dev_priv->ring[RCS];
		1352	else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) {
		1353	if (HAS_BSD2(dev)) {
		1354	int ring_id;
6084	serge	1355
		1356	switch (args->flags & I915_EXEC_BSD_MASK) {
		1357	case I915_EXEC_BSD_DEFAULT:
		1358	ring_id = gen8_dispatch_bsd_ring(dev, file);
		1359	ring = &dev_priv->ring[ring_id];
		1360	break;
		1361	case I915_EXEC_BSD_RING1:
		1362	ring = &dev_priv->ring[VCS];
		1363	break;
		1364	case I915_EXEC_BSD_RING2:
		1365	ring = &dev_priv->ring[VCS2];
		1366	break;
		1367	default:
		1368	DRM_DEBUG("execbuf with unknown bsd ring: %d\n",
		1369	(int)(args->flags & I915_EXEC_BSD_MASK));
		1370	return -EINVAL;
		1371	}
5060	serge	1372	} else
		1373	ring = &dev_priv->ring[VCS];
		1374	} else
		1375	ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
		1376
3263	Serge	1377	if (!intel_ring_initialized(ring)) {
		1378	DRM_DEBUG("execbuf with invalid ring: %d\n",
		1379	(int)(args->flags & I915_EXEC_RING_MASK));
		1380	return -EINVAL;
		1381	}
		1382
		1383	if (args->buffer_count < 1) {
		1384	DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
		1385	return -EINVAL;
		1386	}
		1387
6084	serge	1388	if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
		1389	if (!HAS_RESOURCE_STREAMER(dev)) {
		1390	DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
		1391	return -EINVAL;
		1392	}
		1393	if (ring->id != RCS) {
		1394	DRM_DEBUG("RS is not available on %s\n",
		1395	ring->name);
		1396	return -EINVAL;
		1397	}
		1398
		1399	dispatch_flags \|= I915_DISPATCH_RS;
		1400	}
		1401
4560	Serge	1402	intel_runtime_pm_get(dev_priv);
		1403
3263	Serge	1404	ret = i915_mutex_lock_interruptible(dev);
		1405	if (ret)
		1406	goto pre_mutex_err;
		1407
5060	serge	1408	ctx = i915_gem_validate_context(dev, file, ring, ctx_id);
		1409	if (IS_ERR(ctx)) {
4560	Serge	1410	mutex_unlock(&dev->struct_mutex);
5060	serge	1411	ret = PTR_ERR(ctx);
4560	Serge	1412	goto pre_mutex_err;
		1413	}
		1414
5060	serge	1415	i915_gem_context_reference(ctx);
		1416
5354	serge	1417	if (ctx->ppgtt)
		1418	vm = &ctx->ppgtt->base;
		1419	else
5060	serge	1420	vm = &dev_priv->gtt.base;
		1421
6084	serge	1422	memset(¶ms_master, 0x00, sizeof(params_master));
		1423
3480	Serge	1424	eb = eb_create(args);
3263	Serge	1425	if (eb == NULL) {
5060	serge	1426	i915_gem_context_unreference(ctx);
3263	Serge	1427	mutex_unlock(&dev->struct_mutex);
		1428	ret = -ENOMEM;
6084	serge	1429	goto pre_mutex_err;
3263	Serge	1430	}
		1431
		1432	/* Look up object handles */
4560	Serge	1433	ret = eb_lookup_vmas(eb, exec, args, vm, file);
3480	Serge	1434	if (ret)
6084	serge	1435	goto err;
3263	Serge	1436
		1437	/* take note of the batch buffer before we might reorder the lists */
5060	serge	1438	batch_obj = eb_get_batch(eb);
3263	Serge	1439
		1440	/* Move the objects en-masse into the GTT, evicting if necessary. */
3480	Serge	1441	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
6084	serge	1442	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
3263	Serge	1443	if (ret)
		1444	goto err;
		1445
		1446	/* The objects are in their final locations, apply the relocations. */
3480	Serge	1447	if (need_relocs)
4560	Serge	1448	ret = i915_gem_execbuffer_relocate(eb);
3263	Serge	1449	if (ret) {
		1450	if (ret == -EFAULT) {
3480	Serge	1451	ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
6084	serge	1452	eb, exec, ctx);
3263	Serge	1453	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1454	}
		1455	if (ret)
		1456	goto err;
6084	serge	1457	}
3263	Serge	1458
		1459	/* Set the pending read domains for the batch buffer to COMMAND */
		1460	if (batch_obj->base.pending_write_domain) {
		1461	DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
		1462	ret = -EINVAL;
		1463	goto err;
		1464	}
6084	serge	1465
		1466	params->args_batch_start_offset = args->batch_start_offset;
		1467
5060	serge	1468	#if 0
6084	serge	1469	if (i915_needs_cmd_parser(ring) && args->batch_len) {
		1470	struct drm_i915_gem_object *parsed_batch_obj;
		1471
		1472	parsed_batch_obj = i915_gem_execbuffer_parse(ring,
		1473	&shadow_exec_entry,
		1474	eb,
		1475	batch_obj,
		1476	args->batch_start_offset,
		1477	args->batch_len,
		1478	file->is_master);
		1479	if (IS_ERR(parsed_batch_obj)) {
		1480	ret = PTR_ERR(parsed_batch_obj);
5060	serge	1481	goto err;
6084	serge	1482	}
		1483
5060	serge	1484	/*
6084	serge	1485	* parsed_batch_obj == batch_obj means batch not fully parsed:
		1486	* Accept, but don't promote to secure.
5060	serge	1487	*/
6084	serge	1488
		1489	if (parsed_batch_obj != batch_obj) {
		1490	/*
		1491	* Batch parsed and accepted:
		1492	*
		1493	* Set the DISPATCH_SECURE bit to remove the NON_SECURE
		1494	* bit from MI_BATCH_BUFFER_START commands issued in
		1495	* the dispatch_execbuffer implementations. We
		1496	* specifically don't want that set on batches the
		1497	* command parser has accepted.
		1498	*/
		1499	dispatch_flags \|= I915_DISPATCH_SECURE;
		1500	params->args_batch_start_offset = 0;
		1501	batch_obj = parsed_batch_obj;
		1502	}
5060	serge	1503	}
6084	serge	1504	#endif
5367	serge	1505
6084	serge	1506	batch_obj->base.pending_read_domains \|= I915_GEM_DOMAIN_COMMAND;
		1507
3263	Serge	1508	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
		1509	* batch" bit. Hence we need to pin secure batches into the global gtt.
4560	Serge	1510	* hsw should have this fixed, but bdw mucks it up again. */
6084	serge	1511	if (dispatch_flags & I915_DISPATCH_SECURE) {
5354	serge	1512	/*
		1513	* So on first glance it looks freaky that we pin the batch here
		1514	* outside of the reservation loop. But:
		1515	* - The batch is already pinned into the relevant ppgtt, so we
		1516	* already have the backing storage fully allocated.
		1517	* - No other BO uses the global gtt (well contexts, but meh),
6084	serge	1518	* so we don't really have issues with multiple objects not
5354	serge	1519	* fitting due to fragmentation.
		1520	* So this is actually safe.
		1521	*/
		1522	ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
		1523	if (ret)
		1524	goto err;
3263	Serge	1525
6084	serge	1526	params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
5354	serge	1527	} else
6084	serge	1528	params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
3263	Serge	1529
6084	serge	1530	/* Allocate a request for this batch buffer nice and early. */
		1531	ret = i915_gem_request_alloc(ring, ctx, ¶ms->request);
		1532	if (ret)
		1533	goto err_batch_unpin;
3263	Serge	1534
6084	serge	1535	ret = i915_gem_request_add_to_client(params->request, file);
		1536	if (ret)
		1537	goto err_batch_unpin;
		1538
5354	serge	1539	/*
6084	serge	1540	* Save assorted stuff away to pass through to *_submission().
		1541	* NB: This data should be 'persistent' and not local as it will
		1542	* kept around beyond the duration of the IOCTL once the GPU
		1543	* scheduler arrives.
		1544	*/
		1545	params->dev = dev;
		1546	params->file = file;
		1547	params->ring = ring;
		1548	params->dispatch_flags = dispatch_flags;
		1549	params->batch_obj = batch_obj;
		1550	params->ctx = ctx;
		1551
		1552	ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
		1553
		1554	err_batch_unpin:
		1555	/*
5354	serge	1556	* FIXME: We crucially rely upon the active tracking for the (ppgtt)
		1557	* batch vma for correctness. For less ugly and less fragility this
		1558	* needs to be adjusted to also track the ggtt batch vma properly as
		1559	* active.
		1560	*/
6084	serge	1561	if (dispatch_flags & I915_DISPATCH_SECURE)
5354	serge	1562	i915_gem_object_ggtt_unpin(batch_obj);
6084	serge	1563
3263	Serge	1564	err:
5060	serge	1565	/* the request owns the ref now */
		1566	i915_gem_context_unreference(ctx);
3263	Serge	1567	eb_destroy(eb);
		1568
6084	serge	1569	/*
		1570	* If the request was created but not successfully submitted then it
		1571	* must be freed again. If it was submitted then it is being tracked
		1572	* on the active request list and no clean up is required here.
		1573	*/
		1574	if (ret && params->request)
		1575	i915_gem_request_cancel(params->request);
		1576
3263	Serge	1577	mutex_unlock(&dev->struct_mutex);
		1578
		1579	pre_mutex_err:
4560	Serge	1580	/* intel_gpu_busy should also get a ref, so it will free when the device
		1581	* is really idle. */
		1582	intel_runtime_pm_put(dev_priv);
3263	Serge	1583	return ret;
		1584	}
		1585
4246	Serge	1586	#if 0
		1587	/*
		1588	* Legacy execbuffer just creates an exec2 list from the original exec object
		1589	* list array and passes it to the real function.
		1590	*/
		1591	int
		1592	i915_gem_execbuffer(struct drm_device dev, void data,
		1593	struct drm_file *file)
		1594	{
		1595	struct drm_i915_gem_execbuffer *args = data;
		1596	struct drm_i915_gem_execbuffer2 exec2;
		1597	struct drm_i915_gem_exec_object *exec_list = NULL;
		1598	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
		1599	int ret, i;
3480	Serge	1600
4246	Serge	1601	if (args->buffer_count < 1) {
		1602	DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
		1603	return -EINVAL;
		1604	}
3480	Serge	1605
4246	Serge	1606	/* Copy in the exec list from userland */
		1607	exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
		1608	exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
		1609	if (exec_list == NULL \|\| exec2_list == NULL) {
		1610	DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
		1611	args->buffer_count);
		1612	drm_free_large(exec_list);
		1613	drm_free_large(exec2_list);
		1614	return -ENOMEM;
		1615	}
		1616	ret = copy_from_user(exec_list,
		1617	to_user_ptr(args->buffers_ptr),
		1618	sizeof(exec_list) args->buffer_count);
		1619	if (ret != 0) {
		1620	DRM_DEBUG("copy %d exec entries failed %d\n",
		1621	args->buffer_count, ret);
		1622	drm_free_large(exec_list);
		1623	drm_free_large(exec2_list);
		1624	return -EFAULT;
		1625	}
		1626
		1627	for (i = 0; i < args->buffer_count; i++) {
		1628	exec2_list[i].handle = exec_list[i].handle;
		1629	exec2_list[i].relocation_count = exec_list[i].relocation_count;
		1630	exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
		1631	exec2_list[i].alignment = exec_list[i].alignment;
		1632	exec2_list[i].offset = exec_list[i].offset;
		1633	if (INTEL_INFO(dev)->gen < 4)
		1634	exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
		1635	else
		1636	exec2_list[i].flags = 0;
		1637	}
		1638
		1639	exec2.buffers_ptr = args->buffers_ptr;
		1640	exec2.buffer_count = args->buffer_count;
		1641	exec2.batch_start_offset = args->batch_start_offset;
		1642	exec2.batch_len = args->batch_len;
		1643	exec2.DR1 = args->DR1;
		1644	exec2.DR4 = args->DR4;
		1645	exec2.num_cliprects = args->num_cliprects;
		1646	exec2.cliprects_ptr = args->cliprects_ptr;
		1647	exec2.flags = I915_EXEC_RENDER;
		1648	i915_execbuffer2_set_context_id(exec2, 0);
		1649
5060	serge	1650	ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
4246	Serge	1651	if (!ret) {
5060	serge	1652	struct drm_i915_gem_exec_object __user *user_exec_list =
		1653	to_user_ptr(args->buffers_ptr);
		1654
4246	Serge	1655	/* Copy the new buffer offsets back to the user's exec list. */
5060	serge	1656	for (i = 0; i < args->buffer_count; i++) {
		1657	ret = __copy_to_user(&user_exec_list[i].offset,
		1658	&exec2_list[i].offset,
		1659	sizeof(user_exec_list[i].offset));
6084	serge	1660	if (ret) {
		1661	ret = -EFAULT;
		1662	DRM_DEBUG("failed to copy %d exec entries "
		1663	"back to user (%d)\n",
		1664	args->buffer_count, ret);
5060	serge	1665	break;
		1666	}
4246	Serge	1667	}
		1668	}
		1669
		1670	drm_free_large(exec_list);
		1671	drm_free_large(exec2_list);
		1672	return ret;
		1673	}
		1674	#endif
		1675
3263	Serge	1676	int
		1677	i915_gem_execbuffer2(struct drm_device dev, void data,
		1678	struct drm_file *file)
		1679	{
		1680	struct drm_i915_gem_execbuffer2 *args = data;
		1681	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
		1682	int ret;
		1683
		1684	if (args->buffer_count < 1 \|\|
		1685	args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
		1686	DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
		1687	return -EINVAL;
		1688	}
		1689
5060	serge	1690	if (args->rsvd2 != 0) {
		1691	DRM_DEBUG("dirty rvsd2 field\n");
		1692	return -EINVAL;
		1693	}
		1694
3480	Serge	1695	exec2_list = kmalloc(sizeof(exec2_list)args->buffer_count,
		1696	GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
3263	Serge	1697	if (exec2_list == NULL) {
		1698	DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
		1699	args->buffer_count);
		1700	return -ENOMEM;
		1701	}
		1702	ret = copy_from_user(exec2_list,
4539	Serge	1703	to_user_ptr(args->buffers_ptr),
3263	Serge	1704	sizeof(exec2_list) args->buffer_count);
		1705	if (ret != 0) {
		1706	DRM_DEBUG("copy %d exec entries failed %d\n",
		1707	args->buffer_count, ret);
3266	Serge	1708	kfree(exec2_list);
		1709	FAIL();
3263	Serge	1710	return -EFAULT;
		1711	}
		1712
5060	serge	1713	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
3263	Serge	1714	if (!ret) {
		1715	/* Copy the new buffer offsets back to the user's exec list. */
5060	serge	1716	struct drm_i915_gem_exec_object2 __user *user_exec_list =
		1717	to_user_ptr(args->buffers_ptr);
		1718	int i;
		1719
		1720	for (i = 0; i < args->buffer_count; i++) {
		1721	ret = __copy_to_user(&user_exec_list[i].offset,
		1722	&exec2_list[i].offset,
		1723	sizeof(user_exec_list[i].offset));
6084	serge	1724	if (ret) {
		1725	ret = -EFAULT;
		1726	DRM_DEBUG("failed to copy %d exec entries "
5060	serge	1727	"back to user\n",
		1728	args->buffer_count);
		1729	break;
		1730	}
3263	Serge	1731	}
		1732	}
		1733
3266	Serge	1734	kfree(exec2_list);
3263	Serge	1735	return ret;
		1736	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem_execbuffer.c – Rev 6935