WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem_execbuffer.c

Rev	Author	Line No.	Line
3263	Serge	1	/*
		2	* Copyright © 2008,2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Chris Wilson
		26	*
		27	*/
		28
		29	#include
		30	#include
		31	#include "i915_drv.h"
		32	#include "i915_trace.h"
		33	#include "intel_drv.h"
5060	serge	34	#include
3263	Serge	35
4560	Serge	36	#define __EXEC_OBJECT_HAS_PIN (1<<31)
		37	#define __EXEC_OBJECT_HAS_FENCE (1<<30)
5354	serge	38	#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
5060	serge	39	#define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
3263	Serge	40
5060	serge	41	#define BATCH_OFFSET_BIAS (256*1024)
3263	Serge	42
		43	static unsigned long
		44	copy_from_user(void to, const void __user from, unsigned long n)
		45	{
		46	memcpy(to, from, n);
		47	return 0;
		48	}
		49
4560	Serge	50	struct eb_vmas {
		51	struct list_head vmas;
3263	Serge	52	int and;
3480	Serge	53	union {
4560	Serge	54	struct i915_vma *lut[0];
5060	serge	55	struct hlist_head buckets[0];
3480	Serge	56	};
3263	Serge	57	};
		58
4560	Serge	59	static struct eb_vmas *
3480	Serge	60	eb_create(struct drm_i915_gem_execbuffer2 *args)
3263	Serge	61	{
4560	Serge	62	struct eb_vmas *eb = NULL;
3480	Serge	63
		64	if (args->flags & I915_EXEC_HANDLE_LUT) {
4560	Serge	65	unsigned size = args->buffer_count;
		66	size = sizeof(struct i915_vma );
		67	size += sizeof(struct eb_vmas);
3480	Serge	68	eb = kmalloc(size, GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
		69	}
		70
		71	if (eb == NULL) {
4560	Serge	72	unsigned size = args->buffer_count;
		73	unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
3480	Serge	74	BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
		75	while (count > 2*size)
6084	serge	76	count >>= 1;
		77	eb = kzalloc(count*sizeof(struct hlist_head) +
4560	Serge	78	sizeof(struct eb_vmas),
3480	Serge	79	GFP_TEMPORARY);
6084	serge	80	if (eb == NULL)
		81	return eb;
3263	Serge	82
6084	serge	83	eb->and = count - 1;
3480	Serge	84	} else
		85	eb->and = -args->buffer_count;
		86
4560	Serge	87	INIT_LIST_HEAD(&eb->vmas);
3263	Serge	88	return eb;
		89	}
		90
		91	static void
4560	Serge	92	eb_reset(struct eb_vmas *eb)
3263	Serge	93	{
3480	Serge	94	if (eb->and >= 0)
6084	serge	95	memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
3263	Serge	96	}
		97
3480	Serge	98	static int
4560	Serge	99	eb_lookup_vmas(struct eb_vmas *eb,
6084	serge	100	struct drm_i915_gem_exec_object2 *exec,
		101	const struct drm_i915_gem_execbuffer2 *args,
4560	Serge	102	struct i915_address_space *vm,
6084	serge	103	struct drm_file *file)
3263	Serge	104	{
4560	Serge	105	struct drm_i915_gem_object *obj;
		106	struct list_head objects;
		107	int i, ret;
3480	Serge	108
4560	Serge	109	INIT_LIST_HEAD(&objects);
3480	Serge	110	spin_lock(&file->table_lock);
4560	Serge	111	/* Grab a reference to the object and release the lock so we can lookup
		112	* or create the VMA without using GFP_ATOMIC */
3480	Serge	113	for (i = 0; i < args->buffer_count; i++) {
6084	serge	114	obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
3480	Serge	115	if (obj == NULL) {
		116	spin_unlock(&file->table_lock);
		117	DRM_DEBUG("Invalid object handle %d at index %d\n",
		118	exec[i].handle, i);
4560	Serge	119	ret = -ENOENT;
		120	goto err;
3480	Serge	121	}
		122
4560	Serge	123	if (!list_empty(&obj->obj_exec_link)) {
3480	Serge	124	spin_unlock(&file->table_lock);
		125	DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
		126	obj, exec[i].handle, i);
4560	Serge	127	ret = -EINVAL;
		128	goto err;
3480	Serge	129	}
		130
		131	drm_gem_object_reference(&obj->base);
4560	Serge	132	list_add_tail(&obj->obj_exec_link, &objects);
		133	}
		134	spin_unlock(&file->table_lock);
3480	Serge	135
4560	Serge	136	i = 0;
		137	while (!list_empty(&objects)) {
		138	struct i915_vma *vma;
		139
		140	obj = list_first_entry(&objects,
		141	struct drm_i915_gem_object,
		142	obj_exec_link);
		143
		144	/*
		145	* NOTE: We can leak any vmas created here when something fails
		146	* later on. But that's no issue since vma_unbind can deal with
		147	* vmas which are not actually bound. And since only
		148	* lookup_or_create exists as an interface to get at the vma
		149	* from the (obj, vm) we don't run the risk of creating
		150	* duplicated vmas for the same vm.
		151	*/
5354	serge	152	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
4560	Serge	153	if (IS_ERR(vma)) {
		154	DRM_DEBUG("Failed to lookup VMA\n");
		155	ret = PTR_ERR(vma);
		156	goto err;
		157	}
		158
		159	/* Transfer ownership from the objects list to the vmas list. */
		160	list_add_tail(&vma->exec_list, &eb->vmas);
		161	list_del_init(&obj->obj_exec_link);
		162
		163	vma->exec_entry = &exec[i];
3480	Serge	164	if (eb->and < 0) {
4560	Serge	165	eb->lut[i] = vma;
3480	Serge	166	} else {
		167	uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
4560	Serge	168	vma->exec_handle = handle;
		169	hlist_add_head(&vma->exec_node,
3480	Serge	170	&eb->buckets[handle & eb->and]);
		171	}
4560	Serge	172	++i;
3480	Serge	173	}
		174
		175	return 0;
4560	Serge	176
		177
		178	err:
		179	while (!list_empty(&objects)) {
		180	obj = list_first_entry(&objects,
		181	struct drm_i915_gem_object,
		182	obj_exec_link);
		183	list_del_init(&obj->obj_exec_link);
		184	drm_gem_object_unreference(&obj->base);
		185	}
		186	/*
		187	* Objects already transfered to the vmas list will be unreferenced by
		188	* eb_destroy.
		189	*/
		190
		191	return ret;
3263	Serge	192	}
		193
4560	Serge	194	static struct i915_vma eb_get_vma(struct eb_vmas eb, unsigned long handle)
3263	Serge	195	{
3480	Serge	196	if (eb->and < 0) {
		197	if (handle >= -eb->and)
		198	return NULL;
		199	return eb->lut[handle];
		200	} else {
6084	serge	201	struct hlist_head *head;
		202	struct hlist_node *node;
3263	Serge	203
6084	serge	204	head = &eb->buckets[handle & eb->and];
		205	hlist_for_each(node, head) {
4560	Serge	206	struct i915_vma *vma;
3480	Serge	207
4560	Serge	208	vma = hlist_entry(node, struct i915_vma, exec_node);
		209	if (vma->exec_handle == handle)
		210	return vma;
6084	serge	211	}
		212	return NULL;
3263	Serge	213	}
		214	}
		215
		216	static void
4560	Serge	217	i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
3263	Serge	218	{
4560	Serge	219	struct drm_i915_gem_exec_object2 *entry;
		220	struct drm_i915_gem_object *obj = vma->obj;
3480	Serge	221
4560	Serge	222	if (!drm_mm_node_allocated(&vma->node))
		223	return;
		224
		225	entry = vma->exec_entry;
		226
		227	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
		228	i915_gem_object_unpin_fence(obj);
		229
		230	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
5060	serge	231	vma->pin_count--;
4560	Serge	232
		233	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE \| __EXEC_OBJECT_HAS_PIN);
		234	}
		235
		236	static void eb_destroy(struct eb_vmas *eb)
		237	{
		238	while (!list_empty(&eb->vmas)) {
		239	struct i915_vma *vma;
		240
		241	vma = list_first_entry(&eb->vmas,
		242	struct i915_vma,
3480	Serge	243	exec_list);
4560	Serge	244	list_del_init(&vma->exec_list);
		245	i915_gem_execbuffer_unreserve_vma(vma);
		246	drm_gem_object_unreference(&vma->obj->base);
3480	Serge	247	}
3263	Serge	248	kfree(eb);
		249	}
		250
		251	static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
		252	{
4560	Serge	253	return (HAS_LLC(obj->base.dev) \|\|
		254	obj->base.write_domain == I915_GEM_DOMAIN_CPU \|\|
3263	Serge	255	obj->cache_level != I915_CACHE_NONE);
		256	}
		257
		258	static int
4371	Serge	259	relocate_entry_cpu(struct drm_i915_gem_object *obj,
5060	serge	260	struct drm_i915_gem_relocation_entry *reloc,
		261	uint64_t target_offset)
4371	Serge	262	{
6084	serge	263	struct drm_device *dev = obj->base.dev;
		264	uint32_t page_offset = offset_in_page(reloc->offset);
5060	serge	265	uint64_t delta = reloc->delta + target_offset;
4371	Serge	266	char *vaddr;
4560	Serge	267	int ret;
4371	Serge	268
4560	Serge	269	ret = i915_gem_object_set_to_cpu_domain(obj, true);
4371	Serge	270	if (ret)
		271	return ret;
		272
5354	serge	273	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		274	reloc->offset >> PAGE_SHIFT));
5060	serge	275	(uint32_t )(vaddr + page_offset) = lower_32_bits(delta);
4371	Serge	276
5060	serge	277	if (INTEL_INFO(dev)->gen >= 8) {
		278	page_offset = offset_in_page(page_offset + sizeof(uint32_t));
		279
		280	if (page_offset == 0) {
5354	serge	281	kunmap_atomic(vaddr);
		282	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		283	(reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
5060	serge	284	}
		285
		286	(uint32_t )(vaddr + page_offset) = upper_32_bits(delta);
		287	}
		288
5354	serge	289	kunmap_atomic(vaddr);
		290
4371	Serge	291	return 0;
		292	}
		293
		294	static int
		295	relocate_entry_gtt(struct drm_i915_gem_object *obj,
5060	serge	296	struct drm_i915_gem_relocation_entry *reloc,
		297	uint64_t target_offset)
4371	Serge	298	{
		299	struct drm_device *dev = obj->base.dev;
		300	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	301	uint64_t delta = reloc->delta + target_offset;
5354	serge	302	uint64_t offset;
4371	Serge	303	void __iomem *reloc_page;
4560	Serge	304	int ret;
4371	Serge	305
		306	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		307	if (ret)
		308	return ret;
		309
		310	ret = i915_gem_object_put_fence(obj);
		311	if (ret)
		312	return ret;
		313
		314	/* Map the page containing the relocation we're going to perform. */
5354	serge	315	offset = i915_gem_obj_ggtt_offset(obj);
		316	offset += reloc->offset;
4539	Serge	317	MapPage(dev_priv->gtt.mappable,dev_priv->gtt.mappable_base +
5354	serge	318	(offset & PAGE_MASK), PG_SW);
4539	Serge	319	reloc_page = dev_priv->gtt.mappable;
5354	serge	320	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
4371	Serge	321
6084	serge	322	// io_mapping_unmap_atomic(reloc_page);
5060	serge	323
6084	serge	324	return 0;
		325	}
5354	serge	326
6084	serge	327	static void
		328	clflush_write32(void *addr, uint32_t value)
		329	{
		330	/* This is not a fast path, so KISS. */
		331	drm_clflush_virt_range(addr, sizeof(uint32_t));
		332	(uint32_t )addr = value;
		333	drm_clflush_virt_range(addr, sizeof(uint32_t));
		334	}
		335
		336	static int
		337	relocate_entry_clflush(struct drm_i915_gem_object *obj,
		338	struct drm_i915_gem_relocation_entry *reloc,
		339	uint64_t target_offset)
		340	{
		341	struct drm_device *dev = obj->base.dev;
		342	uint32_t page_offset = offset_in_page(reloc->offset);
		343	uint64_t delta = (int)reloc->delta + target_offset;
		344	char *vaddr;
		345	int ret;
		346
		347	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		348	if (ret)
		349	return ret;
		350
		351	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		352	reloc->offset >> PAGE_SHIFT));
		353	clflush_write32(vaddr + page_offset, lower_32_bits(delta));
		354
		355	if (INTEL_INFO(dev)->gen >= 8) {
		356	page_offset = offset_in_page(page_offset + sizeof(uint32_t));
		357
		358	if (page_offset == 0) {
		359	kunmap_atomic(vaddr);
		360	vaddr = kmap_atomic(i915_gem_object_get_page(obj,
		361	(reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
		362	}
		363
		364	clflush_write32(vaddr + page_offset, upper_32_bits(delta));
		365	}
		366
		367	kunmap_atomic(vaddr);
		368
4371	Serge	369	return 0;
		370	}
		371
		372	static int
3263	Serge	373	i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
4560	Serge	374	struct eb_vmas *eb,
5060	serge	375	struct drm_i915_gem_relocation_entry *reloc)
3263	Serge	376	{
		377	struct drm_device *dev = obj->base.dev;
		378	struct drm_gem_object *target_obj;
		379	struct drm_i915_gem_object *target_i915_obj;
4560	Serge	380	struct i915_vma *target_vma;
5060	serge	381	uint64_t target_offset;
4560	Serge	382	int ret;
3263	Serge	383
		384	/* we've already hold a reference to all valid objects */
4560	Serge	385	target_vma = eb_get_vma(eb, reloc->target_handle);
		386	if (unlikely(target_vma == NULL))
3263	Serge	387	return -ENOENT;
4560	Serge	388	target_i915_obj = target_vma->obj;
		389	target_obj = &target_vma->obj->base;
3263	Serge	390
4560	Serge	391	target_offset = target_vma->node.start;
3263	Serge	392
		393	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
		394	* pipe_control writes because the gpu doesn't properly redirect them
		395	* through the ppgtt for non_secure batchbuffers. */
		396	if (unlikely(IS_GEN6(dev) &&
6084	serge	397	reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
		398	ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
		399	PIN_GLOBAL);
		400	if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
		401	return ret;
		402	}
3263	Serge	403
		404	/* Validate that the target is in a valid r/w GPU domain */
		405	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
		406	DRM_DEBUG("reloc with multiple write domains: "
		407	"obj %p target %d offset %d "
		408	"read %08x write %08x",
		409	obj, reloc->target_handle,
		410	(int) reloc->offset,
		411	reloc->read_domains,
		412	reloc->write_domain);
4560	Serge	413	return -EINVAL;
3263	Serge	414	}
		415	if (unlikely((reloc->write_domain \| reloc->read_domains)
		416	& ~I915_GEM_GPU_DOMAINS)) {
		417	DRM_DEBUG("reloc with read/write non-GPU domains: "
		418	"obj %p target %d offset %d "
		419	"read %08x write %08x",
		420	obj, reloc->target_handle,
		421	(int) reloc->offset,
		422	reloc->read_domains,
		423	reloc->write_domain);
4560	Serge	424	return -EINVAL;
3263	Serge	425	}
		426
		427	target_obj->pending_read_domains \|= reloc->read_domains;
		428	target_obj->pending_write_domain \|= reloc->write_domain;
		429
		430	/* If the relocation already has the right value in it, no
		431	* more work needs to be done.
		432	*/
		433	if (target_offset == reloc->presumed_offset)
		434	return 0;
		435
		436	/* Check that the relocation address is valid... */
4560	Serge	437	if (unlikely(reloc->offset >
		438	obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
3263	Serge	439	DRM_DEBUG("Relocation beyond object bounds: "
		440	"obj %p target %d offset %d size %d.\n",
		441	obj, reloc->target_handle,
		442	(int) reloc->offset,
		443	(int) obj->base.size);
4560	Serge	444	return -EINVAL;
3263	Serge	445	}
		446	if (unlikely(reloc->offset & 3)) {
		447	DRM_DEBUG("Relocation not 4-byte aligned: "
		448	"obj %p target %d offset %d.\n",
		449	obj, reloc->target_handle,
		450	(int) reloc->offset);
4560	Serge	451	return -EINVAL;
3263	Serge	452	}
		453
		454	/* We can't wait for rendering with pagefaults disabled */
		455
4371	Serge	456	if (use_cpu_reloc(obj))
5060	serge	457	ret = relocate_entry_cpu(obj, reloc, target_offset);
6084	serge	458	else if (obj->map_and_fenceable)
5060	serge	459	ret = relocate_entry_gtt(obj, reloc, target_offset);
6084	serge	460	else if (1)
		461	ret = relocate_entry_clflush(obj, reloc, target_offset);
		462	else {
		463	WARN_ONCE(1, "Impossible case in relocation handling\n");
		464	ret = -ENODEV;
		465	}
3263	Serge	466
6084	serge	467	if (ret)
		468	return ret;
3263	Serge	469
		470	/* and update the user's relocation entry */
		471	reloc->presumed_offset = target_offset;
		472
		473	return 0;
		474	}
		475
		476	static int
4560	Serge	477	i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
		478	struct eb_vmas *eb)
3263	Serge	479	{
		480	#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
3266	Serge	481	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(64)];
3263	Serge	482	struct drm_i915_gem_relocation_entry __user *user_relocs;
4560	Serge	483	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
3263	Serge	484	int remain, ret;
		485
4539	Serge	486	user_relocs = to_user_ptr(entry->relocs_ptr);
3263	Serge	487
		488	remain = entry->relocation_count;
		489	while (remain) {
		490	struct drm_i915_gem_relocation_entry *r = stack_reloc;
		491	int count = remain;
		492	if (count > ARRAY_SIZE(stack_reloc))
		493	count = ARRAY_SIZE(stack_reloc);
		494	remain -= count;
		495
		496	memcpy(r, user_relocs, count*sizeof(r[0]));
		497
		498	do {
		499	u64 offset = r->presumed_offset;
		500
5060	serge	501	ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
3263	Serge	502	if (ret)
		503	return ret;
		504
4392	Serge	505	if (r->presumed_offset != offset)
		506	{
3263	Serge	507	memcpy(&user_relocs->presumed_offset,
		508	&r->presumed_offset,
		509	sizeof(r->presumed_offset));
4392	Serge	510	}
3263	Serge	511
		512	user_relocs++;
		513	r++;
		514	} while (--count);
		515	}
		516
		517	return 0;
		518	#undef N_RELOC
		519	}
		520
		521	static int
4560	Serge	522	i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
		523	struct eb_vmas *eb,
		524	struct drm_i915_gem_relocation_entry *relocs)
3263	Serge	525	{
4560	Serge	526	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
3263	Serge	527	int i, ret;
		528
		529	for (i = 0; i < entry->relocation_count; i++) {
5060	serge	530	ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
3263	Serge	531	if (ret)
		532	return ret;
		533	}
		534
		535	return 0;
		536	}
		537
		538	static int
4560	Serge	539	i915_gem_execbuffer_relocate(struct eb_vmas *eb)
3263	Serge	540	{
4560	Serge	541	struct i915_vma *vma;
3263	Serge	542	int ret = 0;
		543
		544	/* This is the fast path and we cannot handle a pagefault whilst
		545	* holding the struct mutex lest the user pass in the relocations
		546	* contained within a mmaped bo. For in such a case we, the page
		547	* fault handler would call i915_gem_fault() and we would try to
		548	* acquire the struct mutex again. Obviously this is bad and so
		549	* lockdep complains vehemently.
		550	*/
4104	Serge	551	// pagefault_disable();
4560	Serge	552	list_for_each_entry(vma, &eb->vmas, exec_list) {
		553	ret = i915_gem_execbuffer_relocate_vma(vma, eb);
3263	Serge	554	if (ret)
		555	break;
		556	}
		557	// pagefault_enable();
		558
		559	return ret;
		560	}
		561
6084	serge	562	static bool only_mappable_for_reloc(unsigned int flags)
		563	{
		564	return (flags & (EXEC_OBJECT_NEEDS_FENCE \| __EXEC_OBJECT_NEEDS_MAP)) ==
		565	__EXEC_OBJECT_NEEDS_MAP;
		566	}
		567
3263	Serge	568	static int
4560	Serge	569	i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
5060	serge	570	struct intel_engine_cs *ring,
6084	serge	571	bool *need_reloc)
3263	Serge	572	{
5060	serge	573	struct drm_i915_gem_object *obj = vma->obj;
4560	Serge	574	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5060	serge	575	uint64_t flags;
3263	Serge	576	int ret;
		577
6084	serge	578	flags = PIN_USER;
5060	serge	579	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
		580	flags \|= PIN_GLOBAL;
		581
6084	serge	582	if (!drm_mm_node_allocated(&vma->node)) {
		583	/* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
		584	* limit address to the first 4GBs for unflagged objects.
		585	*/
		586	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
		587	flags \|= PIN_ZONE_4G;
		588	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
		589	flags \|= PIN_GLOBAL \| PIN_MAPPABLE;
		590	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
		591	flags \|= BATCH_OFFSET_BIAS \| PIN_OFFSET_BIAS;
		592	if ((flags & PIN_MAPPABLE) == 0)
		593	flags \|= PIN_HIGH;
		594	}
		595
5060	serge	596	ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
6084	serge	597	if ((ret == -ENOSPC \|\| ret == -E2BIG) &&
		598	only_mappable_for_reloc(entry->flags))
		599	ret = i915_gem_object_pin(obj, vma->vm,
		600	entry->alignment,
		601	flags & ~PIN_MAPPABLE);
3263	Serge	602	if (ret)
		603	return ret;
		604
		605	entry->flags \|= __EXEC_OBJECT_HAS_PIN;
		606
6084	serge	607	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
		608	ret = i915_gem_object_get_fence(obj);
		609	if (ret)
		610	return ret;
3263	Serge	611
6084	serge	612	if (i915_gem_object_pin_fence(obj))
		613	entry->flags \|= __EXEC_OBJECT_HAS_FENCE;
3263	Serge	614	}
		615
4560	Serge	616	if (entry->offset != vma->node.start) {
		617	entry->offset = vma->node.start;
3480	Serge	618	*need_reloc = true;
		619	}
3266	Serge	620
3480	Serge	621	if (entry->flags & EXEC_OBJECT_WRITE) {
		622	obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
		623	obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
		624	}
		625
3263	Serge	626	return 0;
		627	}
		628
5060	serge	629	static bool
5354	serge	630	need_reloc_mappable(struct i915_vma *vma)
5060	serge	631	{
		632	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5354	serge	633
		634	if (entry->relocation_count == 0)
		635	return false;
		636
		637	if (!i915_is_ggtt(vma->vm))
		638	return false;
		639
		640	/* See also use_cpu_reloc() */
		641	if (HAS_LLC(vma->obj->base.dev))
		642	return false;
		643
		644	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		645	return false;
		646
		647	return true;
		648	}
		649
		650	static bool
		651	eb_vma_misplaced(struct i915_vma *vma)
		652	{
		653	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5060	serge	654	struct drm_i915_gem_object *obj = vma->obj;
		655
5354	serge	656	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
5060	serge	657	!i915_is_ggtt(vma->vm));
		658
		659	if (entry->alignment &&
		660	vma->node.start & (entry->alignment - 1))
		661	return true;
		662
		663	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
		664	vma->node.start < BATCH_OFFSET_BIAS)
		665	return true;
		666
6084	serge	667	/* avoid costly ping-pong once a batch bo ended up non-mappable */
		668	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
		669	return !only_mappable_for_reloc(entry->flags);
		670
		671	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
		672	(vma->node.start + vma->node.size - 1) >> 32)
		673	return true;
		674
5060	serge	675	return false;
		676	}
		677
3263	Serge	678	static int
5060	serge	679	i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
4560	Serge	680	struct list_head *vmas,
6084	serge	681	struct intel_context *ctx,
3480	Serge	682	bool *need_relocs)
3263	Serge	683	{
		684	struct drm_i915_gem_object *obj;
4560	Serge	685	struct i915_vma *vma;
		686	struct i915_address_space *vm;
		687	struct list_head ordered_vmas;
3263	Serge	688	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
		689	int retry;
		690
5060	serge	691	i915_gem_retire_requests_ring(ring);
		692
4560	Serge	693	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
		694
		695	INIT_LIST_HEAD(&ordered_vmas);
		696	while (!list_empty(vmas)) {
3263	Serge	697	struct drm_i915_gem_exec_object2 *entry;
		698	bool need_fence, need_mappable;
		699
4560	Serge	700	vma = list_first_entry(vmas, struct i915_vma, exec_list);
		701	obj = vma->obj;
		702	entry = vma->exec_entry;
3263	Serge	703
6084	serge	704	if (ctx->flags & CONTEXT_NO_ZEROMAP)
		705	entry->flags \|= __EXEC_OBJECT_NEEDS_BIAS;
		706
5354	serge	707	if (!has_fenced_gpu_access)
		708	entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
3263	Serge	709	need_fence =
		710	entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
		711	obj->tiling_mode != I915_TILING_NONE;
4560	Serge	712	need_mappable = need_fence \|\| need_reloc_mappable(vma);
3263	Serge	713
5354	serge	714	if (need_mappable) {
		715	entry->flags \|= __EXEC_OBJECT_NEEDS_MAP;
4560	Serge	716	list_move(&vma->exec_list, &ordered_vmas);
5354	serge	717	} else
4560	Serge	718	list_move_tail(&vma->exec_list, &ordered_vmas);
3263	Serge	719
3480	Serge	720	obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
3263	Serge	721	obj->base.pending_write_domain = 0;
		722	}
4560	Serge	723	list_splice(&ordered_vmas, vmas);
3263	Serge	724
		725	/* Attempt to pin all of the buffers into the GTT.
		726	* This is done in 3 phases:
		727	*
		728	* 1a. Unbind all objects that do not match the GTT constraints for
		729	* the execbuffer (fenceable, mappable, alignment etc).
		730	* 1b. Increment pin count for already bound objects.
		731	* 2. Bind new objects.
		732	* 3. Decrement pin count.
		733	*
		734	* This avoid unnecessary unbinding of later objects in order to make
		735	* room for the earlier objects unless we need to defragment.
		736	*/
		737	retry = 0;
		738	do {
		739	int ret = 0;
		740
		741	/* Unbind any ill-fitting objects or pin. */
4560	Serge	742	list_for_each_entry(vma, vmas, exec_list) {
		743	if (!drm_mm_node_allocated(&vma->node))
3263	Serge	744	continue;
		745
5354	serge	746	if (eb_vma_misplaced(vma))
4560	Serge	747	ret = i915_vma_unbind(vma);
3263	Serge	748	else
4560	Serge	749	ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
3263	Serge	750	if (ret)
		751	goto err;
		752	}
		753
		754	/* Bind fresh objects */
4560	Serge	755	list_for_each_entry(vma, vmas, exec_list) {
		756	if (drm_mm_node_allocated(&vma->node))
3263	Serge	757	continue;
		758
4560	Serge	759	ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
3263	Serge	760	if (ret)
		761	goto err;
		762	}
		763
4560	Serge	764	err:
3263	Serge	765	if (ret != -ENOSPC \|\| retry++)
		766	return ret;
		767
4560	Serge	768	/* Decrement pin count for bound objects */
		769	list_for_each_entry(vma, vmas, exec_list)
		770	i915_gem_execbuffer_unreserve_vma(vma);
		771
5060	serge	772	ret = i915_gem_evict_vm(vm, true);
3263	Serge	773	if (ret)
		774	return ret;
		775	} while (1);
		776	}
		777
		778	static int
		779	i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3480	Serge	780	struct drm_i915_gem_execbuffer2 *args,
3263	Serge	781	struct drm_file *file,
5060	serge	782	struct intel_engine_cs *ring,
4560	Serge	783	struct eb_vmas *eb,
6084	serge	784	struct drm_i915_gem_exec_object2 *exec,
		785	struct intel_context *ctx)
3263	Serge	786	{
		787	struct drm_i915_gem_relocation_entry *reloc;
4560	Serge	788	struct i915_address_space *vm;
		789	struct i915_vma *vma;
3480	Serge	790	bool need_relocs;
3263	Serge	791	int *reloc_offset;
		792	int i, total, ret;
4560	Serge	793	unsigned count = args->buffer_count;
3263	Serge	794
4560	Serge	795	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
		796
3263	Serge	797	/* We may process another execbuffer during the unlock... */
4560	Serge	798	while (!list_empty(&eb->vmas)) {
		799	vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
		800	list_del_init(&vma->exec_list);
		801	i915_gem_execbuffer_unreserve_vma(vma);
		802	drm_gem_object_unreference(&vma->obj->base);
3263	Serge	803	}
		804
		805	mutex_unlock(&dev->struct_mutex);
		806
		807	total = 0;
		808	for (i = 0; i < count; i++)
		809	total += exec[i].relocation_count;
		810
6084	serge	811	reloc_offset = __builtin_malloc(count * sizeof(*reloc_offset));
		812	reloc = __builtin_malloc(total * sizeof(*reloc));
3263	Serge	813	if (reloc == NULL \|\| reloc_offset == NULL) {
3266	Serge	814	kfree(reloc);
		815	kfree(reloc_offset);
3263	Serge	816	mutex_lock(&dev->struct_mutex);
		817	return -ENOMEM;
		818	}
		819
		820	total = 0;
		821	for (i = 0; i < count; i++) {
		822	struct drm_i915_gem_relocation_entry __user *user_relocs;
		823	u64 invalid_offset = (u64)-1;
		824	int j;
		825
4539	Serge	826	user_relocs = to_user_ptr(exec[i].relocs_ptr);
3263	Serge	827
		828	if (copy_from_user(reloc+total, user_relocs,
		829	exec[i].relocation_count * sizeof(*reloc))) {
		830	ret = -EFAULT;
		831	mutex_lock(&dev->struct_mutex);
		832	goto err;
		833	}
		834
		835	/* As we do not update the known relocation offsets after
		836	* relocating (due to the complexities in lock handling),
		837	* we need to mark them as invalid now so that we force the
		838	* relocation processing next time. Just in case the target
		839	* object is evicted and then rebound into its old
		840	* presumed_offset before the next execbuffer - if that
		841	* happened we would make the mistake of assuming that the
		842	* relocations were valid.
		843	*/
		844	for (j = 0; j < exec[i].relocation_count; j++) {
5060	serge	845	if (__copy_to_user(&user_relocs[j].presumed_offset,
6084	serge	846	&invalid_offset,
		847	sizeof(invalid_offset))) {
3263	Serge	848	ret = -EFAULT;
		849	mutex_lock(&dev->struct_mutex);
		850	goto err;
		851	}
		852	}
		853
		854	reloc_offset[i] = total;
		855	total += exec[i].relocation_count;
		856	}
		857
		858	ret = i915_mutex_lock_interruptible(dev);
		859	if (ret) {
		860	mutex_lock(&dev->struct_mutex);
		861	goto err;
		862	}
		863
		864	/* reacquire the objects */
		865	eb_reset(eb);
4560	Serge	866	ret = eb_lookup_vmas(eb, exec, args, vm, file);
3480	Serge	867	if (ret)
6084	serge	868	goto err;
3263	Serge	869
3480	Serge	870	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
6084	serge	871	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
3263	Serge	872	if (ret)
		873	goto err;
		874
4560	Serge	875	list_for_each_entry(vma, &eb->vmas, exec_list) {
		876	int offset = vma->exec_entry - exec;
		877	ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
		878	reloc + reloc_offset[offset]);
3263	Serge	879	if (ret)
		880	goto err;
		881	}
		882
		883	/* Leave the user relocations as are, this is the painfully slow path,
		884	* and we want to avoid the complication of dropping the lock whilst
		885	* having buffers reserved in the aperture and so causing spurious
		886	* ENOSPC for random operations.
		887	*/
		888
		889	err:
3266	Serge	890	kfree(reloc);
		891	kfree(reloc_offset);
3263	Serge	892	return ret;
		893	}
		894
		895	static int
6084	serge	896	i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
4560	Serge	897	struct list_head *vmas)
3263	Serge	898	{
6084	serge	899	const unsigned other_rings = ~intel_ring_flag(req->ring);
4560	Serge	900	struct i915_vma *vma;
3263	Serge	901	uint32_t flush_domains = 0;
4104	Serge	902	bool flush_chipset = false;
3263	Serge	903	int ret;
		904
4560	Serge	905	list_for_each_entry(vma, vmas, exec_list) {
		906	struct drm_i915_gem_object *obj = vma->obj;
3263	Serge	907
6084	serge	908	if (obj->active & other_rings) {
		909	ret = i915_gem_object_sync(obj, req->ring, &req);
		910	if (ret)
		911	return ret;
		912	}
		913
3263	Serge	914	if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
4104	Serge	915	flush_chipset \|= i915_gem_clflush_object(obj, false);
3263	Serge	916
		917	flush_domains \|= obj->base.write_domain;
		918	}
		919
4104	Serge	920	if (flush_chipset)
6084	serge	921	i915_gem_chipset_flush(req->ring->dev);
3263	Serge	922
		923	if (flush_domains & I915_GEM_DOMAIN_GTT)
		924	wmb();
		925
		926	/* Unconditionally invalidate gpu caches and ensure that we do flush
		927	* any residual writes from the previous batch.
		928	*/
6084	serge	929	return intel_ring_invalidate_all_caches(req);
3263	Serge	930	}
		931
		932	static bool
		933	i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
		934	{
3480	Serge	935	if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
		936	return false;
		937
6084	serge	938	/* Kernel clipping was a DRI1 misfeature */
		939	if (exec->num_cliprects \|\| exec->cliprects_ptr)
		940	return false;
		941
		942	if (exec->DR4 == 0xffffffff) {
		943	DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
		944	exec->DR4 = 0;
		945	}
		946	if (exec->DR1 \|\| exec->DR4)
		947	return false;
		948
		949	if ((exec->batch_start_offset \| exec->batch_len) & 0x7)
		950	return false;
		951
		952	return true;
3263	Serge	953	}
		954
		955	static int
5354	serge	956	validate_exec_list(struct drm_device *dev,
		957	struct drm_i915_gem_exec_object2 *exec,
3263	Serge	958	int count)
		959	{
4560	Serge	960	unsigned relocs_total = 0;
		961	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
5354	serge	962	unsigned invalid_flags;
		963	int i;
3263	Serge	964
5354	serge	965	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
		966	if (USES_FULL_PPGTT(dev))
		967	invalid_flags \|= EXEC_OBJECT_NEEDS_GTT;
		968
3263	Serge	969	for (i = 0; i < count; i++) {
3746	Serge	970	char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
3263	Serge	971	int length; /* limited by fault_in_pages_readable() */
		972
5354	serge	973	if (exec[i].flags & invalid_flags)
3263	Serge	974	return -EINVAL;
		975
6084	serge	976	if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
		977	return -EINVAL;
		978
3480	Serge	979	/* First check for malicious input causing overflow in
		980	* the worst case where we need to allocate the entire
		981	* relocation tree as a single array.
		982	*/
		983	if (exec[i].relocation_count > relocs_max - relocs_total)
		984	return -EINVAL;
		985	relocs_total += exec[i].relocation_count;
		986
3263	Serge	987	length = exec[i].relocation_count *
		988	sizeof(struct drm_i915_gem_relocation_entry);
3746	Serge	989	/*
		990	* We must check that the entire relocation array is safe
		991	* to read, but since we may need to update the presumed
		992	* offsets during execution, check for full write access.
		993	*/
4560	Serge	994	}
3263	Serge	995
4560	Serge	996	return 0;
		997	}
		998
5060	serge	999	static struct intel_context *
4560	Serge	1000	i915_gem_validate_context(struct drm_device dev, struct drm_file file,
5060	serge	1001	struct intel_engine_cs *ring, const u32 ctx_id)
4560	Serge	1002	{
5060	serge	1003	struct intel_context *ctx = NULL;
4560	Serge	1004	struct i915_ctx_hang_stats *hs;
		1005
5060	serge	1006	if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
		1007	return ERR_PTR(-EINVAL);
4560	Serge	1008
5060	serge	1009	ctx = i915_gem_context_get(file->driver_priv, ctx_id);
		1010	if (IS_ERR(ctx))
		1011	return ctx;
		1012
		1013	hs = &ctx->hang_stats;
4560	Serge	1014	if (hs->banned) {
		1015	DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
5060	serge	1016	return ERR_PTR(-EIO);
3263	Serge	1017	}
		1018
5354	serge	1019	if (i915.enable_execlists && !ctx->engine[ring->id].state) {
6084	serge	1020	int ret = intel_lr_context_deferred_alloc(ctx, ring);
5354	serge	1021	if (ret) {
		1022	DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
		1023	return ERR_PTR(ret);
		1024	}
		1025	}
		1026
5060	serge	1027	return ctx;
3263	Serge	1028	}
		1029
5354	serge	1030	void
4560	Serge	1031	i915_gem_execbuffer_move_to_active(struct list_head *vmas,
6084	serge	1032	struct drm_i915_gem_request *req)
3263	Serge	1033	{
6084	serge	1034	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
4560	Serge	1035	struct i915_vma *vma;
3263	Serge	1036
4560	Serge	1037	list_for_each_entry(vma, vmas, exec_list) {
5354	serge	1038	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
4560	Serge	1039	struct drm_i915_gem_object *obj = vma->obj;
3263	Serge	1040	u32 old_read = obj->base.read_domains;
		1041	u32 old_write = obj->base.write_domain;
		1042
6084	serge	1043	obj->dirty = 1; /* be paranoid */
3480	Serge	1044	obj->base.write_domain = obj->base.pending_write_domain;
		1045	if (obj->base.write_domain == 0)
		1046	obj->base.pending_read_domains \|= obj->base.read_domains;
3263	Serge	1047	obj->base.read_domains = obj->base.pending_read_domains;
		1048
6084	serge	1049	i915_vma_move_to_active(vma, req);
3263	Serge	1050	if (obj->base.write_domain) {
6084	serge	1051	i915_gem_request_assign(&obj->last_write_req, req);
5060	serge	1052
6084	serge	1053	intel_fb_obj_invalidate(obj, ORIGIN_CS);
5060	serge	1054
		1055	/* update for the implicit flush after a batch */
		1056	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
3263	Serge	1057	}
5354	serge	1058	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
6084	serge	1059	i915_gem_request_assign(&obj->last_fenced_req, req);
5354	serge	1060	if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
		1061	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		1062	list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
		1063	&dev_priv->mm.fence_list);
		1064	}
		1065	}
3263	Serge	1066
		1067	trace_i915_gem_object_change_domain(obj, old_read, old_write);
		1068	}
		1069	}
		1070
5354	serge	1071	void
6084	serge	1072	i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
3263	Serge	1073	{
		1074	/* Unconditionally force add_request to emit a full flush. */
6084	serge	1075	params->ring->gpu_caches_dirty = true;
3263	Serge	1076
		1077	/* Add a breadcrumb for the completion of the batch buffer */
6084	serge	1078	__i915_add_request(params->request, params->batch_obj, true);
3263	Serge	1079	}
		1080
		1081	static int
		1082	i915_reset_gen7_sol_offsets(struct drm_device *dev,
6084	serge	1083	struct drm_i915_gem_request *req)
3263	Serge	1084	{
6084	serge	1085	struct intel_engine_cs *ring = req->ring;
5060	serge	1086	struct drm_i915_private *dev_priv = dev->dev_private;
3263	Serge	1087	int ret, i;
		1088
5060	serge	1089	if (!IS_GEN7(dev) \|\| ring != &dev_priv->ring[RCS]) {
		1090	DRM_DEBUG("sol reset is gen7/rcs only\n");
		1091	return -EINVAL;
		1092	}
3263	Serge	1093
6084	serge	1094	ret = intel_ring_begin(req, 4 * 3);
3263	Serge	1095	if (ret)
		1096	return ret;
		1097
		1098	for (i = 0; i < 4; i++) {
		1099	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		1100	intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
		1101	intel_ring_emit(ring, 0);
		1102	}
		1103
		1104	intel_ring_advance(ring);
		1105
		1106	return 0;
		1107	}
		1108
6084	serge	1109	static struct drm_i915_gem_object*
		1110	i915_gem_execbuffer_parse(struct intel_engine_cs *ring,
		1111	struct drm_i915_gem_exec_object2 *shadow_exec_entry,
		1112	struct eb_vmas *eb,
		1113	struct drm_i915_gem_object *batch_obj,
		1114	u32 batch_start_offset,
		1115	u32 batch_len,
		1116	bool is_master)
5354	serge	1117	{
6084	serge	1118	struct drm_i915_gem_object *shadow_batch_obj;
		1119	struct i915_vma *vma;
5354	serge	1120	int ret;
		1121
6084	serge	1122	shadow_batch_obj = i915_gem_batch_pool_get(&ring->batch_pool,
		1123	PAGE_ALIGN(batch_len));
		1124	if (IS_ERR(shadow_batch_obj))
		1125	return shadow_batch_obj;
5354	serge	1126
6084	serge	1127	ret = i915_parse_cmds(ring,
		1128	batch_obj,
		1129	shadow_batch_obj,
		1130	batch_start_offset,
		1131	batch_len,
		1132	is_master);
		1133	if (ret)
		1134	goto err;
5354	serge	1135
6084	serge	1136	ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
		1137	if (ret)
		1138	goto err;
5354	serge	1139
6084	serge	1140	i915_gem_object_unpin_pages(shadow_batch_obj);
5354	serge	1141
6084	serge	1142	memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
		1143
		1144	vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
		1145	vma->exec_entry = shadow_exec_entry;
		1146	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
		1147	drm_gem_object_reference(&shadow_batch_obj->base);
		1148	list_add_tail(&vma->exec_list, &eb->vmas);
		1149
		1150	shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
		1151
		1152	return shadow_batch_obj;
		1153
		1154	err:
		1155	i915_gem_object_unpin_pages(shadow_batch_obj);
		1156	if (ret == -EACCES) /* unhandled chained batch */
		1157	return batch_obj;
		1158	else
		1159	return ERR_PTR(ret);
5354	serge	1160	}
		1161
		1162	int
6084	serge	1163	i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
		1164	struct drm_i915_gem_execbuffer2 *args,
		1165	struct list_head *vmas)
5060	serge	1166	{
6084	serge	1167	struct drm_device *dev = params->dev;
		1168	struct intel_engine_cs *ring = params->ring;
5060	serge	1169	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1170	u64 exec_start, exec_len;
5060	serge	1171	int instp_mode;
		1172	u32 instp_mask;
6084	serge	1173	int ret;
5060	serge	1174
6084	serge	1175	ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
5060	serge	1176	if (ret)
6084	serge	1177	return ret;
5060	serge	1178
6084	serge	1179	ret = i915_switch_context(params->request);
5060	serge	1180	if (ret)
6084	serge	1181	return ret;
5060	serge	1182
6084	serge	1183	WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<id),
		1184	"%s didn't clear reload\n", ring->name);
		1185
5060	serge	1186	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
		1187	instp_mask = I915_EXEC_CONSTANTS_MASK;
		1188	switch (instp_mode) {
		1189	case I915_EXEC_CONSTANTS_REL_GENERAL:
		1190	case I915_EXEC_CONSTANTS_ABSOLUTE:
		1191	case I915_EXEC_CONSTANTS_REL_SURFACE:
		1192	if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
		1193	DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
6084	serge	1194	return -EINVAL;
5060	serge	1195	}
		1196
		1197	if (instp_mode != dev_priv->relative_constants_mode) {
		1198	if (INTEL_INFO(dev)->gen < 4) {
		1199	DRM_DEBUG("no rel constants on pre-gen4\n");
6084	serge	1200	return -EINVAL;
5060	serge	1201	}
		1202
		1203	if (INTEL_INFO(dev)->gen > 5 &&
		1204	instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
		1205	DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
6084	serge	1206	return -EINVAL;
5060	serge	1207	}
		1208
		1209	/* The HW changed the meaning on this bit on gen6 */
		1210	if (INTEL_INFO(dev)->gen >= 6)
		1211	instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
		1212	}
		1213	break;
		1214	default:
		1215	DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
6084	serge	1216	return -EINVAL;
5060	serge	1217	}
		1218
		1219	if (ring == &dev_priv->ring[RCS] &&
6084	serge	1220	instp_mode != dev_priv->relative_constants_mode) {
		1221	ret = intel_ring_begin(params->request, 4);
5060	serge	1222	if (ret)
6084	serge	1223	return ret;
5060	serge	1224
		1225	intel_ring_emit(ring, MI_NOOP);
		1226	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		1227	intel_ring_emit(ring, INSTPM);
		1228	intel_ring_emit(ring, instp_mask << 16 \| instp_mode);
		1229	intel_ring_advance(ring);
		1230
		1231	dev_priv->relative_constants_mode = instp_mode;
		1232	}
		1233
		1234	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
6084	serge	1235	ret = i915_reset_gen7_sol_offsets(dev, params->request);
5060	serge	1236	if (ret)
6084	serge	1237	return ret;
5060	serge	1238	}
		1239
6084	serge	1240	exec_len = args->batch_len;
		1241	exec_start = params->batch_obj_vm_offset +
		1242	params->args_batch_start_offset;
5060	serge	1243
6084	serge	1244	ret = ring->dispatch_execbuffer(params->request,
		1245	exec_start, exec_len,
		1246	params->dispatch_flags);
		1247	if (ret)
		1248	return ret;
5060	serge	1249
6084	serge	1250	trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
5060	serge	1251
6084	serge	1252	i915_gem_execbuffer_move_to_active(vmas, params->request);
		1253	i915_gem_execbuffer_retire_commands(params);
5060	serge	1254
6084	serge	1255	return 0;
5060	serge	1256	}
		1257
		1258	/**
		1259	* Find one BSD ring to dispatch the corresponding BSD command.
		1260	* The Ring ID is returned.
		1261	*/
		1262	static int gen8_dispatch_bsd_ring(struct drm_device *dev,
		1263	struct drm_file *file)
		1264	{
		1265	struct drm_i915_private *dev_priv = dev->dev_private;
		1266	struct drm_i915_file_private *file_priv = file->driver_priv;
		1267
		1268	/* Check whether the file_priv is using one ring */
		1269	if (file_priv->bsd_ring)
		1270	return file_priv->bsd_ring->id;
		1271	else {
		1272	/* If no, use the ping-pong mechanism to select one ring */
		1273	int ring_id;
		1274
		1275	mutex_lock(&dev->struct_mutex);
		1276	if (dev_priv->mm.bsd_ring_dispatch_index == 0) {
		1277	ring_id = VCS;
		1278	dev_priv->mm.bsd_ring_dispatch_index = 1;
		1279	} else {
		1280	ring_id = VCS2;
		1281	dev_priv->mm.bsd_ring_dispatch_index = 0;
		1282	}
		1283	file_priv->bsd_ring = &dev_priv->ring[ring_id];
		1284	mutex_unlock(&dev->struct_mutex);
		1285	return ring_id;
		1286	}
		1287	}
		1288
		1289	static struct drm_i915_gem_object *
		1290	eb_get_batch(struct eb_vmas *eb)
		1291	{
		1292	struct i915_vma vma = list_entry(eb->vmas.prev, typeof(vma), exec_list);
		1293
		1294	/*
		1295	* SNA is doing fancy tricks with compressing batch buffers, which leads
		1296	* to negative relocation deltas. Usually that works out ok since the
		1297	* relocate address is still positive, except when the batch is placed
		1298	* very low in the GTT. Ensure this doesn't happen.
		1299	*
		1300	* Note that actual hangs have only been observed on gen7, but for
		1301	* paranoia do it everywhere.
		1302	*/
		1303	vma->exec_entry->flags \|= __EXEC_OBJECT_NEEDS_BIAS;
		1304
		1305	return vma->obj;
		1306	}
		1307
		1308	static int
3263	Serge	1309	i915_gem_do_execbuffer(struct drm_device dev, void data,
		1310	struct drm_file *file,
		1311	struct drm_i915_gem_execbuffer2 *args,
5060	serge	1312	struct drm_i915_gem_exec_object2 *exec)
3263	Serge	1313	{
5060	serge	1314	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1315	struct eb_vmas *eb;
3263	Serge	1316	struct drm_i915_gem_object *batch_obj;
6084	serge	1317	struct drm_i915_gem_exec_object2 shadow_exec_entry;
5060	serge	1318	struct intel_engine_cs *ring;
		1319	struct intel_context *ctx;
		1320	struct i915_address_space *vm;
6084	serge	1321	struct i915_execbuffer_params params_master; /* XXX: will be removed later */
		1322	struct i915_execbuffer_params *params = ¶ms_master;
4560	Serge	1323	const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
6084	serge	1324	u32 dispatch_flags;
5060	serge	1325	int ret;
3480	Serge	1326	bool need_relocs;
3263	Serge	1327
3480	Serge	1328	if (!i915_gem_check_execbuffer(args))
3263	Serge	1329	return -EINVAL;
		1330
5354	serge	1331	ret = validate_exec_list(dev, exec, args->buffer_count);
3263	Serge	1332	if (ret)
		1333	return ret;
		1334
6084	serge	1335	dispatch_flags = 0;
3263	Serge	1336	if (args->flags & I915_EXEC_SECURE) {
		1337
6084	serge	1338	dispatch_flags \|= I915_DISPATCH_SECURE;
3263	Serge	1339	}
		1340	if (args->flags & I915_EXEC_IS_PINNED)
6084	serge	1341	dispatch_flags \|= I915_DISPATCH_PINNED;
3263	Serge	1342
5060	serge	1343	if ((args->flags & I915_EXEC_RING_MASK) > LAST_USER_RING) {
3263	Serge	1344	DRM_DEBUG("execbuf with unknown ring: %d\n",
		1345	(int)(args->flags & I915_EXEC_RING_MASK));
		1346	return -EINVAL;
		1347	}
5060	serge	1348
6084	serge	1349	if (((args->flags & I915_EXEC_RING_MASK) != I915_EXEC_BSD) &&
		1350	((args->flags & I915_EXEC_BSD_MASK) != 0)) {
		1351	DRM_DEBUG("execbuf with non bsd ring but with invalid "
		1352	"bsd dispatch flags: %d\n", (int)(args->flags));
		1353	return -EINVAL;
		1354	}
		1355
5060	serge	1356	if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
		1357	ring = &dev_priv->ring[RCS];
		1358	else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) {
		1359	if (HAS_BSD2(dev)) {
		1360	int ring_id;
6084	serge	1361
		1362	switch (args->flags & I915_EXEC_BSD_MASK) {
		1363	case I915_EXEC_BSD_DEFAULT:
		1364	ring_id = gen8_dispatch_bsd_ring(dev, file);
		1365	ring = &dev_priv->ring[ring_id];
		1366	break;
		1367	case I915_EXEC_BSD_RING1:
		1368	ring = &dev_priv->ring[VCS];
		1369	break;
		1370	case I915_EXEC_BSD_RING2:
		1371	ring = &dev_priv->ring[VCS2];
		1372	break;
		1373	default:
		1374	DRM_DEBUG("execbuf with unknown bsd ring: %d\n",
		1375	(int)(args->flags & I915_EXEC_BSD_MASK));
		1376	return -EINVAL;
		1377	}
5060	serge	1378	} else
		1379	ring = &dev_priv->ring[VCS];
		1380	} else
		1381	ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
		1382
3263	Serge	1383	if (!intel_ring_initialized(ring)) {
		1384	DRM_DEBUG("execbuf with invalid ring: %d\n",
		1385	(int)(args->flags & I915_EXEC_RING_MASK));
		1386	return -EINVAL;
		1387	}
		1388
		1389	if (args->buffer_count < 1) {
		1390	DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
		1391	return -EINVAL;
		1392	}
		1393
6084	serge	1394	if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
		1395	if (!HAS_RESOURCE_STREAMER(dev)) {
		1396	DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
		1397	return -EINVAL;
		1398	}
		1399	if (ring->id != RCS) {
		1400	DRM_DEBUG("RS is not available on %s\n",
		1401	ring->name);
		1402	return -EINVAL;
		1403	}
		1404
		1405	dispatch_flags \|= I915_DISPATCH_RS;
		1406	}
		1407
4560	Serge	1408	intel_runtime_pm_get(dev_priv);
		1409
3263	Serge	1410	ret = i915_mutex_lock_interruptible(dev);
		1411	if (ret)
		1412	goto pre_mutex_err;
		1413
5060	serge	1414	ctx = i915_gem_validate_context(dev, file, ring, ctx_id);
		1415	if (IS_ERR(ctx)) {
4560	Serge	1416	mutex_unlock(&dev->struct_mutex);
5060	serge	1417	ret = PTR_ERR(ctx);
4560	Serge	1418	goto pre_mutex_err;
		1419	}
		1420
5060	serge	1421	i915_gem_context_reference(ctx);
		1422
5354	serge	1423	if (ctx->ppgtt)
		1424	vm = &ctx->ppgtt->base;
		1425	else
5060	serge	1426	vm = &dev_priv->gtt.base;
		1427
6084	serge	1428	memset(¶ms_master, 0x00, sizeof(params_master));
		1429
3480	Serge	1430	eb = eb_create(args);
3263	Serge	1431	if (eb == NULL) {
5060	serge	1432	i915_gem_context_unreference(ctx);
3263	Serge	1433	mutex_unlock(&dev->struct_mutex);
		1434	ret = -ENOMEM;
6084	serge	1435	goto pre_mutex_err;
3263	Serge	1436	}
		1437
		1438	/* Look up object handles */
4560	Serge	1439	ret = eb_lookup_vmas(eb, exec, args, vm, file);
3480	Serge	1440	if (ret)
6084	serge	1441	goto err;
3263	Serge	1442
		1443	/* take note of the batch buffer before we might reorder the lists */
5060	serge	1444	batch_obj = eb_get_batch(eb);
3263	Serge	1445
		1446	/* Move the objects en-masse into the GTT, evicting if necessary. */
3480	Serge	1447	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
6084	serge	1448	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
3263	Serge	1449	if (ret)
		1450	goto err;
		1451
		1452	/* The objects are in their final locations, apply the relocations. */
3480	Serge	1453	if (need_relocs)
4560	Serge	1454	ret = i915_gem_execbuffer_relocate(eb);
3263	Serge	1455	if (ret) {
		1456	if (ret == -EFAULT) {
3480	Serge	1457	ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
6084	serge	1458	eb, exec, ctx);
3263	Serge	1459	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1460	}
		1461	if (ret)
		1462	goto err;
6084	serge	1463	}
3263	Serge	1464
		1465	/* Set the pending read domains for the batch buffer to COMMAND */
		1466	if (batch_obj->base.pending_write_domain) {
		1467	DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
		1468	ret = -EINVAL;
		1469	goto err;
		1470	}
6084	serge	1471
		1472	params->args_batch_start_offset = args->batch_start_offset;
		1473
5060	serge	1474	#if 0
6084	serge	1475	if (i915_needs_cmd_parser(ring) && args->batch_len) {
		1476	struct drm_i915_gem_object *parsed_batch_obj;
		1477
		1478	parsed_batch_obj = i915_gem_execbuffer_parse(ring,
		1479	&shadow_exec_entry,
		1480	eb,
		1481	batch_obj,
		1482	args->batch_start_offset,
		1483	args->batch_len,
		1484	file->is_master);
		1485	if (IS_ERR(parsed_batch_obj)) {
		1486	ret = PTR_ERR(parsed_batch_obj);
5060	serge	1487	goto err;
6084	serge	1488	}
		1489
5060	serge	1490	/*
6084	serge	1491	* parsed_batch_obj == batch_obj means batch not fully parsed:
		1492	* Accept, but don't promote to secure.
5060	serge	1493	*/
6084	serge	1494
		1495	if (parsed_batch_obj != batch_obj) {
		1496	/*
		1497	* Batch parsed and accepted:
		1498	*
		1499	* Set the DISPATCH_SECURE bit to remove the NON_SECURE
		1500	* bit from MI_BATCH_BUFFER_START commands issued in
		1501	* the dispatch_execbuffer implementations. We
		1502	* specifically don't want that set on batches the
		1503	* command parser has accepted.
		1504	*/
		1505	dispatch_flags \|= I915_DISPATCH_SECURE;
		1506	params->args_batch_start_offset = 0;
		1507	batch_obj = parsed_batch_obj;
		1508	}
5060	serge	1509	}
6084	serge	1510	#endif
5367	serge	1511
6084	serge	1512	batch_obj->base.pending_read_domains \|= I915_GEM_DOMAIN_COMMAND;
		1513
3263	Serge	1514	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
		1515	* batch" bit. Hence we need to pin secure batches into the global gtt.
4560	Serge	1516	* hsw should have this fixed, but bdw mucks it up again. */
6084	serge	1517	if (dispatch_flags & I915_DISPATCH_SECURE) {
5354	serge	1518	/*
		1519	* So on first glance it looks freaky that we pin the batch here
		1520	* outside of the reservation loop. But:
		1521	* - The batch is already pinned into the relevant ppgtt, so we
		1522	* already have the backing storage fully allocated.
		1523	* - No other BO uses the global gtt (well contexts, but meh),
6084	serge	1524	* so we don't really have issues with multiple objects not
5354	serge	1525	* fitting due to fragmentation.
		1526	* So this is actually safe.
		1527	*/
		1528	ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
		1529	if (ret)
		1530	goto err;
3263	Serge	1531
6084	serge	1532	params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
5354	serge	1533	} else
6084	serge	1534	params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
3263	Serge	1535
6084	serge	1536	/* Allocate a request for this batch buffer nice and early. */
		1537	ret = i915_gem_request_alloc(ring, ctx, ¶ms->request);
		1538	if (ret)
		1539	goto err_batch_unpin;
3263	Serge	1540
6084	serge	1541	ret = i915_gem_request_add_to_client(params->request, file);
		1542	if (ret)
		1543	goto err_batch_unpin;
		1544
5354	serge	1545	/*
6084	serge	1546	* Save assorted stuff away to pass through to *_submission().
		1547	* NB: This data should be 'persistent' and not local as it will
		1548	* kept around beyond the duration of the IOCTL once the GPU
		1549	* scheduler arrives.
		1550	*/
		1551	params->dev = dev;
		1552	params->file = file;
		1553	params->ring = ring;
		1554	params->dispatch_flags = dispatch_flags;
		1555	params->batch_obj = batch_obj;
		1556	params->ctx = ctx;
		1557
		1558	ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
		1559
		1560	err_batch_unpin:
		1561	/*
5354	serge	1562	* FIXME: We crucially rely upon the active tracking for the (ppgtt)
		1563	* batch vma for correctness. For less ugly and less fragility this
		1564	* needs to be adjusted to also track the ggtt batch vma properly as
		1565	* active.
		1566	*/
6084	serge	1567	if (dispatch_flags & I915_DISPATCH_SECURE)
5354	serge	1568	i915_gem_object_ggtt_unpin(batch_obj);
6084	serge	1569
3263	Serge	1570	err:
5060	serge	1571	/* the request owns the ref now */
		1572	i915_gem_context_unreference(ctx);
3263	Serge	1573	eb_destroy(eb);
		1574
6084	serge	1575	/*
		1576	* If the request was created but not successfully submitted then it
		1577	* must be freed again. If it was submitted then it is being tracked
		1578	* on the active request list and no clean up is required here.
		1579	*/
		1580	if (ret && params->request)
		1581	i915_gem_request_cancel(params->request);
		1582
3263	Serge	1583	mutex_unlock(&dev->struct_mutex);
		1584
		1585	pre_mutex_err:
4560	Serge	1586	/* intel_gpu_busy should also get a ref, so it will free when the device
		1587	* is really idle. */
		1588	intel_runtime_pm_put(dev_priv);
3263	Serge	1589	return ret;
		1590	}
		1591
4246	Serge	1592	#if 0
		1593	/*
		1594	* Legacy execbuffer just creates an exec2 list from the original exec object
		1595	* list array and passes it to the real function.
		1596	*/
		1597	int
		1598	i915_gem_execbuffer(struct drm_device dev, void data,
		1599	struct drm_file *file)
		1600	{
		1601	struct drm_i915_gem_execbuffer *args = data;
		1602	struct drm_i915_gem_execbuffer2 exec2;
		1603	struct drm_i915_gem_exec_object *exec_list = NULL;
		1604	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
		1605	int ret, i;
3480	Serge	1606
4246	Serge	1607	if (args->buffer_count < 1) {
		1608	DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
		1609	return -EINVAL;
		1610	}
3480	Serge	1611
4246	Serge	1612	/* Copy in the exec list from userland */
		1613	exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
		1614	exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
		1615	if (exec_list == NULL \|\| exec2_list == NULL) {
		1616	DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
		1617	args->buffer_count);
		1618	drm_free_large(exec_list);
		1619	drm_free_large(exec2_list);
		1620	return -ENOMEM;
		1621	}
		1622	ret = copy_from_user(exec_list,
		1623	to_user_ptr(args->buffers_ptr),
		1624	sizeof(exec_list) args->buffer_count);
		1625	if (ret != 0) {
		1626	DRM_DEBUG("copy %d exec entries failed %d\n",
		1627	args->buffer_count, ret);
		1628	drm_free_large(exec_list);
		1629	drm_free_large(exec2_list);
		1630	return -EFAULT;
		1631	}
		1632
		1633	for (i = 0; i < args->buffer_count; i++) {
		1634	exec2_list[i].handle = exec_list[i].handle;
		1635	exec2_list[i].relocation_count = exec_list[i].relocation_count;
		1636	exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
		1637	exec2_list[i].alignment = exec_list[i].alignment;
		1638	exec2_list[i].offset = exec_list[i].offset;
		1639	if (INTEL_INFO(dev)->gen < 4)
		1640	exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
		1641	else
		1642	exec2_list[i].flags = 0;
		1643	}
		1644
		1645	exec2.buffers_ptr = args->buffers_ptr;
		1646	exec2.buffer_count = args->buffer_count;
		1647	exec2.batch_start_offset = args->batch_start_offset;
		1648	exec2.batch_len = args->batch_len;
		1649	exec2.DR1 = args->DR1;
		1650	exec2.DR4 = args->DR4;
		1651	exec2.num_cliprects = args->num_cliprects;
		1652	exec2.cliprects_ptr = args->cliprects_ptr;
		1653	exec2.flags = I915_EXEC_RENDER;
		1654	i915_execbuffer2_set_context_id(exec2, 0);
		1655
5060	serge	1656	ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
4246	Serge	1657	if (!ret) {
5060	serge	1658	struct drm_i915_gem_exec_object __user *user_exec_list =
		1659	to_user_ptr(args->buffers_ptr);
		1660
4246	Serge	1661	/* Copy the new buffer offsets back to the user's exec list. */
5060	serge	1662	for (i = 0; i < args->buffer_count; i++) {
		1663	ret = __copy_to_user(&user_exec_list[i].offset,
		1664	&exec2_list[i].offset,
		1665	sizeof(user_exec_list[i].offset));
6084	serge	1666	if (ret) {
		1667	ret = -EFAULT;
		1668	DRM_DEBUG("failed to copy %d exec entries "
		1669	"back to user (%d)\n",
		1670	args->buffer_count, ret);
5060	serge	1671	break;
		1672	}
4246	Serge	1673	}
		1674	}
		1675
		1676	drm_free_large(exec_list);
		1677	drm_free_large(exec2_list);
		1678	return ret;
		1679	}
		1680	#endif
		1681
3263	Serge	1682	int
		1683	i915_gem_execbuffer2(struct drm_device dev, void data,
		1684	struct drm_file *file)
		1685	{
		1686	struct drm_i915_gem_execbuffer2 *args = data;
		1687	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
		1688	int ret;
		1689
		1690	if (args->buffer_count < 1 \|\|
		1691	args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
		1692	DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
		1693	return -EINVAL;
		1694	}
		1695
5060	serge	1696	if (args->rsvd2 != 0) {
		1697	DRM_DEBUG("dirty rvsd2 field\n");
		1698	return -EINVAL;
		1699	}
		1700
3480	Serge	1701	exec2_list = kmalloc(sizeof(exec2_list)args->buffer_count,
		1702	GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
3263	Serge	1703	if (exec2_list == NULL) {
		1704	DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
		1705	args->buffer_count);
		1706	return -ENOMEM;
		1707	}
		1708	ret = copy_from_user(exec2_list,
4539	Serge	1709	to_user_ptr(args->buffers_ptr),
3263	Serge	1710	sizeof(exec2_list) args->buffer_count);
		1711	if (ret != 0) {
		1712	DRM_DEBUG("copy %d exec entries failed %d\n",
		1713	args->buffer_count, ret);
3266	Serge	1714	kfree(exec2_list);
		1715	FAIL();
3263	Serge	1716	return -EFAULT;
		1717	}
		1718
5060	serge	1719	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
3263	Serge	1720	if (!ret) {
		1721	/* Copy the new buffer offsets back to the user's exec list. */
5060	serge	1722	struct drm_i915_gem_exec_object2 __user *user_exec_list =
		1723	to_user_ptr(args->buffers_ptr);
		1724	int i;
		1725
		1726	for (i = 0; i < args->buffer_count; i++) {
		1727	ret = __copy_to_user(&user_exec_list[i].offset,
		1728	&exec2_list[i].offset,
		1729	sizeof(user_exec_list[i].offset));
6084	serge	1730	if (ret) {
		1731	ret = -EFAULT;
		1732	DRM_DEBUG("failed to copy %d exec entries "
5060	serge	1733	"back to user\n",
		1734	args->buffer_count);
		1735	break;
		1736	}
3263	Serge	1737	}
		1738	}
		1739
3266	Serge	1740	kfree(exec2_list);
3263	Serge	1741	return ret;
		1742	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem_execbuffer.c @ 5367 – Rev 6084