WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem_execbuffer.c

Rev	Author	Line No.	Line
3263	Serge	1	/*
		2	* Copyright © 2008,2010 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	* Chris Wilson
		26	*
		27	*/
		28
		29	#include
		30	#include
		31	#include "i915_drv.h"
		32	#include "i915_trace.h"
		33	#include "intel_drv.h"
5060	serge	34	#include
6935	serge	35	#include
3263	Serge	36
4560	Serge	37	#define __EXEC_OBJECT_HAS_PIN (1<<31)
		38	#define __EXEC_OBJECT_HAS_FENCE (1<<30)
5354	serge	39	#define __EXEC_OBJECT_NEEDS_MAP (1<<29)
5060	serge	40	#define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
3263	Serge	41
5060	serge	42	#define BATCH_OFFSET_BIAS (256*1024)
3263	Serge	43
4560	Serge	44	struct eb_vmas {
		45	struct list_head vmas;
3263	Serge	46	int and;
3480	Serge	47	union {
4560	Serge	48	struct i915_vma *lut[0];
5060	serge	49	struct hlist_head buckets[0];
3480	Serge	50	};
3263	Serge	51	};
		52
4560	Serge	53	static struct eb_vmas *
3480	Serge	54	eb_create(struct drm_i915_gem_execbuffer2 *args)
3263	Serge	55	{
4560	Serge	56	struct eb_vmas *eb = NULL;
3480	Serge	57
		58	if (args->flags & I915_EXEC_HANDLE_LUT) {
4560	Serge	59	unsigned size = args->buffer_count;
		60	size = sizeof(struct i915_vma );
		61	size += sizeof(struct eb_vmas);
3480	Serge	62	eb = kmalloc(size, GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
		63	}
		64
		65	if (eb == NULL) {
4560	Serge	66	unsigned size = args->buffer_count;
		67	unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
3480	Serge	68	BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
		69	while (count > 2*size)
6084	serge	70	count >>= 1;
		71	eb = kzalloc(count*sizeof(struct hlist_head) +
4560	Serge	72	sizeof(struct eb_vmas),
3480	Serge	73	GFP_TEMPORARY);
6084	serge	74	if (eb == NULL)
		75	return eb;
3263	Serge	76
6084	serge	77	eb->and = count - 1;
3480	Serge	78	} else
		79	eb->and = -args->buffer_count;
		80
4560	Serge	81	INIT_LIST_HEAD(&eb->vmas);
3263	Serge	82	return eb;
		83	}
		84
		85	static void
4560	Serge	86	eb_reset(struct eb_vmas *eb)
3263	Serge	87	{
3480	Serge	88	if (eb->and >= 0)
6084	serge	89	memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
3263	Serge	90	}
		91
3480	Serge	92	static int
4560	Serge	93	eb_lookup_vmas(struct eb_vmas *eb,
6084	serge	94	struct drm_i915_gem_exec_object2 *exec,
		95	const struct drm_i915_gem_execbuffer2 *args,
4560	Serge	96	struct i915_address_space *vm,
6084	serge	97	struct drm_file *file)
3263	Serge	98	{
4560	Serge	99	struct drm_i915_gem_object *obj;
		100	struct list_head objects;
		101	int i, ret;
3480	Serge	102
4560	Serge	103	INIT_LIST_HEAD(&objects);
3480	Serge	104	spin_lock(&file->table_lock);
4560	Serge	105	/* Grab a reference to the object and release the lock so we can lookup
		106	* or create the VMA without using GFP_ATOMIC */
3480	Serge	107	for (i = 0; i < args->buffer_count; i++) {
6084	serge	108	obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
3480	Serge	109	if (obj == NULL) {
		110	spin_unlock(&file->table_lock);
		111	DRM_DEBUG("Invalid object handle %d at index %d\n",
		112	exec[i].handle, i);
4560	Serge	113	ret = -ENOENT;
		114	goto err;
3480	Serge	115	}
		116
4560	Serge	117	if (!list_empty(&obj->obj_exec_link)) {
3480	Serge	118	spin_unlock(&file->table_lock);
		119	DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
		120	obj, exec[i].handle, i);
4560	Serge	121	ret = -EINVAL;
		122	goto err;
3480	Serge	123	}
		124
		125	drm_gem_object_reference(&obj->base);
4560	Serge	126	list_add_tail(&obj->obj_exec_link, &objects);
		127	}
		128	spin_unlock(&file->table_lock);
3480	Serge	129
4560	Serge	130	i = 0;
		131	while (!list_empty(&objects)) {
		132	struct i915_vma *vma;
		133
		134	obj = list_first_entry(&objects,
		135	struct drm_i915_gem_object,
		136	obj_exec_link);
		137
		138	/*
		139	* NOTE: We can leak any vmas created here when something fails
		140	* later on. But that's no issue since vma_unbind can deal with
		141	* vmas which are not actually bound. And since only
		142	* lookup_or_create exists as an interface to get at the vma
		143	* from the (obj, vm) we don't run the risk of creating
		144	* duplicated vmas for the same vm.
		145	*/
5354	serge	146	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
4560	Serge	147	if (IS_ERR(vma)) {
		148	DRM_DEBUG("Failed to lookup VMA\n");
		149	ret = PTR_ERR(vma);
		150	goto err;
		151	}
		152
		153	/* Transfer ownership from the objects list to the vmas list. */
		154	list_add_tail(&vma->exec_list, &eb->vmas);
		155	list_del_init(&obj->obj_exec_link);
		156
		157	vma->exec_entry = &exec[i];
3480	Serge	158	if (eb->and < 0) {
4560	Serge	159	eb->lut[i] = vma;
3480	Serge	160	} else {
		161	uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
4560	Serge	162	vma->exec_handle = handle;
		163	hlist_add_head(&vma->exec_node,
3480	Serge	164	&eb->buckets[handle & eb->and]);
		165	}
4560	Serge	166	++i;
3480	Serge	167	}
		168
		169	return 0;
4560	Serge	170
		171
		172	err:
		173	while (!list_empty(&objects)) {
		174	obj = list_first_entry(&objects,
		175	struct drm_i915_gem_object,
		176	obj_exec_link);
		177	list_del_init(&obj->obj_exec_link);
		178	drm_gem_object_unreference(&obj->base);
		179	}
		180	/*
		181	* Objects already transfered to the vmas list will be unreferenced by
		182	* eb_destroy.
		183	*/
		184
		185	return ret;
3263	Serge	186	}
		187
4560	Serge	188	static struct i915_vma eb_get_vma(struct eb_vmas eb, unsigned long handle)
3263	Serge	189	{
3480	Serge	190	if (eb->and < 0) {
		191	if (handle >= -eb->and)
		192	return NULL;
		193	return eb->lut[handle];
		194	} else {
6084	serge	195	struct hlist_head *head;
		196	struct hlist_node *node;
3263	Serge	197
6084	serge	198	head = &eb->buckets[handle & eb->and];
		199	hlist_for_each(node, head) {
4560	Serge	200	struct i915_vma *vma;
3480	Serge	201
4560	Serge	202	vma = hlist_entry(node, struct i915_vma, exec_node);
		203	if (vma->exec_handle == handle)
		204	return vma;
6084	serge	205	}
		206	return NULL;
3263	Serge	207	}
		208	}
		209
		210	static void
4560	Serge	211	i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
3263	Serge	212	{
4560	Serge	213	struct drm_i915_gem_exec_object2 *entry;
		214	struct drm_i915_gem_object *obj = vma->obj;
3480	Serge	215
4560	Serge	216	if (!drm_mm_node_allocated(&vma->node))
		217	return;
		218
		219	entry = vma->exec_entry;
		220
		221	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
		222	i915_gem_object_unpin_fence(obj);
		223
		224	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
5060	serge	225	vma->pin_count--;
4560	Serge	226
		227	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE \| __EXEC_OBJECT_HAS_PIN);
		228	}
		229
		230	static void eb_destroy(struct eb_vmas *eb)
		231	{
		232	while (!list_empty(&eb->vmas)) {
		233	struct i915_vma *vma;
		234
		235	vma = list_first_entry(&eb->vmas,
		236	struct i915_vma,
3480	Serge	237	exec_list);
4560	Serge	238	list_del_init(&vma->exec_list);
		239	i915_gem_execbuffer_unreserve_vma(vma);
		240	drm_gem_object_unreference(&vma->obj->base);
3480	Serge	241	}
3263	Serge	242	kfree(eb);
		243	}
		244
		245	static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
		246	{
4560	Serge	247	return (HAS_LLC(obj->base.dev) \|\|
		248	obj->base.write_domain == I915_GEM_DOMAIN_CPU \|\|
3263	Serge	249	obj->cache_level != I915_CACHE_NONE);
		250	}
		251
6937	serge	252	/* Used to convert any address to canonical form.
		253	* Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
		254	* MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
		255	* addresses to be in a canonical form:
		256	* "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
		257	* canonical form [63:48] == [47]."
		258	*/
		259	#define GEN8_HIGH_ADDRESS_BIT 47
		260	static inline uint64_t gen8_canonical_addr(uint64_t address)
		261	{
		262	return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
		263	}
		264
		265	static inline uint64_t gen8_noncanonical_addr(uint64_t address)
		266	{
		267	return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
		268	}
		269
		270	static inline uint64_t
		271	relocation_target(struct drm_i915_gem_relocation_entry *reloc,
		272	uint64_t target_offset)
		273	{
		274	return gen8_canonical_addr((int)reloc->delta + target_offset);
		275	}
		276
3263	Serge	277	static int
4371	Serge	278	relocate_entry_cpu(struct drm_i915_gem_object *obj,
5060	serge	279	struct drm_i915_gem_relocation_entry *reloc,
		280	uint64_t target_offset)
4371	Serge	281	{
6084	serge	282	struct drm_device *dev = obj->base.dev;
		283	uint32_t page_offset = offset_in_page(reloc->offset);
6937	serge	284	uint64_t delta = relocation_target(reloc, target_offset);
4371	Serge	285	char *vaddr;
4560	Serge	286	int ret;
4371	Serge	287
4560	Serge	288	ret = i915_gem_object_set_to_cpu_domain(obj, true);
4371	Serge	289	if (ret)
		290	return ret;
		291
6937	serge	292	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
5354	serge	293	reloc->offset >> PAGE_SHIFT));
5060	serge	294	(uint32_t )(vaddr + page_offset) = lower_32_bits(delta);
4371	Serge	295
5060	serge	296	if (INTEL_INFO(dev)->gen >= 8) {
		297	page_offset = offset_in_page(page_offset + sizeof(uint32_t));
		298
		299	if (page_offset == 0) {
5354	serge	300	kunmap_atomic(vaddr);
6937	serge	301	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
5354	serge	302	(reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
5060	serge	303	}
		304
		305	(uint32_t )(vaddr + page_offset) = upper_32_bits(delta);
		306	}
		307
5354	serge	308	kunmap_atomic(vaddr);
		309
4371	Serge	310	return 0;
		311	}
		312
		313	static int
		314	relocate_entry_gtt(struct drm_i915_gem_object *obj,
5060	serge	315	struct drm_i915_gem_relocation_entry *reloc,
		316	uint64_t target_offset)
4371	Serge	317	{
		318	struct drm_device *dev = obj->base.dev;
		319	struct drm_i915_private *dev_priv = dev->dev_private;
6937	serge	320	uint64_t delta = relocation_target(reloc, target_offset);
5354	serge	321	uint64_t offset;
4371	Serge	322	void __iomem *reloc_page;
4560	Serge	323	int ret;
4371	Serge	324
		325	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		326	if (ret)
		327	return ret;
		328
		329	ret = i915_gem_object_put_fence(obj);
		330	if (ret)
		331	return ret;
		332
		333	/* Map the page containing the relocation we're going to perform. */
5354	serge	334	offset = i915_gem_obj_ggtt_offset(obj);
		335	offset += reloc->offset;
4539	Serge	336	MapPage(dev_priv->gtt.mappable,dev_priv->gtt.mappable_base +
5354	serge	337	(offset & PAGE_MASK), PG_SW);
4539	Serge	338	reloc_page = dev_priv->gtt.mappable;
5354	serge	339	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
4371	Serge	340
6084	serge	341	// io_mapping_unmap_atomic(reloc_page);
5060	serge	342
6084	serge	343	return 0;
		344	}
5354	serge	345
6084	serge	346	static void
		347	clflush_write32(void *addr, uint32_t value)
		348	{
		349	/* This is not a fast path, so KISS. */
		350	drm_clflush_virt_range(addr, sizeof(uint32_t));
		351	(uint32_t )addr = value;
		352	drm_clflush_virt_range(addr, sizeof(uint32_t));
		353	}
		354
		355	static int
		356	relocate_entry_clflush(struct drm_i915_gem_object *obj,
		357	struct drm_i915_gem_relocation_entry *reloc,
		358	uint64_t target_offset)
		359	{
		360	struct drm_device *dev = obj->base.dev;
		361	uint32_t page_offset = offset_in_page(reloc->offset);
6937	serge	362	uint64_t delta = relocation_target(reloc, target_offset);
6084	serge	363	char *vaddr;
		364	int ret;
		365
		366	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		367	if (ret)
		368	return ret;
		369
6937	serge	370	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
6084	serge	371	reloc->offset >> PAGE_SHIFT));
		372	clflush_write32(vaddr + page_offset, lower_32_bits(delta));
		373
		374	if (INTEL_INFO(dev)->gen >= 8) {
		375	page_offset = offset_in_page(page_offset + sizeof(uint32_t));
		376
		377	if (page_offset == 0) {
		378	kunmap_atomic(vaddr);
6937	serge	379	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
6084	serge	380	(reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
		381	}
		382
		383	clflush_write32(vaddr + page_offset, upper_32_bits(delta));
		384	}
		385
		386	kunmap_atomic(vaddr);
		387
4371	Serge	388	return 0;
		389	}
		390
		391	static int
3263	Serge	392	i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
4560	Serge	393	struct eb_vmas *eb,
5060	serge	394	struct drm_i915_gem_relocation_entry *reloc)
3263	Serge	395	{
		396	struct drm_device *dev = obj->base.dev;
		397	struct drm_gem_object *target_obj;
		398	struct drm_i915_gem_object *target_i915_obj;
4560	Serge	399	struct i915_vma *target_vma;
5060	serge	400	uint64_t target_offset;
4560	Serge	401	int ret;
3263	Serge	402
		403	/* we've already hold a reference to all valid objects */
4560	Serge	404	target_vma = eb_get_vma(eb, reloc->target_handle);
		405	if (unlikely(target_vma == NULL))
3263	Serge	406	return -ENOENT;
4560	Serge	407	target_i915_obj = target_vma->obj;
		408	target_obj = &target_vma->obj->base;
3263	Serge	409
6937	serge	410	target_offset = gen8_canonical_addr(target_vma->node.start);
3263	Serge	411
		412	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
		413	* pipe_control writes because the gpu doesn't properly redirect them
		414	* through the ppgtt for non_secure batchbuffers. */
		415	if (unlikely(IS_GEN6(dev) &&
6084	serge	416	reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
		417	ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
		418	PIN_GLOBAL);
		419	if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
		420	return ret;
		421	}
3263	Serge	422
		423	/* Validate that the target is in a valid r/w GPU domain */
		424	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
		425	DRM_DEBUG("reloc with multiple write domains: "
		426	"obj %p target %d offset %d "
		427	"read %08x write %08x",
		428	obj, reloc->target_handle,
		429	(int) reloc->offset,
		430	reloc->read_domains,
		431	reloc->write_domain);
4560	Serge	432	return -EINVAL;
3263	Serge	433	}
		434	if (unlikely((reloc->write_domain \| reloc->read_domains)
		435	& ~I915_GEM_GPU_DOMAINS)) {
		436	DRM_DEBUG("reloc with read/write non-GPU domains: "
		437	"obj %p target %d offset %d "
		438	"read %08x write %08x",
		439	obj, reloc->target_handle,
		440	(int) reloc->offset,
		441	reloc->read_domains,
		442	reloc->write_domain);
4560	Serge	443	return -EINVAL;
3263	Serge	444	}
		445
		446	target_obj->pending_read_domains \|= reloc->read_domains;
		447	target_obj->pending_write_domain \|= reloc->write_domain;
		448
		449	/* If the relocation already has the right value in it, no
		450	* more work needs to be done.
		451	*/
		452	if (target_offset == reloc->presumed_offset)
		453	return 0;
		454
		455	/* Check that the relocation address is valid... */
4560	Serge	456	if (unlikely(reloc->offset >
		457	obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
3263	Serge	458	DRM_DEBUG("Relocation beyond object bounds: "
		459	"obj %p target %d offset %d size %d.\n",
		460	obj, reloc->target_handle,
		461	(int) reloc->offset,
		462	(int) obj->base.size);
4560	Serge	463	return -EINVAL;
3263	Serge	464	}
		465	if (unlikely(reloc->offset & 3)) {
		466	DRM_DEBUG("Relocation not 4-byte aligned: "
		467	"obj %p target %d offset %d.\n",
		468	obj, reloc->target_handle,
		469	(int) reloc->offset);
4560	Serge	470	return -EINVAL;
3263	Serge	471	}
		472
		473	/* We can't wait for rendering with pagefaults disabled */
		474
4371	Serge	475	if (use_cpu_reloc(obj))
5060	serge	476	ret = relocate_entry_cpu(obj, reloc, target_offset);
6084	serge	477	else if (obj->map_and_fenceable)
5060	serge	478	ret = relocate_entry_gtt(obj, reloc, target_offset);
6084	serge	479	else if (1)
		480	ret = relocate_entry_clflush(obj, reloc, target_offset);
		481	else {
		482	WARN_ONCE(1, "Impossible case in relocation handling\n");
		483	ret = -ENODEV;
		484	}
3263	Serge	485
6084	serge	486	if (ret)
		487	return ret;
3263	Serge	488
		489	/* and update the user's relocation entry */
		490	reloc->presumed_offset = target_offset;
		491
		492	return 0;
		493	}
		494
		495	static int
4560	Serge	496	i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
		497	struct eb_vmas *eb)
3263	Serge	498	{
		499	#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
3266	Serge	500	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(64)];
3263	Serge	501	struct drm_i915_gem_relocation_entry __user *user_relocs;
4560	Serge	502	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
3263	Serge	503	int remain, ret;
		504
4539	Serge	505	user_relocs = to_user_ptr(entry->relocs_ptr);
3263	Serge	506
		507	remain = entry->relocation_count;
		508	while (remain) {
		509	struct drm_i915_gem_relocation_entry *r = stack_reloc;
		510	int count = remain;
		511	if (count > ARRAY_SIZE(stack_reloc))
		512	count = ARRAY_SIZE(stack_reloc);
		513	remain -= count;
		514
		515	memcpy(r, user_relocs, count*sizeof(r[0]));
		516
		517	do {
		518	u64 offset = r->presumed_offset;
		519
5060	serge	520	ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
3263	Serge	521	if (ret)
		522	return ret;
		523
4392	Serge	524	if (r->presumed_offset != offset)
		525	{
3263	Serge	526	memcpy(&user_relocs->presumed_offset,
		527	&r->presumed_offset,
		528	sizeof(r->presumed_offset));
4392	Serge	529	}
3263	Serge	530
		531	user_relocs++;
		532	r++;
		533	} while (--count);
		534	}
		535
		536	return 0;
		537	#undef N_RELOC
		538	}
		539
		540	static int
4560	Serge	541	i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
		542	struct eb_vmas *eb,
		543	struct drm_i915_gem_relocation_entry *relocs)
3263	Serge	544	{
4560	Serge	545	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
3263	Serge	546	int i, ret;
		547
		548	for (i = 0; i < entry->relocation_count; i++) {
5060	serge	549	ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
3263	Serge	550	if (ret)
		551	return ret;
		552	}
		553
		554	return 0;
		555	}
		556
		557	static int
4560	Serge	558	i915_gem_execbuffer_relocate(struct eb_vmas *eb)
3263	Serge	559	{
4560	Serge	560	struct i915_vma *vma;
3263	Serge	561	int ret = 0;
		562
		563	/* This is the fast path and we cannot handle a pagefault whilst
		564	* holding the struct mutex lest the user pass in the relocations
		565	* contained within a mmaped bo. For in such a case we, the page
		566	* fault handler would call i915_gem_fault() and we would try to
		567	* acquire the struct mutex again. Obviously this is bad and so
		568	* lockdep complains vehemently.
		569	*/
6935	serge	570	pagefault_disable();
4560	Serge	571	list_for_each_entry(vma, &eb->vmas, exec_list) {
		572	ret = i915_gem_execbuffer_relocate_vma(vma, eb);
3263	Serge	573	if (ret)
		574	break;
		575	}
6935	serge	576	pagefault_enable();
3263	Serge	577
		578	return ret;
		579	}
		580
6084	serge	581	static bool only_mappable_for_reloc(unsigned int flags)
		582	{
		583	return (flags & (EXEC_OBJECT_NEEDS_FENCE \| __EXEC_OBJECT_NEEDS_MAP)) ==
		584	__EXEC_OBJECT_NEEDS_MAP;
		585	}
		586
3263	Serge	587	static int
4560	Serge	588	i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
5060	serge	589	struct intel_engine_cs *ring,
6084	serge	590	bool *need_reloc)
3263	Serge	591	{
5060	serge	592	struct drm_i915_gem_object *obj = vma->obj;
4560	Serge	593	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5060	serge	594	uint64_t flags;
3263	Serge	595	int ret;
		596
6084	serge	597	flags = PIN_USER;
5060	serge	598	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
		599	flags \|= PIN_GLOBAL;
		600
6084	serge	601	if (!drm_mm_node_allocated(&vma->node)) {
		602	/* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
		603	* limit address to the first 4GBs for unflagged objects.
		604	*/
		605	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
		606	flags \|= PIN_ZONE_4G;
		607	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
		608	flags \|= PIN_GLOBAL \| PIN_MAPPABLE;
		609	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
		610	flags \|= BATCH_OFFSET_BIAS \| PIN_OFFSET_BIAS;
6937	serge	611	if (entry->flags & EXEC_OBJECT_PINNED)
		612	flags \|= entry->offset \| PIN_OFFSET_FIXED;
6084	serge	613	if ((flags & PIN_MAPPABLE) == 0)
		614	flags \|= PIN_HIGH;
		615	}
		616
5060	serge	617	ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
6084	serge	618	if ((ret == -ENOSPC \|\| ret == -E2BIG) &&
		619	only_mappable_for_reloc(entry->flags))
		620	ret = i915_gem_object_pin(obj, vma->vm,
		621	entry->alignment,
		622	flags & ~PIN_MAPPABLE);
3263	Serge	623	if (ret)
		624	return ret;
		625
		626	entry->flags \|= __EXEC_OBJECT_HAS_PIN;
		627
6084	serge	628	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
		629	ret = i915_gem_object_get_fence(obj);
		630	if (ret)
		631	return ret;
3263	Serge	632
6084	serge	633	if (i915_gem_object_pin_fence(obj))
		634	entry->flags \|= __EXEC_OBJECT_HAS_FENCE;
3263	Serge	635	}
		636
4560	Serge	637	if (entry->offset != vma->node.start) {
		638	entry->offset = vma->node.start;
3480	Serge	639	*need_reloc = true;
		640	}
3266	Serge	641
3480	Serge	642	if (entry->flags & EXEC_OBJECT_WRITE) {
		643	obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
		644	obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
		645	}
		646
3263	Serge	647	return 0;
		648	}
		649
5060	serge	650	static bool
5354	serge	651	need_reloc_mappable(struct i915_vma *vma)
5060	serge	652	{
		653	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5354	serge	654
		655	if (entry->relocation_count == 0)
		656	return false;
		657
		658	if (!i915_is_ggtt(vma->vm))
		659	return false;
		660
		661	/* See also use_cpu_reloc() */
		662	if (HAS_LLC(vma->obj->base.dev))
		663	return false;
		664
		665	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		666	return false;
		667
		668	return true;
		669	}
		670
		671	static bool
		672	eb_vma_misplaced(struct i915_vma *vma)
		673	{
		674	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
5060	serge	675	struct drm_i915_gem_object *obj = vma->obj;
		676
5354	serge	677	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
5060	serge	678	!i915_is_ggtt(vma->vm));
		679
		680	if (entry->alignment &&
		681	vma->node.start & (entry->alignment - 1))
		682	return true;
		683
6937	serge	684	if (entry->flags & EXEC_OBJECT_PINNED &&
		685	vma->node.start != entry->offset)
		686	return true;
		687
5060	serge	688	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
		689	vma->node.start < BATCH_OFFSET_BIAS)
		690	return true;
		691
6084	serge	692	/* avoid costly ping-pong once a batch bo ended up non-mappable */
		693	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
		694	return !only_mappable_for_reloc(entry->flags);
		695
		696	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
		697	(vma->node.start + vma->node.size - 1) >> 32)
		698	return true;
		699
5060	serge	700	return false;
		701	}
		702
3263	Serge	703	static int
5060	serge	704	i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
4560	Serge	705	struct list_head *vmas,
6084	serge	706	struct intel_context *ctx,
3480	Serge	707	bool *need_relocs)
3263	Serge	708	{
		709	struct drm_i915_gem_object *obj;
4560	Serge	710	struct i915_vma *vma;
		711	struct i915_address_space *vm;
		712	struct list_head ordered_vmas;
6937	serge	713	struct list_head pinned_vmas;
3263	Serge	714	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
		715	int retry;
		716
5060	serge	717	i915_gem_retire_requests_ring(ring);
		718
4560	Serge	719	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
		720
		721	INIT_LIST_HEAD(&ordered_vmas);
6937	serge	722	INIT_LIST_HEAD(&pinned_vmas);
4560	Serge	723	while (!list_empty(vmas)) {
3263	Serge	724	struct drm_i915_gem_exec_object2 *entry;
		725	bool need_fence, need_mappable;
		726
4560	Serge	727	vma = list_first_entry(vmas, struct i915_vma, exec_list);
		728	obj = vma->obj;
		729	entry = vma->exec_entry;
3263	Serge	730
6084	serge	731	if (ctx->flags & CONTEXT_NO_ZEROMAP)
		732	entry->flags \|= __EXEC_OBJECT_NEEDS_BIAS;
		733
5354	serge	734	if (!has_fenced_gpu_access)
		735	entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
3263	Serge	736	need_fence =
		737	entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
		738	obj->tiling_mode != I915_TILING_NONE;
4560	Serge	739	need_mappable = need_fence \|\| need_reloc_mappable(vma);
3263	Serge	740
6937	serge	741	if (entry->flags & EXEC_OBJECT_PINNED)
		742	list_move_tail(&vma->exec_list, &pinned_vmas);
		743	else if (need_mappable) {
5354	serge	744	entry->flags \|= __EXEC_OBJECT_NEEDS_MAP;
4560	Serge	745	list_move(&vma->exec_list, &ordered_vmas);
5354	serge	746	} else
4560	Serge	747	list_move_tail(&vma->exec_list, &ordered_vmas);
3263	Serge	748
3480	Serge	749	obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
3263	Serge	750	obj->base.pending_write_domain = 0;
		751	}
4560	Serge	752	list_splice(&ordered_vmas, vmas);
6937	serge	753	list_splice(&pinned_vmas, vmas);
3263	Serge	754
		755	/* Attempt to pin all of the buffers into the GTT.
		756	* This is done in 3 phases:
		757	*
		758	* 1a. Unbind all objects that do not match the GTT constraints for
		759	* the execbuffer (fenceable, mappable, alignment etc).
		760	* 1b. Increment pin count for already bound objects.
		761	* 2. Bind new objects.
		762	* 3. Decrement pin count.
		763	*
		764	* This avoid unnecessary unbinding of later objects in order to make
		765	* room for the earlier objects unless we need to defragment.
		766	*/
		767	retry = 0;
		768	do {
		769	int ret = 0;
		770
		771	/* Unbind any ill-fitting objects or pin. */
4560	Serge	772	list_for_each_entry(vma, vmas, exec_list) {
		773	if (!drm_mm_node_allocated(&vma->node))
3263	Serge	774	continue;
		775
5354	serge	776	if (eb_vma_misplaced(vma))
4560	Serge	777	ret = i915_vma_unbind(vma);
3263	Serge	778	else
4560	Serge	779	ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
3263	Serge	780	if (ret)
		781	goto err;
		782	}
		783
		784	/* Bind fresh objects */
4560	Serge	785	list_for_each_entry(vma, vmas, exec_list) {
		786	if (drm_mm_node_allocated(&vma->node))
3263	Serge	787	continue;
		788
4560	Serge	789	ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
3263	Serge	790	if (ret)
		791	goto err;
		792	}
		793
4560	Serge	794	err:
3263	Serge	795	if (ret != -ENOSPC \|\| retry++)
		796	return ret;
		797
4560	Serge	798	/* Decrement pin count for bound objects */
		799	list_for_each_entry(vma, vmas, exec_list)
		800	i915_gem_execbuffer_unreserve_vma(vma);
		801
5060	serge	802	ret = i915_gem_evict_vm(vm, true);
3263	Serge	803	if (ret)
		804	return ret;
		805	} while (1);
		806	}
		807
		808	static int
		809	i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3480	Serge	810	struct drm_i915_gem_execbuffer2 *args,
3263	Serge	811	struct drm_file *file,
5060	serge	812	struct intel_engine_cs *ring,
4560	Serge	813	struct eb_vmas *eb,
6084	serge	814	struct drm_i915_gem_exec_object2 *exec,
		815	struct intel_context *ctx)
3263	Serge	816	{
		817	struct drm_i915_gem_relocation_entry *reloc;
4560	Serge	818	struct i915_address_space *vm;
		819	struct i915_vma *vma;
3480	Serge	820	bool need_relocs;
3263	Serge	821	int *reloc_offset;
		822	int i, total, ret;
4560	Serge	823	unsigned count = args->buffer_count;
3263	Serge	824
4560	Serge	825	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
		826
3263	Serge	827	/* We may process another execbuffer during the unlock... */
4560	Serge	828	while (!list_empty(&eb->vmas)) {
		829	vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
		830	list_del_init(&vma->exec_list);
		831	i915_gem_execbuffer_unreserve_vma(vma);
		832	drm_gem_object_unreference(&vma->obj->base);
3263	Serge	833	}
		834
		835	mutex_unlock(&dev->struct_mutex);
		836
		837	total = 0;
		838	for (i = 0; i < count; i++)
		839	total += exec[i].relocation_count;
		840
6084	serge	841	reloc_offset = __builtin_malloc(count * sizeof(*reloc_offset));
		842	reloc = __builtin_malloc(total * sizeof(*reloc));
3263	Serge	843	if (reloc == NULL \|\| reloc_offset == NULL) {
3266	Serge	844	kfree(reloc);
		845	kfree(reloc_offset);
3263	Serge	846	mutex_lock(&dev->struct_mutex);
		847	return -ENOMEM;
		848	}
		849
		850	total = 0;
		851	for (i = 0; i < count; i++) {
		852	struct drm_i915_gem_relocation_entry __user *user_relocs;
		853	u64 invalid_offset = (u64)-1;
		854	int j;
		855
4539	Serge	856	user_relocs = to_user_ptr(exec[i].relocs_ptr);
3263	Serge	857
		858	if (copy_from_user(reloc+total, user_relocs,
		859	exec[i].relocation_count * sizeof(*reloc))) {
		860	ret = -EFAULT;
		861	mutex_lock(&dev->struct_mutex);
		862	goto err;
		863	}
		864
		865	/* As we do not update the known relocation offsets after
		866	* relocating (due to the complexities in lock handling),
		867	* we need to mark them as invalid now so that we force the
		868	* relocation processing next time. Just in case the target
		869	* object is evicted and then rebound into its old
		870	* presumed_offset before the next execbuffer - if that
		871	* happened we would make the mistake of assuming that the
		872	* relocations were valid.
		873	*/
		874	for (j = 0; j < exec[i].relocation_count; j++) {
5060	serge	875	if (__copy_to_user(&user_relocs[j].presumed_offset,
6084	serge	876	&invalid_offset,
		877	sizeof(invalid_offset))) {
3263	Serge	878	ret = -EFAULT;
		879	mutex_lock(&dev->struct_mutex);
		880	goto err;
		881	}
		882	}
		883
		884	reloc_offset[i] = total;
		885	total += exec[i].relocation_count;
		886	}
		887
		888	ret = i915_mutex_lock_interruptible(dev);
		889	if (ret) {
		890	mutex_lock(&dev->struct_mutex);
		891	goto err;
		892	}
		893
		894	/* reacquire the objects */
		895	eb_reset(eb);
4560	Serge	896	ret = eb_lookup_vmas(eb, exec, args, vm, file);
3480	Serge	897	if (ret)
6084	serge	898	goto err;
3263	Serge	899
3480	Serge	900	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
6084	serge	901	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
3263	Serge	902	if (ret)
		903	goto err;
		904
4560	Serge	905	list_for_each_entry(vma, &eb->vmas, exec_list) {
		906	int offset = vma->exec_entry - exec;
		907	ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
		908	reloc + reloc_offset[offset]);
3263	Serge	909	if (ret)
		910	goto err;
		911	}
		912
		913	/* Leave the user relocations as are, this is the painfully slow path,
		914	* and we want to avoid the complication of dropping the lock whilst
		915	* having buffers reserved in the aperture and so causing spurious
		916	* ENOSPC for random operations.
		917	*/
		918
		919	err:
3266	Serge	920	kfree(reloc);
		921	kfree(reloc_offset);
3263	Serge	922	return ret;
		923	}
		924
		925	static int
6084	serge	926	i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
4560	Serge	927	struct list_head *vmas)
3263	Serge	928	{
6084	serge	929	const unsigned other_rings = ~intel_ring_flag(req->ring);
4560	Serge	930	struct i915_vma *vma;
3263	Serge	931	uint32_t flush_domains = 0;
4104	Serge	932	bool flush_chipset = false;
3263	Serge	933	int ret;
		934
4560	Serge	935	list_for_each_entry(vma, vmas, exec_list) {
		936	struct drm_i915_gem_object *obj = vma->obj;
3263	Serge	937
6084	serge	938	if (obj->active & other_rings) {
		939	ret = i915_gem_object_sync(obj, req->ring, &req);
		940	if (ret)
		941	return ret;
		942	}
		943
3263	Serge	944	if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
4104	Serge	945	flush_chipset \|= i915_gem_clflush_object(obj, false);
3263	Serge	946
		947	flush_domains \|= obj->base.write_domain;
		948	}
		949
4104	Serge	950	if (flush_chipset)
6084	serge	951	i915_gem_chipset_flush(req->ring->dev);
3263	Serge	952
		953	if (flush_domains & I915_GEM_DOMAIN_GTT)
		954	wmb();
		955
		956	/* Unconditionally invalidate gpu caches and ensure that we do flush
		957	* any residual writes from the previous batch.
		958	*/
6084	serge	959	return intel_ring_invalidate_all_caches(req);
3263	Serge	960	}
		961
		962	static bool
		963	i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
		964	{
3480	Serge	965	if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
		966	return false;
		967
6084	serge	968	/* Kernel clipping was a DRI1 misfeature */
		969	if (exec->num_cliprects \|\| exec->cliprects_ptr)
		970	return false;
		971
		972	if (exec->DR4 == 0xffffffff) {
		973	DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
		974	exec->DR4 = 0;
		975	}
		976	if (exec->DR1 \|\| exec->DR4)
		977	return false;
		978
		979	if ((exec->batch_start_offset \| exec->batch_len) & 0x7)
		980	return false;
		981
		982	return true;
3263	Serge	983	}
		984
		985	static int
5354	serge	986	validate_exec_list(struct drm_device *dev,
		987	struct drm_i915_gem_exec_object2 *exec,
3263	Serge	988	int count)
		989	{
4560	Serge	990	unsigned relocs_total = 0;
		991	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
5354	serge	992	unsigned invalid_flags;
		993	int i;
3263	Serge	994
5354	serge	995	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
		996	if (USES_FULL_PPGTT(dev))
		997	invalid_flags \|= EXEC_OBJECT_NEEDS_GTT;
		998
3263	Serge	999	for (i = 0; i < count; i++) {
3746	Serge	1000	char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
3263	Serge	1001	int length; /* limited by fault_in_pages_readable() */
		1002
5354	serge	1003	if (exec[i].flags & invalid_flags)
3263	Serge	1004	return -EINVAL;
		1005
6937	serge	1006	/* Offset can be used as input (EXEC_OBJECT_PINNED), reject
		1007	* any non-page-aligned or non-canonical addresses.
		1008	*/
		1009	if (exec[i].flags & EXEC_OBJECT_PINNED) {
		1010	if (exec[i].offset !=
		1011	gen8_canonical_addr(exec[i].offset & PAGE_MASK))
		1012	return -EINVAL;
		1013
		1014	/* From drm_mm perspective address space is continuous,
		1015	* so from this point we're always using non-canonical
		1016	* form internally.
		1017	*/
		1018	exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
		1019	}
		1020
6084	serge	1021	if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
		1022	return -EINVAL;
		1023
3480	Serge	1024	/* First check for malicious input causing overflow in
		1025	* the worst case where we need to allocate the entire
		1026	* relocation tree as a single array.
		1027	*/
		1028	if (exec[i].relocation_count > relocs_max - relocs_total)
		1029	return -EINVAL;
		1030	relocs_total += exec[i].relocation_count;
		1031
3263	Serge	1032	length = exec[i].relocation_count *
		1033	sizeof(struct drm_i915_gem_relocation_entry);
3746	Serge	1034	/*
		1035	* We must check that the entire relocation array is safe
		1036	* to read, but since we may need to update the presumed
		1037	* offsets during execution, check for full write access.
		1038	*/
4560	Serge	1039	}
3263	Serge	1040
4560	Serge	1041	return 0;
		1042	}
		1043
5060	serge	1044	static struct intel_context *
4560	Serge	1045	i915_gem_validate_context(struct drm_device dev, struct drm_file file,
5060	serge	1046	struct intel_engine_cs *ring, const u32 ctx_id)
4560	Serge	1047	{
5060	serge	1048	struct intel_context *ctx = NULL;
4560	Serge	1049	struct i915_ctx_hang_stats *hs;
		1050
5060	serge	1051	if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
		1052	return ERR_PTR(-EINVAL);
4560	Serge	1053
5060	serge	1054	ctx = i915_gem_context_get(file->driver_priv, ctx_id);
		1055	if (IS_ERR(ctx))
		1056	return ctx;
		1057
		1058	hs = &ctx->hang_stats;
4560	Serge	1059	if (hs->banned) {
		1060	DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
5060	serge	1061	return ERR_PTR(-EIO);
3263	Serge	1062	}
		1063
5354	serge	1064	if (i915.enable_execlists && !ctx->engine[ring->id].state) {
6084	serge	1065	int ret = intel_lr_context_deferred_alloc(ctx, ring);
5354	serge	1066	if (ret) {
		1067	DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
		1068	return ERR_PTR(ret);
		1069	}
		1070	}
		1071
5060	serge	1072	return ctx;
3263	Serge	1073	}
		1074
5354	serge	1075	void
4560	Serge	1076	i915_gem_execbuffer_move_to_active(struct list_head *vmas,
6084	serge	1077	struct drm_i915_gem_request *req)
3263	Serge	1078	{
6084	serge	1079	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
4560	Serge	1080	struct i915_vma *vma;
3263	Serge	1081
4560	Serge	1082	list_for_each_entry(vma, vmas, exec_list) {
5354	serge	1083	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
4560	Serge	1084	struct drm_i915_gem_object *obj = vma->obj;
3263	Serge	1085	u32 old_read = obj->base.read_domains;
		1086	u32 old_write = obj->base.write_domain;
		1087
6084	serge	1088	obj->dirty = 1; /* be paranoid */
3480	Serge	1089	obj->base.write_domain = obj->base.pending_write_domain;
		1090	if (obj->base.write_domain == 0)
		1091	obj->base.pending_read_domains \|= obj->base.read_domains;
3263	Serge	1092	obj->base.read_domains = obj->base.pending_read_domains;
		1093
6084	serge	1094	i915_vma_move_to_active(vma, req);
3263	Serge	1095	if (obj->base.write_domain) {
6084	serge	1096	i915_gem_request_assign(&obj->last_write_req, req);
5060	serge	1097
6084	serge	1098	intel_fb_obj_invalidate(obj, ORIGIN_CS);
5060	serge	1099
		1100	/* update for the implicit flush after a batch */
		1101	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
3263	Serge	1102	}
5354	serge	1103	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
6084	serge	1104	i915_gem_request_assign(&obj->last_fenced_req, req);
5354	serge	1105	if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
		1106	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		1107	list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
		1108	&dev_priv->mm.fence_list);
		1109	}
		1110	}
3263	Serge	1111
		1112	trace_i915_gem_object_change_domain(obj, old_read, old_write);
		1113	}
		1114	}
		1115
5354	serge	1116	void
6084	serge	1117	i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
3263	Serge	1118	{
		1119	/* Unconditionally force add_request to emit a full flush. */
6084	serge	1120	params->ring->gpu_caches_dirty = true;
3263	Serge	1121
		1122	/* Add a breadcrumb for the completion of the batch buffer */
6084	serge	1123	__i915_add_request(params->request, params->batch_obj, true);
3263	Serge	1124	}
		1125
		1126	static int
		1127	i915_reset_gen7_sol_offsets(struct drm_device *dev,
6084	serge	1128	struct drm_i915_gem_request *req)
3263	Serge	1129	{
6084	serge	1130	struct intel_engine_cs *ring = req->ring;
5060	serge	1131	struct drm_i915_private *dev_priv = dev->dev_private;
3263	Serge	1132	int ret, i;
		1133
5060	serge	1134	if (!IS_GEN7(dev) \|\| ring != &dev_priv->ring[RCS]) {
		1135	DRM_DEBUG("sol reset is gen7/rcs only\n");
		1136	return -EINVAL;
		1137	}
3263	Serge	1138
6084	serge	1139	ret = intel_ring_begin(req, 4 * 3);
3263	Serge	1140	if (ret)
		1141	return ret;
		1142
		1143	for (i = 0; i < 4; i++) {
		1144	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
6937	serge	1145	intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
3263	Serge	1146	intel_ring_emit(ring, 0);
		1147	}
		1148
		1149	intel_ring_advance(ring);
		1150
		1151	return 0;
		1152	}
		1153
6084	serge	1154	static struct drm_i915_gem_object*
		1155	i915_gem_execbuffer_parse(struct intel_engine_cs *ring,
		1156	struct drm_i915_gem_exec_object2 *shadow_exec_entry,
		1157	struct eb_vmas *eb,
		1158	struct drm_i915_gem_object *batch_obj,
		1159	u32 batch_start_offset,
		1160	u32 batch_len,
		1161	bool is_master)
5354	serge	1162	{
6084	serge	1163	struct drm_i915_gem_object *shadow_batch_obj;
		1164	struct i915_vma *vma;
5354	serge	1165	int ret;
		1166
6084	serge	1167	shadow_batch_obj = i915_gem_batch_pool_get(&ring->batch_pool,
		1168	PAGE_ALIGN(batch_len));
		1169	if (IS_ERR(shadow_batch_obj))
		1170	return shadow_batch_obj;
5354	serge	1171
6084	serge	1172	ret = i915_parse_cmds(ring,
		1173	batch_obj,
		1174	shadow_batch_obj,
		1175	batch_start_offset,
		1176	batch_len,
		1177	is_master);
		1178	if (ret)
		1179	goto err;
5354	serge	1180
6084	serge	1181	ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
		1182	if (ret)
		1183	goto err;
5354	serge	1184
6084	serge	1185	i915_gem_object_unpin_pages(shadow_batch_obj);
5354	serge	1186
6084	serge	1187	memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
		1188
		1189	vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
		1190	vma->exec_entry = shadow_exec_entry;
		1191	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
		1192	drm_gem_object_reference(&shadow_batch_obj->base);
		1193	list_add_tail(&vma->exec_list, &eb->vmas);
		1194
		1195	shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
		1196
		1197	return shadow_batch_obj;
		1198
		1199	err:
		1200	i915_gem_object_unpin_pages(shadow_batch_obj);
		1201	if (ret == -EACCES) /* unhandled chained batch */
		1202	return batch_obj;
		1203	else
		1204	return ERR_PTR(ret);
5354	serge	1205	}
		1206
		1207	int
6084	serge	1208	i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
		1209	struct drm_i915_gem_execbuffer2 *args,
		1210	struct list_head *vmas)
5060	serge	1211	{
6084	serge	1212	struct drm_device *dev = params->dev;
		1213	struct intel_engine_cs *ring = params->ring;
5060	serge	1214	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1215	u64 exec_start, exec_len;
5060	serge	1216	int instp_mode;
		1217	u32 instp_mask;
6084	serge	1218	int ret;
5060	serge	1219
6084	serge	1220	ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
5060	serge	1221	if (ret)
6084	serge	1222	return ret;
5060	serge	1223
6084	serge	1224	ret = i915_switch_context(params->request);
5060	serge	1225	if (ret)
6084	serge	1226	return ret;
5060	serge	1227
6084	serge	1228	WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<id),
		1229	"%s didn't clear reload\n", ring->name);
		1230
5060	serge	1231	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
		1232	instp_mask = I915_EXEC_CONSTANTS_MASK;
		1233	switch (instp_mode) {
		1234	case I915_EXEC_CONSTANTS_REL_GENERAL:
		1235	case I915_EXEC_CONSTANTS_ABSOLUTE:
		1236	case I915_EXEC_CONSTANTS_REL_SURFACE:
		1237	if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
		1238	DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
6084	serge	1239	return -EINVAL;
5060	serge	1240	}
		1241
		1242	if (instp_mode != dev_priv->relative_constants_mode) {
		1243	if (INTEL_INFO(dev)->gen < 4) {
		1244	DRM_DEBUG("no rel constants on pre-gen4\n");
6084	serge	1245	return -EINVAL;
5060	serge	1246	}
		1247
		1248	if (INTEL_INFO(dev)->gen > 5 &&
		1249	instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
		1250	DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
6084	serge	1251	return -EINVAL;
5060	serge	1252	}
		1253
		1254	/* The HW changed the meaning on this bit on gen6 */
		1255	if (INTEL_INFO(dev)->gen >= 6)
		1256	instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
		1257	}
		1258	break;
		1259	default:
		1260	DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
6084	serge	1261	return -EINVAL;
5060	serge	1262	}
		1263
		1264	if (ring == &dev_priv->ring[RCS] &&
6084	serge	1265	instp_mode != dev_priv->relative_constants_mode) {
		1266	ret = intel_ring_begin(params->request, 4);
5060	serge	1267	if (ret)
6084	serge	1268	return ret;
5060	serge	1269
		1270	intel_ring_emit(ring, MI_NOOP);
		1271	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
6937	serge	1272	intel_ring_emit_reg(ring, INSTPM);
5060	serge	1273	intel_ring_emit(ring, instp_mask << 16 \| instp_mode);
		1274	intel_ring_advance(ring);
		1275
		1276	dev_priv->relative_constants_mode = instp_mode;
		1277	}
		1278
		1279	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
6084	serge	1280	ret = i915_reset_gen7_sol_offsets(dev, params->request);
5060	serge	1281	if (ret)
6084	serge	1282	return ret;
5060	serge	1283	}
		1284
6084	serge	1285	exec_len = args->batch_len;
		1286	exec_start = params->batch_obj_vm_offset +
		1287	params->args_batch_start_offset;
5060	serge	1288
6084	serge	1289	ret = ring->dispatch_execbuffer(params->request,
		1290	exec_start, exec_len,
		1291	params->dispatch_flags);
		1292	if (ret)
		1293	return ret;
5060	serge	1294
6084	serge	1295	trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
5060	serge	1296
6084	serge	1297	i915_gem_execbuffer_move_to_active(vmas, params->request);
		1298	i915_gem_execbuffer_retire_commands(params);
5060	serge	1299
6084	serge	1300	return 0;
5060	serge	1301	}
		1302
		1303	/**
		1304	* Find one BSD ring to dispatch the corresponding BSD command.
		1305	* The Ring ID is returned.
		1306	*/
		1307	static int gen8_dispatch_bsd_ring(struct drm_device *dev,
		1308	struct drm_file *file)
		1309	{
		1310	struct drm_i915_private *dev_priv = dev->dev_private;
		1311	struct drm_i915_file_private *file_priv = file->driver_priv;
		1312
		1313	/* Check whether the file_priv is using one ring */
		1314	if (file_priv->bsd_ring)
		1315	return file_priv->bsd_ring->id;
		1316	else {
		1317	/* If no, use the ping-pong mechanism to select one ring */
		1318	int ring_id;
		1319
		1320	mutex_lock(&dev->struct_mutex);
		1321	if (dev_priv->mm.bsd_ring_dispatch_index == 0) {
		1322	ring_id = VCS;
		1323	dev_priv->mm.bsd_ring_dispatch_index = 1;
		1324	} else {
		1325	ring_id = VCS2;
		1326	dev_priv->mm.bsd_ring_dispatch_index = 0;
		1327	}
		1328	file_priv->bsd_ring = &dev_priv->ring[ring_id];
		1329	mutex_unlock(&dev->struct_mutex);
		1330	return ring_id;
		1331	}
		1332	}
		1333
		1334	static struct drm_i915_gem_object *
		1335	eb_get_batch(struct eb_vmas *eb)
		1336	{
		1337	struct i915_vma vma = list_entry(eb->vmas.prev, typeof(vma), exec_list);
		1338
		1339	/*
		1340	* SNA is doing fancy tricks with compressing batch buffers, which leads
		1341	* to negative relocation deltas. Usually that works out ok since the
		1342	* relocate address is still positive, except when the batch is placed
		1343	* very low in the GTT. Ensure this doesn't happen.
		1344	*
		1345	* Note that actual hangs have only been observed on gen7, but for
		1346	* paranoia do it everywhere.
		1347	*/
6937	serge	1348	if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
5060	serge	1349	vma->exec_entry->flags \|= __EXEC_OBJECT_NEEDS_BIAS;
		1350
		1351	return vma->obj;
		1352	}
		1353
		1354	static int
3263	Serge	1355	i915_gem_do_execbuffer(struct drm_device dev, void data,
		1356	struct drm_file *file,
		1357	struct drm_i915_gem_execbuffer2 *args,
5060	serge	1358	struct drm_i915_gem_exec_object2 *exec)
3263	Serge	1359	{
5060	serge	1360	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1361	struct eb_vmas *eb;
3263	Serge	1362	struct drm_i915_gem_object *batch_obj;
6084	serge	1363	struct drm_i915_gem_exec_object2 shadow_exec_entry;
5060	serge	1364	struct intel_engine_cs *ring;
		1365	struct intel_context *ctx;
		1366	struct i915_address_space *vm;
6084	serge	1367	struct i915_execbuffer_params params_master; /* XXX: will be removed later */
		1368	struct i915_execbuffer_params *params = ¶ms_master;
4560	Serge	1369	const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
6084	serge	1370	u32 dispatch_flags;
5060	serge	1371	int ret;
3480	Serge	1372	bool need_relocs;
3263	Serge	1373
3480	Serge	1374	if (!i915_gem_check_execbuffer(args))
3263	Serge	1375	return -EINVAL;
		1376
5354	serge	1377	ret = validate_exec_list(dev, exec, args->buffer_count);
3263	Serge	1378	if (ret)
		1379	return ret;
		1380
6084	serge	1381	dispatch_flags = 0;
3263	Serge	1382	if (args->flags & I915_EXEC_SECURE) {
		1383
6084	serge	1384	dispatch_flags \|= I915_DISPATCH_SECURE;
3263	Serge	1385	}
		1386	if (args->flags & I915_EXEC_IS_PINNED)
6084	serge	1387	dispatch_flags \|= I915_DISPATCH_PINNED;
3263	Serge	1388
5060	serge	1389	if ((args->flags & I915_EXEC_RING_MASK) > LAST_USER_RING) {
3263	Serge	1390	DRM_DEBUG("execbuf with unknown ring: %d\n",
		1391	(int)(args->flags & I915_EXEC_RING_MASK));
		1392	return -EINVAL;
		1393	}
5060	serge	1394
6084	serge	1395	if (((args->flags & I915_EXEC_RING_MASK) != I915_EXEC_BSD) &&
		1396	((args->flags & I915_EXEC_BSD_MASK) != 0)) {
		1397	DRM_DEBUG("execbuf with non bsd ring but with invalid "
		1398	"bsd dispatch flags: %d\n", (int)(args->flags));
		1399	return -EINVAL;
		1400	}
		1401
5060	serge	1402	if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
		1403	ring = &dev_priv->ring[RCS];
		1404	else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) {
		1405	if (HAS_BSD2(dev)) {
		1406	int ring_id;
6084	serge	1407
		1408	switch (args->flags & I915_EXEC_BSD_MASK) {
		1409	case I915_EXEC_BSD_DEFAULT:
		1410	ring_id = gen8_dispatch_bsd_ring(dev, file);
		1411	ring = &dev_priv->ring[ring_id];
		1412	break;
		1413	case I915_EXEC_BSD_RING1:
		1414	ring = &dev_priv->ring[VCS];
		1415	break;
		1416	case I915_EXEC_BSD_RING2:
		1417	ring = &dev_priv->ring[VCS2];
		1418	break;
		1419	default:
		1420	DRM_DEBUG("execbuf with unknown bsd ring: %d\n",
		1421	(int)(args->flags & I915_EXEC_BSD_MASK));
		1422	return -EINVAL;
		1423	}
5060	serge	1424	} else
		1425	ring = &dev_priv->ring[VCS];
		1426	} else
		1427	ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
		1428
3263	Serge	1429	if (!intel_ring_initialized(ring)) {
		1430	DRM_DEBUG("execbuf with invalid ring: %d\n",
		1431	(int)(args->flags & I915_EXEC_RING_MASK));
		1432	return -EINVAL;
		1433	}
		1434
		1435	if (args->buffer_count < 1) {
		1436	DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
		1437	return -EINVAL;
		1438	}
		1439
6084	serge	1440	if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
		1441	if (!HAS_RESOURCE_STREAMER(dev)) {
		1442	DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
		1443	return -EINVAL;
		1444	}
		1445	if (ring->id != RCS) {
		1446	DRM_DEBUG("RS is not available on %s\n",
		1447	ring->name);
		1448	return -EINVAL;
		1449	}
		1450
		1451	dispatch_flags \|= I915_DISPATCH_RS;
		1452	}
		1453
4560	Serge	1454	intel_runtime_pm_get(dev_priv);
		1455
3263	Serge	1456	ret = i915_mutex_lock_interruptible(dev);
		1457	if (ret)
		1458	goto pre_mutex_err;
		1459
5060	serge	1460	ctx = i915_gem_validate_context(dev, file, ring, ctx_id);
		1461	if (IS_ERR(ctx)) {
4560	Serge	1462	mutex_unlock(&dev->struct_mutex);
5060	serge	1463	ret = PTR_ERR(ctx);
4560	Serge	1464	goto pre_mutex_err;
		1465	}
		1466
5060	serge	1467	i915_gem_context_reference(ctx);
		1468
5354	serge	1469	if (ctx->ppgtt)
		1470	vm = &ctx->ppgtt->base;
		1471	else
5060	serge	1472	vm = &dev_priv->gtt.base;
		1473
6084	serge	1474	memset(¶ms_master, 0x00, sizeof(params_master));
		1475
3480	Serge	1476	eb = eb_create(args);
3263	Serge	1477	if (eb == NULL) {
5060	serge	1478	i915_gem_context_unreference(ctx);
3263	Serge	1479	mutex_unlock(&dev->struct_mutex);
		1480	ret = -ENOMEM;
6084	serge	1481	goto pre_mutex_err;
3263	Serge	1482	}
		1483
		1484	/* Look up object handles */
4560	Serge	1485	ret = eb_lookup_vmas(eb, exec, args, vm, file);
3480	Serge	1486	if (ret)
6084	serge	1487	goto err;
3263	Serge	1488
		1489	/* take note of the batch buffer before we might reorder the lists */
5060	serge	1490	batch_obj = eb_get_batch(eb);
3263	Serge	1491
		1492	/* Move the objects en-masse into the GTT, evicting if necessary. */
3480	Serge	1493	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
6084	serge	1494	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, ctx, &need_relocs);
3263	Serge	1495	if (ret)
		1496	goto err;
		1497
		1498	/* The objects are in their final locations, apply the relocations. */
3480	Serge	1499	if (need_relocs)
4560	Serge	1500	ret = i915_gem_execbuffer_relocate(eb);
3263	Serge	1501	if (ret) {
		1502	if (ret == -EFAULT) {
3480	Serge	1503	ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
6084	serge	1504	eb, exec, ctx);
3263	Serge	1505	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1506	}
		1507	if (ret)
		1508	goto err;
6084	serge	1509	}
3263	Serge	1510
		1511	/* Set the pending read domains for the batch buffer to COMMAND */
		1512	if (batch_obj->base.pending_write_domain) {
		1513	DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
		1514	ret = -EINVAL;
		1515	goto err;
		1516	}
6084	serge	1517
		1518	params->args_batch_start_offset = args->batch_start_offset;
		1519
5060	serge	1520	#if 0
6084	serge	1521	if (i915_needs_cmd_parser(ring) && args->batch_len) {
		1522	struct drm_i915_gem_object *parsed_batch_obj;
		1523
		1524	parsed_batch_obj = i915_gem_execbuffer_parse(ring,
		1525	&shadow_exec_entry,
		1526	eb,
		1527	batch_obj,
		1528	args->batch_start_offset,
		1529	args->batch_len,
		1530	file->is_master);
		1531	if (IS_ERR(parsed_batch_obj)) {
		1532	ret = PTR_ERR(parsed_batch_obj);
5060	serge	1533	goto err;
6084	serge	1534	}
		1535
5060	serge	1536	/*
6084	serge	1537	* parsed_batch_obj == batch_obj means batch not fully parsed:
		1538	* Accept, but don't promote to secure.
5060	serge	1539	*/
6084	serge	1540
		1541	if (parsed_batch_obj != batch_obj) {
		1542	/*
		1543	* Batch parsed and accepted:
		1544	*
		1545	* Set the DISPATCH_SECURE bit to remove the NON_SECURE
		1546	* bit from MI_BATCH_BUFFER_START commands issued in
		1547	* the dispatch_execbuffer implementations. We
		1548	* specifically don't want that set on batches the
		1549	* command parser has accepted.
		1550	*/
		1551	dispatch_flags \|= I915_DISPATCH_SECURE;
		1552	params->args_batch_start_offset = 0;
		1553	batch_obj = parsed_batch_obj;
		1554	}
5060	serge	1555	}
6084	serge	1556	#endif
5367	serge	1557
6084	serge	1558	batch_obj->base.pending_read_domains \|= I915_GEM_DOMAIN_COMMAND;
		1559
3263	Serge	1560	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
		1561	* batch" bit. Hence we need to pin secure batches into the global gtt.
4560	Serge	1562	* hsw should have this fixed, but bdw mucks it up again. */
6084	serge	1563	if (dispatch_flags & I915_DISPATCH_SECURE) {
5354	serge	1564	/*
		1565	* So on first glance it looks freaky that we pin the batch here
		1566	* outside of the reservation loop. But:
		1567	* - The batch is already pinned into the relevant ppgtt, so we
		1568	* already have the backing storage fully allocated.
		1569	* - No other BO uses the global gtt (well contexts, but meh),
6084	serge	1570	* so we don't really have issues with multiple objects not
5354	serge	1571	* fitting due to fragmentation.
		1572	* So this is actually safe.
		1573	*/
		1574	ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
		1575	if (ret)
		1576	goto err;
3263	Serge	1577
6084	serge	1578	params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
5354	serge	1579	} else
6084	serge	1580	params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
3263	Serge	1581
6084	serge	1582	/* Allocate a request for this batch buffer nice and early. */
		1583	ret = i915_gem_request_alloc(ring, ctx, ¶ms->request);
		1584	if (ret)
		1585	goto err_batch_unpin;
3263	Serge	1586
6084	serge	1587	ret = i915_gem_request_add_to_client(params->request, file);
		1588	if (ret)
		1589	goto err_batch_unpin;
		1590
5354	serge	1591	/*
6084	serge	1592	* Save assorted stuff away to pass through to *_submission().
		1593	* NB: This data should be 'persistent' and not local as it will
		1594	* kept around beyond the duration of the IOCTL once the GPU
		1595	* scheduler arrives.
		1596	*/
		1597	params->dev = dev;
		1598	params->file = file;
		1599	params->ring = ring;
		1600	params->dispatch_flags = dispatch_flags;
		1601	params->batch_obj = batch_obj;
		1602	params->ctx = ctx;
		1603
		1604	ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
		1605
		1606	err_batch_unpin:
		1607	/*
5354	serge	1608	* FIXME: We crucially rely upon the active tracking for the (ppgtt)
		1609	* batch vma for correctness. For less ugly and less fragility this
		1610	* needs to be adjusted to also track the ggtt batch vma properly as
		1611	* active.
		1612	*/
6084	serge	1613	if (dispatch_flags & I915_DISPATCH_SECURE)
5354	serge	1614	i915_gem_object_ggtt_unpin(batch_obj);
6084	serge	1615
3263	Serge	1616	err:
5060	serge	1617	/* the request owns the ref now */
		1618	i915_gem_context_unreference(ctx);
3263	Serge	1619	eb_destroy(eb);
		1620
6084	serge	1621	/*
		1622	* If the request was created but not successfully submitted then it
		1623	* must be freed again. If it was submitted then it is being tracked
		1624	* on the active request list and no clean up is required here.
		1625	*/
		1626	if (ret && params->request)
		1627	i915_gem_request_cancel(params->request);
		1628
3263	Serge	1629	mutex_unlock(&dev->struct_mutex);
		1630
		1631	pre_mutex_err:
4560	Serge	1632	/* intel_gpu_busy should also get a ref, so it will free when the device
		1633	* is really idle. */
		1634	intel_runtime_pm_put(dev_priv);
3263	Serge	1635	return ret;
		1636	}
		1637
4246	Serge	1638	#if 0
		1639	/*
		1640	* Legacy execbuffer just creates an exec2 list from the original exec object
		1641	* list array and passes it to the real function.
		1642	*/
		1643	int
		1644	i915_gem_execbuffer(struct drm_device dev, void data,
		1645	struct drm_file *file)
		1646	{
		1647	struct drm_i915_gem_execbuffer *args = data;
		1648	struct drm_i915_gem_execbuffer2 exec2;
		1649	struct drm_i915_gem_exec_object *exec_list = NULL;
		1650	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
		1651	int ret, i;
3480	Serge	1652
4246	Serge	1653	if (args->buffer_count < 1) {
		1654	DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
		1655	return -EINVAL;
		1656	}
3480	Serge	1657
4246	Serge	1658	/* Copy in the exec list from userland */
		1659	exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
		1660	exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
		1661	if (exec_list == NULL \|\| exec2_list == NULL) {
		1662	DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
		1663	args->buffer_count);
		1664	drm_free_large(exec_list);
		1665	drm_free_large(exec2_list);
		1666	return -ENOMEM;
		1667	}
		1668	ret = copy_from_user(exec_list,
		1669	to_user_ptr(args->buffers_ptr),
		1670	sizeof(exec_list) args->buffer_count);
		1671	if (ret != 0) {
		1672	DRM_DEBUG("copy %d exec entries failed %d\n",
		1673	args->buffer_count, ret);
		1674	drm_free_large(exec_list);
		1675	drm_free_large(exec2_list);
		1676	return -EFAULT;
		1677	}
		1678
		1679	for (i = 0; i < args->buffer_count; i++) {
		1680	exec2_list[i].handle = exec_list[i].handle;
		1681	exec2_list[i].relocation_count = exec_list[i].relocation_count;
		1682	exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
		1683	exec2_list[i].alignment = exec_list[i].alignment;
		1684	exec2_list[i].offset = exec_list[i].offset;
		1685	if (INTEL_INFO(dev)->gen < 4)
		1686	exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
		1687	else
		1688	exec2_list[i].flags = 0;
		1689	}
		1690
		1691	exec2.buffers_ptr = args->buffers_ptr;
		1692	exec2.buffer_count = args->buffer_count;
		1693	exec2.batch_start_offset = args->batch_start_offset;
		1694	exec2.batch_len = args->batch_len;
		1695	exec2.DR1 = args->DR1;
		1696	exec2.DR4 = args->DR4;
		1697	exec2.num_cliprects = args->num_cliprects;
		1698	exec2.cliprects_ptr = args->cliprects_ptr;
		1699	exec2.flags = I915_EXEC_RENDER;
		1700	i915_execbuffer2_set_context_id(exec2, 0);
		1701
5060	serge	1702	ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
4246	Serge	1703	if (!ret) {
5060	serge	1704	struct drm_i915_gem_exec_object __user *user_exec_list =
		1705	to_user_ptr(args->buffers_ptr);
		1706
4246	Serge	1707	/* Copy the new buffer offsets back to the user's exec list. */
5060	serge	1708	for (i = 0; i < args->buffer_count; i++) {
6937	serge	1709	exec2_list[i].offset =
		1710	gen8_canonical_addr(exec2_list[i].offset);
5060	serge	1711	ret = __copy_to_user(&user_exec_list[i].offset,
		1712	&exec2_list[i].offset,
		1713	sizeof(user_exec_list[i].offset));
6084	serge	1714	if (ret) {
		1715	ret = -EFAULT;
		1716	DRM_DEBUG("failed to copy %d exec entries "
		1717	"back to user (%d)\n",
		1718	args->buffer_count, ret);
5060	serge	1719	break;
		1720	}
4246	Serge	1721	}
		1722	}
		1723
		1724	drm_free_large(exec_list);
		1725	drm_free_large(exec2_list);
		1726	return ret;
		1727	}
		1728	#endif
		1729
3263	Serge	1730	int
		1731	i915_gem_execbuffer2(struct drm_device dev, void data,
		1732	struct drm_file *file)
		1733	{
		1734	struct drm_i915_gem_execbuffer2 *args = data;
		1735	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
		1736	int ret;
		1737
		1738	if (args->buffer_count < 1 \|\|
		1739	args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
		1740	DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
		1741	return -EINVAL;
		1742	}
		1743
5060	serge	1744	if (args->rsvd2 != 0) {
		1745	DRM_DEBUG("dirty rvsd2 field\n");
		1746	return -EINVAL;
		1747	}
		1748
3480	Serge	1749	exec2_list = kmalloc(sizeof(exec2_list)args->buffer_count,
		1750	GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
3263	Serge	1751	if (exec2_list == NULL) {
		1752	DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
		1753	args->buffer_count);
		1754	return -ENOMEM;
		1755	}
		1756	ret = copy_from_user(exec2_list,
4539	Serge	1757	to_user_ptr(args->buffers_ptr),
3263	Serge	1758	sizeof(exec2_list) args->buffer_count);
		1759	if (ret != 0) {
		1760	DRM_DEBUG("copy %d exec entries failed %d\n",
		1761	args->buffer_count, ret);
3266	Serge	1762	kfree(exec2_list);
		1763	FAIL();
3263	Serge	1764	return -EFAULT;
		1765	}
		1766
5060	serge	1767	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
3263	Serge	1768	if (!ret) {
		1769	/* Copy the new buffer offsets back to the user's exec list. */
5060	serge	1770	struct drm_i915_gem_exec_object2 __user *user_exec_list =
		1771	to_user_ptr(args->buffers_ptr);
		1772	int i;
		1773
		1774	for (i = 0; i < args->buffer_count; i++) {
6937	serge	1775	exec2_list[i].offset =
		1776	gen8_canonical_addr(exec2_list[i].offset);
5060	serge	1777	ret = __copy_to_user(&user_exec_list[i].offset,
		1778	&exec2_list[i].offset,
		1779	sizeof(user_exec_list[i].offset));
6084	serge	1780	if (ret) {
		1781	ret = -EFAULT;
		1782	DRM_DEBUG("failed to copy %d exec entries "
5060	serge	1783	"back to user\n",
		1784	args->buffer_count);
		1785	break;
		1786	}
3263	Serge	1787	}
		1788	}
		1789
3266	Serge	1790	kfree(exec2_list);
3263	Serge	1791	return ret;
		1792	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem_execbuffer.c – Rev 6937