WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
6084	serge	2	* Copyright © 2008-2015 Intel Corporation
2326	Serge	3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
6084	serge	32	#include "i915_vgpu.h"
2351	Serge	33	#include "i915_trace.h"
2326	Serge	34	#include "intel_drv.h"
3260	Serge	35	#include
2330	Serge	36	#include
2326	Serge	37	//#include
3746	Serge	38	#include
2326	Serge	39	#include
6084	serge	40	#define RQ_BUG_ON(expr)
2326	Serge	41
2344	Serge	42	extern int x86_clflush_size;
6131	serge	43	#define __copy_to_user_inatomic __copy_to_user
2332	Serge	44
3263	Serge	45	#define PROT_READ 0x1 /* page can be read */
		46	#define PROT_WRITE 0x2 /* page can be written */
		47	#define MAP_SHARED 0x01 /* Share changes */
		48
2344	Serge	49
5060	serge	50
3266	Serge	51	struct drm_i915_gem_object *get_fb_obj();
		52
3263	Serge	53	unsigned long vm_mmap(struct file *file, unsigned long addr,
		54	unsigned long len, unsigned long prot,
		55	unsigned long flag, unsigned long offset);
		56
2344	Serge	57
2332	Serge	58	#define MAX_ERRNO 4095
		59
		60	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
6131	serge	61	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
2332	Serge	62
		63	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
6084	serge	64	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
5060	serge	65	static void
6084	serge	66	i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
		67	static void
		68	i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
2326	Serge	69
4104	Serge	70	static bool cpu_cache_is_coherent(struct drm_device *dev,
		71	enum i915_cache_level level)
		72	{
		73	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		74	}
		75
		76	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		77	{
		78	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		79	return true;
		80
		81	return obj->pin_display;
		82	}
		83
2332	Serge	84	/* some bookkeeping */
		85	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		86	size_t size)
		87	{
4104	Serge	88	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	89	dev_priv->mm.object_count++;
		90	dev_priv->mm.object_memory += size;
4104	Serge	91	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	92	}
		93
		94	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		95	size_t size)
		96	{
4104	Serge	97	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	98	dev_priv->mm.object_count--;
		99	dev_priv->mm.object_memory -= size;
4104	Serge	100	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	101	}
		102
		103	static int
3480	Serge	104	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	105	{
		106	int ret;
		107
3480	Serge	108	#define EXIT_COND (!i915_reset_in_progress(error))
		109	if (EXIT_COND)
2332	Serge	110	return 0;
3255	Serge	111	#if 0
3031	serge	112	/*
		113	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		114	* userspace. If it takes that long something really bad is going on and
		115	* we should simply try to bail out and fail as gracefully as possible.
		116	*/
3480	Serge	117	ret = wait_event_interruptible_timeout(error->reset_queue,
		118	EXIT_COND,
		119	10*HZ);
3031	serge	120	if (ret == 0) {
		121	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		122	return -EIO;
		123	} else if (ret < 0) {
2332	Serge	124	return ret;
3031	serge	125	}
2332	Serge	126
3255	Serge	127	#endif
3480	Serge	128	#undef EXIT_COND
3255	Serge	129
2332	Serge	130	return 0;
		131	}
		132
		133	int i915_mutex_lock_interruptible(struct drm_device *dev)
		134	{
3480	Serge	135	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	136	int ret;
		137
3480	Serge	138	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	139	if (ret)
		140	return ret;
		141
3480	Serge	142	ret = mutex_lock_interruptible(&dev->struct_mutex);
		143	if (ret)
		144	return ret;
2332	Serge	145
		146	WARN_ON(i915_verify_lists(dev));
		147	return 0;
		148	}
		149
		150	int
		151	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		152	struct drm_file *file)
		153	{
		154	struct drm_i915_private *dev_priv = dev->dev_private;
		155	struct drm_i915_gem_get_aperture *args = data;
6084	serge	156	struct i915_gtt *ggtt = &dev_priv->gtt;
		157	struct i915_vma *vma;
2332	Serge	158	size_t pinned;
		159
		160	pinned = 0;
		161	mutex_lock(&dev->struct_mutex);
6084	serge	162	list_for_each_entry(vma, &ggtt->base.active_list, mm_list)
		163	if (vma->pin_count)
		164	pinned += vma->node.size;
		165	list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list)
		166	if (vma->pin_count)
		167	pinned += vma->node.size;
2332	Serge	168	mutex_unlock(&dev->struct_mutex);
		169
4104	Serge	170	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	171	args->aper_available_size = args->aper_size - pinned;
2332	Serge	172
		173	return 0;
		174	}
		175
6296	serge	176	static int
		177	i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
		178	{
		179	char *vaddr = obj->phys_handle->vaddr;
		180	struct sg_table *st;
		181	struct scatterlist *sg;
		182	int i;
		183
		184	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
		185	return -EINVAL;
		186
		187
		188	st = kmalloc(sizeof(*st), GFP_KERNEL);
		189	if (st == NULL)
		190	return -ENOMEM;
		191
		192	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
		193	kfree(st);
		194	return -ENOMEM;
		195	}
		196
		197	sg = st->sgl;
		198	sg->offset = 0;
		199	sg->length = obj->base.size;
		200
		201	sg_dma_address(sg) = obj->phys_handle->busaddr;
		202	sg_dma_len(sg) = obj->base.size;
		203
		204	obj->pages = st;
		205	return 0;
		206	}
		207
		208	static void
		209	i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
		210	{
		211	int ret;
		212
		213	BUG_ON(obj->madv == __I915_MADV_PURGED);
		214
		215	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		216	if (ret) {
		217	/* In the event of a disaster, abandon all caches and
		218	* hope for the best.
		219	*/
		220	WARN_ON(ret != -EIO);
		221	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		222	}
		223
		224	if (obj->madv == I915_MADV_DONTNEED)
		225	obj->dirty = 0;
		226
		227	if (obj->dirty) {
		228	obj->dirty = 0;
		229	}
		230
		231	sg_free_table(obj->pages);
		232	kfree(obj->pages);
		233	}
		234
		235	static void
		236	i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
		237	{
		238	drm_pci_free(obj->base.dev, obj->phys_handle);
		239	}
		240
		241	static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
		242	.get_pages = i915_gem_object_get_pages_phys,
		243	.put_pages = i915_gem_object_put_pages_phys,
		244	.release = i915_gem_object_release_phys,
		245	};
		246
		247	static int
		248	drop_pages(struct drm_i915_gem_object *obj)
		249	{
		250	struct i915_vma vma, next;
		251	int ret;
		252
		253	drm_gem_object_reference(&obj->base);
		254	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
		255	if (i915_vma_unbind(vma))
		256	break;
		257
		258	ret = i915_gem_object_put_pages(obj);
		259	drm_gem_object_unreference(&obj->base);
		260
		261	return ret;
		262	}
		263
		264	int
		265	i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
		266	int align)
		267	{
		268	drm_dma_handle_t *phys;
		269	int ret;
		270
		271	if (obj->phys_handle) {
		272	if ((unsigned long)obj->phys_handle->vaddr & (align -1))
		273	return -EBUSY;
		274
		275	return 0;
		276	}
		277
		278	if (obj->madv != I915_MADV_WILLNEED)
		279	return -EFAULT;
		280
		281	if (obj->base.filp == NULL)
		282	return -EINVAL;
		283
		284	ret = drop_pages(obj);
		285	if (ret)
		286	return ret;
		287
		288	/* create a new object */
		289	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
		290	if (!phys)
		291	return -ENOMEM;
		292
		293	obj->phys_handle = phys;
		294	obj->ops = &i915_gem_phys_ops;
		295
		296	return i915_gem_object_get_pages(obj);
		297	}
3480	Serge	298	void i915_gem_object_alloc(struct drm_device dev)
		299	{
		300	struct drm_i915_private *dev_priv = dev->dev_private;
5367	serge	301	return kzalloc(sizeof(struct drm_i915_gem_object), 0);
3480	Serge	302	}
		303
		304	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		305	{
		306	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		307	kfree(obj);
		308	}
		309
3031	serge	310	static int
		311	i915_gem_create(struct drm_file *file,
2332	Serge	312	struct drm_device *dev,
		313	uint64_t size,
		314	uint32_t *handle_p)
		315	{
		316	struct drm_i915_gem_object *obj;
		317	int ret;
		318	u32 handle;
		319
		320	size = roundup(size, PAGE_SIZE);
2342	Serge	321	if (size == 0)
		322	return -EINVAL;
2332	Serge	323
		324	/* Allocate the new object */
		325	obj = i915_gem_alloc_object(dev, size);
		326	if (obj == NULL)
		327	return -ENOMEM;
		328
		329	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	330	/* drop reference from allocate - handle holds it now */
		331	drm_gem_object_unreference_unlocked(&obj->base);
		332	if (ret)
2332	Serge	333	return ret;
		334
		335	*handle_p = handle;
		336	return 0;
		337	}
		338
		339	int
		340	i915_gem_dumb_create(struct drm_file *file,
		341	struct drm_device *dev,
		342	struct drm_mode_create_dumb *args)
		343	{
		344	/* have to work out size/pitch and return them */
4560	Serge	345	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
2332	Serge	346	args->size = args->pitch * args->height;
		347	return i915_gem_create(file, dev,
		348	args->size, &args->handle);
		349	}
		350
2326	Serge	351	/**
2332	Serge	352	* Creates a new mm object and returns a handle to it.
		353	*/
		354	int
		355	i915_gem_create_ioctl(struct drm_device dev, void data,
		356	struct drm_file *file)
		357	{
		358	struct drm_i915_gem_create *args = data;
3031	serge	359
2332	Serge	360	return i915_gem_create(file, dev,
		361	args->size, &args->handle);
		362	}
		363
3031	serge	364	static inline int
		365	__copy_to_user_swizzled(char __user *cpu_vaddr,
		366	const char *gpu_vaddr, int gpu_offset,
6084	serge	367	int length)
2332	Serge	368	{
3031	serge	369	int ret, cpu_offset = 0;
2332	Serge	370
3031	serge	371	while (length > 0) {
		372	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		373	int this_length = min(cacheline_end - gpu_offset, length);
		374	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	375
3031	serge	376	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		377	gpu_vaddr + swizzled_gpu_offset,
		378	this_length);
		379	if (ret)
		380	return ret + length;
2332	Serge	381
3031	serge	382	cpu_offset += this_length;
		383	gpu_offset += this_length;
		384	length -= this_length;
		385	}
		386
		387	return 0;
2332	Serge	388	}
		389
3031	serge	390	static inline int
		391	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		392	const char __user *cpu_vaddr,
		393	int length)
2332	Serge	394	{
3031	serge	395	int ret, cpu_offset = 0;
2332	Serge	396
		397	while (length > 0) {
		398	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		399	int this_length = min(cacheline_end - gpu_offset, length);
		400	int swizzled_gpu_offset = gpu_offset ^ 64;
		401
3031	serge	402	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
6084	serge	403	cpu_vaddr + cpu_offset,
		404	this_length);
3031	serge	405	if (ret)
		406	return ret + length;
		407
2332	Serge	408	cpu_offset += this_length;
		409	gpu_offset += this_length;
		410	length -= this_length;
		411	}
		412
3031	serge	413	return 0;
2332	Serge	414	}
		415
6131	serge	416	/*
		417	* Pins the specified object's pages and synchronizes the object with
		418	* GPU accesses. Sets needs_clflush to non-zero if the caller should
		419	* flush the object from the CPU cache.
		420	*/
		421	int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
		422	int *needs_clflush)
		423	{
		424	int ret;
		425
		426	*needs_clflush = 0;
		427
		428	if (!obj->base.filp)
		429	return -EINVAL;
		430
		431	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		432	/* If we're not in the cpu read domain, set ourself into the gtt
		433	* read domain and manually flush cachelines (if required). This
		434	* optimizes for the case when the gpu will dirty the data
		435	* anyway again before the next pread happens. */
		436	*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
		437	obj->cache_level);
		438	ret = i915_gem_object_wait_rendering(obj, true);
		439	if (ret)
		440	return ret;
		441	}
		442
		443	ret = i915_gem_object_get_pages(obj);
		444	if (ret)
		445	return ret;
		446
		447	i915_gem_object_pin_pages(obj);
		448
		449	return ret;
		450	}
		451
3031	serge	452	/* Per-page copy function for the shmem pread fastpath.
		453	* Flushes invalid cachelines before reading the target if
		454	* needs_clflush is set. */
2332	Serge	455	static int
3031	serge	456	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		457	char __user *user_data,
		458	bool page_do_bit17_swizzling, bool needs_clflush)
		459	{
6084	serge	460	char *vaddr;
		461	int ret;
3031	serge	462
		463	if (unlikely(page_do_bit17_swizzling))
		464	return -EINVAL;
		465
6084	serge	466	vaddr = kmap_atomic(page);
3031	serge	467	if (needs_clflush)
		468	drm_clflush_virt_range(vaddr + shmem_page_offset,
		469	page_length);
6084	serge	470	ret = __copy_to_user_inatomic(user_data,
3031	serge	471	vaddr + shmem_page_offset,
6084	serge	472	page_length);
		473	kunmap_atomic(vaddr);
3031	serge	474
		475	return ret ? -EFAULT : 0;
		476	}
		477
		478	static void
		479	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		480	bool swizzled)
		481	{
		482	if (unlikely(swizzled)) {
		483	unsigned long start = (unsigned long) addr;
		484	unsigned long end = (unsigned long) addr + length;
		485
		486	/* For swizzling simply ensure that we always flush both
		487	* channels. Lame, but simple and it works. Swizzled
		488	* pwrite/pread is far from a hotpath - current userspace
		489	* doesn't use it at all. */
		490	start = round_down(start, 128);
		491	end = round_up(end, 128);
		492
		493	drm_clflush_virt_range((void *)start, end - start);
		494	} else {
		495	drm_clflush_virt_range(addr, length);
		496	}
		497
		498	}
		499
		500	/* Only difference to the fast-path function is that this can handle bit17
		501	* and uses non-atomic copy and kmap functions. */
		502	static int
		503	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		504	char __user *user_data,
		505	bool page_do_bit17_swizzling, bool needs_clflush)
		506	{
		507	char *vaddr;
		508	int ret;
		509
		510	vaddr = kmap(page);
		511	if (needs_clflush)
		512	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		513	page_length,
		514	page_do_bit17_swizzling);
		515
		516	if (page_do_bit17_swizzling)
		517	ret = __copy_to_user_swizzled(user_data,
		518	vaddr, shmem_page_offset,
		519	page_length);
		520	else
		521	ret = __copy_to_user(user_data,
		522	vaddr + shmem_page_offset,
		523	page_length);
		524	kunmap(page);
		525
		526	return ret ? - EFAULT : 0;
		527	}
		528
		529	static int
		530	i915_gem_shmem_pread(struct drm_device *dev,
6084	serge	531	struct drm_i915_gem_object *obj,
		532	struct drm_i915_gem_pread *args,
		533	struct drm_file *file)
2332	Serge	534	{
3031	serge	535	char __user *user_data;
2332	Serge	536	ssize_t remain;
		537	loff_t offset;
3031	serge	538	int shmem_page_offset, page_length, ret = 0;
		539	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		540	int prefaulted = 0;
		541	int needs_clflush = 0;
3746	Serge	542	struct sg_page_iter sg_iter;
2332	Serge	543
3746	Serge	544	user_data = to_user_ptr(args->data_ptr);
2332	Serge	545	remain = args->size;
		546
3031	serge	547	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		548
5060	serge	549	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
3031	serge	550	if (ret)
		551	return ret;
		552
2332	Serge	553	offset = args->offset;
		554
3746	Serge	555	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		556	offset >> PAGE_SHIFT) {
		557	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	558
3031	serge	559	if (remain <= 0)
		560	break;
		561
2332	Serge	562	/* Operation in this page
		563	*
3031	serge	564	* shmem_page_offset = offset within page in shmem file
2332	Serge	565	* page_length = bytes to copy for this page
		566	*/
3031	serge	567	shmem_page_offset = offset_in_page(offset);
2332	Serge	568	page_length = remain;
3031	serge	569	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		570	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	571
3031	serge	572	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		573	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	574
3031	serge	575	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		576	user_data, page_do_bit17_swizzling,
		577	needs_clflush);
		578	if (ret == 0)
		579	goto next_page;
2332	Serge	580
3031	serge	581	mutex_unlock(&dev->struct_mutex);
		582
		583	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		584	user_data, page_do_bit17_swizzling,
		585	needs_clflush);
		586
		587	mutex_lock(&dev->struct_mutex);
		588
2332	Serge	589	if (ret)
3031	serge	590	goto out;
2332	Serge	591
5060	serge	592	next_page:
2332	Serge	593	remain -= page_length;
		594	user_data += page_length;
		595	offset += page_length;
		596	}
		597
3031	serge	598	out:
		599	i915_gem_object_unpin_pages(obj);
		600
		601	return ret;
2332	Serge	602	}
		603
		604	/**
3031	serge	605	* Reads data from the object referenced by handle.
		606	*
		607	* On error, the contents of *data are undefined.
2332	Serge	608	*/
3031	serge	609	int
		610	i915_gem_pread_ioctl(struct drm_device dev, void data,
		611	struct drm_file *file)
		612	{
		613	struct drm_i915_gem_pread *args = data;
		614	struct drm_i915_gem_object *obj;
		615	int ret = 0;
		616
		617	if (args->size == 0)
		618	return 0;
		619
		620	ret = i915_mutex_lock_interruptible(dev);
		621	if (ret)
		622	return ret;
		623
		624	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		625	if (&obj->base == NULL) {
		626	ret = -ENOENT;
		627	goto unlock;
		628	}
		629
		630	/* Bounds check source. */
		631	if (args->offset > obj->base.size \|\|
		632	args->size > obj->base.size - args->offset) {
		633	ret = -EINVAL;
		634	goto out;
		635	}
		636
		637	/* prime objects have no backing filp to GEM pread/pwrite
		638	* pages from.
		639	*/
		640	if (!obj->base.filp) {
		641	ret = -EINVAL;
		642	goto out;
		643	}
		644
		645	trace_i915_gem_object_pread(obj, args->offset, args->size);
		646
		647	ret = i915_gem_shmem_pread(dev, obj, args, file);
		648
		649	out:
		650	drm_gem_object_unreference(&obj->base);
		651	unlock:
		652	mutex_unlock(&dev->struct_mutex);
		653	return ret;
		654	}
		655
		656	/* This is the fast write path which cannot handle
		657	* page faults in the source data
		658	*/
		659
		660
		661	/**
		662	* This is the fast pwrite path, where we copy the data directly from the
		663	* user into the GTT, uncached.
		664	*/
2332	Serge	665	static int
3031	serge	666	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		667	struct drm_i915_gem_object *obj,
		668	struct drm_i915_gem_pwrite *args,
		669	struct drm_file *file)
2332	Serge	670	{
5060	serge	671	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	672	ssize_t remain;
3031	serge	673	loff_t offset, page_base;
		674	char __user *user_data;
		675	int page_offset, page_length, ret;
2332	Serge	676
5060	serge	677	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3031	serge	678	if (ret)
		679	goto out;
		680
		681	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		682	if (ret)
		683	goto out_unpin;
		684
		685	ret = i915_gem_object_put_fence(obj);
		686	if (ret)
		687	goto out_unpin;
		688
4539	Serge	689	user_data = to_user_ptr(args->data_ptr);
2332	Serge	690	remain = args->size;
		691
4104	Serge	692	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	693
6084	serge	694	intel_fb_obj_invalidate(obj, ORIGIN_GTT);
		695
3031	serge	696	while (remain > 0) {
		697	/* Operation in this page
		698	*
		699	* page_base = page offset within aperture
		700	* page_offset = offset within page
		701	* page_length = bytes to copy for this page
		702	*/
		703	page_base = offset & PAGE_MASK;
		704	page_offset = offset_in_page(offset);
		705	page_length = remain;
		706	if ((page_offset + remain) > PAGE_SIZE)
		707	page_length = PAGE_SIZE - page_offset;
2332	Serge	708
6131	serge	709	MapPage(dev_priv->gtt.mappable,
		710	dev_priv->gtt.mappable_base+page_base, PG_WRITEC\|PG_SW);
3031	serge	711
6131	serge	712	memcpy((char*)dev_priv->gtt.mappable+page_offset, user_data, page_length);
3260	Serge	713
3031	serge	714	remain -= page_length;
		715	user_data += page_length;
		716	offset += page_length;
2332	Serge	717	}
		718
6084	serge	719	out_flush:
		720	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3031	serge	721	out_unpin:
5060	serge	722	i915_gem_object_ggtt_unpin(obj);
3031	serge	723	out:
6084	serge	724	return ret;
3031	serge	725	}
		726
		727	/* Per-page copy function for the shmem pwrite fastpath.
		728	* Flushes invalid cachelines before writing to the target if
		729	* needs_clflush_before is set and flushes out any written cachelines after
		730	* writing if needs_clflush is set. */
		731	static int
		732	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		733	char __user *user_data,
		734	bool page_do_bit17_swizzling,
		735	bool needs_clflush_before,
		736	bool needs_clflush_after)
		737	{
		738	char *vaddr;
5354	serge	739	int ret;
3031	serge	740
		741	if (unlikely(page_do_bit17_swizzling))
		742	return -EINVAL;
		743
5354	serge	744	vaddr = kmap_atomic(page);
3031	serge	745	if (needs_clflush_before)
		746	drm_clflush_virt_range(vaddr + shmem_page_offset,
		747	page_length);
3260	Serge	748	memcpy(vaddr + shmem_page_offset,
3031	serge	749	user_data,
		750	page_length);
		751	if (needs_clflush_after)
		752	drm_clflush_virt_range(vaddr + shmem_page_offset,
		753	page_length);
5354	serge	754	kunmap_atomic(vaddr);
3031	serge	755
		756	return ret ? -EFAULT : 0;
		757	}
		758
		759	/* Only difference to the fast-path function is that this can handle bit17
		760	* and uses non-atomic copy and kmap functions. */
		761	static int
		762	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		763	char __user *user_data,
		764	bool page_do_bit17_swizzling,
		765	bool needs_clflush_before,
		766	bool needs_clflush_after)
		767	{
		768	char *vaddr;
		769	int ret;
		770
		771	vaddr = kmap(page);
		772	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		773	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		774	page_length,
		775	page_do_bit17_swizzling);
		776	if (page_do_bit17_swizzling)
		777	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		778	user_data,
		779	page_length);
		780	else
		781	ret = __copy_from_user(vaddr + shmem_page_offset,
		782	user_data,
		783	page_length);
		784	if (needs_clflush_after)
		785	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		786	page_length,
		787	page_do_bit17_swizzling);
		788	kunmap(page);
		789
		790	return ret ? -EFAULT : 0;
		791	}
		792
		793	static int
		794	i915_gem_shmem_pwrite(struct drm_device *dev,
		795	struct drm_i915_gem_object *obj,
		796	struct drm_i915_gem_pwrite *args,
		797	struct drm_file *file)
		798	{
		799	ssize_t remain;
		800	loff_t offset;
		801	char __user *user_data;
		802	int shmem_page_offset, page_length, ret = 0;
		803	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		804	int hit_slowpath = 0;
		805	int needs_clflush_after = 0;
		806	int needs_clflush_before = 0;
3746	Serge	807	struct sg_page_iter sg_iter;
3031	serge	808
3746	Serge	809	user_data = to_user_ptr(args->data_ptr);
3031	serge	810	remain = args->size;
		811
		812	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		813
		814	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		815	/* If we're not in the cpu write domain, set ourself into the gtt
		816	* write domain and manually flush cachelines (if required). This
		817	* optimizes for the case when the gpu will use the data
		818	* right away and we therefore have to clflush anyway. */
4104	Serge	819	needs_clflush_after = cpu_write_needs_clflush(obj);
4560	Serge	820	ret = i915_gem_object_wait_rendering(obj, false);
6084	serge	821	if (ret)
		822	return ret;
		823	}
4104	Serge	824	/* Same trick applies to invalidate partially written cachelines read
		825	* before writing. */
		826	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		827	needs_clflush_before =
		828	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	829
		830	ret = i915_gem_object_get_pages(obj);
2332	Serge	831	if (ret)
3031	serge	832	return ret;
2332	Serge	833
6084	serge	834	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
		835
3031	serge	836	i915_gem_object_pin_pages(obj);
2332	Serge	837
		838	offset = args->offset;
3031	serge	839	obj->dirty = 1;
2332	Serge	840
3746	Serge	841	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		842	offset >> PAGE_SHIFT) {
		843	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	844	int partial_cacheline_write;
2332	Serge	845
3031	serge	846	if (remain <= 0)
		847	break;
		848
2332	Serge	849	/* Operation in this page
		850	*
		851	* shmem_page_offset = offset within page in shmem file
		852	* page_length = bytes to copy for this page
		853	*/
		854	shmem_page_offset = offset_in_page(offset);
		855
		856	page_length = remain;
		857	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		858	page_length = PAGE_SIZE - shmem_page_offset;
		859
3031	serge	860	/* If we don't overwrite a cacheline completely we need to be
		861	* careful to have up-to-date data by first clflushing. Don't
		862	* overcomplicate things and flush the entire patch. */
		863	partial_cacheline_write = needs_clflush_before &&
		864	((shmem_page_offset \| page_length)
3260	Serge	865	& (x86_clflush_size - 1));
2332	Serge	866
3031	serge	867	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		868	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	869
3031	serge	870	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		871	user_data, page_do_bit17_swizzling,
		872	partial_cacheline_write,
		873	needs_clflush_after);
		874	if (ret == 0)
		875	goto next_page;
		876
		877	hit_slowpath = 1;
		878	mutex_unlock(&dev->struct_mutex);
6296	serge	879	ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		880	user_data, page_do_bit17_swizzling,
		881	partial_cacheline_write,
		882	needs_clflush_after);
3031	serge	883
		884	mutex_lock(&dev->struct_mutex);
		885
		886	if (ret)
		887	goto out;
		888
5354	serge	889	next_page:
2332	Serge	890	remain -= page_length;
3031	serge	891	user_data += page_length;
2332	Serge	892	offset += page_length;
		893	}
		894
		895	out:
3031	serge	896	i915_gem_object_unpin_pages(obj);
		897
		898	if (hit_slowpath) {
3480	Serge	899	/*
		900	* Fixup: Flush cpu caches in case we didn't flush the dirty
		901	* cachelines in-line while writing and the object moved
		902	* out of the cpu write domain while we've dropped the lock.
		903	*/
		904	if (!needs_clflush_after &&
		905	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	906	if (i915_gem_clflush_object(obj, obj->pin_display))
6084	serge	907	needs_clflush_after = true;
3031	serge	908	}
2332	Serge	909	}
		910
3031	serge	911	if (needs_clflush_after)
3243	Serge	912	i915_gem_chipset_flush(dev);
6084	serge	913	else
		914	obj->cache_dirty = true;
3031	serge	915
6084	serge	916	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
2332	Serge	917	return ret;
		918	}
3031	serge	919
		920	/**
		921	* Writes data to the object referenced by handle.
		922	*
		923	* On error, the contents of the buffer that were to be modified are undefined.
		924	*/
		925	int
		926	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		927	struct drm_file *file)
		928	{
6084	serge	929	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	930	struct drm_i915_gem_pwrite *args = data;
		931	struct drm_i915_gem_object *obj;
		932	int ret;
		933
4104	Serge	934	if (args->size == 0)
		935	return 0;
		936
6084	serge	937	intel_runtime_pm_get(dev_priv);
3480	Serge	938
3031	serge	939	ret = i915_mutex_lock_interruptible(dev);
		940	if (ret)
6084	serge	941	goto put_rpm;
3031	serge	942
		943	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		944	if (&obj->base == NULL) {
		945	ret = -ENOENT;
		946	goto unlock;
		947	}
		948
		949	/* Bounds check destination. */
		950	if (args->offset > obj->base.size \|\|
		951	args->size > obj->base.size - args->offset) {
		952	ret = -EINVAL;
		953	goto out;
		954	}
		955
		956	/* prime objects have no backing filp to GEM pread/pwrite
		957	* pages from.
		958	*/
		959	if (!obj->base.filp) {
		960	ret = -EINVAL;
		961	goto out;
		962	}
		963
		964	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		965
		966	ret = -EFAULT;
		967	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		968	* it would end up going through the fenced access, and we'll get
		969	* different detiling behavior between reading and writing.
		970	* pread/pwrite currently are reading and writing from the CPU
		971	* perspective, requiring manual detiling by the client.
		972	*/
4104	Serge	973	if (obj->tiling_mode == I915_TILING_NONE &&
		974	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		975	cpu_write_needs_clflush(obj)) {
3031	serge	976	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		977	/* Note that the gtt paths might fail with non-page-backed user
		978	* pointers (e.g. gtt mappings when moving data between
		979	* textures). Fallback to the shmem path in that case. */
		980	}
		981
6296	serge	982	if (ret == -EFAULT \|\| ret == -ENOSPC) {
6084	serge	983	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
6296	serge	984	}
3031	serge	985
		986	out:
		987	drm_gem_object_unreference(&obj->base);
		988	unlock:
		989	mutex_unlock(&dev->struct_mutex);
6084	serge	990	put_rpm:
		991	intel_runtime_pm_put(dev_priv);
		992
3031	serge	993	return ret;
		994	}
		995
		996	int
3480	Serge	997	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	998	bool interruptible)
		999	{
3480	Serge	1000	if (i915_reset_in_progress(error)) {
3031	serge	1001	/* Non-interruptible callers can't handle -EAGAIN, hence return
		1002	* -EIO unconditionally for these. */
		1003	if (!interruptible)
		1004	return -EIO;
2332	Serge	1005
3480	Serge	1006	/* Recovery complete, but the reset failed ... */
		1007	if (i915_terminally_wedged(error))
3031	serge	1008	return -EIO;
2332	Serge	1009
6084	serge	1010	/*
		1011	* Check if GPU Reset is in progress - we need intel_ring_begin
		1012	* to work properly to reinit the hw state while the gpu is
		1013	* still marked as reset-in-progress. Handle this with a flag.
		1014	*/
		1015	if (!error->reload_in_reset)
		1016	return -EAGAIN;
3031	serge	1017	}
2332	Serge	1018
3031	serge	1019	return 0;
		1020	}
2332	Serge	1021
4560	Serge	1022	static void fake_irq(unsigned long data)
		1023	{
		1024	// wake_up_process((struct task_struct *)data);
		1025	}
		1026
		1027	static bool missed_irq(struct drm_i915_private *dev_priv,
5060	serge	1028	struct intel_engine_cs *ring)
4560	Serge	1029	{
		1030	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
		1031	}
		1032
6084	serge	1033	static unsigned long local_clock_us(unsigned *cpu)
4560	Serge	1034	{
6084	serge	1035	unsigned long t;
		1036
		1037	/* Cheaply and approximately convert from nanoseconds to microseconds.
		1038	* The result and subsequent calculations are also defined in the same
		1039	* approximate microseconds units. The principal source of timing
		1040	* error here is from the simple truncation.
		1041	*
		1042	* Note that local_clock() is only defined wrt to the current CPU;
		1043	* the comparisons are no longer valid if we switch CPUs. Instead of
		1044	* blocking preemption for the entire busywait, we can detect the CPU
		1045	* switch and use that as indicator of system load and a reason to
		1046	* stop busywaiting, see busywait_stop().
		1047	*/
		1048	t = GetClockNs() >> 10;
		1049
		1050	return t;
		1051	}
		1052
		1053	static bool busywait_stop(unsigned long timeout, unsigned cpu)
		1054	{
		1055	unsigned this_cpu = 0;
		1056
		1057	if (time_after(local_clock_us(&this_cpu), timeout))
4560	Serge	1058	return true;
		1059
6084	serge	1060	return this_cpu != cpu;
4560	Serge	1061	}
		1062
6084	serge	1063	static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
		1064	{
		1065	unsigned long timeout;
		1066	unsigned cpu;
		1067
		1068	/* When waiting for high frequency requests, e.g. during synchronous
		1069	* rendering split between the CPU and GPU, the finite amount of time
		1070	* required to set up the irq and wait upon it limits the response
		1071	* rate. By busywaiting on the request completion for a short while we
		1072	* can service the high frequency waits as quick as possible. However,
		1073	* if it is a slow request, we want to sleep as quickly as possible.
		1074	* The tradeoff between waiting and sleeping is roughly the time it
		1075	* takes to sleep on a request, on the order of a microsecond.
		1076	*/
		1077
		1078	if (req->ring->irq_refcount)
		1079	return -EBUSY;
		1080
		1081	/* Only spin if we know the GPU is processing this request */
		1082	if (!i915_gem_request_started(req, true))
		1083	return -EAGAIN;
		1084
		1085	timeout = local_clock_us(&cpu) + 5;
		1086	while (1 /!need_resched()/) {
		1087	if (i915_gem_request_completed(req, true))
		1088	return 0;
		1089
		1090	if (busywait_stop(timeout, cpu))
		1091	break;
		1092
		1093	cpu_relax_lowlatency();
		1094	}
		1095
		1096	if (i915_gem_request_completed(req, false))
		1097	return 0;
		1098
		1099	return -EAGAIN;
		1100	}
		1101
3031	serge	1102	/**
6084	serge	1103	* __i915_wait_request - wait until execution of request has finished
		1104	* @req: duh!
		1105	* @reset_counter: reset sequence associated with the given request
3031	serge	1106	* @interruptible: do an interruptible wait (normally yes)
		1107	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1108	*
3480	Serge	1109	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1110	* values have been read by the caller in an smp safe manner. Where read-side
		1111	* locks are involved, it is sufficient to read the reset_counter before
		1112	* unlocking the lock that protects the seqno. For lockless tricks, the
		1113	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1114	* inserted.
		1115	*
6084	serge	1116	* Returns 0 if the request was found within the alloted time. Else returns the
3031	serge	1117	* errno with remaining time filled in timeout argument.
		1118	*/
6084	serge	1119	int __i915_wait_request(struct drm_i915_gem_request *req,
3480	Serge	1120	unsigned reset_counter,
4560	Serge	1121	bool interruptible,
5060	serge	1122	s64 *timeout,
6084	serge	1123	struct intel_rps_client *rps)
3031	serge	1124	{
6084	serge	1125	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
5060	serge	1126	struct drm_device *dev = ring->dev;
		1127	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1128	const bool irq_test_in_progress =
		1129	ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
6084	serge	1130	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
6088	serge	1131	wait_queue_t wait;
5060	serge	1132	unsigned long timeout_expire;
		1133	s64 before, now;
3031	serge	1134	int ret;
2332	Serge	1135
5060	serge	1136	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
4104	Serge	1137
6084	serge	1138	if (list_empty(&req->list))
3031	serge	1139	return 0;
2332	Serge	1140
6084	serge	1141	if (i915_gem_request_completed(req, true))
		1142	return 0;
2332	Serge	1143
6084	serge	1144	timeout_expire = 0;
		1145	if (timeout) {
		1146	if (WARN_ON(*timeout < 0))
		1147	return -EINVAL;
		1148
		1149	if (*timeout == 0)
		1150	return -ETIME;
		1151
		1152	timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
3031	serge	1153	}
2332	Serge	1154
6084	serge	1155	if (INTEL_INFO(dev_priv)->gen >= 6)
		1156	gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
2332	Serge	1157
6084	serge	1158	/* Record current time in case interrupted by signal, or wedged */
		1159	trace_i915_gem_request_wait_begin(req);
		1160	before = ktime_get_raw_ns();
		1161
		1162	/* Optimistic spin for the next jiffie before touching IRQs */
		1163	ret = __i915_spin_request(req, state);
		1164	if (ret == 0)
		1165	goto out;
		1166
		1167	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
		1168	ret = -ENODEV;
		1169	goto out;
		1170	}
		1171
6088	serge	1172	INIT_LIST_HEAD(&wait.task_list);
		1173	wait.evnt = CreateEvent(NULL, MANUAL_DESTROY);
2332	Serge	1174
4560	Serge	1175	for (;;) {
6103	serge	1176	unsigned long flags;
4560	Serge	1177
3480	Serge	1178	/* We need to check whether any gpu reset happened in between
		1179	* the caller grabbing the seqno and now ... */
4560	Serge	1180	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
		1181	/* ... but upgrade the -EAGAIN to an -EIO if the gpu
		1182	* is truely gone. */
		1183	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
		1184	if (ret == 0)
		1185	ret = -EAGAIN;
		1186	break;
		1187	}
3480	Serge	1188
6084	serge	1189	if (i915_gem_request_completed(req, false)) {
4560	Serge	1190	ret = 0;
		1191	break;
		1192	}
2332	Serge	1193
6088	serge	1194	if (timeout && time_after_eq(jiffies, timeout_expire)) {
4560	Serge	1195	ret = -ETIME;
		1196	break;
		1197	}
2332	Serge	1198
4560	Serge	1199	spin_lock_irqsave(&ring->irq_queue.lock, flags);
6088	serge	1200	if (list_empty(&wait.task_list))
		1201	__add_wait_queue(&ring->irq_queue, &wait);
4560	Serge	1202	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1203
6088	serge	1204	WaitEventTimeout(wait.evnt, 1);
4560	Serge	1205
6088	serge	1206	if (!list_empty(&wait.task_list)) {
4560	Serge	1207	spin_lock_irqsave(&ring->irq_queue.lock, flags);
6088	serge	1208	list_del_init(&wait.task_list);
4560	Serge	1209	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1210	}
		1211
6088	serge	1212	};
4560	Serge	1213
		1214	if (!irq_test_in_progress)
6084	serge	1215	ring->irq_put(ring);
2332	Serge	1216
6088	serge	1217	DestroyEvent(wait.evnt);
		1218
6084	serge	1219	out:
		1220	now = ktime_get_raw_ns();
		1221	trace_i915_gem_request_wait_end(req);
		1222
		1223	if (timeout) {
		1224	s64 tres = *timeout - (now - before);
		1225
		1226	*timeout = tres < 0 ? 0 : tres;
		1227
		1228	/*
		1229	* Apparently ktime isn't accurate enough and occasionally has a
		1230	* bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
		1231	* things up to make the test happy. We allow up to 1 jiffy.
		1232	*
		1233	* This is a regrssion from the timespec->ktime conversion.
		1234	*/
		1235	if (ret == -ETIME && timeout < jiffies_to_usecs(1)1000)
		1236	*timeout = 0;
		1237	}
		1238
4560	Serge	1239	return ret;
3031	serge	1240	}
2332	Serge	1241
6084	serge	1242	int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
		1243	struct drm_file *file)
		1244	{
		1245	struct drm_i915_private *dev_private;
		1246	struct drm_i915_file_private *file_priv;
		1247
		1248	WARN_ON(!req \|\| !file \|\| req->file_priv);
		1249
		1250	if (!req \|\| !file)
		1251	return -EINVAL;
		1252
		1253	if (req->file_priv)
		1254	return -EINVAL;
		1255
		1256	dev_private = req->ring->dev->dev_private;
		1257	file_priv = file->driver_priv;
		1258
		1259	spin_lock(&file_priv->mm.lock);
		1260	req->file_priv = file_priv;
		1261	list_add_tail(&req->client_list, &file_priv->mm.request_list);
		1262	spin_unlock(&file_priv->mm.lock);
		1263
		1264	req->pid = 1;
		1265
		1266	return 0;
		1267	}
		1268
		1269	static inline void
		1270	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1271	{
		1272	struct drm_i915_file_private *file_priv = request->file_priv;
		1273
		1274	if (!file_priv)
		1275	return;
		1276
		1277	spin_lock(&file_priv->mm.lock);
		1278	list_del(&request->client_list);
		1279	request->file_priv = NULL;
		1280	spin_unlock(&file_priv->mm.lock);
		1281	}
		1282
		1283	static void i915_gem_request_retire(struct drm_i915_gem_request *request)
		1284	{
		1285	trace_i915_gem_request_retire(request);
		1286
		1287	/* We know the GPU must have read the request to have
		1288	* sent us the seqno + interrupt, so use the position
		1289	* of tail of the request to update the last known position
		1290	* of the GPU head.
		1291	*
		1292	* Note this requires that we are always called in request
		1293	* completion order.
		1294	*/
		1295	request->ringbuf->last_retired_head = request->postfix;
		1296
		1297	list_del_init(&request->list);
		1298	i915_gem_request_remove_from_client(request);
		1299
		1300	i915_gem_request_unreference(request);
		1301	}
		1302
		1303	static void
		1304	__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
		1305	{
		1306	struct intel_engine_cs *engine = req->ring;
		1307	struct drm_i915_gem_request *tmp;
		1308
		1309
		1310	if (list_empty(&req->list))
		1311	return;
		1312
		1313	do {
		1314	tmp = list_first_entry(&engine->request_list,
		1315	typeof(*tmp), list);
		1316
		1317	i915_gem_request_retire(tmp);
		1318	} while (tmp != req);
		1319
		1320	WARN_ON(i915_verify_lists(engine->dev));
		1321	}
		1322
3031	serge	1323	/**
6084	serge	1324	* Waits for a request to be signaled, and cleans up the
3031	serge	1325	* request and object lists appropriately for that event.
		1326	*/
		1327	int
6084	serge	1328	i915_wait_request(struct drm_i915_gem_request *req)
3031	serge	1329	{
6084	serge	1330	struct drm_device *dev;
		1331	struct drm_i915_private *dev_priv;
		1332	bool interruptible;
3031	serge	1333	int ret;
2332	Serge	1334
6084	serge	1335	BUG_ON(req == NULL);
		1336
		1337	dev = req->ring->dev;
		1338	dev_priv = dev->dev_private;
		1339	interruptible = dev_priv->mm.interruptible;
		1340
3031	serge	1341	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2332	Serge	1342
3480	Serge	1343	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1344	if (ret)
		1345	return ret;
2332	Serge	1346
6084	serge	1347	ret = __i915_wait_request(req,
		1348	atomic_read(&dev_priv->gpu_error.reset_counter),
		1349	interruptible, NULL, NULL);
3031	serge	1350	if (ret)
		1351	return ret;
2332	Serge	1352
6084	serge	1353	__i915_gem_request_retire__upto(req);
4104	Serge	1354	return 0;
		1355	}
		1356
3031	serge	1357	/**
		1358	* Ensures that all rendering to the object has completed and the object is
		1359	* safe to unbind from the GTT or access from the CPU.
		1360	*/
6084	serge	1361	int
3031	serge	1362	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1363	bool readonly)
		1364	{
6084	serge	1365	int ret, i;
2332	Serge	1366
6084	serge	1367	if (!obj->active)
3031	serge	1368	return 0;
2332	Serge	1369
6084	serge	1370	if (readonly) {
		1371	if (obj->last_write_req != NULL) {
		1372	ret = i915_wait_request(obj->last_write_req);
		1373	if (ret)
		1374	return ret;
2332	Serge	1375
6084	serge	1376	i = obj->last_write_req->ring->id;
		1377	if (obj->last_read_req[i] == obj->last_write_req)
		1378	i915_gem_object_retire__read(obj, i);
		1379	else
		1380	i915_gem_object_retire__write(obj);
		1381	}
		1382	} else {
		1383	for (i = 0; i < I915_NUM_RINGS; i++) {
		1384	if (obj->last_read_req[i] == NULL)
		1385	continue;
		1386
		1387	ret = i915_wait_request(obj->last_read_req[i]);
		1388	if (ret)
		1389	return ret;
		1390
		1391	i915_gem_object_retire__read(obj, i);
		1392	}
		1393	RQ_BUG_ON(obj->active);
		1394	}
		1395
		1396	return 0;
3031	serge	1397	}
2332	Serge	1398
6084	serge	1399	static void
		1400	i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
		1401	struct drm_i915_gem_request *req)
		1402	{
		1403	int ring = req->ring->id;
		1404
		1405	if (obj->last_read_req[ring] == req)
		1406	i915_gem_object_retire__read(obj, ring);
		1407	else if (obj->last_write_req == req)
		1408	i915_gem_object_retire__write(obj);
		1409
		1410	__i915_gem_request_retire__upto(req);
		1411	}
		1412
3260	Serge	1413	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1414	* as the object state may change during this call.
		1415	*/
		1416	static __must_check int
		1417	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
6084	serge	1418	struct intel_rps_client *rps,
3260	Serge	1419	bool readonly)
		1420	{
		1421	struct drm_device *dev = obj->base.dev;
		1422	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1423	struct drm_i915_gem_request *requests[I915_NUM_RINGS];
3480	Serge	1424	unsigned reset_counter;
6084	serge	1425	int ret, i, n = 0;
2332	Serge	1426
3260	Serge	1427	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1428	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1429
6084	serge	1430	if (!obj->active)
3260	Serge	1431	return 0;
2332	Serge	1432
3480	Serge	1433	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1434	if (ret)
		1435	return ret;
2332	Serge	1436
6084	serge	1437	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2332	Serge	1438
6084	serge	1439	if (readonly) {
		1440	struct drm_i915_gem_request *req;
		1441
		1442	req = obj->last_write_req;
		1443	if (req == NULL)
		1444	return 0;
		1445
		1446	requests[n++] = i915_gem_request_reference(req);
		1447	} else {
		1448	for (i = 0; i < I915_NUM_RINGS; i++) {
		1449	struct drm_i915_gem_request *req;
		1450
		1451	req = obj->last_read_req[i];
		1452	if (req == NULL)
		1453	continue;
		1454
		1455	requests[n++] = i915_gem_request_reference(req);
		1456	}
		1457	}
		1458
3260	Serge	1459	mutex_unlock(&dev->struct_mutex);
6084	serge	1460	for (i = 0; ret == 0 && i < n; i++)
		1461	ret = __i915_wait_request(requests[i], reset_counter, true,
		1462	NULL, rps);
3260	Serge	1463	mutex_lock(&dev->struct_mutex);
2332	Serge	1464
6084	serge	1465	for (i = 0; i < n; i++) {
		1466	if (ret == 0)
		1467	i915_gem_object_retire_request(obj, requests[i]);
		1468	i915_gem_request_unreference(requests[i]);
		1469	}
		1470
		1471	return ret;
3260	Serge	1472	}
2332	Serge	1473
6084	serge	1474	static struct intel_rps_client to_rps_client(struct drm_file file)
		1475	{
		1476	struct drm_i915_file_private *fpriv = file->driver_priv;
		1477	return &fpriv->rps;
		1478	}
		1479
3260	Serge	1480	/**
		1481	* Called when user space prepares to use an object with the CPU, either
		1482	* through the mmap ioctl's mapping or a GTT mapping.
		1483	*/
		1484	int
		1485	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1486	struct drm_file *file)
		1487	{
		1488	struct drm_i915_gem_set_domain *args = data;
		1489	struct drm_i915_gem_object *obj;
		1490	uint32_t read_domains = args->read_domains;
		1491	uint32_t write_domain = args->write_domain;
		1492	int ret;
2332	Serge	1493
3260	Serge	1494	/* Only handle setting domains to types used by the CPU. */
		1495	if (write_domain & I915_GEM_GPU_DOMAINS)
		1496	return -EINVAL;
2332	Serge	1497
3260	Serge	1498	if (read_domains & I915_GEM_GPU_DOMAINS)
		1499	return -EINVAL;
2332	Serge	1500
3260	Serge	1501	/* Having something in the write domain implies it's in the read
		1502	* domain, and only that read domain. Enforce that in the request.
		1503	*/
		1504	if (write_domain != 0 && read_domains != write_domain)
		1505	return -EINVAL;
2332	Serge	1506
3260	Serge	1507	ret = i915_mutex_lock_interruptible(dev);
		1508	if (ret)
		1509	return ret;
2332	Serge	1510
3260	Serge	1511	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1512	if (&obj->base == NULL) {
		1513	ret = -ENOENT;
		1514	goto unlock;
		1515	}
2332	Serge	1516
3260	Serge	1517	/* Try to flush the object off the GPU without holding the lock.
		1518	* We will repeat the flush holding the lock in the normal manner
		1519	* to catch cases where we are gazumped.
		1520	*/
5060	serge	1521	ret = i915_gem_object_wait_rendering__nonblocking(obj,
6084	serge	1522	to_rps_client(file),
5060	serge	1523	!write_domain);
3260	Serge	1524	if (ret)
		1525	goto unref;
2332	Serge	1526
6084	serge	1527	if (read_domains & I915_GEM_DOMAIN_GTT)
3260	Serge	1528	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
6084	serge	1529	else
3260	Serge	1530	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2332	Serge	1531
6084	serge	1532	if (write_domain != 0)
		1533	intel_fb_obj_invalidate(obj,
		1534	write_domain == I915_GEM_DOMAIN_GTT ?
		1535	ORIGIN_GTT : ORIGIN_CPU);
		1536
3260	Serge	1537	unref:
		1538	drm_gem_object_unreference(&obj->base);
		1539	unlock:
		1540	mutex_unlock(&dev->struct_mutex);
		1541	return ret;
		1542	}
2332	Serge	1543
4293	Serge	1544	/**
		1545	* Called when user space has done writes to this buffer
		1546	*/
		1547	int
		1548	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
		1549	struct drm_file *file)
		1550	{
		1551	struct drm_i915_gem_sw_finish *args = data;
		1552	struct drm_i915_gem_object *obj;
		1553	int ret = 0;
2332	Serge	1554
4293	Serge	1555	ret = i915_mutex_lock_interruptible(dev);
		1556	if (ret)
		1557	return ret;
2332	Serge	1558
4293	Serge	1559	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1560	if (&obj->base == NULL) {
		1561	ret = -ENOENT;
		1562	goto unlock;
		1563	}
2332	Serge	1564
4293	Serge	1565	/* Pinned buffers may be scanout, so flush the cache */
		1566	if (obj->pin_display)
6084	serge	1567	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	1568
4293	Serge	1569	drm_gem_object_unreference(&obj->base);
		1570	unlock:
		1571	mutex_unlock(&dev->struct_mutex);
		1572	return ret;
		1573	}
		1574
3260	Serge	1575	/**
		1576	* Maps the contents of an object, returning the address it is mapped
		1577	* into.
		1578	*
		1579	* While the mapping holds a reference on the contents of the object, it doesn't
		1580	* imply a ref on the object itself.
5354	serge	1581	*
		1582	* IMPORTANT:
		1583	*
		1584	* DRM driver writers who look a this function as an example for how to do GEM
		1585	* mmap support, please don't implement mmap support like here. The modern way
		1586	* to implement DRM mmap support is with an mmap offset ioctl (like
		1587	* i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
		1588	* That way debug tooling like valgrind will understand what's going on, hiding
		1589	* the mmap call in a driver private ioctl will break that. The i915 driver only
		1590	* does cpu mmaps this way because we didn't know better.
3260	Serge	1591	*/
		1592	int
		1593	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1594	struct drm_file *file)
		1595	{
		1596	struct drm_i915_gem_mmap *args = data;
		1597	struct drm_gem_object *obj;
4392	Serge	1598	unsigned long addr;
2332	Serge	1599
6084	serge	1600	// if (args->flags & ~(I915_MMAP_WC))
		1601	// return -EINVAL;
3260	Serge	1602	obj = drm_gem_object_lookup(dev, file, args->handle);
		1603	if (obj == NULL)
		1604	return -ENOENT;
4104	Serge	1605
3260	Serge	1606	/* prime objects have no backing filp to GEM mmap
		1607	* pages from.
		1608	*/
		1609	if (!obj->filp) {
		1610	drm_gem_object_unreference_unlocked(obj);
		1611	return -EINVAL;
		1612	}
2332	Serge	1613
6084	serge	1614	addr = vm_mmap(obj->filp, 0, args->size,
		1615	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1616	args->offset);
3260	Serge	1617	drm_gem_object_unreference_unlocked(obj);
6084	serge	1618	if (IS_ERR((void *)addr))
		1619	return addr;
2332	Serge	1620
3260	Serge	1621	args->addr_ptr = (uint64_t) addr;
2332	Serge	1622
6084	serge	1623	return 0;
3260	Serge	1624	}
2332	Serge	1625
		1626
		1627
		1628
		1629
		1630
		1631
		1632
3031	serge	1633
		1634
		1635
		1636
		1637
		1638	/**
		1639	* i915_gem_release_mmap - remove physical page mappings
		1640	* @obj: obj in question
		1641	*
		1642	* Preserve the reservation of the mmapping with the DRM core code, but
		1643	* relinquish ownership of the pages back to the system.
		1644	*
		1645	* It is vital that we remove the page mapping if we have mapped a tiled
		1646	* object through the GTT and then lose the fence register due to
		1647	* resource pressure. Similarly if the object has been moved out of the
		1648	* aperture, than pages mapped into userspace must be revoked. Removing the
		1649	* mapping will then trigger a page fault on the next user access, allowing
		1650	* fixup by i915_gem_fault().
		1651	*/
		1652	void
		1653	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1654	{
		1655	if (!obj->fault_mappable)
		1656	return;
		1657
4104	Serge	1658	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1659	obj->fault_mappable = false;
		1660	}
		1661
6084	serge	1662	void
		1663	i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
		1664	{
		1665	struct drm_i915_gem_object *obj;
		1666
		1667	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
		1668	i915_gem_release_mmap(obj);
		1669	}
		1670
3480	Serge	1671	uint32_t
2332	Serge	1672	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1673	{
		1674	uint32_t gtt_size;
		1675
		1676	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1677	tiling_mode == I915_TILING_NONE)
		1678	return size;
		1679
		1680	/* Previous chips need a power-of-two fence region when tiling */
		1681	if (INTEL_INFO(dev)->gen == 3)
		1682	gtt_size = 1024*1024;
		1683	else
		1684	gtt_size = 512*1024;
		1685
		1686	while (gtt_size < size)
		1687	gtt_size <<= 1;
		1688
		1689	return gtt_size;
		1690	}
		1691
		1692	/**
		1693	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1694	* @obj: object to check
		1695	*
		1696	* Return the required GTT alignment for an object, taking into account
		1697	* potential fence register mapping.
		1698	*/
3480	Serge	1699	uint32_t
		1700	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1701	int tiling_mode, bool fenced)
2332	Serge	1702	{
		1703	/*
		1704	* Minimum alignment is 4k (GTT page size), but might be greater
		1705	* if a fence register is needed for the object.
		1706	*/
3480	Serge	1707	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1708	tiling_mode == I915_TILING_NONE)
		1709	return 4096;
		1710
		1711	/*
		1712	* Previous chips need to be aligned to the size of the smallest
		1713	* fence register that can contain the object.
		1714	*/
		1715	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1716	}
		1717
		1718
		1719
3480	Serge	1720	int
		1721	i915_gem_mmap_gtt(struct drm_file *file,
		1722	struct drm_device *dev,
6084	serge	1723	uint32_t handle,
3480	Serge	1724	uint64_t *offset)
		1725	{
		1726	struct drm_i915_private *dev_priv = dev->dev_private;
		1727	struct drm_i915_gem_object *obj;
		1728	unsigned long pfn;
		1729	char mem, ptr;
		1730	int ret;
		1731
		1732	ret = i915_mutex_lock_interruptible(dev);
		1733	if (ret)
		1734	return ret;
		1735
		1736	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1737	if (&obj->base == NULL) {
		1738	ret = -ENOENT;
		1739	goto unlock;
		1740	}
		1741
		1742	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1743	DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
		1744	ret = -EFAULT;
3480	Serge	1745	goto out;
		1746	}
		1747	/* Now bind it into the GTT if needed */
5060	serge	1748	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3480	Serge	1749	if (ret)
		1750	goto out;
		1751
		1752	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1753	if (ret)
		1754	goto unpin;
		1755
		1756	ret = i915_gem_object_get_fence(obj);
		1757	if (ret)
		1758	goto unpin;
		1759
		1760	obj->fault_mappable = true;
		1761
4104	Serge	1762	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1763
		1764	/* Finally, remap it using the new GTT offset */
		1765
		1766	mem = UserAlloc(obj->base.size);
		1767	if(unlikely(mem == NULL))
		1768	{
		1769	ret = -ENOMEM;
		1770	goto unpin;
		1771	}
		1772
		1773	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1774	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1775
		1776	unpin:
5060	serge	1777	i915_gem_object_unpin_pages(obj);
3480	Serge	1778
		1779
5367	serge	1780	*offset = (uint32_t)mem;
3480	Serge	1781
		1782	out:
6088	serge	1783	drm_gem_object_unreference(&obj->base);
3480	Serge	1784	unlock:
6088	serge	1785	mutex_unlock(&dev->struct_mutex);
		1786	return ret;
3480	Serge	1787	}
		1788
		1789	/**
		1790	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1791	* @dev: DRM device
		1792	* @data: GTT mapping ioctl data
		1793	* @file: GEM object info
		1794	*
		1795	* Simply returns the fake offset to userspace so it can mmap it.
		1796	* The mmap call will end up in drm_gem_mmap(), which will set things
		1797	* up so we can get faults in the handler above.
		1798	*
		1799	* The fault handler will take care of binding the object into the GTT
		1800	* (since it may have been evicted to make room for something), allocating
		1801	* a fence register, and mapping the appropriate aperture address into
		1802	* userspace.
		1803	*/
		1804	int
		1805	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
6084	serge	1806	struct drm_file *file)
3480	Serge	1807	{
6084	serge	1808	struct drm_i915_gem_mmap_gtt *args = data;
3480	Serge	1809
6084	serge	1810	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
3480	Serge	1811	}
		1812
3031	serge	1813	/* Immediately discard the backing storage */
		1814	static void
		1815	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1816	{
		1817	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1818
3263	Serge	1819	if (obj->base.filp == NULL)
		1820	return;
2332	Serge	1821
3031	serge	1822	/* Our goal here is to return as much of the memory as
		1823	* is possible back to the system as we are called from OOM.
		1824	* To do this we must instruct the shmfs to drop all of its
		1825	* backing pages, now.
		1826	*/
5060	serge	1827	// shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
3031	serge	1828	obj->madv = __I915_MADV_PURGED;
		1829	}
2332	Serge	1830
5060	serge	1831	/* Try to discard unwanted pages */
		1832	static void
		1833	i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
3031	serge	1834	{
5060	serge	1835	struct address_space *mapping;
		1836
		1837	switch (obj->madv) {
		1838	case I915_MADV_DONTNEED:
		1839	i915_gem_object_truncate(obj);
		1840	case __I915_MADV_PURGED:
		1841	return;
		1842	}
		1843
		1844	if (obj->base.filp == NULL)
		1845	return;
		1846
3031	serge	1847	}
2332	Serge	1848
3031	serge	1849	static void
		1850	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1851	{
3746	Serge	1852	struct sg_page_iter sg_iter;
		1853	int ret;
2332	Serge	1854
3031	serge	1855	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1856
3031	serge	1857	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1858	if (ret) {
		1859	/* In the event of a disaster, abandon all caches and
		1860	* hope for the best.
		1861	*/
		1862	WARN_ON(ret != -EIO);
4104	Serge	1863	i915_gem_clflush_object(obj, true);
3031	serge	1864	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1865	}
2332	Serge	1866
6084	serge	1867	i915_gem_gtt_finish_object(obj);
6296	serge	1868
		1869	if (i915_gem_object_needs_bit17_swizzle(obj))
		1870	i915_gem_object_save_bit_17_swizzle(obj);
		1871
3031	serge	1872	if (obj->madv == I915_MADV_DONTNEED)
		1873	obj->dirty = 0;
2332	Serge	1874
3746	Serge	1875	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1876	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1877
6084	serge	1878	page_cache_release(page);
3243	Serge	1879	}
6084	serge	1880	obj->dirty = 0;
3243	Serge	1881
		1882	sg_free_table(obj->pages);
		1883	kfree(obj->pages);
3031	serge	1884	}
2332	Serge	1885
3480	Serge	1886	int
3031	serge	1887	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1888	{
		1889	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1890
3243	Serge	1891	if (obj->pages == NULL)
3031	serge	1892	return 0;
2332	Serge	1893
3031	serge	1894	if (obj->pages_pin_count)
		1895	return -EBUSY;
		1896
4104	Serge	1897	BUG_ON(i915_gem_obj_bound_any(obj));
		1898
3243	Serge	1899	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1900	* array, hence protect them from being reaped by removing them from gtt
		1901	* lists early. */
4104	Serge	1902	list_del(&obj->global_list);
3243	Serge	1903
3031	serge	1904	ops->put_pages(obj);
3243	Serge	1905	obj->pages = NULL;
3031	serge	1906
5060	serge	1907	i915_gem_object_invalidate(obj);
3031	serge	1908
		1909	return 0;
		1910	}
		1911
2332	Serge	1912	static int
3031	serge	1913	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1914	{
3260	Serge	1915	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
6084	serge	1916	int page_count, i;
		1917	struct address_space *mapping;
		1918	struct sg_table *st;
3243	Serge	1919	struct scatterlist *sg;
3746	Serge	1920	struct sg_page_iter sg_iter;
3243	Serge	1921	struct page *page;
3746	Serge	1922	unsigned long last_pfn = 0; /* suppress gcc warning */
6084	serge	1923	int ret;
3243	Serge	1924	gfp_t gfp;
2332	Serge	1925
3243	Serge	1926	/* Assert that the object is not currently in any GPU domain. As it
		1927	* wasn't in the GTT, there shouldn't be any way it could have been in
		1928	* a GPU cache
2332	Serge	1929	*/
3243	Serge	1930	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1931	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1932
		1933	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1934	if (st == NULL)
		1935	return -ENOMEM;
		1936
2332	Serge	1937	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1938	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1939	kfree(st);
2332	Serge	1940	return -ENOMEM;
3243	Serge	1941	}
2332	Serge	1942
3243	Serge	1943	/* Get the list of pages out of our struct file. They'll be pinned
		1944	* at this point until we release them.
		1945	*
		1946	* Fail silently without starting the shrinker
		1947	*/
3746	Serge	1948	sg = st->sgl;
		1949	st->nents = 0;
		1950	for (i = 0; i < page_count; i++) {
4104	Serge	1951	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1952	if (IS_ERR(page)) {
		1953	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1954	goto err_pages;
3260	Serge	1955	}
5354	serge	1956	#ifdef CONFIG_SWIOTLB
		1957	if (swiotlb_nr_tbl()) {
		1958	st->nents++;
		1959	sg_set_page(sg, page, PAGE_SIZE, 0);
		1960	sg = sg_next(sg);
		1961	continue;
		1962	}
		1963	#endif
3746	Serge	1964	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1965	if (i)
		1966	sg = sg_next(sg);
		1967	st->nents++;
6084	serge	1968	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1969	} else {
		1970	sg->length += PAGE_SIZE;
		1971	}
		1972	last_pfn = page_to_pfn(page);
3243	Serge	1973	}
5354	serge	1974	#ifdef CONFIG_SWIOTLB
		1975	if (!swiotlb_nr_tbl())
		1976	#endif
3746	Serge	1977	sg_mark_end(sg);
3243	Serge	1978	obj->pages = st;
3031	serge	1979
6084	serge	1980	ret = i915_gem_gtt_prepare_object(obj);
		1981	if (ret)
		1982	goto err_pages;
5367	serge	1983
6296	serge	1984	if (i915_gem_object_needs_bit17_swizzle(obj))
		1985	i915_gem_object_do_bit_17_swizzle(obj);
		1986
5367	serge	1987	if (obj->tiling_mode != I915_TILING_NONE &&
		1988	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
		1989	i915_gem_object_pin_pages(obj);
		1990
2332	Serge	1991	return 0;
		1992
		1993	err_pages:
3746	Serge	1994	sg_mark_end(sg);
		1995	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1996	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1997	sg_free_table(st);
		1998	kfree(st);
6084	serge	1999
3243	Serge	2000	return PTR_ERR(page);
2332	Serge	2001	}
		2002
3031	serge	2003	/* Ensure that the associated pages are gathered from the backing storage
		2004	* and pinned into our object. i915_gem_object_get_pages() may be called
		2005	* multiple times before they are released by a single call to
		2006	* i915_gem_object_put_pages() - once the pages are no longer referenced
		2007	* either as a result of memory pressure (reaping pages under the shrinker)
		2008	* or as the object is itself released.
		2009	*/
		2010	int
		2011	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	2012	{
3031	serge	2013	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2014	const struct drm_i915_gem_object_ops *ops = obj->ops;
		2015	int ret;
2332	Serge	2016
3243	Serge	2017	if (obj->pages)
3031	serge	2018	return 0;
2332	Serge	2019
4392	Serge	2020	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	2021	DRM_DEBUG("Attempting to obtain a purgeable object\n");
		2022	return -EFAULT;
4392	Serge	2023	}
		2024
3031	serge	2025	BUG_ON(obj->pages_pin_count);
2332	Serge	2026
3031	serge	2027	ret = ops->get_pages(obj);
		2028	if (ret)
		2029	return ret;
2344	Serge	2030
4104	Serge	2031	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
6084	serge	2032
		2033	obj->get_page.sg = obj->pages->sgl;
		2034	obj->get_page.last = 0;
		2035
		2036	return 0;
2332	Serge	2037	}
		2038
6084	serge	2039	void i915_vma_move_to_active(struct i915_vma *vma,
		2040	struct drm_i915_gem_request *req)
2332	Serge	2041	{
6084	serge	2042	struct drm_i915_gem_object *obj = vma->obj;
		2043	struct intel_engine_cs *ring;
2332	Serge	2044
6084	serge	2045	ring = i915_gem_request_get_ring(req);
2332	Serge	2046
		2047	/* Add a reference if we're newly entering the active list. */
6084	serge	2048	if (obj->active == 0)
2344	Serge	2049	drm_gem_object_reference(&obj->base);
6084	serge	2050	obj->active \|= intel_ring_flag(ring);
2332	Serge	2051
6084	serge	2052	list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
		2053	i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2332	Serge	2054
6084	serge	2055	list_move_tail(&vma->mm_list, &vma->vm->active_list);
2332	Serge	2056	}
		2057
6084	serge	2058	static void
		2059	i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
4560	Serge	2060	{
6084	serge	2061	RQ_BUG_ON(obj->last_write_req == NULL);
		2062	RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
		2063
		2064	i915_gem_request_assign(&obj->last_write_req, NULL);
		2065	intel_fb_obj_flush(obj, true, ORIGIN_CS);
4560	Serge	2066	}
		2067
2344	Serge	2068	static void
6084	serge	2069	i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2344	Serge	2070	{
5060	serge	2071	struct i915_vma *vma;
2332	Serge	2072
6084	serge	2073	RQ_BUG_ON(obj->last_read_req[ring] == NULL);
		2074	RQ_BUG_ON(!(obj->active & (1 << ring)));
2332	Serge	2075
6084	serge	2076	list_del_init(&obj->ring_list[ring]);
		2077	i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2344	Serge	2078
6084	serge	2079	if (obj->last_write_req && obj->last_write_req->ring->id == ring)
		2080	i915_gem_object_retire__write(obj);
5354	serge	2081
6084	serge	2082	obj->active &= ~(1 << ring);
		2083	if (obj->active)
		2084	return;
2344	Serge	2085
6084	serge	2086	/* Bump our place on the bound list to keep it roughly in LRU order
		2087	* so that we don't steal from recently used but inactive objects
		2088	* (unless we are forced to ofc!)
		2089	*/
		2090	list_move_tail(&obj->global_list,
		2091	&to_i915(obj->base.dev)->mm.bound_list);
3031	serge	2092
6084	serge	2093	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		2094	if (!list_empty(&vma->mm_list))
		2095	list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
		2096	}
2344	Serge	2097
6084	serge	2098	i915_gem_request_assign(&obj->last_fenced_req, NULL);
2352	Serge	2099	drm_gem_object_unreference(&obj->base);
		2100	}
		2101
3243	Serge	2102	static int
3480	Serge	2103	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	2104	{
3243	Serge	2105	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2106	struct intel_engine_cs *ring;
3243	Serge	2107	int ret, i, j;
2344	Serge	2108
3480	Serge	2109	/* Carefully retire all requests without writing to the rings */
3243	Serge	2110	for_each_ring(ring, dev_priv, i) {
3480	Serge	2111	ret = intel_ring_idle(ring);
6084	serge	2112	if (ret)
		2113	return ret;
3480	Serge	2114	}
		2115	i915_gem_retire_requests(dev);
3243	Serge	2116
3480	Serge	2117	/* Finally reset hw state */
3243	Serge	2118	for_each_ring(ring, dev_priv, i) {
3480	Serge	2119	intel_ring_init_seqno(ring, seqno);
		2120
5060	serge	2121	for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
		2122	ring->semaphore.sync_seqno[j] = 0;
3243	Serge	2123	}
		2124
		2125	return 0;
2344	Serge	2126	}
		2127
3480	Serge	2128	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		2129	{
		2130	struct drm_i915_private *dev_priv = dev->dev_private;
		2131	int ret;
		2132
		2133	if (seqno == 0)
		2134	return -EINVAL;
		2135
		2136	/* HWS page needs to be set less than what we
		2137	* will inject to ring
		2138	*/
		2139	ret = i915_gem_init_seqno(dev, seqno - 1);
		2140	if (ret)
		2141	return ret;
		2142
		2143	/* Carefully set the last_seqno value so that wrap
		2144	* detection still works
		2145	*/
		2146	dev_priv->next_seqno = seqno;
		2147	dev_priv->last_seqno = seqno - 1;
		2148	if (dev_priv->last_seqno == 0)
		2149	dev_priv->last_seqno--;
		2150
		2151	return 0;
		2152	}
		2153
3243	Serge	2154	int
		2155	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	2156	{
3243	Serge	2157	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	2158
3243	Serge	2159	/* reserve 0 for non-seqno */
		2160	if (dev_priv->next_seqno == 0) {
3480	Serge	2161	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	2162	if (ret)
		2163	return ret;
		2164
		2165	dev_priv->next_seqno = 1;
		2166	}
		2167
3480	Serge	2168	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	2169	return 0;
2332	Serge	2170	}
		2171
6084	serge	2172	/*
		2173	* NB: This function is not allowed to fail. Doing so would mean the the
		2174	* request is not being tracked for completion but the work itself is
		2175	* going to happen on the hardware. This would be a Bad Thing(tm).
		2176	*/
		2177	void __i915_add_request(struct drm_i915_gem_request *request,
		2178	struct drm_i915_gem_object *obj,
		2179	bool flush_caches)
2352	Serge	2180	{
6084	serge	2181	struct intel_engine_cs *ring;
		2182	struct drm_i915_private *dev_priv;
5354	serge	2183	struct intel_ringbuffer *ringbuf;
6084	serge	2184	u32 request_start;
2352	Serge	2185	int ret;
2332	Serge	2186
5354	serge	2187	if (WARN_ON(request == NULL))
6084	serge	2188	return;
5354	serge	2189
6084	serge	2190	ring = request->ring;
		2191	dev_priv = ring->dev->dev_private;
		2192	ringbuf = request->ringbuf;
5354	serge	2193
6084	serge	2194	/*
		2195	* To ensure that this call will not fail, space for its emissions
		2196	* should already have been reserved in the ring buffer. Let the ring
		2197	* know that it is time to use that space up.
		2198	*/
		2199	intel_ring_reserved_space_use(ringbuf);
		2200
5354	serge	2201	request_start = intel_ring_get_tail(ringbuf);
3031	serge	2202	/*
		2203	* Emit any outstanding flushes - execbuf can fail to emit the flush
		2204	* after having emitted the batchbuffer command. Hence we need to fix
		2205	* things up similar to emitting the lazy request. The difference here
		2206	* is that the flush _must_ happen before the next request, no matter
		2207	* what.
		2208	*/
6084	serge	2209	if (flush_caches) {
		2210	if (i915.enable_execlists)
		2211	ret = logical_ring_flush_all_caches(request);
		2212	else
		2213	ret = intel_ring_flush_all_caches(request);
		2214	/* Not allowed to fail! */
		2215	WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
5354	serge	2216	}
2332	Serge	2217
3031	serge	2218	/* Record the position of the start of the request so that
		2219	* should we detect the updated seqno part-way through the
6084	serge	2220	* GPU processing the request, we never over-estimate the
3031	serge	2221	* position of the head.
		2222	*/
6084	serge	2223	request->postfix = intel_ring_get_tail(ringbuf);
3031	serge	2224
6084	serge	2225	if (i915.enable_execlists)
		2226	ret = ring->emit_request(request);
		2227	else {
		2228	ret = ring->add_request(request);
		2229
		2230	request->tail = intel_ring_get_tail(ringbuf);
5354	serge	2231	}
6084	serge	2232	/* Not allowed to fail! */
		2233	WARN(ret, "emit\|add_request failed: %d!\n", ret);
2332	Serge	2234
4104	Serge	2235	request->head = request_start;
		2236
		2237	/* Whilst this request exists, batch_obj will be on the
		2238	* active_list, and so will hold the active reference. Only when this
		2239	* request is retired will the the batch_obj be moved onto the
		2240	* inactive_list and lose its active reference. Hence we do not need
		2241	* to explicitly hold another reference here.
		2242	*/
4560	Serge	2243	request->batch_obj = obj;
4104	Serge	2244
5060	serge	2245	request->emitted_jiffies = jiffies;
6084	serge	2246	request->previous_seqno = ring->last_submitted_seqno;
		2247	ring->last_submitted_seqno = request->seqno;
2352	Serge	2248	list_add_tail(&request->list, &ring->request_list);
2332	Serge	2249
6084	serge	2250	trace_i915_gem_request_add(request);
2332	Serge	2251
6084	serge	2252	// i915_queue_hangcheck(ring->dev);
3263	Serge	2253
6084	serge	2254	queue_delayed_work(dev_priv->wq,
		2255	&dev_priv->mm.retire_work,
		2256	round_jiffies_up_relative(HZ));
		2257	intel_mark_busy(dev_priv->dev);
2332	Serge	2258
6084	serge	2259	/* Sanity check that the reserved size was large enough. */
		2260	intel_ring_reserved_space_end(ringbuf);
2352	Serge	2261	}
2332	Serge	2262
5060	serge	2263	static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
		2264	const struct intel_context *ctx)
4104	Serge	2265	{
5060	serge	2266	unsigned long elapsed;
4104	Serge	2267
5060	serge	2268	elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts;
4104	Serge	2269
5060	serge	2270	if (ctx->hang_stats.banned)
		2271	return true;
4104	Serge	2272
6084	serge	2273	if (ctx->hang_stats.ban_period_seconds &&
		2274	elapsed <= ctx->hang_stats.ban_period_seconds) {
5060	serge	2275	if (!i915_gem_context_is_default(ctx)) {
		2276	DRM_DEBUG("context hanging too fast, banning!\n");
4104	Serge	2277	return true;
5060	serge	2278	} else if (i915_stop_ring_allow_ban(dev_priv)) {
		2279	if (i915_stop_ring_allow_warn(dev_priv))
6084	serge	2280	DRM_ERROR("gpu hanging too fast, banning!\n");
4104	Serge	2281	return true;
6084	serge	2282	}
4104	Serge	2283	}
		2284
		2285	return false;
		2286	}
		2287
5060	serge	2288	static void i915_set_reset_status(struct drm_i915_private *dev_priv,
		2289	struct intel_context *ctx,
		2290	const bool guilty)
4560	Serge	2291	{
5060	serge	2292	struct i915_ctx_hang_stats *hs;
4560	Serge	2293
5060	serge	2294	if (WARN_ON(!ctx))
		2295	return;
4560	Serge	2296
5060	serge	2297	hs = &ctx->hang_stats;
4560	Serge	2298
5060	serge	2299	if (guilty) {
		2300	hs->banned = i915_context_is_banned(dev_priv, ctx);
		2301	hs->batch_active++;
		2302	hs->guilty_ts = GetTimerTicks()/100;
		2303	} else {
		2304	hs->batch_pending++;
4104	Serge	2305	}
		2306	}
		2307
6084	serge	2308	void i915_gem_request_free(struct kref *req_ref)
4104	Serge	2309	{
6084	serge	2310	struct drm_i915_gem_request *req = container_of(req_ref,
		2311	typeof(*req), ref);
		2312	struct intel_context *ctx = req->ctx;
5354	serge	2313
6084	serge	2314	if (req->file_priv)
		2315	i915_gem_request_remove_from_client(req);
4104	Serge	2316
5354	serge	2317	if (ctx) {
		2318	if (i915.enable_execlists) {
6084	serge	2319	if (ctx != req->ring->default_context)
		2320	intel_lr_context_unpin(req);
		2321	}
4104	Serge	2322
5354	serge	2323	i915_gem_context_unreference(ctx);
		2324	}
6084	serge	2325
		2326	kfree(req);
4104	Serge	2327	}
		2328
6084	serge	2329	int i915_gem_request_alloc(struct intel_engine_cs *ring,
		2330	struct intel_context *ctx,
		2331	struct drm_i915_gem_request **req_out)
		2332	{
		2333	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		2334	struct drm_i915_gem_request *req;
		2335	int ret;
		2336
		2337	if (!req_out)
		2338	return -EINVAL;
		2339
		2340	*req_out = NULL;
		2341
		2342	// req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
		2343	req = kzalloc(sizeof(*req),0);
		2344	if (req == NULL)
		2345	return -ENOMEM;
		2346
		2347	ret = i915_gem_get_seqno(ring->dev, &req->seqno);
		2348	if (ret)
		2349	goto err;
		2350
		2351	kref_init(&req->ref);
		2352	req->i915 = dev_priv;
		2353	req->ring = ring;
		2354	req->ctx = ctx;
		2355	i915_gem_context_reference(req->ctx);
		2356
		2357	if (i915.enable_execlists)
		2358	ret = intel_logical_ring_alloc_request_extras(req);
		2359	else
		2360	ret = intel_ring_alloc_request_extras(req);
		2361	if (ret) {
		2362	i915_gem_context_unreference(req->ctx);
		2363	goto err;
		2364	}
		2365
		2366	/*
		2367	* Reserve space in the ring buffer for all the commands required to
		2368	* eventually emit this request. This is to guarantee that the
		2369	* i915_add_request() call can't fail. Note that the reserve may need
		2370	* to be redone if the request is not actually submitted straight
		2371	* away, e.g. because a GPU scheduler has deferred it.
		2372	*/
		2373	if (i915.enable_execlists)
		2374	ret = intel_logical_ring_reserve_space(req);
		2375	else
		2376	ret = intel_ring_reserve_space(req);
		2377	if (ret) {
		2378	/*
		2379	* At this point, the request is fully allocated even if not
		2380	* fully prepared. Thus it can be cleaned up using the proper
		2381	* free code.
		2382	*/
		2383	i915_gem_request_cancel(req);
		2384	return ret;
		2385	}
		2386
		2387	*req_out = req;
		2388	return 0;
		2389
		2390	err:
		2391	kfree(req);
		2392	return ret;
		2393	}
		2394
		2395	void i915_gem_request_cancel(struct drm_i915_gem_request *req)
		2396	{
		2397	intel_ring_reserved_space_cancel(req->ringbuf);
		2398
		2399	i915_gem_request_unreference(req);
		2400	}
		2401
5060	serge	2402	struct drm_i915_gem_request *
		2403	i915_gem_find_active_request(struct intel_engine_cs *ring)
3031	serge	2404	{
4539	Serge	2405	struct drm_i915_gem_request *request;
4104	Serge	2406
4539	Serge	2407	list_for_each_entry(request, &ring->request_list, list) {
6084	serge	2408	if (i915_gem_request_completed(request, false))
4539	Serge	2409	continue;
4104	Serge	2410
5060	serge	2411	return request;
4539	Serge	2412	}
5060	serge	2413
		2414	return NULL;
4539	Serge	2415	}
		2416
5060	serge	2417	static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
		2418	struct intel_engine_cs *ring)
		2419	{
		2420	struct drm_i915_gem_request *request;
		2421	bool ring_hung;
		2422
		2423	request = i915_gem_find_active_request(ring);
		2424
		2425	if (request == NULL)
		2426	return;
		2427
		2428	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
		2429
		2430	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
		2431
		2432	list_for_each_entry_continue(request, &ring->request_list, list)
		2433	i915_set_reset_status(dev_priv, request->ctx, false);
		2434	}
		2435
4539	Serge	2436	static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
5060	serge	2437	struct intel_engine_cs *ring)
4539	Serge	2438	{
4560	Serge	2439	while (!list_empty(&ring->active_list)) {
		2440	struct drm_i915_gem_object *obj;
		2441
		2442	obj = list_first_entry(&ring->active_list,
		2443	struct drm_i915_gem_object,
6084	serge	2444	ring_list[ring->id]);
4560	Serge	2445
6084	serge	2446	i915_gem_object_retire__read(obj, ring->id);
4560	Serge	2447	}
		2448
		2449	/*
5354	serge	2450	* Clear the execlists queue up before freeing the requests, as those
		2451	* are the ones that keep the context and ringbuffer backing objects
		2452	* pinned in place.
		2453	*/
		2454	while (!list_empty(&ring->execlist_queue)) {
6084	serge	2455	struct drm_i915_gem_request *submit_req;
5354	serge	2456
		2457	submit_req = list_first_entry(&ring->execlist_queue,
6084	serge	2458	struct drm_i915_gem_request,
5354	serge	2459	execlist_link);
		2460	list_del(&submit_req->execlist_link);
6084	serge	2461
		2462	if (submit_req->ctx != ring->default_context)
		2463	intel_lr_context_unpin(submit_req);
		2464
		2465	i915_gem_request_unreference(submit_req);
5354	serge	2466	}
		2467
		2468	/*
4560	Serge	2469	* We must free the requests after all the corresponding objects have
		2470	* been moved off active lists. Which is the same order as the normal
		2471	* retire_requests function does. This is important if object hold
		2472	* implicit references on things like e.g. ppgtt address spaces through
		2473	* the request.
		2474	*/
3031	serge	2475	while (!list_empty(&ring->request_list)) {
		2476	struct drm_i915_gem_request *request;
2332	Serge	2477
3031	serge	2478	request = list_first_entry(&ring->request_list,
		2479	struct drm_i915_gem_request,
		2480	list);
2332	Serge	2481
6084	serge	2482	i915_gem_request_retire(request);
3031	serge	2483	}
		2484	}
2332	Serge	2485
3031	serge	2486	void i915_gem_reset(struct drm_device *dev)
		2487	{
		2488	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2489	struct intel_engine_cs *ring;
3031	serge	2490	int i;
2360	Serge	2491
4539	Serge	2492	/*
		2493	* Before we free the objects from the requests, we need to inspect
		2494	* them for finding the guilty party. As the requests only borrow
		2495	* their reference to the objects, the inspection must be done first.
		2496	*/
3031	serge	2497	for_each_ring(ring, dev_priv, i)
4539	Serge	2498	i915_gem_reset_ring_status(dev_priv, ring);
2360	Serge	2499
4539	Serge	2500	for_each_ring(ring, dev_priv, i)
		2501	i915_gem_reset_ring_cleanup(dev_priv, ring);
		2502
5060	serge	2503	i915_gem_context_reset(dev);
4560	Serge	2504
3746	Serge	2505	i915_gem_restore_fences(dev);
6084	serge	2506
		2507	WARN_ON(i915_verify_lists(dev));
3031	serge	2508	}
2360	Serge	2509
2352	Serge	2510	/**
		2511	* This function clears the request list as sequence numbers are passed.
		2512	*/
3031	serge	2513	void
5060	serge	2514	i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2352	Serge	2515	{
6084	serge	2516	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2517
6084	serge	2518	/* Retire requests first as we use it above for the early return.
		2519	* If we retire requests last, we may use a later seqno and so clear
		2520	* the requests lists without clearing the active list, leading to
		2521	* confusion.
		2522	*/
		2523	while (!list_empty(&ring->request_list)) {
		2524	struct drm_i915_gem_request *request;
2332	Serge	2525
6084	serge	2526	request = list_first_entry(&ring->request_list,
		2527	struct drm_i915_gem_request,
		2528	list);
2332	Serge	2529
6084	serge	2530	if (!i915_gem_request_completed(request, true))
		2531	break;
2332	Serge	2532
6084	serge	2533	i915_gem_request_retire(request);
		2534	}
		2535
5060	serge	2536	/* Move any buffers on the active list that are no longer referenced
		2537	* by the ringbuffer to the flushing/inactive lists as appropriate,
		2538	* before we free the context associated with the requests.
		2539	*/
		2540	while (!list_empty(&ring->active_list)) {
		2541	struct drm_i915_gem_object *obj;
		2542
		2543	obj = list_first_entry(&ring->active_list,
		2544	struct drm_i915_gem_object,
6084	serge	2545	ring_list[ring->id]);
5060	serge	2546
6084	serge	2547	if (!list_empty(&obj->last_read_req[ring->id]->list))
5060	serge	2548	break;
		2549
6084	serge	2550	i915_gem_object_retire__read(obj, ring->id);
5060	serge	2551	}
		2552
6084	serge	2553	if (unlikely(ring->trace_irq_req &&
		2554	i915_gem_request_completed(ring->trace_irq_req, true))) {
2352	Serge	2555	ring->irq_put(ring);
6084	serge	2556	i915_gem_request_assign(&ring->trace_irq_req, NULL);
2352	Serge	2557	}
2332	Serge	2558
2352	Serge	2559	WARN_ON(i915_verify_lists(ring->dev));
		2560	}
2332	Serge	2561
4560	Serge	2562	bool
2352	Serge	2563	i915_gem_retire_requests(struct drm_device *dev)
		2564	{
5060	serge	2565	struct drm_i915_private *dev_priv = dev->dev_private;
		2566	struct intel_engine_cs *ring;
4560	Serge	2567	bool idle = true;
2352	Serge	2568	int i;
2332	Serge	2569
4560	Serge	2570	for_each_ring(ring, dev_priv, i) {
3031	serge	2571	i915_gem_retire_requests_ring(ring);
4560	Serge	2572	idle &= list_empty(&ring->request_list);
5354	serge	2573	if (i915.enable_execlists) {
		2574	unsigned long flags;
		2575
		2576	spin_lock_irqsave(&ring->execlist_lock, flags);
		2577	idle &= list_empty(&ring->execlist_queue);
		2578	spin_unlock_irqrestore(&ring->execlist_lock, flags);
		2579
		2580	intel_execlists_retire_requests(ring);
		2581	}
4560	Serge	2582	}
		2583
		2584	if (idle)
		2585	mod_delayed_work(dev_priv->wq,
		2586	&dev_priv->mm.idle_work,
		2587	msecs_to_jiffies(100));
		2588
		2589	return idle;
2352	Serge	2590	}
		2591
2360	Serge	2592	static void
		2593	i915_gem_retire_work_handler(struct work_struct *work)
		2594	{
4560	Serge	2595	struct drm_i915_private *dev_priv =
		2596	container_of(work, typeof(*dev_priv), mm.retire_work.work);
		2597	struct drm_device *dev = dev_priv->dev;
2360	Serge	2598	bool idle;
2352	Serge	2599
2360	Serge	2600	/* Come back later if the device is busy... */
4560	Serge	2601	idle = false;
		2602	if (mutex_trylock(&dev->struct_mutex)) {
		2603	idle = i915_gem_retire_requests(dev);
		2604	mutex_unlock(&dev->struct_mutex);
		2605	}
		2606	if (!idle)
3482	Serge	2607	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2608	round_jiffies_up_relative(HZ));
4560	Serge	2609	}
2352	Serge	2610
4560	Serge	2611	static void
		2612	i915_gem_idle_work_handler(struct work_struct *work)
		2613	{
		2614	struct drm_i915_private *dev_priv =
		2615	container_of(work, typeof(*dev_priv), mm.idle_work.work);
6084	serge	2616	struct drm_device *dev = dev_priv->dev;
		2617	struct intel_engine_cs *ring;
		2618	int i;
2352	Serge	2619
6084	serge	2620	for_each_ring(ring, dev_priv, i)
		2621	if (!list_empty(&ring->request_list))
		2622	return;
		2623
		2624	intel_mark_idle(dev);
		2625
		2626	if (mutex_trylock(&dev->struct_mutex)) {
		2627	struct intel_engine_cs *ring;
		2628	int i;
		2629
		2630	for_each_ring(ring, dev_priv, i)
		2631	i915_gem_batch_pool_fini(&ring->batch_pool);
		2632
		2633	mutex_unlock(&dev->struct_mutex);
		2634	}
2360	Serge	2635	}
		2636
2344	Serge	2637	/**
3031	serge	2638	* Ensures that an object will eventually get non-busy by flushing any required
		2639	* write domains, emitting any outstanding lazy request and retiring and
		2640	* completed requests.
2352	Serge	2641	*/
3031	serge	2642	static int
		2643	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2644	{
6084	serge	2645	int i;
2352	Serge	2646
6084	serge	2647	if (!obj->active)
		2648	return 0;
2352	Serge	2649
6084	serge	2650	for (i = 0; i < I915_NUM_RINGS; i++) {
		2651	struct drm_i915_gem_request *req;
		2652
		2653	req = obj->last_read_req[i];
		2654	if (req == NULL)
		2655	continue;
		2656
		2657	if (list_empty(&req->list))
		2658	goto retire;
		2659
		2660	if (i915_gem_request_completed(req, true)) {
		2661	__i915_gem_request_retire__upto(req);
		2662	retire:
		2663	i915_gem_object_retire__read(obj, i);
		2664	}
3031	serge	2665	}
2352	Serge	2666
3031	serge	2667	return 0;
		2668	}
2352	Serge	2669
3243	Serge	2670	/**
		2671	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2672	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2673	*
		2674	* Returns 0 if successful, else an error is returned with the remaining time in
		2675	* the timeout parameter.
		2676	* -ETIME: object is still busy after timeout
		2677	* -ERESTARTSYS: signal interrupted the wait
		2678	* -ENONENT: object doesn't exist
		2679	* Also possible, but rare:
		2680	* -EAGAIN: GPU wedged
		2681	* -ENOMEM: damn
		2682	* -ENODEV: Internal IRQ fail
		2683	* -E?: The add request failed
		2684	*
		2685	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2686	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2687	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2688	* without holding struct_mutex the object may become re-busied before this
		2689	* function completes. A similar but shorter * race condition exists in the busy
		2690	* ioctl
		2691	*/
4246	Serge	2692	int
		2693	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2694	{
5060	serge	2695	struct drm_i915_private *dev_priv = dev->dev_private;
4246	Serge	2696	struct drm_i915_gem_wait *args = data;
		2697	struct drm_i915_gem_object *obj;
6084	serge	2698	struct drm_i915_gem_request *req[I915_NUM_RINGS];
4246	Serge	2699	unsigned reset_counter;
6084	serge	2700	int i, n = 0;
		2701	int ret;
2352	Serge	2702
5354	serge	2703	if (args->flags != 0)
		2704	return -EINVAL;
		2705
4246	Serge	2706	ret = i915_mutex_lock_interruptible(dev);
		2707	if (ret)
		2708	return ret;
2352	Serge	2709
4246	Serge	2710	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2711	if (&obj->base == NULL) {
		2712	mutex_unlock(&dev->struct_mutex);
		2713	return -ENOENT;
		2714	}
2352	Serge	2715
4246	Serge	2716	/* Need to make sure the object gets inactive eventually. */
		2717	ret = i915_gem_object_flush_active(obj);
		2718	if (ret)
		2719	goto out;
2352	Serge	2720
6084	serge	2721	if (!obj->active)
		2722	goto out;
2352	Serge	2723
4246	Serge	2724	/* Do this after OLR check to make sure we make forward progress polling
6084	serge	2725	* on this IOCTL with a timeout == 0 (like busy ioctl)
4246	Serge	2726	*/
6084	serge	2727	if (args->timeout_ns == 0) {
4246	Serge	2728	ret = -ETIME;
		2729	goto out;
		2730	}
2352	Serge	2731
4246	Serge	2732	drm_gem_object_unreference(&obj->base);
		2733	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
6084	serge	2734
		2735	for (i = 0; i < I915_NUM_RINGS; i++) {
		2736	if (obj->last_read_req[i] == NULL)
		2737	continue;
		2738
		2739	req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
		2740	}
		2741
4246	Serge	2742	mutex_unlock(&dev->struct_mutex);
2352	Serge	2743
6084	serge	2744	for (i = 0; i < n; i++) {
		2745	if (ret == 0)
		2746	ret = __i915_wait_request(req[i], reset_counter, true,
		2747	args->timeout_ns > 0 ? &args->timeout_ns : NULL,
		2748	file->driver_priv);
		2749	i915_gem_request_unreference__unlocked(req[i]);
		2750	}
		2751	return ret;
3243	Serge	2752
4246	Serge	2753	out:
		2754	drm_gem_object_unreference(&obj->base);
		2755	mutex_unlock(&dev->struct_mutex);
		2756	return ret;
		2757	}
3243	Serge	2758
6084	serge	2759	static int
		2760	__i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2761	struct intel_engine_cs *to,
		2762	struct drm_i915_gem_request *from_req,
		2763	struct drm_i915_gem_request **to_req)
		2764	{
		2765	struct intel_engine_cs *from;
		2766	int ret;
		2767
		2768	from = i915_gem_request_get_ring(from_req);
		2769	if (to == from)
		2770	return 0;
		2771
		2772	if (i915_gem_request_completed(from_req, true))
		2773	return 0;
		2774
		2775	if (!i915_semaphore_is_enabled(obj->base.dev)) {
		2776	struct drm_i915_private *i915 = to_i915(obj->base.dev);
		2777	ret = __i915_wait_request(from_req,
		2778	atomic_read(&i915->gpu_error.reset_counter),
		2779	i915->mm.interruptible,
		2780	NULL,
		2781	&i915->rps.semaphores);
		2782	if (ret)
		2783	return ret;
		2784
		2785	i915_gem_object_retire_request(obj, from_req);
		2786	} else {
		2787	int idx = intel_ring_sync_index(from, to);
		2788	u32 seqno = i915_gem_request_get_seqno(from_req);
		2789
		2790	WARN_ON(!to_req);
		2791
		2792	if (seqno <= from->semaphore.sync_seqno[idx])
		2793	return 0;
		2794
		2795	if (*to_req == NULL) {
		2796	ret = i915_gem_request_alloc(to, to->default_context, to_req);
		2797	if (ret)
		2798	return ret;
		2799	}
		2800
		2801	trace_i915_gem_ring_sync_to(*to_req, from, from_req);
		2802	ret = to->semaphore.sync_to(*to_req, from, seqno);
		2803	if (ret)
		2804	return ret;
		2805
		2806	/* We use last_read_req because sync_to()
		2807	* might have just caused seqno wrap under
		2808	* the radar.
		2809	*/
		2810	from->semaphore.sync_seqno[idx] =
		2811	i915_gem_request_get_seqno(obj->last_read_req[from->id]);
		2812	}
		2813
		2814	return 0;
		2815	}
		2816
2352	Serge	2817	/**
3031	serge	2818	* i915_gem_object_sync - sync an object to a ring.
		2819	*
		2820	* @obj: object which may be in use on another ring.
		2821	* @to: ring we wish to use the object on. May be NULL.
6084	serge	2822	* @to_req: request we wish to use the object for. See below.
		2823	* This will be allocated and returned if a request is
		2824	* required but not passed in.
3031	serge	2825	*
		2826	* This code is meant to abstract object synchronization with the GPU.
		2827	* Calling with NULL implies synchronizing the object with the CPU
6084	serge	2828	* rather than a particular GPU ring. Conceptually we serialise writes
		2829	* between engines inside the GPU. We only allow one engine to write
		2830	* into a buffer at any time, but multiple readers. To ensure each has
		2831	* a coherent view of memory, we must:
3031	serge	2832	*
6084	serge	2833	* - If there is an outstanding write request to the object, the new
		2834	* request must wait for it to complete (either CPU or in hw, requests
		2835	* on the same ring will be naturally ordered).
		2836	*
		2837	* - If we are a write request (pending_write_domain is set), the new
		2838	* request must wait for outstanding read requests to complete.
		2839	*
		2840	* For CPU synchronisation (NULL to) no request is required. For syncing with
		2841	* rings to_req must be non-NULL. However, a request does not have to be
		2842	* pre-allocated. If *to_req is NULL and sync commands will be emitted then a
		2843	* request will be allocated automatically and returned through *to_req. Note
		2844	* that it is not guaranteed that commands will be emitted (because the system
		2845	* might already be idle). Hence there is no need to create a request that
		2846	* might never have any work submitted. Note further that if a request is
		2847	* returned in *to_req, it is the responsibility of the caller to submit
		2848	* that request (after potentially adding more work to it).
		2849	*
3031	serge	2850	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2851	*/
		2852	int
3031	serge	2853	i915_gem_object_sync(struct drm_i915_gem_object *obj,
6084	serge	2854	struct intel_engine_cs *to,
		2855	struct drm_i915_gem_request **to_req)
2344	Serge	2856	{
6084	serge	2857	const bool readonly = obj->base.pending_write_domain == 0;
		2858	struct drm_i915_gem_request *req[I915_NUM_RINGS];
		2859	int ret, i, n;
2332	Serge	2860
6084	serge	2861	if (!obj->active)
3031	serge	2862	return 0;
2332	Serge	2863
6084	serge	2864	if (to == NULL)
		2865	return i915_gem_object_wait_rendering(obj, readonly);
2332	Serge	2866
6084	serge	2867	n = 0;
		2868	if (readonly) {
		2869	if (obj->last_write_req)
		2870	req[n++] = obj->last_write_req;
		2871	} else {
		2872	for (i = 0; i < I915_NUM_RINGS; i++)
		2873	if (obj->last_read_req[i])
		2874	req[n++] = obj->last_read_req[i];
		2875	}
		2876	for (i = 0; i < n; i++) {
		2877	ret = __i915_gem_object_sync(obj, to, req[i], to_req);
		2878	if (ret)
		2879	return ret;
		2880	}
3031	serge	2881
6084	serge	2882	return 0;
2344	Serge	2883	}
2332	Serge	2884
2344	Serge	2885	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2886	{
		2887	u32 old_write_domain, old_read_domains;
2332	Serge	2888
2344	Serge	2889	/* Force a pagefault for domain tracking on next user access */
6084	serge	2890	i915_gem_release_mmap(obj);
2332	Serge	2891
2344	Serge	2892	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2893	return;
2332	Serge	2894
3480	Serge	2895	/* Wait for any direct GTT access to complete */
		2896	mb();
		2897
2344	Serge	2898	old_read_domains = obj->base.read_domains;
		2899	old_write_domain = obj->base.write_domain;
2351	Serge	2900
2344	Serge	2901	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2902	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2903
2351	Serge	2904	trace_i915_gem_object_change_domain(obj,
		2905	old_read_domains,
		2906	old_write_domain);
2344	Serge	2907	}
2332	Serge	2908
6084	serge	2909	static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
2344	Serge	2910	{
4104	Serge	2911	struct drm_i915_gem_object *obj = vma->obj;
5060	serge	2912	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3480	Serge	2913	int ret;
3263	Serge	2914
4104	Serge	2915	if (list_empty(&vma->vma_link))
2344	Serge	2916	return 0;
2332	Serge	2917
4560	Serge	2918	if (!drm_mm_node_allocated(&vma->node)) {
		2919	i915_gem_vma_destroy(vma);
		2920	return 0;
		2921	}
		2922
5060	serge	2923	if (vma->pin_count)
3031	serge	2924	return -EBUSY;
2332	Serge	2925
3243	Serge	2926	BUG_ON(obj->pages == NULL);
3031	serge	2927
6084	serge	2928	if (wait) {
		2929	ret = i915_gem_object_wait_rendering(obj, false);
		2930	if (ret)
		2931	return ret;
		2932	}
2332	Serge	2933
6084	serge	2934	if (i915_is_ggtt(vma->vm) &&
		2935	vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
		2936	i915_gem_object_finish_gtt(obj);
5354	serge	2937
6084	serge	2938	/* release the fence reg _after_ flushing */
		2939	ret = i915_gem_object_put_fence(obj);
		2940	if (ret)
		2941	return ret;
5060	serge	2942	}
2332	Serge	2943
4104	Serge	2944	trace_i915_vma_unbind(vma);
2332	Serge	2945
6084	serge	2946	vma->vm->unbind_vma(vma);
		2947	vma->bound = 0;
2332	Serge	2948
5060	serge	2949	list_del_init(&vma->mm_list);
6084	serge	2950	if (i915_is_ggtt(vma->vm)) {
		2951	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
		2952	obj->map_and_fenceable = false;
		2953	} else if (vma->ggtt_view.pages) {
		2954	sg_free_table(vma->ggtt_view.pages);
		2955	kfree(vma->ggtt_view.pages);
		2956	}
		2957	vma->ggtt_view.pages = NULL;
		2958	}
2332	Serge	2959
4104	Serge	2960	drm_mm_remove_node(&vma->node);
		2961	i915_gem_vma_destroy(vma);
		2962
		2963	/* Since the unbound list is global, only move to that list if
4560	Serge	2964	* no more VMAs exist. */
6084	serge	2965	if (list_empty(&obj->vma_list))
4104	Serge	2966	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2967
4560	Serge	2968	/* And finally now the object is completely decoupled from this vma,
		2969	* we can drop its hold on the backing storage and allow it to be
		2970	* reaped by the shrinker.
		2971	*/
		2972	i915_gem_object_unpin_pages(obj);
		2973
2344	Serge	2974	return 0;
		2975	}
2332	Serge	2976
6084	serge	2977	int i915_vma_unbind(struct i915_vma *vma)
		2978	{
		2979	return __i915_vma_unbind(vma, true);
		2980	}
		2981
		2982	int __i915_vma_unbind_no_wait(struct i915_vma *vma)
		2983	{
		2984	return __i915_vma_unbind(vma, false);
		2985	}
		2986
3031	serge	2987	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2988	{
5060	serge	2989	struct drm_i915_private *dev_priv = dev->dev_private;
		2990	struct intel_engine_cs *ring;
2344	Serge	2991	int ret, i;
2332	Serge	2992
2344	Serge	2993	/* Flush everything onto the inactive list. */
3031	serge	2994	for_each_ring(ring, dev_priv, i) {
5354	serge	2995	if (!i915.enable_execlists) {
6084	serge	2996	struct drm_i915_gem_request *req;
3031	serge	2997
6084	serge	2998	ret = i915_gem_request_alloc(ring, ring->default_context, &req);
2352	Serge	2999	if (ret)
		3000	return ret;
2344	Serge	3001
6084	serge	3002	ret = i915_switch_context(req);
		3003	if (ret) {
		3004	i915_gem_request_cancel(req);
		3005	return ret;
		3006	}
2344	Serge	3007
6084	serge	3008	i915_add_request_no_flush(req);
		3009	}
2332	Serge	3010
6084	serge	3011	ret = intel_ring_idle(ring);
3031	serge	3012	if (ret)
		3013	return ret;
		3014	}
2332	Serge	3015
6084	serge	3016	WARN_ON(i915_verify_lists(dev));
3031	serge	3017	return 0;
		3018	}
2332	Serge	3019
5354	serge	3020	static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3031	serge	3021	unsigned long cache_level)
		3022	{
5354	serge	3023	struct drm_mm_node *gtt_space = &vma->node;
3031	serge	3024	struct drm_mm_node *other;
2332	Serge	3025
5354	serge	3026	/*
		3027	* On some machines we have to be careful when putting differing types
		3028	* of snoopable memory together to avoid the prefetcher crossing memory
		3029	* domains and dying. During vm initialisation, we decide whether or not
		3030	* these constraints apply and set the drm_mm.color_adjust
		3031	* appropriately.
3031	serge	3032	*/
5354	serge	3033	if (vma->vm->mm.color_adjust == NULL)
3031	serge	3034	return true;
2332	Serge	3035
4104	Serge	3036	if (!drm_mm_node_allocated(gtt_space))
3031	serge	3037	return true;
2332	Serge	3038
3031	serge	3039	if (list_empty(>t_space->node_list))
		3040	return true;
2332	Serge	3041
3031	serge	3042	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		3043	if (other->allocated && !other->hole_follows && other->color != cache_level)
		3044	return false;
2344	Serge	3045
3031	serge	3046	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		3047	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		3048	return false;
2344	Serge	3049
3031	serge	3050	return true;
		3051	}
2344	Serge	3052
2332	Serge	3053	/**
6084	serge	3054	* Finds free space in the GTT aperture and binds the object or a view of it
		3055	* there.
2332	Serge	3056	*/
5060	serge	3057	static struct i915_vma *
4104	Serge	3058	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		3059	struct i915_address_space *vm,
6084	serge	3060	const struct i915_ggtt_view *ggtt_view,
		3061	unsigned alignment,
5060	serge	3062	uint64_t flags)
2332	Serge	3063	{
		3064	struct drm_device *dev = obj->base.dev;
5060	serge	3065	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	3066	u32 fence_alignment, unfenced_alignment;
		3067	u32 search_flag, alloc_flag;
		3068	u64 start, end;
		3069	u64 size, fence_size;
4104	Serge	3070	struct i915_vma *vma;
2332	Serge	3071	int ret;
2326	Serge	3072
6084	serge	3073	if (i915_is_ggtt(vm)) {
		3074	u32 view_size;
2332	Serge	3075
6084	serge	3076	if (WARN_ON(!ggtt_view))
		3077	return ERR_PTR(-EINVAL);
		3078
		3079	view_size = i915_ggtt_view_size(obj, ggtt_view);
		3080
		3081	fence_size = i915_gem_get_gtt_size(dev,
		3082	view_size,
		3083	obj->tiling_mode);
		3084	fence_alignment = i915_gem_get_gtt_alignment(dev,
		3085	view_size,
		3086	obj->tiling_mode,
		3087	true);
		3088	unfenced_alignment = i915_gem_get_gtt_alignment(dev,
		3089	view_size,
		3090	obj->tiling_mode,
		3091	false);
		3092	size = flags & PIN_MAPPABLE ? fence_size : view_size;
		3093	} else {
		3094	fence_size = i915_gem_get_gtt_size(dev,
		3095	obj->base.size,
		3096	obj->tiling_mode);
		3097	fence_alignment = i915_gem_get_gtt_alignment(dev,
		3098	obj->base.size,
		3099	obj->tiling_mode,
		3100	true);
		3101	unfenced_alignment =
		3102	i915_gem_get_gtt_alignment(dev,
		3103	obj->base.size,
		3104	obj->tiling_mode,
		3105	false);
		3106	size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
		3107	}
		3108
		3109	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
		3110	end = vm->total;
		3111	if (flags & PIN_MAPPABLE)
		3112	end = min_t(u64, end, dev_priv->gtt.mappable_end);
		3113	if (flags & PIN_ZONE_4G)
		3114	end = min_t(u64, end, (1ULL << 32));
		3115
2332	Serge	3116	if (alignment == 0)
5060	serge	3117	alignment = flags & PIN_MAPPABLE ? fence_alignment :
2332	Serge	3118	unfenced_alignment;
5060	serge	3119	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
6084	serge	3120	DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
		3121	ggtt_view ? ggtt_view->type : 0,
		3122	alignment);
5060	serge	3123	return ERR_PTR(-EINVAL);
2332	Serge	3124	}
		3125
6084	serge	3126	/* If binding the object/GGTT view requires more space than the entire
		3127	* aperture has, reject it early before evicting everything in a vain
		3128	* attempt to find space.
2332	Serge	3129	*/
6084	serge	3130	if (size > end) {
		3131	DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
		3132	ggtt_view ? ggtt_view->type : 0,
		3133	size,
5060	serge	3134	flags & PIN_MAPPABLE ? "mappable" : "total",
		3135	end);
		3136	return ERR_PTR(-E2BIG);
2332	Serge	3137	}
		3138
3031	serge	3139	ret = i915_gem_object_get_pages(obj);
		3140	if (ret)
5060	serge	3141	return ERR_PTR(ret);
3031	serge	3142
3243	Serge	3143	i915_gem_object_pin_pages(obj);
		3144
6084	serge	3145	vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
		3146	i915_gem_obj_lookup_or_create_vma(obj, vm);
		3147
5060	serge	3148	if (IS_ERR(vma))
4104	Serge	3149	goto err_unpin;
3243	Serge	3150
6084	serge	3151	if (flags & PIN_HIGH) {
		3152	search_flag = DRM_MM_SEARCH_BELOW;
		3153	alloc_flag = DRM_MM_CREATE_TOP;
		3154	} else {
		3155	search_flag = DRM_MM_SEARCH_DEFAULT;
		3156	alloc_flag = DRM_MM_CREATE_DEFAULT;
		3157	}
		3158
4104	Serge	3159	search_free:
		3160	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3161	size, alignment,
5060	serge	3162	obj->cache_level,
		3163	start, end,
6084	serge	3164	search_flag,
		3165	alloc_flag);
3243	Serge	3166	if (ret) {
2332	Serge	3167
4104	Serge	3168	goto err_free_vma;
2332	Serge	3169	}
5354	serge	3170	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
4104	Serge	3171	ret = -EINVAL;
		3172	goto err_remove_node;
3031	serge	3173	}
2332	Serge	3174
6084	serge	3175	trace_i915_vma_bind(vma, flags);
		3176	ret = i915_vma_bind(vma, obj->cache_level, flags);
4104	Serge	3177	if (ret)
		3178	goto err_remove_node;
2332	Serge	3179
4104	Serge	3180	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3181	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3182
5060	serge	3183	return vma;
4104	Serge	3184
		3185	err_remove_node:
		3186	drm_mm_remove_node(&vma->node);
		3187	err_free_vma:
		3188	i915_gem_vma_destroy(vma);
5060	serge	3189	vma = ERR_PTR(ret);
4104	Serge	3190	err_unpin:
		3191	i915_gem_object_unpin_pages(obj);
5060	serge	3192	return vma;
2332	Serge	3193	}
		3194
4104	Serge	3195	bool
		3196	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3197	bool force)
2332	Serge	3198	{
		3199	/* If we don't have a page list set up, then we're not pinned
		3200	* to GPU, and we can ignore the cache flush because it'll happen
		3201	* again at bind time.
		3202	*/
3243	Serge	3203	if (obj->pages == NULL)
4104	Serge	3204	return false;
2332	Serge	3205
3480	Serge	3206	/*
		3207	* Stolen memory is always coherent with the GPU as it is explicitly
		3208	* marked as wc by the system, or the system is cache-coherent.
		3209	*/
5354	serge	3210	if (obj->stolen \|\| obj->phys_handle)
4104	Serge	3211	return false;
3480	Serge	3212
2332	Serge	3213	/* If the GPU is snooping the contents of the CPU cache,
		3214	* we do not need to manually clear the CPU cache lines. However,
		3215	* the caches are only snooped when the render cache is
		3216	* flushed/invalidated. As we always have to emit invalidations
		3217	* and flushes when moving into and out of the RENDER domain, correct
		3218	* snooping behaviour occurs naturally as the result of our domain
		3219	* tracking.
		3220	*/
6084	serge	3221	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
		3222	obj->cache_dirty = true;
4104	Serge	3223	return false;
6084	serge	3224	}
2332	Serge	3225
4293	Serge	3226	trace_i915_gem_object_clflush(obj);
		3227	drm_clflush_sg(obj->pages);
6084	serge	3228	obj->cache_dirty = false;
2344	Serge	3229
4104	Serge	3230	return true;
2332	Serge	3231	}
		3232
2344	Serge	3233	/** Flushes the GTT write domain for the object if it's dirty. */
		3234	static void
		3235	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3236	{
		3237	uint32_t old_write_domain;
2332	Serge	3238
2344	Serge	3239	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3240	return;
2332	Serge	3241
2344	Serge	3242	/* No actual flushing is required for the GTT write domain. Writes
		3243	* to it immediately go to main memory as far as we know, so there's
		3244	* no chipset flush. It also doesn't land in render cache.
		3245	*
		3246	* However, we do have to enforce the order so that all writes through
		3247	* the GTT land before any writes to the device, such as updates to
		3248	* the GATT itself.
		3249	*/
		3250	wmb();
2332	Serge	3251
2344	Serge	3252	old_write_domain = obj->base.write_domain;
		3253	obj->base.write_domain = 0;
2332	Serge	3254
6084	serge	3255	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
5354	serge	3256
2351	Serge	3257	trace_i915_gem_object_change_domain(obj,
		3258	obj->base.read_domains,
		3259	old_write_domain);
2344	Serge	3260	}
2332	Serge	3261
		3262	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3263	static void
6084	serge	3264	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2332	Serge	3265	{
		3266	uint32_t old_write_domain;
		3267
		3268	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3269	return;
		3270
6084	serge	3271	if (i915_gem_clflush_object(obj, obj->pin_display))
		3272	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3273
2332	Serge	3274	old_write_domain = obj->base.write_domain;
		3275	obj->base.write_domain = 0;
		3276
6084	serge	3277	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
5354	serge	3278
2351	Serge	3279	trace_i915_gem_object_change_domain(obj,
		3280	obj->base.read_domains,
		3281	old_write_domain);
2332	Serge	3282	}
		3283
		3284	/**
		3285	* Moves a single object to the GTT read, and possibly write domain.
		3286	*
		3287	* This function returns when the move is complete, including waiting on
		3288	* flushes to occur.
		3289	*/
		3290	int
		3291	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3292	{
		3293	uint32_t old_write_domain, old_read_domains;
6084	serge	3294	struct i915_vma *vma;
2332	Serge	3295	int ret;
		3296
		3297	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3298	return 0;
		3299
3031	serge	3300	ret = i915_gem_object_wait_rendering(obj, !write);
6084	serge	3301	if (ret)
		3302	return ret;
2332	Serge	3303
6084	serge	3304	/* Flush and acquire obj->pages so that we are coherent through
		3305	* direct access in memory with previous cached writes through
		3306	* shmemfs and that our cache domain tracking remains valid.
		3307	* For example, if the obj->filp was moved to swap without us
		3308	* being notified and releasing the pages, we would mistakenly
		3309	* continue to assume that the obj remained out of the CPU cached
		3310	* domain.
		3311	*/
		3312	ret = i915_gem_object_get_pages(obj);
		3313	if (ret)
		3314	return ret;
2332	Serge	3315
6084	serge	3316	i915_gem_object_flush_cpu_write_domain(obj);
		3317
3480	Serge	3318	/* Serialise direct access to this object with the barriers for
		3319	* coherent writes from the GPU, by effectively invalidating the
		3320	* GTT domain upon first access.
		3321	*/
		3322	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3323	mb();
		3324
2332	Serge	3325	old_write_domain = obj->base.write_domain;
		3326	old_read_domains = obj->base.read_domains;
		3327
		3328	/* It should now be out of any other write domains, and we can update
		3329	* the domain values for our changes.
		3330	*/
		3331	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3332	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3333	if (write) {
		3334	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3335	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3336	obj->dirty = 1;
		3337	}
		3338
2351	Serge	3339	trace_i915_gem_object_change_domain(obj,
		3340	old_read_domains,
		3341	old_write_domain);
		3342
3031	serge	3343	/* And bump the LRU for this access */
6084	serge	3344	vma = i915_gem_obj_to_ggtt(obj);
		3345	if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
		3346	list_move_tail(&vma->mm_list,
		3347	&to_i915(obj->base.dev)->gtt.base.inactive_list);
3031	serge	3348
2332	Serge	3349	return 0;
		3350	}
		3351
6084	serge	3352	/**
		3353	* Changes the cache-level of an object across all VMA.
		3354	*
		3355	* After this function returns, the object will be in the new cache-level
		3356	* across all GTT and the contents of the backing storage will be coherent,
		3357	* with respect to the new cache-level. In order to keep the backing storage
		3358	* coherent for all users, we only allow a single cache level to be set
		3359	* globally on the object and prevent it from being changed whilst the
		3360	* hardware is reading from the object. That is if the object is currently
		3361	* on the scanout it will be set to uncached (or equivalent display
		3362	* cache coherency) and all non-MOCS GPU access will also be uncached so
		3363	* that all direct access to the scanout remains coherent.
		3364	*/
2335	Serge	3365	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3366	enum i915_cache_level cache_level)
		3367	{
3031	serge	3368	struct drm_device *dev = obj->base.dev;
5060	serge	3369	struct i915_vma vma, next;
6084	serge	3370	bool bound = false;
		3371	int ret = 0;
2332	Serge	3372
2335	Serge	3373	if (obj->cache_level == cache_level)
6084	serge	3374	goto out;
2332	Serge	3375
6084	serge	3376	/* Inspect the list of currently bound VMA and unbind any that would
		3377	* be invalid given the new cache-level. This is principally to
		3378	* catch the issue of the CS prefetch crossing page boundaries and
		3379	* reading an invalid PTE on older architectures.
		3380	*/
		3381	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3382	if (!drm_mm_node_allocated(&vma->node))
		3383	continue;
2332	Serge	3384
6084	serge	3385	if (vma->pin_count) {
		3386	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3387	return -EBUSY;
		3388	}
		3389
5354	serge	3390	if (!i915_gem_valid_gtt_space(vma, cache_level)) {
4104	Serge	3391	ret = i915_vma_unbind(vma);
6084	serge	3392	if (ret)
		3393	return ret;
		3394	} else
		3395	bound = true;
3031	serge	3396	}
		3397
6084	serge	3398	/* We can reuse the existing drm_mm nodes but need to change the
		3399	* cache-level on the PTE. We could simply unbind them all and
		3400	* rebind with the correct cache-level on next use. However since
		3401	* we already have a valid slot, dma mapping, pages etc, we may as
		3402	* rewrite the PTE in the belief that doing so tramples upon less
		3403	* state and so involves less work.
		3404	*/
		3405	if (bound) {
		3406	/* Before we change the PTE, the GPU must not be accessing it.
		3407	* If we wait upon the object, we know that all the bound
		3408	* VMA are no longer active.
		3409	*/
		3410	ret = i915_gem_object_wait_rendering(obj, false);
2335	Serge	3411	if (ret)
		3412	return ret;
2332	Serge	3413
6084	serge	3414	if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
		3415	/* Access to snoopable pages through the GTT is
		3416	* incoherent and on some machines causes a hard
		3417	* lockup. Relinquish the CPU mmaping to force
		3418	* userspace to refault in the pages and we can
		3419	* then double check if the GTT mapping is still
		3420	* valid for that pointer access.
		3421	*/
		3422	i915_gem_release_mmap(obj);
2332	Serge	3423
6084	serge	3424	/* As we no longer need a fence for GTT access,
		3425	* we can relinquish it now (and so prevent having
		3426	* to steal a fence from someone else on the next
		3427	* fence request). Note GPU activity would have
		3428	* dropped the fence as all snoopable access is
		3429	* supposed to be linear.
		3430	*/
2335	Serge	3431	ret = i915_gem_object_put_fence(obj);
		3432	if (ret)
		3433	return ret;
6084	serge	3434	} else {
		3435	/* We either have incoherent backing store and
		3436	* so no GTT access or the architecture is fully
		3437	* coherent. In such cases, existing GTT mmaps
		3438	* ignore the cache bit in the PTE and we can
		3439	* rewrite it without confusing the GPU or having
		3440	* to force userspace to fault back in its mmaps.
		3441	*/
		3442	}
2332	Serge	3443
6084	serge	3444	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3445	if (!drm_mm_node_allocated(&vma->node))
		3446	continue;
		3447
		3448	ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
		3449	if (ret)
		3450	return ret;
		3451	}
2335	Serge	3452	}
2332	Serge	3453
4104	Serge	3454	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3455	vma->node.color = cache_level;
		3456	obj->cache_level = cache_level;
		3457
6084	serge	3458	out:
		3459	/* Flush the dirty CPU caches to the backing storage so that the
		3460	* object is now coherent at its new cache level (with respect
		3461	* to the access domain).
		3462	*/
		3463	if (obj->cache_dirty &&
		3464	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		3465	cpu_write_needs_clflush(obj)) {
		3466	if (i915_gem_clflush_object(obj, true))
		3467	i915_gem_chipset_flush(obj->base.dev);
		3468	}
2332	Serge	3469
2335	Serge	3470	return 0;
		3471	}
2332	Serge	3472
3260	Serge	3473	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3474	struct drm_file *file)
		3475	{
		3476	struct drm_i915_gem_caching *args = data;
		3477	struct drm_i915_gem_object *obj;
		3478
		3479	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
6084	serge	3480	if (&obj->base == NULL)
		3481	return -ENOENT;
3260	Serge	3482
4104	Serge	3483	switch (obj->cache_level) {
		3484	case I915_CACHE_LLC:
		3485	case I915_CACHE_L3_LLC:
		3486	args->caching = I915_CACHING_CACHED;
		3487	break;
3260	Serge	3488
4104	Serge	3489	case I915_CACHE_WT:
		3490	args->caching = I915_CACHING_DISPLAY;
		3491	break;
		3492
		3493	default:
		3494	args->caching = I915_CACHING_NONE;
		3495	break;
		3496	}
		3497
6084	serge	3498	drm_gem_object_unreference_unlocked(&obj->base);
		3499	return 0;
3260	Serge	3500	}
		3501
		3502	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3503	struct drm_file *file)
		3504	{
6084	serge	3505	struct drm_i915_private *dev_priv = dev->dev_private;
3260	Serge	3506	struct drm_i915_gem_caching *args = data;
		3507	struct drm_i915_gem_object *obj;
		3508	enum i915_cache_level level;
		3509	int ret;
		3510
		3511	switch (args->caching) {
		3512	case I915_CACHING_NONE:
		3513	level = I915_CACHE_NONE;
		3514	break;
		3515	case I915_CACHING_CACHED:
6084	serge	3516	/*
		3517	* Due to a HW issue on BXT A stepping, GPU stores via a
		3518	* snooped mapping may leave stale data in a corresponding CPU
		3519	* cacheline, whereas normally such cachelines would get
		3520	* invalidated.
		3521	*/
		3522	if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
		3523	return -ENODEV;
		3524
3260	Serge	3525	level = I915_CACHE_LLC;
		3526	break;
4104	Serge	3527	case I915_CACHING_DISPLAY:
		3528	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3529	break;
3260	Serge	3530	default:
		3531	return -EINVAL;
		3532	}
		3533
6084	serge	3534	intel_runtime_pm_get(dev_priv);
		3535
3260	Serge	3536	ret = i915_mutex_lock_interruptible(dev);
		3537	if (ret)
6084	serge	3538	goto rpm_put;
3260	Serge	3539
		3540	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3541	if (&obj->base == NULL) {
		3542	ret = -ENOENT;
		3543	goto unlock;
		3544	}
		3545
		3546	ret = i915_gem_object_set_cache_level(obj, level);
		3547
		3548	drm_gem_object_unreference(&obj->base);
		3549	unlock:
		3550	mutex_unlock(&dev->struct_mutex);
6084	serge	3551	rpm_put:
		3552	intel_runtime_pm_put(dev_priv);
		3553
3260	Serge	3554	return ret;
		3555	}
		3556
2335	Serge	3557	/*
		3558	* Prepare buffer for display plane (scanout, cursors, etc).
		3559	* Can be called from an uninterruptible phase (modesetting) and allows
		3560	* any flushes to be pipelined (for pageflips).
		3561	*/
		3562	int
		3563	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3564	u32 alignment,
6084	serge	3565	struct intel_engine_cs *pipelined,
		3566	struct drm_i915_gem_request **pipelined_request,
		3567	const struct i915_ggtt_view *view)
2335	Serge	3568	{
		3569	u32 old_read_domains, old_write_domain;
		3570	int ret;
2332	Serge	3571
6084	serge	3572	ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
2335	Serge	3573	if (ret)
		3574	return ret;
2332	Serge	3575
4104	Serge	3576	/* Mark the pin_display early so that we account for the
		3577	* display coherency whilst setting up the cache domains.
		3578	*/
6084	serge	3579	obj->pin_display++;
4104	Serge	3580
2335	Serge	3581	/* The display engine is not coherent with the LLC cache on gen6. As
		3582	* a result, we make sure that the pinning that is about to occur is
		3583	* done with uncached PTEs. This is lowest common denominator for all
		3584	* chipsets.
		3585	*
		3586	* However for gen6+, we could do better by using the GFDT bit instead
		3587	* of uncaching, which would allow us to flush all the LLC-cached data
		3588	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3589	*/
4104	Serge	3590	ret = i915_gem_object_set_cache_level(obj,
		3591	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3592	if (ret)
4104	Serge	3593	goto err_unpin_display;
2332	Serge	3594
2335	Serge	3595	/* As the user may map the buffer once pinned in the display plane
		3596	* (e.g. libkms for the bootup splash), we have to ensure that we
		3597	* always use map_and_fenceable for all scanout buffers.
		3598	*/
6084	serge	3599	ret = i915_gem_object_ggtt_pin(obj, view, alignment,
		3600	view->type == I915_GGTT_VIEW_NORMAL ?
		3601	PIN_MAPPABLE : 0);
2335	Serge	3602	if (ret)
4104	Serge	3603	goto err_unpin_display;
2332	Serge	3604
6084	serge	3605	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3606
2335	Serge	3607	old_write_domain = obj->base.write_domain;
		3608	old_read_domains = obj->base.read_domains;
2332	Serge	3609
2335	Serge	3610	/* It should now be out of any other write domains, and we can update
		3611	* the domain values for our changes.
		3612	*/
3031	serge	3613	obj->base.write_domain = 0;
2335	Serge	3614	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3615
2351	Serge	3616	trace_i915_gem_object_change_domain(obj,
		3617	old_read_domains,
		3618	old_write_domain);
2332	Serge	3619
2335	Serge	3620	return 0;
4104	Serge	3621
		3622	err_unpin_display:
6084	serge	3623	obj->pin_display--;
4104	Serge	3624	return ret;
2335	Serge	3625	}
2332	Serge	3626
4104	Serge	3627	void
6084	serge	3628	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
		3629	const struct i915_ggtt_view *view)
4104	Serge	3630	{
6084	serge	3631	if (WARN_ON(obj->pin_display == 0))
		3632	return;
4104	Serge	3633
6084	serge	3634	i915_gem_object_ggtt_unpin_view(obj, view);
2332	Serge	3635
6084	serge	3636	obj->pin_display--;
2344	Serge	3637	}
2332	Serge	3638
2344	Serge	3639	/**
		3640	* Moves a single object to the CPU read, and possibly write domain.
		3641	*
		3642	* This function returns when the move is complete, including waiting on
		3643	* flushes to occur.
		3644	*/
3031	serge	3645	int
2344	Serge	3646	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3647	{
		3648	uint32_t old_write_domain, old_read_domains;
		3649	int ret;
2332	Serge	3650
2344	Serge	3651	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3652	return 0;
2332	Serge	3653
3031	serge	3654	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3655	if (ret)
		3656	return ret;
2332	Serge	3657
2344	Serge	3658	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3659
2344	Serge	3660	old_write_domain = obj->base.write_domain;
		3661	old_read_domains = obj->base.read_domains;
2332	Serge	3662
2344	Serge	3663	/* Flush the CPU cache if it's still invalid. */
		3664	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3665	i915_gem_clflush_object(obj, false);
2332	Serge	3666
2344	Serge	3667	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3668	}
2332	Serge	3669
2344	Serge	3670	/* It should now be out of any other write domains, and we can update
		3671	* the domain values for our changes.
		3672	*/
		3673	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3674
2344	Serge	3675	/* If we're writing through the CPU, then the GPU read domains will
		3676	* need to be invalidated at next use.
		3677	*/
		3678	if (write) {
		3679	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3680	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3681	}
2332	Serge	3682
2351	Serge	3683	trace_i915_gem_object_change_domain(obj,
		3684	old_read_domains,
		3685	old_write_domain);
2332	Serge	3686
2344	Serge	3687	return 0;
		3688	}
2332	Serge	3689
3031	serge	3690	/* Throttle our rendering by waiting until the ring has completed our requests
		3691	* emitted over 20 msec ago.
2344	Serge	3692	*
3031	serge	3693	* Note that if we were to use the current jiffies each time around the loop,
		3694	* we wouldn't escape the function with any frames outstanding if the time to
		3695	* render a frame was over 20ms.
		3696	*
		3697	* This should get us reasonable parallelism between CPU and GPU but also
		3698	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3699	*/
3031	serge	3700	static int
		3701	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3702	{
3031	serge	3703	struct drm_i915_private *dev_priv = dev->dev_private;
		3704	struct drm_i915_file_private *file_priv = file->driver_priv;
6084	serge	3705	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
		3706	struct drm_i915_gem_request request, target = NULL;
3480	Serge	3707	unsigned reset_counter;
3031	serge	3708	int ret;
2332	Serge	3709
3480	Serge	3710	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3711	if (ret)
		3712	return ret;
2332	Serge	3713
3480	Serge	3714	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3715	if (ret)
		3716	return ret;
		3717
3031	serge	3718	spin_lock(&file_priv->mm.lock);
		3719	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3720	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3721	break;
2332	Serge	3722
6084	serge	3723	/*
		3724	* Note that the request might not have been submitted yet.
		3725	* In which case emitted_jiffies will be zero.
		3726	*/
		3727	if (!request->emitted_jiffies)
		3728	continue;
		3729
		3730	target = request;
3031	serge	3731	}
3480	Serge	3732	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
6084	serge	3733	if (target)
		3734	i915_gem_request_reference(target);
3031	serge	3735	spin_unlock(&file_priv->mm.lock);
2332	Serge	3736
6084	serge	3737	if (target == NULL)
3031	serge	3738	return 0;
2332	Serge	3739
6084	serge	3740	ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
3031	serge	3741	if (ret == 0)
		3742	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3743
6084	serge	3744	i915_gem_request_unreference__unlocked(target);
		3745
3031	serge	3746	return ret;
2352	Serge	3747	}
2332	Serge	3748
5060	serge	3749	static bool
		3750	i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
		3751	{
		3752	struct drm_i915_gem_object *obj = vma->obj;
		3753
		3754	if (alignment &&
		3755	vma->node.start & (alignment - 1))
		3756	return true;
		3757
		3758	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
		3759	return true;
		3760
		3761	if (flags & PIN_OFFSET_BIAS &&
		3762	vma->node.start < (flags & PIN_OFFSET_MASK))
		3763	return true;
		3764
		3765	return false;
		3766	}
		3767
6084	serge	3768	void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
2332	Serge	3769	{
6084	serge	3770	struct drm_i915_gem_object *obj = vma->obj;
		3771	bool mappable, fenceable;
		3772	u32 fence_size, fence_alignment;
		3773
		3774	fence_size = i915_gem_get_gtt_size(obj->base.dev,
		3775	obj->base.size,
		3776	obj->tiling_mode);
		3777	fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
		3778	obj->base.size,
		3779	obj->tiling_mode,
		3780	true);
		3781
		3782	fenceable = (vma->node.size == fence_size &&
		3783	(vma->node.start & (fence_alignment - 1)) == 0);
		3784
		3785	mappable = (vma->node.start + fence_size <=
		3786	to_i915(obj->base.dev)->gtt.mappable_end);
		3787
		3788	obj->map_and_fenceable = mappable && fenceable;
		3789	}
		3790
		3791	static int
		3792	i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
		3793	struct i915_address_space *vm,
		3794	const struct i915_ggtt_view *ggtt_view,
		3795	uint32_t alignment,
		3796	uint64_t flags)
		3797	{
5060	serge	3798	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4104	Serge	3799	struct i915_vma *vma;
5354	serge	3800	unsigned bound;
2332	Serge	3801	int ret;
		3802
5060	serge	3803	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
		3804	return -ENODEV;
2332	Serge	3805
5060	serge	3806	if (WARN_ON(flags & (PIN_GLOBAL \| PIN_MAPPABLE) && !i915_is_ggtt(vm)))
		3807	return -EINVAL;
4104	Serge	3808
5354	serge	3809	if (WARN_ON((flags & (PIN_MAPPABLE \| PIN_GLOBAL)) == PIN_MAPPABLE))
		3810	return -EINVAL;
		3811
6084	serge	3812	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
		3813	return -EINVAL;
		3814
		3815	vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
		3816	i915_gem_obj_to_vma(obj, vm);
		3817
		3818	if (IS_ERR(vma))
		3819	return PTR_ERR(vma);
		3820
5060	serge	3821	if (vma) {
		3822	if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3823	return -EBUSY;
4104	Serge	3824
5060	serge	3825	if (i915_vma_misplaced(vma, alignment, flags)) {
		3826	WARN(vma->pin_count,
6084	serge	3827	"bo is already pinned in %s with incorrect alignment:"
		3828	" offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3829	" obj->map_and_fenceable=%d\n",
6084	serge	3830	ggtt_view ? "ggtt" : "ppgtt",
		3831	upper_32_bits(vma->node.start),
		3832	lower_32_bits(vma->node.start),
		3833	alignment,
5060	serge	3834	!!(flags & PIN_MAPPABLE),
2332	Serge	3835	obj->map_and_fenceable);
4104	Serge	3836	ret = i915_vma_unbind(vma);
2332	Serge	3837	if (ret)
		3838	return ret;
5060	serge	3839
		3840	vma = NULL;
2332	Serge	3841	}
		3842	}
		3843
5354	serge	3844	bound = vma ? vma->bound : 0;
5060	serge	3845	if (vma == NULL \|\| !drm_mm_node_allocated(&vma->node)) {
6084	serge	3846	vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
		3847	flags);
5060	serge	3848	if (IS_ERR(vma))
		3849	return PTR_ERR(vma);
6084	serge	3850	} else {
		3851	ret = i915_vma_bind(vma, obj->cache_level, flags);
		3852	if (ret)
		3853	return ret;
2332	Serge	3854	}
		3855
6084	serge	3856	if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
		3857	(bound ^ vma->bound) & GLOBAL_BIND) {
		3858	__i915_vma_set_map_and_fenceable(vma);
		3859	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
5354	serge	3860	}
		3861
5060	serge	3862	vma->pin_count++;
2332	Serge	3863	return 0;
		3864	}
		3865
6084	serge	3866	int
		3867	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3868	struct i915_address_space *vm,
		3869	uint32_t alignment,
		3870	uint64_t flags)
2344	Serge	3871	{
6084	serge	3872	return i915_gem_object_do_pin(obj, vm,
		3873	i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
		3874	alignment, flags);
2344	Serge	3875	}
2332	Serge	3876
6084	serge	3877	int
		3878	i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
		3879	const struct i915_ggtt_view *view,
		3880	uint32_t alignment,
		3881	uint64_t flags)
5060	serge	3882	{
6084	serge	3883	if (WARN_ONCE(!view, "no view specified"))
		3884	return -EINVAL;
5060	serge	3885
6084	serge	3886	return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
		3887	alignment, flags \| PIN_GLOBAL);
5060	serge	3888	}
		3889
		3890	void
6084	serge	3891	i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
		3892	const struct i915_ggtt_view *view)
5060	serge	3893	{
6084	serge	3894	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
5060	serge	3895
6084	serge	3896	BUG_ON(!vma);
		3897	WARN_ON(vma->pin_count == 0);
		3898	WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
2332	Serge	3899
6084	serge	3900	--vma->pin_count;
3031	serge	3901	}
2332	Serge	3902
3031	serge	3903	int
		3904	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3905	struct drm_file *file)
		3906	{
		3907	struct drm_i915_gem_busy *args = data;
		3908	struct drm_i915_gem_object *obj;
		3909	int ret;
2332	Serge	3910
3031	serge	3911	ret = i915_mutex_lock_interruptible(dev);
		3912	if (ret)
		3913	return ret;
2332	Serge	3914
5060	serge	3915	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3916	if (&obj->base == NULL) {
		3917	ret = -ENOENT;
		3918	goto unlock;
		3919	}
2332	Serge	3920
3031	serge	3921	/* Count all active objects as busy, even if they are currently not used
		3922	* by the gpu. Users of this interface expect objects to eventually
		3923	* become non-busy without any further actions, therefore emit any
		3924	* necessary flushes here.
		3925	*/
		3926	ret = i915_gem_object_flush_active(obj);
6084	serge	3927	if (ret)
		3928	goto unref;
2332	Serge	3929
6084	serge	3930	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3931	args->busy = obj->active << 16;
		3932	if (obj->last_write_req)
		3933	args->busy \|= obj->last_write_req->ring->id;
2332	Serge	3934
6084	serge	3935	unref:
3031	serge	3936	drm_gem_object_unreference(&obj->base);
		3937	unlock:
		3938	mutex_unlock(&dev->struct_mutex);
		3939	return ret;
		3940	}
2332	Serge	3941
3031	serge	3942	int
		3943	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3944	struct drm_file *file_priv)
		3945	{
		3946	return i915_gem_ring_throttle(dev, file_priv);
		3947	}
2332	Serge	3948
3263	Serge	3949	#if 0
		3950
3031	serge	3951	int
		3952	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3953	struct drm_file *file_priv)
		3954	{
5354	serge	3955	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3956	struct drm_i915_gem_madvise *args = data;
		3957	struct drm_i915_gem_object *obj;
		3958	int ret;
2332	Serge	3959
3031	serge	3960	switch (args->madv) {
		3961	case I915_MADV_DONTNEED:
		3962	case I915_MADV_WILLNEED:
		3963	break;
		3964	default:
		3965	return -EINVAL;
		3966	}
2332	Serge	3967
3031	serge	3968	ret = i915_mutex_lock_interruptible(dev);
		3969	if (ret)
		3970	return ret;
2332	Serge	3971
3031	serge	3972	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3973	if (&obj->base == NULL) {
		3974	ret = -ENOENT;
		3975	goto unlock;
		3976	}
2332	Serge	3977
5060	serge	3978	if (i915_gem_obj_is_pinned(obj)) {
3031	serge	3979	ret = -EINVAL;
		3980	goto out;
		3981	}
2332	Serge	3982
5354	serge	3983	if (obj->pages &&
		3984	obj->tiling_mode != I915_TILING_NONE &&
		3985	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		3986	if (obj->madv == I915_MADV_WILLNEED)
		3987	i915_gem_object_unpin_pages(obj);
		3988	if (args->madv == I915_MADV_WILLNEED)
		3989	i915_gem_object_pin_pages(obj);
		3990	}
		3991
3031	serge	3992	if (obj->madv != __I915_MADV_PURGED)
		3993	obj->madv = args->madv;
2332	Serge	3994
3031	serge	3995	/* if the object is no longer attached, discard its backing storage */
6084	serge	3996	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
3031	serge	3997	i915_gem_object_truncate(obj);
2332	Serge	3998
3031	serge	3999	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	4000
3031	serge	4001	out:
		4002	drm_gem_object_unreference(&obj->base);
		4003	unlock:
		4004	mutex_unlock(&dev->struct_mutex);
		4005	return ret;
		4006	}
		4007	#endif
2332	Serge	4008
3031	serge	4009	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		4010	const struct drm_i915_gem_object_ops *ops)
		4011	{
6084	serge	4012	int i;
		4013
4104	Serge	4014	INIT_LIST_HEAD(&obj->global_list);
6084	serge	4015	for (i = 0; i < I915_NUM_RINGS; i++)
		4016	INIT_LIST_HEAD(&obj->ring_list[i]);
4104	Serge	4017	INIT_LIST_HEAD(&obj->obj_exec_link);
		4018	INIT_LIST_HEAD(&obj->vma_list);
6084	serge	4019	INIT_LIST_HEAD(&obj->batch_pool_link);
2332	Serge	4020
3031	serge	4021	obj->ops = ops;
		4022
		4023	obj->fence_reg = I915_FENCE_REG_NONE;
		4024	obj->madv = I915_MADV_WILLNEED;
		4025
		4026	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		4027	}
		4028
		4029	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		4030	.get_pages = i915_gem_object_get_pages_gtt,
		4031	.put_pages = i915_gem_object_put_pages_gtt,
		4032	};
		4033
2332	Serge	4034	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		4035	size_t size)
		4036	{
		4037	struct drm_i915_gem_object *obj;
3031	serge	4038	struct address_space *mapping;
3480	Serge	4039	gfp_t mask;
2340	Serge	4040
3746	Serge	4041	obj = i915_gem_object_alloc(dev);
2332	Serge	4042	if (obj == NULL)
		4043	return NULL;
		4044
		4045	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	4046	i915_gem_object_free(obj);
2332	Serge	4047	return NULL;
		4048	}
		4049
		4050
3031	serge	4051	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	4052
		4053	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		4054	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		4055
3031	serge	4056	if (HAS_LLC(dev)) {
		4057	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	4058	* cache) for about a 10% performance improvement
		4059	* compared to uncached. Graphics requests other than
		4060	* display scanout are coherent with the CPU in
		4061	* accessing this cache. This means in this mode we
		4062	* don't need to clflush on the CPU side, and on the
		4063	* GPU side we only need to flush internal caches to
		4064	* get data visible to the CPU.
		4065	*
		4066	* However, we maintain the display planes as UC, and so
		4067	* need to rebind when first used as such.
		4068	*/
		4069	obj->cache_level = I915_CACHE_LLC;
		4070	} else
		4071	obj->cache_level = I915_CACHE_NONE;
		4072
4560	Serge	4073	trace_i915_gem_object_create(obj);
		4074
2332	Serge	4075	return obj;
		4076	}
		4077
6283	serge	4078	static bool discard_backing_storage(struct drm_i915_gem_object *obj)
		4079	{
		4080	/* If we are the last user of the backing storage (be it shmemfs
		4081	* pages or stolen etc), we know that the pages are going to be
		4082	* immediately released. In this case, we can then skip copying
		4083	* back the contents from the GPU.
		4084	*/
		4085
		4086	if (obj->madv != I915_MADV_WILLNEED)
		4087	return false;
		4088
		4089	if (obj->base.filp == NULL)
		4090	return true;
		4091
		4092	// printf("filp %p\n", obj->base.filp);
		4093	shmem_file_delete(obj->base.filp);
		4094	return true;
		4095	}
		4096
3031	serge	4097	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	4098	{
3031	serge	4099	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	4100	struct drm_device *dev = obj->base.dev;
5060	serge	4101	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	4102	struct i915_vma vma, next;
2332	Serge	4103
4560	Serge	4104	intel_runtime_pm_get(dev_priv);
		4105
3031	serge	4106	trace_i915_gem_object_destroy(obj);
		4107
5060	serge	4108	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		4109	int ret;
3031	serge	4110
5060	serge	4111	vma->pin_count = 0;
		4112	ret = i915_vma_unbind(vma);
4104	Serge	4113	if (WARN_ON(ret == -ERESTARTSYS)) {
6084	serge	4114	bool was_interruptible;
3031	serge	4115
6084	serge	4116	was_interruptible = dev_priv->mm.interruptible;
		4117	dev_priv->mm.interruptible = false;
3031	serge	4118
4104	Serge	4119	WARN_ON(i915_vma_unbind(vma));
3031	serge	4120
6084	serge	4121	dev_priv->mm.interruptible = was_interruptible;
		4122	}
2344	Serge	4123	}
2332	Serge	4124
4104	Serge	4125	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		4126	* before progressing. */
		4127	if (obj->stolen)
		4128	i915_gem_object_unpin_pages(obj);
		4129
5060	serge	4130	WARN_ON(obj->frontbuffer_bits);
		4131
5354	serge	4132	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
		4133	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
		4134	obj->tiling_mode != I915_TILING_NONE)
		4135	i915_gem_object_unpin_pages(obj);
		4136
4104	Serge	4137	if (WARN_ON(obj->pages_pin_count))
6084	serge	4138	obj->pages_pin_count = 0;
6283	serge	4139	if (discard_backing_storage(obj))
		4140	obj->madv = I915_MADV_DONTNEED;
3031	serge	4141	i915_gem_object_put_pages(obj);
		4142	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	4143
3243	Serge	4144	BUG_ON(obj->pages);
2332	Serge	4145
6283	serge	4146	if (obj->ops->release)
		4147	obj->ops->release(obj);
3031	serge	4148
2344	Serge	4149	drm_gem_object_release(&obj->base);
		4150	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	4151
2344	Serge	4152	kfree(obj->bit_17);
4104	Serge	4153	i915_gem_object_free(obj);
4560	Serge	4154
		4155	intel_runtime_pm_put(dev_priv);
2344	Serge	4156	}
2332	Serge	4157
4560	Serge	4158	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
4104	Serge	4159	struct i915_address_space *vm)
		4160	{
4560	Serge	4161	struct i915_vma *vma;
6084	serge	4162	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		4163	if (i915_is_ggtt(vma->vm) &&
		4164	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4165	continue;
4560	Serge	4166	if (vma->vm == vm)
		4167	return vma;
6084	serge	4168	}
		4169	return NULL;
		4170	}
4560	Serge	4171
6084	serge	4172	struct i915_vma i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object obj,
		4173	const struct i915_ggtt_view *view)
		4174	{
		4175	struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
		4176	struct i915_vma *vma;
		4177
		4178	if (WARN_ONCE(!view, "no view specified"))
		4179	return ERR_PTR(-EINVAL);
		4180
		4181	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4182	if (vma->vm == ggtt &&
		4183	i915_ggtt_view_equal(&vma->ggtt_view, view))
		4184	return vma;
4560	Serge	4185	return NULL;
		4186	}
		4187
4104	Serge	4188	void i915_gem_vma_destroy(struct i915_vma *vma)
		4189	{
5354	serge	4190	struct i915_address_space *vm = NULL;
4104	Serge	4191	WARN_ON(vma->node.allocated);
4560	Serge	4192
		4193	/* Keep the vma as a placeholder in the execbuffer reservation lists */
		4194	if (!list_empty(&vma->exec_list))
		4195	return;
		4196
5354	serge	4197	vm = vma->vm;
		4198
		4199	if (!i915_is_ggtt(vm))
		4200	i915_ppgtt_put(i915_vm_to_ppgtt(vm));
		4201
4104	Serge	4202	list_del(&vma->vma_link);
4560	Serge	4203
4104	Serge	4204	kfree(vma);
		4205	}
		4206
6084	serge	4207	static void
		4208	i915_gem_stop_ringbuffers(struct drm_device *dev)
		4209	{
		4210	struct drm_i915_private *dev_priv = dev->dev_private;
		4211	struct intel_engine_cs *ring;
		4212	int i;
		4213
		4214	for_each_ring(ring, dev_priv, i)
		4215	dev_priv->gt.stop_ring(ring);
		4216	}
		4217
3031	serge	4218	#if 0
		4219	int
4560	Serge	4220	i915_gem_suspend(struct drm_device *dev)
2344	Serge	4221	{
5060	serge	4222	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4223	int ret = 0;
2332	Serge	4224
4560	Serge	4225	mutex_lock(&dev->struct_mutex);
3031	serge	4226	ret = i915_gpu_idle(dev);
4560	Serge	4227	if (ret)
		4228	goto err;
		4229
3031	serge	4230	i915_gem_retire_requests(dev);
		4231
5060	serge	4232	i915_gem_stop_ringbuffers(dev);
4560	Serge	4233	mutex_unlock(&dev->struct_mutex);
		4234
6084	serge	4235	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
3263	Serge	4236	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
5060	serge	4237	flush_delayed_work(&dev_priv->mm.idle_work);
3031	serge	4238
6084	serge	4239	/* Assert that we sucessfully flushed all the work and
		4240	* reset the GPU back to its idle, low power state.
		4241	*/
		4242	WARN_ON(dev_priv->mm.busy);
		4243
3031	serge	4244	return 0;
4560	Serge	4245
		4246	err:
		4247	mutex_unlock(&dev->struct_mutex);
		4248	return ret;
2344	Serge	4249	}
3031	serge	4250	#endif
2332	Serge	4251
6084	serge	4252	int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
3031	serge	4253	{
6084	serge	4254	struct intel_engine_cs *ring = req->ring;
4560	Serge	4255	struct drm_device *dev = ring->dev;
5060	serge	4256	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4257	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
		4258	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
		4259	int i, ret;
2332	Serge	4260
4560	Serge	4261	if (!HAS_L3_DPF(dev) \|\| !remap_info)
		4262	return 0;
2332	Serge	4263
6084	serge	4264	ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
4560	Serge	4265	if (ret)
		4266	return ret;
2332	Serge	4267
4560	Serge	4268	/*
		4269	* Note: We do not worry about the concurrent register cacheline hang
		4270	* here because no other code should access these registers other than
		4271	* at initialization time.
		4272	*/
3031	serge	4273	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4560	Serge	4274	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		4275	intel_ring_emit(ring, reg_base + i);
		4276	intel_ring_emit(ring, remap_info[i/4]);
3031	serge	4277	}
2332	Serge	4278
4560	Serge	4279	intel_ring_advance(ring);
2332	Serge	4280
4560	Serge	4281	return ret;
3031	serge	4282	}
2332	Serge	4283
3031	serge	4284	void i915_gem_init_swizzling(struct drm_device *dev)
		4285	{
5060	serge	4286	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4287
3031	serge	4288	if (INTEL_INFO(dev)->gen < 5 \|\|
		4289	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4290	return;
2332	Serge	4291
3031	serge	4292	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4293	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4294
3031	serge	4295	if (IS_GEN5(dev))
		4296	return;
2344	Serge	4297
3031	serge	4298	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4299	if (IS_GEN6(dev))
		4300	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4301	else if (IS_GEN7(dev))
		4302	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4560	Serge	4303	else if (IS_GEN8(dev))
		4304	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
3031	serge	4305	else
3480	Serge	4306	BUG();
3031	serge	4307	}
		4308
5354	serge	4309	static void init_unused_ring(struct drm_device *dev, u32 base)
2332	Serge	4310	{
3480	Serge	4311	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4312
		4313	I915_WRITE(RING_CTL(base), 0);
		4314	I915_WRITE(RING_HEAD(base), 0);
		4315	I915_WRITE(RING_TAIL(base), 0);
		4316	I915_WRITE(RING_START(base), 0);
		4317	}
		4318
		4319	static void init_unused_rings(struct drm_device *dev)
		4320	{
		4321	if (IS_I830(dev)) {
		4322	init_unused_ring(dev, PRB1_BASE);
		4323	init_unused_ring(dev, SRB0_BASE);
		4324	init_unused_ring(dev, SRB1_BASE);
		4325	init_unused_ring(dev, SRB2_BASE);
		4326	init_unused_ring(dev, SRB3_BASE);
		4327	} else if (IS_GEN2(dev)) {
		4328	init_unused_ring(dev, SRB0_BASE);
		4329	init_unused_ring(dev, SRB1_BASE);
		4330	} else if (IS_GEN3(dev)) {
		4331	init_unused_ring(dev, PRB1_BASE);
		4332	init_unused_ring(dev, PRB2_BASE);
		4333	}
		4334	}
		4335
		4336	int i915_gem_init_rings(struct drm_device *dev)
		4337	{
		4338	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4339	int ret;
2351	Serge	4340
2332	Serge	4341	ret = intel_init_render_ring_buffer(dev);
		4342	if (ret)
		4343	return ret;
		4344
6084	serge	4345	if (HAS_BSD(dev)) {
2332	Serge	4346	ret = intel_init_bsd_ring_buffer(dev);
		4347	if (ret)
		4348	goto cleanup_render_ring;
		4349	}
		4350
6084	serge	4351	if (HAS_BLT(dev)) {
2332	Serge	4352	ret = intel_init_blt_ring_buffer(dev);
		4353	if (ret)
		4354	goto cleanup_bsd_ring;
		4355	}
		4356
4104	Serge	4357	if (HAS_VEBOX(dev)) {
		4358	ret = intel_init_vebox_ring_buffer(dev);
		4359	if (ret)
		4360	goto cleanup_blt_ring;
		4361	}
		4362
5060	serge	4363	if (HAS_BSD2(dev)) {
		4364	ret = intel_init_bsd2_ring_buffer(dev);
		4365	if (ret)
		4366	goto cleanup_vebox_ring;
		4367	}
4104	Serge	4368
2332	Serge	4369	return 0;
		4370
4104	Serge	4371	cleanup_vebox_ring:
		4372	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4373	cleanup_blt_ring:
		4374	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4375	cleanup_bsd_ring:
		4376	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4377	cleanup_render_ring:
		4378	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4379
2332	Serge	4380	return ret;
		4381	}
		4382
3480	Serge	4383	int
		4384	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4385	{
5060	serge	4386	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4387	struct intel_engine_cs *ring;
		4388	int ret, i, j;
3031	serge	4389
3480	Serge	4390	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4391	return -EIO;
3031	serge	4392
6084	serge	4393	/* Double layer security blanket, see i915_gem_init() */
		4394	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4395
4104	Serge	4396	if (dev_priv->ellc_size)
		4397	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4398
4560	Serge	4399	if (IS_HASWELL(dev))
		4400	I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
		4401	LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
		4402
3746	Serge	4403	if (HAS_PCH_NOP(dev)) {
5060	serge	4404	if (IS_IVYBRIDGE(dev)) {
6084	serge	4405	u32 temp = I915_READ(GEN7_MSG_CTL);
		4406	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4407	I915_WRITE(GEN7_MSG_CTL, temp);
5060	serge	4408	} else if (INTEL_INFO(dev)->gen >= 7) {
		4409	u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
		4410	temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
		4411	I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
		4412	}
3746	Serge	4413	}
		4414
3480	Serge	4415	i915_gem_init_swizzling(dev);
		4416
6084	serge	4417	/*
		4418	* At least 830 can leave some of the unused rings
		4419	* "active" (ie. head != tail) after resume which
		4420	* will prevent c3 entry. Makes sure all unused rings
		4421	* are totally idle.
		4422	*/
		4423	init_unused_rings(dev);
3480	Serge	4424
6084	serge	4425	BUG_ON(!dev_priv->ring[RCS].default_context);
4560	Serge	4426
6084	serge	4427	ret = i915_ppgtt_init_hw(dev);
		4428	if (ret) {
		4429	DRM_ERROR("PPGTT enable HW failed %d\n", ret);
		4430	goto out;
		4431	}
		4432
		4433	/* Need to do basic initialisation of all rings first: */
		4434	for_each_ring(ring, dev_priv, i) {
		4435	ret = ring->init_hw(ring);
		4436	if (ret)
		4437	goto out;
		4438	}
		4439
		4440	/* We can't enable contexts until all firmware is loaded */
		4441	if (HAS_GUC_UCODE(dev)) {
		4442	ret = intel_guc_ucode_load(dev);
		4443	if (ret) {
		4444	/*
		4445	* If we got an error and GuC submission is enabled, map
		4446	* the error to -EIO so the GPU will be declared wedged.
		4447	* OTOH, if we didn't intend to use the GuC anyway, just
		4448	* discard the error and carry on.
		4449	*/
		4450	DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
		4451	i915.enable_guc_submission ? "" :
		4452	" (ignored)");
		4453	ret = i915.enable_guc_submission ? -EIO : 0;
		4454	if (ret)
		4455	goto out;
		4456	}
		4457	}
		4458
3480	Serge	4459	/*
6084	serge	4460	* Increment the next seqno by 0x100 so we have a visible break
		4461	* on re-initialisation
3480	Serge	4462	*/
6084	serge	4463	ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
		4464	if (ret)
		4465	goto out;
5354	serge	4466
6084	serge	4467	/* Now it is safe to go back round and do everything else: */
		4468	for_each_ring(ring, dev_priv, i) {
		4469	struct drm_i915_gem_request *req;
4560	Serge	4470
6084	serge	4471	WARN_ON(!ring->default_context);
		4472
		4473	ret = i915_gem_request_alloc(ring, ring->default_context, &req);
		4474	if (ret) {
		4475	i915_gem_cleanup_ringbuffer(dev);
		4476	goto out;
		4477	}
		4478
		4479	if (ring->id == RCS) {
		4480	for (j = 0; j < NUM_L3_SLICES(dev); j++)
		4481	i915_gem_l3_remap(req, j);
		4482	}
		4483
		4484	ret = i915_ppgtt_init_ring(req);
		4485	if (ret && ret != -EIO) {
		4486	DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
		4487	i915_gem_request_cancel(req);
		4488	i915_gem_cleanup_ringbuffer(dev);
		4489	goto out;
		4490	}
		4491
		4492	ret = i915_gem_context_enable(req);
		4493	if (ret && ret != -EIO) {
		4494	DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
		4495	i915_gem_request_cancel(req);
		4496	i915_gem_cleanup_ringbuffer(dev);
		4497	goto out;
		4498	}
		4499
		4500	i915_add_request_no_flush(req);
5354	serge	4501	}
		4502
6084	serge	4503	out:
		4504	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	4505	return ret;
3031	serge	4506	}
		4507
		4508	int i915_gem_init(struct drm_device *dev)
		4509	{
		4510	struct drm_i915_private *dev_priv = dev->dev_private;
		4511	int ret;
		4512
5354	serge	4513	i915.enable_execlists = intel_sanitize_enable_execlists(dev,
		4514	i915.enable_execlists);
		4515
3031	serge	4516	mutex_lock(&dev->struct_mutex);
3746	Serge	4517
		4518	if (IS_VALLEYVIEW(dev)) {
		4519	/* VLVA0 (potential hack), BIOS isn't actually waking us */
5060	serge	4520	I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
		4521	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
		4522	VLV_GTLC_ALLOWWAKEACK), 10))
3746	Serge	4523	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4524	}
		4525
5354	serge	4526	if (!i915.enable_execlists) {
6084	serge	4527	dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
5354	serge	4528	dev_priv->gt.init_rings = i915_gem_init_rings;
		4529	dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
		4530	dev_priv->gt.stop_ring = intel_stop_ring_buffer;
		4531	} else {
6084	serge	4532	dev_priv->gt.execbuf_submit = intel_execlists_submission;
5354	serge	4533	dev_priv->gt.init_rings = intel_logical_rings_init;
		4534	dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
		4535	dev_priv->gt.stop_ring = intel_logical_ring_stop;
		4536	}
		4537
6084	serge	4538	/* This is just a security blanket to placate dragons.
		4539	* On some systems, we very sporadically observe that the first TLBs
		4540	* used by the CS may be stale, despite us poking the TLB reset. If
		4541	* we hold the forcewake during initialisation these problems
		4542	* just magically go away.
		4543	*/
		4544	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4545
6084	serge	4546	// ret = i915_gem_init_userptr(dev);
		4547	// if (ret)
		4548	// goto out_unlock;
3746	Serge	4549
6084	serge	4550	i915_gem_init_global_gtt(dev);
		4551
5060	serge	4552	ret = i915_gem_context_init(dev);
6084	serge	4553	if (ret)
		4554	goto out_unlock;
3031	serge	4555
6084	serge	4556	ret = dev_priv->gt.init_rings(dev);
		4557	if (ret)
		4558	goto out_unlock;
		4559
5060	serge	4560	ret = i915_gem_init_hw(dev);
		4561	if (ret == -EIO) {
		4562	/* Allow ring initialisation to fail by marking the GPU as
		4563	* wedged. But we only want to do this where the GPU is angry,
		4564	* for all other failure, such as an allocation failure, bail.
		4565	*/
		4566	DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
6084	serge	4567	atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
5060	serge	4568	ret = 0;
		4569	}
6084	serge	4570
		4571	out_unlock:
		4572	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	4573	mutex_unlock(&dev->struct_mutex);
3746	Serge	4574
6084	serge	4575	return ret;
3031	serge	4576	}
		4577
2332	Serge	4578	void
		4579	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4580	{
5060	serge	4581	struct drm_i915_private *dev_priv = dev->dev_private;
		4582	struct intel_engine_cs *ring;
2332	Serge	4583	int i;
		4584
3031	serge	4585	for_each_ring(ring, dev_priv, i)
5354	serge	4586	dev_priv->gt.cleanup_ring(ring);
2332	Serge	4587	}
		4588
		4589	static void
5060	serge	4590	init_ring_lists(struct intel_engine_cs *ring)
2326	Serge	4591	{
6084	serge	4592	INIT_LIST_HEAD(&ring->active_list);
		4593	INIT_LIST_HEAD(&ring->request_list);
2326	Serge	4594	}
		4595
		4596	void
		4597	i915_gem_load(struct drm_device *dev)
		4598	{
5060	serge	4599	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4600	int i;
2326	Serge	4601
4104	Serge	4602	INIT_LIST_HEAD(&dev_priv->vm_list);
4560	Serge	4603	INIT_LIST_HEAD(&dev_priv->context_list);
3031	serge	4604	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4605	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
6084	serge	4606	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4607	for (i = 0; i < I915_NUM_RINGS; i++)
		4608	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4609	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
6084	serge	4610	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4611	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4612	i915_gem_retire_work_handler);
4560	Serge	4613	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
		4614	i915_gem_idle_work_handler);
3480	Serge	4615	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4616
6084	serge	4617	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
2326	Serge	4618
3746	Serge	4619	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4620	dev_priv->num_fence_regs = 32;
		4621	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
6084	serge	4622	dev_priv->num_fence_regs = 16;
		4623	else
		4624	dev_priv->num_fence_regs = 8;
2326	Serge	4625
6084	serge	4626	if (intel_vgpu_active(dev))
		4627	dev_priv->num_fence_regs =
		4628	I915_READ(vgtif_reg(avail_rs.fence_num));
		4629
		4630	/*
		4631	* Set initial sequence number for requests.
		4632	* Using this number allows the wraparound to happen early,
		4633	* catching any obvious problems.
		4634	*/
		4635	dev_priv->next_seqno = ((u32)~0 - 0x1100);
		4636	dev_priv->last_seqno = ((u32)~0 - 0x1101);
		4637
		4638	/* Initialize fence registers to zero */
3746	Serge	4639	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4640	i915_gem_restore_fences(dev);
2326	Serge	4641
6084	serge	4642	i915_gem_detect_bit_6_swizzle(dev);
2326	Serge	4643
6084	serge	4644	dev_priv->mm.interruptible = true;
2326	Serge	4645
5060	serge	4646	mutex_init(&dev_priv->fb_tracking.lock);
2326	Serge	4647	}
		4648
6084	serge	4649	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4650	{
		4651	struct drm_i915_file_private *file_priv = file->driver_priv;
		4652
		4653	/* Clean up our request list when the client is going away, so that
		4654	* later retire_requests won't dereference our soon-to-be-gone
		4655	* file_priv.
		4656	*/
		4657	spin_lock(&file_priv->mm.lock);
		4658	while (!list_empty(&file_priv->mm.request_list)) {
		4659	struct drm_i915_gem_request *request;
		4660
		4661	request = list_first_entry(&file_priv->mm.request_list,
		4662	struct drm_i915_gem_request,
		4663	client_list);
		4664	list_del(&request->client_list);
		4665	request->file_priv = NULL;
		4666	}
		4667	spin_unlock(&file_priv->mm.lock);
		4668
		4669	if (!list_empty(&file_priv->rps.link)) {
		4670	spin_lock(&to_i915(dev)->rps.client_lock);
		4671	list_del(&file_priv->rps.link);
		4672	spin_unlock(&to_i915(dev)->rps.client_lock);
		4673	}
		4674	}
		4675
5060	serge	4676	int i915_gem_open(struct drm_device dev, struct drm_file file)
4104	Serge	4677	{
5060	serge	4678	struct drm_i915_file_private *file_priv;
4104	Serge	4679	int ret;
2326	Serge	4680
5060	serge	4681	DRM_DEBUG_DRIVER("\n");
4104	Serge	4682
5060	serge	4683	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
		4684	if (!file_priv)
4104	Serge	4685	return -ENOMEM;
		4686
5060	serge	4687	file->driver_priv = file_priv;
		4688	file_priv->dev_priv = dev->dev_private;
		4689	file_priv->file = file;
6084	serge	4690	INIT_LIST_HEAD(&file_priv->rps.link);
4104	Serge	4691
5060	serge	4692	spin_lock_init(&file_priv->mm.lock);
		4693	INIT_LIST_HEAD(&file_priv->mm.request_list);
4104	Serge	4694
5060	serge	4695	ret = i915_gem_context_open(dev, file);
		4696	if (ret)
		4697	kfree(file_priv);
4104	Serge	4698
		4699	return ret;
		4700	}
		4701
5354	serge	4702	/**
		4703	* i915_gem_track_fb - update frontbuffer tracking
6084	serge	4704	* @old: current GEM buffer for the frontbuffer slots
		4705	* @new: new GEM buffer for the frontbuffer slots
		4706	* @frontbuffer_bits: bitmask of frontbuffer slots
5354	serge	4707	*
		4708	* This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
		4709	* from @old and setting them in @new. Both @old and @new can be NULL.
		4710	*/
5060	serge	4711	void i915_gem_track_fb(struct drm_i915_gem_object *old,
		4712	struct drm_i915_gem_object *new,
		4713	unsigned frontbuffer_bits)
4104	Serge	4714	{
5060	serge	4715	if (old) {
		4716	WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
		4717	WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
		4718	old->frontbuffer_bits &= ~frontbuffer_bits;
4104	Serge	4719	}
		4720
5060	serge	4721	if (new) {
		4722	WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
		4723	WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
		4724	new->frontbuffer_bits \|= frontbuffer_bits;
4104	Serge	4725	}
		4726	}
		4727
		4728	/* All the new VM stuff */
6084	serge	4729	u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4730	struct i915_address_space *vm)
4104	Serge	4731	{
		4732	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4733	struct i915_vma *vma;
		4734
5354	serge	4735	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4104	Serge	4736
		4737	list_for_each_entry(vma, &o->vma_list, vma_link) {
6084	serge	4738	if (i915_is_ggtt(vma->vm) &&
		4739	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4740	continue;
4104	Serge	4741	if (vma->vm == vm)
		4742	return vma->node.start;
6084	serge	4743	}
4104	Serge	4744
5060	serge	4745	WARN(1, "%s vma for this object not found.\n",
		4746	i915_is_ggtt(vm) ? "global" : "ppgtt");
		4747	return -1;
4104	Serge	4748	}
		4749
6084	serge	4750	u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
		4751	const struct i915_ggtt_view *view)
		4752	{
		4753	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
		4754	struct i915_vma *vma;
		4755
		4756	list_for_each_entry(vma, &o->vma_list, vma_link)
		4757	if (vma->vm == ggtt &&
		4758	i915_ggtt_view_equal(&vma->ggtt_view, view))
		4759	return vma->node.start;
		4760
		4761	WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
		4762	return -1;
		4763	}
		4764
4104	Serge	4765	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4766	struct i915_address_space *vm)
		4767	{
		4768	struct i915_vma *vma;
		4769
6084	serge	4770	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4771	if (i915_is_ggtt(vma->vm) &&
		4772	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4773	continue;
4104	Serge	4774	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4775	return true;
6084	serge	4776	}
4104	Serge	4777
		4778	return false;
		4779	}
		4780
6084	serge	4781	bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
		4782	const struct i915_ggtt_view *view)
		4783	{
		4784	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
		4785	struct i915_vma *vma;
		4786
		4787	list_for_each_entry(vma, &o->vma_list, vma_link)
		4788	if (vma->vm == ggtt &&
		4789	i915_ggtt_view_equal(&vma->ggtt_view, view) &&
		4790	drm_mm_node_allocated(&vma->node))
		4791	return true;
		4792
		4793	return false;
		4794	}
		4795
4104	Serge	4796	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4797	{
4560	Serge	4798	struct i915_vma *vma;
4104	Serge	4799
4560	Serge	4800	list_for_each_entry(vma, &o->vma_list, vma_link)
		4801	if (drm_mm_node_allocated(&vma->node))
4104	Serge	4802	return true;
		4803
		4804	return false;
		4805	}
		4806
		4807	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4808	struct i915_address_space *vm)
		4809	{
		4810	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4811	struct i915_vma *vma;
		4812
5354	serge	4813	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4104	Serge	4814
		4815	BUG_ON(list_empty(&o->vma_list));
		4816
6084	serge	4817	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4818	if (i915_is_ggtt(vma->vm) &&
		4819	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4820	continue;
4104	Serge	4821	if (vma->vm == vm)
		4822	return vma->node.size;
6084	serge	4823	}
4104	Serge	4824	return 0;
		4825	}
4560	Serge	4826
6084	serge	4827	bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
		4828	{
		4829	struct i915_vma *vma;
		4830	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4831	if (vma->pin_count > 0)
		4832	return true;
4560	Serge	4833
6084	serge	4834	return false;
		4835	}
5060	serge	4836
6084	serge	4837	/* Allocate a new GEM object and fill it with the supplied data */
		4838	struct drm_i915_gem_object *
		4839	i915_gem_object_create_from_data(struct drm_device *dev,
		4840	const void *data, size_t size)
4104	Serge	4841	{
6084	serge	4842	struct drm_i915_gem_object *obj;
		4843	struct sg_table *sg;
		4844	size_t bytes;
		4845	int ret;
4104	Serge	4846
6084	serge	4847	obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
		4848	if (IS_ERR_OR_NULL(obj))
		4849	return obj;
4104	Serge	4850
6084	serge	4851	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		4852	if (ret)
		4853	goto fail;
		4854
		4855	ret = i915_gem_object_get_pages(obj);
		4856	if (ret)
		4857	goto fail;
		4858
		4859	i915_gem_object_pin_pages(obj);
		4860	sg = obj->pages;
		4861	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
		4862	i915_gem_object_unpin_pages(obj);
		4863
		4864	if (WARN_ON(bytes != size)) {
		4865	DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
		4866	ret = -EFAULT;
		4867	goto fail;
		4868	}
		4869
		4870	return obj;
		4871
		4872	fail:
		4873	drm_gem_object_unreference(&obj->base);
		4874	return ERR_PTR(ret);
4104	Serge	4875	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 6296