WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
3260	Serge	34	#include
2330	Serge	35	#include
2326	Serge	36	//#include
3746	Serge	37	#include
2326	Serge	38	#include
		39
2344	Serge	40	extern int x86_clflush_size;
2332	Serge	41
3263	Serge	42	#define PROT_READ 0x1 /* page can be read */
		43	#define PROT_WRITE 0x2 /* page can be written */
		44	#define MAP_SHARED 0x01 /* Share changes */
		45
2344	Serge	46
5060	serge	47	u64 nsecs_to_jiffies64(u64 n)
		48	{
		49	#if (NSEC_PER_SEC % HZ) == 0
		50	/* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */
		51	return div_u64(n, NSEC_PER_SEC / HZ);
		52	#elif (HZ % 512) == 0
		53	/* overflow after 292 years if HZ = 1024 */
		54	return div_u64(n * HZ / 512, NSEC_PER_SEC / 512);
		55	#else
		56	/*
		57	* Generic case - optimized for cases where HZ is a multiple of 3.
		58	* overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc.
		59	*/
		60	return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ);
		61	#endif
		62	}
		63
		64	unsigned long nsecs_to_jiffies(u64 n)
		65	{
		66	return (unsigned long)nsecs_to_jiffies64(n);
		67	}
		68
		69
3266	Serge	70	struct drm_i915_gem_object *get_fb_obj();
		71
3263	Serge	72	unsigned long vm_mmap(struct file *file, unsigned long addr,
		73	unsigned long len, unsigned long prot,
		74	unsigned long flag, unsigned long offset);
		75
2344	Serge	76	static inline void clflush(volatile void *__p)
		77	{
		78	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		79	}
		80
2332	Serge	81	#define MAX_ERRNO 4095
		82
		83	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		84
		85
		86	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	87	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		88	bool force);
		89	static __must_check int
4560	Serge	90	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		91	bool readonly);
5060	serge	92	static void
		93	i915_gem_object_retire(struct drm_i915_gem_object *obj);
2326	Serge	94
3031	serge	95	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		96	struct drm_i915_gem_object *obj);
		97	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		98	struct drm_i915_fence_reg *fence,
		99	bool enable);
2332	Serge	100
4560	Serge	101	static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		102	static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	103
4104	Serge	104	static bool cpu_cache_is_coherent(struct drm_device *dev,
		105	enum i915_cache_level level)
		106	{
		107	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		108	}
		109
		110	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		111	{
		112	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		113	return true;
		114
		115	return obj->pin_display;
		116	}
		117
3031	serge	118	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		119	{
		120	if (obj->tiling_mode)
		121	i915_gem_release_mmap(obj);
		122
		123	/* As we do not have an associated fence register, we will force
		124	* a tiling change if we ever need to acquire one.
		125	*/
		126	obj->fence_dirty = false;
		127	obj->fence_reg = I915_FENCE_REG_NONE;
		128	}
		129
2332	Serge	130	/* some bookkeeping */
		131	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		132	size_t size)
		133	{
4104	Serge	134	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	135	dev_priv->mm.object_count++;
		136	dev_priv->mm.object_memory += size;
4104	Serge	137	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	138	}
		139
		140	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		141	size_t size)
		142	{
4104	Serge	143	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	144	dev_priv->mm.object_count--;
		145	dev_priv->mm.object_memory -= size;
4104	Serge	146	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	147	}
		148
		149	static int
3480	Serge	150	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	151	{
		152	int ret;
		153
3480	Serge	154	#define EXIT_COND (!i915_reset_in_progress(error))
		155	if (EXIT_COND)
2332	Serge	156	return 0;
3255	Serge	157	#if 0
3031	serge	158	/*
		159	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		160	* userspace. If it takes that long something really bad is going on and
		161	* we should simply try to bail out and fail as gracefully as possible.
		162	*/
3480	Serge	163	ret = wait_event_interruptible_timeout(error->reset_queue,
		164	EXIT_COND,
		165	10*HZ);
3031	serge	166	if (ret == 0) {
		167	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		168	return -EIO;
		169	} else if (ret < 0) {
2332	Serge	170	return ret;
3031	serge	171	}
2332	Serge	172
3255	Serge	173	#endif
3480	Serge	174	#undef EXIT_COND
3255	Serge	175
2332	Serge	176	return 0;
		177	}
		178
		179	int i915_mutex_lock_interruptible(struct drm_device *dev)
		180	{
3480	Serge	181	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	182	int ret;
		183
3480	Serge	184	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	185	if (ret)
		186	return ret;
		187
3480	Serge	188	ret = mutex_lock_interruptible(&dev->struct_mutex);
		189	if (ret)
		190	return ret;
2332	Serge	191
		192	WARN_ON(i915_verify_lists(dev));
		193	return 0;
		194	}
		195
		196	static inline bool
		197	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		198	{
4104	Serge	199	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	200	}
		201
		202
		203	#if 0
		204
		205	int
		206	i915_gem_init_ioctl(struct drm_device dev, void data,
		207	struct drm_file *file)
		208	{
3480	Serge	209	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	210	struct drm_i915_gem_init *args = data;
		211
3031	serge	212	if (drm_core_check_feature(dev, DRIVER_MODESET))
		213	return -ENODEV;
		214
2332	Serge	215	if (args->gtt_start >= args->gtt_end \|\|
		216	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		217	return -EINVAL;
		218
3031	serge	219	/* GEM with user mode setting was never supported on ilk and later. */
		220	if (INTEL_INFO(dev)->gen >= 5)
		221	return -ENODEV;
		222
2332	Serge	223	mutex_lock(&dev->struct_mutex);
3480	Serge	224	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		225	args->gtt_end);
		226	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	227	mutex_unlock(&dev->struct_mutex);
		228
		229	return 0;
		230	}
2351	Serge	231	#endif
2332	Serge	232
		233	int
		234	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		235	struct drm_file *file)
		236	{
		237	struct drm_i915_private *dev_priv = dev->dev_private;
		238	struct drm_i915_gem_get_aperture *args = data;
		239	struct drm_i915_gem_object *obj;
		240	size_t pinned;
		241
		242	pinned = 0;
		243	mutex_lock(&dev->struct_mutex);
4104	Serge	244	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
5060	serge	245	if (i915_gem_obj_is_pinned(obj))
4104	Serge	246	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	247	mutex_unlock(&dev->struct_mutex);
		248
4104	Serge	249	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	250	args->aper_available_size = args->aper_size - pinned;
2332	Serge	251
		252	return 0;
		253	}
		254
3480	Serge	255	void i915_gem_object_alloc(struct drm_device dev)
		256	{
		257	struct drm_i915_private *dev_priv = dev->dev_private;
		258	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		259	}
		260
		261	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		262	{
		263	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		264	kfree(obj);
		265	}
		266
3031	serge	267	static int
		268	i915_gem_create(struct drm_file *file,
2332	Serge	269	struct drm_device *dev,
		270	uint64_t size,
		271	uint32_t *handle_p)
		272	{
		273	struct drm_i915_gem_object *obj;
		274	int ret;
		275	u32 handle;
		276
		277	size = roundup(size, PAGE_SIZE);
2342	Serge	278	if (size == 0)
		279	return -EINVAL;
2332	Serge	280
		281	/* Allocate the new object */
		282	obj = i915_gem_alloc_object(dev, size);
		283	if (obj == NULL)
		284	return -ENOMEM;
		285
		286	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	287	/* drop reference from allocate - handle holds it now */
		288	drm_gem_object_unreference_unlocked(&obj->base);
		289	if (ret)
2332	Serge	290	return ret;
		291
		292	*handle_p = handle;
		293	return 0;
		294	}
		295
		296	int
		297	i915_gem_dumb_create(struct drm_file *file,
		298	struct drm_device *dev,
		299	struct drm_mode_create_dumb *args)
		300	{
		301	/* have to work out size/pitch and return them */
4560	Serge	302	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
2332	Serge	303	args->size = args->pitch * args->height;
		304	return i915_gem_create(file, dev,
		305	args->size, &args->handle);
		306	}
		307
2326	Serge	308	/**
2332	Serge	309	* Creates a new mm object and returns a handle to it.
		310	*/
		311	int
		312	i915_gem_create_ioctl(struct drm_device dev, void data,
		313	struct drm_file *file)
		314	{
		315	struct drm_i915_gem_create *args = data;
3031	serge	316
2332	Serge	317	return i915_gem_create(file, dev,
		318	args->size, &args->handle);
		319	}
		320
		321
3260	Serge	322	#if 0
2332	Serge	323
3031	serge	324	static inline int
		325	__copy_to_user_swizzled(char __user *cpu_vaddr,
		326	const char *gpu_vaddr, int gpu_offset,
2332	Serge	327	int length)
		328	{
3031	serge	329	int ret, cpu_offset = 0;
2332	Serge	330
3031	serge	331	while (length > 0) {
		332	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		333	int this_length = min(cacheline_end - gpu_offset, length);
		334	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	335
3031	serge	336	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		337	gpu_vaddr + swizzled_gpu_offset,
		338	this_length);
		339	if (ret)
		340	return ret + length;
2332	Serge	341
3031	serge	342	cpu_offset += this_length;
		343	gpu_offset += this_length;
		344	length -= this_length;
		345	}
		346
		347	return 0;
2332	Serge	348	}
		349
3031	serge	350	static inline int
		351	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		352	const char __user *cpu_vaddr,
		353	int length)
2332	Serge	354	{
3031	serge	355	int ret, cpu_offset = 0;
2332	Serge	356
		357	while (length > 0) {
		358	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		359	int this_length = min(cacheline_end - gpu_offset, length);
		360	int swizzled_gpu_offset = gpu_offset ^ 64;
		361
3031	serge	362	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	363	cpu_vaddr + cpu_offset,
		364	this_length);
3031	serge	365	if (ret)
		366	return ret + length;
		367
2332	Serge	368	cpu_offset += this_length;
		369	gpu_offset += this_length;
		370	length -= this_length;
		371	}
		372
3031	serge	373	return 0;
2332	Serge	374	}
		375
3031	serge	376	/* Per-page copy function for the shmem pread fastpath.
		377	* Flushes invalid cachelines before reading the target if
		378	* needs_clflush is set. */
2332	Serge	379	static int
3031	serge	380	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		381	char __user *user_data,
		382	bool page_do_bit17_swizzling, bool needs_clflush)
		383	{
		384	char *vaddr;
		385	int ret;
		386
		387	if (unlikely(page_do_bit17_swizzling))
		388	return -EINVAL;
		389
		390	vaddr = kmap_atomic(page);
		391	if (needs_clflush)
		392	drm_clflush_virt_range(vaddr + shmem_page_offset,
		393	page_length);
		394	ret = __copy_to_user_inatomic(user_data,
		395	vaddr + shmem_page_offset,
		396	page_length);
		397	kunmap_atomic(vaddr);
		398
		399	return ret ? -EFAULT : 0;
		400	}
		401
		402	static void
		403	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		404	bool swizzled)
		405	{
		406	if (unlikely(swizzled)) {
		407	unsigned long start = (unsigned long) addr;
		408	unsigned long end = (unsigned long) addr + length;
		409
		410	/* For swizzling simply ensure that we always flush both
		411	* channels. Lame, but simple and it works. Swizzled
		412	* pwrite/pread is far from a hotpath - current userspace
		413	* doesn't use it at all. */
		414	start = round_down(start, 128);
		415	end = round_up(end, 128);
		416
		417	drm_clflush_virt_range((void *)start, end - start);
		418	} else {
		419	drm_clflush_virt_range(addr, length);
		420	}
		421
		422	}
		423
		424	/* Only difference to the fast-path function is that this can handle bit17
		425	* and uses non-atomic copy and kmap functions. */
		426	static int
		427	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		428	char __user *user_data,
		429	bool page_do_bit17_swizzling, bool needs_clflush)
		430	{
		431	char *vaddr;
		432	int ret;
		433
		434	vaddr = kmap(page);
		435	if (needs_clflush)
		436	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		437	page_length,
		438	page_do_bit17_swizzling);
		439
		440	if (page_do_bit17_swizzling)
		441	ret = __copy_to_user_swizzled(user_data,
		442	vaddr, shmem_page_offset,
		443	page_length);
		444	else
		445	ret = __copy_to_user(user_data,
		446	vaddr + shmem_page_offset,
		447	page_length);
		448	kunmap(page);
		449
		450	return ret ? - EFAULT : 0;
		451	}
		452
		453	static int
		454	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	455	struct drm_i915_gem_object *obj,
		456	struct drm_i915_gem_pread *args,
		457	struct drm_file *file)
		458	{
3031	serge	459	char __user *user_data;
2332	Serge	460	ssize_t remain;
		461	loff_t offset;
3031	serge	462	int shmem_page_offset, page_length, ret = 0;
		463	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		464	int prefaulted = 0;
		465	int needs_clflush = 0;
3746	Serge	466	struct sg_page_iter sg_iter;
2332	Serge	467
3746	Serge	468	user_data = to_user_ptr(args->data_ptr);
2332	Serge	469	remain = args->size;
		470
3031	serge	471	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		472
5060	serge	473	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
3031	serge	474	if (ret)
		475	return ret;
		476
2332	Serge	477	offset = args->offset;
		478
3746	Serge	479	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		480	offset >> PAGE_SHIFT) {
		481	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	482
3031	serge	483	if (remain <= 0)
		484	break;
		485
2332	Serge	486	/* Operation in this page
		487	*
3031	serge	488	* shmem_page_offset = offset within page in shmem file
2332	Serge	489	* page_length = bytes to copy for this page
		490	*/
3031	serge	491	shmem_page_offset = offset_in_page(offset);
2332	Serge	492	page_length = remain;
3031	serge	493	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		494	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	495
3031	serge	496	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		497	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	498
3031	serge	499	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		500	user_data, page_do_bit17_swizzling,
		501	needs_clflush);
		502	if (ret == 0)
		503	goto next_page;
2332	Serge	504
3031	serge	505	mutex_unlock(&dev->struct_mutex);
		506
5060	serge	507	if (likely(!i915.prefault_disable) && !prefaulted) {
3031	serge	508	ret = fault_in_multipages_writeable(user_data, remain);
		509	/* Userspace is tricking us, but we've already clobbered
		510	* its pages with the prefault and promised to write the
		511	* data up to the first fault. Hence ignore any errors
		512	* and just continue. */
		513	(void)ret;
		514	prefaulted = 1;
		515	}
		516
		517	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		518	user_data, page_do_bit17_swizzling,
		519	needs_clflush);
		520
		521	mutex_lock(&dev->struct_mutex);
		522
2332	Serge	523	if (ret)
3031	serge	524	goto out;
2332	Serge	525
5060	serge	526	next_page:
2332	Serge	527	remain -= page_length;
		528	user_data += page_length;
		529	offset += page_length;
		530	}
		531
3031	serge	532	out:
		533	i915_gem_object_unpin_pages(obj);
		534
		535	return ret;
2332	Serge	536	}
		537
		538	/**
3031	serge	539	* Reads data from the object referenced by handle.
		540	*
		541	* On error, the contents of *data are undefined.
2332	Serge	542	*/
3031	serge	543	int
		544	i915_gem_pread_ioctl(struct drm_device dev, void data,
		545	struct drm_file *file)
		546	{
		547	struct drm_i915_gem_pread *args = data;
		548	struct drm_i915_gem_object *obj;
		549	int ret = 0;
		550
		551	if (args->size == 0)
		552	return 0;
		553
		554	if (!access_ok(VERIFY_WRITE,
3746	Serge	555	to_user_ptr(args->data_ptr),
3031	serge	556	args->size))
		557	return -EFAULT;
		558
		559	ret = i915_mutex_lock_interruptible(dev);
		560	if (ret)
		561	return ret;
		562
		563	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		564	if (&obj->base == NULL) {
		565	ret = -ENOENT;
		566	goto unlock;
		567	}
		568
		569	/* Bounds check source. */
		570	if (args->offset > obj->base.size \|\|
		571	args->size > obj->base.size - args->offset) {
		572	ret = -EINVAL;
		573	goto out;
		574	}
		575
		576	/* prime objects have no backing filp to GEM pread/pwrite
		577	* pages from.
		578	*/
		579	if (!obj->base.filp) {
		580	ret = -EINVAL;
		581	goto out;
		582	}
		583
		584	trace_i915_gem_object_pread(obj, args->offset, args->size);
		585
		586	ret = i915_gem_shmem_pread(dev, obj, args, file);
		587
		588	out:
		589	drm_gem_object_unreference(&obj->base);
		590	unlock:
		591	mutex_unlock(&dev->struct_mutex);
		592	return ret;
		593	}
		594
		595	/* This is the fast write path which cannot handle
		596	* page faults in the source data
		597	*/
		598
		599	static inline int
		600	fast_user_write(struct io_mapping *mapping,
		601	loff_t page_base, int page_offset,
		602	char __user *user_data,
		603	int length)
		604	{
		605	void __iomem *vaddr_atomic;
		606	void *vaddr;
		607	unsigned long unwritten;
		608
		609	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		610	/* We can use the cpu mem copy function because this is X86. */
		611	vaddr = (void __force*)vaddr_atomic + page_offset;
		612	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		613	user_data, length);
		614	io_mapping_unmap_atomic(vaddr_atomic);
		615	return unwritten;
		616	}
3260	Serge	617	#endif
3031	serge	618
3260	Serge	619	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	620	/**
		621	* This is the fast pwrite path, where we copy the data directly from the
		622	* user into the GTT, uncached.
		623	*/
2332	Serge	624	static int
3031	serge	625	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		626	struct drm_i915_gem_object *obj,
		627	struct drm_i915_gem_pwrite *args,
		628	struct drm_file *file)
2332	Serge	629	{
5060	serge	630	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	631	ssize_t remain;
3031	serge	632	loff_t offset, page_base;
		633	char __user *user_data;
		634	int page_offset, page_length, ret;
2332	Serge	635
5060	serge	636	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3031	serge	637	if (ret)
		638	goto out;
		639
		640	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		641	if (ret)
		642	goto out_unpin;
		643
		644	ret = i915_gem_object_put_fence(obj);
		645	if (ret)
		646	goto out_unpin;
		647
4539	Serge	648	user_data = to_user_ptr(args->data_ptr);
2332	Serge	649	remain = args->size;
		650
4104	Serge	651	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	652
3031	serge	653	while (remain > 0) {
		654	/* Operation in this page
		655	*
		656	* page_base = page offset within aperture
		657	* page_offset = offset within page
		658	* page_length = bytes to copy for this page
		659	*/
		660	page_base = offset & PAGE_MASK;
		661	page_offset = offset_in_page(offset);
		662	page_length = remain;
		663	if ((page_offset + remain) > PAGE_SIZE)
		664	page_length = PAGE_SIZE - page_offset;
2332	Serge	665
4539	Serge	666	MapPage(dev_priv->gtt.mappable, dev_priv->gtt.mappable_base+page_base, PG_SW);
3031	serge	667
5060	serge	668	memcpy((char*)dev_priv->gtt.mappable+page_offset, user_data, page_length);
3260	Serge	669
3031	serge	670	remain -= page_length;
		671	user_data += page_length;
		672	offset += page_length;
2332	Serge	673	}
		674
3031	serge	675	out_unpin:
5060	serge	676	i915_gem_object_ggtt_unpin(obj);
3031	serge	677	out:
5060	serge	678	return ret;
3031	serge	679	}
		680
		681	/* Per-page copy function for the shmem pwrite fastpath.
		682	* Flushes invalid cachelines before writing to the target if
		683	* needs_clflush_before is set and flushes out any written cachelines after
		684	* writing if needs_clflush is set. */
		685	static int
		686	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		687	char __user *user_data,
		688	bool page_do_bit17_swizzling,
		689	bool needs_clflush_before,
		690	bool needs_clflush_after)
		691	{
		692	char *vaddr;
3260	Serge	693	int ret = 0;
3031	serge	694
		695	if (unlikely(page_do_bit17_swizzling))
		696	return -EINVAL;
		697
4539	Serge	698	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	699	if (needs_clflush_before)
		700	drm_clflush_virt_range(vaddr + shmem_page_offset,
		701	page_length);
3260	Serge	702	memcpy(vaddr + shmem_page_offset,
3031	serge	703	user_data,
		704	page_length);
		705	if (needs_clflush_after)
		706	drm_clflush_virt_range(vaddr + shmem_page_offset,
		707	page_length);
3260	Serge	708	FreeKernelSpace(vaddr);
3031	serge	709
		710	return ret ? -EFAULT : 0;
		711	}
3260	Serge	712	#if 0
3031	serge	713
		714	/* Only difference to the fast-path function is that this can handle bit17
		715	* and uses non-atomic copy and kmap functions. */
		716	static int
		717	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		718	char __user *user_data,
		719	bool page_do_bit17_swizzling,
		720	bool needs_clflush_before,
		721	bool needs_clflush_after)
		722	{
		723	char *vaddr;
		724	int ret;
		725
		726	vaddr = kmap(page);
		727	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		728	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		729	page_length,
		730	page_do_bit17_swizzling);
		731	if (page_do_bit17_swizzling)
		732	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		733	user_data,
		734	page_length);
		735	else
		736	ret = __copy_from_user(vaddr + shmem_page_offset,
		737	user_data,
		738	page_length);
		739	if (needs_clflush_after)
		740	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		741	page_length,
		742	page_do_bit17_swizzling);
		743	kunmap(page);
		744
		745	return ret ? -EFAULT : 0;
		746	}
3260	Serge	747	#endif
3031	serge	748
3260	Serge	749
3031	serge	750	static int
		751	i915_gem_shmem_pwrite(struct drm_device *dev,
		752	struct drm_i915_gem_object *obj,
		753	struct drm_i915_gem_pwrite *args,
		754	struct drm_file *file)
		755	{
		756	ssize_t remain;
		757	loff_t offset;
		758	char __user *user_data;
		759	int shmem_page_offset, page_length, ret = 0;
		760	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		761	int hit_slowpath = 0;
		762	int needs_clflush_after = 0;
		763	int needs_clflush_before = 0;
3746	Serge	764	struct sg_page_iter sg_iter;
3031	serge	765
3746	Serge	766	user_data = to_user_ptr(args->data_ptr);
3031	serge	767	remain = args->size;
		768
		769	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		770
		771	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		772	/* If we're not in the cpu write domain, set ourself into the gtt
		773	* write domain and manually flush cachelines (if required). This
		774	* optimizes for the case when the gpu will use the data
		775	* right away and we therefore have to clflush anyway. */
4104	Serge	776	needs_clflush_after = cpu_write_needs_clflush(obj);
4560	Serge	777	ret = i915_gem_object_wait_rendering(obj, false);
3031	serge	778	if (ret)
		779	return ret;
5060	serge	780
		781	i915_gem_object_retire(obj);
3031	serge	782	}
4104	Serge	783	/* Same trick applies to invalidate partially written cachelines read
		784	* before writing. */
		785	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		786	needs_clflush_before =
		787	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	788
		789	ret = i915_gem_object_get_pages(obj);
2332	Serge	790	if (ret)
3031	serge	791	return ret;
2332	Serge	792
3031	serge	793	i915_gem_object_pin_pages(obj);
2332	Serge	794
		795	offset = args->offset;
3031	serge	796	obj->dirty = 1;
2332	Serge	797
3746	Serge	798	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		799	offset >> PAGE_SHIFT) {
		800	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	801	int partial_cacheline_write;
2332	Serge	802
3031	serge	803	if (remain <= 0)
		804	break;
		805
2332	Serge	806	/* Operation in this page
		807	*
		808	* shmem_page_offset = offset within page in shmem file
		809	* page_length = bytes to copy for this page
		810	*/
		811	shmem_page_offset = offset_in_page(offset);
		812
		813	page_length = remain;
		814	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		815	page_length = PAGE_SIZE - shmem_page_offset;
		816
3031	serge	817	/* If we don't overwrite a cacheline completely we need to be
		818	* careful to have up-to-date data by first clflushing. Don't
		819	* overcomplicate things and flush the entire patch. */
		820	partial_cacheline_write = needs_clflush_before &&
		821	((shmem_page_offset \| page_length)
3260	Serge	822	& (x86_clflush_size - 1));
2332	Serge	823
3031	serge	824	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		825	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	826
3031	serge	827	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		828	user_data, page_do_bit17_swizzling,
		829	partial_cacheline_write,
		830	needs_clflush_after);
		831	if (ret == 0)
		832	goto next_page;
		833
		834	hit_slowpath = 1;
		835	mutex_unlock(&dev->struct_mutex);
3260	Serge	836	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	837
3260	Serge	838	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		839	// user_data, page_do_bit17_swizzling,
		840	// partial_cacheline_write,
		841	// needs_clflush_after);
		842
3031	serge	843	mutex_lock(&dev->struct_mutex);
		844
		845	next_page:
2332	Serge	846
3031	serge	847	if (ret)
		848	goto out;
		849
2332	Serge	850	remain -= page_length;
3031	serge	851	user_data += page_length;
2332	Serge	852	offset += page_length;
		853	}
		854
		855	out:
3031	serge	856	i915_gem_object_unpin_pages(obj);
		857
		858	if (hit_slowpath) {
3480	Serge	859	/*
		860	* Fixup: Flush cpu caches in case we didn't flush the dirty
		861	* cachelines in-line while writing and the object moved
		862	* out of the cpu write domain while we've dropped the lock.
		863	*/
		864	if (!needs_clflush_after &&
		865	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	866	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	867	i915_gem_chipset_flush(dev);
3031	serge	868	}
2332	Serge	869	}
		870
3031	serge	871	if (needs_clflush_after)
3243	Serge	872	i915_gem_chipset_flush(dev);
3031	serge	873
2332	Serge	874	return ret;
		875	}
3031	serge	876
		877	/**
		878	* Writes data to the object referenced by handle.
		879	*
		880	* On error, the contents of the buffer that were to be modified are undefined.
		881	*/
		882	int
		883	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		884	struct drm_file *file)
		885	{
		886	struct drm_i915_gem_pwrite *args = data;
		887	struct drm_i915_gem_object *obj;
		888	int ret;
		889
4104	Serge	890	if (args->size == 0)
		891	return 0;
		892
3480	Serge	893
3031	serge	894	ret = i915_mutex_lock_interruptible(dev);
		895	if (ret)
		896	return ret;
		897
		898	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		899	if (&obj->base == NULL) {
		900	ret = -ENOENT;
		901	goto unlock;
		902	}
		903
		904	/* Bounds check destination. */
		905	if (args->offset > obj->base.size \|\|
		906	args->size > obj->base.size - args->offset) {
		907	ret = -EINVAL;
		908	goto out;
		909	}
		910
		911	/* prime objects have no backing filp to GEM pread/pwrite
		912	* pages from.
		913	*/
		914	if (!obj->base.filp) {
		915	ret = -EINVAL;
		916	goto out;
		917	}
		918
		919	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		920
		921	ret = -EFAULT;
		922	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		923	* it would end up going through the fenced access, and we'll get
		924	* different detiling behavior between reading and writing.
		925	* pread/pwrite currently are reading and writing from the CPU
		926	* perspective, requiring manual detiling by the client.
		927	*/
3260	Serge	928	// if (obj->phys_obj) {
		929	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		930	// goto out;
		931	// }
3031	serge	932
4104	Serge	933	if (obj->tiling_mode == I915_TILING_NONE &&
		934	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		935	cpu_write_needs_clflush(obj)) {
3031	serge	936	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		937	/* Note that the gtt paths might fail with non-page-backed user
		938	* pointers (e.g. gtt mappings when moving data between
		939	* textures). Fallback to the shmem path in that case. */
		940	}
		941
		942	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	943	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	944
		945	out:
		946	drm_gem_object_unreference(&obj->base);
		947	unlock:
		948	mutex_unlock(&dev->struct_mutex);
		949	return ret;
		950	}
		951
		952	int
3480	Serge	953	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	954	bool interruptible)
		955	{
3480	Serge	956	if (i915_reset_in_progress(error)) {
3031	serge	957	/* Non-interruptible callers can't handle -EAGAIN, hence return
		958	* -EIO unconditionally for these. */
		959	if (!interruptible)
		960	return -EIO;
2332	Serge	961
3480	Serge	962	/* Recovery complete, but the reset failed ... */
		963	if (i915_terminally_wedged(error))
3031	serge	964	return -EIO;
2332	Serge	965
3031	serge	966	return -EAGAIN;
		967	}
2332	Serge	968
3031	serge	969	return 0;
		970	}
2332	Serge	971
3031	serge	972	/*
		973	* Compare seqno against outstanding lazy request. Emit a request if they are
		974	* equal.
		975	*/
5060	serge	976	int
		977	i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno)
3031	serge	978	{
		979	int ret;
2332	Serge	980
3031	serge	981	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	982
3031	serge	983	ret = 0;
4560	Serge	984	if (seqno == ring->outstanding_lazy_seqno)
4104	Serge	985	ret = i915_add_request(ring, NULL);
2332	Serge	986
3031	serge	987	return ret;
		988	}
2332	Serge	989
4560	Serge	990	static void fake_irq(unsigned long data)
		991	{
		992	// wake_up_process((struct task_struct *)data);
		993	}
		994
		995	static bool missed_irq(struct drm_i915_private *dev_priv,
5060	serge	996	struct intel_engine_cs *ring)
4560	Serge	997	{
		998	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
		999	}
		1000
		1001	static bool can_wait_boost(struct drm_i915_file_private *file_priv)
		1002	{
		1003	if (file_priv == NULL)
		1004	return true;
		1005
		1006	return !atomic_xchg(&file_priv->rps_wait_boost, true);
		1007	}
		1008
3031	serge	1009	/**
		1010	* __wait_seqno - wait until execution of seqno has finished
		1011	* @ring: the ring expected to report seqno
		1012	* @seqno: duh!
3480	Serge	1013	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1014	* @interruptible: do an interruptible wait (normally yes)
		1015	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1016	*
3480	Serge	1017	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1018	* values have been read by the caller in an smp safe manner. Where read-side
		1019	* locks are involved, it is sufficient to read the reset_counter before
		1020	* unlocking the lock that protects the seqno. For lockless tricks, the
		1021	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1022	* inserted.
		1023	*
3031	serge	1024	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1025	* errno with remaining time filled in timeout argument.
		1026	*/
5060	serge	1027	static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
3480	Serge	1028	unsigned reset_counter,
4560	Serge	1029	bool interruptible,
5060	serge	1030	s64 *timeout,
4560	Serge	1031	struct drm_i915_file_private *file_priv)
3031	serge	1032	{
5060	serge	1033	struct drm_device *dev = ring->dev;
		1034	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1035	const bool irq_test_in_progress =
		1036	ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
5060	serge	1037	unsigned long timeout_expire;
		1038	s64 before, now;
		1039
4560	Serge	1040	wait_queue_t __wait;
3031	serge	1041	int ret;
2332	Serge	1042
5060	serge	1043	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
4104	Serge	1044
3031	serge	1045	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1046	return 0;
2332	Serge	1047
5060	serge	1048	timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0;
2332	Serge	1049
5060	serge	1050	if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
4560	Serge	1051	gen6_rps_boost(dev_priv);
		1052	if (file_priv)
		1053	mod_delayed_work(dev_priv->wq,
		1054	&file_priv->mm.idle_work,
		1055	msecs_to_jiffies(100));
3031	serge	1056	}
2332	Serge	1057
4560	Serge	1058	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
3031	serge	1059	return -ENODEV;
2332	Serge	1060
4560	Serge	1061	INIT_LIST_HEAD(&__wait.task_list);
		1062	__wait.evnt = CreateEvent(NULL, MANUAL_DESTROY);
2332	Serge	1063
4560	Serge	1064	/* Record current time in case interrupted by signal, or wedged */
		1065	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1066
4560	Serge	1067	for (;;) {
		1068	unsigned long flags;
		1069
3480	Serge	1070	/* We need to check whether any gpu reset happened in between
		1071	* the caller grabbing the seqno and now ... */
4560	Serge	1072	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
		1073	/* ... but upgrade the -EAGAIN to an -EIO if the gpu
		1074	* is truely gone. */
		1075	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
		1076	if (ret == 0)
		1077	ret = -EAGAIN;
		1078	break;
		1079	}
3480	Serge	1080
4560	Serge	1081	if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
		1082	ret = 0;
		1083	break;
		1084	}
2332	Serge	1085
5060	serge	1086	if (timeout && time_after_eq(jiffies, timeout_expire)) {
4560	Serge	1087	ret = -ETIME;
		1088	break;
		1089	}
2332	Serge	1090
4560	Serge	1091	spin_lock_irqsave(&ring->irq_queue.lock, flags);
		1092	if (list_empty(&__wait.task_list))
		1093	__add_wait_queue(&ring->irq_queue, &__wait);
		1094	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1095
		1096	WaitEventTimeout(__wait.evnt, 1);
		1097
		1098	if (!list_empty(&__wait.task_list)) {
		1099	spin_lock_irqsave(&ring->irq_queue.lock, flags);
		1100	list_del_init(&__wait.task_list);
		1101	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1102	}
		1103	};
		1104	trace_i915_gem_request_wait_end(ring, seqno);
		1105
		1106	DestroyEvent(__wait.evnt);
		1107
		1108	if (!irq_test_in_progress)
5060	serge	1109	ring->irq_put(ring);
2332	Serge	1110
5060	serge	1111	// finish_wait(&ring->irq_queue, &wait);
4560	Serge	1112	return ret;
3031	serge	1113	}
2332	Serge	1114
3031	serge	1115	/**
		1116	* Waits for a sequence number to be signaled, and cleans up the
		1117	* request and object lists appropriately for that event.
		1118	*/
		1119	int
5060	serge	1120	i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
3031	serge	1121	{
		1122	struct drm_device *dev = ring->dev;
		1123	struct drm_i915_private *dev_priv = dev->dev_private;
		1124	bool interruptible = dev_priv->mm.interruptible;
		1125	int ret;
2332	Serge	1126
3031	serge	1127	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1128	BUG_ON(seqno == 0);
2332	Serge	1129
3480	Serge	1130	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1131	if (ret)
		1132	return ret;
2332	Serge	1133
3031	serge	1134	ret = i915_gem_check_olr(ring, seqno);
		1135	if (ret)
		1136	return ret;
2332	Serge	1137
3480	Serge	1138	return __wait_seqno(ring, seqno,
		1139	atomic_read(&dev_priv->gpu_error.reset_counter),
4560	Serge	1140	interruptible, NULL, NULL);
3031	serge	1141	}
2332	Serge	1142
4104	Serge	1143	static int
		1144	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
5060	serge	1145	struct intel_engine_cs *ring)
4104	Serge	1146	{
5060	serge	1147	if (!obj->active)
		1148	return 0;
4104	Serge	1149
		1150	/* Manually manage the write flush as we may have not yet
		1151	* retired the buffer.
		1152	*
		1153	* Note that the last_write_seqno is always the earlier of
		1154	* the two (read/write) seqno, so if we haved successfully waited,
		1155	* we know we have passed the last write.
		1156	*/
		1157	obj->last_write_seqno = 0;
		1158
		1159	return 0;
		1160	}
		1161
3031	serge	1162	/**
		1163	* Ensures that all rendering to the object has completed and the object is
		1164	* safe to unbind from the GTT or access from the CPU.
		1165	*/
		1166	static __must_check int
		1167	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1168	bool readonly)
		1169	{
5060	serge	1170	struct intel_engine_cs *ring = obj->ring;
3031	serge	1171	u32 seqno;
		1172	int ret;
2332	Serge	1173
3031	serge	1174	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1175	if (seqno == 0)
		1176	return 0;
2332	Serge	1177
3031	serge	1178	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1179	if (ret)
		1180	return ret;
2332	Serge	1181
4104	Serge	1182	return i915_gem_object_wait_rendering__tail(obj, ring);
3031	serge	1183	}
2332	Serge	1184
3260	Serge	1185	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1186	* as the object state may change during this call.
		1187	*/
		1188	static __must_check int
		1189	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
5060	serge	1190	struct drm_i915_file_private *file_priv,
3260	Serge	1191	bool readonly)
		1192	{
		1193	struct drm_device *dev = obj->base.dev;
		1194	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	1195	struct intel_engine_cs *ring = obj->ring;
3480	Serge	1196	unsigned reset_counter;
3260	Serge	1197	u32 seqno;
		1198	int ret;
2332	Serge	1199
3260	Serge	1200	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1201	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1202
3260	Serge	1203	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1204	if (seqno == 0)
		1205	return 0;
2332	Serge	1206
3480	Serge	1207	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1208	if (ret)
		1209	return ret;
2332	Serge	1210
3260	Serge	1211	ret = i915_gem_check_olr(ring, seqno);
		1212	if (ret)
		1213	return ret;
2332	Serge	1214
3480	Serge	1215	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1216	mutex_unlock(&dev->struct_mutex);
5060	serge	1217	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
3260	Serge	1218	mutex_lock(&dev->struct_mutex);
4104	Serge	1219	if (ret)
		1220	return ret;
2332	Serge	1221
4104	Serge	1222	return i915_gem_object_wait_rendering__tail(obj, ring);
3260	Serge	1223	}
2332	Serge	1224
3260	Serge	1225	/**
		1226	* Called when user space prepares to use an object with the CPU, either
		1227	* through the mmap ioctl's mapping or a GTT mapping.
		1228	*/
		1229	int
		1230	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1231	struct drm_file *file)
		1232	{
		1233	struct drm_i915_gem_set_domain *args = data;
		1234	struct drm_i915_gem_object *obj;
		1235	uint32_t read_domains = args->read_domains;
		1236	uint32_t write_domain = args->write_domain;
		1237	int ret;
2332	Serge	1238
3260	Serge	1239	/* Only handle setting domains to types used by the CPU. */
		1240	if (write_domain & I915_GEM_GPU_DOMAINS)
		1241	return -EINVAL;
2332	Serge	1242
3260	Serge	1243	if (read_domains & I915_GEM_GPU_DOMAINS)
		1244	return -EINVAL;
2332	Serge	1245
3260	Serge	1246	/* Having something in the write domain implies it's in the read
		1247	* domain, and only that read domain. Enforce that in the request.
		1248	*/
		1249	if (write_domain != 0 && read_domains != write_domain)
		1250	return -EINVAL;
2332	Serge	1251
3260	Serge	1252	ret = i915_mutex_lock_interruptible(dev);
		1253	if (ret)
		1254	return ret;
2332	Serge	1255
3260	Serge	1256	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1257	if (&obj->base == NULL) {
		1258	ret = -ENOENT;
		1259	goto unlock;
		1260	}
2332	Serge	1261
3260	Serge	1262	/* Try to flush the object off the GPU without holding the lock.
		1263	* We will repeat the flush holding the lock in the normal manner
		1264	* to catch cases where we are gazumped.
		1265	*/
5060	serge	1266	ret = i915_gem_object_wait_rendering__nonblocking(obj,
		1267	file->driver_priv,
		1268	!write_domain);
3260	Serge	1269	if (ret)
		1270	goto unref;
2332	Serge	1271
3260	Serge	1272	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1273	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1274
3260	Serge	1275	/* Silently promote "you're not bound, there was nothing to do"
		1276	* to success, since the client was just asking us to
		1277	* make sure everything was done.
		1278	*/
		1279	if (ret == -EINVAL)
		1280	ret = 0;
		1281	} else {
		1282	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1283	}
2332	Serge	1284
3260	Serge	1285	unref:
		1286	drm_gem_object_unreference(&obj->base);
		1287	unlock:
		1288	mutex_unlock(&dev->struct_mutex);
		1289	return ret;
		1290	}
2332	Serge	1291
4293	Serge	1292	/**
		1293	* Called when user space has done writes to this buffer
		1294	*/
		1295	int
		1296	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
		1297	struct drm_file *file)
		1298	{
		1299	struct drm_i915_gem_sw_finish *args = data;
		1300	struct drm_i915_gem_object *obj;
		1301	int ret = 0;
2332	Serge	1302
4293	Serge	1303	ret = i915_mutex_lock_interruptible(dev);
		1304	if (ret)
		1305	return ret;
2332	Serge	1306
4293	Serge	1307	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1308	if (&obj->base == NULL) {
		1309	ret = -ENOENT;
		1310	goto unlock;
		1311	}
2332	Serge	1312
4293	Serge	1313	/* Pinned buffers may be scanout, so flush the cache */
		1314	if (obj->pin_display)
		1315	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	1316
4293	Serge	1317	drm_gem_object_unreference(&obj->base);
		1318	unlock:
		1319	mutex_unlock(&dev->struct_mutex);
		1320	return ret;
		1321	}
		1322
3260	Serge	1323	/**
		1324	* Maps the contents of an object, returning the address it is mapped
		1325	* into.
		1326	*
		1327	* While the mapping holds a reference on the contents of the object, it doesn't
		1328	* imply a ref on the object itself.
		1329	*/
		1330	int
		1331	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1332	struct drm_file *file)
		1333	{
		1334	struct drm_i915_gem_mmap *args = data;
		1335	struct drm_gem_object *obj;
4392	Serge	1336	unsigned long addr;
2332	Serge	1337
3260	Serge	1338	obj = drm_gem_object_lookup(dev, file, args->handle);
		1339	if (obj == NULL)
		1340	return -ENOENT;
4104	Serge	1341
3260	Serge	1342	/* prime objects have no backing filp to GEM mmap
		1343	* pages from.
		1344	*/
		1345	if (!obj->filp) {
		1346	drm_gem_object_unreference_unlocked(obj);
		1347	return -EINVAL;
		1348	}
2332	Serge	1349
3263	Serge	1350	addr = vm_mmap(obj->filp, 0, args->size,
		1351	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1352	args->offset);
3260	Serge	1353	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1354	if (IS_ERR((void *)addr))
		1355	return addr;
2332	Serge	1356
3260	Serge	1357	args->addr_ptr = (uint64_t) addr;
2332	Serge	1358
3263	Serge	1359	return 0;
3260	Serge	1360	}
2332	Serge	1361
		1362
		1363
		1364
		1365
		1366
		1367
		1368
3031	serge	1369
		1370
		1371
		1372
		1373
		1374	/**
		1375	* i915_gem_release_mmap - remove physical page mappings
		1376	* @obj: obj in question
		1377	*
		1378	* Preserve the reservation of the mmapping with the DRM core code, but
		1379	* relinquish ownership of the pages back to the system.
		1380	*
		1381	* It is vital that we remove the page mapping if we have mapped a tiled
		1382	* object through the GTT and then lose the fence register due to
		1383	* resource pressure. Similarly if the object has been moved out of the
		1384	* aperture, than pages mapped into userspace must be revoked. Removing the
		1385	* mapping will then trigger a page fault on the next user access, allowing
		1386	* fixup by i915_gem_fault().
		1387	*/
		1388	void
		1389	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1390	{
		1391	if (!obj->fault_mappable)
		1392	return;
		1393
4104	Serge	1394	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1395	obj->fault_mappable = false;
		1396	}
		1397
3480	Serge	1398	uint32_t
2332	Serge	1399	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1400	{
		1401	uint32_t gtt_size;
		1402
		1403	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1404	tiling_mode == I915_TILING_NONE)
		1405	return size;
		1406
		1407	/* Previous chips need a power-of-two fence region when tiling */
		1408	if (INTEL_INFO(dev)->gen == 3)
		1409	gtt_size = 1024*1024;
		1410	else
		1411	gtt_size = 512*1024;
		1412
		1413	while (gtt_size < size)
		1414	gtt_size <<= 1;
		1415
		1416	return gtt_size;
		1417	}
		1418
		1419	/**
		1420	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1421	* @obj: object to check
		1422	*
		1423	* Return the required GTT alignment for an object, taking into account
		1424	* potential fence register mapping.
		1425	*/
3480	Serge	1426	uint32_t
		1427	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1428	int tiling_mode, bool fenced)
2332	Serge	1429	{
		1430	/*
		1431	* Minimum alignment is 4k (GTT page size), but might be greater
		1432	* if a fence register is needed for the object.
		1433	*/
3480	Serge	1434	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1435	tiling_mode == I915_TILING_NONE)
		1436	return 4096;
		1437
		1438	/*
		1439	* Previous chips need to be aligned to the size of the smallest
		1440	* fence register that can contain the object.
		1441	*/
		1442	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1443	}
		1444
		1445
		1446
3480	Serge	1447	int
		1448	i915_gem_mmap_gtt(struct drm_file *file,
		1449	struct drm_device *dev,
		1450	uint32_t handle,
		1451	uint64_t *offset)
		1452	{
		1453	struct drm_i915_private *dev_priv = dev->dev_private;
		1454	struct drm_i915_gem_object *obj;
		1455	unsigned long pfn;
		1456	char mem, ptr;
		1457	int ret;
		1458
		1459	ret = i915_mutex_lock_interruptible(dev);
		1460	if (ret)
		1461	return ret;
		1462
		1463	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1464	if (&obj->base == NULL) {
		1465	ret = -ENOENT;
		1466	goto unlock;
		1467	}
		1468
		1469	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1470	ret = -E2BIG;
		1471	goto out;
		1472	}
		1473
		1474	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1475	DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
		1476	ret = -EFAULT;
3480	Serge	1477	goto out;
		1478	}
		1479	/* Now bind it into the GTT if needed */
5060	serge	1480	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3480	Serge	1481	if (ret)
		1482	goto out;
		1483
		1484	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1485	if (ret)
		1486	goto unpin;
		1487
		1488	ret = i915_gem_object_get_fence(obj);
		1489	if (ret)
		1490	goto unpin;
		1491
		1492	obj->fault_mappable = true;
		1493
4104	Serge	1494	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1495
		1496	/* Finally, remap it using the new GTT offset */
		1497
		1498	mem = UserAlloc(obj->base.size);
		1499	if(unlikely(mem == NULL))
		1500	{
		1501	ret = -ENOMEM;
		1502	goto unpin;
		1503	}
		1504
		1505	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1506	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1507
		1508	unpin:
5060	serge	1509	i915_gem_object_unpin_pages(obj);
3480	Serge	1510
		1511
4104	Serge	1512	*offset = mem;
3480	Serge	1513
		1514	out:
		1515	drm_gem_object_unreference(&obj->base);
		1516	unlock:
		1517	mutex_unlock(&dev->struct_mutex);
		1518	return ret;
		1519	}
		1520
		1521	/**
		1522	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1523	* @dev: DRM device
		1524	* @data: GTT mapping ioctl data
		1525	* @file: GEM object info
		1526	*
		1527	* Simply returns the fake offset to userspace so it can mmap it.
		1528	* The mmap call will end up in drm_gem_mmap(), which will set things
		1529	* up so we can get faults in the handler above.
		1530	*
		1531	* The fault handler will take care of binding the object into the GTT
		1532	* (since it may have been evicted to make room for something), allocating
		1533	* a fence register, and mapping the appropriate aperture address into
		1534	* userspace.
		1535	*/
		1536	int
		1537	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1538	struct drm_file *file)
		1539	{
		1540	struct drm_i915_gem_mmap_gtt *args = data;
		1541
		1542	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1543	}
		1544
5060	serge	1545	static inline int
		1546	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1547	{
		1548	return obj->madv == I915_MADV_DONTNEED;
		1549	}
		1550
3031	serge	1551	/* Immediately discard the backing storage */
		1552	static void
		1553	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1554	{
		1555	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1556
3263	Serge	1557	if (obj->base.filp == NULL)
		1558	return;
2332	Serge	1559
3031	serge	1560	/* Our goal here is to return as much of the memory as
		1561	* is possible back to the system as we are called from OOM.
		1562	* To do this we must instruct the shmfs to drop all of its
		1563	* backing pages, now.
		1564	*/
5060	serge	1565	// shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
3031	serge	1566	obj->madv = __I915_MADV_PURGED;
		1567	}
2332	Serge	1568
5060	serge	1569	/* Try to discard unwanted pages */
		1570	static void
		1571	i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
3031	serge	1572	{
5060	serge	1573	struct address_space *mapping;
		1574
		1575	switch (obj->madv) {
		1576	case I915_MADV_DONTNEED:
		1577	i915_gem_object_truncate(obj);
		1578	case __I915_MADV_PURGED:
		1579	return;
		1580	}
		1581
		1582	if (obj->base.filp == NULL)
		1583	return;
		1584
3031	serge	1585	}
2332	Serge	1586
3031	serge	1587	static void
		1588	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1589	{
3746	Serge	1590	struct sg_page_iter sg_iter;
		1591	int ret;
2332	Serge	1592
3031	serge	1593	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1594
3031	serge	1595	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1596	if (ret) {
		1597	/* In the event of a disaster, abandon all caches and
		1598	* hope for the best.
		1599	*/
		1600	WARN_ON(ret != -EIO);
4104	Serge	1601	i915_gem_clflush_object(obj, true);
3031	serge	1602	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1603	}
2332	Serge	1604
3031	serge	1605	if (obj->madv == I915_MADV_DONTNEED)
		1606	obj->dirty = 0;
2332	Serge	1607
3746	Serge	1608	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1609	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1610
3290	Serge	1611	page_cache_release(page);
3243	Serge	1612	}
4104	Serge	1613	obj->dirty = 0;
3243	Serge	1614
		1615	sg_free_table(obj->pages);
		1616	kfree(obj->pages);
3031	serge	1617	}
2332	Serge	1618
3480	Serge	1619	int
3031	serge	1620	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1621	{
		1622	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1623
3243	Serge	1624	if (obj->pages == NULL)
3031	serge	1625	return 0;
2332	Serge	1626
3031	serge	1627	if (obj->pages_pin_count)
		1628	return -EBUSY;
		1629
4104	Serge	1630	BUG_ON(i915_gem_obj_bound_any(obj));
		1631
3243	Serge	1632	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1633	* array, hence protect them from being reaped by removing them from gtt
		1634	* lists early. */
4104	Serge	1635	list_del(&obj->global_list);
3243	Serge	1636
3031	serge	1637	ops->put_pages(obj);
3243	Serge	1638	obj->pages = NULL;
3031	serge	1639
5060	serge	1640	i915_gem_object_invalidate(obj);
3031	serge	1641
		1642	return 0;
		1643	}
		1644
		1645
		1646
		1647
		1648
		1649
		1650
		1651
2332	Serge	1652	static int
3031	serge	1653	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1654	{
3260	Serge	1655	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1656	int page_count, i;
4104	Serge	1657	struct sg_table *st;
3243	Serge	1658	struct scatterlist *sg;
3746	Serge	1659	struct sg_page_iter sg_iter;
3243	Serge	1660	struct page *page;
3746	Serge	1661	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1662	gfp_t gfp;
2332	Serge	1663
3243	Serge	1664	/* Assert that the object is not currently in any GPU domain. As it
		1665	* wasn't in the GTT, there shouldn't be any way it could have been in
		1666	* a GPU cache
2332	Serge	1667	*/
3243	Serge	1668	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1669	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1670
		1671	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1672	if (st == NULL)
		1673	return -ENOMEM;
		1674
2332	Serge	1675	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1676	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1677	kfree(st);
3746	Serge	1678	FAIL();
2332	Serge	1679	return -ENOMEM;
3243	Serge	1680	}
2332	Serge	1681
3243	Serge	1682	/* Get the list of pages out of our struct file. They'll be pinned
		1683	* at this point until we release them.
		1684	*
		1685	* Fail silently without starting the shrinker
		1686	*/
3746	Serge	1687	sg = st->sgl;
		1688	st->nents = 0;
		1689	for (i = 0; i < page_count; i++) {
4104	Serge	1690	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1691	if (IS_ERR(page)) {
		1692	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1693	goto err_pages;
		1694
3260	Serge	1695	}
3746	Serge	1696
		1697	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1698	if (i)
		1699	sg = sg_next(sg);
		1700	st->nents++;
3243	Serge	1701	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1702	} else {
		1703	sg->length += PAGE_SIZE;
		1704	}
		1705	last_pfn = page_to_pfn(page);
3243	Serge	1706	}
3031	serge	1707
3746	Serge	1708	sg_mark_end(sg);
3243	Serge	1709	obj->pages = st;
3031	serge	1710
2332	Serge	1711	return 0;
		1712
		1713	err_pages:
3746	Serge	1714	sg_mark_end(sg);
		1715	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1716	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1717	sg_free_table(st);
		1718	kfree(st);
3746	Serge	1719	FAIL();
3243	Serge	1720	return PTR_ERR(page);
2332	Serge	1721	}
		1722
3031	serge	1723	/* Ensure that the associated pages are gathered from the backing storage
		1724	* and pinned into our object. i915_gem_object_get_pages() may be called
		1725	* multiple times before they are released by a single call to
		1726	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1727	* either as a result of memory pressure (reaping pages under the shrinker)
		1728	* or as the object is itself released.
		1729	*/
		1730	int
		1731	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1732	{
3031	serge	1733	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1734	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1735	int ret;
2332	Serge	1736
3243	Serge	1737	if (obj->pages)
3031	serge	1738	return 0;
2332	Serge	1739
4392	Serge	1740	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1741	DRM_DEBUG("Attempting to obtain a purgeable object\n");
		1742	return -EFAULT;
4392	Serge	1743	}
		1744
3031	serge	1745	BUG_ON(obj->pages_pin_count);
2332	Serge	1746
3031	serge	1747	ret = ops->get_pages(obj);
		1748	if (ret)
		1749	return ret;
2344	Serge	1750
4104	Serge	1751	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1752	return 0;
2332	Serge	1753	}
		1754
5060	serge	1755	static void
2332	Serge	1756	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
5060	serge	1757	struct intel_engine_cs *ring)
2332	Serge	1758	{
		1759	struct drm_device *dev = obj->base.dev;
		1760	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1761	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1762
		1763	BUG_ON(ring == NULL);
4104	Serge	1764	if (obj->ring != ring && obj->last_write_seqno) {
		1765	/* Keep the seqno relative to the current ring */
		1766	obj->last_write_seqno = seqno;
		1767	}
2332	Serge	1768	obj->ring = ring;
		1769
		1770	/* Add a reference if we're newly entering the active list. */
		1771	if (!obj->active) {
2344	Serge	1772	drm_gem_object_reference(&obj->base);
2332	Serge	1773	obj->active = 1;
		1774	}
		1775
		1776	list_move_tail(&obj->ring_list, &ring->active_list);
		1777
3031	serge	1778	obj->last_read_seqno = seqno;
		1779
2332	Serge	1780	if (obj->fenced_gpu_access) {
3031	serge	1781	obj->last_fenced_seqno = seqno;
		1782
		1783	/* Bump MRU to take account of the delayed flush */
		1784	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1785	struct drm_i915_fence_reg *reg;
		1786
		1787	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1788	list_move_tail(®->lru_list,
		1789	&dev_priv->mm.fence_list);
		1790	}
2332	Serge	1791	}
		1792	}
		1793
4560	Serge	1794	void i915_vma_move_to_active(struct i915_vma *vma,
5060	serge	1795	struct intel_engine_cs *ring)
4560	Serge	1796	{
		1797	list_move_tail(&vma->mm_list, &vma->vm->active_list);
		1798	return i915_gem_object_move_to_active(vma->obj, ring);
		1799	}
		1800
2344	Serge	1801	static void
3031	serge	1802	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1803	{
4104	Serge	1804	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
5060	serge	1805	struct i915_address_space *vm;
		1806	struct i915_vma *vma;
2332	Serge	1807
3031	serge	1808	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1809	BUG_ON(!obj->active);
2332	Serge	1810
5060	serge	1811	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
		1812	vma = i915_gem_obj_to_vma(obj, vm);
		1813	if (vma && !list_empty(&vma->mm_list))
		1814	list_move_tail(&vma->mm_list, &vm->inactive_list);
		1815	}
2344	Serge	1816
3031	serge	1817	list_del_init(&obj->ring_list);
2352	Serge	1818	obj->ring = NULL;
2344	Serge	1819
3031	serge	1820	obj->last_read_seqno = 0;
		1821	obj->last_write_seqno = 0;
		1822	obj->base.write_domain = 0;
		1823
		1824	obj->last_fenced_seqno = 0;
2352	Serge	1825	obj->fenced_gpu_access = false;
2344	Serge	1826
2352	Serge	1827	obj->active = 0;
		1828	drm_gem_object_unreference(&obj->base);
		1829
		1830	WARN_ON(i915_verify_lists(dev));
		1831	}
		1832
5060	serge	1833	static void
		1834	i915_gem_object_retire(struct drm_i915_gem_object *obj)
		1835	{
		1836	struct intel_engine_cs *ring = obj->ring;
		1837
		1838	if (ring == NULL)
		1839	return;
		1840
		1841	if (i915_seqno_passed(ring->get_seqno(ring, true),
		1842	obj->last_read_seqno))
		1843	i915_gem_object_move_to_inactive(obj);
		1844	}
		1845
3243	Serge	1846	static int
3480	Serge	1847	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1848	{
3243	Serge	1849	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	1850	struct intel_engine_cs *ring;
3243	Serge	1851	int ret, i, j;
2344	Serge	1852
3480	Serge	1853	/* Carefully retire all requests without writing to the rings */
3243	Serge	1854	for_each_ring(ring, dev_priv, i) {
3480	Serge	1855	ret = intel_ring_idle(ring);
3243	Serge	1856	if (ret)
		1857	return ret;
3480	Serge	1858	}
		1859	i915_gem_retire_requests(dev);
3243	Serge	1860
3480	Serge	1861	/* Finally reset hw state */
3243	Serge	1862	for_each_ring(ring, dev_priv, i) {
3480	Serge	1863	intel_ring_init_seqno(ring, seqno);
		1864
5060	serge	1865	for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
		1866	ring->semaphore.sync_seqno[j] = 0;
3243	Serge	1867	}
		1868
		1869	return 0;
2344	Serge	1870	}
		1871
3480	Serge	1872	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1873	{
		1874	struct drm_i915_private *dev_priv = dev->dev_private;
		1875	int ret;
		1876
		1877	if (seqno == 0)
		1878	return -EINVAL;
		1879
		1880	/* HWS page needs to be set less than what we
		1881	* will inject to ring
		1882	*/
		1883	ret = i915_gem_init_seqno(dev, seqno - 1);
		1884	if (ret)
		1885	return ret;
		1886
		1887	/* Carefully set the last_seqno value so that wrap
		1888	* detection still works
		1889	*/
		1890	dev_priv->next_seqno = seqno;
		1891	dev_priv->last_seqno = seqno - 1;
		1892	if (dev_priv->last_seqno == 0)
		1893	dev_priv->last_seqno--;
		1894
		1895	return 0;
		1896	}
		1897
3243	Serge	1898	int
		1899	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1900	{
3243	Serge	1901	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1902
3243	Serge	1903	/* reserve 0 for non-seqno */
		1904	if (dev_priv->next_seqno == 0) {
3480	Serge	1905	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1906	if (ret)
		1907	return ret;
		1908
		1909	dev_priv->next_seqno = 1;
		1910	}
		1911
3480	Serge	1912	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1913	return 0;
2332	Serge	1914	}
		1915
5060	serge	1916	int __i915_add_request(struct intel_engine_cs *ring,
2352	Serge	1917	struct drm_file *file,
4104	Serge	1918	struct drm_i915_gem_object *obj,
3031	serge	1919	u32 *out_seqno)
2352	Serge	1920	{
5060	serge	1921	struct drm_i915_private *dev_priv = ring->dev->dev_private;
3031	serge	1922	struct drm_i915_gem_request *request;
4104	Serge	1923	u32 request_ring_position, request_start;
2352	Serge	1924	int ret;
2332	Serge	1925
5060	serge	1926	request_start = intel_ring_get_tail(ring->buffer);
3031	serge	1927	/*
		1928	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1929	* after having emitted the batchbuffer command. Hence we need to fix
		1930	* things up similar to emitting the lazy request. The difference here
		1931	* is that the flush _must_ happen before the next request, no matter
		1932	* what.
		1933	*/
4104	Serge	1934	ret = intel_ring_flush_all_caches(ring);
		1935	if (ret)
		1936	return ret;
2332	Serge	1937
4560	Serge	1938	request = ring->preallocated_lazy_request;
		1939	if (WARN_ON(request == NULL))
3031	serge	1940	return -ENOMEM;
		1941
		1942	/* Record the position of the start of the request so that
		1943	* should we detect the updated seqno part-way through the
4104	Serge	1944	* GPU processing the request, we never over-estimate the
3031	serge	1945	* position of the head.
		1946	*/
5060	serge	1947	request_ring_position = intel_ring_get_tail(ring->buffer);
3031	serge	1948
3243	Serge	1949	ret = ring->add_request(ring);
4560	Serge	1950	if (ret)
4104	Serge	1951	return ret;
2332	Serge	1952
3243	Serge	1953	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1954	request->ring = ring;
4104	Serge	1955	request->head = request_start;
3031	serge	1956	request->tail = request_ring_position;
4104	Serge	1957
		1958	/* Whilst this request exists, batch_obj will be on the
		1959	* active_list, and so will hold the active reference. Only when this
		1960	* request is retired will the the batch_obj be moved onto the
		1961	* inactive_list and lose its active reference. Hence we do not need
		1962	* to explicitly hold another reference here.
		1963	*/
4560	Serge	1964	request->batch_obj = obj;
4104	Serge	1965
4560	Serge	1966	/* Hold a reference to the current context so that we can inspect
		1967	* it later in case a hangcheck error event fires.
		1968	*/
		1969	request->ctx = ring->last_context;
4104	Serge	1970	if (request->ctx)
		1971	i915_gem_context_reference(request->ctx);
		1972
5060	serge	1973	request->emitted_jiffies = jiffies;
2352	Serge	1974	list_add_tail(&request->list, &ring->request_list);
3031	serge	1975	request->file_priv = NULL;
2332	Serge	1976
3263	Serge	1977	if (file) {
		1978	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1979
3263	Serge	1980	spin_lock(&file_priv->mm.lock);
		1981	request->file_priv = file_priv;
		1982	list_add_tail(&request->client_list,
		1983	&file_priv->mm.request_list);
		1984	spin_unlock(&file_priv->mm.lock);
		1985	}
		1986
		1987	trace_i915_gem_request_add(ring, request->seqno);
4560	Serge	1988	ring->outstanding_lazy_seqno = 0;
		1989	ring->preallocated_lazy_request = NULL;
2332	Serge	1990
4104	Serge	1991	if (!dev_priv->ums.mm_suspended) {
		1992	// i915_queue_hangcheck(ring->dev);
		1993
2360	Serge	1994	queue_delayed_work(dev_priv->wq,
3482	Serge	1995	&dev_priv->mm.retire_work,
		1996	round_jiffies_up_relative(HZ));
4104	Serge	1997	intel_mark_busy(dev_priv->dev);
		1998	}
3031	serge	1999
		2000	if (out_seqno)
3243	Serge	2001	*out_seqno = request->seqno;
2352	Serge	2002	return 0;
		2003	}
2332	Serge	2004
3263	Serge	2005	static inline void
		2006	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		2007	{
		2008	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	2009
3263	Serge	2010	if (!file_priv)
		2011	return;
2332	Serge	2012
3263	Serge	2013	spin_lock(&file_priv->mm.lock);
		2014	list_del(&request->client_list);
		2015	request->file_priv = NULL;
		2016	spin_unlock(&file_priv->mm.lock);
		2017	}
2332	Serge	2018
5060	serge	2019	static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
		2020	const struct intel_context *ctx)
4104	Serge	2021	{
5060	serge	2022	unsigned long elapsed;
4104	Serge	2023
5060	serge	2024	elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts;
4104	Serge	2025
5060	serge	2026	if (ctx->hang_stats.banned)
		2027	return true;
4104	Serge	2028
5060	serge	2029	if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
		2030	if (!i915_gem_context_is_default(ctx)) {
		2031	DRM_DEBUG("context hanging too fast, banning!\n");
4104	Serge	2032	return true;
5060	serge	2033	} else if (i915_stop_ring_allow_ban(dev_priv)) {
		2034	if (i915_stop_ring_allow_warn(dev_priv))
		2035	DRM_ERROR("gpu hanging too fast, banning!\n");
4104	Serge	2036	return true;
		2037	}
		2038	}
		2039
		2040	return false;
		2041	}
		2042
5060	serge	2043	static void i915_set_reset_status(struct drm_i915_private *dev_priv,
		2044	struct intel_context *ctx,
		2045	const bool guilty)
4560	Serge	2046	{
5060	serge	2047	struct i915_ctx_hang_stats *hs;
4560	Serge	2048
5060	serge	2049	if (WARN_ON(!ctx))
		2050	return;
4560	Serge	2051
5060	serge	2052	hs = &ctx->hang_stats;
4560	Serge	2053
5060	serge	2054	if (guilty) {
		2055	hs->banned = i915_context_is_banned(dev_priv, ctx);
		2056	hs->batch_active++;
		2057	hs->guilty_ts = GetTimerTicks()/100;
		2058	} else {
		2059	hs->batch_pending++;
4104	Serge	2060	}
		2061	}
		2062
		2063	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2064	{
		2065	list_del(&request->list);
		2066	i915_gem_request_remove_from_client(request);
		2067
		2068	if (request->ctx)
		2069	i915_gem_context_unreference(request->ctx);
		2070
		2071	kfree(request);
		2072	}
		2073
5060	serge	2074	struct drm_i915_gem_request *
		2075	i915_gem_find_active_request(struct intel_engine_cs *ring)
3031	serge	2076	{
4539	Serge	2077	struct drm_i915_gem_request *request;
5060	serge	2078	u32 completed_seqno;
4104	Serge	2079
5060	serge	2080	completed_seqno = ring->get_seqno(ring, false);
		2081
4539	Serge	2082	list_for_each_entry(request, &ring->request_list, list) {
		2083	if (i915_seqno_passed(completed_seqno, request->seqno))
		2084	continue;
4104	Serge	2085
5060	serge	2086	return request;
4539	Serge	2087	}
5060	serge	2088
		2089	return NULL;
4539	Serge	2090	}
		2091
5060	serge	2092	static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
		2093	struct intel_engine_cs *ring)
		2094	{
		2095	struct drm_i915_gem_request *request;
		2096	bool ring_hung;
		2097
		2098	request = i915_gem_find_active_request(ring);
		2099
		2100	if (request == NULL)
		2101	return;
		2102
		2103	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
		2104
		2105	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
		2106
		2107	list_for_each_entry_continue(request, &ring->request_list, list)
		2108	i915_set_reset_status(dev_priv, request->ctx, false);
		2109	}
		2110
4539	Serge	2111	static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
5060	serge	2112	struct intel_engine_cs *ring)
4539	Serge	2113	{
4560	Serge	2114	while (!list_empty(&ring->active_list)) {
		2115	struct drm_i915_gem_object *obj;
		2116
		2117	obj = list_first_entry(&ring->active_list,
		2118	struct drm_i915_gem_object,
		2119	ring_list);
		2120
		2121	i915_gem_object_move_to_inactive(obj);
		2122	}
		2123
		2124	/*
		2125	* We must free the requests after all the corresponding objects have
		2126	* been moved off active lists. Which is the same order as the normal
		2127	* retire_requests function does. This is important if object hold
		2128	* implicit references on things like e.g. ppgtt address spaces through
		2129	* the request.
		2130	*/
3031	serge	2131	while (!list_empty(&ring->request_list)) {
		2132	struct drm_i915_gem_request *request;
2332	Serge	2133
3031	serge	2134	request = list_first_entry(&ring->request_list,
		2135	struct drm_i915_gem_request,
		2136	list);
2332	Serge	2137
4104	Serge	2138	i915_gem_free_request(request);
3031	serge	2139	}
5060	serge	2140
		2141	/* These may not have been flush before the reset, do so now */
		2142	kfree(ring->preallocated_lazy_request);
		2143	ring->preallocated_lazy_request = NULL;
		2144	ring->outstanding_lazy_seqno = 0;
3031	serge	2145	}
2332	Serge	2146
3746	Serge	2147	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2148	{
		2149	struct drm_i915_private *dev_priv = dev->dev_private;
		2150	int i;
2332	Serge	2151
3031	serge	2152	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2153	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2154
		2155	/*
		2156	* Commit delayed tiling changes if we have an object still
		2157	* attached to the fence, otherwise just clear the fence.
		2158	*/
		2159	if (reg->obj) {
		2160	i915_gem_object_update_fence(reg->obj, reg,
		2161	reg->obj->tiling_mode);
		2162	} else {
		2163	i915_gem_write_fence(dev, i, NULL);
		2164	}
3031	serge	2165	}
		2166	}
2360	Serge	2167
3031	serge	2168	void i915_gem_reset(struct drm_device *dev)
		2169	{
		2170	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2171	struct intel_engine_cs *ring;
3031	serge	2172	int i;
2360	Serge	2173
4539	Serge	2174	/*
		2175	* Before we free the objects from the requests, we need to inspect
		2176	* them for finding the guilty party. As the requests only borrow
		2177	* their reference to the objects, the inspection must be done first.
		2178	*/
3031	serge	2179	for_each_ring(ring, dev_priv, i)
4539	Serge	2180	i915_gem_reset_ring_status(dev_priv, ring);
2360	Serge	2181
4539	Serge	2182	for_each_ring(ring, dev_priv, i)
		2183	i915_gem_reset_ring_cleanup(dev_priv, ring);
		2184
5060	serge	2185	i915_gem_context_reset(dev);
4560	Serge	2186
3746	Serge	2187	i915_gem_restore_fences(dev);
3031	serge	2188	}
2360	Serge	2189
2352	Serge	2190	/**
		2191	* This function clears the request list as sequence numbers are passed.
		2192	*/
3031	serge	2193	void
5060	serge	2194	i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2352	Serge	2195	{
		2196	uint32_t seqno;
2332	Serge	2197
2352	Serge	2198	if (list_empty(&ring->request_list))
		2199	return;
2332	Serge	2200
2352	Serge	2201	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2202
3031	serge	2203	seqno = ring->get_seqno(ring, true);
2332	Serge	2204
5060	serge	2205	/* Move any buffers on the active list that are no longer referenced
		2206	* by the ringbuffer to the flushing/inactive lists as appropriate,
		2207	* before we free the context associated with the requests.
		2208	*/
		2209	while (!list_empty(&ring->active_list)) {
		2210	struct drm_i915_gem_object *obj;
		2211
		2212	obj = list_first_entry(&ring->active_list,
		2213	struct drm_i915_gem_object,
		2214	ring_list);
		2215
		2216	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
		2217	break;
		2218
		2219	i915_gem_object_move_to_inactive(obj);
		2220	}
		2221
		2222
2352	Serge	2223	while (!list_empty(&ring->request_list)) {
		2224	struct drm_i915_gem_request *request;
2332	Serge	2225
2352	Serge	2226	request = list_first_entry(&ring->request_list,
		2227	struct drm_i915_gem_request,
		2228	list);
2332	Serge	2229
2352	Serge	2230	if (!i915_seqno_passed(seqno, request->seqno))
		2231	break;
2332	Serge	2232
2352	Serge	2233	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2234	/* We know the GPU must have read the request to have
		2235	* sent us the seqno + interrupt, so use the position
		2236	* of tail of the request to update the last known position
		2237	* of the GPU head.
		2238	*/
5060	serge	2239	ring->buffer->last_retired_head = request->tail;
2332	Serge	2240
4104	Serge	2241	i915_gem_free_request(request);
2352	Serge	2242	}
2332	Serge	2243
2352	Serge	2244	if (unlikely(ring->trace_irq_seqno &&
		2245	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2246	ring->irq_put(ring);
		2247	ring->trace_irq_seqno = 0;
		2248	}
2332	Serge	2249
2352	Serge	2250	WARN_ON(i915_verify_lists(ring->dev));
		2251	}
2332	Serge	2252
4560	Serge	2253	bool
2352	Serge	2254	i915_gem_retire_requests(struct drm_device *dev)
		2255	{
5060	serge	2256	struct drm_i915_private *dev_priv = dev->dev_private;
		2257	struct intel_engine_cs *ring;
4560	Serge	2258	bool idle = true;
2352	Serge	2259	int i;
2332	Serge	2260
4560	Serge	2261	for_each_ring(ring, dev_priv, i) {
3031	serge	2262	i915_gem_retire_requests_ring(ring);
4560	Serge	2263	idle &= list_empty(&ring->request_list);
		2264	}
		2265
		2266	if (idle)
		2267	mod_delayed_work(dev_priv->wq,
		2268	&dev_priv->mm.idle_work,
		2269	msecs_to_jiffies(100));
		2270
		2271	return idle;
2352	Serge	2272	}
		2273
2360	Serge	2274	static void
		2275	i915_gem_retire_work_handler(struct work_struct *work)
		2276	{
4560	Serge	2277	struct drm_i915_private *dev_priv =
		2278	container_of(work, typeof(*dev_priv), mm.retire_work.work);
		2279	struct drm_device *dev = dev_priv->dev;
2360	Serge	2280	bool idle;
2352	Serge	2281
2360	Serge	2282	/* Come back later if the device is busy... */
4560	Serge	2283	idle = false;
		2284	if (mutex_trylock(&dev->struct_mutex)) {
		2285	idle = i915_gem_retire_requests(dev);
		2286	mutex_unlock(&dev->struct_mutex);
		2287	}
		2288	if (!idle)
3482	Serge	2289	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2290	round_jiffies_up_relative(HZ));
4560	Serge	2291	}
2352	Serge	2292
4560	Serge	2293	static void
		2294	i915_gem_idle_work_handler(struct work_struct *work)
		2295	{
		2296	struct drm_i915_private *dev_priv =
		2297	container_of(work, typeof(*dev_priv), mm.idle_work.work);
2352	Serge	2298
4560	Serge	2299	intel_mark_idle(dev_priv->dev);
2360	Serge	2300	}
		2301
2344	Serge	2302	/**
3031	serge	2303	* Ensures that an object will eventually get non-busy by flushing any required
		2304	* write domains, emitting any outstanding lazy request and retiring and
		2305	* completed requests.
2352	Serge	2306	*/
3031	serge	2307	static int
		2308	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2309	{
3031	serge	2310	int ret;
2352	Serge	2311
3031	serge	2312	if (obj->active) {
		2313	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2314	if (ret)
		2315	return ret;
2352	Serge	2316
3031	serge	2317	i915_gem_retire_requests_ring(obj->ring);
		2318	}
2352	Serge	2319
3031	serge	2320	return 0;
		2321	}
2352	Serge	2322
3243	Serge	2323	/**
		2324	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2325	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2326	*
		2327	* Returns 0 if successful, else an error is returned with the remaining time in
		2328	* the timeout parameter.
		2329	* -ETIME: object is still busy after timeout
		2330	* -ERESTARTSYS: signal interrupted the wait
		2331	* -ENONENT: object doesn't exist
		2332	* Also possible, but rare:
		2333	* -EAGAIN: GPU wedged
		2334	* -ENOMEM: damn
		2335	* -ENODEV: Internal IRQ fail
		2336	* -E?: The add request failed
		2337	*
		2338	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2339	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2340	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2341	* without holding struct_mutex the object may become re-busied before this
		2342	* function completes. A similar but shorter * race condition exists in the busy
		2343	* ioctl
		2344	*/
4246	Serge	2345	int
		2346	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2347	{
5060	serge	2348	struct drm_i915_private *dev_priv = dev->dev_private;
4246	Serge	2349	struct drm_i915_gem_wait *args = data;
		2350	struct drm_i915_gem_object *obj;
5060	serge	2351	struct intel_engine_cs *ring = NULL;
4246	Serge	2352	unsigned reset_counter;
		2353	u32 seqno = 0;
		2354	int ret = 0;
2352	Serge	2355
4246	Serge	2356	ret = i915_mutex_lock_interruptible(dev);
		2357	if (ret)
		2358	return ret;
2352	Serge	2359
4246	Serge	2360	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2361	if (&obj->base == NULL) {
		2362	mutex_unlock(&dev->struct_mutex);
		2363	return -ENOENT;
		2364	}
2352	Serge	2365
4246	Serge	2366	/* Need to make sure the object gets inactive eventually. */
		2367	ret = i915_gem_object_flush_active(obj);
		2368	if (ret)
		2369	goto out;
2352	Serge	2370
4246	Serge	2371	if (obj->active) {
		2372	seqno = obj->last_read_seqno;
		2373	ring = obj->ring;
		2374	}
2352	Serge	2375
4246	Serge	2376	if (seqno == 0)
		2377	goto out;
2352	Serge	2378
4246	Serge	2379	/* Do this after OLR check to make sure we make forward progress polling
5060	serge	2380	* on this IOCTL with a timeout <=0 (like busy ioctl)
4246	Serge	2381	*/
5060	serge	2382	if (args->timeout_ns <= 0) {
4246	Serge	2383	ret = -ETIME;
		2384	goto out;
		2385	}
2352	Serge	2386
4246	Serge	2387	drm_gem_object_unreference(&obj->base);
		2388	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		2389	mutex_unlock(&dev->struct_mutex);
2352	Serge	2390
5060	serge	2391	return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns,
		2392	file->driver_priv);
3243	Serge	2393
4246	Serge	2394	out:
		2395	drm_gem_object_unreference(&obj->base);
		2396	mutex_unlock(&dev->struct_mutex);
		2397	return ret;
		2398	}
3243	Serge	2399
2352	Serge	2400	/**
3031	serge	2401	* i915_gem_object_sync - sync an object to a ring.
		2402	*
		2403	* @obj: object which may be in use on another ring.
		2404	* @to: ring we wish to use the object on. May be NULL.
		2405	*
		2406	* This code is meant to abstract object synchronization with the GPU.
		2407	* Calling with NULL implies synchronizing the object with the CPU
		2408	* rather than a particular GPU ring.
		2409	*
		2410	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2411	*/
		2412	int
3031	serge	2413	i915_gem_object_sync(struct drm_i915_gem_object *obj,
5060	serge	2414	struct intel_engine_cs *to)
2344	Serge	2415	{
5060	serge	2416	struct intel_engine_cs *from = obj->ring;
3031	serge	2417	u32 seqno;
		2418	int ret, idx;
2332	Serge	2419
3031	serge	2420	if (from == NULL \|\| to == from)
		2421	return 0;
2332	Serge	2422
3031	serge	2423	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2424	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2425
3031	serge	2426	idx = intel_ring_sync_index(from, to);
		2427
		2428	seqno = obj->last_read_seqno;
5060	serge	2429	/* Optimization: Avoid semaphore sync when we are sure we already
		2430	* waited for an object with higher seqno */
		2431	if (seqno <= from->semaphore.sync_seqno[idx])
3031	serge	2432	return 0;
		2433
		2434	ret = i915_gem_check_olr(obj->ring, seqno);
		2435	if (ret)
		2436	return ret;
		2437
4560	Serge	2438	trace_i915_gem_ring_sync_to(from, to, seqno);
5060	serge	2439	ret = to->semaphore.sync_to(to, from, seqno);
3031	serge	2440	if (!ret)
3243	Serge	2441	/* We use last_read_seqno because sync_to()
		2442	* might have just caused seqno wrap under
		2443	* the radar.
		2444	*/
5060	serge	2445	from->semaphore.sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2446
		2447	return ret;
2344	Serge	2448	}
2332	Serge	2449
2344	Serge	2450	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2451	{
		2452	u32 old_write_domain, old_read_domains;
2332	Serge	2453
2344	Serge	2454	/* Force a pagefault for domain tracking on next user access */
		2455	// i915_gem_release_mmap(obj);
2332	Serge	2456
2344	Serge	2457	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2458	return;
2332	Serge	2459
3480	Serge	2460	/* Wait for any direct GTT access to complete */
		2461	mb();
		2462
2344	Serge	2463	old_read_domains = obj->base.read_domains;
		2464	old_write_domain = obj->base.write_domain;
2351	Serge	2465
2344	Serge	2466	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2467	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2468
2351	Serge	2469	trace_i915_gem_object_change_domain(obj,
		2470	old_read_domains,
		2471	old_write_domain);
2344	Serge	2472	}
2332	Serge	2473
4104	Serge	2474	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2475	{
4104	Serge	2476	struct drm_i915_gem_object *obj = vma->obj;
5060	serge	2477	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3480	Serge	2478	int ret;
2332	Serge	2479
3263	Serge	2480	if(obj == get_fb_obj())
		2481	return 0;
		2482
4104	Serge	2483	if (list_empty(&vma->vma_link))
2344	Serge	2484	return 0;
2332	Serge	2485
4560	Serge	2486	if (!drm_mm_node_allocated(&vma->node)) {
		2487	i915_gem_vma_destroy(vma);
		2488	return 0;
		2489	}
		2490
5060	serge	2491	if (vma->pin_count)
3031	serge	2492	return -EBUSY;
2332	Serge	2493
3243	Serge	2494	BUG_ON(obj->pages == NULL);
3031	serge	2495
2344	Serge	2496	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2497	if (ret)
2344	Serge	2498	return ret;
		2499	/* Continue on if we fail due to EIO, the GPU is hung so we
		2500	* should be safe and we need to cleanup or else we might
		2501	* cause memory corruption through use-after-free.
		2502	*/
2332	Serge	2503
5060	serge	2504	if (i915_is_ggtt(vma->vm)) {
2344	Serge	2505	i915_gem_object_finish_gtt(obj);
2332	Serge	2506
2344	Serge	2507	/* release the fence reg _after_ flushing */
		2508	ret = i915_gem_object_put_fence(obj);
3031	serge	2509	if (ret)
2344	Serge	2510	return ret;
5060	serge	2511	}
2332	Serge	2512
4104	Serge	2513	trace_i915_vma_unbind(vma);
2332	Serge	2514
5060	serge	2515	vma->unbind_vma(vma);
2332	Serge	2516
5060	serge	2517	list_del_init(&vma->mm_list);
2344	Serge	2518	/* Avoid an unnecessary call to unbind on rebind. */
4104	Serge	2519	if (i915_is_ggtt(vma->vm))
2344	Serge	2520	obj->map_and_fenceable = true;
2332	Serge	2521
4104	Serge	2522	drm_mm_remove_node(&vma->node);
		2523	i915_gem_vma_destroy(vma);
		2524
		2525	/* Since the unbound list is global, only move to that list if
4560	Serge	2526	* no more VMAs exist. */
5060	serge	2527	if (list_empty(&obj->vma_list)) {
		2528	i915_gem_gtt_finish_object(obj);
4104	Serge	2529	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
5060	serge	2530	}
4104	Serge	2531
4560	Serge	2532	/* And finally now the object is completely decoupled from this vma,
		2533	* we can drop its hold on the backing storage and allow it to be
		2534	* reaped by the shrinker.
		2535	*/
		2536	i915_gem_object_unpin_pages(obj);
		2537
2344	Serge	2538	return 0;
		2539	}
2332	Serge	2540
3031	serge	2541	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2542	{
5060	serge	2543	struct drm_i915_private *dev_priv = dev->dev_private;
		2544	struct intel_engine_cs *ring;
2344	Serge	2545	int ret, i;
2332	Serge	2546
2344	Serge	2547	/* Flush everything onto the inactive list. */
3031	serge	2548	for_each_ring(ring, dev_priv, i) {
5060	serge	2549	ret = i915_switch_context(ring, ring->default_context);
2344	Serge	2550	if (ret)
		2551	return ret;
3031	serge	2552
3243	Serge	2553	ret = intel_ring_idle(ring);
3031	serge	2554	if (ret)
		2555	return ret;
2344	Serge	2556	}
2332	Serge	2557
2344	Serge	2558	return 0;
		2559	}
2332	Serge	2560
3480	Serge	2561	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2562	struct drm_i915_gem_object *obj)
		2563	{
5060	serge	2564	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	2565	int fence_reg;
		2566	int fence_pitch_shift;
2332	Serge	2567
3480	Serge	2568	if (INTEL_INFO(dev)->gen >= 6) {
		2569	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2570	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2571	} else {
		2572	fence_reg = FENCE_REG_965_0;
		2573	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2574	}
2332	Serge	2575
4104	Serge	2576	fence_reg += reg * 8;
		2577
		2578	/* To w/a incoherency with non-atomic 64-bit register updates,
		2579	* we split the 64-bit update into two 32-bit writes. In order
		2580	* for a partial fence not to be evaluated between writes, we
		2581	* precede the update with write to turn off the fence register,
		2582	* and only enable the fence as the last step.
		2583	*
		2584	* For extra levels of paranoia, we make sure each step lands
		2585	* before applying the next step.
		2586	*/
		2587	I915_WRITE(fence_reg, 0);
		2588	POSTING_READ(fence_reg);
		2589
3031	serge	2590	if (obj) {
4104	Serge	2591	u32 size = i915_gem_obj_ggtt_size(obj);
		2592	uint64_t val;
2332	Serge	2593
4104	Serge	2594	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2595	0xfffff000) << 32;
4104	Serge	2596	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2597	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2598	if (obj->tiling_mode == I915_TILING_Y)
		2599	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2600	val \|= I965_FENCE_REG_VALID;
2332	Serge	2601
4104	Serge	2602	I915_WRITE(fence_reg + 4, val >> 32);
		2603	POSTING_READ(fence_reg + 4);
		2604
		2605	I915_WRITE(fence_reg + 0, val);
5060	serge	2606	POSTING_READ(fence_reg);
4104	Serge	2607	} else {
		2608	I915_WRITE(fence_reg + 4, 0);
		2609	POSTING_READ(fence_reg + 4);
		2610	}
3031	serge	2611	}
2332	Serge	2612
3031	serge	2613	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2614	struct drm_i915_gem_object *obj)
		2615	{
5060	serge	2616	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2617	u32 val;
2332	Serge	2618
3031	serge	2619	if (obj) {
4104	Serge	2620	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2621	int pitch_val;
		2622	int tile_width;
2332	Serge	2623
4104	Serge	2624	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2625	(size & -size) != size \|\|
4104	Serge	2626	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2627	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2628	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2629
3031	serge	2630	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2631	tile_width = 128;
		2632	else
		2633	tile_width = 512;
2332	Serge	2634
3031	serge	2635	/* Note: pitch better be a power of two tile widths */
		2636	pitch_val = obj->stride / tile_width;
		2637	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2638
4104	Serge	2639	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2640	if (obj->tiling_mode == I915_TILING_Y)
		2641	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2642	val \|= I915_FENCE_SIZE_BITS(size);
		2643	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2644	val \|= I830_FENCE_REG_VALID;
		2645	} else
		2646	val = 0;
2332	Serge	2647
3031	serge	2648	if (reg < 8)
		2649	reg = FENCE_REG_830_0 + reg * 4;
		2650	else
		2651	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2652
3031	serge	2653	I915_WRITE(reg, val);
		2654	POSTING_READ(reg);
		2655	}
2332	Serge	2656
3031	serge	2657	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2658	struct drm_i915_gem_object *obj)
		2659	{
5060	serge	2660	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2661	uint32_t val;
2344	Serge	2662
3031	serge	2663	if (obj) {
4104	Serge	2664	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2665	uint32_t pitch_val;
2344	Serge	2666
4104	Serge	2667	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2668	(size & -size) != size \|\|
4104	Serge	2669	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2670	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2671	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2672
3031	serge	2673	pitch_val = obj->stride / 128;
		2674	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2675
4104	Serge	2676	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2677	if (obj->tiling_mode == I915_TILING_Y)
		2678	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2679	val \|= I830_FENCE_SIZE_BITS(size);
		2680	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2681	val \|= I830_FENCE_REG_VALID;
		2682	} else
		2683	val = 0;
		2684
		2685	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2686	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2687	}
		2688
3480	Serge	2689	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2690	{
		2691	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2692	}
		2693
3031	serge	2694	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2695	struct drm_i915_gem_object *obj)
2332	Serge	2696	{
3480	Serge	2697	struct drm_i915_private *dev_priv = dev->dev_private;
		2698
		2699	/* Ensure that all CPU reads are completed before installing a fence
		2700	* and all writes before removing the fence.
		2701	*/
		2702	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2703	mb();
		2704
4104	Serge	2705	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2706	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2707	obj->stride, obj->tiling_mode);
		2708
3031	serge	2709	switch (INTEL_INFO(dev)->gen) {
4560	Serge	2710	case 8:
3031	serge	2711	case 7:
3480	Serge	2712	case 6:
3031	serge	2713	case 5:
		2714	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2715	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2716	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2717	default: BUG();
3031	serge	2718	}
3480	Serge	2719
		2720	/* And similarly be paranoid that no direct access to this region
		2721	* is reordered to before the fence is installed.
		2722	*/
		2723	if (i915_gem_object_needs_mb(obj))
		2724	mb();
2344	Serge	2725	}
		2726
3031	serge	2727	static inline int fence_number(struct drm_i915_private *dev_priv,
		2728	struct drm_i915_fence_reg *fence)
2344	Serge	2729	{
3031	serge	2730	return fence - dev_priv->fence_regs;
		2731	}
2332	Serge	2732
3031	serge	2733	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2734	struct drm_i915_fence_reg *fence,
		2735	bool enable)
		2736	{
4104	Serge	2737	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2738	int reg = fence_number(dev_priv, fence);
2332	Serge	2739
4104	Serge	2740	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2741
		2742	if (enable) {
4104	Serge	2743	obj->fence_reg = reg;
3031	serge	2744	fence->obj = obj;
		2745	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2746	} else {
		2747	obj->fence_reg = I915_FENCE_REG_NONE;
		2748	fence->obj = NULL;
		2749	list_del_init(&fence->lru_list);
2344	Serge	2750	}
4104	Serge	2751	obj->fence_dirty = false;
3031	serge	2752	}
2344	Serge	2753
3031	serge	2754	static int
3480	Serge	2755	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2756	{
		2757	if (obj->last_fenced_seqno) {
		2758	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2759	if (ret)
		2760	return ret;
2344	Serge	2761
		2762	obj->last_fenced_seqno = 0;
		2763	}
		2764
3031	serge	2765	obj->fenced_gpu_access = false;
2332	Serge	2766	return 0;
		2767	}
		2768
		2769	int
2344	Serge	2770	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2771	{
3031	serge	2772	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2773	struct drm_i915_fence_reg *fence;
2332	Serge	2774	int ret;
		2775
3480	Serge	2776	ret = i915_gem_object_wait_fence(obj);
5060	serge	2777	if (ret)
		2778	return ret;
2332	Serge	2779
3031	serge	2780	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2781	return 0;
2332	Serge	2782
3746	Serge	2783	fence = &dev_priv->fence_regs[obj->fence_reg];
		2784
5060	serge	2785	if (WARN_ON(fence->pin_count))
		2786	return -EBUSY;
		2787
3031	serge	2788	i915_gem_object_fence_lost(obj);
3746	Serge	2789	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2790
2332	Serge	2791	return 0;
		2792	}
		2793
3031	serge	2794	static struct drm_i915_fence_reg *
		2795	i915_find_fence_reg(struct drm_device *dev)
		2796	{
		2797	struct drm_i915_private *dev_priv = dev->dev_private;
		2798	struct drm_i915_fence_reg reg, avail;
		2799	int i;
2332	Serge	2800
3031	serge	2801	/* First try to find a free reg */
		2802	avail = NULL;
		2803	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2804	reg = &dev_priv->fence_regs[i];
		2805	if (!reg->obj)
		2806	return reg;
2332	Serge	2807
3031	serge	2808	if (!reg->pin_count)
		2809	avail = reg;
		2810	}
2332	Serge	2811
3031	serge	2812	if (avail == NULL)
4560	Serge	2813	goto deadlock;
2332	Serge	2814
3031	serge	2815	/* None available, try to steal one or wait for a user to finish */
		2816	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2817	if (reg->pin_count)
		2818	continue;
2332	Serge	2819
3031	serge	2820	return reg;
		2821	}
2332	Serge	2822
4560	Serge	2823	deadlock:
		2824	/* Wait for completion of pending flips which consume fences */
		2825	// if (intel_has_pending_fb_unpin(dev))
		2826	// return ERR_PTR(-EAGAIN);
		2827
		2828	return ERR_PTR(-EDEADLK);
3031	serge	2829	}
2332	Serge	2830
3031	serge	2831	/**
		2832	* i915_gem_object_get_fence - set up fencing for an object
		2833	* @obj: object to map through a fence reg
		2834	*
		2835	* When mapping objects through the GTT, userspace wants to be able to write
		2836	* to them without having to worry about swizzling if the object is tiled.
		2837	* This function walks the fence regs looking for a free one for @obj,
		2838	* stealing one if it can't find any.
		2839	*
		2840	* It then sets up the reg based on the object's properties: address, pitch
		2841	* and tiling format.
		2842	*
		2843	* For an untiled surface, this removes any existing fence.
		2844	*/
		2845	int
		2846	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2847	{
		2848	struct drm_device *dev = obj->base.dev;
		2849	struct drm_i915_private *dev_priv = dev->dev_private;
		2850	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2851	struct drm_i915_fence_reg *reg;
		2852	int ret;
2332	Serge	2853
3031	serge	2854	/* Have we updated the tiling parameters upon the object and so
		2855	* will need to serialise the write to the associated fence register?
		2856	*/
		2857	if (obj->fence_dirty) {
3480	Serge	2858	ret = i915_gem_object_wait_fence(obj);
3031	serge	2859	if (ret)
		2860	return ret;
		2861	}
2332	Serge	2862
3031	serge	2863	/* Just update our place in the LRU if our fence is getting reused. */
		2864	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2865	reg = &dev_priv->fence_regs[obj->fence_reg];
		2866	if (!obj->fence_dirty) {
		2867	list_move_tail(®->lru_list,
		2868	&dev_priv->mm.fence_list);
		2869	return 0;
		2870	}
		2871	} else if (enable) {
		2872	reg = i915_find_fence_reg(dev);
4560	Serge	2873	if (IS_ERR(reg))
		2874	return PTR_ERR(reg);
2332	Serge	2875
3031	serge	2876	if (reg->obj) {
		2877	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2878
3480	Serge	2879	ret = i915_gem_object_wait_fence(old);
3031	serge	2880	if (ret)
		2881	return ret;
2332	Serge	2882
3031	serge	2883	i915_gem_object_fence_lost(old);
		2884	}
		2885	} else
		2886	return 0;
2332	Serge	2887
3031	serge	2888	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2889
3031	serge	2890	return 0;
		2891	}
2332	Serge	2892
3031	serge	2893	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2894	struct drm_mm_node *gtt_space,
		2895	unsigned long cache_level)
		2896	{
		2897	struct drm_mm_node *other;
2332	Serge	2898
3031	serge	2899	/* On non-LLC machines we have to be careful when putting differing
		2900	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2901	* crossing memory domains and dying.
3031	serge	2902	*/
		2903	if (HAS_LLC(dev))
		2904	return true;
2332	Serge	2905
4104	Serge	2906	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2907	return true;
2332	Serge	2908
3031	serge	2909	if (list_empty(>t_space->node_list))
		2910	return true;
2332	Serge	2911
3031	serge	2912	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2913	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2914	return false;
2344	Serge	2915
3031	serge	2916	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2917	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2918	return false;
2344	Serge	2919
3031	serge	2920	return true;
		2921	}
2344	Serge	2922
3031	serge	2923	static void i915_gem_verify_gtt(struct drm_device *dev)
		2924	{
		2925	#if WATCH_GTT
		2926	struct drm_i915_private *dev_priv = dev->dev_private;
		2927	struct drm_i915_gem_object *obj;
		2928	int err = 0;
2344	Serge	2929
4104	Serge	2930	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031	serge	2931	if (obj->gtt_space == NULL) {
		2932	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2933	err++;
		2934	continue;
		2935	}
2344	Serge	2936
3031	serge	2937	if (obj->cache_level != obj->gtt_space->color) {
		2938	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
4104	Serge	2939	i915_gem_obj_ggtt_offset(obj),
		2940	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2941	obj->cache_level,
		2942	obj->gtt_space->color);
		2943	err++;
		2944	continue;
		2945	}
2344	Serge	2946
3031	serge	2947	if (!i915_gem_valid_gtt_space(dev,
		2948	obj->gtt_space,
		2949	obj->cache_level)) {
		2950	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
4104	Serge	2951	i915_gem_obj_ggtt_offset(obj),
		2952	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2953	obj->cache_level);
		2954	err++;
		2955	continue;
		2956	}
		2957	}
2344	Serge	2958
3031	serge	2959	WARN_ON(err);
		2960	#endif
2326	Serge	2961	}
		2962
2332	Serge	2963	/**
		2964	* Finds free space in the GTT aperture and binds the object there.
		2965	*/
5060	serge	2966	static struct i915_vma *
4104	Serge	2967	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2968	struct i915_address_space *vm,
2332	Serge	2969	unsigned alignment,
5060	serge	2970	uint64_t flags)
2332	Serge	2971	{
		2972	struct drm_device *dev = obj->base.dev;
5060	serge	2973	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	2974	u32 size, fence_size, fence_alignment, unfenced_alignment;
5060	serge	2975	unsigned long start =
		2976	flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
		2977	unsigned long end =
		2978	flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
4104	Serge	2979	struct i915_vma *vma;
2332	Serge	2980	int ret;
2326	Serge	2981
2332	Serge	2982	fence_size = i915_gem_get_gtt_size(dev,
		2983	obj->base.size,
		2984	obj->tiling_mode);
		2985	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2986	obj->base.size,
3480	Serge	2987	obj->tiling_mode, true);
2332	Serge	2988	unfenced_alignment =
3480	Serge	2989	i915_gem_get_gtt_alignment(dev,
2332	Serge	2990	obj->base.size,
3480	Serge	2991	obj->tiling_mode, false);
2332	Serge	2992
		2993	if (alignment == 0)
5060	serge	2994	alignment = flags & PIN_MAPPABLE ? fence_alignment :
2332	Serge	2995	unfenced_alignment;
5060	serge	2996	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
		2997	DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
		2998	return ERR_PTR(-EINVAL);
2332	Serge	2999	}
		3000
5060	serge	3001	size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
2332	Serge	3002
		3003	/* If the object is bigger than the entire aperture, reject it early
		3004	* before evicting everything in a vain attempt to find space.
		3005	*/
5060	serge	3006	if (obj->base.size > end) {
		3007	DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n",
4104	Serge	3008	obj->base.size,
5060	serge	3009	flags & PIN_MAPPABLE ? "mappable" : "total",
		3010	end);
		3011	return ERR_PTR(-E2BIG);
2332	Serge	3012	}
		3013
3031	serge	3014	ret = i915_gem_object_get_pages(obj);
		3015	if (ret)
5060	serge	3016	return ERR_PTR(ret);
3031	serge	3017
3243	Serge	3018	i915_gem_object_pin_pages(obj);
		3019
4104	Serge	3020	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
5060	serge	3021	if (IS_ERR(vma))
4104	Serge	3022	goto err_unpin;
3243	Serge	3023
4104	Serge	3024	search_free:
		3025	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3026	size, alignment,
5060	serge	3027	obj->cache_level,
		3028	start, end,
		3029	DRM_MM_SEARCH_DEFAULT,
		3030	DRM_MM_CREATE_DEFAULT);
3243	Serge	3031	if (ret) {
2332	Serge	3032
4104	Serge	3033	goto err_free_vma;
2332	Serge	3034	}
4104	Serge	3035	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
		3036	obj->cache_level))) {
		3037	ret = -EINVAL;
		3038	goto err_remove_node;
3031	serge	3039	}
2332	Serge	3040
3031	serge	3041	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	3042	if (ret)
		3043	goto err_remove_node;
2332	Serge	3044
4104	Serge	3045	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3046	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3047
4104	Serge	3048	if (i915_is_ggtt(vm)) {
		3049	bool mappable, fenceable;
2332	Serge	3050
4104	Serge	3051	fenceable = (vma->node.size == fence_size &&
		3052	(vma->node.start & (fence_alignment - 1)) == 0);
2332	Serge	3053
4104	Serge	3054	mappable = (vma->node.start + obj->base.size <=
		3055	dev_priv->gtt.mappable_end);
2332	Serge	3056
		3057	obj->map_and_fenceable = mappable && fenceable;
4104	Serge	3058	}
2332	Serge	3059
5060	serge	3060	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4104	Serge	3061
5060	serge	3062	trace_i915_vma_bind(vma, flags);
		3063	vma->bind_vma(vma, obj->cache_level,
		3064	flags & (PIN_MAPPABLE \| PIN_GLOBAL) ? GLOBAL_BIND : 0);
		3065
3031	serge	3066	i915_gem_verify_gtt(dev);
5060	serge	3067	return vma;
4104	Serge	3068
		3069	err_remove_node:
		3070	drm_mm_remove_node(&vma->node);
		3071	err_free_vma:
		3072	i915_gem_vma_destroy(vma);
5060	serge	3073	vma = ERR_PTR(ret);
4104	Serge	3074	err_unpin:
		3075	i915_gem_object_unpin_pages(obj);
5060	serge	3076	return vma;
2332	Serge	3077	}
		3078
4104	Serge	3079	bool
		3080	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3081	bool force)
2332	Serge	3082	{
		3083	/* If we don't have a page list set up, then we're not pinned
		3084	* to GPU, and we can ignore the cache flush because it'll happen
		3085	* again at bind time.
		3086	*/
3243	Serge	3087	if (obj->pages == NULL)
4104	Serge	3088	return false;
2332	Serge	3089
3480	Serge	3090	/*
		3091	* Stolen memory is always coherent with the GPU as it is explicitly
		3092	* marked as wc by the system, or the system is cache-coherent.
		3093	*/
		3094	if (obj->stolen)
4104	Serge	3095	return false;
3480	Serge	3096
2332	Serge	3097	/* If the GPU is snooping the contents of the CPU cache,
		3098	* we do not need to manually clear the CPU cache lines. However,
		3099	* the caches are only snooped when the render cache is
		3100	* flushed/invalidated. As we always have to emit invalidations
		3101	* and flushes when moving into and out of the RENDER domain, correct
		3102	* snooping behaviour occurs naturally as the result of our domain
		3103	* tracking.
		3104	*/
4104	Serge	3105	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3106	return false;
2332	Serge	3107
4293	Serge	3108	trace_i915_gem_object_clflush(obj);
		3109	drm_clflush_sg(obj->pages);
2344	Serge	3110
4104	Serge	3111	return true;
2332	Serge	3112	}
		3113
2344	Serge	3114	/** Flushes the GTT write domain for the object if it's dirty. */
		3115	static void
		3116	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3117	{
		3118	uint32_t old_write_domain;
2332	Serge	3119
2344	Serge	3120	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3121	return;
2332	Serge	3122
2344	Serge	3123	/* No actual flushing is required for the GTT write domain. Writes
		3124	* to it immediately go to main memory as far as we know, so there's
		3125	* no chipset flush. It also doesn't land in render cache.
		3126	*
		3127	* However, we do have to enforce the order so that all writes through
		3128	* the GTT land before any writes to the device, such as updates to
		3129	* the GATT itself.
		3130	*/
		3131	wmb();
2332	Serge	3132
2344	Serge	3133	old_write_domain = obj->base.write_domain;
		3134	obj->base.write_domain = 0;
2332	Serge	3135
2351	Serge	3136	trace_i915_gem_object_change_domain(obj,
		3137	obj->base.read_domains,
		3138	old_write_domain);
2344	Serge	3139	}
2332	Serge	3140
		3141	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3142	static void
4104	Serge	3143	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3144	bool force)
2332	Serge	3145	{
		3146	uint32_t old_write_domain;
		3147
		3148	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3149	return;
		3150
4104	Serge	3151	if (i915_gem_clflush_object(obj, force))
3243	Serge	3152	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3153
2332	Serge	3154	old_write_domain = obj->base.write_domain;
		3155	obj->base.write_domain = 0;
		3156
2351	Serge	3157	trace_i915_gem_object_change_domain(obj,
		3158	obj->base.read_domains,
		3159	old_write_domain);
2332	Serge	3160	}
		3161
		3162	/**
		3163	* Moves a single object to the GTT read, and possibly write domain.
		3164	*
		3165	* This function returns when the move is complete, including waiting on
		3166	* flushes to occur.
		3167	*/
		3168	int
		3169	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3170	{
5060	serge	3171	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	3172	uint32_t old_write_domain, old_read_domains;
		3173	int ret;
		3174
		3175	/* Not valid to be called on unbound objects. */
4104	Serge	3176	if (!i915_gem_obj_bound_any(obj))
2332	Serge	3177	return -EINVAL;
		3178
		3179	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3180	return 0;
		3181
3031	serge	3182	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3183	if (ret)
		3184	return ret;
		3185
5060	serge	3186	i915_gem_object_retire(obj);
4104	Serge	3187	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3188
3480	Serge	3189	/* Serialise direct access to this object with the barriers for
		3190	* coherent writes from the GPU, by effectively invalidating the
		3191	* GTT domain upon first access.
		3192	*/
		3193	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3194	mb();
		3195
2332	Serge	3196	old_write_domain = obj->base.write_domain;
		3197	old_read_domains = obj->base.read_domains;
		3198
		3199	/* It should now be out of any other write domains, and we can update
		3200	* the domain values for our changes.
		3201	*/
		3202	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3203	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3204	if (write) {
		3205	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3206	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3207	obj->dirty = 1;
		3208	}
		3209
2351	Serge	3210	trace_i915_gem_object_change_domain(obj,
		3211	old_read_domains,
		3212	old_write_domain);
		3213
3031	serge	3214	/* And bump the LRU for this access */
4104	Serge	3215	if (i915_gem_object_is_inactive(obj)) {
4560	Serge	3216	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
4104	Serge	3217	if (vma)
		3218	list_move_tail(&vma->mm_list,
		3219	&dev_priv->gtt.base.inactive_list);
3031	serge	3220
4104	Serge	3221	}
		3222
2332	Serge	3223	return 0;
		3224	}
		3225
2335	Serge	3226	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3227	enum i915_cache_level cache_level)
		3228	{
3031	serge	3229	struct drm_device *dev = obj->base.dev;
5060	serge	3230	struct i915_vma vma, next;
2335	Serge	3231	int ret;
2332	Serge	3232
2335	Serge	3233	if (obj->cache_level == cache_level)
		3234	return 0;
2332	Serge	3235
5060	serge	3236	if (i915_gem_obj_is_pinned(obj)) {
2335	Serge	3237	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3238	return -EBUSY;
		3239	}
2332	Serge	3240
5060	serge	3241	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4104	Serge	3242	if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
		3243	ret = i915_vma_unbind(vma);
3031	serge	3244	if (ret)
		3245	return ret;
4104	Serge	3246	}
3031	serge	3247	}
		3248
4104	Serge	3249	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3250	ret = i915_gem_object_finish_gpu(obj);
		3251	if (ret)
		3252	return ret;
2332	Serge	3253
2335	Serge	3254	i915_gem_object_finish_gtt(obj);
2332	Serge	3255
2335	Serge	3256	/* Before SandyBridge, you could not use tiling or fence
		3257	* registers with snooped memory, so relinquish any fences
		3258	* currently pointing to our region in the aperture.
		3259	*/
3031	serge	3260	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3261	ret = i915_gem_object_put_fence(obj);
		3262	if (ret)
		3263	return ret;
5060	serge	3264	}
2332	Serge	3265
5060	serge	3266	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3267	if (drm_mm_node_allocated(&vma->node))
		3268	vma->bind_vma(vma, cache_level,
		3269	obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
2335	Serge	3270	}
2332	Serge	3271
4104	Serge	3272	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3273	vma->node.color = cache_level;
		3274	obj->cache_level = cache_level;
		3275
		3276	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3277	u32 old_read_domains, old_write_domain;
2332	Serge	3278
2335	Serge	3279	/* If we're coming from LLC cached, then we haven't
		3280	* actually been tracking whether the data is in the
		3281	* CPU cache or not, since we only allow one bit set
		3282	* in obj->write_domain and have been skipping the clflushes.
		3283	* Just set it to the CPU cache for now.
		3284	*/
5060	serge	3285	i915_gem_object_retire(obj);
2335	Serge	3286	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3287
2335	Serge	3288	old_read_domains = obj->base.read_domains;
		3289	old_write_domain = obj->base.write_domain;
2332	Serge	3290
2335	Serge	3291	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3292	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3293
2351	Serge	3294	trace_i915_gem_object_change_domain(obj,
		3295	old_read_domains,
		3296	old_write_domain);
2344	Serge	3297	}
2332	Serge	3298
3031	serge	3299	i915_gem_verify_gtt(dev);
2335	Serge	3300	return 0;
		3301	}
2332	Serge	3302
3260	Serge	3303	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3304	struct drm_file *file)
		3305	{
		3306	struct drm_i915_gem_caching *args = data;
		3307	struct drm_i915_gem_object *obj;
		3308	int ret;
		3309
		3310	ret = i915_mutex_lock_interruptible(dev);
		3311	if (ret)
		3312	return ret;
		3313
		3314	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3315	if (&obj->base == NULL) {
		3316	ret = -ENOENT;
		3317	goto unlock;
		3318	}
		3319
4104	Serge	3320	switch (obj->cache_level) {
		3321	case I915_CACHE_LLC:
		3322	case I915_CACHE_L3_LLC:
		3323	args->caching = I915_CACHING_CACHED;
		3324	break;
3260	Serge	3325
4104	Serge	3326	case I915_CACHE_WT:
		3327	args->caching = I915_CACHING_DISPLAY;
		3328	break;
		3329
		3330	default:
		3331	args->caching = I915_CACHING_NONE;
		3332	break;
		3333	}
		3334
3260	Serge	3335	drm_gem_object_unreference(&obj->base);
		3336	unlock:
		3337	mutex_unlock(&dev->struct_mutex);
		3338	return ret;
		3339	}
		3340
		3341	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3342	struct drm_file *file)
		3343	{
		3344	struct drm_i915_gem_caching *args = data;
		3345	struct drm_i915_gem_object *obj;
		3346	enum i915_cache_level level;
		3347	int ret;
		3348
		3349	switch (args->caching) {
		3350	case I915_CACHING_NONE:
		3351	level = I915_CACHE_NONE;
		3352	break;
		3353	case I915_CACHING_CACHED:
		3354	level = I915_CACHE_LLC;
		3355	break;
4104	Serge	3356	case I915_CACHING_DISPLAY:
		3357	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3358	break;
3260	Serge	3359	default:
		3360	return -EINVAL;
		3361	}
		3362
		3363	ret = i915_mutex_lock_interruptible(dev);
		3364	if (ret)
		3365	return ret;
		3366
		3367	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3368	if (&obj->base == NULL) {
		3369	ret = -ENOENT;
		3370	goto unlock;
		3371	}
		3372
		3373	ret = i915_gem_object_set_cache_level(obj, level);
		3374
		3375	drm_gem_object_unreference(&obj->base);
		3376	unlock:
		3377	mutex_unlock(&dev->struct_mutex);
		3378	return ret;
		3379	}
		3380
4104	Serge	3381	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3382	{
5060	serge	3383	struct i915_vma *vma;
		3384
		3385	if (list_empty(&obj->vma_list))
		3386	return false;
		3387
		3388	vma = i915_gem_obj_to_ggtt(obj);
		3389	if (!vma)
		3390	return false;
		3391
4104	Serge	3392	/* There are 3 sources that pin objects:
		3393	* 1. The display engine (scanouts, sprites, cursors);
		3394	* 2. Reservations for execbuffer;
		3395	* 3. The user.
		3396	*
		3397	* We can ignore reservations as we hold the struct_mutex and
		3398	* are only called outside of the reservation path. The user
		3399	* can only increment pin_count once, and so if after
		3400	* subtracting the potential reference by the user, any pin_count
		3401	* remains, it must be due to another use by the display engine.
		3402	*/
5060	serge	3403	return vma->pin_count - !!obj->user_pin_count;
4104	Serge	3404	}
		3405
2335	Serge	3406	/*
		3407	* Prepare buffer for display plane (scanout, cursors, etc).
		3408	* Can be called from an uninterruptible phase (modesetting) and allows
		3409	* any flushes to be pipelined (for pageflips).
		3410	*/
		3411	int
		3412	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3413	u32 alignment,
5060	serge	3414	struct intel_engine_cs *pipelined)
2335	Serge	3415	{
		3416	u32 old_read_domains, old_write_domain;
5060	serge	3417	bool was_pin_display;
2335	Serge	3418	int ret;
2332	Serge	3419
3031	serge	3420	if (pipelined != obj->ring) {
		3421	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3422	if (ret)
		3423	return ret;
		3424	}
2332	Serge	3425
4104	Serge	3426	/* Mark the pin_display early so that we account for the
		3427	* display coherency whilst setting up the cache domains.
		3428	*/
5060	serge	3429	was_pin_display = obj->pin_display;
4104	Serge	3430	obj->pin_display = true;
		3431
2335	Serge	3432	/* The display engine is not coherent with the LLC cache on gen6. As
		3433	* a result, we make sure that the pinning that is about to occur is
		3434	* done with uncached PTEs. This is lowest common denominator for all
		3435	* chipsets.
		3436	*
		3437	* However for gen6+, we could do better by using the GFDT bit instead
		3438	* of uncaching, which would allow us to flush all the LLC-cached data
		3439	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3440	*/
4104	Serge	3441	ret = i915_gem_object_set_cache_level(obj,
		3442	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3443	if (ret)
4104	Serge	3444	goto err_unpin_display;
2332	Serge	3445
2335	Serge	3446	/* As the user may map the buffer once pinned in the display plane
		3447	* (e.g. libkms for the bootup splash), we have to ensure that we
		3448	* always use map_and_fenceable for all scanout buffers.
		3449	*/
5060	serge	3450	ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
2335	Serge	3451	if (ret)
4104	Serge	3452	goto err_unpin_display;
2332	Serge	3453
4104	Serge	3454	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3455
2335	Serge	3456	old_write_domain = obj->base.write_domain;
		3457	old_read_domains = obj->base.read_domains;
2332	Serge	3458
2335	Serge	3459	/* It should now be out of any other write domains, and we can update
		3460	* the domain values for our changes.
		3461	*/
3031	serge	3462	obj->base.write_domain = 0;
2335	Serge	3463	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3464
2351	Serge	3465	trace_i915_gem_object_change_domain(obj,
		3466	old_read_domains,
		3467	old_write_domain);
2332	Serge	3468
2335	Serge	3469	return 0;
4104	Serge	3470
		3471	err_unpin_display:
5060	serge	3472	WARN_ON(was_pin_display != is_pin_display(obj));
		3473	obj->pin_display = was_pin_display;
4104	Serge	3474	return ret;
2335	Serge	3475	}
2332	Serge	3476
4104	Serge	3477	void
		3478	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3479	{
5060	serge	3480	i915_gem_object_ggtt_unpin(obj);
4104	Serge	3481	obj->pin_display = is_pin_display(obj);
		3482	}
		3483
2344	Serge	3484	int
		3485	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3486	{
		3487	int ret;
2332	Serge	3488
2344	Serge	3489	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3490	return 0;
2332	Serge	3491
3031	serge	3492	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3493	if (ret)
		3494	return ret;
2332	Serge	3495
2344	Serge	3496	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3497	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3498	return 0;
2344	Serge	3499	}
2332	Serge	3500
2344	Serge	3501	/**
		3502	* Moves a single object to the CPU read, and possibly write domain.
		3503	*
		3504	* This function returns when the move is complete, including waiting on
		3505	* flushes to occur.
		3506	*/
3031	serge	3507	int
2344	Serge	3508	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3509	{
		3510	uint32_t old_write_domain, old_read_domains;
		3511	int ret;
2332	Serge	3512
2344	Serge	3513	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3514	return 0;
2332	Serge	3515
3031	serge	3516	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3517	if (ret)
		3518	return ret;
2332	Serge	3519
5060	serge	3520	i915_gem_object_retire(obj);
2344	Serge	3521	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3522
2344	Serge	3523	old_write_domain = obj->base.write_domain;
		3524	old_read_domains = obj->base.read_domains;
2332	Serge	3525
2344	Serge	3526	/* Flush the CPU cache if it's still invalid. */
		3527	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3528	i915_gem_clflush_object(obj, false);
2332	Serge	3529
2344	Serge	3530	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3531	}
2332	Serge	3532
2344	Serge	3533	/* It should now be out of any other write domains, and we can update
		3534	* the domain values for our changes.
		3535	*/
		3536	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3537
2344	Serge	3538	/* If we're writing through the CPU, then the GPU read domains will
		3539	* need to be invalidated at next use.
		3540	*/
		3541	if (write) {
		3542	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3543	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3544	}
2332	Serge	3545
2351	Serge	3546	trace_i915_gem_object_change_domain(obj,
		3547	old_read_domains,
		3548	old_write_domain);
2332	Serge	3549
2344	Serge	3550	return 0;
		3551	}
2332	Serge	3552
3031	serge	3553	/* Throttle our rendering by waiting until the ring has completed our requests
		3554	* emitted over 20 msec ago.
2344	Serge	3555	*
3031	serge	3556	* Note that if we were to use the current jiffies each time around the loop,
		3557	* we wouldn't escape the function with any frames outstanding if the time to
		3558	* render a frame was over 20ms.
		3559	*
		3560	* This should get us reasonable parallelism between CPU and GPU but also
		3561	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3562	*/
3031	serge	3563	static int
		3564	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3565	{
3031	serge	3566	struct drm_i915_private *dev_priv = dev->dev_private;
		3567	struct drm_i915_file_private *file_priv = file->driver_priv;
5060	serge	3568	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3031	serge	3569	struct drm_i915_gem_request *request;
5060	serge	3570	struct intel_engine_cs *ring = NULL;
3480	Serge	3571	unsigned reset_counter;
3031	serge	3572	u32 seqno = 0;
		3573	int ret;
2332	Serge	3574
3480	Serge	3575	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3576	if (ret)
		3577	return ret;
2332	Serge	3578
3480	Serge	3579	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3580	if (ret)
		3581	return ret;
		3582
3031	serge	3583	spin_lock(&file_priv->mm.lock);
		3584	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3585	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3586	break;
2332	Serge	3587
3031	serge	3588	ring = request->ring;
		3589	seqno = request->seqno;
		3590	}
3480	Serge	3591	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3592	spin_unlock(&file_priv->mm.lock);
2332	Serge	3593
3031	serge	3594	if (seqno == 0)
		3595	return 0;
2332	Serge	3596
4560	Serge	3597	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
3031	serge	3598	if (ret == 0)
		3599	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3600
3031	serge	3601	return ret;
2352	Serge	3602	}
2332	Serge	3603
5060	serge	3604	static bool
		3605	i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
		3606	{
		3607	struct drm_i915_gem_object *obj = vma->obj;
		3608
		3609	if (alignment &&
		3610	vma->node.start & (alignment - 1))
		3611	return true;
		3612
		3613	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
		3614	return true;
		3615
		3616	if (flags & PIN_OFFSET_BIAS &&
		3617	vma->node.start < (flags & PIN_OFFSET_MASK))
		3618	return true;
		3619
		3620	return false;
		3621	}
		3622
2332	Serge	3623	int
		3624	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3625	struct i915_address_space *vm,
2332	Serge	3626	uint32_t alignment,
5060	serge	3627	uint64_t flags)
2332	Serge	3628	{
5060	serge	3629	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4104	Serge	3630	struct i915_vma *vma;
2332	Serge	3631	int ret;
		3632
5060	serge	3633	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
		3634	return -ENODEV;
2332	Serge	3635
5060	serge	3636	if (WARN_ON(flags & (PIN_GLOBAL \| PIN_MAPPABLE) && !i915_is_ggtt(vm)))
		3637	return -EINVAL;
4104	Serge	3638
		3639	vma = i915_gem_obj_to_vma(obj, vm);
5060	serge	3640	if (vma) {
		3641	if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3642	return -EBUSY;
4104	Serge	3643
5060	serge	3644	if (i915_vma_misplaced(vma, alignment, flags)) {
		3645	WARN(vma->pin_count,
2332	Serge	3646	"bo is already pinned with incorrect alignment:"
4104	Serge	3647	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3648	" obj->map_and_fenceable=%d\n",
4104	Serge	3649	i915_gem_obj_offset(obj, vm), alignment,
5060	serge	3650	!!(flags & PIN_MAPPABLE),
2332	Serge	3651	obj->map_and_fenceable);
4104	Serge	3652	ret = i915_vma_unbind(vma);
2332	Serge	3653	if (ret)
		3654	return ret;
5060	serge	3655
		3656	vma = NULL;
2332	Serge	3657	}
		3658	}
		3659
5060	serge	3660	if (vma == NULL \|\| !drm_mm_node_allocated(&vma->node)) {
		3661	vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
		3662	if (IS_ERR(vma))
		3663	return PTR_ERR(vma);
2332	Serge	3664	}
		3665
5060	serge	3666	if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
		3667	vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
3031	serge	3668
5060	serge	3669	vma->pin_count++;
		3670	if (flags & PIN_MAPPABLE)
		3671	obj->pin_mappable \|= true;
2332	Serge	3672
		3673	return 0;
		3674	}
		3675
2344	Serge	3676	void
5060	serge	3677	i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
2344	Serge	3678	{
5060	serge	3679	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
2332	Serge	3680
5060	serge	3681	BUG_ON(!vma);
		3682	BUG_ON(vma->pin_count == 0);
		3683	BUG_ON(!i915_gem_obj_ggtt_bound(obj));
		3684
		3685	if (--vma->pin_count == 0)
2344	Serge	3686	obj->pin_mappable = false;
		3687	}
2332	Serge	3688
5060	serge	3689	bool
		3690	i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
		3691	{
		3692	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		3693	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3694	struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
		3695
		3696	WARN_ON(!ggtt_vma \|\|
		3697	dev_priv->fence_regs[obj->fence_reg].pin_count >
		3698	ggtt_vma->pin_count);
		3699	dev_priv->fence_regs[obj->fence_reg].pin_count++;
		3700	return true;
		3701	} else
		3702	return false;
		3703	}
		3704
		3705	void
		3706	i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
		3707	{
		3708	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		3709	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3710	WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
		3711	dev_priv->fence_regs[obj->fence_reg].pin_count--;
		3712	}
		3713	}
		3714
3031	serge	3715	int
		3716	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3717	struct drm_file *file)
		3718	{
		3719	struct drm_i915_gem_pin *args = data;
		3720	struct drm_i915_gem_object *obj;
		3721	int ret;
2332	Serge	3722
5060	serge	3723	if (INTEL_INFO(dev)->gen >= 6)
		3724	return -ENODEV;
		3725
3031	serge	3726	ret = i915_mutex_lock_interruptible(dev);
		3727	if (ret)
		3728	return ret;
2332	Serge	3729
3031	serge	3730	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3731	if (&obj->base == NULL) {
		3732	ret = -ENOENT;
		3733	goto unlock;
		3734	}
2332	Serge	3735
3031	serge	3736	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	3737	DRM_DEBUG("Attempting to pin a purgeable buffer\n");
		3738	ret = -EFAULT;
3031	serge	3739	goto out;
		3740	}
2332	Serge	3741
3031	serge	3742	if (obj->pin_filp != NULL && obj->pin_filp != file) {
5060	serge	3743	DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
3031	serge	3744	args->handle);
		3745	ret = -EINVAL;
		3746	goto out;
		3747	}
2332	Serge	3748
4560	Serge	3749	if (obj->user_pin_count == ULONG_MAX) {
		3750	ret = -EBUSY;
		3751	goto out;
		3752	}
		3753
3243	Serge	3754	if (obj->user_pin_count == 0) {
5060	serge	3755	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
3031	serge	3756	if (ret)
		3757	goto out;
		3758	}
2332	Serge	3759
3243	Serge	3760	obj->user_pin_count++;
		3761	obj->pin_filp = file;
		3762
4104	Serge	3763	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3764	out:
		3765	drm_gem_object_unreference(&obj->base);
		3766	unlock:
		3767	mutex_unlock(&dev->struct_mutex);
		3768	return ret;
		3769	}
2332	Serge	3770
3031	serge	3771	int
		3772	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3773	struct drm_file *file)
		3774	{
		3775	struct drm_i915_gem_pin *args = data;
		3776	struct drm_i915_gem_object *obj;
		3777	int ret;
2332	Serge	3778
3031	serge	3779	ret = i915_mutex_lock_interruptible(dev);
		3780	if (ret)
		3781	return ret;
2332	Serge	3782
3031	serge	3783	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3784	if (&obj->base == NULL) {
		3785	ret = -ENOENT;
		3786	goto unlock;
		3787	}
2332	Serge	3788
3031	serge	3789	if (obj->pin_filp != file) {
5060	serge	3790	DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3031	serge	3791	args->handle);
		3792	ret = -EINVAL;
		3793	goto out;
		3794	}
		3795	obj->user_pin_count--;
		3796	if (obj->user_pin_count == 0) {
		3797	obj->pin_filp = NULL;
5060	serge	3798	i915_gem_object_ggtt_unpin(obj);
3031	serge	3799	}
2332	Serge	3800
3031	serge	3801	out:
		3802	drm_gem_object_unreference(&obj->base);
		3803	unlock:
		3804	mutex_unlock(&dev->struct_mutex);
		3805	return ret;
		3806	}
2332	Serge	3807
3031	serge	3808	int
		3809	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3810	struct drm_file *file)
		3811	{
		3812	struct drm_i915_gem_busy *args = data;
		3813	struct drm_i915_gem_object *obj;
		3814	int ret;
2332	Serge	3815
3031	serge	3816	ret = i915_mutex_lock_interruptible(dev);
		3817	if (ret)
		3818	return ret;
2332	Serge	3819
5060	serge	3820	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3821	if (&obj->base == NULL) {
		3822	ret = -ENOENT;
		3823	goto unlock;
		3824	}
2332	Serge	3825
3031	serge	3826	/* Count all active objects as busy, even if they are currently not used
		3827	* by the gpu. Users of this interface expect objects to eventually
		3828	* become non-busy without any further actions, therefore emit any
		3829	* necessary flushes here.
		3830	*/
		3831	ret = i915_gem_object_flush_active(obj);
2332	Serge	3832
3031	serge	3833	args->busy = obj->active;
		3834	if (obj->ring) {
		3835	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3836	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3837	}
2332	Serge	3838
3031	serge	3839	drm_gem_object_unreference(&obj->base);
		3840	unlock:
		3841	mutex_unlock(&dev->struct_mutex);
		3842	return ret;
		3843	}
2332	Serge	3844
3031	serge	3845	int
		3846	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3847	struct drm_file *file_priv)
		3848	{
		3849	return i915_gem_ring_throttle(dev, file_priv);
		3850	}
2332	Serge	3851
3263	Serge	3852	#if 0
		3853
3031	serge	3854	int
		3855	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3856	struct drm_file *file_priv)
		3857	{
		3858	struct drm_i915_gem_madvise *args = data;
		3859	struct drm_i915_gem_object *obj;
		3860	int ret;
2332	Serge	3861
3031	serge	3862	switch (args->madv) {
		3863	case I915_MADV_DONTNEED:
		3864	case I915_MADV_WILLNEED:
		3865	break;
		3866	default:
		3867	return -EINVAL;
		3868	}
2332	Serge	3869
3031	serge	3870	ret = i915_mutex_lock_interruptible(dev);
		3871	if (ret)
		3872	return ret;
2332	Serge	3873
3031	serge	3874	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3875	if (&obj->base == NULL) {
		3876	ret = -ENOENT;
		3877	goto unlock;
		3878	}
2332	Serge	3879
5060	serge	3880	if (i915_gem_obj_is_pinned(obj)) {
3031	serge	3881	ret = -EINVAL;
		3882	goto out;
		3883	}
2332	Serge	3884
3031	serge	3885	if (obj->madv != __I915_MADV_PURGED)
		3886	obj->madv = args->madv;
2332	Serge	3887
3031	serge	3888	/* if the object is no longer attached, discard its backing storage */
		3889	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3890	i915_gem_object_truncate(obj);
2332	Serge	3891
3031	serge	3892	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3893
3031	serge	3894	out:
		3895	drm_gem_object_unreference(&obj->base);
		3896	unlock:
		3897	mutex_unlock(&dev->struct_mutex);
		3898	return ret;
		3899	}
		3900	#endif
2332	Serge	3901
3031	serge	3902	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3903	const struct drm_i915_gem_object_ops *ops)
		3904	{
4104	Serge	3905	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3906	INIT_LIST_HEAD(&obj->ring_list);
4104	Serge	3907	INIT_LIST_HEAD(&obj->obj_exec_link);
		3908	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3909
3031	serge	3910	obj->ops = ops;
		3911
		3912	obj->fence_reg = I915_FENCE_REG_NONE;
		3913	obj->madv = I915_MADV_WILLNEED;
		3914	/* Avoid an unnecessary call to unbind on the first bind. */
		3915	obj->map_and_fenceable = true;
		3916
		3917	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3918	}
		3919
		3920	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3921	.get_pages = i915_gem_object_get_pages_gtt,
		3922	.put_pages = i915_gem_object_put_pages_gtt,
		3923	};
		3924
2332	Serge	3925	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3926	size_t size)
		3927	{
		3928	struct drm_i915_gem_object *obj;
3031	serge	3929	struct address_space *mapping;
3480	Serge	3930	gfp_t mask;
2340	Serge	3931
3746	Serge	3932	obj = i915_gem_object_alloc(dev);
2332	Serge	3933	if (obj == NULL)
		3934	return NULL;
		3935
		3936	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3937	i915_gem_object_free(obj);
2332	Serge	3938	return NULL;
		3939	}
		3940
		3941
3031	serge	3942	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3943
		3944	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3945	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3946
3031	serge	3947	if (HAS_LLC(dev)) {
		3948	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3949	* cache) for about a 10% performance improvement
		3950	* compared to uncached. Graphics requests other than
		3951	* display scanout are coherent with the CPU in
		3952	* accessing this cache. This means in this mode we
		3953	* don't need to clflush on the CPU side, and on the
		3954	* GPU side we only need to flush internal caches to
		3955	* get data visible to the CPU.
		3956	*
		3957	* However, we maintain the display planes as UC, and so
		3958	* need to rebind when first used as such.
		3959	*/
		3960	obj->cache_level = I915_CACHE_LLC;
		3961	} else
		3962	obj->cache_level = I915_CACHE_NONE;
		3963
4560	Serge	3964	trace_i915_gem_object_create(obj);
		3965
2332	Serge	3966	return obj;
		3967	}
		3968
3031	serge	3969	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3970	{
3031	serge	3971	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3972	struct drm_device *dev = obj->base.dev;
5060	serge	3973	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3974	struct i915_vma vma, next;
2332	Serge	3975
4560	Serge	3976	intel_runtime_pm_get(dev_priv);
		3977
3031	serge	3978	trace_i915_gem_object_destroy(obj);
		3979
5060	serge	3980	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3981	int ret;
3031	serge	3982
5060	serge	3983	vma->pin_count = 0;
		3984	ret = i915_vma_unbind(vma);
4104	Serge	3985	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	3986	bool was_interruptible;
		3987
		3988	was_interruptible = dev_priv->mm.interruptible;
		3989	dev_priv->mm.interruptible = false;
		3990
4104	Serge	3991	WARN_ON(i915_vma_unbind(vma));
3031	serge	3992
		3993	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3994	}
4104	Serge	3995	}
2332	Serge	3996
4104	Serge	3997	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		3998	* before progressing. */
		3999	if (obj->stolen)
		4000	i915_gem_object_unpin_pages(obj);
		4001
5060	serge	4002	WARN_ON(obj->frontbuffer_bits);
		4003
4104	Serge	4004	if (WARN_ON(obj->pages_pin_count))
3031	serge	4005	obj->pages_pin_count = 0;
		4006	i915_gem_object_put_pages(obj);
		4007	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	4008
3243	Serge	4009	BUG_ON(obj->pages);
2332	Serge	4010
3031	serge	4011
3290	Serge	4012	if(obj->base.filp != NULL)
		4013	{
3298	Serge	4014	// printf("filp %p\n", obj->base.filp);
3290	Serge	4015	shmem_file_delete(obj->base.filp);
		4016	}
		4017
2344	Serge	4018	drm_gem_object_release(&obj->base);
		4019	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	4020
2344	Serge	4021	kfree(obj->bit_17);
4104	Serge	4022	i915_gem_object_free(obj);
4560	Serge	4023
		4024	intel_runtime_pm_put(dev_priv);
2344	Serge	4025	}
2332	Serge	4026
4560	Serge	4027	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
4104	Serge	4028	struct i915_address_space *vm)
		4029	{
4560	Serge	4030	struct i915_vma *vma;
		4031	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4032	if (vma->vm == vm)
		4033	return vma;
		4034
		4035	return NULL;
		4036	}
		4037
4104	Serge	4038	void i915_gem_vma_destroy(struct i915_vma *vma)
		4039	{
		4040	WARN_ON(vma->node.allocated);
4560	Serge	4041
		4042	/* Keep the vma as a placeholder in the execbuffer reservation lists */
		4043	if (!list_empty(&vma->exec_list))
		4044	return;
		4045
4104	Serge	4046	list_del(&vma->vma_link);
4560	Serge	4047
4104	Serge	4048	kfree(vma);
		4049	}
		4050
3031	serge	4051	#if 0
		4052	int
4560	Serge	4053	i915_gem_suspend(struct drm_device *dev)
2344	Serge	4054	{
5060	serge	4055	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4056	int ret = 0;
2332	Serge	4057
4560	Serge	4058	mutex_lock(&dev->struct_mutex);
		4059	if (dev_priv->ums.mm_suspended)
		4060	goto err;
2332	Serge	4061
3031	serge	4062	ret = i915_gpu_idle(dev);
4560	Serge	4063	if (ret)
		4064	goto err;
		4065
3031	serge	4066	i915_gem_retire_requests(dev);
		4067
3480	Serge	4068	/* Under UMS, be paranoid and evict. */
		4069	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		4070	i915_gem_evict_everything(dev);
		4071
3031	serge	4072	i915_kernel_lost_context(dev);
5060	serge	4073	i915_gem_stop_ringbuffers(dev);
3031	serge	4074
4560	Serge	4075	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		4076	* We need to replace this with a semaphore, or something.
		4077	* And not confound ums.mm_suspended!
		4078	*/
		4079	dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
		4080	DRIVER_MODESET);
		4081	mutex_unlock(&dev->struct_mutex);
		4082
		4083	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3263	Serge	4084	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
5060	serge	4085	flush_delayed_work(&dev_priv->mm.idle_work);
3031	serge	4086
		4087	return 0;
4560	Serge	4088
		4089	err:
		4090	mutex_unlock(&dev->struct_mutex);
		4091	return ret;
2344	Serge	4092	}
3031	serge	4093	#endif
2332	Serge	4094
5060	serge	4095	int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice)
3031	serge	4096	{
4560	Serge	4097	struct drm_device *dev = ring->dev;
5060	serge	4098	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4099	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
		4100	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
		4101	int i, ret;
2332	Serge	4102
4560	Serge	4103	if (!HAS_L3_DPF(dev) \|\| !remap_info)
		4104	return 0;
2332	Serge	4105
4560	Serge	4106	ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
		4107	if (ret)
		4108	return ret;
2332	Serge	4109
4560	Serge	4110	/*
		4111	* Note: We do not worry about the concurrent register cacheline hang
		4112	* here because no other code should access these registers other than
		4113	* at initialization time.
		4114	*/
3031	serge	4115	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4560	Serge	4116	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		4117	intel_ring_emit(ring, reg_base + i);
		4118	intel_ring_emit(ring, remap_info[i/4]);
3031	serge	4119	}
2332	Serge	4120
4560	Serge	4121	intel_ring_advance(ring);
2332	Serge	4122
4560	Serge	4123	return ret;
3031	serge	4124	}
2332	Serge	4125
3031	serge	4126	void i915_gem_init_swizzling(struct drm_device *dev)
		4127	{
5060	serge	4128	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4129
3031	serge	4130	if (INTEL_INFO(dev)->gen < 5 \|\|
		4131	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4132	return;
2332	Serge	4133
3031	serge	4134	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4135	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4136
3031	serge	4137	if (IS_GEN5(dev))
		4138	return;
2344	Serge	4139
3031	serge	4140	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4141	if (IS_GEN6(dev))
		4142	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4143	else if (IS_GEN7(dev))
		4144	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4560	Serge	4145	else if (IS_GEN8(dev))
		4146	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
3031	serge	4147	else
3480	Serge	4148	BUG();
3031	serge	4149	}
		4150
		4151	static bool
		4152	intel_enable_blt(struct drm_device *dev)
		4153	{
		4154	if (!HAS_BLT(dev))
		4155	return false;
		4156
		4157	/* The blitter was dysfunctional on early prototypes */
		4158	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4159	DRM_INFO("BLT not supported on this pre-production hardware;"
		4160	" graphics performance will be degraded.\n");
		4161	return false;
		4162	}
		4163
		4164	return true;
		4165	}
		4166
3480	Serge	4167	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	4168	{
3480	Serge	4169	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4170	int ret;
2351	Serge	4171
2332	Serge	4172	ret = intel_init_render_ring_buffer(dev);
		4173	if (ret)
		4174	return ret;
		4175
		4176	if (HAS_BSD(dev)) {
		4177	ret = intel_init_bsd_ring_buffer(dev);
		4178	if (ret)
		4179	goto cleanup_render_ring;
		4180	}
		4181
3031	serge	4182	if (intel_enable_blt(dev)) {
2332	Serge	4183	ret = intel_init_blt_ring_buffer(dev);
		4184	if (ret)
		4185	goto cleanup_bsd_ring;
		4186	}
		4187
4104	Serge	4188	if (HAS_VEBOX(dev)) {
		4189	ret = intel_init_vebox_ring_buffer(dev);
		4190	if (ret)
		4191	goto cleanup_blt_ring;
		4192	}
		4193
5060	serge	4194	if (HAS_BSD2(dev)) {
		4195	ret = intel_init_bsd2_ring_buffer(dev);
		4196	if (ret)
		4197	goto cleanup_vebox_ring;
		4198	}
4104	Serge	4199
3480	Serge	4200	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4201	if (ret)
5060	serge	4202	goto cleanup_bsd2_ring;
2351	Serge	4203
2332	Serge	4204	return 0;
		4205
5060	serge	4206	cleanup_bsd2_ring:
		4207	intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
4104	Serge	4208	cleanup_vebox_ring:
		4209	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4210	cleanup_blt_ring:
		4211	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4212	cleanup_bsd_ring:
		4213	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4214	cleanup_render_ring:
		4215	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4216
2332	Serge	4217	return ret;
		4218	}
		4219
3480	Serge	4220	int
		4221	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4222	{
5060	serge	4223	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4224	int ret, i;
3031	serge	4225
3480	Serge	4226	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4227	return -EIO;
3031	serge	4228
4104	Serge	4229	if (dev_priv->ellc_size)
		4230	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4231
4560	Serge	4232	if (IS_HASWELL(dev))
		4233	I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
		4234	LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
		4235
3746	Serge	4236	if (HAS_PCH_NOP(dev)) {
5060	serge	4237	if (IS_IVYBRIDGE(dev)) {
3746	Serge	4238	u32 temp = I915_READ(GEN7_MSG_CTL);
		4239	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4240	I915_WRITE(GEN7_MSG_CTL, temp);
5060	serge	4241	} else if (INTEL_INFO(dev)->gen >= 7) {
		4242	u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
		4243	temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
		4244	I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
		4245	}
3746	Serge	4246	}
		4247
3480	Serge	4248	i915_gem_init_swizzling(dev);
		4249
		4250	ret = i915_gem_init_rings(dev);
		4251	if (ret)
		4252	return ret;
		4253
4560	Serge	4254	for (i = 0; i < NUM_L3_SLICES(dev); i++)
		4255	i915_gem_l3_remap(&dev_priv->ring[RCS], i);
		4256
3480	Serge	4257	/*
5060	serge	4258	* XXX: Contexts should only be initialized once. Doing a switch to the
		4259	* default context switch however is something we'd like to do after
		4260	* reset or thaw (the latter may not actually be necessary for HW, but
		4261	* goes with our code better). Context switching requires rings (for
		4262	* the do_switch), but before enabling PPGTT. So don't move this.
3480	Serge	4263	*/
5060	serge	4264	ret = i915_gem_context_enable(dev_priv);
		4265	if (ret && ret != -EIO) {
		4266	DRM_ERROR("Context enable failed %d\n", ret);
4560	Serge	4267	i915_gem_cleanup_ringbuffer(dev);
		4268	}
		4269
5060	serge	4270	return ret;
3031	serge	4271	}
		4272
		4273	int i915_gem_init(struct drm_device *dev)
		4274	{
		4275	struct drm_i915_private *dev_priv = dev->dev_private;
		4276	int ret;
		4277
		4278	mutex_lock(&dev->struct_mutex);
3746	Serge	4279
		4280	if (IS_VALLEYVIEW(dev)) {
		4281	/* VLVA0 (potential hack), BIOS isn't actually waking us */
5060	serge	4282	I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
		4283	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
		4284	VLV_GTLC_ALLOWWAKEACK), 10))
3746	Serge	4285	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4286	}
		4287
5060	serge	4288	i915_gem_init_global_gtt(dev);
3746	Serge	4289
5060	serge	4290	ret = i915_gem_context_init(dev);
3031	serge	4291	if (ret) {
5060	serge	4292	mutex_unlock(&dev->struct_mutex);
3031	serge	4293	return ret;
		4294	}
		4295
5060	serge	4296	ret = i915_gem_init_hw(dev);
		4297	if (ret == -EIO) {
		4298	/* Allow ring initialisation to fail by marking the GPU as
		4299	* wedged. But we only want to do this where the GPU is angry,
		4300	* for all other failure, such as an allocation failure, bail.
		4301	*/
		4302	DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
		4303	atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
		4304	ret = 0;
		4305	}
		4306	mutex_unlock(&dev->struct_mutex);
3746	Serge	4307
5060	serge	4308	return ret;
3031	serge	4309	}
		4310
2332	Serge	4311	void
		4312	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4313	{
5060	serge	4314	struct drm_i915_private *dev_priv = dev->dev_private;
		4315	struct intel_engine_cs *ring;
2332	Serge	4316	int i;
		4317
3031	serge	4318	for_each_ring(ring, dev_priv, i)
		4319	intel_cleanup_ring_buffer(ring);
2332	Serge	4320	}
		4321
3031	serge	4322	#if 0
		4323
2332	Serge	4324	int
		4325	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		4326	struct drm_file *file_priv)
		4327	{
4104	Serge	4328	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4329	int ret;
2332	Serge	4330
		4331	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4332	return 0;
		4333
3480	Serge	4334	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	4335	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	4336	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	4337	}
		4338
		4339	mutex_lock(&dev->struct_mutex);
4104	Serge	4340	dev_priv->ums.mm_suspended = 0;
2332	Serge	4341
3031	serge	4342	ret = i915_gem_init_hw(dev);
2332	Serge	4343	if (ret != 0) {
		4344	mutex_unlock(&dev->struct_mutex);
		4345	return ret;
		4346	}
		4347
4104	Serge	4348	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
2332	Serge	4349
5060	serge	4350	ret = drm_irq_install(dev, dev->pdev->irq);
2332	Serge	4351	if (ret)
		4352	goto cleanup_ringbuffer;
5060	serge	4353	mutex_unlock(&dev->struct_mutex);
2332	Serge	4354
		4355	return 0;
		4356
		4357	cleanup_ringbuffer:
		4358	i915_gem_cleanup_ringbuffer(dev);
4104	Serge	4359	dev_priv->ums.mm_suspended = 1;
2332	Serge	4360	mutex_unlock(&dev->struct_mutex);
		4361
		4362	return ret;
		4363	}
		4364
		4365	int
		4366	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4367	struct drm_file *file_priv)
		4368	{
		4369	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4370	return 0;
		4371
5060	serge	4372	mutex_lock(&dev->struct_mutex);
2332	Serge	4373	drm_irq_uninstall(dev);
5060	serge	4374	mutex_unlock(&dev->struct_mutex);
4104	Serge	4375
4560	Serge	4376	return i915_gem_suspend(dev);
2332	Serge	4377	}
		4378
		4379	void
		4380	i915_gem_lastclose(struct drm_device *dev)
		4381	{
		4382	int ret;
		4383
		4384	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4385	return;
		4386
4560	Serge	4387	ret = i915_gem_suspend(dev);
2332	Serge	4388	if (ret)
		4389	DRM_ERROR("failed to idle hardware: %d\n", ret);
		4390	}
		4391	#endif
		4392
		4393	static void
5060	serge	4394	init_ring_lists(struct intel_engine_cs *ring)
2326	Serge	4395	{
		4396	INIT_LIST_HEAD(&ring->active_list);
		4397	INIT_LIST_HEAD(&ring->request_list);
		4398	}
		4399
5060	serge	4400	void i915_init_vm(struct drm_i915_private *dev_priv,
4104	Serge	4401	struct i915_address_space *vm)
		4402	{
5060	serge	4403	if (!i915_is_ggtt(vm))
		4404	drm_mm_init(&vm->mm, vm->start, vm->total);
4104	Serge	4405	vm->dev = dev_priv->dev;
		4406	INIT_LIST_HEAD(&vm->active_list);
		4407	INIT_LIST_HEAD(&vm->inactive_list);
		4408	INIT_LIST_HEAD(&vm->global_link);
5060	serge	4409	list_add_tail(&vm->global_link, &dev_priv->vm_list);
4104	Serge	4410	}
		4411
2326	Serge	4412	void
		4413	i915_gem_load(struct drm_device *dev)
		4414	{
5060	serge	4415	struct drm_i915_private *dev_priv = dev->dev_private;
2326	Serge	4416	int i;
		4417
4104	Serge	4418	INIT_LIST_HEAD(&dev_priv->vm_list);
		4419	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4420
4560	Serge	4421	INIT_LIST_HEAD(&dev_priv->context_list);
3031	serge	4422	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4423	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4424	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4425	for (i = 0; i < I915_NUM_RINGS; i++)
		4426	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4427	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4428	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4429	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4430	i915_gem_retire_work_handler);
4560	Serge	4431	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
		4432	i915_gem_idle_work_handler);
3480	Serge	4433	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4434
		4435	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4436	if (IS_GEN3(dev)) {
3031	serge	4437	I915_WRITE(MI_ARB_STATE,
		4438	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4439	}
		4440
		4441	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4442
3746	Serge	4443	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4444	dev_priv->num_fence_regs = 32;
		4445	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4446	dev_priv->num_fence_regs = 16;
		4447	else
		4448	dev_priv->num_fence_regs = 8;
		4449
		4450	/* Initialize fence registers to zero */
3746	Serge	4451	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4452	i915_gem_restore_fences(dev);
2326	Serge	4453
		4454	i915_gem_detect_bit_6_swizzle(dev);
		4455
		4456	dev_priv->mm.interruptible = true;
		4457
5060	serge	4458	mutex_init(&dev_priv->fb_tracking.lock);
2326	Serge	4459	}
		4460
5060	serge	4461	int i915_gem_open(struct drm_device dev, struct drm_file file)
4104	Serge	4462	{
5060	serge	4463	struct drm_i915_file_private *file_priv;
4104	Serge	4464	int ret;
2326	Serge	4465
5060	serge	4466	DRM_DEBUG_DRIVER("\n");
4104	Serge	4467
5060	serge	4468	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
		4469	if (!file_priv)
4104	Serge	4470	return -ENOMEM;
		4471
5060	serge	4472	file->driver_priv = file_priv;
		4473	file_priv->dev_priv = dev->dev_private;
		4474	file_priv->file = file;
4104	Serge	4475
5060	serge	4476	spin_lock_init(&file_priv->mm.lock);
		4477	INIT_LIST_HEAD(&file_priv->mm.request_list);
		4478	// INIT_DELAYED_WORK(&file_priv->mm.idle_work,
		4479	// i915_gem_file_idle_work_handler);
4104	Serge	4480
5060	serge	4481	ret = i915_gem_context_open(dev, file);
		4482	if (ret)
		4483	kfree(file_priv);
4104	Serge	4484
		4485	return ret;
		4486	}
		4487
5060	serge	4488	void i915_gem_track_fb(struct drm_i915_gem_object *old,
		4489	struct drm_i915_gem_object *new,
		4490	unsigned frontbuffer_bits)
4104	Serge	4491	{
5060	serge	4492	if (old) {
		4493	WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
		4494	WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
		4495	old->frontbuffer_bits &= ~frontbuffer_bits;
4104	Serge	4496	}
		4497
5060	serge	4498	if (new) {
		4499	WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
		4500	WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
		4501	new->frontbuffer_bits \|= frontbuffer_bits;
4104	Serge	4502	}
		4503	}
		4504
		4505	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4506	{
		4507	if (!mutex_is_locked(mutex))
		4508	return false;
		4509
		4510	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4511	return mutex->owner == task;
		4512	#else
		4513	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4514	return false;
		4515	#endif
		4516	}
		4517
		4518	/* All the new VM stuff */
		4519	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4520	struct i915_address_space *vm)
		4521	{
		4522	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4523	struct i915_vma *vma;
		4524
5060	serge	4525	if (!dev_priv->mm.aliasing_ppgtt \|\|
		4526	vm == &dev_priv->mm.aliasing_ppgtt->base)
4104	Serge	4527	vm = &dev_priv->gtt.base;
		4528
		4529	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4530	if (vma->vm == vm)
		4531	return vma->node.start;
		4532
		4533	}
5060	serge	4534	WARN(1, "%s vma for this object not found.\n",
		4535	i915_is_ggtt(vm) ? "global" : "ppgtt");
		4536	return -1;
4104	Serge	4537	}
		4538
		4539	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4540	struct i915_address_space *vm)
		4541	{
		4542	struct i915_vma *vma;
		4543
		4544	list_for_each_entry(vma, &o->vma_list, vma_link)
		4545	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4546	return true;
		4547
		4548	return false;
		4549	}
		4550
		4551	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4552	{
4560	Serge	4553	struct i915_vma *vma;
4104	Serge	4554
4560	Serge	4555	list_for_each_entry(vma, &o->vma_list, vma_link)
		4556	if (drm_mm_node_allocated(&vma->node))
4104	Serge	4557	return true;
		4558
		4559	return false;
		4560	}
		4561
		4562	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4563	struct i915_address_space *vm)
		4564	{
		4565	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4566	struct i915_vma *vma;
		4567
5060	serge	4568	if (!dev_priv->mm.aliasing_ppgtt \|\|
		4569	vm == &dev_priv->mm.aliasing_ppgtt->base)
4104	Serge	4570	vm = &dev_priv->gtt.base;
		4571
		4572	BUG_ON(list_empty(&o->vma_list));
		4573
		4574	list_for_each_entry(vma, &o->vma_list, vma_link)
		4575	if (vma->vm == vm)
		4576	return vma->node.size;
		4577
		4578	return 0;
		4579	}
4560	Serge	4580
		4581
5060	serge	4582
4560	Serge	4583	struct i915_vma i915_gem_obj_to_ggtt(struct drm_i915_gem_object obj)
4104	Serge	4584	{
		4585	struct i915_vma *vma;
		4586
5060	serge	4587	/* This WARN has probably outlived its usefulness (callers already
		4588	* WARN if they don't find the GGTT vma they expect). When removing,
		4589	* remember to remove the pre-check in is_pin_display() as well */
4560	Serge	4590	if (WARN_ON(list_empty(&obj->vma_list)))
5060	serge	4591	return NULL;
4104	Serge	4592
4560	Serge	4593	vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5060	serge	4594	if (vma->vm != obj_to_ggtt(obj))
4560	Serge	4595	return NULL;
4104	Serge	4596
		4597	return vma;
		4598	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 5367 – Rev 5060