WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
3746	Serge	36	#include
2326	Serge	37	#include
		38
2344	Serge	39	extern int x86_clflush_size;
2332	Serge	40
3263	Serge	41	#define PROT_READ 0x1 /* page can be read */
		42	#define PROT_WRITE 0x2 /* page can be written */
		43	#define MAP_SHARED 0x01 /* Share changes */
		44
2344	Serge	45	#undef mb
		46	#undef rmb
		47	#undef wmb
		48	#define mb() asm volatile("mfence")
		49	#define rmb() asm volatile ("lfence")
		50	#define wmb() asm volatile ("sfence")
		51
3266	Serge	52	struct drm_i915_gem_object *get_fb_obj();
		53
3263	Serge	54	unsigned long vm_mmap(struct file *file, unsigned long addr,
		55	unsigned long len, unsigned long prot,
		56	unsigned long flag, unsigned long offset);
		57
2344	Serge	58	static inline void clflush(volatile void *__p)
		59	{
		60	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		61	}
		62
2332	Serge	63	#define MAX_ERRNO 4095
		64
		65	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		66
		67
2326	Serge	68	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		69	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		70	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		71	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		72
2332	Serge	73	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	74	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		75	bool force);
		76	static __must_check int
		77	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		78	struct i915_address_space *vm,
2332	Serge	79	unsigned alignment,
3031	serge	80	bool map_and_fenceable,
		81	bool nonblocking);
2332	Serge	82	static int i915_gem_phys_pwrite(struct drm_device *dev,
		83	struct drm_i915_gem_object *obj,
		84	struct drm_i915_gem_pwrite *args,
		85	struct drm_file *file);
2326	Serge	86
3031	serge	87	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		88	struct drm_i915_gem_object *obj);
		89	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		90	struct drm_i915_fence_reg *fence,
		91	bool enable);
2332	Serge	92
3031	serge	93	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
4104	Serge	94	static long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	95	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		96
4104	Serge	97	static bool cpu_cache_is_coherent(struct drm_device *dev,
		98	enum i915_cache_level level)
		99	{
		100	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		101	}
		102
		103	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		104	{
		105	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		106	return true;
		107
		108	return obj->pin_display;
		109	}
		110
3031	serge	111	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		112	{
		113	if (obj->tiling_mode)
		114	i915_gem_release_mmap(obj);
		115
		116	/* As we do not have an associated fence register, we will force
		117	* a tiling change if we ever need to acquire one.
		118	*/
		119	obj->fence_dirty = false;
		120	obj->fence_reg = I915_FENCE_REG_NONE;
		121	}
		122
2332	Serge	123	/* some bookkeeping */
		124	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		125	size_t size)
		126	{
4104	Serge	127	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	128	dev_priv->mm.object_count++;
		129	dev_priv->mm.object_memory += size;
4104	Serge	130	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	131	}
		132
		133	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		134	size_t size)
		135	{
4104	Serge	136	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	137	dev_priv->mm.object_count--;
		138	dev_priv->mm.object_memory -= size;
4104	Serge	139	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	140	}
		141
		142	static int
3480	Serge	143	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	144	{
		145	int ret;
		146
3480	Serge	147	#define EXIT_COND (!i915_reset_in_progress(error))
		148	if (EXIT_COND)
2332	Serge	149	return 0;
3255	Serge	150	#if 0
3031	serge	151	/*
		152	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		153	* userspace. If it takes that long something really bad is going on and
		154	* we should simply try to bail out and fail as gracefully as possible.
		155	*/
3480	Serge	156	ret = wait_event_interruptible_timeout(error->reset_queue,
		157	EXIT_COND,
		158	10*HZ);
3031	serge	159	if (ret == 0) {
		160	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		161	return -EIO;
		162	} else if (ret < 0) {
2332	Serge	163	return ret;
3031	serge	164	}
2332	Serge	165
3255	Serge	166	#endif
3480	Serge	167	#undef EXIT_COND
3255	Serge	168
2332	Serge	169	return 0;
		170	}
		171
		172	int i915_mutex_lock_interruptible(struct drm_device *dev)
		173	{
3480	Serge	174	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	175	int ret;
		176
3480	Serge	177	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	178	if (ret)
		179	return ret;
		180
3480	Serge	181	ret = mutex_lock_interruptible(&dev->struct_mutex);
		182	if (ret)
		183	return ret;
2332	Serge	184
		185	WARN_ON(i915_verify_lists(dev));
		186	return 0;
		187	}
		188
		189	static inline bool
		190	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		191	{
4104	Serge	192	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	193	}
		194
		195
		196	#if 0
		197
		198	int
		199	i915_gem_init_ioctl(struct drm_device dev, void data,
		200	struct drm_file *file)
		201	{
3480	Serge	202	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	203	struct drm_i915_gem_init *args = data;
		204
3031	serge	205	if (drm_core_check_feature(dev, DRIVER_MODESET))
		206	return -ENODEV;
		207
2332	Serge	208	if (args->gtt_start >= args->gtt_end \|\|
		209	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		210	return -EINVAL;
		211
3031	serge	212	/* GEM with user mode setting was never supported on ilk and later. */
		213	if (INTEL_INFO(dev)->gen >= 5)
		214	return -ENODEV;
		215
2332	Serge	216	mutex_lock(&dev->struct_mutex);
3480	Serge	217	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		218	args->gtt_end);
		219	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	220	mutex_unlock(&dev->struct_mutex);
		221
		222	return 0;
		223	}
2351	Serge	224	#endif
2332	Serge	225
		226	int
		227	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		228	struct drm_file *file)
		229	{
		230	struct drm_i915_private *dev_priv = dev->dev_private;
		231	struct drm_i915_gem_get_aperture *args = data;
		232	struct drm_i915_gem_object *obj;
		233	size_t pinned;
		234
		235	pinned = 0;
		236	mutex_lock(&dev->struct_mutex);
4104	Serge	237	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
3031	serge	238	if (obj->pin_count)
4104	Serge	239	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	240	mutex_unlock(&dev->struct_mutex);
		241
4104	Serge	242	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	243	args->aper_available_size = args->aper_size - pinned;
2332	Serge	244
		245	return 0;
		246	}
		247
3480	Serge	248	void i915_gem_object_alloc(struct drm_device dev)
		249	{
		250	struct drm_i915_private *dev_priv = dev->dev_private;
		251	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		252	}
		253
		254	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		255	{
		256	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		257	kfree(obj);
		258	}
		259
3031	serge	260	static int
		261	i915_gem_create(struct drm_file *file,
2332	Serge	262	struct drm_device *dev,
		263	uint64_t size,
		264	uint32_t *handle_p)
		265	{
		266	struct drm_i915_gem_object *obj;
		267	int ret;
		268	u32 handle;
		269
		270	size = roundup(size, PAGE_SIZE);
2342	Serge	271	if (size == 0)
		272	return -EINVAL;
2332	Serge	273
		274	/* Allocate the new object */
		275	obj = i915_gem_alloc_object(dev, size);
		276	if (obj == NULL)
		277	return -ENOMEM;
		278
		279	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	280	/* drop reference from allocate - handle holds it now */
		281	drm_gem_object_unreference_unlocked(&obj->base);
		282	if (ret)
2332	Serge	283	return ret;
		284
		285	*handle_p = handle;
		286	return 0;
		287	}
		288
		289	int
		290	i915_gem_dumb_create(struct drm_file *file,
		291	struct drm_device *dev,
		292	struct drm_mode_create_dumb *args)
		293	{
		294	/* have to work out size/pitch and return them */
		295	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		296	args->size = args->pitch * args->height;
		297	return i915_gem_create(file, dev,
		298	args->size, &args->handle);
		299	}
		300
2326	Serge	301	/**
2332	Serge	302	* Creates a new mm object and returns a handle to it.
		303	*/
		304	int
		305	i915_gem_create_ioctl(struct drm_device dev, void data,
		306	struct drm_file *file)
		307	{
		308	struct drm_i915_gem_create *args = data;
3031	serge	309
2332	Serge	310	return i915_gem_create(file, dev,
		311	args->size, &args->handle);
		312	}
		313
		314
3260	Serge	315	#if 0
2332	Serge	316
3031	serge	317	static inline int
		318	__copy_to_user_swizzled(char __user *cpu_vaddr,
		319	const char *gpu_vaddr, int gpu_offset,
2332	Serge	320	int length)
		321	{
3031	serge	322	int ret, cpu_offset = 0;
2332	Serge	323
3031	serge	324	while (length > 0) {
		325	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		326	int this_length = min(cacheline_end - gpu_offset, length);
		327	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	328
3031	serge	329	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		330	gpu_vaddr + swizzled_gpu_offset,
		331	this_length);
		332	if (ret)
		333	return ret + length;
2332	Serge	334
3031	serge	335	cpu_offset += this_length;
		336	gpu_offset += this_length;
		337	length -= this_length;
		338	}
		339
		340	return 0;
2332	Serge	341	}
		342
3031	serge	343	static inline int
		344	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		345	const char __user *cpu_vaddr,
		346	int length)
2332	Serge	347	{
3031	serge	348	int ret, cpu_offset = 0;
2332	Serge	349
		350	while (length > 0) {
		351	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		352	int this_length = min(cacheline_end - gpu_offset, length);
		353	int swizzled_gpu_offset = gpu_offset ^ 64;
		354
3031	serge	355	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	356	cpu_vaddr + cpu_offset,
		357	this_length);
3031	serge	358	if (ret)
		359	return ret + length;
		360
2332	Serge	361	cpu_offset += this_length;
		362	gpu_offset += this_length;
		363	length -= this_length;
		364	}
		365
3031	serge	366	return 0;
2332	Serge	367	}
		368
3031	serge	369	/* Per-page copy function for the shmem pread fastpath.
		370	* Flushes invalid cachelines before reading the target if
		371	* needs_clflush is set. */
2332	Serge	372	static int
3031	serge	373	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		374	char __user *user_data,
		375	bool page_do_bit17_swizzling, bool needs_clflush)
		376	{
		377	char *vaddr;
		378	int ret;
		379
		380	if (unlikely(page_do_bit17_swizzling))
		381	return -EINVAL;
		382
		383	vaddr = kmap_atomic(page);
		384	if (needs_clflush)
		385	drm_clflush_virt_range(vaddr + shmem_page_offset,
		386	page_length);
		387	ret = __copy_to_user_inatomic(user_data,
		388	vaddr + shmem_page_offset,
		389	page_length);
		390	kunmap_atomic(vaddr);
		391
		392	return ret ? -EFAULT : 0;
		393	}
		394
		395	static void
		396	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		397	bool swizzled)
		398	{
		399	if (unlikely(swizzled)) {
		400	unsigned long start = (unsigned long) addr;
		401	unsigned long end = (unsigned long) addr + length;
		402
		403	/* For swizzling simply ensure that we always flush both
		404	* channels. Lame, but simple and it works. Swizzled
		405	* pwrite/pread is far from a hotpath - current userspace
		406	* doesn't use it at all. */
		407	start = round_down(start, 128);
		408	end = round_up(end, 128);
		409
		410	drm_clflush_virt_range((void *)start, end - start);
		411	} else {
		412	drm_clflush_virt_range(addr, length);
		413	}
		414
		415	}
		416
		417	/* Only difference to the fast-path function is that this can handle bit17
		418	* and uses non-atomic copy and kmap functions. */
		419	static int
		420	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		421	char __user *user_data,
		422	bool page_do_bit17_swizzling, bool needs_clflush)
		423	{
		424	char *vaddr;
		425	int ret;
		426
		427	vaddr = kmap(page);
		428	if (needs_clflush)
		429	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		430	page_length,
		431	page_do_bit17_swizzling);
		432
		433	if (page_do_bit17_swizzling)
		434	ret = __copy_to_user_swizzled(user_data,
		435	vaddr, shmem_page_offset,
		436	page_length);
		437	else
		438	ret = __copy_to_user(user_data,
		439	vaddr + shmem_page_offset,
		440	page_length);
		441	kunmap(page);
		442
		443	return ret ? - EFAULT : 0;
		444	}
		445
		446	static int
		447	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	448	struct drm_i915_gem_object *obj,
		449	struct drm_i915_gem_pread *args,
		450	struct drm_file *file)
		451	{
3031	serge	452	char __user *user_data;
2332	Serge	453	ssize_t remain;
		454	loff_t offset;
3031	serge	455	int shmem_page_offset, page_length, ret = 0;
		456	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		457	int prefaulted = 0;
		458	int needs_clflush = 0;
3746	Serge	459	struct sg_page_iter sg_iter;
2332	Serge	460
3746	Serge	461	user_data = to_user_ptr(args->data_ptr);
2332	Serge	462	remain = args->size;
		463
3031	serge	464	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		465
		466	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		467	/* If we're not in the cpu read domain, set ourself into the gtt
		468	* read domain and manually flush cachelines (if required). This
		469	* optimizes for the case when the gpu will dirty the data
		470	* anyway again before the next pread happens. */
4104	Serge	471	needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
		472	if (i915_gem_obj_bound_any(obj)) {
3031	serge	473	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		474	if (ret)
		475	return ret;
		476	}
		477	}
		478
		479	ret = i915_gem_object_get_pages(obj);
		480	if (ret)
		481	return ret;
		482
		483	i915_gem_object_pin_pages(obj);
		484
2332	Serge	485	offset = args->offset;
		486
3746	Serge	487	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		488	offset >> PAGE_SHIFT) {
		489	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	490
3031	serge	491	if (remain <= 0)
		492	break;
		493
2332	Serge	494	/* Operation in this page
		495	*
3031	serge	496	* shmem_page_offset = offset within page in shmem file
2332	Serge	497	* page_length = bytes to copy for this page
		498	*/
3031	serge	499	shmem_page_offset = offset_in_page(offset);
2332	Serge	500	page_length = remain;
3031	serge	501	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		502	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	503
3031	serge	504	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		505	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	506
3031	serge	507	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		508	user_data, page_do_bit17_swizzling,
		509	needs_clflush);
		510	if (ret == 0)
		511	goto next_page;
2332	Serge	512
3031	serge	513	mutex_unlock(&dev->struct_mutex);
		514
4104	Serge	515	if (likely(!i915_prefault_disable) && !prefaulted) {
3031	serge	516	ret = fault_in_multipages_writeable(user_data, remain);
		517	/* Userspace is tricking us, but we've already clobbered
		518	* its pages with the prefault and promised to write the
		519	* data up to the first fault. Hence ignore any errors
		520	* and just continue. */
		521	(void)ret;
		522	prefaulted = 1;
		523	}
		524
		525	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		526	user_data, page_do_bit17_swizzling,
		527	needs_clflush);
		528
		529	mutex_lock(&dev->struct_mutex);
		530
		531	next_page:
2332	Serge	532	mark_page_accessed(page);
3031	serge	533
2332	Serge	534	if (ret)
3031	serge	535	goto out;
2332	Serge	536
		537	remain -= page_length;
		538	user_data += page_length;
		539	offset += page_length;
		540	}
		541
3031	serge	542	out:
		543	i915_gem_object_unpin_pages(obj);
		544
		545	return ret;
2332	Serge	546	}
		547
		548	/**
3031	serge	549	* Reads data from the object referenced by handle.
		550	*
		551	* On error, the contents of *data are undefined.
2332	Serge	552	*/
3031	serge	553	int
		554	i915_gem_pread_ioctl(struct drm_device dev, void data,
		555	struct drm_file *file)
		556	{
		557	struct drm_i915_gem_pread *args = data;
		558	struct drm_i915_gem_object *obj;
		559	int ret = 0;
		560
		561	if (args->size == 0)
		562	return 0;
		563
		564	if (!access_ok(VERIFY_WRITE,
3746	Serge	565	to_user_ptr(args->data_ptr),
3031	serge	566	args->size))
		567	return -EFAULT;
		568
		569	ret = i915_mutex_lock_interruptible(dev);
		570	if (ret)
		571	return ret;
		572
		573	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		574	if (&obj->base == NULL) {
		575	ret = -ENOENT;
		576	goto unlock;
		577	}
		578
		579	/* Bounds check source. */
		580	if (args->offset > obj->base.size \|\|
		581	args->size > obj->base.size - args->offset) {
		582	ret = -EINVAL;
		583	goto out;
		584	}
		585
		586	/* prime objects have no backing filp to GEM pread/pwrite
		587	* pages from.
		588	*/
		589	if (!obj->base.filp) {
		590	ret = -EINVAL;
		591	goto out;
		592	}
		593
		594	trace_i915_gem_object_pread(obj, args->offset, args->size);
		595
		596	ret = i915_gem_shmem_pread(dev, obj, args, file);
		597
		598	out:
		599	drm_gem_object_unreference(&obj->base);
		600	unlock:
		601	mutex_unlock(&dev->struct_mutex);
		602	return ret;
		603	}
		604
		605	/* This is the fast write path which cannot handle
		606	* page faults in the source data
		607	*/
		608
		609	static inline int
		610	fast_user_write(struct io_mapping *mapping,
		611	loff_t page_base, int page_offset,
		612	char __user *user_data,
		613	int length)
		614	{
		615	void __iomem *vaddr_atomic;
		616	void *vaddr;
		617	unsigned long unwritten;
		618
		619	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		620	/* We can use the cpu mem copy function because this is X86. */
		621	vaddr = (void __force*)vaddr_atomic + page_offset;
		622	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		623	user_data, length);
		624	io_mapping_unmap_atomic(vaddr_atomic);
		625	return unwritten;
		626	}
3260	Serge	627	#endif
3031	serge	628
3260	Serge	629	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	630	/**
		631	* This is the fast pwrite path, where we copy the data directly from the
		632	* user into the GTT, uncached.
		633	*/
2332	Serge	634	static int
3031	serge	635	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		636	struct drm_i915_gem_object *obj,
		637	struct drm_i915_gem_pwrite *args,
		638	struct drm_file *file)
2332	Serge	639	{
3031	serge	640	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	641	ssize_t remain;
3031	serge	642	loff_t offset, page_base;
		643	char __user *user_data;
		644	int page_offset, page_length, ret;
3260	Serge	645	char *vaddr;
2332	Serge	646
4104	Serge	647	ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
3031	serge	648	if (ret)
		649	goto out;
		650
		651	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		652	if (ret)
		653	goto out_unpin;
		654
		655	ret = i915_gem_object_put_fence(obj);
		656	if (ret)
		657	goto out_unpin;
		658
3260	Serge	659	vaddr = AllocKernelSpace(4096);
		660	if(vaddr == NULL)
		661	{
		662	ret = -ENOSPC;
		663	goto out_unpin;
		664	};
		665
3031	serge	666	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	667	remain = args->size;
		668
4104	Serge	669	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	670
3031	serge	671	while (remain > 0) {
		672	/* Operation in this page
		673	*
		674	* page_base = page offset within aperture
		675	* page_offset = offset within page
		676	* page_length = bytes to copy for this page
		677	*/
		678	page_base = offset & PAGE_MASK;
		679	page_offset = offset_in_page(offset);
		680	page_length = remain;
		681	if ((page_offset + remain) > PAGE_SIZE)
		682	page_length = PAGE_SIZE - page_offset;
2332	Serge	683
3260	Serge	684	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	685
3260	Serge	686	memcpy(vaddr+page_offset, user_data, page_length);
		687
3031	serge	688	remain -= page_length;
		689	user_data += page_length;
		690	offset += page_length;
2332	Serge	691	}
		692
3260	Serge	693	FreeKernelSpace(vaddr);
		694
3031	serge	695	out_unpin:
		696	i915_gem_object_unpin(obj);
		697	out:
		698	return ret;
		699	}
		700
		701	/* Per-page copy function for the shmem pwrite fastpath.
		702	* Flushes invalid cachelines before writing to the target if
		703	* needs_clflush_before is set and flushes out any written cachelines after
		704	* writing if needs_clflush is set. */
		705	static int
		706	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		707	char __user *user_data,
		708	bool page_do_bit17_swizzling,
		709	bool needs_clflush_before,
		710	bool needs_clflush_after)
		711	{
		712	char *vaddr;
3260	Serge	713	int ret = 0;
3031	serge	714
		715	if (unlikely(page_do_bit17_swizzling))
		716	return -EINVAL;
		717
3260	Serge	718	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	719	if (needs_clflush_before)
		720	drm_clflush_virt_range(vaddr + shmem_page_offset,
		721	page_length);
3260	Serge	722	memcpy(vaddr + shmem_page_offset,
3031	serge	723	user_data,
		724	page_length);
		725	if (needs_clflush_after)
		726	drm_clflush_virt_range(vaddr + shmem_page_offset,
		727	page_length);
3260	Serge	728	FreeKernelSpace(vaddr);
3031	serge	729
		730	return ret ? -EFAULT : 0;
		731	}
3260	Serge	732	#if 0
3031	serge	733
		734	/* Only difference to the fast-path function is that this can handle bit17
		735	* and uses non-atomic copy and kmap functions. */
		736	static int
		737	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		738	char __user *user_data,
		739	bool page_do_bit17_swizzling,
		740	bool needs_clflush_before,
		741	bool needs_clflush_after)
		742	{
		743	char *vaddr;
		744	int ret;
		745
		746	vaddr = kmap(page);
		747	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		748	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		749	page_length,
		750	page_do_bit17_swizzling);
		751	if (page_do_bit17_swizzling)
		752	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		753	user_data,
		754	page_length);
		755	else
		756	ret = __copy_from_user(vaddr + shmem_page_offset,
		757	user_data,
		758	page_length);
		759	if (needs_clflush_after)
		760	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		761	page_length,
		762	page_do_bit17_swizzling);
		763	kunmap(page);
		764
		765	return ret ? -EFAULT : 0;
		766	}
3260	Serge	767	#endif
3031	serge	768
3260	Serge	769
3031	serge	770	static int
		771	i915_gem_shmem_pwrite(struct drm_device *dev,
		772	struct drm_i915_gem_object *obj,
		773	struct drm_i915_gem_pwrite *args,
		774	struct drm_file *file)
		775	{
		776	ssize_t remain;
		777	loff_t offset;
		778	char __user *user_data;
		779	int shmem_page_offset, page_length, ret = 0;
		780	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		781	int hit_slowpath = 0;
		782	int needs_clflush_after = 0;
		783	int needs_clflush_before = 0;
3746	Serge	784	struct sg_page_iter sg_iter;
3031	serge	785
3746	Serge	786	user_data = to_user_ptr(args->data_ptr);
3031	serge	787	remain = args->size;
		788
		789	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		790
		791	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		792	/* If we're not in the cpu write domain, set ourself into the gtt
		793	* write domain and manually flush cachelines (if required). This
		794	* optimizes for the case when the gpu will use the data
		795	* right away and we therefore have to clflush anyway. */
4104	Serge	796	needs_clflush_after = cpu_write_needs_clflush(obj);
		797	if (i915_gem_obj_bound_any(obj)) {
3031	serge	798	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		799	if (ret)
		800	return ret;
		801	}
		802	}
4104	Serge	803	/* Same trick applies to invalidate partially written cachelines read
		804	* before writing. */
		805	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		806	needs_clflush_before =
		807	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	808
		809	ret = i915_gem_object_get_pages(obj);
2332	Serge	810	if (ret)
3031	serge	811	return ret;
2332	Serge	812
3031	serge	813	i915_gem_object_pin_pages(obj);
2332	Serge	814
		815	offset = args->offset;
3031	serge	816	obj->dirty = 1;
2332	Serge	817
3746	Serge	818	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		819	offset >> PAGE_SHIFT) {
		820	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	821	int partial_cacheline_write;
2332	Serge	822
3031	serge	823	if (remain <= 0)
		824	break;
		825
2332	Serge	826	/* Operation in this page
		827	*
		828	* shmem_page_offset = offset within page in shmem file
		829	* page_length = bytes to copy for this page
		830	*/
		831	shmem_page_offset = offset_in_page(offset);
		832
		833	page_length = remain;
		834	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		835	page_length = PAGE_SIZE - shmem_page_offset;
		836
3031	serge	837	/* If we don't overwrite a cacheline completely we need to be
		838	* careful to have up-to-date data by first clflushing. Don't
		839	* overcomplicate things and flush the entire patch. */
		840	partial_cacheline_write = needs_clflush_before &&
		841	((shmem_page_offset \| page_length)
3260	Serge	842	& (x86_clflush_size - 1));
2332	Serge	843
3031	serge	844	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		845	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	846
3031	serge	847	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		848	user_data, page_do_bit17_swizzling,
		849	partial_cacheline_write,
		850	needs_clflush_after);
		851	if (ret == 0)
		852	goto next_page;
		853
		854	hit_slowpath = 1;
		855	mutex_unlock(&dev->struct_mutex);
3260	Serge	856	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	857
3260	Serge	858	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		859	// user_data, page_do_bit17_swizzling,
		860	// partial_cacheline_write,
		861	// needs_clflush_after);
		862
3031	serge	863	mutex_lock(&dev->struct_mutex);
		864
		865	next_page:
2332	Serge	866
3031	serge	867	if (ret)
		868	goto out;
		869
2332	Serge	870	remain -= page_length;
3031	serge	871	user_data += page_length;
2332	Serge	872	offset += page_length;
		873	}
		874
		875	out:
3031	serge	876	i915_gem_object_unpin_pages(obj);
		877
		878	if (hit_slowpath) {
3480	Serge	879	/*
		880	* Fixup: Flush cpu caches in case we didn't flush the dirty
		881	* cachelines in-line while writing and the object moved
		882	* out of the cpu write domain while we've dropped the lock.
		883	*/
		884	if (!needs_clflush_after &&
		885	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	886	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	887	i915_gem_chipset_flush(dev);
3031	serge	888	}
2332	Serge	889	}
		890
3031	serge	891	if (needs_clflush_after)
3243	Serge	892	i915_gem_chipset_flush(dev);
3031	serge	893
2332	Serge	894	return ret;
		895	}
3031	serge	896
		897	/**
		898	* Writes data to the object referenced by handle.
		899	*
		900	* On error, the contents of the buffer that were to be modified are undefined.
		901	*/
		902	int
		903	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		904	struct drm_file *file)
		905	{
		906	struct drm_i915_gem_pwrite *args = data;
		907	struct drm_i915_gem_object *obj;
		908	int ret;
		909
4104	Serge	910	if (args->size == 0)
		911	return 0;
		912
3480	Serge	913	if(args->handle == -2)
		914	{
		915	printf("%s handle %d\n", __FUNCTION__, args->handle);
		916	return 0;
		917	}
		918
3031	serge	919	ret = i915_mutex_lock_interruptible(dev);
		920	if (ret)
		921	return ret;
		922
		923	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		924	if (&obj->base == NULL) {
		925	ret = -ENOENT;
		926	goto unlock;
		927	}
		928
		929	/* Bounds check destination. */
		930	if (args->offset > obj->base.size \|\|
		931	args->size > obj->base.size - args->offset) {
		932	ret = -EINVAL;
		933	goto out;
		934	}
		935
		936	/* prime objects have no backing filp to GEM pread/pwrite
		937	* pages from.
		938	*/
		939	if (!obj->base.filp) {
		940	ret = -EINVAL;
		941	goto out;
		942	}
		943
		944	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		945
		946	ret = -EFAULT;
		947	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		948	* it would end up going through the fenced access, and we'll get
		949	* different detiling behavior between reading and writing.
		950	* pread/pwrite currently are reading and writing from the CPU
		951	* perspective, requiring manual detiling by the client.
		952	*/
3260	Serge	953	// if (obj->phys_obj) {
		954	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		955	// goto out;
		956	// }
3031	serge	957
4104	Serge	958	if (obj->tiling_mode == I915_TILING_NONE &&
		959	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		960	cpu_write_needs_clflush(obj)) {
3031	serge	961	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		962	/* Note that the gtt paths might fail with non-page-backed user
		963	* pointers (e.g. gtt mappings when moving data between
		964	* textures). Fallback to the shmem path in that case. */
		965	}
		966
		967	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	968	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	969
		970	out:
		971	drm_gem_object_unreference(&obj->base);
		972	unlock:
		973	mutex_unlock(&dev->struct_mutex);
		974	return ret;
		975	}
		976
		977	int
3480	Serge	978	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	979	bool interruptible)
		980	{
3480	Serge	981	if (i915_reset_in_progress(error)) {
3031	serge	982	/* Non-interruptible callers can't handle -EAGAIN, hence return
		983	* -EIO unconditionally for these. */
		984	if (!interruptible)
		985	return -EIO;
2332	Serge	986
3480	Serge	987	/* Recovery complete, but the reset failed ... */
		988	if (i915_terminally_wedged(error))
3031	serge	989	return -EIO;
2332	Serge	990
3031	serge	991	return -EAGAIN;
		992	}
2332	Serge	993
3031	serge	994	return 0;
		995	}
2332	Serge	996
3031	serge	997	/*
		998	* Compare seqno against outstanding lazy request. Emit a request if they are
		999	* equal.
		1000	*/
		1001	static int
		1002	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1003	{
		1004	int ret;
2332	Serge	1005
3031	serge	1006	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1007
3031	serge	1008	ret = 0;
		1009	if (seqno == ring->outstanding_lazy_request)
4104	Serge	1010	ret = i915_add_request(ring, NULL);
2332	Serge	1011
3031	serge	1012	return ret;
		1013	}
2332	Serge	1014
3031	serge	1015	/**
		1016	* __wait_seqno - wait until execution of seqno has finished
		1017	* @ring: the ring expected to report seqno
		1018	* @seqno: duh!
3480	Serge	1019	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1020	* @interruptible: do an interruptible wait (normally yes)
		1021	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1022	*
3480	Serge	1023	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1024	* values have been read by the caller in an smp safe manner. Where read-side
		1025	* locks are involved, it is sufficient to read the reset_counter before
		1026	* unlocking the lock that protects the seqno. For lockless tricks, the
		1027	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1028	* inserted.
		1029	*
3031	serge	1030	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1031	* errno with remaining time filled in timeout argument.
		1032	*/
		1033	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1034	unsigned reset_counter,
3031	serge	1035	bool interruptible, struct timespec *timeout)
		1036	{
		1037	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1038	struct timespec before, now, wait_time={1,0};
		1039	unsigned long timeout_jiffies;
		1040	long end;
		1041	bool wait_forever = true;
		1042	int ret;
2332	Serge	1043
4104	Serge	1044	WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
		1045
3031	serge	1046	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1047	return 0;
2332	Serge	1048
3031	serge	1049	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1050
3031	serge	1051	if (timeout != NULL) {
		1052	wait_time = *timeout;
		1053	wait_forever = false;
		1054	}
2332	Serge	1055
4104	Serge	1056	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
2332	Serge	1057
3031	serge	1058	if (WARN_ON(!ring->irq_get(ring)))
		1059	return -ENODEV;
2332	Serge	1060
3031	serge	1061	/* Record current time in case interrupted by signal, or wedged * */
		1062	getrawmonotonic(&before);
2332	Serge	1063
3031	serge	1064	#define EXIT_COND \
		1065	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1066	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1067	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1068	do {
3266	Serge	1069	if (interruptible)
		1070	end = wait_event_interruptible_timeout(ring->irq_queue,
		1071	EXIT_COND,
		1072	timeout_jiffies);
		1073	else
3031	serge	1074	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1075	timeout_jiffies);
2332	Serge	1076
3480	Serge	1077	/* We need to check whether any gpu reset happened in between
		1078	* the caller grabbing the seqno and now ... */
		1079	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1080	end = -EAGAIN;
		1081
		1082	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1083	* gone. */
		1084	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1085	if (ret)
		1086	end = ret;
		1087	} while (end == 0 && wait_forever);
2332	Serge	1088
3031	serge	1089	getrawmonotonic(&now);
2332	Serge	1090
3031	serge	1091	ring->irq_put(ring);
		1092	trace_i915_gem_request_wait_end(ring, seqno);
		1093	#undef EXIT_COND
2332	Serge	1094
3031	serge	1095	if (timeout) {
4104	Serge	1096	// struct timespec sleep_time = timespec_sub(now, before);
		1097	// timeout = timespec_sub(timeout, sleep_time);
3031	serge	1098	}
2332	Serge	1099
3031	serge	1100	switch (end) {
		1101	case -EIO:
		1102	case -EAGAIN: /* Wedged */
		1103	case -ERESTARTSYS: /* Signal */
		1104	return (int)end;
		1105	case 0: /* Timeout */
		1106	return -ETIME;
		1107	default: /* Completed */
		1108	WARN_ON(end < 0); /* We're not aware of other errors */
		1109	return 0;
		1110	}
		1111	}
2332	Serge	1112
3031	serge	1113	/**
		1114	* Waits for a sequence number to be signaled, and cleans up the
		1115	* request and object lists appropriately for that event.
		1116	*/
		1117	int
		1118	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1119	{
		1120	struct drm_device *dev = ring->dev;
		1121	struct drm_i915_private *dev_priv = dev->dev_private;
		1122	bool interruptible = dev_priv->mm.interruptible;
		1123	int ret;
2332	Serge	1124
3031	serge	1125	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1126	BUG_ON(seqno == 0);
2332	Serge	1127
3480	Serge	1128	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1129	if (ret)
		1130	return ret;
2332	Serge	1131
3031	serge	1132	ret = i915_gem_check_olr(ring, seqno);
		1133	if (ret)
		1134	return ret;
2332	Serge	1135
3480	Serge	1136	return __wait_seqno(ring, seqno,
		1137	atomic_read(&dev_priv->gpu_error.reset_counter),
		1138	interruptible, NULL);
3031	serge	1139	}
2332	Serge	1140
4104	Serge	1141	static int
		1142	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
		1143	struct intel_ring_buffer *ring)
		1144	{
		1145	i915_gem_retire_requests_ring(ring);
		1146
		1147	/* Manually manage the write flush as we may have not yet
		1148	* retired the buffer.
		1149	*
		1150	* Note that the last_write_seqno is always the earlier of
		1151	* the two (read/write) seqno, so if we haved successfully waited,
		1152	* we know we have passed the last write.
		1153	*/
		1154	obj->last_write_seqno = 0;
		1155	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1156
		1157	return 0;
		1158	}
		1159
3031	serge	1160	/**
		1161	* Ensures that all rendering to the object has completed and the object is
		1162	* safe to unbind from the GTT or access from the CPU.
		1163	*/
		1164	static __must_check int
		1165	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1166	bool readonly)
		1167	{
		1168	struct intel_ring_buffer *ring = obj->ring;
		1169	u32 seqno;
		1170	int ret;
2332	Serge	1171
3031	serge	1172	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1173	if (seqno == 0)
		1174	return 0;
2332	Serge	1175
3031	serge	1176	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1177	if (ret)
		1178	return ret;
2332	Serge	1179
4104	Serge	1180	return i915_gem_object_wait_rendering__tail(obj, ring);
3031	serge	1181	}
2332	Serge	1182
3260	Serge	1183	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1184	* as the object state may change during this call.
		1185	*/
		1186	static __must_check int
		1187	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1188	bool readonly)
		1189	{
		1190	struct drm_device *dev = obj->base.dev;
		1191	struct drm_i915_private *dev_priv = dev->dev_private;
		1192	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1193	unsigned reset_counter;
3260	Serge	1194	u32 seqno;
		1195	int ret;
2332	Serge	1196
3260	Serge	1197	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1198	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1199
3260	Serge	1200	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1201	if (seqno == 0)
		1202	return 0;
2332	Serge	1203
3480	Serge	1204	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1205	if (ret)
		1206	return ret;
2332	Serge	1207
3260	Serge	1208	ret = i915_gem_check_olr(ring, seqno);
		1209	if (ret)
		1210	return ret;
2332	Serge	1211
3480	Serge	1212	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1213	mutex_unlock(&dev->struct_mutex);
3480	Serge	1214	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1215	mutex_lock(&dev->struct_mutex);
4104	Serge	1216	if (ret)
		1217	return ret;
2332	Serge	1218
4104	Serge	1219	return i915_gem_object_wait_rendering__tail(obj, ring);
3260	Serge	1220	}
2332	Serge	1221
3260	Serge	1222	/**
		1223	* Called when user space prepares to use an object with the CPU, either
		1224	* through the mmap ioctl's mapping or a GTT mapping.
		1225	*/
		1226	int
		1227	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1228	struct drm_file *file)
		1229	{
		1230	struct drm_i915_gem_set_domain *args = data;
		1231	struct drm_i915_gem_object *obj;
		1232	uint32_t read_domains = args->read_domains;
		1233	uint32_t write_domain = args->write_domain;
		1234	int ret;
2332	Serge	1235
3480	Serge	1236
		1237	if(args->handle == -2)
		1238	{
		1239	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1240	return 0;
		1241	}
		1242
3260	Serge	1243	/* Only handle setting domains to types used by the CPU. */
		1244	if (write_domain & I915_GEM_GPU_DOMAINS)
		1245	return -EINVAL;
2332	Serge	1246
3260	Serge	1247	if (read_domains & I915_GEM_GPU_DOMAINS)
		1248	return -EINVAL;
2332	Serge	1249
3260	Serge	1250	/* Having something in the write domain implies it's in the read
		1251	* domain, and only that read domain. Enforce that in the request.
		1252	*/
		1253	if (write_domain != 0 && read_domains != write_domain)
		1254	return -EINVAL;
2332	Serge	1255
3260	Serge	1256	ret = i915_mutex_lock_interruptible(dev);
		1257	if (ret)
		1258	return ret;
2332	Serge	1259
3260	Serge	1260	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1261	if (&obj->base == NULL) {
		1262	ret = -ENOENT;
		1263	goto unlock;
		1264	}
2332	Serge	1265
3260	Serge	1266	/* Try to flush the object off the GPU without holding the lock.
		1267	* We will repeat the flush holding the lock in the normal manner
		1268	* to catch cases where we are gazumped.
		1269	*/
		1270	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1271	if (ret)
		1272	goto unref;
2332	Serge	1273
3260	Serge	1274	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1275	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1276
3260	Serge	1277	/* Silently promote "you're not bound, there was nothing to do"
		1278	* to success, since the client was just asking us to
		1279	* make sure everything was done.
		1280	*/
		1281	if (ret == -EINVAL)
		1282	ret = 0;
		1283	} else {
		1284	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1285	}
2332	Serge	1286
3260	Serge	1287	unref:
		1288	drm_gem_object_unreference(&obj->base);
		1289	unlock:
		1290	mutex_unlock(&dev->struct_mutex);
		1291	return ret;
		1292	}
2332	Serge	1293
		1294
		1295
		1296
		1297
		1298
3260	Serge	1299	/**
		1300	* Maps the contents of an object, returning the address it is mapped
		1301	* into.
		1302	*
		1303	* While the mapping holds a reference on the contents of the object, it doesn't
		1304	* imply a ref on the object itself.
		1305	*/
		1306	int
		1307	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1308	struct drm_file *file)
		1309	{
		1310	struct drm_i915_gem_mmap *args = data;
		1311	struct drm_gem_object *obj;
		1312	unsigned long addr = 0;
2332	Serge	1313
3480	Serge	1314	if(args->handle == -2)
		1315	{
		1316	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1317	return 0;
		1318	}
		1319
3260	Serge	1320	obj = drm_gem_object_lookup(dev, file, args->handle);
		1321	if (obj == NULL)
		1322	return -ENOENT;
4104	Serge	1323
3260	Serge	1324	/* prime objects have no backing filp to GEM mmap
		1325	* pages from.
		1326	*/
		1327	if (!obj->filp) {
		1328	drm_gem_object_unreference_unlocked(obj);
		1329	return -EINVAL;
		1330	}
2332	Serge	1331
3263	Serge	1332	addr = vm_mmap(obj->filp, 0, args->size,
		1333	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1334	args->offset);
3260	Serge	1335	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1336	if (IS_ERR((void *)addr))
		1337	return addr;
2332	Serge	1338
3260	Serge	1339	args->addr_ptr = (uint64_t) addr;
2332	Serge	1340
3263	Serge	1341	return 0;
3260	Serge	1342	}
2332	Serge	1343
		1344
		1345
		1346
		1347
		1348
		1349
		1350
3031	serge	1351
		1352
		1353
		1354
		1355
		1356	/**
		1357	* i915_gem_release_mmap - remove physical page mappings
		1358	* @obj: obj in question
		1359	*
		1360	* Preserve the reservation of the mmapping with the DRM core code, but
		1361	* relinquish ownership of the pages back to the system.
		1362	*
		1363	* It is vital that we remove the page mapping if we have mapped a tiled
		1364	* object through the GTT and then lose the fence register due to
		1365	* resource pressure. Similarly if the object has been moved out of the
		1366	* aperture, than pages mapped into userspace must be revoked. Removing the
		1367	* mapping will then trigger a page fault on the next user access, allowing
		1368	* fixup by i915_gem_fault().
		1369	*/
		1370	void
		1371	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1372	{
		1373	if (!obj->fault_mappable)
		1374	return;
		1375
4104	Serge	1376	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1377	obj->fault_mappable = false;
		1378	}
		1379
3480	Serge	1380	uint32_t
2332	Serge	1381	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1382	{
		1383	uint32_t gtt_size;
		1384
		1385	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1386	tiling_mode == I915_TILING_NONE)
		1387	return size;
		1388
		1389	/* Previous chips need a power-of-two fence region when tiling */
		1390	if (INTEL_INFO(dev)->gen == 3)
		1391	gtt_size = 1024*1024;
		1392	else
		1393	gtt_size = 512*1024;
		1394
		1395	while (gtt_size < size)
		1396	gtt_size <<= 1;
		1397
		1398	return gtt_size;
		1399	}
		1400
		1401	/**
		1402	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1403	* @obj: object to check
		1404	*
		1405	* Return the required GTT alignment for an object, taking into account
		1406	* potential fence register mapping.
		1407	*/
3480	Serge	1408	uint32_t
		1409	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1410	int tiling_mode, bool fenced)
2332	Serge	1411	{
		1412	/*
		1413	* Minimum alignment is 4k (GTT page size), but might be greater
		1414	* if a fence register is needed for the object.
		1415	*/
3480	Serge	1416	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1417	tiling_mode == I915_TILING_NONE)
		1418	return 4096;
		1419
		1420	/*
		1421	* Previous chips need to be aligned to the size of the smallest
		1422	* fence register that can contain the object.
		1423	*/
		1424	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1425	}
		1426
		1427	/**
		1428	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1429	* unfenced object
		1430	* @dev: the device
		1431	* @size: size of the object
		1432	* @tiling_mode: tiling mode of the object
		1433	*
		1434	* Return the required GTT alignment for an object, only taking into account
		1435	* unfenced tiled surface requirements.
		1436	*/
		1437	uint32_t
		1438	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1439	uint32_t size,
		1440	int tiling_mode)
		1441	{
		1442	/*
		1443	* Minimum alignment is 4k (GTT page size) for sane hw.
		1444	*/
		1445	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1446	tiling_mode == I915_TILING_NONE)
		1447	return 4096;
		1448
		1449	/* Previous hardware however needs to be aligned to a power-of-two
		1450	* tile height. The simplest method for determining this is to reuse
		1451	* the power-of-tile object size.
		1452	*/
		1453	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1454	}
		1455
3480	Serge	1456	int
		1457	i915_gem_mmap_gtt(struct drm_file *file,
		1458	struct drm_device *dev,
		1459	uint32_t handle,
		1460	uint64_t *offset)
		1461	{
		1462	struct drm_i915_private *dev_priv = dev->dev_private;
		1463	struct drm_i915_gem_object *obj;
		1464	unsigned long pfn;
		1465	char mem, ptr;
		1466	int ret;
		1467
		1468	ret = i915_mutex_lock_interruptible(dev);
		1469	if (ret)
		1470	return ret;
		1471
		1472	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1473	if (&obj->base == NULL) {
		1474	ret = -ENOENT;
		1475	goto unlock;
		1476	}
		1477
		1478	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1479	ret = -E2BIG;
		1480	goto out;
		1481	}
		1482
		1483	if (obj->madv != I915_MADV_WILLNEED) {
		1484	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1485	ret = -EINVAL;
		1486	goto out;
		1487	}
		1488	/* Now bind it into the GTT if needed */
4104	Serge	1489	ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
3480	Serge	1490	if (ret)
		1491	goto out;
		1492
		1493	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1494	if (ret)
		1495	goto unpin;
		1496
		1497	ret = i915_gem_object_get_fence(obj);
		1498	if (ret)
		1499	goto unpin;
		1500
		1501	obj->fault_mappable = true;
		1502
4104	Serge	1503	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1504
		1505	/* Finally, remap it using the new GTT offset */
		1506
		1507	mem = UserAlloc(obj->base.size);
		1508	if(unlikely(mem == NULL))
		1509	{
		1510	ret = -ENOMEM;
		1511	goto unpin;
		1512	}
		1513
		1514	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1515	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1516
		1517	unpin:
		1518	i915_gem_object_unpin(obj);
		1519
		1520
4104	Serge	1521	*offset = mem;
3480	Serge	1522
		1523	out:
		1524	drm_gem_object_unreference(&obj->base);
		1525	unlock:
		1526	mutex_unlock(&dev->struct_mutex);
		1527	return ret;
		1528	}
		1529
		1530	/**
		1531	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1532	* @dev: DRM device
		1533	* @data: GTT mapping ioctl data
		1534	* @file: GEM object info
		1535	*
		1536	* Simply returns the fake offset to userspace so it can mmap it.
		1537	* The mmap call will end up in drm_gem_mmap(), which will set things
		1538	* up so we can get faults in the handler above.
		1539	*
		1540	* The fault handler will take care of binding the object into the GTT
		1541	* (since it may have been evicted to make room for something), allocating
		1542	* a fence register, and mapping the appropriate aperture address into
		1543	* userspace.
		1544	*/
		1545	int
		1546	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1547	struct drm_file *file)
		1548	{
		1549	struct drm_i915_gem_mmap_gtt *args = data;
		1550
		1551	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1552	}
		1553
3031	serge	1554	/* Immediately discard the backing storage */
		1555	static void
		1556	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1557	{
		1558	// struct inode *inode;
2332	Serge	1559
3031	serge	1560	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1561
3263	Serge	1562	if (obj->base.filp == NULL)
		1563	return;
2332	Serge	1564
3031	serge	1565	/* Our goal here is to return as much of the memory as
		1566	* is possible back to the system as we are called from OOM.
		1567	* To do this we must instruct the shmfs to drop all of its
		1568	* backing pages, now.
		1569	*/
		1570	// inode = obj->base.filp->f_path.dentry->d_inode;
		1571	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1572
3031	serge	1573	obj->madv = __I915_MADV_PURGED;
		1574	}
2332	Serge	1575
3031	serge	1576	static inline int
		1577	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1578	{
		1579	return obj->madv == I915_MADV_DONTNEED;
		1580	}
2332	Serge	1581
3031	serge	1582	static void
		1583	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1584	{
3746	Serge	1585	struct sg_page_iter sg_iter;
		1586	int ret;
2332	Serge	1587
3031	serge	1588	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1589
3031	serge	1590	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1591	if (ret) {
		1592	/* In the event of a disaster, abandon all caches and
		1593	* hope for the best.
		1594	*/
		1595	WARN_ON(ret != -EIO);
4104	Serge	1596	i915_gem_clflush_object(obj, true);
3031	serge	1597	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1598	}
2332	Serge	1599
3031	serge	1600	if (obj->madv == I915_MADV_DONTNEED)
		1601	obj->dirty = 0;
2332	Serge	1602
3746	Serge	1603	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1604	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1605
3290	Serge	1606	page_cache_release(page);
3243	Serge	1607	}
		1608	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1609
4104	Serge	1610	obj->dirty = 0;
3243	Serge	1611
		1612	sg_free_table(obj->pages);
		1613	kfree(obj->pages);
3031	serge	1614	}
2332	Serge	1615
3480	Serge	1616	int
3031	serge	1617	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1618	{
		1619	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1620
3243	Serge	1621	if (obj->pages == NULL)
3031	serge	1622	return 0;
2332	Serge	1623
3031	serge	1624	if (obj->pages_pin_count)
		1625	return -EBUSY;
		1626
4104	Serge	1627	BUG_ON(i915_gem_obj_bound_any(obj));
		1628
3243	Serge	1629	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1630	* array, hence protect them from being reaped by removing them from gtt
		1631	* lists early. */
4104	Serge	1632	list_del(&obj->global_list);
3243	Serge	1633
3031	serge	1634	ops->put_pages(obj);
3243	Serge	1635	obj->pages = NULL;
3031	serge	1636
		1637	if (i915_gem_object_is_purgeable(obj))
		1638	i915_gem_object_truncate(obj);
		1639
		1640	return 0;
		1641	}
		1642
		1643
		1644
		1645
		1646
		1647
		1648
		1649
2332	Serge	1650	static int
3031	serge	1651	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1652	{
3260	Serge	1653	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1654	int page_count, i;
4104	Serge	1655	struct sg_table *st;
3243	Serge	1656	struct scatterlist *sg;
3746	Serge	1657	struct sg_page_iter sg_iter;
3243	Serge	1658	struct page *page;
3746	Serge	1659	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1660	gfp_t gfp;
2332	Serge	1661
3243	Serge	1662	/* Assert that the object is not currently in any GPU domain. As it
		1663	* wasn't in the GTT, there shouldn't be any way it could have been in
		1664	* a GPU cache
2332	Serge	1665	*/
3243	Serge	1666	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1667	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1668
		1669	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1670	if (st == NULL)
		1671	return -ENOMEM;
		1672
2332	Serge	1673	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1674	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1675	kfree(st);
3746	Serge	1676	FAIL();
2332	Serge	1677	return -ENOMEM;
3243	Serge	1678	}
2332	Serge	1679
3243	Serge	1680	/* Get the list of pages out of our struct file. They'll be pinned
		1681	* at this point until we release them.
		1682	*
		1683	* Fail silently without starting the shrinker
		1684	*/
3746	Serge	1685	sg = st->sgl;
		1686	st->nents = 0;
		1687	for (i = 0; i < page_count; i++) {
4104	Serge	1688	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1689	if (IS_ERR(page)) {
		1690	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1691	goto err_pages;
		1692
3260	Serge	1693	}
3746	Serge	1694
		1695	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1696	if (i)
		1697	sg = sg_next(sg);
		1698	st->nents++;
3243	Serge	1699	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1700	} else {
		1701	sg->length += PAGE_SIZE;
		1702	}
		1703	last_pfn = page_to_pfn(page);
3243	Serge	1704	}
3031	serge	1705
3746	Serge	1706	sg_mark_end(sg);
3243	Serge	1707	obj->pages = st;
3031	serge	1708
2332	Serge	1709	return 0;
		1710
		1711	err_pages:
3746	Serge	1712	sg_mark_end(sg);
		1713	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1714	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1715	sg_free_table(st);
		1716	kfree(st);
3746	Serge	1717	FAIL();
3243	Serge	1718	return PTR_ERR(page);
2332	Serge	1719	}
		1720
3031	serge	1721	/* Ensure that the associated pages are gathered from the backing storage
		1722	* and pinned into our object. i915_gem_object_get_pages() may be called
		1723	* multiple times before they are released by a single call to
		1724	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1725	* either as a result of memory pressure (reaping pages under the shrinker)
		1726	* or as the object is itself released.
		1727	*/
		1728	int
		1729	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1730	{
3031	serge	1731	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1732	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1733	int ret;
2332	Serge	1734
3243	Serge	1735	if (obj->pages)
3031	serge	1736	return 0;
2332	Serge	1737
3031	serge	1738	BUG_ON(obj->pages_pin_count);
2332	Serge	1739
3031	serge	1740	ret = ops->get_pages(obj);
		1741	if (ret)
		1742	return ret;
2344	Serge	1743
4104	Serge	1744	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1745	return 0;
2332	Serge	1746	}
		1747
		1748	void
		1749	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1750	struct intel_ring_buffer *ring)
2332	Serge	1751	{
		1752	struct drm_device *dev = obj->base.dev;
		1753	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1754	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1755
		1756	BUG_ON(ring == NULL);
4104	Serge	1757	if (obj->ring != ring && obj->last_write_seqno) {
		1758	/* Keep the seqno relative to the current ring */
		1759	obj->last_write_seqno = seqno;
		1760	}
2332	Serge	1761	obj->ring = ring;
		1762
		1763	/* Add a reference if we're newly entering the active list. */
		1764	if (!obj->active) {
2344	Serge	1765	drm_gem_object_reference(&obj->base);
2332	Serge	1766	obj->active = 1;
		1767	}
		1768
		1769	list_move_tail(&obj->ring_list, &ring->active_list);
		1770
3031	serge	1771	obj->last_read_seqno = seqno;
		1772
2332	Serge	1773	if (obj->fenced_gpu_access) {
3031	serge	1774	obj->last_fenced_seqno = seqno;
		1775
		1776	/* Bump MRU to take account of the delayed flush */
		1777	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1778	struct drm_i915_fence_reg *reg;
		1779
		1780	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1781	list_move_tail(®->lru_list,
		1782	&dev_priv->mm.fence_list);
		1783	}
2332	Serge	1784	}
		1785	}
		1786
2344	Serge	1787	static void
3031	serge	1788	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1789	{
4104	Serge	1790	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1791	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
		1792	struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2332	Serge	1793
3031	serge	1794	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1795	BUG_ON(!obj->active);
2332	Serge	1796
4104	Serge	1797	list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2344	Serge	1798
3031	serge	1799	list_del_init(&obj->ring_list);
2352	Serge	1800	obj->ring = NULL;
2344	Serge	1801
3031	serge	1802	obj->last_read_seqno = 0;
		1803	obj->last_write_seqno = 0;
		1804	obj->base.write_domain = 0;
		1805
		1806	obj->last_fenced_seqno = 0;
2352	Serge	1807	obj->fenced_gpu_access = false;
2344	Serge	1808
2352	Serge	1809	obj->active = 0;
		1810	drm_gem_object_unreference(&obj->base);
		1811
		1812	WARN_ON(i915_verify_lists(dev));
		1813	}
		1814
3243	Serge	1815	static int
3480	Serge	1816	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1817	{
3243	Serge	1818	struct drm_i915_private *dev_priv = dev->dev_private;
		1819	struct intel_ring_buffer *ring;
		1820	int ret, i, j;
2344	Serge	1821
3480	Serge	1822	/* Carefully retire all requests without writing to the rings */
3243	Serge	1823	for_each_ring(ring, dev_priv, i) {
3480	Serge	1824	ret = intel_ring_idle(ring);
3243	Serge	1825	if (ret)
		1826	return ret;
3480	Serge	1827	}
		1828	i915_gem_retire_requests(dev);
3243	Serge	1829
3480	Serge	1830	/* Finally reset hw state */
3243	Serge	1831	for_each_ring(ring, dev_priv, i) {
3480	Serge	1832	intel_ring_init_seqno(ring, seqno);
		1833
3243	Serge	1834	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1835	ring->sync_seqno[j] = 0;
		1836	}
		1837
		1838	return 0;
2344	Serge	1839	}
		1840
3480	Serge	1841	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1842	{
		1843	struct drm_i915_private *dev_priv = dev->dev_private;
		1844	int ret;
		1845
		1846	if (seqno == 0)
		1847	return -EINVAL;
		1848
		1849	/* HWS page needs to be set less than what we
		1850	* will inject to ring
		1851	*/
		1852	ret = i915_gem_init_seqno(dev, seqno - 1);
		1853	if (ret)
		1854	return ret;
		1855
		1856	/* Carefully set the last_seqno value so that wrap
		1857	* detection still works
		1858	*/
		1859	dev_priv->next_seqno = seqno;
		1860	dev_priv->last_seqno = seqno - 1;
		1861	if (dev_priv->last_seqno == 0)
		1862	dev_priv->last_seqno--;
		1863
		1864	return 0;
		1865	}
		1866
3243	Serge	1867	int
		1868	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1869	{
3243	Serge	1870	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1871
3243	Serge	1872	/* reserve 0 for non-seqno */
		1873	if (dev_priv->next_seqno == 0) {
3480	Serge	1874	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1875	if (ret)
		1876	return ret;
		1877
		1878	dev_priv->next_seqno = 1;
		1879	}
		1880
3480	Serge	1881	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1882	return 0;
2332	Serge	1883	}
		1884
4104	Serge	1885	int __i915_add_request(struct intel_ring_buffer *ring,
2352	Serge	1886	struct drm_file *file,
4104	Serge	1887	struct drm_i915_gem_object *obj,
3031	serge	1888	u32 *out_seqno)
2352	Serge	1889	{
		1890	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1891	struct drm_i915_gem_request *request;
4104	Serge	1892	u32 request_ring_position, request_start;
2352	Serge	1893	int was_empty;
		1894	int ret;
2332	Serge	1895
4104	Serge	1896	request_start = intel_ring_get_tail(ring);
3031	serge	1897	/*
		1898	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1899	* after having emitted the batchbuffer command. Hence we need to fix
		1900	* things up similar to emitting the lazy request. The difference here
		1901	* is that the flush _must_ happen before the next request, no matter
		1902	* what.
		1903	*/
4104	Serge	1904	ret = intel_ring_flush_all_caches(ring);
		1905	if (ret)
		1906	return ret;
2332	Serge	1907
3031	serge	1908	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1909	if (request == NULL)
		1910	return -ENOMEM;
		1911
		1912
		1913	/* Record the position of the start of the request so that
		1914	* should we detect the updated seqno part-way through the
4104	Serge	1915	* GPU processing the request, we never over-estimate the
3031	serge	1916	* position of the head.
		1917	*/
4104	Serge	1918	request_ring_position = intel_ring_get_tail(ring);
3031	serge	1919
3243	Serge	1920	ret = ring->add_request(ring);
3031	serge	1921	if (ret) {
		1922	kfree(request);
4104	Serge	1923	return ret;
3031	serge	1924	}
2332	Serge	1925
3243	Serge	1926	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1927	request->ring = ring;
4104	Serge	1928	request->head = request_start;
3031	serge	1929	request->tail = request_ring_position;
4104	Serge	1930	request->ctx = ring->last_context;
		1931	request->batch_obj = obj;
		1932
		1933	/* Whilst this request exists, batch_obj will be on the
		1934	* active_list, and so will hold the active reference. Only when this
		1935	* request is retired will the the batch_obj be moved onto the
		1936	* inactive_list and lose its active reference. Hence we do not need
		1937	* to explicitly hold another reference here.
		1938	*/
		1939
		1940	if (request->ctx)
		1941	i915_gem_context_reference(request->ctx);
		1942
3031	serge	1943	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1944	was_empty = list_empty(&ring->request_list);
		1945	list_add_tail(&request->list, &ring->request_list);
3031	serge	1946	request->file_priv = NULL;
2332	Serge	1947
3263	Serge	1948	if (file) {
		1949	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1950
3263	Serge	1951	spin_lock(&file_priv->mm.lock);
		1952	request->file_priv = file_priv;
		1953	list_add_tail(&request->client_list,
		1954	&file_priv->mm.request_list);
		1955	spin_unlock(&file_priv->mm.lock);
		1956	}
		1957
		1958	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1959	ring->outstanding_lazy_request = 0;
2332	Serge	1960
4104	Serge	1961	if (!dev_priv->ums.mm_suspended) {
		1962	// i915_queue_hangcheck(ring->dev);
		1963
		1964	if (was_empty) {
2360	Serge	1965	queue_delayed_work(dev_priv->wq,
3482	Serge	1966	&dev_priv->mm.retire_work,
		1967	round_jiffies_up_relative(HZ));
4104	Serge	1968	intel_mark_busy(dev_priv->dev);
		1969	}
		1970	}
3031	serge	1971
		1972	if (out_seqno)
3243	Serge	1973	*out_seqno = request->seqno;
2352	Serge	1974	return 0;
		1975	}
2332	Serge	1976
3263	Serge	1977	static inline void
		1978	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1979	{
		1980	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1981
3263	Serge	1982	if (!file_priv)
		1983	return;
2332	Serge	1984
3263	Serge	1985	spin_lock(&file_priv->mm.lock);
		1986	if (request->file_priv) {
		1987	list_del(&request->client_list);
		1988	request->file_priv = NULL;
		1989	}
		1990	spin_unlock(&file_priv->mm.lock);
		1991	}
2332	Serge	1992
4104	Serge	1993	static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
		1994	struct i915_address_space *vm)
		1995	{
		1996	if (acthd >= i915_gem_obj_offset(obj, vm) &&
		1997	acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
		1998	return true;
		1999
		2000	return false;
		2001	}
		2002
		2003	static bool i915_head_inside_request(const u32 acthd_unmasked,
		2004	const u32 request_start,
		2005	const u32 request_end)
		2006	{
		2007	const u32 acthd = acthd_unmasked & HEAD_ADDR;
		2008
		2009	if (request_start < request_end) {
		2010	if (acthd >= request_start && acthd < request_end)
		2011	return true;
		2012	} else if (request_start > request_end) {
		2013	if (acthd >= request_start \|\| acthd < request_end)
		2014	return true;
		2015	}
		2016
		2017	return false;
		2018	}
		2019
		2020	static struct i915_address_space *
		2021	request_to_vm(struct drm_i915_gem_request *request)
		2022	{
		2023	struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
		2024	struct i915_address_space *vm;
		2025
		2026	vm = &dev_priv->gtt.base;
		2027
		2028	return vm;
		2029	}
		2030
		2031	static bool i915_request_guilty(struct drm_i915_gem_request *request,
		2032	const u32 acthd, bool *inside)
		2033	{
		2034	/* There is a possibility that unmasked head address
		2035	* pointing inside the ring, matches the batch_obj address range.
		2036	* However this is extremely unlikely.
		2037	*/
		2038	if (request->batch_obj) {
		2039	if (i915_head_inside_object(acthd, request->batch_obj,
		2040	request_to_vm(request))) {
		2041	*inside = true;
		2042	return true;
		2043	}
		2044	}
		2045
		2046	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		2047	*inside = false;
		2048	return true;
		2049	}
		2050
		2051	return false;
		2052	}
		2053
		2054	static void i915_set_reset_status(struct intel_ring_buffer *ring,
		2055	struct drm_i915_gem_request *request,
		2056	u32 acthd)
		2057	{
		2058	struct i915_ctx_hang_stats *hs = NULL;
		2059	bool inside, guilty;
		2060	unsigned long offset = 0;
		2061
		2062	/* Innocent until proven guilty */
		2063	guilty = false;
		2064
		2065	if (request->batch_obj)
		2066	offset = i915_gem_obj_offset(request->batch_obj,
		2067	request_to_vm(request));
		2068
		2069	if (ring->hangcheck.action != HANGCHECK_WAIT &&
		2070	i915_request_guilty(request, acthd, &inside)) {
		2071	DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
		2072	ring->name,
		2073	inside ? "inside" : "flushing",
		2074	offset,
		2075	request->ctx ? request->ctx->id : 0,
		2076	acthd);
		2077
		2078	guilty = true;
		2079	}
		2080
		2081	/* If contexts are disabled or this is the default context, use
		2082	* file_priv->reset_state
		2083	*/
		2084	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		2085	hs = &request->ctx->hang_stats;
		2086	else if (request->file_priv)
		2087	hs = &request->file_priv->hang_stats;
		2088
		2089	if (hs) {
		2090	if (guilty)
		2091	hs->batch_active++;
		2092	else
		2093	hs->batch_pending++;
		2094	}
		2095	}
		2096
		2097	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2098	{
		2099	list_del(&request->list);
		2100	i915_gem_request_remove_from_client(request);
		2101
		2102	if (request->ctx)
		2103	i915_gem_context_unreference(request->ctx);
		2104
		2105	kfree(request);
		2106	}
		2107
3031	serge	2108	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		2109	struct intel_ring_buffer *ring)
		2110	{
4104	Serge	2111	u32 completed_seqno;
		2112	u32 acthd;
		2113
		2114	acthd = intel_ring_get_active_head(ring);
		2115	completed_seqno = ring->get_seqno(ring, false);
		2116
3031	serge	2117	while (!list_empty(&ring->request_list)) {
		2118	struct drm_i915_gem_request *request;
2332	Serge	2119
3031	serge	2120	request = list_first_entry(&ring->request_list,
		2121	struct drm_i915_gem_request,
		2122	list);
2332	Serge	2123
4104	Serge	2124	if (request->seqno > completed_seqno)
		2125	i915_set_reset_status(ring, request, acthd);
		2126
		2127	i915_gem_free_request(request);
3031	serge	2128	}
2332	Serge	2129
3031	serge	2130	while (!list_empty(&ring->active_list)) {
		2131	struct drm_i915_gem_object *obj;
2332	Serge	2132
3031	serge	2133	obj = list_first_entry(&ring->active_list,
		2134	struct drm_i915_gem_object,
		2135	ring_list);
2332	Serge	2136
3031	serge	2137	i915_gem_object_move_to_inactive(obj);
		2138	}
		2139	}
2332	Serge	2140
3746	Serge	2141	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2142	{
		2143	struct drm_i915_private *dev_priv = dev->dev_private;
		2144	int i;
2332	Serge	2145
3031	serge	2146	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2147	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2148
		2149	/*
		2150	* Commit delayed tiling changes if we have an object still
		2151	* attached to the fence, otherwise just clear the fence.
		2152	*/
		2153	if (reg->obj) {
		2154	i915_gem_object_update_fence(reg->obj, reg,
		2155	reg->obj->tiling_mode);
		2156	} else {
		2157	i915_gem_write_fence(dev, i, NULL);
		2158	}
3031	serge	2159	}
		2160	}
2360	Serge	2161
3031	serge	2162	void i915_gem_reset(struct drm_device *dev)
		2163	{
		2164	struct drm_i915_private *dev_priv = dev->dev_private;
		2165	struct intel_ring_buffer *ring;
		2166	int i;
2360	Serge	2167
3031	serge	2168	for_each_ring(ring, dev_priv, i)
		2169	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	2170
3746	Serge	2171	i915_gem_restore_fences(dev);
3031	serge	2172	}
2360	Serge	2173
2352	Serge	2174	/**
		2175	* This function clears the request list as sequence numbers are passed.
		2176	*/
3031	serge	2177	void
2352	Serge	2178	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2179	{
		2180	uint32_t seqno;
2332	Serge	2181
2352	Serge	2182	if (list_empty(&ring->request_list))
		2183	return;
2332	Serge	2184
2352	Serge	2185	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2186
3031	serge	2187	seqno = ring->get_seqno(ring, true);
2332	Serge	2188
2352	Serge	2189	while (!list_empty(&ring->request_list)) {
		2190	struct drm_i915_gem_request *request;
2332	Serge	2191
2352	Serge	2192	request = list_first_entry(&ring->request_list,
		2193	struct drm_i915_gem_request,
		2194	list);
2332	Serge	2195
2352	Serge	2196	if (!i915_seqno_passed(seqno, request->seqno))
		2197	break;
2332	Serge	2198
2352	Serge	2199	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2200	/* We know the GPU must have read the request to have
		2201	* sent us the seqno + interrupt, so use the position
		2202	* of tail of the request to update the last known position
		2203	* of the GPU head.
		2204	*/
		2205	ring->last_retired_head = request->tail;
2332	Serge	2206
4104	Serge	2207	i915_gem_free_request(request);
2352	Serge	2208	}
2332	Serge	2209
2352	Serge	2210	/* Move any buffers on the active list that are no longer referenced
		2211	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2212	*/
		2213	while (!list_empty(&ring->active_list)) {
		2214	struct drm_i915_gem_object *obj;
2332	Serge	2215
2352	Serge	2216	obj = list_first_entry(&ring->active_list,
		2217	struct drm_i915_gem_object,
		2218	ring_list);
2332	Serge	2219
3031	serge	2220	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2221	break;
2332	Serge	2222
2352	Serge	2223	i915_gem_object_move_to_inactive(obj);
		2224	}
2332	Serge	2225
2352	Serge	2226	if (unlikely(ring->trace_irq_seqno &&
		2227	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2228	ring->irq_put(ring);
		2229	ring->trace_irq_seqno = 0;
		2230	}
2332	Serge	2231
2352	Serge	2232	WARN_ON(i915_verify_lists(ring->dev));
		2233	}
2332	Serge	2234
2352	Serge	2235	void
		2236	i915_gem_retire_requests(struct drm_device *dev)
		2237	{
		2238	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2239	struct intel_ring_buffer *ring;
2352	Serge	2240	int i;
2332	Serge	2241
3031	serge	2242	for_each_ring(ring, dev_priv, i)
		2243	i915_gem_retire_requests_ring(ring);
2352	Serge	2244	}
		2245
2360	Serge	2246	static void
		2247	i915_gem_retire_work_handler(struct work_struct *work)
		2248	{
		2249	drm_i915_private_t *dev_priv;
		2250	struct drm_device *dev;
3031	serge	2251	struct intel_ring_buffer *ring;
2360	Serge	2252	bool idle;
		2253	int i;
2352	Serge	2254
2360	Serge	2255	dev_priv = container_of(work, drm_i915_private_t,
		2256	mm.retire_work.work);
		2257	dev = dev_priv->dev;
2352	Serge	2258
2360	Serge	2259	/* Come back later if the device is busy... */
		2260	if (!mutex_trylock(&dev->struct_mutex)) {
3482	Serge	2261	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2262	round_jiffies_up_relative(HZ));
3243	Serge	2263	return;
2360	Serge	2264	}
2352	Serge	2265
2360	Serge	2266	i915_gem_retire_requests(dev);
2352	Serge	2267
2360	Serge	2268	/* Send a periodic flush down the ring so we don't hold onto GEM
		2269	* objects indefinitely.
		2270	*/
		2271	idle = true;
3031	serge	2272	for_each_ring(ring, dev_priv, i) {
		2273	if (ring->gpu_caches_dirty)
4104	Serge	2274	i915_add_request(ring, NULL);
2352	Serge	2275
2360	Serge	2276	idle &= list_empty(&ring->request_list);
		2277	}
2352	Serge	2278
4104	Serge	2279	if (!dev_priv->ums.mm_suspended && !idle)
3482	Serge	2280	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2281	round_jiffies_up_relative(HZ));
3031	serge	2282	if (idle)
		2283	intel_mark_idle(dev);
2360	Serge	2284
		2285	mutex_unlock(&dev->struct_mutex);
		2286	}
		2287
2344	Serge	2288	/**
3031	serge	2289	* Ensures that an object will eventually get non-busy by flushing any required
		2290	* write domains, emitting any outstanding lazy request and retiring and
		2291	* completed requests.
2352	Serge	2292	*/
3031	serge	2293	static int
		2294	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2295	{
3031	serge	2296	int ret;
2352	Serge	2297
3031	serge	2298	if (obj->active) {
		2299	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2300	if (ret)
		2301	return ret;
2352	Serge	2302
3031	serge	2303	i915_gem_retire_requests_ring(obj->ring);
		2304	}
2352	Serge	2305
3031	serge	2306	return 0;
		2307	}
2352	Serge	2308
3243	Serge	2309	/**
		2310	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2311	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2312	*
		2313	* Returns 0 if successful, else an error is returned with the remaining time in
		2314	* the timeout parameter.
		2315	* -ETIME: object is still busy after timeout
		2316	* -ERESTARTSYS: signal interrupted the wait
		2317	* -ENONENT: object doesn't exist
		2318	* Also possible, but rare:
		2319	* -EAGAIN: GPU wedged
		2320	* -ENOMEM: damn
		2321	* -ENODEV: Internal IRQ fail
		2322	* -E?: The add request failed
		2323	*
		2324	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2325	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2326	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2327	* without holding struct_mutex the object may become re-busied before this
		2328	* function completes. A similar but shorter * race condition exists in the busy
		2329	* ioctl
		2330	*/
4246	Serge	2331	int
		2332	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2333	{
		2334	drm_i915_private_t *dev_priv = dev->dev_private;
		2335	struct drm_i915_gem_wait *args = data;
		2336	struct drm_i915_gem_object *obj;
		2337	struct intel_ring_buffer *ring = NULL;
		2338	struct timespec timeout_stack, *timeout = NULL;
		2339	unsigned reset_counter;
		2340	u32 seqno = 0;
		2341	int ret = 0;
2352	Serge	2342
4246	Serge	2343	if (args->timeout_ns >= 0) {
		2344	timeout_stack = ns_to_timespec(args->timeout_ns);
		2345	timeout = &timeout_stack;
		2346	}
2352	Serge	2347
4246	Serge	2348	ret = i915_mutex_lock_interruptible(dev);
		2349	if (ret)
		2350	return ret;
2352	Serge	2351
4246	Serge	2352	if(args->bo_handle == -2)
		2353	{
		2354	obj = get_fb_obj();
		2355	drm_gem_object_reference(&obj->base);
		2356	}
		2357	else
		2358	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2359	if (&obj->base == NULL) {
		2360	mutex_unlock(&dev->struct_mutex);
		2361	return -ENOENT;
		2362	}
2352	Serge	2363
4246	Serge	2364	/* Need to make sure the object gets inactive eventually. */
		2365	ret = i915_gem_object_flush_active(obj);
		2366	if (ret)
		2367	goto out;
2352	Serge	2368
4246	Serge	2369	if (obj->active) {
		2370	seqno = obj->last_read_seqno;
		2371	ring = obj->ring;
		2372	}
2352	Serge	2373
4246	Serge	2374	if (seqno == 0)
		2375	goto out;
2352	Serge	2376
4246	Serge	2377	/* Do this after OLR check to make sure we make forward progress polling
		2378	* on this IOCTL with a 0 timeout (like busy ioctl)
		2379	*/
		2380	if (!args->timeout_ns) {
		2381	ret = -ETIME;
		2382	goto out;
		2383	}
2352	Serge	2384
4246	Serge	2385	drm_gem_object_unreference(&obj->base);
		2386	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		2387	mutex_unlock(&dev->struct_mutex);
2352	Serge	2388
4246	Serge	2389	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
		2390	if (timeout)
		2391	args->timeout_ns = timespec_to_ns(timeout);
		2392	return ret;
3243	Serge	2393
4246	Serge	2394	out:
		2395	drm_gem_object_unreference(&obj->base);
		2396	mutex_unlock(&dev->struct_mutex);
		2397	return ret;
		2398	}
3243	Serge	2399
2352	Serge	2400	/**
3031	serge	2401	* i915_gem_object_sync - sync an object to a ring.
		2402	*
		2403	* @obj: object which may be in use on another ring.
		2404	* @to: ring we wish to use the object on. May be NULL.
		2405	*
		2406	* This code is meant to abstract object synchronization with the GPU.
		2407	* Calling with NULL implies synchronizing the object with the CPU
		2408	* rather than a particular GPU ring.
		2409	*
		2410	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2411	*/
		2412	int
3031	serge	2413	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2414	struct intel_ring_buffer *to)
2344	Serge	2415	{
3031	serge	2416	struct intel_ring_buffer *from = obj->ring;
		2417	u32 seqno;
		2418	int ret, idx;
2332	Serge	2419
3031	serge	2420	if (from == NULL \|\| to == from)
		2421	return 0;
2332	Serge	2422
3031	serge	2423	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2424	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2425
3031	serge	2426	idx = intel_ring_sync_index(from, to);
		2427
		2428	seqno = obj->last_read_seqno;
		2429	if (seqno <= from->sync_seqno[idx])
		2430	return 0;
		2431
		2432	ret = i915_gem_check_olr(obj->ring, seqno);
		2433	if (ret)
		2434	return ret;
		2435
		2436	ret = to->sync_to(to, from, seqno);
		2437	if (!ret)
3243	Serge	2438	/* We use last_read_seqno because sync_to()
		2439	* might have just caused seqno wrap under
		2440	* the radar.
		2441	*/
		2442	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2443
		2444	return ret;
2344	Serge	2445	}
2332	Serge	2446
2344	Serge	2447	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2448	{
		2449	u32 old_write_domain, old_read_domains;
2332	Serge	2450
2344	Serge	2451	/* Force a pagefault for domain tracking on next user access */
		2452	// i915_gem_release_mmap(obj);
2332	Serge	2453
2344	Serge	2454	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2455	return;
2332	Serge	2456
3480	Serge	2457	/* Wait for any direct GTT access to complete */
		2458	mb();
		2459
2344	Serge	2460	old_read_domains = obj->base.read_domains;
		2461	old_write_domain = obj->base.write_domain;
2351	Serge	2462
2344	Serge	2463	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2464	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2465
2351	Serge	2466	trace_i915_gem_object_change_domain(obj,
		2467	old_read_domains,
		2468	old_write_domain);
2344	Serge	2469	}
2332	Serge	2470
4104	Serge	2471	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2472	{
4104	Serge	2473	struct drm_i915_gem_object *obj = vma->obj;
3031	serge	2474	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2475	int ret;
2332	Serge	2476
3263	Serge	2477	if(obj == get_fb_obj())
		2478	return 0;
		2479
4104	Serge	2480	if (list_empty(&vma->vma_link))
2344	Serge	2481	return 0;
2332	Serge	2482
4104	Serge	2483	if (!drm_mm_node_allocated(&vma->node))
		2484	goto destroy;
		2485
3031	serge	2486	if (obj->pin_count)
		2487	return -EBUSY;
2332	Serge	2488
3243	Serge	2489	BUG_ON(obj->pages == NULL);
3031	serge	2490
2344	Serge	2491	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2492	if (ret)
2344	Serge	2493	return ret;
		2494	/* Continue on if we fail due to EIO, the GPU is hung so we
		2495	* should be safe and we need to cleanup or else we might
		2496	* cause memory corruption through use-after-free.
		2497	*/
2332	Serge	2498
2344	Serge	2499	i915_gem_object_finish_gtt(obj);
2332	Serge	2500
2344	Serge	2501	/* release the fence reg _after_ flushing */
		2502	ret = i915_gem_object_put_fence(obj);
3031	serge	2503	if (ret)
2344	Serge	2504	return ret;
2332	Serge	2505
4104	Serge	2506	trace_i915_vma_unbind(vma);
2332	Serge	2507
3031	serge	2508	if (obj->has_global_gtt_mapping)
3243	Serge	2509	i915_gem_gtt_unbind_object(obj);
3031	serge	2510	if (obj->has_aliasing_ppgtt_mapping) {
		2511	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2512	obj->has_aliasing_ppgtt_mapping = 0;
		2513	}
		2514	i915_gem_gtt_finish_object(obj);
4104	Serge	2515	i915_gem_object_unpin_pages(obj);
2332	Serge	2516
4104	Serge	2517	list_del(&vma->mm_list);
2344	Serge	2518	/* Avoid an unnecessary call to unbind on rebind. */
4104	Serge	2519	if (i915_is_ggtt(vma->vm))
2344	Serge	2520	obj->map_and_fenceable = true;
2332	Serge	2521
4104	Serge	2522	drm_mm_remove_node(&vma->node);
2332	Serge	2523
4104	Serge	2524	destroy:
		2525	i915_gem_vma_destroy(vma);
		2526
		2527	/* Since the unbound list is global, only move to that list if
		2528	* no more VMAs exist.
		2529	* NB: Until we have real VMAs there will only ever be one */
		2530	WARN_ON(!list_empty(&obj->vma_list));
		2531	if (list_empty(&obj->vma_list))
		2532	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2533
2344	Serge	2534	return 0;
		2535	}
2332	Serge	2536
4104	Serge	2537	/**
		2538	* Unbinds an object from the global GTT aperture.
		2539	*/
		2540	int
		2541	i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
		2542	{
		2543	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2544	struct i915_address_space *ggtt = &dev_priv->gtt.base;
		2545
		2546	if (!i915_gem_obj_ggtt_bound(obj))
		2547	return 0;
		2548
		2549	if (obj->pin_count)
		2550	return -EBUSY;
		2551
		2552	BUG_ON(obj->pages == NULL);
		2553
		2554	return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
		2555	}
		2556
3031	serge	2557	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2558	{
		2559	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2560	struct intel_ring_buffer *ring;
2344	Serge	2561	int ret, i;
2332	Serge	2562
2344	Serge	2563	/* Flush everything onto the inactive list. */
3031	serge	2564	for_each_ring(ring, dev_priv, i) {
		2565	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2566	if (ret)
		2567	return ret;
3031	serge	2568
3243	Serge	2569	ret = intel_ring_idle(ring);
3031	serge	2570	if (ret)
		2571	return ret;
2344	Serge	2572	}
2332	Serge	2573
2344	Serge	2574	return 0;
		2575	}
2332	Serge	2576
3480	Serge	2577	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2578	struct drm_i915_gem_object *obj)
		2579	{
		2580	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2581	int fence_reg;
		2582	int fence_pitch_shift;
2332	Serge	2583
3480	Serge	2584	if (INTEL_INFO(dev)->gen >= 6) {
		2585	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2586	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2587	} else {
		2588	fence_reg = FENCE_REG_965_0;
		2589	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2590	}
2332	Serge	2591
4104	Serge	2592	fence_reg += reg * 8;
		2593
		2594	/* To w/a incoherency with non-atomic 64-bit register updates,
		2595	* we split the 64-bit update into two 32-bit writes. In order
		2596	* for a partial fence not to be evaluated between writes, we
		2597	* precede the update with write to turn off the fence register,
		2598	* and only enable the fence as the last step.
		2599	*
		2600	* For extra levels of paranoia, we make sure each step lands
		2601	* before applying the next step.
		2602	*/
		2603	I915_WRITE(fence_reg, 0);
		2604	POSTING_READ(fence_reg);
		2605
3031	serge	2606	if (obj) {
4104	Serge	2607	u32 size = i915_gem_obj_ggtt_size(obj);
		2608	uint64_t val;
2332	Serge	2609
4104	Serge	2610	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2611	0xfffff000) << 32;
4104	Serge	2612	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2613	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2614	if (obj->tiling_mode == I915_TILING_Y)
		2615	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2616	val \|= I965_FENCE_REG_VALID;
2332	Serge	2617
4104	Serge	2618	I915_WRITE(fence_reg + 4, val >> 32);
		2619	POSTING_READ(fence_reg + 4);
		2620
		2621	I915_WRITE(fence_reg + 0, val);
3480	Serge	2622	POSTING_READ(fence_reg);
4104	Serge	2623	} else {
		2624	I915_WRITE(fence_reg + 4, 0);
		2625	POSTING_READ(fence_reg + 4);
		2626	}
3031	serge	2627	}
2332	Serge	2628
3031	serge	2629	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2630	struct drm_i915_gem_object *obj)
		2631	{
		2632	drm_i915_private_t *dev_priv = dev->dev_private;
		2633	u32 val;
2332	Serge	2634
3031	serge	2635	if (obj) {
4104	Serge	2636	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2637	int pitch_val;
		2638	int tile_width;
2332	Serge	2639
4104	Serge	2640	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2641	(size & -size) != size \|\|
4104	Serge	2642	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2643	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2644	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2645
3031	serge	2646	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2647	tile_width = 128;
		2648	else
		2649	tile_width = 512;
2332	Serge	2650
3031	serge	2651	/* Note: pitch better be a power of two tile widths */
		2652	pitch_val = obj->stride / tile_width;
		2653	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2654
4104	Serge	2655	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2656	if (obj->tiling_mode == I915_TILING_Y)
		2657	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2658	val \|= I915_FENCE_SIZE_BITS(size);
		2659	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2660	val \|= I830_FENCE_REG_VALID;
		2661	} else
		2662	val = 0;
2332	Serge	2663
3031	serge	2664	if (reg < 8)
		2665	reg = FENCE_REG_830_0 + reg * 4;
		2666	else
		2667	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2668
3031	serge	2669	I915_WRITE(reg, val);
		2670	POSTING_READ(reg);
		2671	}
2332	Serge	2672
3031	serge	2673	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2674	struct drm_i915_gem_object *obj)
		2675	{
		2676	drm_i915_private_t *dev_priv = dev->dev_private;
		2677	uint32_t val;
2344	Serge	2678
3031	serge	2679	if (obj) {
4104	Serge	2680	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2681	uint32_t pitch_val;
2344	Serge	2682
4104	Serge	2683	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2684	(size & -size) != size \|\|
4104	Serge	2685	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2686	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2687	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2688
3031	serge	2689	pitch_val = obj->stride / 128;
		2690	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2691
4104	Serge	2692	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2693	if (obj->tiling_mode == I915_TILING_Y)
		2694	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2695	val \|= I830_FENCE_SIZE_BITS(size);
		2696	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2697	val \|= I830_FENCE_REG_VALID;
		2698	} else
		2699	val = 0;
		2700
		2701	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2702	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2703	}
		2704
3480	Serge	2705	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2706	{
		2707	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2708	}
		2709
3031	serge	2710	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2711	struct drm_i915_gem_object *obj)
2332	Serge	2712	{
3480	Serge	2713	struct drm_i915_private *dev_priv = dev->dev_private;
		2714
		2715	/* Ensure that all CPU reads are completed before installing a fence
		2716	* and all writes before removing the fence.
		2717	*/
		2718	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2719	mb();
		2720
4104	Serge	2721	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2722	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2723	obj->stride, obj->tiling_mode);
		2724
3031	serge	2725	switch (INTEL_INFO(dev)->gen) {
		2726	case 7:
3480	Serge	2727	case 6:
3031	serge	2728	case 5:
		2729	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2730	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2731	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2732	default: BUG();
3031	serge	2733	}
3480	Serge	2734
		2735	/* And similarly be paranoid that no direct access to this region
		2736	* is reordered to before the fence is installed.
		2737	*/
		2738	if (i915_gem_object_needs_mb(obj))
		2739	mb();
2344	Serge	2740	}
		2741
3031	serge	2742	static inline int fence_number(struct drm_i915_private *dev_priv,
		2743	struct drm_i915_fence_reg *fence)
2344	Serge	2744	{
3031	serge	2745	return fence - dev_priv->fence_regs;
		2746	}
2332	Serge	2747
3031	serge	2748	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2749	struct drm_i915_fence_reg *fence,
		2750	bool enable)
		2751	{
4104	Serge	2752	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2753	int reg = fence_number(dev_priv, fence);
2332	Serge	2754
4104	Serge	2755	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2756
		2757	if (enable) {
4104	Serge	2758	obj->fence_reg = reg;
3031	serge	2759	fence->obj = obj;
		2760	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2761	} else {
		2762	obj->fence_reg = I915_FENCE_REG_NONE;
		2763	fence->obj = NULL;
		2764	list_del_init(&fence->lru_list);
2344	Serge	2765	}
4104	Serge	2766	obj->fence_dirty = false;
3031	serge	2767	}
2344	Serge	2768
3031	serge	2769	static int
3480	Serge	2770	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2771	{
		2772	if (obj->last_fenced_seqno) {
		2773	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2774	if (ret)
		2775	return ret;
2344	Serge	2776
		2777	obj->last_fenced_seqno = 0;
		2778	}
		2779
3031	serge	2780	obj->fenced_gpu_access = false;
2332	Serge	2781	return 0;
		2782	}
		2783
		2784	int
2344	Serge	2785	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2786	{
3031	serge	2787	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2788	struct drm_i915_fence_reg *fence;
2332	Serge	2789	int ret;
		2790
3480	Serge	2791	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2792	if (ret)
		2793	return ret;
		2794
3031	serge	2795	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2796	return 0;
2332	Serge	2797
3746	Serge	2798	fence = &dev_priv->fence_regs[obj->fence_reg];
		2799
3031	serge	2800	i915_gem_object_fence_lost(obj);
3746	Serge	2801	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2802
2332	Serge	2803	return 0;
		2804	}
		2805
3031	serge	2806	static struct drm_i915_fence_reg *
		2807	i915_find_fence_reg(struct drm_device *dev)
		2808	{
		2809	struct drm_i915_private *dev_priv = dev->dev_private;
		2810	struct drm_i915_fence_reg reg, avail;
		2811	int i;
2332	Serge	2812
3031	serge	2813	/* First try to find a free reg */
		2814	avail = NULL;
		2815	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2816	reg = &dev_priv->fence_regs[i];
		2817	if (!reg->obj)
		2818	return reg;
2332	Serge	2819
3031	serge	2820	if (!reg->pin_count)
		2821	avail = reg;
		2822	}
2332	Serge	2823
3031	serge	2824	if (avail == NULL)
		2825	return NULL;
2332	Serge	2826
3031	serge	2827	/* None available, try to steal one or wait for a user to finish */
		2828	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2829	if (reg->pin_count)
		2830	continue;
2332	Serge	2831
3031	serge	2832	return reg;
		2833	}
2332	Serge	2834
3031	serge	2835	return NULL;
		2836	}
2332	Serge	2837
3031	serge	2838	/**
		2839	* i915_gem_object_get_fence - set up fencing for an object
		2840	* @obj: object to map through a fence reg
		2841	*
		2842	* When mapping objects through the GTT, userspace wants to be able to write
		2843	* to them without having to worry about swizzling if the object is tiled.
		2844	* This function walks the fence regs looking for a free one for @obj,
		2845	* stealing one if it can't find any.
		2846	*
		2847	* It then sets up the reg based on the object's properties: address, pitch
		2848	* and tiling format.
		2849	*
		2850	* For an untiled surface, this removes any existing fence.
		2851	*/
		2852	int
		2853	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2854	{
		2855	struct drm_device *dev = obj->base.dev;
		2856	struct drm_i915_private *dev_priv = dev->dev_private;
		2857	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2858	struct drm_i915_fence_reg *reg;
		2859	int ret;
2332	Serge	2860
3031	serge	2861	/* Have we updated the tiling parameters upon the object and so
		2862	* will need to serialise the write to the associated fence register?
		2863	*/
		2864	if (obj->fence_dirty) {
3480	Serge	2865	ret = i915_gem_object_wait_fence(obj);
3031	serge	2866	if (ret)
		2867	return ret;
		2868	}
2332	Serge	2869
3031	serge	2870	/* Just update our place in the LRU if our fence is getting reused. */
		2871	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2872	reg = &dev_priv->fence_regs[obj->fence_reg];
		2873	if (!obj->fence_dirty) {
		2874	list_move_tail(®->lru_list,
		2875	&dev_priv->mm.fence_list);
		2876	return 0;
		2877	}
		2878	} else if (enable) {
		2879	reg = i915_find_fence_reg(dev);
		2880	if (reg == NULL)
		2881	return -EDEADLK;
2332	Serge	2882
3031	serge	2883	if (reg->obj) {
		2884	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2885
3480	Serge	2886	ret = i915_gem_object_wait_fence(old);
3031	serge	2887	if (ret)
		2888	return ret;
2332	Serge	2889
3031	serge	2890	i915_gem_object_fence_lost(old);
		2891	}
		2892	} else
		2893	return 0;
2332	Serge	2894
3031	serge	2895	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2896
3031	serge	2897	return 0;
		2898	}
2332	Serge	2899
3031	serge	2900	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2901	struct drm_mm_node *gtt_space,
		2902	unsigned long cache_level)
		2903	{
		2904	struct drm_mm_node *other;
2332	Serge	2905
3031	serge	2906	/* On non-LLC machines we have to be careful when putting differing
		2907	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2908	* crossing memory domains and dying.
3031	serge	2909	*/
		2910	if (HAS_LLC(dev))
		2911	return true;
2332	Serge	2912
4104	Serge	2913	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2914	return true;
2332	Serge	2915
3031	serge	2916	if (list_empty(>t_space->node_list))
		2917	return true;
2332	Serge	2918
3031	serge	2919	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2920	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2921	return false;
2344	Serge	2922
3031	serge	2923	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2924	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2925	return false;
2344	Serge	2926
3031	serge	2927	return true;
		2928	}
2344	Serge	2929
3031	serge	2930	static void i915_gem_verify_gtt(struct drm_device *dev)
		2931	{
		2932	#if WATCH_GTT
		2933	struct drm_i915_private *dev_priv = dev->dev_private;
		2934	struct drm_i915_gem_object *obj;
		2935	int err = 0;
2344	Serge	2936
4104	Serge	2937	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031	serge	2938	if (obj->gtt_space == NULL) {
		2939	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2940	err++;
		2941	continue;
		2942	}
2344	Serge	2943
3031	serge	2944	if (obj->cache_level != obj->gtt_space->color) {
		2945	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
4104	Serge	2946	i915_gem_obj_ggtt_offset(obj),
		2947	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2948	obj->cache_level,
		2949	obj->gtt_space->color);
		2950	err++;
		2951	continue;
		2952	}
2344	Serge	2953
3031	serge	2954	if (!i915_gem_valid_gtt_space(dev,
		2955	obj->gtt_space,
		2956	obj->cache_level)) {
		2957	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
4104	Serge	2958	i915_gem_obj_ggtt_offset(obj),
		2959	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2960	obj->cache_level);
		2961	err++;
		2962	continue;
		2963	}
		2964	}
2344	Serge	2965
3031	serge	2966	WARN_ON(err);
		2967	#endif
2326	Serge	2968	}
		2969
2332	Serge	2970	/**
		2971	* Finds free space in the GTT aperture and binds the object there.
		2972	*/
		2973	static int
4104	Serge	2974	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2975	struct i915_address_space *vm,
2332	Serge	2976	unsigned alignment,
3031	serge	2977	bool map_and_fenceable,
		2978	bool nonblocking)
2332	Serge	2979	{
		2980	struct drm_device *dev = obj->base.dev;
		2981	drm_i915_private_t *dev_priv = dev->dev_private;
		2982	u32 size, fence_size, fence_alignment, unfenced_alignment;
4104	Serge	2983	size_t gtt_max =
		2984	map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
		2985	struct i915_vma *vma;
2332	Serge	2986	int ret;
2326	Serge	2987
2332	Serge	2988	fence_size = i915_gem_get_gtt_size(dev,
		2989	obj->base.size,
		2990	obj->tiling_mode);
		2991	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2992	obj->base.size,
3480	Serge	2993	obj->tiling_mode, true);
2332	Serge	2994	unfenced_alignment =
3480	Serge	2995	i915_gem_get_gtt_alignment(dev,
2332	Serge	2996	obj->base.size,
3480	Serge	2997	obj->tiling_mode, false);
2332	Serge	2998
		2999	if (alignment == 0)
		3000	alignment = map_and_fenceable ? fence_alignment :
		3001	unfenced_alignment;
		3002	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		3003	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		3004	return -EINVAL;
		3005	}
		3006
		3007	size = map_and_fenceable ? fence_size : obj->base.size;
		3008
		3009	/* If the object is bigger than the entire aperture, reject it early
		3010	* before evicting everything in a vain attempt to find space.
		3011	*/
4104	Serge	3012	if (obj->base.size > gtt_max) {
		3013	DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
		3014	obj->base.size,
		3015	map_and_fenceable ? "mappable" : "total",
		3016	gtt_max);
2332	Serge	3017	return -E2BIG;
		3018	}
		3019
3031	serge	3020	ret = i915_gem_object_get_pages(obj);
		3021	if (ret)
		3022	return ret;
		3023
3243	Serge	3024	i915_gem_object_pin_pages(obj);
		3025
4104	Serge	3026	BUG_ON(!i915_is_ggtt(vm));
		3027
		3028	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
		3029	if (IS_ERR(vma)) {
		3030	ret = PTR_ERR(vma);
		3031	goto err_unpin;
3243	Serge	3032	}
		3033
4104	Serge	3034	/* For now we only ever use 1 vma per object */
		3035	WARN_ON(!list_is_singular(&obj->vma_list));
		3036
		3037	search_free:
		3038	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3039	size, alignment,
		3040	obj->cache_level, 0, gtt_max,
		3041	DRM_MM_SEARCH_DEFAULT);
3243	Serge	3042	if (ret) {
2332	Serge	3043
4104	Serge	3044	goto err_free_vma;
2332	Serge	3045	}
4104	Serge	3046	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
		3047	obj->cache_level))) {
		3048	ret = -EINVAL;
		3049	goto err_remove_node;
3031	serge	3050	}
2332	Serge	3051
3031	serge	3052	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	3053	if (ret)
		3054	goto err_remove_node;
2332	Serge	3055
4104	Serge	3056	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3057	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3058
4104	Serge	3059	if (i915_is_ggtt(vm)) {
		3060	bool mappable, fenceable;
2332	Serge	3061
4104	Serge	3062	fenceable = (vma->node.size == fence_size &&
		3063	(vma->node.start & (fence_alignment - 1)) == 0);
2332	Serge	3064
4104	Serge	3065	mappable = (vma->node.start + obj->base.size <=
		3066	dev_priv->gtt.mappable_end);
2332	Serge	3067
		3068	obj->map_and_fenceable = mappable && fenceable;
4104	Serge	3069	}
2332	Serge	3070
4104	Serge	3071	WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
		3072
		3073	trace_i915_vma_bind(vma, map_and_fenceable);
3031	serge	3074	i915_gem_verify_gtt(dev);
2332	Serge	3075	return 0;
4104	Serge	3076
		3077	err_remove_node:
		3078	drm_mm_remove_node(&vma->node);
		3079	err_free_vma:
		3080	i915_gem_vma_destroy(vma);
		3081	err_unpin:
		3082	i915_gem_object_unpin_pages(obj);
		3083	return ret;
2332	Serge	3084	}
		3085
4104	Serge	3086	bool
		3087	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3088	bool force)
2332	Serge	3089	{
		3090	/* If we don't have a page list set up, then we're not pinned
		3091	* to GPU, and we can ignore the cache flush because it'll happen
		3092	* again at bind time.
		3093	*/
3243	Serge	3094	if (obj->pages == NULL)
4104	Serge	3095	return false;
2332	Serge	3096
3480	Serge	3097	/*
		3098	* Stolen memory is always coherent with the GPU as it is explicitly
		3099	* marked as wc by the system, or the system is cache-coherent.
		3100	*/
		3101	if (obj->stolen)
4104	Serge	3102	return false;
3480	Serge	3103
2332	Serge	3104	/* If the GPU is snooping the contents of the CPU cache,
		3105	* we do not need to manually clear the CPU cache lines. However,
		3106	* the caches are only snooped when the render cache is
		3107	* flushed/invalidated. As we always have to emit invalidations
		3108	* and flushes when moving into and out of the RENDER domain, correct
		3109	* snooping behaviour occurs naturally as the result of our domain
		3110	* tracking.
		3111	*/
4104	Serge	3112	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3113	return false;
3243	Serge	3114	#if 0
2344	Serge	3115	if(obj->mapped != NULL)
		3116	{
		3117	uint8_t *page_virtual;
		3118	unsigned int i;
2332	Serge	3119
2344	Serge	3120	page_virtual = obj->mapped;
		3121	asm volatile("mfence");
		3122	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		3123	clflush(page_virtual + i);
		3124	asm volatile("mfence");
		3125	}
		3126	else
		3127	{
		3128	uint8_t *page_virtual;
		3129	unsigned int i;
		3130	page_virtual = AllocKernelSpace(obj->base.size);
		3131	if(page_virtual != NULL)
		3132	{
3031	serge	3133	dma_addr_t src, dst;
2344	Serge	3134	u32 count;
		3135
		3136	#define page_tabs 0xFDC00000 /* really dirty hack */
		3137
3031	serge	3138	src = obj->pages.page;
		3139	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	3140	count = obj->base.size/4096;
		3141
		3142	while(count--)
		3143	{
		3144	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		3145	};
		3146
		3147	asm volatile("mfence");
		3148	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		3149	clflush(page_virtual + i);
		3150	asm volatile("mfence");
		3151	FreeKernelSpace(page_virtual);
		3152	}
		3153	else
		3154	{
		3155	asm volatile (
		3156	"mfence \n"
		3157	"wbinvd \n" /* this is really ugly */
		3158	"mfence");
		3159	}
		3160	}
3243	Serge	3161	#endif
		3162
4104	Serge	3163	return true;
2332	Serge	3164	}
		3165
2344	Serge	3166	/** Flushes the GTT write domain for the object if it's dirty. */
		3167	static void
		3168	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3169	{
		3170	uint32_t old_write_domain;
2332	Serge	3171
2344	Serge	3172	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3173	return;
2332	Serge	3174
2344	Serge	3175	/* No actual flushing is required for the GTT write domain. Writes
		3176	* to it immediately go to main memory as far as we know, so there's
		3177	* no chipset flush. It also doesn't land in render cache.
		3178	*
		3179	* However, we do have to enforce the order so that all writes through
		3180	* the GTT land before any writes to the device, such as updates to
		3181	* the GATT itself.
		3182	*/
		3183	wmb();
2332	Serge	3184
2344	Serge	3185	old_write_domain = obj->base.write_domain;
		3186	obj->base.write_domain = 0;
2332	Serge	3187
2351	Serge	3188	trace_i915_gem_object_change_domain(obj,
		3189	obj->base.read_domains,
		3190	old_write_domain);
2344	Serge	3191	}
2332	Serge	3192
		3193	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3194	static void
4104	Serge	3195	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3196	bool force)
2332	Serge	3197	{
		3198	uint32_t old_write_domain;
		3199
		3200	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3201	return;
		3202
4104	Serge	3203	if (i915_gem_clflush_object(obj, force))
3243	Serge	3204	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3205
2332	Serge	3206	old_write_domain = obj->base.write_domain;
		3207	obj->base.write_domain = 0;
		3208
2351	Serge	3209	trace_i915_gem_object_change_domain(obj,
		3210	obj->base.read_domains,
		3211	old_write_domain);
2332	Serge	3212	}
		3213
		3214	/**
		3215	* Moves a single object to the GTT read, and possibly write domain.
		3216	*
		3217	* This function returns when the move is complete, including waiting on
		3218	* flushes to occur.
		3219	*/
		3220	int
		3221	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3222	{
3031	serge	3223	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	3224	uint32_t old_write_domain, old_read_domains;
		3225	int ret;
		3226
		3227	/* Not valid to be called on unbound objects. */
4104	Serge	3228	if (!i915_gem_obj_bound_any(obj))
2332	Serge	3229	return -EINVAL;
		3230
		3231	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3232	return 0;
		3233
3031	serge	3234	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3235	if (ret)
		3236	return ret;
		3237
4104	Serge	3238	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3239
3480	Serge	3240	/* Serialise direct access to this object with the barriers for
		3241	* coherent writes from the GPU, by effectively invalidating the
		3242	* GTT domain upon first access.
		3243	*/
		3244	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3245	mb();
		3246
2332	Serge	3247	old_write_domain = obj->base.write_domain;
		3248	old_read_domains = obj->base.read_domains;
		3249
		3250	/* It should now be out of any other write domains, and we can update
		3251	* the domain values for our changes.
		3252	*/
		3253	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3254	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3255	if (write) {
		3256	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3257	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3258	obj->dirty = 1;
		3259	}
		3260
2351	Serge	3261	trace_i915_gem_object_change_domain(obj,
		3262	old_read_domains,
		3263	old_write_domain);
		3264
3031	serge	3265	/* And bump the LRU for this access */
4104	Serge	3266	if (i915_gem_object_is_inactive(obj)) {
		3267	struct i915_vma *vma = i915_gem_obj_to_vma(obj,
		3268	&dev_priv->gtt.base);
		3269	if (vma)
		3270	list_move_tail(&vma->mm_list,
		3271	&dev_priv->gtt.base.inactive_list);
3031	serge	3272
4104	Serge	3273	}
		3274
2332	Serge	3275	return 0;
		3276	}
		3277
2335	Serge	3278	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3279	enum i915_cache_level cache_level)
		3280	{
3031	serge	3281	struct drm_device *dev = obj->base.dev;
		3282	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3283	struct i915_vma *vma;
2335	Serge	3284	int ret;
2332	Serge	3285
2335	Serge	3286	if (obj->cache_level == cache_level)
		3287	return 0;
2332	Serge	3288
2335	Serge	3289	if (obj->pin_count) {
		3290	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3291	return -EBUSY;
		3292	}
2332	Serge	3293
4104	Serge	3294	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3295	if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
		3296	ret = i915_vma_unbind(vma);
3031	serge	3297	if (ret)
		3298	return ret;
4104	Serge	3299
		3300	break;
		3301	}
3031	serge	3302	}
		3303
4104	Serge	3304	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3305	ret = i915_gem_object_finish_gpu(obj);
		3306	if (ret)
		3307	return ret;
2332	Serge	3308
2335	Serge	3309	i915_gem_object_finish_gtt(obj);
2332	Serge	3310
2335	Serge	3311	/* Before SandyBridge, you could not use tiling or fence
		3312	* registers with snooped memory, so relinquish any fences
		3313	* currently pointing to our region in the aperture.
		3314	*/
3031	serge	3315	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3316	ret = i915_gem_object_put_fence(obj);
		3317	if (ret)
		3318	return ret;
		3319	}
2332	Serge	3320
3031	serge	3321	if (obj->has_global_gtt_mapping)
		3322	i915_gem_gtt_bind_object(obj, cache_level);
		3323	if (obj->has_aliasing_ppgtt_mapping)
		3324	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3325	obj, cache_level);
2335	Serge	3326	}
2332	Serge	3327
4104	Serge	3328	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3329	vma->node.color = cache_level;
		3330	obj->cache_level = cache_level;
		3331
		3332	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3333	u32 old_read_domains, old_write_domain;
2332	Serge	3334
2335	Serge	3335	/* If we're coming from LLC cached, then we haven't
		3336	* actually been tracking whether the data is in the
		3337	* CPU cache or not, since we only allow one bit set
		3338	* in obj->write_domain and have been skipping the clflushes.
		3339	* Just set it to the CPU cache for now.
		3340	*/
		3341	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3342
2335	Serge	3343	old_read_domains = obj->base.read_domains;
		3344	old_write_domain = obj->base.write_domain;
2332	Serge	3345
2335	Serge	3346	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3347	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3348
2351	Serge	3349	trace_i915_gem_object_change_domain(obj,
		3350	old_read_domains,
		3351	old_write_domain);
2344	Serge	3352	}
2332	Serge	3353
3031	serge	3354	i915_gem_verify_gtt(dev);
2335	Serge	3355	return 0;
		3356	}
2332	Serge	3357
3260	Serge	3358	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3359	struct drm_file *file)
		3360	{
		3361	struct drm_i915_gem_caching *args = data;
		3362	struct drm_i915_gem_object *obj;
		3363	int ret;
		3364
3480	Serge	3365	if(args->handle == -2)
		3366	{
		3367	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3368	return 0;
		3369	}
		3370
3260	Serge	3371	ret = i915_mutex_lock_interruptible(dev);
		3372	if (ret)
		3373	return ret;
		3374
		3375	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3376	if (&obj->base == NULL) {
		3377	ret = -ENOENT;
		3378	goto unlock;
		3379	}
		3380
4104	Serge	3381	switch (obj->cache_level) {
		3382	case I915_CACHE_LLC:
		3383	case I915_CACHE_L3_LLC:
		3384	args->caching = I915_CACHING_CACHED;
		3385	break;
3260	Serge	3386
4104	Serge	3387	case I915_CACHE_WT:
		3388	args->caching = I915_CACHING_DISPLAY;
		3389	break;
		3390
		3391	default:
		3392	args->caching = I915_CACHING_NONE;
		3393	break;
		3394	}
		3395
3260	Serge	3396	drm_gem_object_unreference(&obj->base);
		3397	unlock:
		3398	mutex_unlock(&dev->struct_mutex);
		3399	return ret;
		3400	}
		3401
		3402	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3403	struct drm_file *file)
		3404	{
		3405	struct drm_i915_gem_caching *args = data;
		3406	struct drm_i915_gem_object *obj;
		3407	enum i915_cache_level level;
		3408	int ret;
		3409
3480	Serge	3410	if(args->handle == -2)
		3411	{
		3412	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3413	return 0;
		3414	}
		3415
3260	Serge	3416	switch (args->caching) {
		3417	case I915_CACHING_NONE:
		3418	level = I915_CACHE_NONE;
		3419	break;
		3420	case I915_CACHING_CACHED:
		3421	level = I915_CACHE_LLC;
		3422	break;
4104	Serge	3423	case I915_CACHING_DISPLAY:
		3424	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3425	break;
3260	Serge	3426	default:
		3427	return -EINVAL;
		3428	}
		3429
		3430	ret = i915_mutex_lock_interruptible(dev);
		3431	if (ret)
		3432	return ret;
		3433
		3434	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3435	if (&obj->base == NULL) {
		3436	ret = -ENOENT;
		3437	goto unlock;
		3438	}
		3439
		3440	ret = i915_gem_object_set_cache_level(obj, level);
		3441
		3442	drm_gem_object_unreference(&obj->base);
		3443	unlock:
		3444	mutex_unlock(&dev->struct_mutex);
		3445	return ret;
		3446	}
		3447
4104	Serge	3448	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3449	{
		3450	/* There are 3 sources that pin objects:
		3451	* 1. The display engine (scanouts, sprites, cursors);
		3452	* 2. Reservations for execbuffer;
		3453	* 3. The user.
		3454	*
		3455	* We can ignore reservations as we hold the struct_mutex and
		3456	* are only called outside of the reservation path. The user
		3457	* can only increment pin_count once, and so if after
		3458	* subtracting the potential reference by the user, any pin_count
		3459	* remains, it must be due to another use by the display engine.
		3460	*/
		3461	return obj->pin_count - !!obj->user_pin_count;
		3462	}
		3463
2335	Serge	3464	/*
		3465	* Prepare buffer for display plane (scanout, cursors, etc).
		3466	* Can be called from an uninterruptible phase (modesetting) and allows
		3467	* any flushes to be pipelined (for pageflips).
		3468	*/
		3469	int
		3470	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3471	u32 alignment,
		3472	struct intel_ring_buffer *pipelined)
		3473	{
		3474	u32 old_read_domains, old_write_domain;
		3475	int ret;
2332	Serge	3476
3031	serge	3477	if (pipelined != obj->ring) {
		3478	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3479	if (ret)
		3480	return ret;
		3481	}
2332	Serge	3482
4104	Serge	3483	/* Mark the pin_display early so that we account for the
		3484	* display coherency whilst setting up the cache domains.
		3485	*/
		3486	obj->pin_display = true;
		3487
2335	Serge	3488	/* The display engine is not coherent with the LLC cache on gen6. As
		3489	* a result, we make sure that the pinning that is about to occur is
		3490	* done with uncached PTEs. This is lowest common denominator for all
		3491	* chipsets.
		3492	*
		3493	* However for gen6+, we could do better by using the GFDT bit instead
		3494	* of uncaching, which would allow us to flush all the LLC-cached data
		3495	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3496	*/
4104	Serge	3497	ret = i915_gem_object_set_cache_level(obj,
		3498	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3499	if (ret)
4104	Serge	3500	goto err_unpin_display;
2332	Serge	3501
2335	Serge	3502	/* As the user may map the buffer once pinned in the display plane
		3503	* (e.g. libkms for the bootup splash), we have to ensure that we
		3504	* always use map_and_fenceable for all scanout buffers.
		3505	*/
4104	Serge	3506	ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
2335	Serge	3507	if (ret)
4104	Serge	3508	goto err_unpin_display;
2332	Serge	3509
4104	Serge	3510	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3511
2335	Serge	3512	old_write_domain = obj->base.write_domain;
		3513	old_read_domains = obj->base.read_domains;
2332	Serge	3514
2335	Serge	3515	/* It should now be out of any other write domains, and we can update
		3516	* the domain values for our changes.
		3517	*/
3031	serge	3518	obj->base.write_domain = 0;
2335	Serge	3519	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3520
2351	Serge	3521	trace_i915_gem_object_change_domain(obj,
		3522	old_read_domains,
		3523	old_write_domain);
2332	Serge	3524
2335	Serge	3525	return 0;
4104	Serge	3526
		3527	err_unpin_display:
		3528	obj->pin_display = is_pin_display(obj);
		3529	return ret;
2335	Serge	3530	}
2332	Serge	3531
4104	Serge	3532	void
		3533	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3534	{
		3535	i915_gem_object_unpin(obj);
		3536	obj->pin_display = is_pin_display(obj);
		3537	}
		3538
2344	Serge	3539	int
		3540	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3541	{
		3542	int ret;
2332	Serge	3543
2344	Serge	3544	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3545	return 0;
2332	Serge	3546
3031	serge	3547	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3548	if (ret)
		3549	return ret;
2332	Serge	3550
2344	Serge	3551	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3552	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3553	return 0;
2344	Serge	3554	}
2332	Serge	3555
2344	Serge	3556	/**
		3557	* Moves a single object to the CPU read, and possibly write domain.
		3558	*
		3559	* This function returns when the move is complete, including waiting on
		3560	* flushes to occur.
		3561	*/
3031	serge	3562	int
2344	Serge	3563	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3564	{
		3565	uint32_t old_write_domain, old_read_domains;
		3566	int ret;
2332	Serge	3567
2344	Serge	3568	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3569	return 0;
2332	Serge	3570
3031	serge	3571	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3572	if (ret)
		3573	return ret;
2332	Serge	3574
2344	Serge	3575	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3576
2344	Serge	3577	old_write_domain = obj->base.write_domain;
		3578	old_read_domains = obj->base.read_domains;
2332	Serge	3579
2344	Serge	3580	/* Flush the CPU cache if it's still invalid. */
		3581	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3582	i915_gem_clflush_object(obj, false);
2332	Serge	3583
2344	Serge	3584	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3585	}
2332	Serge	3586
2344	Serge	3587	/* It should now be out of any other write domains, and we can update
		3588	* the domain values for our changes.
		3589	*/
		3590	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3591
2344	Serge	3592	/* If we're writing through the CPU, then the GPU read domains will
		3593	* need to be invalidated at next use.
		3594	*/
		3595	if (write) {
		3596	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3597	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3598	}
2332	Serge	3599
2351	Serge	3600	trace_i915_gem_object_change_domain(obj,
		3601	old_read_domains,
		3602	old_write_domain);
2332	Serge	3603
2344	Serge	3604	return 0;
		3605	}
2332	Serge	3606
3031	serge	3607	/* Throttle our rendering by waiting until the ring has completed our requests
		3608	* emitted over 20 msec ago.
2344	Serge	3609	*
3031	serge	3610	* Note that if we were to use the current jiffies each time around the loop,
		3611	* we wouldn't escape the function with any frames outstanding if the time to
		3612	* render a frame was over 20ms.
		3613	*
		3614	* This should get us reasonable parallelism between CPU and GPU but also
		3615	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3616	*/
3031	serge	3617	static int
		3618	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3619	{
3031	serge	3620	struct drm_i915_private *dev_priv = dev->dev_private;
		3621	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3622	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3623	struct drm_i915_gem_request *request;
		3624	struct intel_ring_buffer *ring = NULL;
3480	Serge	3625	unsigned reset_counter;
3031	serge	3626	u32 seqno = 0;
		3627	int ret;
2332	Serge	3628
3480	Serge	3629	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3630	if (ret)
		3631	return ret;
2332	Serge	3632
3480	Serge	3633	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3634	if (ret)
		3635	return ret;
		3636
3031	serge	3637	spin_lock(&file_priv->mm.lock);
		3638	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3639	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3640	break;
2332	Serge	3641
3031	serge	3642	ring = request->ring;
		3643	seqno = request->seqno;
		3644	}
3480	Serge	3645	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3646	spin_unlock(&file_priv->mm.lock);
2332	Serge	3647
3031	serge	3648	if (seqno == 0)
		3649	return 0;
2332	Serge	3650
3480	Serge	3651	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3652	if (ret == 0)
		3653	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3654
3031	serge	3655	return ret;
2352	Serge	3656	}
2332	Serge	3657
		3658	int
		3659	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3660	struct i915_address_space *vm,
2332	Serge	3661	uint32_t alignment,
3031	serge	3662	bool map_and_fenceable,
		3663	bool nonblocking)
2332	Serge	3664	{
4104	Serge	3665	struct i915_vma *vma;
2332	Serge	3666	int ret;
		3667
3031	serge	3668	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3669	return -EBUSY;
2332	Serge	3670
4246	Serge	3671	// if( obj == get_fb_obj())
		3672	// return 0;
		3673
4104	Serge	3674	WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
		3675
		3676	vma = i915_gem_obj_to_vma(obj, vm);
		3677
		3678	if (vma) {
		3679	if ((alignment &&
		3680	vma->node.start & (alignment - 1)) \|\|
2332	Serge	3681	(map_and_fenceable && !obj->map_and_fenceable)) {
		3682	WARN(obj->pin_count,
		3683	"bo is already pinned with incorrect alignment:"
4104	Serge	3684	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3685	" obj->map_and_fenceable=%d\n",
4104	Serge	3686	i915_gem_obj_offset(obj, vm), alignment,
2332	Serge	3687	map_and_fenceable,
		3688	obj->map_and_fenceable);
4104	Serge	3689	ret = i915_vma_unbind(vma);
2332	Serge	3690	if (ret)
		3691	return ret;
		3692	}
		3693	}
		3694
4104	Serge	3695	if (!i915_gem_obj_bound(obj, vm)) {
3243	Serge	3696	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3697
4104	Serge	3698	ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3031	serge	3699	map_and_fenceable,
		3700	nonblocking);
2332	Serge	3701	if (ret)
		3702	return ret;
3243	Serge	3703
		3704	if (!dev_priv->mm.aliasing_ppgtt)
		3705	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3706	}
		3707
3031	serge	3708	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3709	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3710
		3711	obj->pin_count++;
2332	Serge	3712	obj->pin_mappable \|= map_and_fenceable;
		3713
		3714	return 0;
		3715	}
		3716
2344	Serge	3717	void
		3718	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3719	{
		3720	BUG_ON(obj->pin_count == 0);
4104	Serge	3721	BUG_ON(!i915_gem_obj_bound_any(obj));
2332	Serge	3722
3031	serge	3723	if (--obj->pin_count == 0)
2344	Serge	3724	obj->pin_mappable = false;
		3725	}
2332	Serge	3726
3031	serge	3727	int
		3728	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3729	struct drm_file *file)
		3730	{
		3731	struct drm_i915_gem_pin *args = data;
		3732	struct drm_i915_gem_object *obj;
		3733	int ret;
2332	Serge	3734
3480	Serge	3735	if(args->handle == -2)
		3736	{
		3737	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3738	return 0;
		3739	}
		3740
3031	serge	3741	ret = i915_mutex_lock_interruptible(dev);
		3742	if (ret)
		3743	return ret;
2332	Serge	3744
3031	serge	3745	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3746	if (&obj->base == NULL) {
		3747	ret = -ENOENT;
		3748	goto unlock;
		3749	}
2332	Serge	3750
3031	serge	3751	if (obj->madv != I915_MADV_WILLNEED) {
		3752	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3753	ret = -EINVAL;
		3754	goto out;
		3755	}
2332	Serge	3756
3031	serge	3757	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3758	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3759	args->handle);
		3760	ret = -EINVAL;
		3761	goto out;
		3762	}
2332	Serge	3763
3243	Serge	3764	if (obj->user_pin_count == 0) {
4104	Serge	3765	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3031	serge	3766	if (ret)
		3767	goto out;
		3768	}
2332	Serge	3769
3243	Serge	3770	obj->user_pin_count++;
		3771	obj->pin_filp = file;
		3772
4104	Serge	3773	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3774	out:
		3775	drm_gem_object_unreference(&obj->base);
		3776	unlock:
		3777	mutex_unlock(&dev->struct_mutex);
		3778	return ret;
		3779	}
2332	Serge	3780
3031	serge	3781	int
		3782	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3783	struct drm_file *file)
		3784	{
		3785	struct drm_i915_gem_pin *args = data;
		3786	struct drm_i915_gem_object *obj;
		3787	int ret;
2332	Serge	3788
3031	serge	3789	ret = i915_mutex_lock_interruptible(dev);
		3790	if (ret)
		3791	return ret;
2332	Serge	3792
4246	Serge	3793	if(args->handle == -2)
		3794	{
		3795	obj = get_fb_obj();
		3796	drm_gem_object_reference(&obj->base);
		3797	}
		3798	else
3031	serge	3799	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3800	if (&obj->base == NULL) {
		3801	ret = -ENOENT;
		3802	goto unlock;
		3803	}
2332	Serge	3804
3031	serge	3805	if (obj->pin_filp != file) {
		3806	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3807	args->handle);
		3808	ret = -EINVAL;
		3809	goto out;
		3810	}
		3811	obj->user_pin_count--;
		3812	if (obj->user_pin_count == 0) {
		3813	obj->pin_filp = NULL;
		3814	i915_gem_object_unpin(obj);
		3815	}
2332	Serge	3816
3031	serge	3817	out:
		3818	drm_gem_object_unreference(&obj->base);
		3819	unlock:
		3820	mutex_unlock(&dev->struct_mutex);
		3821	return ret;
		3822	}
2332	Serge	3823
3031	serge	3824	int
		3825	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3826	struct drm_file *file)
		3827	{
		3828	struct drm_i915_gem_busy *args = data;
		3829	struct drm_i915_gem_object *obj;
		3830	int ret;
2332	Serge	3831
3031	serge	3832	ret = i915_mutex_lock_interruptible(dev);
		3833	if (ret)
		3834	return ret;
2332	Serge	3835
3480	Serge	3836	if(args->handle == -2)
		3837	{
		3838	obj = get_fb_obj();
		3839	drm_gem_object_reference(&obj->base);
		3840	}
		3841	else
4104	Serge	3842	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3843	if (&obj->base == NULL) {
		3844	ret = -ENOENT;
		3845	goto unlock;
		3846	}
2332	Serge	3847
3031	serge	3848	/* Count all active objects as busy, even if they are currently not used
		3849	* by the gpu. Users of this interface expect objects to eventually
		3850	* become non-busy without any further actions, therefore emit any
		3851	* necessary flushes here.
		3852	*/
		3853	ret = i915_gem_object_flush_active(obj);
2332	Serge	3854
3031	serge	3855	args->busy = obj->active;
		3856	if (obj->ring) {
		3857	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3858	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3859	}
2332	Serge	3860
3031	serge	3861	drm_gem_object_unreference(&obj->base);
		3862	unlock:
		3863	mutex_unlock(&dev->struct_mutex);
		3864	return ret;
		3865	}
2332	Serge	3866
3031	serge	3867	int
		3868	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3869	struct drm_file *file_priv)
		3870	{
		3871	return i915_gem_ring_throttle(dev, file_priv);
		3872	}
2332	Serge	3873
3263	Serge	3874	#if 0
		3875
3031	serge	3876	int
		3877	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3878	struct drm_file *file_priv)
		3879	{
		3880	struct drm_i915_gem_madvise *args = data;
		3881	struct drm_i915_gem_object *obj;
		3882	int ret;
2332	Serge	3883
3031	serge	3884	switch (args->madv) {
		3885	case I915_MADV_DONTNEED:
		3886	case I915_MADV_WILLNEED:
		3887	break;
		3888	default:
		3889	return -EINVAL;
		3890	}
2332	Serge	3891
3031	serge	3892	ret = i915_mutex_lock_interruptible(dev);
		3893	if (ret)
		3894	return ret;
2332	Serge	3895
3031	serge	3896	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3897	if (&obj->base == NULL) {
		3898	ret = -ENOENT;
		3899	goto unlock;
		3900	}
2332	Serge	3901
3031	serge	3902	if (obj->pin_count) {
		3903	ret = -EINVAL;
		3904	goto out;
		3905	}
2332	Serge	3906
3031	serge	3907	if (obj->madv != __I915_MADV_PURGED)
		3908	obj->madv = args->madv;
2332	Serge	3909
3031	serge	3910	/* if the object is no longer attached, discard its backing storage */
		3911	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3912	i915_gem_object_truncate(obj);
2332	Serge	3913
3031	serge	3914	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3915
3031	serge	3916	out:
		3917	drm_gem_object_unreference(&obj->base);
		3918	unlock:
		3919	mutex_unlock(&dev->struct_mutex);
		3920	return ret;
		3921	}
		3922	#endif
2332	Serge	3923
3031	serge	3924	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3925	const struct drm_i915_gem_object_ops *ops)
		3926	{
4104	Serge	3927	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3928	INIT_LIST_HEAD(&obj->ring_list);
		3929	INIT_LIST_HEAD(&obj->exec_list);
4104	Serge	3930	INIT_LIST_HEAD(&obj->obj_exec_link);
		3931	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3932
3031	serge	3933	obj->ops = ops;
		3934
		3935	obj->fence_reg = I915_FENCE_REG_NONE;
		3936	obj->madv = I915_MADV_WILLNEED;
		3937	/* Avoid an unnecessary call to unbind on the first bind. */
		3938	obj->map_and_fenceable = true;
		3939
		3940	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3941	}
		3942
		3943	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3944	.get_pages = i915_gem_object_get_pages_gtt,
		3945	.put_pages = i915_gem_object_put_pages_gtt,
		3946	};
		3947
2332	Serge	3948	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3949	size_t size)
		3950	{
		3951	struct drm_i915_gem_object *obj;
3031	serge	3952	struct address_space *mapping;
3480	Serge	3953	gfp_t mask;
2340	Serge	3954
3746	Serge	3955	obj = i915_gem_object_alloc(dev);
2332	Serge	3956	if (obj == NULL)
		3957	return NULL;
		3958
		3959	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3960	i915_gem_object_free(obj);
2332	Serge	3961	return NULL;
		3962	}
		3963
		3964
3031	serge	3965	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3966
		3967	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3968	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3969
3031	serge	3970	if (HAS_LLC(dev)) {
		3971	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3972	* cache) for about a 10% performance improvement
		3973	* compared to uncached. Graphics requests other than
		3974	* display scanout are coherent with the CPU in
		3975	* accessing this cache. This means in this mode we
		3976	* don't need to clflush on the CPU side, and on the
		3977	* GPU side we only need to flush internal caches to
		3978	* get data visible to the CPU.
		3979	*
		3980	* However, we maintain the display planes as UC, and so
		3981	* need to rebind when first used as such.
		3982	*/
		3983	obj->cache_level = I915_CACHE_LLC;
		3984	} else
		3985	obj->cache_level = I915_CACHE_NONE;
		3986
		3987	return obj;
		3988	}
		3989
2344	Serge	3990	int i915_gem_init_object(struct drm_gem_object *obj)
		3991	{
		3992	BUG();
2332	Serge	3993
2344	Serge	3994	return 0;
		3995	}
2332	Serge	3996
3031	serge	3997	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3998	{
3031	serge	3999	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	4000	struct drm_device *dev = obj->base.dev;
		4001	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	4002	struct i915_vma vma, next;
2332	Serge	4003
3031	serge	4004	trace_i915_gem_object_destroy(obj);
		4005
		4006
		4007	obj->pin_count = 0;
4104	Serge	4008	/* NB: 0 or 1 elements */
		4009	WARN_ON(!list_empty(&obj->vma_list) &&
		4010	!list_is_singular(&obj->vma_list));
		4011	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		4012	int ret = i915_vma_unbind(vma);
		4013	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	4014	bool was_interruptible;
		4015
		4016	was_interruptible = dev_priv->mm.interruptible;
		4017	dev_priv->mm.interruptible = false;
		4018
4104	Serge	4019	WARN_ON(i915_vma_unbind(vma));
3031	serge	4020
		4021	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	4022	}
4104	Serge	4023	}
2332	Serge	4024
4104	Serge	4025	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		4026	* before progressing. */
		4027	if (obj->stolen)
		4028	i915_gem_object_unpin_pages(obj);
		4029
		4030	if (WARN_ON(obj->pages_pin_count))
3031	serge	4031	obj->pages_pin_count = 0;
		4032	i915_gem_object_put_pages(obj);
		4033	// i915_gem_object_free_mmap_offset(obj);
4104	Serge	4034	i915_gem_object_release_stolen(obj);
2332	Serge	4035
3243	Serge	4036	BUG_ON(obj->pages);
2332	Serge	4037
3031	serge	4038
3290	Serge	4039	if(obj->base.filp != NULL)
		4040	{
3298	Serge	4041	// printf("filp %p\n", obj->base.filp);
3290	Serge	4042	shmem_file_delete(obj->base.filp);
		4043	}
		4044
2344	Serge	4045	drm_gem_object_release(&obj->base);
		4046	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	4047
2344	Serge	4048	kfree(obj->bit_17);
4104	Serge	4049	i915_gem_object_free(obj);
2344	Serge	4050	}
2332	Serge	4051
4104	Serge	4052	struct i915_vma i915_gem_vma_create(struct drm_i915_gem_object obj,
		4053	struct i915_address_space *vm)
		4054	{
		4055	struct i915_vma vma = kzalloc(sizeof(vma), GFP_KERNEL);
		4056	if (vma == NULL)
		4057	return ERR_PTR(-ENOMEM);
		4058
		4059	INIT_LIST_HEAD(&vma->vma_link);
		4060	INIT_LIST_HEAD(&vma->mm_list);
		4061	INIT_LIST_HEAD(&vma->exec_list);
		4062	vma->vm = vm;
		4063	vma->obj = obj;
		4064
		4065	/* Keep GGTT vmas first to make debug easier */
		4066	if (i915_is_ggtt(vm))
		4067	list_add(&vma->vma_link, &obj->vma_list);
		4068	else
		4069	list_add_tail(&vma->vma_link, &obj->vma_list);
		4070
		4071	return vma;
		4072	}
		4073
		4074	void i915_gem_vma_destroy(struct i915_vma *vma)
		4075	{
		4076	WARN_ON(vma->node.allocated);
		4077	list_del(&vma->vma_link);
		4078	kfree(vma);
		4079	}
		4080
3031	serge	4081	#if 0
		4082	int
		4083	i915_gem_idle(struct drm_device *dev)
2344	Serge	4084	{
3031	serge	4085	drm_i915_private_t *dev_priv = dev->dev_private;
		4086	int ret;
2332	Serge	4087
4104	Serge	4088	if (dev_priv->ums.mm_suspended) {
3031	serge	4089	mutex_unlock(&dev->struct_mutex);
		4090	return 0;
		4091	}
2332	Serge	4092
3031	serge	4093	ret = i915_gpu_idle(dev);
		4094	if (ret) {
		4095	mutex_unlock(&dev->struct_mutex);
		4096	return ret;
		4097	}
		4098	i915_gem_retire_requests(dev);
		4099
3480	Serge	4100	/* Under UMS, be paranoid and evict. */
		4101	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		4102	i915_gem_evict_everything(dev);
		4103
		4104	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	4105
		4106	i915_kernel_lost_context(dev);
		4107	i915_gem_cleanup_ringbuffer(dev);
		4108
		4109	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	4110	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	4111
		4112	return 0;
2344	Serge	4113	}
3031	serge	4114	#endif
2332	Serge	4115
3031	serge	4116	void i915_gem_l3_remap(struct drm_device *dev)
		4117	{
		4118	drm_i915_private_t *dev_priv = dev->dev_private;
		4119	u32 misccpctl;
		4120	int i;
2332	Serge	4121
3480	Serge	4122	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	4123	return;
2332	Serge	4124
3243	Serge	4125	if (!dev_priv->l3_parity.remap_info)
3031	serge	4126	return;
2332	Serge	4127
3031	serge	4128	misccpctl = I915_READ(GEN7_MISCCPCTL);
		4129	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		4130	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	4131
3031	serge	4132	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		4133	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	4134	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	4135	DRM_DEBUG("0x%x was already programmed to %x\n",
		4136	GEN7_L3LOG_BASE + i, remap);
3243	Serge	4137	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	4138	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	4139	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	4140	}
2332	Serge	4141
3031	serge	4142	/* Make sure all the writes land before disabling dop clock gating */
		4143	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	4144
3031	serge	4145	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		4146	}
2332	Serge	4147
3031	serge	4148	void i915_gem_init_swizzling(struct drm_device *dev)
		4149	{
		4150	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	4151
3031	serge	4152	if (INTEL_INFO(dev)->gen < 5 \|\|
		4153	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4154	return;
2332	Serge	4155
3031	serge	4156	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4157	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4158
3031	serge	4159	if (IS_GEN5(dev))
		4160	return;
2344	Serge	4161
3031	serge	4162	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4163	if (IS_GEN6(dev))
		4164	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4165	else if (IS_GEN7(dev))
		4166	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	4167	else
3480	Serge	4168	BUG();
3031	serge	4169	}
		4170
		4171	static bool
		4172	intel_enable_blt(struct drm_device *dev)
		4173	{
		4174	if (!HAS_BLT(dev))
		4175	return false;
		4176
		4177	/* The blitter was dysfunctional on early prototypes */
		4178	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4179	DRM_INFO("BLT not supported on this pre-production hardware;"
		4180	" graphics performance will be degraded.\n");
		4181	return false;
		4182	}
		4183
		4184	return true;
		4185	}
		4186
3480	Serge	4187	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	4188	{
3480	Serge	4189	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4190	int ret;
2351	Serge	4191
2332	Serge	4192	ret = intel_init_render_ring_buffer(dev);
		4193	if (ret)
		4194	return ret;
		4195
		4196	if (HAS_BSD(dev)) {
		4197	ret = intel_init_bsd_ring_buffer(dev);
		4198	if (ret)
		4199	goto cleanup_render_ring;
		4200	}
		4201
3031	serge	4202	if (intel_enable_blt(dev)) {
2332	Serge	4203	ret = intel_init_blt_ring_buffer(dev);
		4204	if (ret)
		4205	goto cleanup_bsd_ring;
		4206	}
		4207
4104	Serge	4208	if (HAS_VEBOX(dev)) {
		4209	ret = intel_init_vebox_ring_buffer(dev);
		4210	if (ret)
		4211	goto cleanup_blt_ring;
		4212	}
		4213
		4214
3480	Serge	4215	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4216	if (ret)
4104	Serge	4217	goto cleanup_vebox_ring;
2351	Serge	4218
2332	Serge	4219	return 0;
		4220
4104	Serge	4221	cleanup_vebox_ring:
		4222	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4223	cleanup_blt_ring:
		4224	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4225	cleanup_bsd_ring:
		4226	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4227	cleanup_render_ring:
		4228	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4229
2332	Serge	4230	return ret;
		4231	}
		4232
3480	Serge	4233	int
		4234	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4235	{
3480	Serge	4236	drm_i915_private_t *dev_priv = dev->dev_private;
		4237	int ret;
3031	serge	4238
3480	Serge	4239	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4240	return -EIO;
3031	serge	4241
4104	Serge	4242	if (dev_priv->ellc_size)
		4243	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4244
3746	Serge	4245	if (HAS_PCH_NOP(dev)) {
		4246	u32 temp = I915_READ(GEN7_MSG_CTL);
		4247	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4248	I915_WRITE(GEN7_MSG_CTL, temp);
		4249	}
		4250
3480	Serge	4251	i915_gem_l3_remap(dev);
		4252
		4253	i915_gem_init_swizzling(dev);
		4254
		4255	ret = i915_gem_init_rings(dev);
		4256	if (ret)
		4257	return ret;
		4258
		4259	/*
		4260	* XXX: There was some w/a described somewhere suggesting loading
		4261	* contexts before PPGTT.
		4262	*/
		4263	i915_gem_context_init(dev);
3746	Serge	4264	if (dev_priv->mm.aliasing_ppgtt) {
		4265	ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		4266	if (ret) {
		4267	i915_gem_cleanup_aliasing_ppgtt(dev);
		4268	DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		4269	}
		4270	}
3480	Serge	4271
		4272	return 0;
3031	serge	4273	}
		4274
		4275	int i915_gem_init(struct drm_device *dev)
		4276	{
		4277	struct drm_i915_private *dev_priv = dev->dev_private;
		4278	int ret;
		4279
		4280	mutex_lock(&dev->struct_mutex);
3746	Serge	4281
		4282	if (IS_VALLEYVIEW(dev)) {
		4283	/* VLVA0 (potential hack), BIOS isn't actually waking us */
		4284	I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		4285	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
		4286	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4287	}
		4288
3480	Serge	4289	i915_gem_init_global_gtt(dev);
3746	Serge	4290
3031	serge	4291	ret = i915_gem_init_hw(dev);
		4292	mutex_unlock(&dev->struct_mutex);
		4293	if (ret) {
		4294	i915_gem_cleanup_aliasing_ppgtt(dev);
		4295	return ret;
		4296	}
		4297
3746	Serge	4298
3031	serge	4299	return 0;
		4300	}
		4301
2332	Serge	4302	void
		4303	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4304	{
		4305	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	4306	struct intel_ring_buffer *ring;
2332	Serge	4307	int i;
		4308
3031	serge	4309	for_each_ring(ring, dev_priv, i)
		4310	intel_cleanup_ring_buffer(ring);
2332	Serge	4311	}
		4312
3031	serge	4313	#if 0
		4314
2332	Serge	4315	int
		4316	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		4317	struct drm_file *file_priv)
		4318	{
4104	Serge	4319	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4320	int ret;
2332	Serge	4321
		4322	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4323	return 0;
		4324
3480	Serge	4325	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	4326	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	4327	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	4328	}
		4329
		4330	mutex_lock(&dev->struct_mutex);
4104	Serge	4331	dev_priv->ums.mm_suspended = 0;
2332	Serge	4332
3031	serge	4333	ret = i915_gem_init_hw(dev);
2332	Serge	4334	if (ret != 0) {
		4335	mutex_unlock(&dev->struct_mutex);
		4336	return ret;
		4337	}
		4338
4104	Serge	4339	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
2332	Serge	4340	mutex_unlock(&dev->struct_mutex);
		4341
		4342	ret = drm_irq_install(dev);
		4343	if (ret)
		4344	goto cleanup_ringbuffer;
		4345
		4346	return 0;
		4347
		4348	cleanup_ringbuffer:
		4349	mutex_lock(&dev->struct_mutex);
		4350	i915_gem_cleanup_ringbuffer(dev);
4104	Serge	4351	dev_priv->ums.mm_suspended = 1;
2332	Serge	4352	mutex_unlock(&dev->struct_mutex);
		4353
		4354	return ret;
		4355	}
		4356
		4357	int
		4358	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4359	struct drm_file *file_priv)
		4360	{
4104	Serge	4361	struct drm_i915_private *dev_priv = dev->dev_private;
		4362	int ret;
		4363
2332	Serge	4364	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4365	return 0;
		4366
		4367	drm_irq_uninstall(dev);
4104	Serge	4368
		4369	mutex_lock(&dev->struct_mutex);
		4370	ret = i915_gem_idle(dev);
		4371
		4372	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		4373	* We need to replace this with a semaphore, or something.
		4374	* And not confound ums.mm_suspended!
		4375	*/
		4376	if (ret != 0)
		4377	dev_priv->ums.mm_suspended = 1;
		4378	mutex_unlock(&dev->struct_mutex);
		4379
		4380	return ret;
2332	Serge	4381	}
		4382
		4383	void
		4384	i915_gem_lastclose(struct drm_device *dev)
		4385	{
		4386	int ret;
		4387
		4388	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4389	return;
		4390
4104	Serge	4391	mutex_lock(&dev->struct_mutex);
2332	Serge	4392	ret = i915_gem_idle(dev);
		4393	if (ret)
		4394	DRM_ERROR("failed to idle hardware: %d\n", ret);
4104	Serge	4395	mutex_unlock(&dev->struct_mutex);
2332	Serge	4396	}
		4397	#endif
		4398
		4399	static void
2326	Serge	4400	init_ring_lists(struct intel_ring_buffer *ring)
		4401	{
		4402	INIT_LIST_HEAD(&ring->active_list);
		4403	INIT_LIST_HEAD(&ring->request_list);
		4404	}
		4405
4104	Serge	4406	static void i915_init_vm(struct drm_i915_private *dev_priv,
		4407	struct i915_address_space *vm)
		4408	{
		4409	vm->dev = dev_priv->dev;
		4410	INIT_LIST_HEAD(&vm->active_list);
		4411	INIT_LIST_HEAD(&vm->inactive_list);
		4412	INIT_LIST_HEAD(&vm->global_link);
		4413	list_add(&vm->global_link, &dev_priv->vm_list);
		4414	}
		4415
2326	Serge	4416	void
		4417	i915_gem_load(struct drm_device *dev)
		4418	{
3480	Serge	4419	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4420	int i;
		4421
4104	Serge	4422	INIT_LIST_HEAD(&dev_priv->vm_list);
		4423	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4424
3031	serge	4425	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4426	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4427	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4428	for (i = 0; i < I915_NUM_RINGS; i++)
		4429	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4430	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4431	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4432	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4433	i915_gem_retire_work_handler);
3480	Serge	4434	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4435
		4436	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4437	if (IS_GEN3(dev)) {
3031	serge	4438	I915_WRITE(MI_ARB_STATE,
		4439	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4440	}
		4441
		4442	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4443
3746	Serge	4444	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4445	dev_priv->num_fence_regs = 32;
		4446	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4447	dev_priv->num_fence_regs = 16;
		4448	else
		4449	dev_priv->num_fence_regs = 8;
		4450
		4451	/* Initialize fence registers to zero */
3746	Serge	4452	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4453	i915_gem_restore_fences(dev);
2326	Serge	4454
		4455	i915_gem_detect_bit_6_swizzle(dev);
		4456
		4457	dev_priv->mm.interruptible = true;
		4458
		4459	}
		4460
4104	Serge	4461	#if 0
		4462	/*
		4463	* Create a physically contiguous memory object for this object
		4464	* e.g. for cursor + overlay regs
		4465	*/
		4466	static int i915_gem_init_phys_object(struct drm_device *dev,
		4467	int id, int size, int align)
		4468	{
		4469	drm_i915_private_t *dev_priv = dev->dev_private;
		4470	struct drm_i915_gem_phys_object *phys_obj;
		4471	int ret;
2326	Serge	4472
4104	Serge	4473	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)
		4474	return 0;
		4475
		4476	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
		4477	if (!phys_obj)
		4478	return -ENOMEM;
		4479
		4480	phys_obj->id = id;
		4481
		4482	phys_obj->handle = drm_pci_alloc(dev, size, align);
		4483	if (!phys_obj->handle) {
		4484	ret = -ENOMEM;
		4485	goto kfree_obj;
		4486	}
		4487	#ifdef CONFIG_X86
		4488	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4489	#endif
		4490
		4491	dev_priv->mm.phys_objs[id - 1] = phys_obj;
		4492
		4493	return 0;
		4494	kfree_obj:
		4495	kfree(phys_obj);
		4496	return ret;
		4497	}
		4498
		4499	static void i915_gem_free_phys_object(struct drm_device *dev, int id)
		4500	{
		4501	drm_i915_private_t *dev_priv = dev->dev_private;
		4502	struct drm_i915_gem_phys_object *phys_obj;
		4503
		4504	if (!dev_priv->mm.phys_objs[id - 1])
		4505	return;
		4506
		4507	phys_obj = dev_priv->mm.phys_objs[id - 1];
		4508	if (phys_obj->cur_obj) {
		4509	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
		4510	}
		4511
		4512	#ifdef CONFIG_X86
		4513	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4514	#endif
		4515	drm_pci_free(dev, phys_obj->handle);
		4516	kfree(phys_obj);
		4517	dev_priv->mm.phys_objs[id - 1] = NULL;
		4518	}
		4519
		4520	void i915_gem_free_all_phys_object(struct drm_device *dev)
		4521	{
		4522	int i;
		4523
		4524	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
		4525	i915_gem_free_phys_object(dev, i);
		4526	}
		4527
		4528	void i915_gem_detach_phys_object(struct drm_device *dev,
		4529	struct drm_i915_gem_object *obj)
		4530	{
		4531	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4532	char *vaddr;
		4533	int i;
		4534	int page_count;
		4535
		4536	if (!obj->phys_obj)
		4537	return;
		4538	vaddr = obj->phys_obj->handle->vaddr;
		4539
		4540	page_count = obj->base.size / PAGE_SIZE;
		4541	for (i = 0; i < page_count; i++) {
		4542	struct page *page = shmem_read_mapping_page(mapping, i);
		4543	if (!IS_ERR(page)) {
		4544	char *dst = kmap_atomic(page);
		4545	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
		4546	kunmap_atomic(dst);
		4547
		4548	drm_clflush_pages(&page, 1);
		4549
		4550	set_page_dirty(page);
		4551	mark_page_accessed(page);
		4552	page_cache_release(page);
		4553	}
		4554	}
		4555	i915_gem_chipset_flush(dev);
		4556
		4557	obj->phys_obj->cur_obj = NULL;
		4558	obj->phys_obj = NULL;
		4559	}
		4560
		4561	int
		4562	i915_gem_attach_phys_object(struct drm_device *dev,
		4563	struct drm_i915_gem_object *obj,
		4564	int id,
		4565	int align)
		4566	{
		4567	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4568	drm_i915_private_t *dev_priv = dev->dev_private;
		4569	int ret = 0;
		4570	int page_count;
		4571	int i;
		4572
		4573	if (id > I915_MAX_PHYS_OBJECT)
		4574	return -EINVAL;
		4575
		4576	if (obj->phys_obj) {
		4577	if (obj->phys_obj->id == id)
		4578	return 0;
		4579	i915_gem_detach_phys_object(dev, obj);
		4580	}
		4581
		4582	/* create a new object */
		4583	if (!dev_priv->mm.phys_objs[id - 1]) {
		4584	ret = i915_gem_init_phys_object(dev, id,
		4585	obj->base.size, align);
		4586	if (ret) {
		4587	DRM_ERROR("failed to init phys object %d size: %zu\n",
		4588	id, obj->base.size);
		4589	return ret;
		4590	}
		4591	}
		4592
		4593	/* bind to the object */
		4594	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
		4595	obj->phys_obj->cur_obj = obj;
		4596
		4597	page_count = obj->base.size / PAGE_SIZE;
		4598
		4599	for (i = 0; i < page_count; i++) {
		4600	struct page *page;
		4601	char dst, src;
		4602
		4603	page = shmem_read_mapping_page(mapping, i);
		4604	if (IS_ERR(page))
		4605	return PTR_ERR(page);
		4606
		4607	src = kmap_atomic(page);
		4608	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
		4609	memcpy(dst, src, PAGE_SIZE);
		4610	kunmap_atomic(src);
		4611
		4612	mark_page_accessed(page);
		4613	page_cache_release(page);
		4614	}
		4615
		4616	return 0;
		4617	}
		4618
		4619	static int
		4620	i915_gem_phys_pwrite(struct drm_device *dev,
		4621	struct drm_i915_gem_object *obj,
		4622	struct drm_i915_gem_pwrite *args,
		4623	struct drm_file *file_priv)
		4624	{
		4625	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
		4626	char __user *user_data = to_user_ptr(args->data_ptr);
		4627
		4628	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		4629	unsigned long unwritten;
		4630
		4631	/* The physical object once assigned is fixed for the lifetime
		4632	* of the obj, so we can safely drop the lock and continue
		4633	* to access vaddr.
		4634	*/
		4635	mutex_unlock(&dev->struct_mutex);
		4636	unwritten = copy_from_user(vaddr, user_data, args->size);
		4637	mutex_lock(&dev->struct_mutex);
		4638	if (unwritten)
		4639	return -EFAULT;
		4640	}
		4641
		4642	i915_gem_chipset_flush(dev);
		4643	return 0;
		4644	}
		4645
		4646	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4647	{
		4648	struct drm_i915_file_private *file_priv = file->driver_priv;
		4649
		4650	/* Clean up our request list when the client is going away, so that
		4651	* later retire_requests won't dereference our soon-to-be-gone
		4652	* file_priv.
		4653	*/
		4654	spin_lock(&file_priv->mm.lock);
		4655	while (!list_empty(&file_priv->mm.request_list)) {
		4656	struct drm_i915_gem_request *request;
		4657
		4658	request = list_first_entry(&file_priv->mm.request_list,
		4659	struct drm_i915_gem_request,
		4660	client_list);
		4661	list_del(&request->client_list);
		4662	request->file_priv = NULL;
		4663	}
		4664	spin_unlock(&file_priv->mm.lock);
		4665	}
		4666	#endif
		4667
		4668	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4669	{
		4670	if (!mutex_is_locked(mutex))
		4671	return false;
		4672
		4673	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4674	return mutex->owner == task;
		4675	#else
		4676	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4677	return false;
		4678	#endif
		4679	}
		4680
		4681	/* All the new VM stuff */
		4682	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4683	struct i915_address_space *vm)
		4684	{
		4685	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4686	struct i915_vma *vma;
		4687
		4688	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4689	vm = &dev_priv->gtt.base;
		4690
		4691	BUG_ON(list_empty(&o->vma_list));
		4692	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4693	if (vma->vm == vm)
		4694	return vma->node.start;
		4695
		4696	}
		4697	return 0; //-1;
		4698	}
		4699
		4700	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4701	struct i915_address_space *vm)
		4702	{
		4703	struct i915_vma *vma;
		4704
		4705	list_for_each_entry(vma, &o->vma_list, vma_link)
		4706	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4707	return true;
		4708
		4709	return false;
		4710	}
		4711
		4712	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4713	{
		4714	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4715	struct i915_address_space *vm;
		4716
		4717	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
		4718	if (i915_gem_obj_bound(o, vm))
		4719	return true;
		4720
		4721	return false;
		4722	}
		4723
		4724	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4725	struct i915_address_space *vm)
		4726	{
		4727	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4728	struct i915_vma *vma;
		4729
		4730	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4731	vm = &dev_priv->gtt.base;
		4732
		4733	BUG_ON(list_empty(&o->vma_list));
		4734
		4735	list_for_each_entry(vma, &o->vma_list, vma_link)
		4736	if (vma->vm == vm)
		4737	return vma->node.size;
		4738
		4739	return 0;
		4740	}
		4741	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
		4742	struct i915_address_space *vm)
		4743	{
		4744	struct i915_vma *vma;
		4745	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4746	if (vma->vm == vm)
		4747	return vma;
		4748
		4749	return NULL;
		4750	}
		4751
		4752	struct i915_vma *
		4753	i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
		4754	struct i915_address_space *vm)
		4755	{
		4756	struct i915_vma *vma;
		4757
		4758	vma = i915_gem_obj_to_vma(obj, vm);
		4759	if (!vma)
		4760	vma = i915_gem_vma_create(obj, vm);
		4761
		4762	return vma;
		4763	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 4246