WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
3260	Serge	34	#include
2330	Serge	35	#include
2326	Serge	36	//#include
3746	Serge	37	#include
2326	Serge	38	#include
		39
2344	Serge	40	extern int x86_clflush_size;
2332	Serge	41
3263	Serge	42	#define PROT_READ 0x1 /* page can be read */
		43	#define PROT_WRITE 0x2 /* page can be written */
		44	#define MAP_SHARED 0x01 /* Share changes */
		45
2344	Serge	46	#undef mb
		47	#undef rmb
		48	#undef wmb
		49	#define mb() asm volatile("mfence")
		50	#define rmb() asm volatile ("lfence")
		51	#define wmb() asm volatile ("sfence")
		52
3266	Serge	53	struct drm_i915_gem_object *get_fb_obj();
		54
3263	Serge	55	unsigned long vm_mmap(struct file *file, unsigned long addr,
		56	unsigned long len, unsigned long prot,
		57	unsigned long flag, unsigned long offset);
		58
2344	Serge	59	static inline void clflush(volatile void *__p)
		60	{
		61	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		62	}
		63
2332	Serge	64	#define MAX_ERRNO 4095
		65
		66	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		67
		68
2326	Serge	69	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		70	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		71	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		72	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		73
2332	Serge	74	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	75	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		76	bool force);
		77	static __must_check int
		78	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		79	struct i915_address_space *vm,
2332	Serge	80	unsigned alignment,
3031	serge	81	bool map_and_fenceable,
		82	bool nonblocking);
2332	Serge	83	static int i915_gem_phys_pwrite(struct drm_device *dev,
		84	struct drm_i915_gem_object *obj,
		85	struct drm_i915_gem_pwrite *args,
		86	struct drm_file *file);
2326	Serge	87
3031	serge	88	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		89	struct drm_i915_gem_object *obj);
		90	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		91	struct drm_i915_fence_reg *fence,
		92	bool enable);
2332	Serge	93
3031	serge	94	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
4104	Serge	95	static long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	96	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		97
4104	Serge	98	static bool cpu_cache_is_coherent(struct drm_device *dev,
		99	enum i915_cache_level level)
		100	{
		101	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		102	}
		103
		104	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		105	{
		106	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		107	return true;
		108
		109	return obj->pin_display;
		110	}
		111
3031	serge	112	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		113	{
		114	if (obj->tiling_mode)
		115	i915_gem_release_mmap(obj);
		116
		117	/* As we do not have an associated fence register, we will force
		118	* a tiling change if we ever need to acquire one.
		119	*/
		120	obj->fence_dirty = false;
		121	obj->fence_reg = I915_FENCE_REG_NONE;
		122	}
		123
2332	Serge	124	/* some bookkeeping */
		125	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		126	size_t size)
		127	{
4104	Serge	128	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	129	dev_priv->mm.object_count++;
		130	dev_priv->mm.object_memory += size;
4104	Serge	131	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	132	}
		133
		134	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		135	size_t size)
		136	{
4104	Serge	137	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	138	dev_priv->mm.object_count--;
		139	dev_priv->mm.object_memory -= size;
4104	Serge	140	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	141	}
		142
		143	static int
3480	Serge	144	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	145	{
		146	int ret;
		147
3480	Serge	148	#define EXIT_COND (!i915_reset_in_progress(error))
		149	if (EXIT_COND)
2332	Serge	150	return 0;
3255	Serge	151	#if 0
3031	serge	152	/*
		153	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		154	* userspace. If it takes that long something really bad is going on and
		155	* we should simply try to bail out and fail as gracefully as possible.
		156	*/
3480	Serge	157	ret = wait_event_interruptible_timeout(error->reset_queue,
		158	EXIT_COND,
		159	10*HZ);
3031	serge	160	if (ret == 0) {
		161	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		162	return -EIO;
		163	} else if (ret < 0) {
2332	Serge	164	return ret;
3031	serge	165	}
2332	Serge	166
3255	Serge	167	#endif
3480	Serge	168	#undef EXIT_COND
3255	Serge	169
2332	Serge	170	return 0;
		171	}
		172
		173	int i915_mutex_lock_interruptible(struct drm_device *dev)
		174	{
3480	Serge	175	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	176	int ret;
		177
3480	Serge	178	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	179	if (ret)
		180	return ret;
		181
3480	Serge	182	ret = mutex_lock_interruptible(&dev->struct_mutex);
		183	if (ret)
		184	return ret;
2332	Serge	185
		186	WARN_ON(i915_verify_lists(dev));
		187	return 0;
		188	}
		189
		190	static inline bool
		191	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		192	{
4104	Serge	193	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	194	}
		195
		196
		197	#if 0
		198
		199	int
		200	i915_gem_init_ioctl(struct drm_device dev, void data,
		201	struct drm_file *file)
		202	{
3480	Serge	203	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	204	struct drm_i915_gem_init *args = data;
		205
3031	serge	206	if (drm_core_check_feature(dev, DRIVER_MODESET))
		207	return -ENODEV;
		208
2332	Serge	209	if (args->gtt_start >= args->gtt_end \|\|
		210	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		211	return -EINVAL;
		212
3031	serge	213	/* GEM with user mode setting was never supported on ilk and later. */
		214	if (INTEL_INFO(dev)->gen >= 5)
		215	return -ENODEV;
		216
2332	Serge	217	mutex_lock(&dev->struct_mutex);
3480	Serge	218	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		219	args->gtt_end);
		220	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	221	mutex_unlock(&dev->struct_mutex);
		222
		223	return 0;
		224	}
2351	Serge	225	#endif
2332	Serge	226
		227	int
		228	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		229	struct drm_file *file)
		230	{
		231	struct drm_i915_private *dev_priv = dev->dev_private;
		232	struct drm_i915_gem_get_aperture *args = data;
		233	struct drm_i915_gem_object *obj;
		234	size_t pinned;
		235
		236	pinned = 0;
		237	mutex_lock(&dev->struct_mutex);
4104	Serge	238	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
3031	serge	239	if (obj->pin_count)
4104	Serge	240	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	241	mutex_unlock(&dev->struct_mutex);
		242
4104	Serge	243	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	244	args->aper_available_size = args->aper_size - pinned;
2332	Serge	245
		246	return 0;
		247	}
		248
3480	Serge	249	void i915_gem_object_alloc(struct drm_device dev)
		250	{
		251	struct drm_i915_private *dev_priv = dev->dev_private;
		252	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		253	}
		254
		255	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		256	{
		257	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		258	kfree(obj);
		259	}
		260
3031	serge	261	static int
		262	i915_gem_create(struct drm_file *file,
2332	Serge	263	struct drm_device *dev,
		264	uint64_t size,
		265	uint32_t *handle_p)
		266	{
		267	struct drm_i915_gem_object *obj;
		268	int ret;
		269	u32 handle;
		270
		271	size = roundup(size, PAGE_SIZE);
2342	Serge	272	if (size == 0)
		273	return -EINVAL;
2332	Serge	274
		275	/* Allocate the new object */
		276	obj = i915_gem_alloc_object(dev, size);
		277	if (obj == NULL)
		278	return -ENOMEM;
		279
		280	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	281	/* drop reference from allocate - handle holds it now */
		282	drm_gem_object_unreference_unlocked(&obj->base);
		283	if (ret)
2332	Serge	284	return ret;
		285
		286	*handle_p = handle;
		287	return 0;
		288	}
		289
		290	int
		291	i915_gem_dumb_create(struct drm_file *file,
		292	struct drm_device *dev,
		293	struct drm_mode_create_dumb *args)
		294	{
		295	/* have to work out size/pitch and return them */
		296	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		297	args->size = args->pitch * args->height;
		298	return i915_gem_create(file, dev,
		299	args->size, &args->handle);
		300	}
		301
2326	Serge	302	/**
2332	Serge	303	* Creates a new mm object and returns a handle to it.
		304	*/
		305	int
		306	i915_gem_create_ioctl(struct drm_device dev, void data,
		307	struct drm_file *file)
		308	{
		309	struct drm_i915_gem_create *args = data;
3031	serge	310
2332	Serge	311	return i915_gem_create(file, dev,
		312	args->size, &args->handle);
		313	}
		314
		315
3260	Serge	316	#if 0
2332	Serge	317
3031	serge	318	static inline int
		319	__copy_to_user_swizzled(char __user *cpu_vaddr,
		320	const char *gpu_vaddr, int gpu_offset,
2332	Serge	321	int length)
		322	{
3031	serge	323	int ret, cpu_offset = 0;
2332	Serge	324
3031	serge	325	while (length > 0) {
		326	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		327	int this_length = min(cacheline_end - gpu_offset, length);
		328	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	329
3031	serge	330	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		331	gpu_vaddr + swizzled_gpu_offset,
		332	this_length);
		333	if (ret)
		334	return ret + length;
2332	Serge	335
3031	serge	336	cpu_offset += this_length;
		337	gpu_offset += this_length;
		338	length -= this_length;
		339	}
		340
		341	return 0;
2332	Serge	342	}
		343
3031	serge	344	static inline int
		345	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		346	const char __user *cpu_vaddr,
		347	int length)
2332	Serge	348	{
3031	serge	349	int ret, cpu_offset = 0;
2332	Serge	350
		351	while (length > 0) {
		352	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		353	int this_length = min(cacheline_end - gpu_offset, length);
		354	int swizzled_gpu_offset = gpu_offset ^ 64;
		355
3031	serge	356	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	357	cpu_vaddr + cpu_offset,
		358	this_length);
3031	serge	359	if (ret)
		360	return ret + length;
		361
2332	Serge	362	cpu_offset += this_length;
		363	gpu_offset += this_length;
		364	length -= this_length;
		365	}
		366
3031	serge	367	return 0;
2332	Serge	368	}
		369
3031	serge	370	/* Per-page copy function for the shmem pread fastpath.
		371	* Flushes invalid cachelines before reading the target if
		372	* needs_clflush is set. */
2332	Serge	373	static int
3031	serge	374	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		375	char __user *user_data,
		376	bool page_do_bit17_swizzling, bool needs_clflush)
		377	{
		378	char *vaddr;
		379	int ret;
		380
		381	if (unlikely(page_do_bit17_swizzling))
		382	return -EINVAL;
		383
		384	vaddr = kmap_atomic(page);
		385	if (needs_clflush)
		386	drm_clflush_virt_range(vaddr + shmem_page_offset,
		387	page_length);
		388	ret = __copy_to_user_inatomic(user_data,
		389	vaddr + shmem_page_offset,
		390	page_length);
		391	kunmap_atomic(vaddr);
		392
		393	return ret ? -EFAULT : 0;
		394	}
		395
		396	static void
		397	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		398	bool swizzled)
		399	{
		400	if (unlikely(swizzled)) {
		401	unsigned long start = (unsigned long) addr;
		402	unsigned long end = (unsigned long) addr + length;
		403
		404	/* For swizzling simply ensure that we always flush both
		405	* channels. Lame, but simple and it works. Swizzled
		406	* pwrite/pread is far from a hotpath - current userspace
		407	* doesn't use it at all. */
		408	start = round_down(start, 128);
		409	end = round_up(end, 128);
		410
		411	drm_clflush_virt_range((void *)start, end - start);
		412	} else {
		413	drm_clflush_virt_range(addr, length);
		414	}
		415
		416	}
		417
		418	/* Only difference to the fast-path function is that this can handle bit17
		419	* and uses non-atomic copy and kmap functions. */
		420	static int
		421	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		422	char __user *user_data,
		423	bool page_do_bit17_swizzling, bool needs_clflush)
		424	{
		425	char *vaddr;
		426	int ret;
		427
		428	vaddr = kmap(page);
		429	if (needs_clflush)
		430	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		431	page_length,
		432	page_do_bit17_swizzling);
		433
		434	if (page_do_bit17_swizzling)
		435	ret = __copy_to_user_swizzled(user_data,
		436	vaddr, shmem_page_offset,
		437	page_length);
		438	else
		439	ret = __copy_to_user(user_data,
		440	vaddr + shmem_page_offset,
		441	page_length);
		442	kunmap(page);
		443
		444	return ret ? - EFAULT : 0;
		445	}
		446
		447	static int
		448	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	449	struct drm_i915_gem_object *obj,
		450	struct drm_i915_gem_pread *args,
		451	struct drm_file *file)
		452	{
3031	serge	453	char __user *user_data;
2332	Serge	454	ssize_t remain;
		455	loff_t offset;
3031	serge	456	int shmem_page_offset, page_length, ret = 0;
		457	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		458	int prefaulted = 0;
		459	int needs_clflush = 0;
3746	Serge	460	struct sg_page_iter sg_iter;
2332	Serge	461
3746	Serge	462	user_data = to_user_ptr(args->data_ptr);
2332	Serge	463	remain = args->size;
		464
3031	serge	465	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		466
		467	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		468	/* If we're not in the cpu read domain, set ourself into the gtt
		469	* read domain and manually flush cachelines (if required). This
		470	* optimizes for the case when the gpu will dirty the data
		471	* anyway again before the next pread happens. */
4104	Serge	472	needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
		473	if (i915_gem_obj_bound_any(obj)) {
3031	serge	474	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		475	if (ret)
		476	return ret;
		477	}
		478	}
		479
		480	ret = i915_gem_object_get_pages(obj);
		481	if (ret)
		482	return ret;
		483
		484	i915_gem_object_pin_pages(obj);
		485
2332	Serge	486	offset = args->offset;
		487
3746	Serge	488	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		489	offset >> PAGE_SHIFT) {
		490	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	491
3031	serge	492	if (remain <= 0)
		493	break;
		494
2332	Serge	495	/* Operation in this page
		496	*
3031	serge	497	* shmem_page_offset = offset within page in shmem file
2332	Serge	498	* page_length = bytes to copy for this page
		499	*/
3031	serge	500	shmem_page_offset = offset_in_page(offset);
2332	Serge	501	page_length = remain;
3031	serge	502	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		503	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	504
3031	serge	505	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		506	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	507
3031	serge	508	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		509	user_data, page_do_bit17_swizzling,
		510	needs_clflush);
		511	if (ret == 0)
		512	goto next_page;
2332	Serge	513
3031	serge	514	mutex_unlock(&dev->struct_mutex);
		515
4104	Serge	516	if (likely(!i915_prefault_disable) && !prefaulted) {
3031	serge	517	ret = fault_in_multipages_writeable(user_data, remain);
		518	/* Userspace is tricking us, but we've already clobbered
		519	* its pages with the prefault and promised to write the
		520	* data up to the first fault. Hence ignore any errors
		521	* and just continue. */
		522	(void)ret;
		523	prefaulted = 1;
		524	}
		525
		526	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		527	user_data, page_do_bit17_swizzling,
		528	needs_clflush);
		529
		530	mutex_lock(&dev->struct_mutex);
		531
		532	next_page:
2332	Serge	533	mark_page_accessed(page);
3031	serge	534
2332	Serge	535	if (ret)
3031	serge	536	goto out;
2332	Serge	537
		538	remain -= page_length;
		539	user_data += page_length;
		540	offset += page_length;
		541	}
		542
3031	serge	543	out:
		544	i915_gem_object_unpin_pages(obj);
		545
		546	return ret;
2332	Serge	547	}
		548
		549	/**
3031	serge	550	* Reads data from the object referenced by handle.
		551	*
		552	* On error, the contents of *data are undefined.
2332	Serge	553	*/
3031	serge	554	int
		555	i915_gem_pread_ioctl(struct drm_device dev, void data,
		556	struct drm_file *file)
		557	{
		558	struct drm_i915_gem_pread *args = data;
		559	struct drm_i915_gem_object *obj;
		560	int ret = 0;
		561
		562	if (args->size == 0)
		563	return 0;
		564
		565	if (!access_ok(VERIFY_WRITE,
3746	Serge	566	to_user_ptr(args->data_ptr),
3031	serge	567	args->size))
		568	return -EFAULT;
		569
		570	ret = i915_mutex_lock_interruptible(dev);
		571	if (ret)
		572	return ret;
		573
		574	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		575	if (&obj->base == NULL) {
		576	ret = -ENOENT;
		577	goto unlock;
		578	}
		579
		580	/* Bounds check source. */
		581	if (args->offset > obj->base.size \|\|
		582	args->size > obj->base.size - args->offset) {
		583	ret = -EINVAL;
		584	goto out;
		585	}
		586
		587	/* prime objects have no backing filp to GEM pread/pwrite
		588	* pages from.
		589	*/
		590	if (!obj->base.filp) {
		591	ret = -EINVAL;
		592	goto out;
		593	}
		594
		595	trace_i915_gem_object_pread(obj, args->offset, args->size);
		596
		597	ret = i915_gem_shmem_pread(dev, obj, args, file);
		598
		599	out:
		600	drm_gem_object_unreference(&obj->base);
		601	unlock:
		602	mutex_unlock(&dev->struct_mutex);
		603	return ret;
		604	}
		605
		606	/* This is the fast write path which cannot handle
		607	* page faults in the source data
		608	*/
		609
		610	static inline int
		611	fast_user_write(struct io_mapping *mapping,
		612	loff_t page_base, int page_offset,
		613	char __user *user_data,
		614	int length)
		615	{
		616	void __iomem *vaddr_atomic;
		617	void *vaddr;
		618	unsigned long unwritten;
		619
		620	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		621	/* We can use the cpu mem copy function because this is X86. */
		622	vaddr = (void __force*)vaddr_atomic + page_offset;
		623	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		624	user_data, length);
		625	io_mapping_unmap_atomic(vaddr_atomic);
		626	return unwritten;
		627	}
3260	Serge	628	#endif
3031	serge	629
3260	Serge	630	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	631	/**
		632	* This is the fast pwrite path, where we copy the data directly from the
		633	* user into the GTT, uncached.
		634	*/
2332	Serge	635	static int
3031	serge	636	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		637	struct drm_i915_gem_object *obj,
		638	struct drm_i915_gem_pwrite *args,
		639	struct drm_file *file)
2332	Serge	640	{
3031	serge	641	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	642	ssize_t remain;
3031	serge	643	loff_t offset, page_base;
		644	char __user *user_data;
		645	int page_offset, page_length, ret;
3260	Serge	646	char *vaddr;
2332	Serge	647
4104	Serge	648	ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
3031	serge	649	if (ret)
		650	goto out;
		651
		652	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		653	if (ret)
		654	goto out_unpin;
		655
		656	ret = i915_gem_object_put_fence(obj);
		657	if (ret)
		658	goto out_unpin;
		659
3260	Serge	660	vaddr = AllocKernelSpace(4096);
		661	if(vaddr == NULL)
		662	{
		663	ret = -ENOSPC;
		664	goto out_unpin;
		665	};
		666
3031	serge	667	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	668	remain = args->size;
		669
4104	Serge	670	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	671
3031	serge	672	while (remain > 0) {
		673	/* Operation in this page
		674	*
		675	* page_base = page offset within aperture
		676	* page_offset = offset within page
		677	* page_length = bytes to copy for this page
		678	*/
		679	page_base = offset & PAGE_MASK;
		680	page_offset = offset_in_page(offset);
		681	page_length = remain;
		682	if ((page_offset + remain) > PAGE_SIZE)
		683	page_length = PAGE_SIZE - page_offset;
2332	Serge	684
3260	Serge	685	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	686
3260	Serge	687	memcpy(vaddr+page_offset, user_data, page_length);
		688
3031	serge	689	remain -= page_length;
		690	user_data += page_length;
		691	offset += page_length;
2332	Serge	692	}
		693
3260	Serge	694	FreeKernelSpace(vaddr);
		695
3031	serge	696	out_unpin:
		697	i915_gem_object_unpin(obj);
		698	out:
		699	return ret;
		700	}
		701
		702	/* Per-page copy function for the shmem pwrite fastpath.
		703	* Flushes invalid cachelines before writing to the target if
		704	* needs_clflush_before is set and flushes out any written cachelines after
		705	* writing if needs_clflush is set. */
		706	static int
		707	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		708	char __user *user_data,
		709	bool page_do_bit17_swizzling,
		710	bool needs_clflush_before,
		711	bool needs_clflush_after)
		712	{
		713	char *vaddr;
3260	Serge	714	int ret = 0;
3031	serge	715
		716	if (unlikely(page_do_bit17_swizzling))
		717	return -EINVAL;
		718
3260	Serge	719	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	720	if (needs_clflush_before)
		721	drm_clflush_virt_range(vaddr + shmem_page_offset,
		722	page_length);
3260	Serge	723	memcpy(vaddr + shmem_page_offset,
3031	serge	724	user_data,
		725	page_length);
		726	if (needs_clflush_after)
		727	drm_clflush_virt_range(vaddr + shmem_page_offset,
		728	page_length);
3260	Serge	729	FreeKernelSpace(vaddr);
3031	serge	730
		731	return ret ? -EFAULT : 0;
		732	}
3260	Serge	733	#if 0
3031	serge	734
		735	/* Only difference to the fast-path function is that this can handle bit17
		736	* and uses non-atomic copy and kmap functions. */
		737	static int
		738	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		739	char __user *user_data,
		740	bool page_do_bit17_swizzling,
		741	bool needs_clflush_before,
		742	bool needs_clflush_after)
		743	{
		744	char *vaddr;
		745	int ret;
		746
		747	vaddr = kmap(page);
		748	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		749	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		750	page_length,
		751	page_do_bit17_swizzling);
		752	if (page_do_bit17_swizzling)
		753	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		754	user_data,
		755	page_length);
		756	else
		757	ret = __copy_from_user(vaddr + shmem_page_offset,
		758	user_data,
		759	page_length);
		760	if (needs_clflush_after)
		761	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		762	page_length,
		763	page_do_bit17_swizzling);
		764	kunmap(page);
		765
		766	return ret ? -EFAULT : 0;
		767	}
3260	Serge	768	#endif
3031	serge	769
3260	Serge	770
3031	serge	771	static int
		772	i915_gem_shmem_pwrite(struct drm_device *dev,
		773	struct drm_i915_gem_object *obj,
		774	struct drm_i915_gem_pwrite *args,
		775	struct drm_file *file)
		776	{
		777	ssize_t remain;
		778	loff_t offset;
		779	char __user *user_data;
		780	int shmem_page_offset, page_length, ret = 0;
		781	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		782	int hit_slowpath = 0;
		783	int needs_clflush_after = 0;
		784	int needs_clflush_before = 0;
3746	Serge	785	struct sg_page_iter sg_iter;
3031	serge	786
3746	Serge	787	user_data = to_user_ptr(args->data_ptr);
3031	serge	788	remain = args->size;
		789
		790	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		791
		792	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		793	/* If we're not in the cpu write domain, set ourself into the gtt
		794	* write domain and manually flush cachelines (if required). This
		795	* optimizes for the case when the gpu will use the data
		796	* right away and we therefore have to clflush anyway. */
4104	Serge	797	needs_clflush_after = cpu_write_needs_clflush(obj);
		798	if (i915_gem_obj_bound_any(obj)) {
3031	serge	799	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		800	if (ret)
		801	return ret;
		802	}
		803	}
4104	Serge	804	/* Same trick applies to invalidate partially written cachelines read
		805	* before writing. */
		806	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		807	needs_clflush_before =
		808	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	809
		810	ret = i915_gem_object_get_pages(obj);
2332	Serge	811	if (ret)
3031	serge	812	return ret;
2332	Serge	813
3031	serge	814	i915_gem_object_pin_pages(obj);
2332	Serge	815
		816	offset = args->offset;
3031	serge	817	obj->dirty = 1;
2332	Serge	818
3746	Serge	819	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		820	offset >> PAGE_SHIFT) {
		821	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	822	int partial_cacheline_write;
2332	Serge	823
3031	serge	824	if (remain <= 0)
		825	break;
		826
2332	Serge	827	/* Operation in this page
		828	*
		829	* shmem_page_offset = offset within page in shmem file
		830	* page_length = bytes to copy for this page
		831	*/
		832	shmem_page_offset = offset_in_page(offset);
		833
		834	page_length = remain;
		835	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		836	page_length = PAGE_SIZE - shmem_page_offset;
		837
3031	serge	838	/* If we don't overwrite a cacheline completely we need to be
		839	* careful to have up-to-date data by first clflushing. Don't
		840	* overcomplicate things and flush the entire patch. */
		841	partial_cacheline_write = needs_clflush_before &&
		842	((shmem_page_offset \| page_length)
3260	Serge	843	& (x86_clflush_size - 1));
2332	Serge	844
3031	serge	845	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		846	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	847
3031	serge	848	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		849	user_data, page_do_bit17_swizzling,
		850	partial_cacheline_write,
		851	needs_clflush_after);
		852	if (ret == 0)
		853	goto next_page;
		854
		855	hit_slowpath = 1;
		856	mutex_unlock(&dev->struct_mutex);
3260	Serge	857	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	858
3260	Serge	859	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		860	// user_data, page_do_bit17_swizzling,
		861	// partial_cacheline_write,
		862	// needs_clflush_after);
		863
3031	serge	864	mutex_lock(&dev->struct_mutex);
		865
		866	next_page:
2332	Serge	867
3031	serge	868	if (ret)
		869	goto out;
		870
2332	Serge	871	remain -= page_length;
3031	serge	872	user_data += page_length;
2332	Serge	873	offset += page_length;
		874	}
		875
		876	out:
3031	serge	877	i915_gem_object_unpin_pages(obj);
		878
		879	if (hit_slowpath) {
3480	Serge	880	/*
		881	* Fixup: Flush cpu caches in case we didn't flush the dirty
		882	* cachelines in-line while writing and the object moved
		883	* out of the cpu write domain while we've dropped the lock.
		884	*/
		885	if (!needs_clflush_after &&
		886	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	887	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	888	i915_gem_chipset_flush(dev);
3031	serge	889	}
2332	Serge	890	}
		891
3031	serge	892	if (needs_clflush_after)
3243	Serge	893	i915_gem_chipset_flush(dev);
3031	serge	894
2332	Serge	895	return ret;
		896	}
3031	serge	897
		898	/**
		899	* Writes data to the object referenced by handle.
		900	*
		901	* On error, the contents of the buffer that were to be modified are undefined.
		902	*/
		903	int
		904	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		905	struct drm_file *file)
		906	{
		907	struct drm_i915_gem_pwrite *args = data;
		908	struct drm_i915_gem_object *obj;
		909	int ret;
		910
4104	Serge	911	if (args->size == 0)
		912	return 0;
		913
3480	Serge	914	if(args->handle == -2)
		915	{
		916	printf("%s handle %d\n", __FUNCTION__, args->handle);
		917	return 0;
		918	}
		919
3031	serge	920	ret = i915_mutex_lock_interruptible(dev);
		921	if (ret)
		922	return ret;
		923
		924	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		925	if (&obj->base == NULL) {
		926	ret = -ENOENT;
		927	goto unlock;
		928	}
		929
		930	/* Bounds check destination. */
		931	if (args->offset > obj->base.size \|\|
		932	args->size > obj->base.size - args->offset) {
		933	ret = -EINVAL;
		934	goto out;
		935	}
		936
		937	/* prime objects have no backing filp to GEM pread/pwrite
		938	* pages from.
		939	*/
		940	if (!obj->base.filp) {
		941	ret = -EINVAL;
		942	goto out;
		943	}
		944
		945	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		946
		947	ret = -EFAULT;
		948	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		949	* it would end up going through the fenced access, and we'll get
		950	* different detiling behavior between reading and writing.
		951	* pread/pwrite currently are reading and writing from the CPU
		952	* perspective, requiring manual detiling by the client.
		953	*/
3260	Serge	954	// if (obj->phys_obj) {
		955	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		956	// goto out;
		957	// }
3031	serge	958
4104	Serge	959	if (obj->tiling_mode == I915_TILING_NONE &&
		960	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		961	cpu_write_needs_clflush(obj)) {
3031	serge	962	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		963	/* Note that the gtt paths might fail with non-page-backed user
		964	* pointers (e.g. gtt mappings when moving data between
		965	* textures). Fallback to the shmem path in that case. */
		966	}
		967
		968	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	969	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	970
		971	out:
		972	drm_gem_object_unreference(&obj->base);
		973	unlock:
		974	mutex_unlock(&dev->struct_mutex);
		975	return ret;
		976	}
		977
		978	int
3480	Serge	979	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	980	bool interruptible)
		981	{
3480	Serge	982	if (i915_reset_in_progress(error)) {
3031	serge	983	/* Non-interruptible callers can't handle -EAGAIN, hence return
		984	* -EIO unconditionally for these. */
		985	if (!interruptible)
		986	return -EIO;
2332	Serge	987
3480	Serge	988	/* Recovery complete, but the reset failed ... */
		989	if (i915_terminally_wedged(error))
3031	serge	990	return -EIO;
2332	Serge	991
3031	serge	992	return -EAGAIN;
		993	}
2332	Serge	994
3031	serge	995	return 0;
		996	}
2332	Serge	997
3031	serge	998	/*
		999	* Compare seqno against outstanding lazy request. Emit a request if they are
		1000	* equal.
		1001	*/
		1002	static int
		1003	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1004	{
		1005	int ret;
2332	Serge	1006
3031	serge	1007	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1008
3031	serge	1009	ret = 0;
		1010	if (seqno == ring->outstanding_lazy_request)
4104	Serge	1011	ret = i915_add_request(ring, NULL);
2332	Serge	1012
3031	serge	1013	return ret;
		1014	}
2332	Serge	1015
3031	serge	1016	/**
		1017	* __wait_seqno - wait until execution of seqno has finished
		1018	* @ring: the ring expected to report seqno
		1019	* @seqno: duh!
3480	Serge	1020	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1021	* @interruptible: do an interruptible wait (normally yes)
		1022	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1023	*
3480	Serge	1024	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1025	* values have been read by the caller in an smp safe manner. Where read-side
		1026	* locks are involved, it is sufficient to read the reset_counter before
		1027	* unlocking the lock that protects the seqno. For lockless tricks, the
		1028	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1029	* inserted.
		1030	*
3031	serge	1031	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1032	* errno with remaining time filled in timeout argument.
		1033	*/
		1034	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1035	unsigned reset_counter,
3031	serge	1036	bool interruptible, struct timespec *timeout)
		1037	{
		1038	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1039	struct timespec before, now, wait_time={1,0};
		1040	unsigned long timeout_jiffies;
		1041	long end;
		1042	bool wait_forever = true;
		1043	int ret;
2332	Serge	1044
4104	Serge	1045	WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
		1046
3031	serge	1047	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1048	return 0;
2332	Serge	1049
3031	serge	1050	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1051
3031	serge	1052	if (timeout != NULL) {
		1053	wait_time = *timeout;
		1054	wait_forever = false;
		1055	}
2332	Serge	1056
4104	Serge	1057	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
2332	Serge	1058
3031	serge	1059	if (WARN_ON(!ring->irq_get(ring)))
		1060	return -ENODEV;
2332	Serge	1061
3031	serge	1062	/* Record current time in case interrupted by signal, or wedged * */
		1063	getrawmonotonic(&before);
2332	Serge	1064
3031	serge	1065	#define EXIT_COND \
		1066	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1067	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1068	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1069	do {
3266	Serge	1070	if (interruptible)
		1071	end = wait_event_interruptible_timeout(ring->irq_queue,
		1072	EXIT_COND,
		1073	timeout_jiffies);
		1074	else
3031	serge	1075	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1076	timeout_jiffies);
2332	Serge	1077
3480	Serge	1078	/* We need to check whether any gpu reset happened in between
		1079	* the caller grabbing the seqno and now ... */
		1080	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1081	end = -EAGAIN;
		1082
		1083	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1084	* gone. */
		1085	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1086	if (ret)
		1087	end = ret;
		1088	} while (end == 0 && wait_forever);
2332	Serge	1089
3031	serge	1090	getrawmonotonic(&now);
2332	Serge	1091
3031	serge	1092	ring->irq_put(ring);
		1093	trace_i915_gem_request_wait_end(ring, seqno);
		1094	#undef EXIT_COND
2332	Serge	1095
3031	serge	1096	if (timeout) {
4104	Serge	1097	// struct timespec sleep_time = timespec_sub(now, before);
		1098	// timeout = timespec_sub(timeout, sleep_time);
3031	serge	1099	}
2332	Serge	1100
3031	serge	1101	switch (end) {
		1102	case -EIO:
		1103	case -EAGAIN: /* Wedged */
		1104	case -ERESTARTSYS: /* Signal */
		1105	return (int)end;
		1106	case 0: /* Timeout */
		1107	return -ETIME;
		1108	default: /* Completed */
		1109	WARN_ON(end < 0); /* We're not aware of other errors */
		1110	return 0;
		1111	}
		1112	}
2332	Serge	1113
3031	serge	1114	/**
		1115	* Waits for a sequence number to be signaled, and cleans up the
		1116	* request and object lists appropriately for that event.
		1117	*/
		1118	int
		1119	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1120	{
		1121	struct drm_device *dev = ring->dev;
		1122	struct drm_i915_private *dev_priv = dev->dev_private;
		1123	bool interruptible = dev_priv->mm.interruptible;
		1124	int ret;
2332	Serge	1125
3031	serge	1126	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1127	BUG_ON(seqno == 0);
2332	Serge	1128
3480	Serge	1129	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1130	if (ret)
		1131	return ret;
2332	Serge	1132
3031	serge	1133	ret = i915_gem_check_olr(ring, seqno);
		1134	if (ret)
		1135	return ret;
2332	Serge	1136
3480	Serge	1137	return __wait_seqno(ring, seqno,
		1138	atomic_read(&dev_priv->gpu_error.reset_counter),
		1139	interruptible, NULL);
3031	serge	1140	}
2332	Serge	1141
4104	Serge	1142	static int
		1143	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
		1144	struct intel_ring_buffer *ring)
		1145	{
		1146	i915_gem_retire_requests_ring(ring);
		1147
		1148	/* Manually manage the write flush as we may have not yet
		1149	* retired the buffer.
		1150	*
		1151	* Note that the last_write_seqno is always the earlier of
		1152	* the two (read/write) seqno, so if we haved successfully waited,
		1153	* we know we have passed the last write.
		1154	*/
		1155	obj->last_write_seqno = 0;
		1156	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1157
		1158	return 0;
		1159	}
		1160
3031	serge	1161	/**
		1162	* Ensures that all rendering to the object has completed and the object is
		1163	* safe to unbind from the GTT or access from the CPU.
		1164	*/
		1165	static __must_check int
		1166	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1167	bool readonly)
		1168	{
		1169	struct intel_ring_buffer *ring = obj->ring;
		1170	u32 seqno;
		1171	int ret;
2332	Serge	1172
3031	serge	1173	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1174	if (seqno == 0)
		1175	return 0;
2332	Serge	1176
3031	serge	1177	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1178	if (ret)
		1179	return ret;
2332	Serge	1180
4104	Serge	1181	return i915_gem_object_wait_rendering__tail(obj, ring);
3031	serge	1182	}
2332	Serge	1183
3260	Serge	1184	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1185	* as the object state may change during this call.
		1186	*/
		1187	static __must_check int
		1188	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1189	bool readonly)
		1190	{
		1191	struct drm_device *dev = obj->base.dev;
		1192	struct drm_i915_private *dev_priv = dev->dev_private;
		1193	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1194	unsigned reset_counter;
3260	Serge	1195	u32 seqno;
		1196	int ret;
2332	Serge	1197
3260	Serge	1198	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1199	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1200
3260	Serge	1201	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1202	if (seqno == 0)
		1203	return 0;
2332	Serge	1204
3480	Serge	1205	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1206	if (ret)
		1207	return ret;
2332	Serge	1208
3260	Serge	1209	ret = i915_gem_check_olr(ring, seqno);
		1210	if (ret)
		1211	return ret;
2332	Serge	1212
3480	Serge	1213	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1214	mutex_unlock(&dev->struct_mutex);
3480	Serge	1215	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1216	mutex_lock(&dev->struct_mutex);
4104	Serge	1217	if (ret)
		1218	return ret;
2332	Serge	1219
4104	Serge	1220	return i915_gem_object_wait_rendering__tail(obj, ring);
3260	Serge	1221	}
2332	Serge	1222
3260	Serge	1223	/**
		1224	* Called when user space prepares to use an object with the CPU, either
		1225	* through the mmap ioctl's mapping or a GTT mapping.
		1226	*/
		1227	int
		1228	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1229	struct drm_file *file)
		1230	{
		1231	struct drm_i915_gem_set_domain *args = data;
		1232	struct drm_i915_gem_object *obj;
		1233	uint32_t read_domains = args->read_domains;
		1234	uint32_t write_domain = args->write_domain;
		1235	int ret;
2332	Serge	1236
3480	Serge	1237
		1238	if(args->handle == -2)
		1239	{
		1240	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1241	return 0;
		1242	}
		1243
3260	Serge	1244	/* Only handle setting domains to types used by the CPU. */
		1245	if (write_domain & I915_GEM_GPU_DOMAINS)
		1246	return -EINVAL;
2332	Serge	1247
3260	Serge	1248	if (read_domains & I915_GEM_GPU_DOMAINS)
		1249	return -EINVAL;
2332	Serge	1250
3260	Serge	1251	/* Having something in the write domain implies it's in the read
		1252	* domain, and only that read domain. Enforce that in the request.
		1253	*/
		1254	if (write_domain != 0 && read_domains != write_domain)
		1255	return -EINVAL;
2332	Serge	1256
3260	Serge	1257	ret = i915_mutex_lock_interruptible(dev);
		1258	if (ret)
		1259	return ret;
2332	Serge	1260
3260	Serge	1261	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1262	if (&obj->base == NULL) {
		1263	ret = -ENOENT;
		1264	goto unlock;
		1265	}
2332	Serge	1266
3260	Serge	1267	/* Try to flush the object off the GPU without holding the lock.
		1268	* We will repeat the flush holding the lock in the normal manner
		1269	* to catch cases where we are gazumped.
		1270	*/
		1271	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1272	if (ret)
		1273	goto unref;
2332	Serge	1274
3260	Serge	1275	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1276	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1277
3260	Serge	1278	/* Silently promote "you're not bound, there was nothing to do"
		1279	* to success, since the client was just asking us to
		1280	* make sure everything was done.
		1281	*/
		1282	if (ret == -EINVAL)
		1283	ret = 0;
		1284	} else {
		1285	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1286	}
2332	Serge	1287
3260	Serge	1288	unref:
		1289	drm_gem_object_unreference(&obj->base);
		1290	unlock:
		1291	mutex_unlock(&dev->struct_mutex);
		1292	return ret;
		1293	}
2332	Serge	1294
		1295
		1296
		1297
		1298
		1299
3260	Serge	1300	/**
		1301	* Maps the contents of an object, returning the address it is mapped
		1302	* into.
		1303	*
		1304	* While the mapping holds a reference on the contents of the object, it doesn't
		1305	* imply a ref on the object itself.
		1306	*/
		1307	int
		1308	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1309	struct drm_file *file)
		1310	{
		1311	struct drm_i915_gem_mmap *args = data;
		1312	struct drm_gem_object *obj;
		1313	unsigned long addr = 0;
2332	Serge	1314
3480	Serge	1315	if(args->handle == -2)
		1316	{
		1317	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1318	return 0;
		1319	}
		1320
3260	Serge	1321	obj = drm_gem_object_lookup(dev, file, args->handle);
		1322	if (obj == NULL)
		1323	return -ENOENT;
4104	Serge	1324
3260	Serge	1325	/* prime objects have no backing filp to GEM mmap
		1326	* pages from.
		1327	*/
		1328	if (!obj->filp) {
		1329	drm_gem_object_unreference_unlocked(obj);
		1330	return -EINVAL;
		1331	}
2332	Serge	1332
3263	Serge	1333	addr = vm_mmap(obj->filp, 0, args->size,
		1334	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1335	args->offset);
3260	Serge	1336	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1337	if (IS_ERR((void *)addr))
		1338	return addr;
2332	Serge	1339
3260	Serge	1340	args->addr_ptr = (uint64_t) addr;
2332	Serge	1341
3263	Serge	1342	return 0;
3260	Serge	1343	}
2332	Serge	1344
		1345
		1346
		1347
		1348
		1349
		1350
		1351
3031	serge	1352
		1353
		1354
		1355
		1356
		1357	/**
		1358	* i915_gem_release_mmap - remove physical page mappings
		1359	* @obj: obj in question
		1360	*
		1361	* Preserve the reservation of the mmapping with the DRM core code, but
		1362	* relinquish ownership of the pages back to the system.
		1363	*
		1364	* It is vital that we remove the page mapping if we have mapped a tiled
		1365	* object through the GTT and then lose the fence register due to
		1366	* resource pressure. Similarly if the object has been moved out of the
		1367	* aperture, than pages mapped into userspace must be revoked. Removing the
		1368	* mapping will then trigger a page fault on the next user access, allowing
		1369	* fixup by i915_gem_fault().
		1370	*/
		1371	void
		1372	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1373	{
		1374	if (!obj->fault_mappable)
		1375	return;
		1376
4104	Serge	1377	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1378	obj->fault_mappable = false;
		1379	}
		1380
3480	Serge	1381	uint32_t
2332	Serge	1382	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1383	{
		1384	uint32_t gtt_size;
		1385
		1386	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1387	tiling_mode == I915_TILING_NONE)
		1388	return size;
		1389
		1390	/* Previous chips need a power-of-two fence region when tiling */
		1391	if (INTEL_INFO(dev)->gen == 3)
		1392	gtt_size = 1024*1024;
		1393	else
		1394	gtt_size = 512*1024;
		1395
		1396	while (gtt_size < size)
		1397	gtt_size <<= 1;
		1398
		1399	return gtt_size;
		1400	}
		1401
		1402	/**
		1403	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1404	* @obj: object to check
		1405	*
		1406	* Return the required GTT alignment for an object, taking into account
		1407	* potential fence register mapping.
		1408	*/
3480	Serge	1409	uint32_t
		1410	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1411	int tiling_mode, bool fenced)
2332	Serge	1412	{
		1413	/*
		1414	* Minimum alignment is 4k (GTT page size), but might be greater
		1415	* if a fence register is needed for the object.
		1416	*/
3480	Serge	1417	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1418	tiling_mode == I915_TILING_NONE)
		1419	return 4096;
		1420
		1421	/*
		1422	* Previous chips need to be aligned to the size of the smallest
		1423	* fence register that can contain the object.
		1424	*/
		1425	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1426	}
		1427
		1428	/**
		1429	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1430	* unfenced object
		1431	* @dev: the device
		1432	* @size: size of the object
		1433	* @tiling_mode: tiling mode of the object
		1434	*
		1435	* Return the required GTT alignment for an object, only taking into account
		1436	* unfenced tiled surface requirements.
		1437	*/
		1438	uint32_t
		1439	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1440	uint32_t size,
		1441	int tiling_mode)
		1442	{
		1443	/*
		1444	* Minimum alignment is 4k (GTT page size) for sane hw.
		1445	*/
		1446	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1447	tiling_mode == I915_TILING_NONE)
		1448	return 4096;
		1449
		1450	/* Previous hardware however needs to be aligned to a power-of-two
		1451	* tile height. The simplest method for determining this is to reuse
		1452	* the power-of-tile object size.
		1453	*/
		1454	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1455	}
		1456
3480	Serge	1457	int
		1458	i915_gem_mmap_gtt(struct drm_file *file,
		1459	struct drm_device *dev,
		1460	uint32_t handle,
		1461	uint64_t *offset)
		1462	{
		1463	struct drm_i915_private *dev_priv = dev->dev_private;
		1464	struct drm_i915_gem_object *obj;
		1465	unsigned long pfn;
		1466	char mem, ptr;
		1467	int ret;
		1468
		1469	ret = i915_mutex_lock_interruptible(dev);
		1470	if (ret)
		1471	return ret;
		1472
		1473	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1474	if (&obj->base == NULL) {
		1475	ret = -ENOENT;
		1476	goto unlock;
		1477	}
		1478
		1479	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1480	ret = -E2BIG;
		1481	goto out;
		1482	}
		1483
		1484	if (obj->madv != I915_MADV_WILLNEED) {
		1485	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1486	ret = -EINVAL;
		1487	goto out;
		1488	}
		1489	/* Now bind it into the GTT if needed */
4104	Serge	1490	ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
3480	Serge	1491	if (ret)
		1492	goto out;
		1493
		1494	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1495	if (ret)
		1496	goto unpin;
		1497
		1498	ret = i915_gem_object_get_fence(obj);
		1499	if (ret)
		1500	goto unpin;
		1501
		1502	obj->fault_mappable = true;
		1503
4104	Serge	1504	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1505
		1506	/* Finally, remap it using the new GTT offset */
		1507
		1508	mem = UserAlloc(obj->base.size);
		1509	if(unlikely(mem == NULL))
		1510	{
		1511	ret = -ENOMEM;
		1512	goto unpin;
		1513	}
		1514
		1515	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1516	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1517
		1518	unpin:
		1519	i915_gem_object_unpin(obj);
		1520
		1521
4104	Serge	1522	*offset = mem;
3480	Serge	1523
		1524	out:
		1525	drm_gem_object_unreference(&obj->base);
		1526	unlock:
		1527	mutex_unlock(&dev->struct_mutex);
		1528	return ret;
		1529	}
		1530
		1531	/**
		1532	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1533	* @dev: DRM device
		1534	* @data: GTT mapping ioctl data
		1535	* @file: GEM object info
		1536	*
		1537	* Simply returns the fake offset to userspace so it can mmap it.
		1538	* The mmap call will end up in drm_gem_mmap(), which will set things
		1539	* up so we can get faults in the handler above.
		1540	*
		1541	* The fault handler will take care of binding the object into the GTT
		1542	* (since it may have been evicted to make room for something), allocating
		1543	* a fence register, and mapping the appropriate aperture address into
		1544	* userspace.
		1545	*/
		1546	int
		1547	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1548	struct drm_file *file)
		1549	{
		1550	struct drm_i915_gem_mmap_gtt *args = data;
		1551
		1552	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1553	}
		1554
3031	serge	1555	/* Immediately discard the backing storage */
		1556	static void
		1557	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1558	{
		1559	// struct inode *inode;
2332	Serge	1560
3031	serge	1561	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1562
3263	Serge	1563	if (obj->base.filp == NULL)
		1564	return;
2332	Serge	1565
3031	serge	1566	/* Our goal here is to return as much of the memory as
		1567	* is possible back to the system as we are called from OOM.
		1568	* To do this we must instruct the shmfs to drop all of its
		1569	* backing pages, now.
		1570	*/
		1571	// inode = obj->base.filp->f_path.dentry->d_inode;
		1572	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1573
3031	serge	1574	obj->madv = __I915_MADV_PURGED;
		1575	}
2332	Serge	1576
3031	serge	1577	static inline int
		1578	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1579	{
		1580	return obj->madv == I915_MADV_DONTNEED;
		1581	}
2332	Serge	1582
3031	serge	1583	static void
		1584	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1585	{
3746	Serge	1586	struct sg_page_iter sg_iter;
		1587	int ret;
2332	Serge	1588
3031	serge	1589	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1590
3031	serge	1591	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1592	if (ret) {
		1593	/* In the event of a disaster, abandon all caches and
		1594	* hope for the best.
		1595	*/
		1596	WARN_ON(ret != -EIO);
4104	Serge	1597	i915_gem_clflush_object(obj, true);
3031	serge	1598	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1599	}
2332	Serge	1600
3031	serge	1601	if (obj->madv == I915_MADV_DONTNEED)
		1602	obj->dirty = 0;
2332	Serge	1603
3746	Serge	1604	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1605	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1606
3290	Serge	1607	page_cache_release(page);
3243	Serge	1608	}
		1609	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1610
4104	Serge	1611	obj->dirty = 0;
3243	Serge	1612
		1613	sg_free_table(obj->pages);
		1614	kfree(obj->pages);
3031	serge	1615	}
2332	Serge	1616
3480	Serge	1617	int
3031	serge	1618	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1619	{
		1620	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1621
3243	Serge	1622	if (obj->pages == NULL)
3031	serge	1623	return 0;
2332	Serge	1624
3031	serge	1625	if (obj->pages_pin_count)
		1626	return -EBUSY;
		1627
4104	Serge	1628	BUG_ON(i915_gem_obj_bound_any(obj));
		1629
3243	Serge	1630	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1631	* array, hence protect them from being reaped by removing them from gtt
		1632	* lists early. */
4104	Serge	1633	list_del(&obj->global_list);
3243	Serge	1634
3031	serge	1635	ops->put_pages(obj);
3243	Serge	1636	obj->pages = NULL;
3031	serge	1637
		1638	if (i915_gem_object_is_purgeable(obj))
		1639	i915_gem_object_truncate(obj);
		1640
		1641	return 0;
		1642	}
		1643
		1644
		1645
		1646
		1647
		1648
		1649
		1650
2332	Serge	1651	static int
3031	serge	1652	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1653	{
3260	Serge	1654	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1655	int page_count, i;
4104	Serge	1656	struct sg_table *st;
3243	Serge	1657	struct scatterlist *sg;
3746	Serge	1658	struct sg_page_iter sg_iter;
3243	Serge	1659	struct page *page;
3746	Serge	1660	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1661	gfp_t gfp;
2332	Serge	1662
3243	Serge	1663	/* Assert that the object is not currently in any GPU domain. As it
		1664	* wasn't in the GTT, there shouldn't be any way it could have been in
		1665	* a GPU cache
2332	Serge	1666	*/
3243	Serge	1667	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1668	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1669
		1670	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1671	if (st == NULL)
		1672	return -ENOMEM;
		1673
2332	Serge	1674	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1675	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1676	kfree(st);
3746	Serge	1677	FAIL();
2332	Serge	1678	return -ENOMEM;
3243	Serge	1679	}
2332	Serge	1680
3243	Serge	1681	/* Get the list of pages out of our struct file. They'll be pinned
		1682	* at this point until we release them.
		1683	*
		1684	* Fail silently without starting the shrinker
		1685	*/
3746	Serge	1686	sg = st->sgl;
		1687	st->nents = 0;
		1688	for (i = 0; i < page_count; i++) {
4104	Serge	1689	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1690	if (IS_ERR(page)) {
		1691	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1692	goto err_pages;
		1693
3260	Serge	1694	}
3746	Serge	1695
		1696	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1697	if (i)
		1698	sg = sg_next(sg);
		1699	st->nents++;
3243	Serge	1700	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1701	} else {
		1702	sg->length += PAGE_SIZE;
		1703	}
		1704	last_pfn = page_to_pfn(page);
3243	Serge	1705	}
3031	serge	1706
3746	Serge	1707	sg_mark_end(sg);
3243	Serge	1708	obj->pages = st;
3031	serge	1709
2332	Serge	1710	return 0;
		1711
		1712	err_pages:
3746	Serge	1713	sg_mark_end(sg);
		1714	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1715	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1716	sg_free_table(st);
		1717	kfree(st);
3746	Serge	1718	FAIL();
3243	Serge	1719	return PTR_ERR(page);
2332	Serge	1720	}
		1721
3031	serge	1722	/* Ensure that the associated pages are gathered from the backing storage
		1723	* and pinned into our object. i915_gem_object_get_pages() may be called
		1724	* multiple times before they are released by a single call to
		1725	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1726	* either as a result of memory pressure (reaping pages under the shrinker)
		1727	* or as the object is itself released.
		1728	*/
		1729	int
		1730	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1731	{
3031	serge	1732	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1733	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1734	int ret;
2332	Serge	1735
3243	Serge	1736	if (obj->pages)
3031	serge	1737	return 0;
2332	Serge	1738
3031	serge	1739	BUG_ON(obj->pages_pin_count);
2332	Serge	1740
3031	serge	1741	ret = ops->get_pages(obj);
		1742	if (ret)
		1743	return ret;
2344	Serge	1744
4104	Serge	1745	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1746	return 0;
2332	Serge	1747	}
		1748
		1749	void
		1750	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1751	struct intel_ring_buffer *ring)
2332	Serge	1752	{
		1753	struct drm_device *dev = obj->base.dev;
		1754	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1755	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1756
		1757	BUG_ON(ring == NULL);
4104	Serge	1758	if (obj->ring != ring && obj->last_write_seqno) {
		1759	/* Keep the seqno relative to the current ring */
		1760	obj->last_write_seqno = seqno;
		1761	}
2332	Serge	1762	obj->ring = ring;
		1763
		1764	/* Add a reference if we're newly entering the active list. */
		1765	if (!obj->active) {
2344	Serge	1766	drm_gem_object_reference(&obj->base);
2332	Serge	1767	obj->active = 1;
		1768	}
		1769
		1770	list_move_tail(&obj->ring_list, &ring->active_list);
		1771
3031	serge	1772	obj->last_read_seqno = seqno;
		1773
2332	Serge	1774	if (obj->fenced_gpu_access) {
3031	serge	1775	obj->last_fenced_seqno = seqno;
		1776
		1777	/* Bump MRU to take account of the delayed flush */
		1778	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1779	struct drm_i915_fence_reg *reg;
		1780
		1781	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1782	list_move_tail(®->lru_list,
		1783	&dev_priv->mm.fence_list);
		1784	}
2332	Serge	1785	}
		1786	}
		1787
2344	Serge	1788	static void
3031	serge	1789	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1790	{
4104	Serge	1791	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1792	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
		1793	struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2332	Serge	1794
3031	serge	1795	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1796	BUG_ON(!obj->active);
2332	Serge	1797
4104	Serge	1798	list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2344	Serge	1799
3031	serge	1800	list_del_init(&obj->ring_list);
2352	Serge	1801	obj->ring = NULL;
2344	Serge	1802
3031	serge	1803	obj->last_read_seqno = 0;
		1804	obj->last_write_seqno = 0;
		1805	obj->base.write_domain = 0;
		1806
		1807	obj->last_fenced_seqno = 0;
2352	Serge	1808	obj->fenced_gpu_access = false;
2344	Serge	1809
2352	Serge	1810	obj->active = 0;
		1811	drm_gem_object_unreference(&obj->base);
		1812
		1813	WARN_ON(i915_verify_lists(dev));
		1814	}
		1815
3243	Serge	1816	static int
3480	Serge	1817	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1818	{
3243	Serge	1819	struct drm_i915_private *dev_priv = dev->dev_private;
		1820	struct intel_ring_buffer *ring;
		1821	int ret, i, j;
2344	Serge	1822
3480	Serge	1823	/* Carefully retire all requests without writing to the rings */
3243	Serge	1824	for_each_ring(ring, dev_priv, i) {
3480	Serge	1825	ret = intel_ring_idle(ring);
3243	Serge	1826	if (ret)
		1827	return ret;
3480	Serge	1828	}
		1829	i915_gem_retire_requests(dev);
3243	Serge	1830
3480	Serge	1831	/* Finally reset hw state */
3243	Serge	1832	for_each_ring(ring, dev_priv, i) {
3480	Serge	1833	intel_ring_init_seqno(ring, seqno);
		1834
3243	Serge	1835	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1836	ring->sync_seqno[j] = 0;
		1837	}
		1838
		1839	return 0;
2344	Serge	1840	}
		1841
3480	Serge	1842	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1843	{
		1844	struct drm_i915_private *dev_priv = dev->dev_private;
		1845	int ret;
		1846
		1847	if (seqno == 0)
		1848	return -EINVAL;
		1849
		1850	/* HWS page needs to be set less than what we
		1851	* will inject to ring
		1852	*/
		1853	ret = i915_gem_init_seqno(dev, seqno - 1);
		1854	if (ret)
		1855	return ret;
		1856
		1857	/* Carefully set the last_seqno value so that wrap
		1858	* detection still works
		1859	*/
		1860	dev_priv->next_seqno = seqno;
		1861	dev_priv->last_seqno = seqno - 1;
		1862	if (dev_priv->last_seqno == 0)
		1863	dev_priv->last_seqno--;
		1864
		1865	return 0;
		1866	}
		1867
3243	Serge	1868	int
		1869	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1870	{
3243	Serge	1871	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1872
3243	Serge	1873	/* reserve 0 for non-seqno */
		1874	if (dev_priv->next_seqno == 0) {
3480	Serge	1875	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1876	if (ret)
		1877	return ret;
		1878
		1879	dev_priv->next_seqno = 1;
		1880	}
		1881
3480	Serge	1882	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1883	return 0;
2332	Serge	1884	}
		1885
4104	Serge	1886	int __i915_add_request(struct intel_ring_buffer *ring,
2352	Serge	1887	struct drm_file *file,
4104	Serge	1888	struct drm_i915_gem_object *obj,
3031	serge	1889	u32 *out_seqno)
2352	Serge	1890	{
		1891	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1892	struct drm_i915_gem_request *request;
4104	Serge	1893	u32 request_ring_position, request_start;
2352	Serge	1894	int was_empty;
		1895	int ret;
2332	Serge	1896
4104	Serge	1897	request_start = intel_ring_get_tail(ring);
3031	serge	1898	/*
		1899	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1900	* after having emitted the batchbuffer command. Hence we need to fix
		1901	* things up similar to emitting the lazy request. The difference here
		1902	* is that the flush _must_ happen before the next request, no matter
		1903	* what.
		1904	*/
4104	Serge	1905	ret = intel_ring_flush_all_caches(ring);
		1906	if (ret)
		1907	return ret;
2332	Serge	1908
3031	serge	1909	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1910	if (request == NULL)
		1911	return -ENOMEM;
		1912
		1913
		1914	/* Record the position of the start of the request so that
		1915	* should we detect the updated seqno part-way through the
4104	Serge	1916	* GPU processing the request, we never over-estimate the
3031	serge	1917	* position of the head.
		1918	*/
4104	Serge	1919	request_ring_position = intel_ring_get_tail(ring);
3031	serge	1920
3243	Serge	1921	ret = ring->add_request(ring);
3031	serge	1922	if (ret) {
		1923	kfree(request);
4104	Serge	1924	return ret;
3031	serge	1925	}
2332	Serge	1926
3243	Serge	1927	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1928	request->ring = ring;
4104	Serge	1929	request->head = request_start;
3031	serge	1930	request->tail = request_ring_position;
4104	Serge	1931	request->ctx = ring->last_context;
		1932	request->batch_obj = obj;
		1933
		1934	/* Whilst this request exists, batch_obj will be on the
		1935	* active_list, and so will hold the active reference. Only when this
		1936	* request is retired will the the batch_obj be moved onto the
		1937	* inactive_list and lose its active reference. Hence we do not need
		1938	* to explicitly hold another reference here.
		1939	*/
		1940
		1941	if (request->ctx)
		1942	i915_gem_context_reference(request->ctx);
		1943
3031	serge	1944	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1945	was_empty = list_empty(&ring->request_list);
		1946	list_add_tail(&request->list, &ring->request_list);
3031	serge	1947	request->file_priv = NULL;
2332	Serge	1948
3263	Serge	1949	if (file) {
		1950	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1951
3263	Serge	1952	spin_lock(&file_priv->mm.lock);
		1953	request->file_priv = file_priv;
		1954	list_add_tail(&request->client_list,
		1955	&file_priv->mm.request_list);
		1956	spin_unlock(&file_priv->mm.lock);
		1957	}
		1958
		1959	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1960	ring->outstanding_lazy_request = 0;
2332	Serge	1961
4104	Serge	1962	if (!dev_priv->ums.mm_suspended) {
		1963	// i915_queue_hangcheck(ring->dev);
		1964
		1965	if (was_empty) {
2360	Serge	1966	queue_delayed_work(dev_priv->wq,
3482	Serge	1967	&dev_priv->mm.retire_work,
		1968	round_jiffies_up_relative(HZ));
4104	Serge	1969	intel_mark_busy(dev_priv->dev);
		1970	}
		1971	}
3031	serge	1972
		1973	if (out_seqno)
3243	Serge	1974	*out_seqno = request->seqno;
2352	Serge	1975	return 0;
		1976	}
2332	Serge	1977
3263	Serge	1978	static inline void
		1979	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1980	{
		1981	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1982
3263	Serge	1983	if (!file_priv)
		1984	return;
2332	Serge	1985
3263	Serge	1986	spin_lock(&file_priv->mm.lock);
		1987	if (request->file_priv) {
		1988	list_del(&request->client_list);
		1989	request->file_priv = NULL;
		1990	}
		1991	spin_unlock(&file_priv->mm.lock);
		1992	}
2332	Serge	1993
4104	Serge	1994	static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
		1995	struct i915_address_space *vm)
		1996	{
		1997	if (acthd >= i915_gem_obj_offset(obj, vm) &&
		1998	acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
		1999	return true;
		2000
		2001	return false;
		2002	}
		2003
		2004	static bool i915_head_inside_request(const u32 acthd_unmasked,
		2005	const u32 request_start,
		2006	const u32 request_end)
		2007	{
		2008	const u32 acthd = acthd_unmasked & HEAD_ADDR;
		2009
		2010	if (request_start < request_end) {
		2011	if (acthd >= request_start && acthd < request_end)
		2012	return true;
		2013	} else if (request_start > request_end) {
		2014	if (acthd >= request_start \|\| acthd < request_end)
		2015	return true;
		2016	}
		2017
		2018	return false;
		2019	}
		2020
		2021	static struct i915_address_space *
		2022	request_to_vm(struct drm_i915_gem_request *request)
		2023	{
		2024	struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
		2025	struct i915_address_space *vm;
		2026
		2027	vm = &dev_priv->gtt.base;
		2028
		2029	return vm;
		2030	}
		2031
		2032	static bool i915_request_guilty(struct drm_i915_gem_request *request,
		2033	const u32 acthd, bool *inside)
		2034	{
		2035	/* There is a possibility that unmasked head address
		2036	* pointing inside the ring, matches the batch_obj address range.
		2037	* However this is extremely unlikely.
		2038	*/
		2039	if (request->batch_obj) {
		2040	if (i915_head_inside_object(acthd, request->batch_obj,
		2041	request_to_vm(request))) {
		2042	*inside = true;
		2043	return true;
		2044	}
		2045	}
		2046
		2047	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		2048	*inside = false;
		2049	return true;
		2050	}
		2051
		2052	return false;
		2053	}
		2054
		2055	static void i915_set_reset_status(struct intel_ring_buffer *ring,
		2056	struct drm_i915_gem_request *request,
		2057	u32 acthd)
		2058	{
		2059	struct i915_ctx_hang_stats *hs = NULL;
		2060	bool inside, guilty;
		2061	unsigned long offset = 0;
		2062
		2063	/* Innocent until proven guilty */
		2064	guilty = false;
		2065
		2066	if (request->batch_obj)
		2067	offset = i915_gem_obj_offset(request->batch_obj,
		2068	request_to_vm(request));
		2069
		2070	if (ring->hangcheck.action != HANGCHECK_WAIT &&
		2071	i915_request_guilty(request, acthd, &inside)) {
		2072	DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
		2073	ring->name,
		2074	inside ? "inside" : "flushing",
		2075	offset,
		2076	request->ctx ? request->ctx->id : 0,
		2077	acthd);
		2078
		2079	guilty = true;
		2080	}
		2081
		2082	/* If contexts are disabled or this is the default context, use
		2083	* file_priv->reset_state
		2084	*/
		2085	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		2086	hs = &request->ctx->hang_stats;
		2087	else if (request->file_priv)
		2088	hs = &request->file_priv->hang_stats;
		2089
		2090	if (hs) {
		2091	if (guilty)
		2092	hs->batch_active++;
		2093	else
		2094	hs->batch_pending++;
		2095	}
		2096	}
		2097
		2098	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2099	{
		2100	list_del(&request->list);
		2101	i915_gem_request_remove_from_client(request);
		2102
		2103	if (request->ctx)
		2104	i915_gem_context_unreference(request->ctx);
		2105
		2106	kfree(request);
		2107	}
		2108
3031	serge	2109	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		2110	struct intel_ring_buffer *ring)
		2111	{
4104	Serge	2112	u32 completed_seqno;
		2113	u32 acthd;
		2114
		2115	acthd = intel_ring_get_active_head(ring);
		2116	completed_seqno = ring->get_seqno(ring, false);
		2117
3031	serge	2118	while (!list_empty(&ring->request_list)) {
		2119	struct drm_i915_gem_request *request;
2332	Serge	2120
3031	serge	2121	request = list_first_entry(&ring->request_list,
		2122	struct drm_i915_gem_request,
		2123	list);
2332	Serge	2124
4104	Serge	2125	if (request->seqno > completed_seqno)
		2126	i915_set_reset_status(ring, request, acthd);
		2127
		2128	i915_gem_free_request(request);
3031	serge	2129	}
2332	Serge	2130
3031	serge	2131	while (!list_empty(&ring->active_list)) {
		2132	struct drm_i915_gem_object *obj;
2332	Serge	2133
3031	serge	2134	obj = list_first_entry(&ring->active_list,
		2135	struct drm_i915_gem_object,
		2136	ring_list);
2332	Serge	2137
3031	serge	2138	i915_gem_object_move_to_inactive(obj);
		2139	}
		2140	}
2332	Serge	2141
3746	Serge	2142	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2143	{
		2144	struct drm_i915_private *dev_priv = dev->dev_private;
		2145	int i;
2332	Serge	2146
3031	serge	2147	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2148	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2149
		2150	/*
		2151	* Commit delayed tiling changes if we have an object still
		2152	* attached to the fence, otherwise just clear the fence.
		2153	*/
		2154	if (reg->obj) {
		2155	i915_gem_object_update_fence(reg->obj, reg,
		2156	reg->obj->tiling_mode);
		2157	} else {
		2158	i915_gem_write_fence(dev, i, NULL);
		2159	}
3031	serge	2160	}
		2161	}
2360	Serge	2162
3031	serge	2163	void i915_gem_reset(struct drm_device *dev)
		2164	{
		2165	struct drm_i915_private *dev_priv = dev->dev_private;
		2166	struct intel_ring_buffer *ring;
		2167	int i;
2360	Serge	2168
3031	serge	2169	for_each_ring(ring, dev_priv, i)
		2170	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	2171
3746	Serge	2172	i915_gem_restore_fences(dev);
3031	serge	2173	}
2360	Serge	2174
2352	Serge	2175	/**
		2176	* This function clears the request list as sequence numbers are passed.
		2177	*/
3031	serge	2178	void
2352	Serge	2179	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2180	{
		2181	uint32_t seqno;
2332	Serge	2182
2352	Serge	2183	if (list_empty(&ring->request_list))
		2184	return;
2332	Serge	2185
2352	Serge	2186	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2187
3031	serge	2188	seqno = ring->get_seqno(ring, true);
2332	Serge	2189
2352	Serge	2190	while (!list_empty(&ring->request_list)) {
		2191	struct drm_i915_gem_request *request;
2332	Serge	2192
2352	Serge	2193	request = list_first_entry(&ring->request_list,
		2194	struct drm_i915_gem_request,
		2195	list);
2332	Serge	2196
2352	Serge	2197	if (!i915_seqno_passed(seqno, request->seqno))
		2198	break;
2332	Serge	2199
2352	Serge	2200	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2201	/* We know the GPU must have read the request to have
		2202	* sent us the seqno + interrupt, so use the position
		2203	* of tail of the request to update the last known position
		2204	* of the GPU head.
		2205	*/
		2206	ring->last_retired_head = request->tail;
2332	Serge	2207
4104	Serge	2208	i915_gem_free_request(request);
2352	Serge	2209	}
2332	Serge	2210
2352	Serge	2211	/* Move any buffers on the active list that are no longer referenced
		2212	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2213	*/
		2214	while (!list_empty(&ring->active_list)) {
		2215	struct drm_i915_gem_object *obj;
2332	Serge	2216
2352	Serge	2217	obj = list_first_entry(&ring->active_list,
		2218	struct drm_i915_gem_object,
		2219	ring_list);
2332	Serge	2220
3031	serge	2221	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2222	break;
2332	Serge	2223
2352	Serge	2224	i915_gem_object_move_to_inactive(obj);
		2225	}
2332	Serge	2226
2352	Serge	2227	if (unlikely(ring->trace_irq_seqno &&
		2228	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2229	ring->irq_put(ring);
		2230	ring->trace_irq_seqno = 0;
		2231	}
2332	Serge	2232
2352	Serge	2233	WARN_ON(i915_verify_lists(ring->dev));
		2234	}
2332	Serge	2235
2352	Serge	2236	void
		2237	i915_gem_retire_requests(struct drm_device *dev)
		2238	{
		2239	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2240	struct intel_ring_buffer *ring;
2352	Serge	2241	int i;
2332	Serge	2242
3031	serge	2243	for_each_ring(ring, dev_priv, i)
		2244	i915_gem_retire_requests_ring(ring);
2352	Serge	2245	}
		2246
2360	Serge	2247	static void
		2248	i915_gem_retire_work_handler(struct work_struct *work)
		2249	{
		2250	drm_i915_private_t *dev_priv;
		2251	struct drm_device *dev;
3031	serge	2252	struct intel_ring_buffer *ring;
2360	Serge	2253	bool idle;
		2254	int i;
2352	Serge	2255
2360	Serge	2256	dev_priv = container_of(work, drm_i915_private_t,
		2257	mm.retire_work.work);
		2258	dev = dev_priv->dev;
2352	Serge	2259
2360	Serge	2260	/* Come back later if the device is busy... */
		2261	if (!mutex_trylock(&dev->struct_mutex)) {
3482	Serge	2262	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2263	round_jiffies_up_relative(HZ));
3243	Serge	2264	return;
2360	Serge	2265	}
2352	Serge	2266
2360	Serge	2267	i915_gem_retire_requests(dev);
2352	Serge	2268
2360	Serge	2269	/* Send a periodic flush down the ring so we don't hold onto GEM
		2270	* objects indefinitely.
		2271	*/
		2272	idle = true;
3031	serge	2273	for_each_ring(ring, dev_priv, i) {
		2274	if (ring->gpu_caches_dirty)
4104	Serge	2275	i915_add_request(ring, NULL);
2352	Serge	2276
2360	Serge	2277	idle &= list_empty(&ring->request_list);
		2278	}
2352	Serge	2279
4104	Serge	2280	if (!dev_priv->ums.mm_suspended && !idle)
3482	Serge	2281	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2282	round_jiffies_up_relative(HZ));
3031	serge	2283	if (idle)
		2284	intel_mark_idle(dev);
2360	Serge	2285
		2286	mutex_unlock(&dev->struct_mutex);
		2287	}
		2288
2344	Serge	2289	/**
3031	serge	2290	* Ensures that an object will eventually get non-busy by flushing any required
		2291	* write domains, emitting any outstanding lazy request and retiring and
		2292	* completed requests.
2352	Serge	2293	*/
3031	serge	2294	static int
		2295	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2296	{
3031	serge	2297	int ret;
2352	Serge	2298
3031	serge	2299	if (obj->active) {
		2300	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2301	if (ret)
		2302	return ret;
2352	Serge	2303
3031	serge	2304	i915_gem_retire_requests_ring(obj->ring);
		2305	}
2352	Serge	2306
3031	serge	2307	return 0;
		2308	}
2352	Serge	2309
3243	Serge	2310	/**
		2311	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2312	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2313	*
		2314	* Returns 0 if successful, else an error is returned with the remaining time in
		2315	* the timeout parameter.
		2316	* -ETIME: object is still busy after timeout
		2317	* -ERESTARTSYS: signal interrupted the wait
		2318	* -ENONENT: object doesn't exist
		2319	* Also possible, but rare:
		2320	* -EAGAIN: GPU wedged
		2321	* -ENOMEM: damn
		2322	* -ENODEV: Internal IRQ fail
		2323	* -E?: The add request failed
		2324	*
		2325	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2326	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2327	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2328	* without holding struct_mutex the object may become re-busied before this
		2329	* function completes. A similar but shorter * race condition exists in the busy
		2330	* ioctl
		2331	*/
4246	Serge	2332	int
		2333	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2334	{
		2335	drm_i915_private_t *dev_priv = dev->dev_private;
		2336	struct drm_i915_gem_wait *args = data;
		2337	struct drm_i915_gem_object *obj;
		2338	struct intel_ring_buffer *ring = NULL;
		2339	struct timespec timeout_stack, *timeout = NULL;
		2340	unsigned reset_counter;
		2341	u32 seqno = 0;
		2342	int ret = 0;
2352	Serge	2343
4246	Serge	2344	if (args->timeout_ns >= 0) {
		2345	timeout_stack = ns_to_timespec(args->timeout_ns);
		2346	timeout = &timeout_stack;
		2347	}
2352	Serge	2348
4246	Serge	2349	ret = i915_mutex_lock_interruptible(dev);
		2350	if (ret)
		2351	return ret;
2352	Serge	2352
4246	Serge	2353	if(args->bo_handle == -2)
		2354	{
		2355	obj = get_fb_obj();
		2356	drm_gem_object_reference(&obj->base);
		2357	}
		2358	else
		2359	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2360	if (&obj->base == NULL) {
		2361	mutex_unlock(&dev->struct_mutex);
		2362	return -ENOENT;
		2363	}
2352	Serge	2364
4246	Serge	2365	/* Need to make sure the object gets inactive eventually. */
		2366	ret = i915_gem_object_flush_active(obj);
		2367	if (ret)
		2368	goto out;
2352	Serge	2369
4246	Serge	2370	if (obj->active) {
		2371	seqno = obj->last_read_seqno;
		2372	ring = obj->ring;
		2373	}
2352	Serge	2374
4246	Serge	2375	if (seqno == 0)
		2376	goto out;
2352	Serge	2377
4246	Serge	2378	/* Do this after OLR check to make sure we make forward progress polling
		2379	* on this IOCTL with a 0 timeout (like busy ioctl)
		2380	*/
		2381	if (!args->timeout_ns) {
		2382	ret = -ETIME;
		2383	goto out;
		2384	}
2352	Serge	2385
4246	Serge	2386	drm_gem_object_unreference(&obj->base);
		2387	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		2388	mutex_unlock(&dev->struct_mutex);
2352	Serge	2389
4246	Serge	2390	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
		2391	if (timeout)
		2392	args->timeout_ns = timespec_to_ns(timeout);
		2393	return ret;
3243	Serge	2394
4246	Serge	2395	out:
		2396	drm_gem_object_unreference(&obj->base);
		2397	mutex_unlock(&dev->struct_mutex);
		2398	return ret;
		2399	}
3243	Serge	2400
2352	Serge	2401	/**
3031	serge	2402	* i915_gem_object_sync - sync an object to a ring.
		2403	*
		2404	* @obj: object which may be in use on another ring.
		2405	* @to: ring we wish to use the object on. May be NULL.
		2406	*
		2407	* This code is meant to abstract object synchronization with the GPU.
		2408	* Calling with NULL implies synchronizing the object with the CPU
		2409	* rather than a particular GPU ring.
		2410	*
		2411	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2412	*/
		2413	int
3031	serge	2414	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2415	struct intel_ring_buffer *to)
2344	Serge	2416	{
3031	serge	2417	struct intel_ring_buffer *from = obj->ring;
		2418	u32 seqno;
		2419	int ret, idx;
2332	Serge	2420
3031	serge	2421	if (from == NULL \|\| to == from)
		2422	return 0;
2332	Serge	2423
3031	serge	2424	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2425	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2426
3031	serge	2427	idx = intel_ring_sync_index(from, to);
		2428
		2429	seqno = obj->last_read_seqno;
		2430	if (seqno <= from->sync_seqno[idx])
		2431	return 0;
		2432
		2433	ret = i915_gem_check_olr(obj->ring, seqno);
		2434	if (ret)
		2435	return ret;
		2436
		2437	ret = to->sync_to(to, from, seqno);
		2438	if (!ret)
3243	Serge	2439	/* We use last_read_seqno because sync_to()
		2440	* might have just caused seqno wrap under
		2441	* the radar.
		2442	*/
		2443	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2444
		2445	return ret;
2344	Serge	2446	}
2332	Serge	2447
2344	Serge	2448	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2449	{
		2450	u32 old_write_domain, old_read_domains;
2332	Serge	2451
2344	Serge	2452	/* Force a pagefault for domain tracking on next user access */
		2453	// i915_gem_release_mmap(obj);
2332	Serge	2454
2344	Serge	2455	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2456	return;
2332	Serge	2457
3480	Serge	2458	/* Wait for any direct GTT access to complete */
		2459	mb();
		2460
2344	Serge	2461	old_read_domains = obj->base.read_domains;
		2462	old_write_domain = obj->base.write_domain;
2351	Serge	2463
2344	Serge	2464	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2465	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2466
2351	Serge	2467	trace_i915_gem_object_change_domain(obj,
		2468	old_read_domains,
		2469	old_write_domain);
2344	Serge	2470	}
2332	Serge	2471
4104	Serge	2472	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2473	{
4104	Serge	2474	struct drm_i915_gem_object *obj = vma->obj;
3031	serge	2475	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2476	int ret;
2332	Serge	2477
3263	Serge	2478	if(obj == get_fb_obj())
		2479	return 0;
		2480
4104	Serge	2481	if (list_empty(&vma->vma_link))
2344	Serge	2482	return 0;
2332	Serge	2483
4104	Serge	2484	if (!drm_mm_node_allocated(&vma->node))
		2485	goto destroy;
		2486
3031	serge	2487	if (obj->pin_count)
		2488	return -EBUSY;
2332	Serge	2489
3243	Serge	2490	BUG_ON(obj->pages == NULL);
3031	serge	2491
2344	Serge	2492	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2493	if (ret)
2344	Serge	2494	return ret;
		2495	/* Continue on if we fail due to EIO, the GPU is hung so we
		2496	* should be safe and we need to cleanup or else we might
		2497	* cause memory corruption through use-after-free.
		2498	*/
2332	Serge	2499
2344	Serge	2500	i915_gem_object_finish_gtt(obj);
2332	Serge	2501
2344	Serge	2502	/* release the fence reg _after_ flushing */
		2503	ret = i915_gem_object_put_fence(obj);
3031	serge	2504	if (ret)
2344	Serge	2505	return ret;
2332	Serge	2506
4104	Serge	2507	trace_i915_vma_unbind(vma);
2332	Serge	2508
3031	serge	2509	if (obj->has_global_gtt_mapping)
3243	Serge	2510	i915_gem_gtt_unbind_object(obj);
3031	serge	2511	if (obj->has_aliasing_ppgtt_mapping) {
		2512	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2513	obj->has_aliasing_ppgtt_mapping = 0;
		2514	}
		2515	i915_gem_gtt_finish_object(obj);
4104	Serge	2516	i915_gem_object_unpin_pages(obj);
2332	Serge	2517
4104	Serge	2518	list_del(&vma->mm_list);
2344	Serge	2519	/* Avoid an unnecessary call to unbind on rebind. */
4104	Serge	2520	if (i915_is_ggtt(vma->vm))
2344	Serge	2521	obj->map_and_fenceable = true;
2332	Serge	2522
4104	Serge	2523	drm_mm_remove_node(&vma->node);
2332	Serge	2524
4104	Serge	2525	destroy:
		2526	i915_gem_vma_destroy(vma);
		2527
		2528	/* Since the unbound list is global, only move to that list if
		2529	* no more VMAs exist.
		2530	* NB: Until we have real VMAs there will only ever be one */
		2531	WARN_ON(!list_empty(&obj->vma_list));
		2532	if (list_empty(&obj->vma_list))
		2533	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2534
2344	Serge	2535	return 0;
		2536	}
2332	Serge	2537
4104	Serge	2538	/**
		2539	* Unbinds an object from the global GTT aperture.
		2540	*/
		2541	int
		2542	i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
		2543	{
		2544	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2545	struct i915_address_space *ggtt = &dev_priv->gtt.base;
		2546
		2547	if (!i915_gem_obj_ggtt_bound(obj))
		2548	return 0;
		2549
		2550	if (obj->pin_count)
		2551	return -EBUSY;
		2552
		2553	BUG_ON(obj->pages == NULL);
		2554
		2555	return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
		2556	}
		2557
3031	serge	2558	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2559	{
		2560	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2561	struct intel_ring_buffer *ring;
2344	Serge	2562	int ret, i;
2332	Serge	2563
2344	Serge	2564	/* Flush everything onto the inactive list. */
3031	serge	2565	for_each_ring(ring, dev_priv, i) {
		2566	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2567	if (ret)
		2568	return ret;
3031	serge	2569
3243	Serge	2570	ret = intel_ring_idle(ring);
3031	serge	2571	if (ret)
		2572	return ret;
2344	Serge	2573	}
2332	Serge	2574
2344	Serge	2575	return 0;
		2576	}
2332	Serge	2577
3480	Serge	2578	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2579	struct drm_i915_gem_object *obj)
		2580	{
		2581	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2582	int fence_reg;
		2583	int fence_pitch_shift;
2332	Serge	2584
3480	Serge	2585	if (INTEL_INFO(dev)->gen >= 6) {
		2586	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2587	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2588	} else {
		2589	fence_reg = FENCE_REG_965_0;
		2590	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2591	}
2332	Serge	2592
4104	Serge	2593	fence_reg += reg * 8;
		2594
		2595	/* To w/a incoherency with non-atomic 64-bit register updates,
		2596	* we split the 64-bit update into two 32-bit writes. In order
		2597	* for a partial fence not to be evaluated between writes, we
		2598	* precede the update with write to turn off the fence register,
		2599	* and only enable the fence as the last step.
		2600	*
		2601	* For extra levels of paranoia, we make sure each step lands
		2602	* before applying the next step.
		2603	*/
		2604	I915_WRITE(fence_reg, 0);
		2605	POSTING_READ(fence_reg);
		2606
3031	serge	2607	if (obj) {
4104	Serge	2608	u32 size = i915_gem_obj_ggtt_size(obj);
		2609	uint64_t val;
2332	Serge	2610
4104	Serge	2611	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2612	0xfffff000) << 32;
4104	Serge	2613	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2614	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2615	if (obj->tiling_mode == I915_TILING_Y)
		2616	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2617	val \|= I965_FENCE_REG_VALID;
2332	Serge	2618
4104	Serge	2619	I915_WRITE(fence_reg + 4, val >> 32);
		2620	POSTING_READ(fence_reg + 4);
		2621
		2622	I915_WRITE(fence_reg + 0, val);
4280	Serge	2623
		2624	dbgprintf("%s val %x%x\n",__FUNCTION__, (int)(val >> 32), (int)val);
		2625
3480	Serge	2626	POSTING_READ(fence_reg);
4104	Serge	2627	} else {
		2628	I915_WRITE(fence_reg + 4, 0);
		2629	POSTING_READ(fence_reg + 4);
		2630	}
3031	serge	2631	}
2332	Serge	2632
3031	serge	2633	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2634	struct drm_i915_gem_object *obj)
		2635	{
		2636	drm_i915_private_t *dev_priv = dev->dev_private;
		2637	u32 val;
2332	Serge	2638
3031	serge	2639	if (obj) {
4104	Serge	2640	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2641	int pitch_val;
		2642	int tile_width;
2332	Serge	2643
4104	Serge	2644	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2645	(size & -size) != size \|\|
4104	Serge	2646	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2647	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2648	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2649
3031	serge	2650	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2651	tile_width = 128;
		2652	else
		2653	tile_width = 512;
2332	Serge	2654
3031	serge	2655	/* Note: pitch better be a power of two tile widths */
		2656	pitch_val = obj->stride / tile_width;
		2657	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2658
4104	Serge	2659	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2660	if (obj->tiling_mode == I915_TILING_Y)
		2661	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2662	val \|= I915_FENCE_SIZE_BITS(size);
		2663	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2664	val \|= I830_FENCE_REG_VALID;
		2665	} else
		2666	val = 0;
2332	Serge	2667
3031	serge	2668	if (reg < 8)
		2669	reg = FENCE_REG_830_0 + reg * 4;
		2670	else
		2671	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2672
3031	serge	2673	I915_WRITE(reg, val);
		2674	POSTING_READ(reg);
		2675	}
2332	Serge	2676
3031	serge	2677	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2678	struct drm_i915_gem_object *obj)
		2679	{
		2680	drm_i915_private_t *dev_priv = dev->dev_private;
		2681	uint32_t val;
2344	Serge	2682
3031	serge	2683	if (obj) {
4104	Serge	2684	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2685	uint32_t pitch_val;
2344	Serge	2686
4104	Serge	2687	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2688	(size & -size) != size \|\|
4104	Serge	2689	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2690	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2691	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2692
3031	serge	2693	pitch_val = obj->stride / 128;
		2694	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2695
4104	Serge	2696	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2697	if (obj->tiling_mode == I915_TILING_Y)
		2698	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2699	val \|= I830_FENCE_SIZE_BITS(size);
		2700	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2701	val \|= I830_FENCE_REG_VALID;
		2702	} else
		2703	val = 0;
		2704
		2705	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2706	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2707	}
		2708
3480	Serge	2709	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2710	{
		2711	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2712	}
		2713
3031	serge	2714	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2715	struct drm_i915_gem_object *obj)
2332	Serge	2716	{
3480	Serge	2717	struct drm_i915_private *dev_priv = dev->dev_private;
		2718
		2719	/* Ensure that all CPU reads are completed before installing a fence
		2720	* and all writes before removing the fence.
		2721	*/
		2722	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2723	mb();
		2724
4104	Serge	2725	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2726	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2727	obj->stride, obj->tiling_mode);
		2728
3031	serge	2729	switch (INTEL_INFO(dev)->gen) {
		2730	case 7:
3480	Serge	2731	case 6:
3031	serge	2732	case 5:
		2733	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2734	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2735	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2736	default: BUG();
3031	serge	2737	}
3480	Serge	2738
		2739	/* And similarly be paranoid that no direct access to this region
		2740	* is reordered to before the fence is installed.
		2741	*/
		2742	if (i915_gem_object_needs_mb(obj))
		2743	mb();
2344	Serge	2744	}
		2745
3031	serge	2746	static inline int fence_number(struct drm_i915_private *dev_priv,
		2747	struct drm_i915_fence_reg *fence)
2344	Serge	2748	{
3031	serge	2749	return fence - dev_priv->fence_regs;
		2750	}
2332	Serge	2751
3031	serge	2752	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2753	struct drm_i915_fence_reg *fence,
		2754	bool enable)
		2755	{
4104	Serge	2756	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2757	int reg = fence_number(dev_priv, fence);
2332	Serge	2758
4104	Serge	2759	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2760
		2761	if (enable) {
4104	Serge	2762	obj->fence_reg = reg;
3031	serge	2763	fence->obj = obj;
		2764	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2765	} else {
		2766	obj->fence_reg = I915_FENCE_REG_NONE;
		2767	fence->obj = NULL;
		2768	list_del_init(&fence->lru_list);
2344	Serge	2769	}
4104	Serge	2770	obj->fence_dirty = false;
3031	serge	2771	}
2344	Serge	2772
3031	serge	2773	static int
3480	Serge	2774	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2775	{
		2776	if (obj->last_fenced_seqno) {
		2777	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2778	if (ret)
		2779	return ret;
2344	Serge	2780
		2781	obj->last_fenced_seqno = 0;
		2782	}
		2783
3031	serge	2784	obj->fenced_gpu_access = false;
2332	Serge	2785	return 0;
		2786	}
		2787
		2788	int
2344	Serge	2789	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2790	{
3031	serge	2791	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2792	struct drm_i915_fence_reg *fence;
2332	Serge	2793	int ret;
		2794
3480	Serge	2795	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2796	if (ret)
		2797	return ret;
		2798
3031	serge	2799	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2800	return 0;
2332	Serge	2801
3746	Serge	2802	fence = &dev_priv->fence_regs[obj->fence_reg];
		2803
3031	serge	2804	i915_gem_object_fence_lost(obj);
3746	Serge	2805	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2806
2332	Serge	2807	return 0;
		2808	}
		2809
3031	serge	2810	static struct drm_i915_fence_reg *
		2811	i915_find_fence_reg(struct drm_device *dev)
		2812	{
		2813	struct drm_i915_private *dev_priv = dev->dev_private;
		2814	struct drm_i915_fence_reg reg, avail;
		2815	int i;
2332	Serge	2816
3031	serge	2817	/* First try to find a free reg */
		2818	avail = NULL;
		2819	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2820	reg = &dev_priv->fence_regs[i];
		2821	if (!reg->obj)
		2822	return reg;
2332	Serge	2823
3031	serge	2824	if (!reg->pin_count)
		2825	avail = reg;
		2826	}
2332	Serge	2827
3031	serge	2828	if (avail == NULL)
		2829	return NULL;
2332	Serge	2830
3031	serge	2831	/* None available, try to steal one or wait for a user to finish */
		2832	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2833	if (reg->pin_count)
		2834	continue;
2332	Serge	2835
3031	serge	2836	return reg;
		2837	}
2332	Serge	2838
3031	serge	2839	return NULL;
		2840	}
2332	Serge	2841
3031	serge	2842	/**
		2843	* i915_gem_object_get_fence - set up fencing for an object
		2844	* @obj: object to map through a fence reg
		2845	*
		2846	* When mapping objects through the GTT, userspace wants to be able to write
		2847	* to them without having to worry about swizzling if the object is tiled.
		2848	* This function walks the fence regs looking for a free one for @obj,
		2849	* stealing one if it can't find any.
		2850	*
		2851	* It then sets up the reg based on the object's properties: address, pitch
		2852	* and tiling format.
		2853	*
		2854	* For an untiled surface, this removes any existing fence.
		2855	*/
		2856	int
		2857	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2858	{
		2859	struct drm_device *dev = obj->base.dev;
		2860	struct drm_i915_private *dev_priv = dev->dev_private;
		2861	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2862	struct drm_i915_fence_reg *reg;
		2863	int ret;
2332	Serge	2864
3031	serge	2865	/* Have we updated the tiling parameters upon the object and so
		2866	* will need to serialise the write to the associated fence register?
		2867	*/
		2868	if (obj->fence_dirty) {
3480	Serge	2869	ret = i915_gem_object_wait_fence(obj);
3031	serge	2870	if (ret)
		2871	return ret;
		2872	}
2332	Serge	2873
3031	serge	2874	/* Just update our place in the LRU if our fence is getting reused. */
		2875	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2876	reg = &dev_priv->fence_regs[obj->fence_reg];
		2877	if (!obj->fence_dirty) {
		2878	list_move_tail(®->lru_list,
		2879	&dev_priv->mm.fence_list);
		2880	return 0;
		2881	}
		2882	} else if (enable) {
		2883	reg = i915_find_fence_reg(dev);
		2884	if (reg == NULL)
		2885	return -EDEADLK;
2332	Serge	2886
3031	serge	2887	if (reg->obj) {
		2888	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2889
3480	Serge	2890	ret = i915_gem_object_wait_fence(old);
3031	serge	2891	if (ret)
		2892	return ret;
2332	Serge	2893
3031	serge	2894	i915_gem_object_fence_lost(old);
		2895	}
		2896	} else
		2897	return 0;
2332	Serge	2898
3031	serge	2899	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2900
3031	serge	2901	return 0;
		2902	}
2332	Serge	2903
3031	serge	2904	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2905	struct drm_mm_node *gtt_space,
		2906	unsigned long cache_level)
		2907	{
		2908	struct drm_mm_node *other;
2332	Serge	2909
3031	serge	2910	/* On non-LLC machines we have to be careful when putting differing
		2911	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2912	* crossing memory domains and dying.
3031	serge	2913	*/
		2914	if (HAS_LLC(dev))
		2915	return true;
2332	Serge	2916
4104	Serge	2917	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2918	return true;
2332	Serge	2919
3031	serge	2920	if (list_empty(>t_space->node_list))
		2921	return true;
2332	Serge	2922
3031	serge	2923	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2924	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2925	return false;
2344	Serge	2926
3031	serge	2927	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2928	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2929	return false;
2344	Serge	2930
3031	serge	2931	return true;
		2932	}
2344	Serge	2933
3031	serge	2934	static void i915_gem_verify_gtt(struct drm_device *dev)
		2935	{
		2936	#if WATCH_GTT
		2937	struct drm_i915_private *dev_priv = dev->dev_private;
		2938	struct drm_i915_gem_object *obj;
		2939	int err = 0;
2344	Serge	2940
4104	Serge	2941	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031	serge	2942	if (obj->gtt_space == NULL) {
		2943	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2944	err++;
		2945	continue;
		2946	}
2344	Serge	2947
3031	serge	2948	if (obj->cache_level != obj->gtt_space->color) {
		2949	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
4104	Serge	2950	i915_gem_obj_ggtt_offset(obj),
		2951	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2952	obj->cache_level,
		2953	obj->gtt_space->color);
		2954	err++;
		2955	continue;
		2956	}
2344	Serge	2957
3031	serge	2958	if (!i915_gem_valid_gtt_space(dev,
		2959	obj->gtt_space,
		2960	obj->cache_level)) {
		2961	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
4104	Serge	2962	i915_gem_obj_ggtt_offset(obj),
		2963	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2964	obj->cache_level);
		2965	err++;
		2966	continue;
		2967	}
		2968	}
2344	Serge	2969
3031	serge	2970	WARN_ON(err);
		2971	#endif
2326	Serge	2972	}
		2973
2332	Serge	2974	/**
		2975	* Finds free space in the GTT aperture and binds the object there.
		2976	*/
		2977	static int
4104	Serge	2978	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2979	struct i915_address_space *vm,
2332	Serge	2980	unsigned alignment,
3031	serge	2981	bool map_and_fenceable,
		2982	bool nonblocking)
2332	Serge	2983	{
		2984	struct drm_device *dev = obj->base.dev;
		2985	drm_i915_private_t *dev_priv = dev->dev_private;
		2986	u32 size, fence_size, fence_alignment, unfenced_alignment;
4104	Serge	2987	size_t gtt_max =
		2988	map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
		2989	struct i915_vma *vma;
2332	Serge	2990	int ret;
2326	Serge	2991
2332	Serge	2992	fence_size = i915_gem_get_gtt_size(dev,
		2993	obj->base.size,
		2994	obj->tiling_mode);
		2995	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2996	obj->base.size,
3480	Serge	2997	obj->tiling_mode, true);
2332	Serge	2998	unfenced_alignment =
3480	Serge	2999	i915_gem_get_gtt_alignment(dev,
2332	Serge	3000	obj->base.size,
3480	Serge	3001	obj->tiling_mode, false);
2332	Serge	3002
		3003	if (alignment == 0)
		3004	alignment = map_and_fenceable ? fence_alignment :
		3005	unfenced_alignment;
		3006	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		3007	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		3008	return -EINVAL;
		3009	}
		3010
		3011	size = map_and_fenceable ? fence_size : obj->base.size;
		3012
		3013	/* If the object is bigger than the entire aperture, reject it early
		3014	* before evicting everything in a vain attempt to find space.
		3015	*/
4104	Serge	3016	if (obj->base.size > gtt_max) {
		3017	DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
		3018	obj->base.size,
		3019	map_and_fenceable ? "mappable" : "total",
		3020	gtt_max);
2332	Serge	3021	return -E2BIG;
		3022	}
		3023
3031	serge	3024	ret = i915_gem_object_get_pages(obj);
		3025	if (ret)
		3026	return ret;
		3027
3243	Serge	3028	i915_gem_object_pin_pages(obj);
		3029
4104	Serge	3030	BUG_ON(!i915_is_ggtt(vm));
		3031
		3032	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
		3033	if (IS_ERR(vma)) {
		3034	ret = PTR_ERR(vma);
		3035	goto err_unpin;
3243	Serge	3036	}
		3037
4104	Serge	3038	/* For now we only ever use 1 vma per object */
		3039	WARN_ON(!list_is_singular(&obj->vma_list));
		3040
		3041	search_free:
		3042	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3043	size, alignment,
		3044	obj->cache_level, 0, gtt_max,
		3045	DRM_MM_SEARCH_DEFAULT);
3243	Serge	3046	if (ret) {
2332	Serge	3047
4104	Serge	3048	goto err_free_vma;
2332	Serge	3049	}
4104	Serge	3050	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
		3051	obj->cache_level))) {
		3052	ret = -EINVAL;
		3053	goto err_remove_node;
3031	serge	3054	}
2332	Serge	3055
3031	serge	3056	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	3057	if (ret)
		3058	goto err_remove_node;
2332	Serge	3059
4104	Serge	3060	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3061	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3062
4104	Serge	3063	if (i915_is_ggtt(vm)) {
		3064	bool mappable, fenceable;
2332	Serge	3065
4104	Serge	3066	fenceable = (vma->node.size == fence_size &&
		3067	(vma->node.start & (fence_alignment - 1)) == 0);
2332	Serge	3068
4104	Serge	3069	mappable = (vma->node.start + obj->base.size <=
		3070	dev_priv->gtt.mappable_end);
2332	Serge	3071
		3072	obj->map_and_fenceable = mappable && fenceable;
4104	Serge	3073	}
2332	Serge	3074
4104	Serge	3075	WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
		3076
		3077	trace_i915_vma_bind(vma, map_and_fenceable);
3031	serge	3078	i915_gem_verify_gtt(dev);
2332	Serge	3079	return 0;
4104	Serge	3080
		3081	err_remove_node:
		3082	drm_mm_remove_node(&vma->node);
		3083	err_free_vma:
		3084	i915_gem_vma_destroy(vma);
		3085	err_unpin:
		3086	i915_gem_object_unpin_pages(obj);
		3087	return ret;
2332	Serge	3088	}
		3089
4104	Serge	3090	bool
		3091	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3092	bool force)
2332	Serge	3093	{
		3094	/* If we don't have a page list set up, then we're not pinned
		3095	* to GPU, and we can ignore the cache flush because it'll happen
		3096	* again at bind time.
		3097	*/
3243	Serge	3098	if (obj->pages == NULL)
4104	Serge	3099	return false;
2332	Serge	3100
3480	Serge	3101	/*
		3102	* Stolen memory is always coherent with the GPU as it is explicitly
		3103	* marked as wc by the system, or the system is cache-coherent.
		3104	*/
		3105	if (obj->stolen)
4104	Serge	3106	return false;
3480	Serge	3107
2332	Serge	3108	/* If the GPU is snooping the contents of the CPU cache,
		3109	* we do not need to manually clear the CPU cache lines. However,
		3110	* the caches are only snooped when the render cache is
		3111	* flushed/invalidated. As we always have to emit invalidations
		3112	* and flushes when moving into and out of the RENDER domain, correct
		3113	* snooping behaviour occurs naturally as the result of our domain
		3114	* tracking.
		3115	*/
4104	Serge	3116	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3117	return false;
3243	Serge	3118	#if 0
2344	Serge	3119	if(obj->mapped != NULL)
		3120	{
		3121	uint8_t *page_virtual;
		3122	unsigned int i;
2332	Serge	3123
2344	Serge	3124	page_virtual = obj->mapped;
		3125	asm volatile("mfence");
		3126	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		3127	clflush(page_virtual + i);
		3128	asm volatile("mfence");
		3129	}
		3130	else
		3131	{
		3132	uint8_t *page_virtual;
		3133	unsigned int i;
		3134	page_virtual = AllocKernelSpace(obj->base.size);
		3135	if(page_virtual != NULL)
		3136	{
3031	serge	3137	dma_addr_t src, dst;
2344	Serge	3138	u32 count;
		3139
		3140	#define page_tabs 0xFDC00000 /* really dirty hack */
		3141
3031	serge	3142	src = obj->pages.page;
		3143	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	3144	count = obj->base.size/4096;
		3145
		3146	while(count--)
		3147	{
		3148	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		3149	};
		3150
		3151	asm volatile("mfence");
		3152	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		3153	clflush(page_virtual + i);
		3154	asm volatile("mfence");
		3155	FreeKernelSpace(page_virtual);
		3156	}
		3157	else
		3158	{
		3159	asm volatile (
		3160	"mfence \n"
		3161	"wbinvd \n" /* this is really ugly */
		3162	"mfence");
		3163	}
		3164	}
3243	Serge	3165	#endif
		3166
4104	Serge	3167	return true;
2332	Serge	3168	}
		3169
2344	Serge	3170	/** Flushes the GTT write domain for the object if it's dirty. */
		3171	static void
		3172	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3173	{
		3174	uint32_t old_write_domain;
2332	Serge	3175
2344	Serge	3176	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3177	return;
2332	Serge	3178
2344	Serge	3179	/* No actual flushing is required for the GTT write domain. Writes
		3180	* to it immediately go to main memory as far as we know, so there's
		3181	* no chipset flush. It also doesn't land in render cache.
		3182	*
		3183	* However, we do have to enforce the order so that all writes through
		3184	* the GTT land before any writes to the device, such as updates to
		3185	* the GATT itself.
		3186	*/
		3187	wmb();
2332	Serge	3188
2344	Serge	3189	old_write_domain = obj->base.write_domain;
		3190	obj->base.write_domain = 0;
2332	Serge	3191
2351	Serge	3192	trace_i915_gem_object_change_domain(obj,
		3193	obj->base.read_domains,
		3194	old_write_domain);
2344	Serge	3195	}
2332	Serge	3196
		3197	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3198	static void
4104	Serge	3199	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3200	bool force)
2332	Serge	3201	{
		3202	uint32_t old_write_domain;
		3203
		3204	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3205	return;
		3206
4104	Serge	3207	if (i915_gem_clflush_object(obj, force))
3243	Serge	3208	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3209
2332	Serge	3210	old_write_domain = obj->base.write_domain;
		3211	obj->base.write_domain = 0;
		3212
2351	Serge	3213	trace_i915_gem_object_change_domain(obj,
		3214	obj->base.read_domains,
		3215	old_write_domain);
2332	Serge	3216	}
		3217
		3218	/**
		3219	* Moves a single object to the GTT read, and possibly write domain.
		3220	*
		3221	* This function returns when the move is complete, including waiting on
		3222	* flushes to occur.
		3223	*/
		3224	int
		3225	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3226	{
3031	serge	3227	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	3228	uint32_t old_write_domain, old_read_domains;
		3229	int ret;
		3230
		3231	/* Not valid to be called on unbound objects. */
4104	Serge	3232	if (!i915_gem_obj_bound_any(obj))
2332	Serge	3233	return -EINVAL;
		3234
		3235	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3236	return 0;
		3237
3031	serge	3238	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3239	if (ret)
		3240	return ret;
		3241
4104	Serge	3242	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3243
3480	Serge	3244	/* Serialise direct access to this object with the barriers for
		3245	* coherent writes from the GPU, by effectively invalidating the
		3246	* GTT domain upon first access.
		3247	*/
		3248	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3249	mb();
		3250
2332	Serge	3251	old_write_domain = obj->base.write_domain;
		3252	old_read_domains = obj->base.read_domains;
		3253
		3254	/* It should now be out of any other write domains, and we can update
		3255	* the domain values for our changes.
		3256	*/
		3257	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3258	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3259	if (write) {
		3260	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3261	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3262	obj->dirty = 1;
		3263	}
		3264
2351	Serge	3265	trace_i915_gem_object_change_domain(obj,
		3266	old_read_domains,
		3267	old_write_domain);
		3268
3031	serge	3269	/* And bump the LRU for this access */
4104	Serge	3270	if (i915_gem_object_is_inactive(obj)) {
		3271	struct i915_vma *vma = i915_gem_obj_to_vma(obj,
		3272	&dev_priv->gtt.base);
		3273	if (vma)
		3274	list_move_tail(&vma->mm_list,
		3275	&dev_priv->gtt.base.inactive_list);
3031	serge	3276
4104	Serge	3277	}
		3278
2332	Serge	3279	return 0;
		3280	}
		3281
2335	Serge	3282	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3283	enum i915_cache_level cache_level)
		3284	{
3031	serge	3285	struct drm_device *dev = obj->base.dev;
		3286	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3287	struct i915_vma *vma;
2335	Serge	3288	int ret;
2332	Serge	3289
2335	Serge	3290	if (obj->cache_level == cache_level)
		3291	return 0;
2332	Serge	3292
2335	Serge	3293	if (obj->pin_count) {
		3294	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3295	return -EBUSY;
		3296	}
2332	Serge	3297
4104	Serge	3298	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3299	if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
		3300	ret = i915_vma_unbind(vma);
3031	serge	3301	if (ret)
		3302	return ret;
4104	Serge	3303
		3304	break;
		3305	}
3031	serge	3306	}
		3307
4104	Serge	3308	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3309	ret = i915_gem_object_finish_gpu(obj);
		3310	if (ret)
		3311	return ret;
2332	Serge	3312
2335	Serge	3313	i915_gem_object_finish_gtt(obj);
2332	Serge	3314
2335	Serge	3315	/* Before SandyBridge, you could not use tiling or fence
		3316	* registers with snooped memory, so relinquish any fences
		3317	* currently pointing to our region in the aperture.
		3318	*/
3031	serge	3319	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3320	ret = i915_gem_object_put_fence(obj);
		3321	if (ret)
		3322	return ret;
		3323	}
2332	Serge	3324
3031	serge	3325	if (obj->has_global_gtt_mapping)
		3326	i915_gem_gtt_bind_object(obj, cache_level);
		3327	if (obj->has_aliasing_ppgtt_mapping)
		3328	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3329	obj, cache_level);
2335	Serge	3330	}
2332	Serge	3331
4104	Serge	3332	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3333	vma->node.color = cache_level;
		3334	obj->cache_level = cache_level;
		3335
		3336	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3337	u32 old_read_domains, old_write_domain;
2332	Serge	3338
2335	Serge	3339	/* If we're coming from LLC cached, then we haven't
		3340	* actually been tracking whether the data is in the
		3341	* CPU cache or not, since we only allow one bit set
		3342	* in obj->write_domain and have been skipping the clflushes.
		3343	* Just set it to the CPU cache for now.
		3344	*/
		3345	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3346
2335	Serge	3347	old_read_domains = obj->base.read_domains;
		3348	old_write_domain = obj->base.write_domain;
2332	Serge	3349
2335	Serge	3350	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3351	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3352
2351	Serge	3353	trace_i915_gem_object_change_domain(obj,
		3354	old_read_domains,
		3355	old_write_domain);
2344	Serge	3356	}
2332	Serge	3357
3031	serge	3358	i915_gem_verify_gtt(dev);
2335	Serge	3359	return 0;
		3360	}
2332	Serge	3361
3260	Serge	3362	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3363	struct drm_file *file)
		3364	{
		3365	struct drm_i915_gem_caching *args = data;
		3366	struct drm_i915_gem_object *obj;
		3367	int ret;
		3368
3480	Serge	3369	if(args->handle == -2)
		3370	{
		3371	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3372	return 0;
		3373	}
		3374
3260	Serge	3375	ret = i915_mutex_lock_interruptible(dev);
		3376	if (ret)
		3377	return ret;
		3378
		3379	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3380	if (&obj->base == NULL) {
		3381	ret = -ENOENT;
		3382	goto unlock;
		3383	}
		3384
4104	Serge	3385	switch (obj->cache_level) {
		3386	case I915_CACHE_LLC:
		3387	case I915_CACHE_L3_LLC:
		3388	args->caching = I915_CACHING_CACHED;
		3389	break;
3260	Serge	3390
4104	Serge	3391	case I915_CACHE_WT:
		3392	args->caching = I915_CACHING_DISPLAY;
		3393	break;
		3394
		3395	default:
		3396	args->caching = I915_CACHING_NONE;
		3397	break;
		3398	}
		3399
3260	Serge	3400	drm_gem_object_unreference(&obj->base);
		3401	unlock:
		3402	mutex_unlock(&dev->struct_mutex);
		3403	return ret;
		3404	}
		3405
		3406	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3407	struct drm_file *file)
		3408	{
		3409	struct drm_i915_gem_caching *args = data;
		3410	struct drm_i915_gem_object *obj;
		3411	enum i915_cache_level level;
		3412	int ret;
		3413
3480	Serge	3414	if(args->handle == -2)
		3415	{
		3416	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3417	return 0;
		3418	}
		3419
3260	Serge	3420	switch (args->caching) {
		3421	case I915_CACHING_NONE:
		3422	level = I915_CACHE_NONE;
		3423	break;
		3424	case I915_CACHING_CACHED:
		3425	level = I915_CACHE_LLC;
		3426	break;
4104	Serge	3427	case I915_CACHING_DISPLAY:
		3428	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3429	break;
3260	Serge	3430	default:
		3431	return -EINVAL;
		3432	}
		3433
		3434	ret = i915_mutex_lock_interruptible(dev);
		3435	if (ret)
		3436	return ret;
		3437
		3438	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3439	if (&obj->base == NULL) {
		3440	ret = -ENOENT;
		3441	goto unlock;
		3442	}
		3443
		3444	ret = i915_gem_object_set_cache_level(obj, level);
		3445
		3446	drm_gem_object_unreference(&obj->base);
		3447	unlock:
		3448	mutex_unlock(&dev->struct_mutex);
		3449	return ret;
		3450	}
		3451
4104	Serge	3452	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3453	{
		3454	/* There are 3 sources that pin objects:
		3455	* 1. The display engine (scanouts, sprites, cursors);
		3456	* 2. Reservations for execbuffer;
		3457	* 3. The user.
		3458	*
		3459	* We can ignore reservations as we hold the struct_mutex and
		3460	* are only called outside of the reservation path. The user
		3461	* can only increment pin_count once, and so if after
		3462	* subtracting the potential reference by the user, any pin_count
		3463	* remains, it must be due to another use by the display engine.
		3464	*/
		3465	return obj->pin_count - !!obj->user_pin_count;
		3466	}
		3467
2335	Serge	3468	/*
		3469	* Prepare buffer for display plane (scanout, cursors, etc).
		3470	* Can be called from an uninterruptible phase (modesetting) and allows
		3471	* any flushes to be pipelined (for pageflips).
		3472	*/
		3473	int
		3474	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3475	u32 alignment,
		3476	struct intel_ring_buffer *pipelined)
		3477	{
		3478	u32 old_read_domains, old_write_domain;
		3479	int ret;
2332	Serge	3480
3031	serge	3481	if (pipelined != obj->ring) {
		3482	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3483	if (ret)
		3484	return ret;
		3485	}
2332	Serge	3486
4104	Serge	3487	/* Mark the pin_display early so that we account for the
		3488	* display coherency whilst setting up the cache domains.
		3489	*/
		3490	obj->pin_display = true;
		3491
2335	Serge	3492	/* The display engine is not coherent with the LLC cache on gen6. As
		3493	* a result, we make sure that the pinning that is about to occur is
		3494	* done with uncached PTEs. This is lowest common denominator for all
		3495	* chipsets.
		3496	*
		3497	* However for gen6+, we could do better by using the GFDT bit instead
		3498	* of uncaching, which would allow us to flush all the LLC-cached data
		3499	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3500	*/
4104	Serge	3501	ret = i915_gem_object_set_cache_level(obj,
		3502	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3503	if (ret)
4104	Serge	3504	goto err_unpin_display;
2332	Serge	3505
2335	Serge	3506	/* As the user may map the buffer once pinned in the display plane
		3507	* (e.g. libkms for the bootup splash), we have to ensure that we
		3508	* always use map_and_fenceable for all scanout buffers.
		3509	*/
4104	Serge	3510	ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
2335	Serge	3511	if (ret)
4104	Serge	3512	goto err_unpin_display;
2332	Serge	3513
4104	Serge	3514	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3515
2335	Serge	3516	old_write_domain = obj->base.write_domain;
		3517	old_read_domains = obj->base.read_domains;
2332	Serge	3518
2335	Serge	3519	/* It should now be out of any other write domains, and we can update
		3520	* the domain values for our changes.
		3521	*/
3031	serge	3522	obj->base.write_domain = 0;
2335	Serge	3523	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3524
2351	Serge	3525	trace_i915_gem_object_change_domain(obj,
		3526	old_read_domains,
		3527	old_write_domain);
2332	Serge	3528
2335	Serge	3529	return 0;
4104	Serge	3530
		3531	err_unpin_display:
		3532	obj->pin_display = is_pin_display(obj);
		3533	return ret;
2335	Serge	3534	}
2332	Serge	3535
4104	Serge	3536	void
		3537	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3538	{
		3539	i915_gem_object_unpin(obj);
		3540	obj->pin_display = is_pin_display(obj);
		3541	}
		3542
2344	Serge	3543	int
		3544	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3545	{
		3546	int ret;
2332	Serge	3547
2344	Serge	3548	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3549	return 0;
2332	Serge	3550
3031	serge	3551	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3552	if (ret)
		3553	return ret;
2332	Serge	3554
2344	Serge	3555	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3556	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3557	return 0;
2344	Serge	3558	}
2332	Serge	3559
2344	Serge	3560	/**
		3561	* Moves a single object to the CPU read, and possibly write domain.
		3562	*
		3563	* This function returns when the move is complete, including waiting on
		3564	* flushes to occur.
		3565	*/
3031	serge	3566	int
2344	Serge	3567	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3568	{
		3569	uint32_t old_write_domain, old_read_domains;
		3570	int ret;
2332	Serge	3571
2344	Serge	3572	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3573	return 0;
2332	Serge	3574
3031	serge	3575	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3576	if (ret)
		3577	return ret;
2332	Serge	3578
2344	Serge	3579	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3580
2344	Serge	3581	old_write_domain = obj->base.write_domain;
		3582	old_read_domains = obj->base.read_domains;
2332	Serge	3583
2344	Serge	3584	/* Flush the CPU cache if it's still invalid. */
		3585	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3586	i915_gem_clflush_object(obj, false);
2332	Serge	3587
2344	Serge	3588	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3589	}
2332	Serge	3590
2344	Serge	3591	/* It should now be out of any other write domains, and we can update
		3592	* the domain values for our changes.
		3593	*/
		3594	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3595
2344	Serge	3596	/* If we're writing through the CPU, then the GPU read domains will
		3597	* need to be invalidated at next use.
		3598	*/
		3599	if (write) {
		3600	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3601	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3602	}
2332	Serge	3603
2351	Serge	3604	trace_i915_gem_object_change_domain(obj,
		3605	old_read_domains,
		3606	old_write_domain);
2332	Serge	3607
2344	Serge	3608	return 0;
		3609	}
2332	Serge	3610
3031	serge	3611	/* Throttle our rendering by waiting until the ring has completed our requests
		3612	* emitted over 20 msec ago.
2344	Serge	3613	*
3031	serge	3614	* Note that if we were to use the current jiffies each time around the loop,
		3615	* we wouldn't escape the function with any frames outstanding if the time to
		3616	* render a frame was over 20ms.
		3617	*
		3618	* This should get us reasonable parallelism between CPU and GPU but also
		3619	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3620	*/
3031	serge	3621	static int
		3622	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3623	{
3031	serge	3624	struct drm_i915_private *dev_priv = dev->dev_private;
		3625	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3626	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3627	struct drm_i915_gem_request *request;
		3628	struct intel_ring_buffer *ring = NULL;
3480	Serge	3629	unsigned reset_counter;
3031	serge	3630	u32 seqno = 0;
		3631	int ret;
2332	Serge	3632
3480	Serge	3633	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3634	if (ret)
		3635	return ret;
2332	Serge	3636
3480	Serge	3637	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3638	if (ret)
		3639	return ret;
		3640
3031	serge	3641	spin_lock(&file_priv->mm.lock);
		3642	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3643	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3644	break;
2332	Serge	3645
3031	serge	3646	ring = request->ring;
		3647	seqno = request->seqno;
		3648	}
3480	Serge	3649	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3650	spin_unlock(&file_priv->mm.lock);
2332	Serge	3651
3031	serge	3652	if (seqno == 0)
		3653	return 0;
2332	Serge	3654
3480	Serge	3655	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3656	if (ret == 0)
		3657	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3658
3031	serge	3659	return ret;
2352	Serge	3660	}
2332	Serge	3661
		3662	int
		3663	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3664	struct i915_address_space *vm,
2332	Serge	3665	uint32_t alignment,
3031	serge	3666	bool map_and_fenceable,
		3667	bool nonblocking)
2332	Serge	3668	{
4104	Serge	3669	struct i915_vma *vma;
2332	Serge	3670	int ret;
		3671
3031	serge	3672	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3673	return -EBUSY;
2332	Serge	3674
4104	Serge	3675	WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
		3676
		3677	vma = i915_gem_obj_to_vma(obj, vm);
		3678
		3679	if (vma) {
		3680	if ((alignment &&
		3681	vma->node.start & (alignment - 1)) \|\|
2332	Serge	3682	(map_and_fenceable && !obj->map_and_fenceable)) {
		3683	WARN(obj->pin_count,
		3684	"bo is already pinned with incorrect alignment:"
4104	Serge	3685	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3686	" obj->map_and_fenceable=%d\n",
4104	Serge	3687	i915_gem_obj_offset(obj, vm), alignment,
2332	Serge	3688	map_and_fenceable,
		3689	obj->map_and_fenceable);
4104	Serge	3690	ret = i915_vma_unbind(vma);
2332	Serge	3691	if (ret)
		3692	return ret;
		3693	}
		3694	}
		3695
4104	Serge	3696	if (!i915_gem_obj_bound(obj, vm)) {
3243	Serge	3697	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3698
4104	Serge	3699	ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3031	serge	3700	map_and_fenceable,
		3701	nonblocking);
2332	Serge	3702	if (ret)
		3703	return ret;
3243	Serge	3704
		3705	if (!dev_priv->mm.aliasing_ppgtt)
		3706	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3707	}
		3708
3031	serge	3709	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3710	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3711
		3712	obj->pin_count++;
2332	Serge	3713	obj->pin_mappable \|= map_and_fenceable;
		3714
		3715	return 0;
		3716	}
		3717
2344	Serge	3718	void
		3719	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3720	{
		3721	BUG_ON(obj->pin_count == 0);
4104	Serge	3722	BUG_ON(!i915_gem_obj_bound_any(obj));
2332	Serge	3723
3031	serge	3724	if (--obj->pin_count == 0)
2344	Serge	3725	obj->pin_mappable = false;
		3726	}
2332	Serge	3727
3031	serge	3728	int
		3729	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3730	struct drm_file *file)
		3731	{
		3732	struct drm_i915_gem_pin *args = data;
		3733	struct drm_i915_gem_object *obj;
		3734	int ret;
2332	Serge	3735
3480	Serge	3736	if(args->handle == -2)
		3737	{
		3738	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3739	return 0;
		3740	}
		3741
3031	serge	3742	ret = i915_mutex_lock_interruptible(dev);
		3743	if (ret)
		3744	return ret;
2332	Serge	3745
3031	serge	3746	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3747	if (&obj->base == NULL) {
		3748	ret = -ENOENT;
		3749	goto unlock;
		3750	}
2332	Serge	3751
3031	serge	3752	if (obj->madv != I915_MADV_WILLNEED) {
		3753	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3754	ret = -EINVAL;
		3755	goto out;
		3756	}
2332	Serge	3757
3031	serge	3758	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3759	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3760	args->handle);
		3761	ret = -EINVAL;
		3762	goto out;
		3763	}
2332	Serge	3764
3243	Serge	3765	if (obj->user_pin_count == 0) {
4104	Serge	3766	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3031	serge	3767	if (ret)
		3768	goto out;
		3769	}
2332	Serge	3770
3243	Serge	3771	obj->user_pin_count++;
		3772	obj->pin_filp = file;
		3773
4104	Serge	3774	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3775	out:
		3776	drm_gem_object_unreference(&obj->base);
		3777	unlock:
		3778	mutex_unlock(&dev->struct_mutex);
		3779	return ret;
		3780	}
2332	Serge	3781
3031	serge	3782	int
		3783	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3784	struct drm_file *file)
		3785	{
		3786	struct drm_i915_gem_pin *args = data;
		3787	struct drm_i915_gem_object *obj;
		3788	int ret;
2332	Serge	3789
3031	serge	3790	ret = i915_mutex_lock_interruptible(dev);
		3791	if (ret)
		3792	return ret;
2332	Serge	3793
4246	Serge	3794	if(args->handle == -2)
		3795	{
		3796	obj = get_fb_obj();
		3797	drm_gem_object_reference(&obj->base);
		3798	}
		3799	else
3031	serge	3800	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3801	if (&obj->base == NULL) {
		3802	ret = -ENOENT;
		3803	goto unlock;
		3804	}
2332	Serge	3805
3031	serge	3806	if (obj->pin_filp != file) {
		3807	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3808	args->handle);
		3809	ret = -EINVAL;
		3810	goto out;
		3811	}
		3812	obj->user_pin_count--;
		3813	if (obj->user_pin_count == 0) {
		3814	obj->pin_filp = NULL;
		3815	i915_gem_object_unpin(obj);
		3816	}
2332	Serge	3817
3031	serge	3818	out:
		3819	drm_gem_object_unreference(&obj->base);
		3820	unlock:
		3821	mutex_unlock(&dev->struct_mutex);
		3822	return ret;
		3823	}
2332	Serge	3824
3031	serge	3825	int
		3826	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3827	struct drm_file *file)
		3828	{
		3829	struct drm_i915_gem_busy *args = data;
		3830	struct drm_i915_gem_object *obj;
		3831	int ret;
2332	Serge	3832
3031	serge	3833	ret = i915_mutex_lock_interruptible(dev);
		3834	if (ret)
		3835	return ret;
2332	Serge	3836
3480	Serge	3837	if(args->handle == -2)
		3838	{
		3839	obj = get_fb_obj();
		3840	drm_gem_object_reference(&obj->base);
		3841	}
		3842	else
4104	Serge	3843	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3844	if (&obj->base == NULL) {
		3845	ret = -ENOENT;
		3846	goto unlock;
		3847	}
2332	Serge	3848
3031	serge	3849	/* Count all active objects as busy, even if they are currently not used
		3850	* by the gpu. Users of this interface expect objects to eventually
		3851	* become non-busy without any further actions, therefore emit any
		3852	* necessary flushes here.
		3853	*/
		3854	ret = i915_gem_object_flush_active(obj);
2332	Serge	3855
3031	serge	3856	args->busy = obj->active;
		3857	if (obj->ring) {
		3858	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3859	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3860	}
2332	Serge	3861
3031	serge	3862	drm_gem_object_unreference(&obj->base);
		3863	unlock:
		3864	mutex_unlock(&dev->struct_mutex);
		3865	return ret;
		3866	}
2332	Serge	3867
3031	serge	3868	int
		3869	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3870	struct drm_file *file_priv)
		3871	{
		3872	return i915_gem_ring_throttle(dev, file_priv);
		3873	}
2332	Serge	3874
3263	Serge	3875	#if 0
		3876
3031	serge	3877	int
		3878	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3879	struct drm_file *file_priv)
		3880	{
		3881	struct drm_i915_gem_madvise *args = data;
		3882	struct drm_i915_gem_object *obj;
		3883	int ret;
2332	Serge	3884
3031	serge	3885	switch (args->madv) {
		3886	case I915_MADV_DONTNEED:
		3887	case I915_MADV_WILLNEED:
		3888	break;
		3889	default:
		3890	return -EINVAL;
		3891	}
2332	Serge	3892
3031	serge	3893	ret = i915_mutex_lock_interruptible(dev);
		3894	if (ret)
		3895	return ret;
2332	Serge	3896
3031	serge	3897	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3898	if (&obj->base == NULL) {
		3899	ret = -ENOENT;
		3900	goto unlock;
		3901	}
2332	Serge	3902
3031	serge	3903	if (obj->pin_count) {
		3904	ret = -EINVAL;
		3905	goto out;
		3906	}
2332	Serge	3907
3031	serge	3908	if (obj->madv != __I915_MADV_PURGED)
		3909	obj->madv = args->madv;
2332	Serge	3910
3031	serge	3911	/* if the object is no longer attached, discard its backing storage */
		3912	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3913	i915_gem_object_truncate(obj);
2332	Serge	3914
3031	serge	3915	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3916
3031	serge	3917	out:
		3918	drm_gem_object_unreference(&obj->base);
		3919	unlock:
		3920	mutex_unlock(&dev->struct_mutex);
		3921	return ret;
		3922	}
		3923	#endif
2332	Serge	3924
3031	serge	3925	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3926	const struct drm_i915_gem_object_ops *ops)
		3927	{
4104	Serge	3928	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3929	INIT_LIST_HEAD(&obj->ring_list);
		3930	INIT_LIST_HEAD(&obj->exec_list);
4104	Serge	3931	INIT_LIST_HEAD(&obj->obj_exec_link);
		3932	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3933
3031	serge	3934	obj->ops = ops;
		3935
		3936	obj->fence_reg = I915_FENCE_REG_NONE;
		3937	obj->madv = I915_MADV_WILLNEED;
		3938	/* Avoid an unnecessary call to unbind on the first bind. */
		3939	obj->map_and_fenceable = true;
		3940
		3941	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3942	}
		3943
		3944	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3945	.get_pages = i915_gem_object_get_pages_gtt,
		3946	.put_pages = i915_gem_object_put_pages_gtt,
		3947	};
		3948
2332	Serge	3949	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3950	size_t size)
		3951	{
		3952	struct drm_i915_gem_object *obj;
3031	serge	3953	struct address_space *mapping;
3480	Serge	3954	gfp_t mask;
2340	Serge	3955
3746	Serge	3956	obj = i915_gem_object_alloc(dev);
2332	Serge	3957	if (obj == NULL)
		3958	return NULL;
		3959
		3960	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3961	i915_gem_object_free(obj);
2332	Serge	3962	return NULL;
		3963	}
		3964
		3965
3031	serge	3966	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3967
		3968	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3969	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3970
3031	serge	3971	if (HAS_LLC(dev)) {
		3972	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3973	* cache) for about a 10% performance improvement
		3974	* compared to uncached. Graphics requests other than
		3975	* display scanout are coherent with the CPU in
		3976	* accessing this cache. This means in this mode we
		3977	* don't need to clflush on the CPU side, and on the
		3978	* GPU side we only need to flush internal caches to
		3979	* get data visible to the CPU.
		3980	*
		3981	* However, we maintain the display planes as UC, and so
		3982	* need to rebind when first used as such.
		3983	*/
		3984	obj->cache_level = I915_CACHE_LLC;
		3985	} else
		3986	obj->cache_level = I915_CACHE_NONE;
		3987
		3988	return obj;
		3989	}
		3990
2344	Serge	3991	int i915_gem_init_object(struct drm_gem_object *obj)
		3992	{
		3993	BUG();
2332	Serge	3994
2344	Serge	3995	return 0;
		3996	}
2332	Serge	3997
3031	serge	3998	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3999	{
3031	serge	4000	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	4001	struct drm_device *dev = obj->base.dev;
		4002	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	4003	struct i915_vma vma, next;
2332	Serge	4004
3031	serge	4005	trace_i915_gem_object_destroy(obj);
		4006
		4007
		4008	obj->pin_count = 0;
4104	Serge	4009	/* NB: 0 or 1 elements */
		4010	WARN_ON(!list_empty(&obj->vma_list) &&
		4011	!list_is_singular(&obj->vma_list));
		4012	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		4013	int ret = i915_vma_unbind(vma);
		4014	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	4015	bool was_interruptible;
		4016
		4017	was_interruptible = dev_priv->mm.interruptible;
		4018	dev_priv->mm.interruptible = false;
		4019
4104	Serge	4020	WARN_ON(i915_vma_unbind(vma));
3031	serge	4021
		4022	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	4023	}
4104	Serge	4024	}
2332	Serge	4025
4104	Serge	4026	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		4027	* before progressing. */
		4028	if (obj->stolen)
		4029	i915_gem_object_unpin_pages(obj);
		4030
		4031	if (WARN_ON(obj->pages_pin_count))
3031	serge	4032	obj->pages_pin_count = 0;
		4033	i915_gem_object_put_pages(obj);
		4034	// i915_gem_object_free_mmap_offset(obj);
4104	Serge	4035	i915_gem_object_release_stolen(obj);
2332	Serge	4036
3243	Serge	4037	BUG_ON(obj->pages);
2332	Serge	4038
3031	serge	4039
3290	Serge	4040	if(obj->base.filp != NULL)
		4041	{
3298	Serge	4042	// printf("filp %p\n", obj->base.filp);
3290	Serge	4043	shmem_file_delete(obj->base.filp);
		4044	}
		4045
2344	Serge	4046	drm_gem_object_release(&obj->base);
		4047	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	4048
2344	Serge	4049	kfree(obj->bit_17);
4104	Serge	4050	i915_gem_object_free(obj);
2344	Serge	4051	}
2332	Serge	4052
4104	Serge	4053	struct i915_vma i915_gem_vma_create(struct drm_i915_gem_object obj,
		4054	struct i915_address_space *vm)
		4055	{
		4056	struct i915_vma vma = kzalloc(sizeof(vma), GFP_KERNEL);
		4057	if (vma == NULL)
		4058	return ERR_PTR(-ENOMEM);
		4059
		4060	INIT_LIST_HEAD(&vma->vma_link);
		4061	INIT_LIST_HEAD(&vma->mm_list);
		4062	INIT_LIST_HEAD(&vma->exec_list);
		4063	vma->vm = vm;
		4064	vma->obj = obj;
		4065
		4066	/* Keep GGTT vmas first to make debug easier */
		4067	if (i915_is_ggtt(vm))
		4068	list_add(&vma->vma_link, &obj->vma_list);
		4069	else
		4070	list_add_tail(&vma->vma_link, &obj->vma_list);
		4071
		4072	return vma;
		4073	}
		4074
		4075	void i915_gem_vma_destroy(struct i915_vma *vma)
		4076	{
		4077	WARN_ON(vma->node.allocated);
		4078	list_del(&vma->vma_link);
		4079	kfree(vma);
		4080	}
		4081
3031	serge	4082	#if 0
		4083	int
		4084	i915_gem_idle(struct drm_device *dev)
2344	Serge	4085	{
3031	serge	4086	drm_i915_private_t *dev_priv = dev->dev_private;
		4087	int ret;
2332	Serge	4088
4104	Serge	4089	if (dev_priv->ums.mm_suspended) {
3031	serge	4090	mutex_unlock(&dev->struct_mutex);
		4091	return 0;
		4092	}
2332	Serge	4093
3031	serge	4094	ret = i915_gpu_idle(dev);
		4095	if (ret) {
		4096	mutex_unlock(&dev->struct_mutex);
		4097	return ret;
		4098	}
		4099	i915_gem_retire_requests(dev);
		4100
3480	Serge	4101	/* Under UMS, be paranoid and evict. */
		4102	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		4103	i915_gem_evict_everything(dev);
		4104
		4105	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	4106
		4107	i915_kernel_lost_context(dev);
		4108	i915_gem_cleanup_ringbuffer(dev);
		4109
		4110	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	4111	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	4112
		4113	return 0;
2344	Serge	4114	}
3031	serge	4115	#endif
2332	Serge	4116
3031	serge	4117	void i915_gem_l3_remap(struct drm_device *dev)
		4118	{
		4119	drm_i915_private_t *dev_priv = dev->dev_private;
		4120	u32 misccpctl;
		4121	int i;
2332	Serge	4122
3480	Serge	4123	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	4124	return;
2332	Serge	4125
3243	Serge	4126	if (!dev_priv->l3_parity.remap_info)
3031	serge	4127	return;
2332	Serge	4128
3031	serge	4129	misccpctl = I915_READ(GEN7_MISCCPCTL);
		4130	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		4131	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	4132
3031	serge	4133	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		4134	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	4135	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	4136	DRM_DEBUG("0x%x was already programmed to %x\n",
		4137	GEN7_L3LOG_BASE + i, remap);
3243	Serge	4138	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	4139	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	4140	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	4141	}
2332	Serge	4142
3031	serge	4143	/* Make sure all the writes land before disabling dop clock gating */
		4144	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	4145
3031	serge	4146	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		4147	}
2332	Serge	4148
3031	serge	4149	void i915_gem_init_swizzling(struct drm_device *dev)
		4150	{
		4151	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	4152
3031	serge	4153	if (INTEL_INFO(dev)->gen < 5 \|\|
		4154	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4155	return;
2332	Serge	4156
3031	serge	4157	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4158	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4159
3031	serge	4160	if (IS_GEN5(dev))
		4161	return;
2344	Serge	4162
3031	serge	4163	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4164	if (IS_GEN6(dev))
		4165	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4166	else if (IS_GEN7(dev))
		4167	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	4168	else
3480	Serge	4169	BUG();
3031	serge	4170	}
		4171
		4172	static bool
		4173	intel_enable_blt(struct drm_device *dev)
		4174	{
		4175	if (!HAS_BLT(dev))
		4176	return false;
		4177
		4178	/* The blitter was dysfunctional on early prototypes */
		4179	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4180	DRM_INFO("BLT not supported on this pre-production hardware;"
		4181	" graphics performance will be degraded.\n");
		4182	return false;
		4183	}
		4184
		4185	return true;
		4186	}
		4187
3480	Serge	4188	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	4189	{
3480	Serge	4190	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4191	int ret;
2351	Serge	4192
2332	Serge	4193	ret = intel_init_render_ring_buffer(dev);
		4194	if (ret)
		4195	return ret;
		4196
		4197	if (HAS_BSD(dev)) {
		4198	ret = intel_init_bsd_ring_buffer(dev);
		4199	if (ret)
		4200	goto cleanup_render_ring;
		4201	}
		4202
3031	serge	4203	if (intel_enable_blt(dev)) {
2332	Serge	4204	ret = intel_init_blt_ring_buffer(dev);
		4205	if (ret)
		4206	goto cleanup_bsd_ring;
		4207	}
		4208
4104	Serge	4209	if (HAS_VEBOX(dev)) {
		4210	ret = intel_init_vebox_ring_buffer(dev);
		4211	if (ret)
		4212	goto cleanup_blt_ring;
		4213	}
		4214
		4215
3480	Serge	4216	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4217	if (ret)
4104	Serge	4218	goto cleanup_vebox_ring;
2351	Serge	4219
2332	Serge	4220	return 0;
		4221
4104	Serge	4222	cleanup_vebox_ring:
		4223	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4224	cleanup_blt_ring:
		4225	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4226	cleanup_bsd_ring:
		4227	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4228	cleanup_render_ring:
		4229	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4230
2332	Serge	4231	return ret;
		4232	}
		4233
3480	Serge	4234	int
		4235	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4236	{
3480	Serge	4237	drm_i915_private_t *dev_priv = dev->dev_private;
		4238	int ret;
3031	serge	4239
3480	Serge	4240	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4241	return -EIO;
3031	serge	4242
4104	Serge	4243	if (dev_priv->ellc_size)
		4244	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4245
3746	Serge	4246	if (HAS_PCH_NOP(dev)) {
		4247	u32 temp = I915_READ(GEN7_MSG_CTL);
		4248	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4249	I915_WRITE(GEN7_MSG_CTL, temp);
		4250	}
		4251
3480	Serge	4252	i915_gem_l3_remap(dev);
		4253
		4254	i915_gem_init_swizzling(dev);
		4255
		4256	ret = i915_gem_init_rings(dev);
		4257	if (ret)
		4258	return ret;
		4259
		4260	/*
		4261	* XXX: There was some w/a described somewhere suggesting loading
		4262	* contexts before PPGTT.
		4263	*/
		4264	i915_gem_context_init(dev);
3746	Serge	4265	if (dev_priv->mm.aliasing_ppgtt) {
		4266	ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		4267	if (ret) {
		4268	i915_gem_cleanup_aliasing_ppgtt(dev);
		4269	DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		4270	}
		4271	}
3480	Serge	4272
		4273	return 0;
3031	serge	4274	}
		4275
		4276	int i915_gem_init(struct drm_device *dev)
		4277	{
		4278	struct drm_i915_private *dev_priv = dev->dev_private;
		4279	int ret;
		4280
		4281	mutex_lock(&dev->struct_mutex);
3746	Serge	4282
		4283	if (IS_VALLEYVIEW(dev)) {
		4284	/* VLVA0 (potential hack), BIOS isn't actually waking us */
		4285	I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		4286	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
		4287	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4288	}
		4289
3480	Serge	4290	i915_gem_init_global_gtt(dev);
3746	Serge	4291
3031	serge	4292	ret = i915_gem_init_hw(dev);
		4293	mutex_unlock(&dev->struct_mutex);
		4294	if (ret) {
		4295	i915_gem_cleanup_aliasing_ppgtt(dev);
		4296	return ret;
		4297	}
		4298
3746	Serge	4299
3031	serge	4300	return 0;
		4301	}
		4302
2332	Serge	4303	void
		4304	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4305	{
		4306	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	4307	struct intel_ring_buffer *ring;
2332	Serge	4308	int i;
		4309
3031	serge	4310	for_each_ring(ring, dev_priv, i)
		4311	intel_cleanup_ring_buffer(ring);
2332	Serge	4312	}
		4313
3031	serge	4314	#if 0
		4315
2332	Serge	4316	int
		4317	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		4318	struct drm_file *file_priv)
		4319	{
4104	Serge	4320	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4321	int ret;
2332	Serge	4322
		4323	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4324	return 0;
		4325
3480	Serge	4326	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	4327	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	4328	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	4329	}
		4330
		4331	mutex_lock(&dev->struct_mutex);
4104	Serge	4332	dev_priv->ums.mm_suspended = 0;
2332	Serge	4333
3031	serge	4334	ret = i915_gem_init_hw(dev);
2332	Serge	4335	if (ret != 0) {
		4336	mutex_unlock(&dev->struct_mutex);
		4337	return ret;
		4338	}
		4339
4104	Serge	4340	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
2332	Serge	4341	mutex_unlock(&dev->struct_mutex);
		4342
		4343	ret = drm_irq_install(dev);
		4344	if (ret)
		4345	goto cleanup_ringbuffer;
		4346
		4347	return 0;
		4348
		4349	cleanup_ringbuffer:
		4350	mutex_lock(&dev->struct_mutex);
		4351	i915_gem_cleanup_ringbuffer(dev);
4104	Serge	4352	dev_priv->ums.mm_suspended = 1;
2332	Serge	4353	mutex_unlock(&dev->struct_mutex);
		4354
		4355	return ret;
		4356	}
		4357
		4358	int
		4359	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4360	struct drm_file *file_priv)
		4361	{
4104	Serge	4362	struct drm_i915_private *dev_priv = dev->dev_private;
		4363	int ret;
		4364
2332	Serge	4365	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4366	return 0;
		4367
		4368	drm_irq_uninstall(dev);
4104	Serge	4369
		4370	mutex_lock(&dev->struct_mutex);
		4371	ret = i915_gem_idle(dev);
		4372
		4373	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		4374	* We need to replace this with a semaphore, or something.
		4375	* And not confound ums.mm_suspended!
		4376	*/
		4377	if (ret != 0)
		4378	dev_priv->ums.mm_suspended = 1;
		4379	mutex_unlock(&dev->struct_mutex);
		4380
		4381	return ret;
2332	Serge	4382	}
		4383
		4384	void
		4385	i915_gem_lastclose(struct drm_device *dev)
		4386	{
		4387	int ret;
		4388
		4389	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4390	return;
		4391
4104	Serge	4392	mutex_lock(&dev->struct_mutex);
2332	Serge	4393	ret = i915_gem_idle(dev);
		4394	if (ret)
		4395	DRM_ERROR("failed to idle hardware: %d\n", ret);
4104	Serge	4396	mutex_unlock(&dev->struct_mutex);
2332	Serge	4397	}
		4398	#endif
		4399
		4400	static void
2326	Serge	4401	init_ring_lists(struct intel_ring_buffer *ring)
		4402	{
		4403	INIT_LIST_HEAD(&ring->active_list);
		4404	INIT_LIST_HEAD(&ring->request_list);
		4405	}
		4406
4104	Serge	4407	static void i915_init_vm(struct drm_i915_private *dev_priv,
		4408	struct i915_address_space *vm)
		4409	{
		4410	vm->dev = dev_priv->dev;
		4411	INIT_LIST_HEAD(&vm->active_list);
		4412	INIT_LIST_HEAD(&vm->inactive_list);
		4413	INIT_LIST_HEAD(&vm->global_link);
		4414	list_add(&vm->global_link, &dev_priv->vm_list);
		4415	}
		4416
2326	Serge	4417	void
		4418	i915_gem_load(struct drm_device *dev)
		4419	{
3480	Serge	4420	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4421	int i;
		4422
4104	Serge	4423	INIT_LIST_HEAD(&dev_priv->vm_list);
		4424	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4425
3031	serge	4426	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4427	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4428	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4429	for (i = 0; i < I915_NUM_RINGS; i++)
		4430	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4431	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4432	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4433	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4434	i915_gem_retire_work_handler);
3480	Serge	4435	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4436
		4437	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4438	if (IS_GEN3(dev)) {
3031	serge	4439	I915_WRITE(MI_ARB_STATE,
		4440	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4441	}
		4442
		4443	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4444
3746	Serge	4445	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4446	dev_priv->num_fence_regs = 32;
		4447	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4448	dev_priv->num_fence_regs = 16;
		4449	else
		4450	dev_priv->num_fence_regs = 8;
		4451
		4452	/* Initialize fence registers to zero */
3746	Serge	4453	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4454	i915_gem_restore_fences(dev);
2326	Serge	4455
		4456	i915_gem_detect_bit_6_swizzle(dev);
		4457
		4458	dev_priv->mm.interruptible = true;
		4459
		4460	}
		4461
4104	Serge	4462	#if 0
		4463	/*
		4464	* Create a physically contiguous memory object for this object
		4465	* e.g. for cursor + overlay regs
		4466	*/
		4467	static int i915_gem_init_phys_object(struct drm_device *dev,
		4468	int id, int size, int align)
		4469	{
		4470	drm_i915_private_t *dev_priv = dev->dev_private;
		4471	struct drm_i915_gem_phys_object *phys_obj;
		4472	int ret;
2326	Serge	4473
4104	Serge	4474	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)
		4475	return 0;
		4476
		4477	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
		4478	if (!phys_obj)
		4479	return -ENOMEM;
		4480
		4481	phys_obj->id = id;
		4482
		4483	phys_obj->handle = drm_pci_alloc(dev, size, align);
		4484	if (!phys_obj->handle) {
		4485	ret = -ENOMEM;
		4486	goto kfree_obj;
		4487	}
		4488	#ifdef CONFIG_X86
		4489	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4490	#endif
		4491
		4492	dev_priv->mm.phys_objs[id - 1] = phys_obj;
		4493
		4494	return 0;
		4495	kfree_obj:
		4496	kfree(phys_obj);
		4497	return ret;
		4498	}
		4499
		4500	static void i915_gem_free_phys_object(struct drm_device *dev, int id)
		4501	{
		4502	drm_i915_private_t *dev_priv = dev->dev_private;
		4503	struct drm_i915_gem_phys_object *phys_obj;
		4504
		4505	if (!dev_priv->mm.phys_objs[id - 1])
		4506	return;
		4507
		4508	phys_obj = dev_priv->mm.phys_objs[id - 1];
		4509	if (phys_obj->cur_obj) {
		4510	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
		4511	}
		4512
		4513	#ifdef CONFIG_X86
		4514	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4515	#endif
		4516	drm_pci_free(dev, phys_obj->handle);
		4517	kfree(phys_obj);
		4518	dev_priv->mm.phys_objs[id - 1] = NULL;
		4519	}
		4520
		4521	void i915_gem_free_all_phys_object(struct drm_device *dev)
		4522	{
		4523	int i;
		4524
		4525	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
		4526	i915_gem_free_phys_object(dev, i);
		4527	}
		4528
		4529	void i915_gem_detach_phys_object(struct drm_device *dev,
		4530	struct drm_i915_gem_object *obj)
		4531	{
		4532	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4533	char *vaddr;
		4534	int i;
		4535	int page_count;
		4536
		4537	if (!obj->phys_obj)
		4538	return;
		4539	vaddr = obj->phys_obj->handle->vaddr;
		4540
		4541	page_count = obj->base.size / PAGE_SIZE;
		4542	for (i = 0; i < page_count; i++) {
		4543	struct page *page = shmem_read_mapping_page(mapping, i);
		4544	if (!IS_ERR(page)) {
		4545	char *dst = kmap_atomic(page);
		4546	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
		4547	kunmap_atomic(dst);
		4548
		4549	drm_clflush_pages(&page, 1);
		4550
		4551	set_page_dirty(page);
		4552	mark_page_accessed(page);
		4553	page_cache_release(page);
		4554	}
		4555	}
		4556	i915_gem_chipset_flush(dev);
		4557
		4558	obj->phys_obj->cur_obj = NULL;
		4559	obj->phys_obj = NULL;
		4560	}
		4561
		4562	int
		4563	i915_gem_attach_phys_object(struct drm_device *dev,
		4564	struct drm_i915_gem_object *obj,
		4565	int id,
		4566	int align)
		4567	{
		4568	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4569	drm_i915_private_t *dev_priv = dev->dev_private;
		4570	int ret = 0;
		4571	int page_count;
		4572	int i;
		4573
		4574	if (id > I915_MAX_PHYS_OBJECT)
		4575	return -EINVAL;
		4576
		4577	if (obj->phys_obj) {
		4578	if (obj->phys_obj->id == id)
		4579	return 0;
		4580	i915_gem_detach_phys_object(dev, obj);
		4581	}
		4582
		4583	/* create a new object */
		4584	if (!dev_priv->mm.phys_objs[id - 1]) {
		4585	ret = i915_gem_init_phys_object(dev, id,
		4586	obj->base.size, align);
		4587	if (ret) {
		4588	DRM_ERROR("failed to init phys object %d size: %zu\n",
		4589	id, obj->base.size);
		4590	return ret;
		4591	}
		4592	}
		4593
		4594	/* bind to the object */
		4595	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
		4596	obj->phys_obj->cur_obj = obj;
		4597
		4598	page_count = obj->base.size / PAGE_SIZE;
		4599
		4600	for (i = 0; i < page_count; i++) {
		4601	struct page *page;
		4602	char dst, src;
		4603
		4604	page = shmem_read_mapping_page(mapping, i);
		4605	if (IS_ERR(page))
		4606	return PTR_ERR(page);
		4607
		4608	src = kmap_atomic(page);
		4609	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
		4610	memcpy(dst, src, PAGE_SIZE);
		4611	kunmap_atomic(src);
		4612
		4613	mark_page_accessed(page);
		4614	page_cache_release(page);
		4615	}
		4616
		4617	return 0;
		4618	}
		4619
		4620	static int
		4621	i915_gem_phys_pwrite(struct drm_device *dev,
		4622	struct drm_i915_gem_object *obj,
		4623	struct drm_i915_gem_pwrite *args,
		4624	struct drm_file *file_priv)
		4625	{
		4626	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
		4627	char __user *user_data = to_user_ptr(args->data_ptr);
		4628
		4629	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		4630	unsigned long unwritten;
		4631
		4632	/* The physical object once assigned is fixed for the lifetime
		4633	* of the obj, so we can safely drop the lock and continue
		4634	* to access vaddr.
		4635	*/
		4636	mutex_unlock(&dev->struct_mutex);
		4637	unwritten = copy_from_user(vaddr, user_data, args->size);
		4638	mutex_lock(&dev->struct_mutex);
		4639	if (unwritten)
		4640	return -EFAULT;
		4641	}
		4642
		4643	i915_gem_chipset_flush(dev);
		4644	return 0;
		4645	}
		4646
		4647	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4648	{
		4649	struct drm_i915_file_private *file_priv = file->driver_priv;
		4650
		4651	/* Clean up our request list when the client is going away, so that
		4652	* later retire_requests won't dereference our soon-to-be-gone
		4653	* file_priv.
		4654	*/
		4655	spin_lock(&file_priv->mm.lock);
		4656	while (!list_empty(&file_priv->mm.request_list)) {
		4657	struct drm_i915_gem_request *request;
		4658
		4659	request = list_first_entry(&file_priv->mm.request_list,
		4660	struct drm_i915_gem_request,
		4661	client_list);
		4662	list_del(&request->client_list);
		4663	request->file_priv = NULL;
		4664	}
		4665	spin_unlock(&file_priv->mm.lock);
		4666	}
		4667	#endif
		4668
		4669	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4670	{
		4671	if (!mutex_is_locked(mutex))
		4672	return false;
		4673
		4674	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4675	return mutex->owner == task;
		4676	#else
		4677	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4678	return false;
		4679	#endif
		4680	}
		4681
		4682	/* All the new VM stuff */
		4683	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4684	struct i915_address_space *vm)
		4685	{
		4686	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4687	struct i915_vma *vma;
		4688
		4689	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4690	vm = &dev_priv->gtt.base;
		4691
		4692	BUG_ON(list_empty(&o->vma_list));
		4693	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4694	if (vma->vm == vm)
		4695	return vma->node.start;
		4696
		4697	}
		4698	return 0; //-1;
		4699	}
		4700
		4701	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4702	struct i915_address_space *vm)
		4703	{
		4704	struct i915_vma *vma;
		4705
		4706	list_for_each_entry(vma, &o->vma_list, vma_link)
		4707	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4708	return true;
		4709
		4710	return false;
		4711	}
		4712
		4713	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4714	{
		4715	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4716	struct i915_address_space *vm;
		4717
		4718	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
		4719	if (i915_gem_obj_bound(o, vm))
		4720	return true;
		4721
		4722	return false;
		4723	}
		4724
		4725	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4726	struct i915_address_space *vm)
		4727	{
		4728	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4729	struct i915_vma *vma;
		4730
		4731	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4732	vm = &dev_priv->gtt.base;
		4733
		4734	BUG_ON(list_empty(&o->vma_list));
		4735
		4736	list_for_each_entry(vma, &o->vma_list, vma_link)
		4737	if (vma->vm == vm)
		4738	return vma->node.size;
		4739
		4740	return 0;
		4741	}
		4742	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
		4743	struct i915_address_space *vm)
		4744	{
		4745	struct i915_vma *vma;
		4746	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4747	if (vma->vm == vm)
		4748	return vma;
		4749
		4750	return NULL;
		4751	}
		4752
		4753	struct i915_vma *
		4754	i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
		4755	struct i915_address_space *vm)
		4756	{
		4757	struct i915_vma *vma;
		4758
		4759	vma = i915_gem_obj_to_vma(obj, vm);
		4760	if (!vma)
		4761	vma = i915_gem_vma_create(obj, vm);
		4762
		4763	return vma;
		4764	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 4280