WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
3263	Serge	40	#define PROT_READ 0x1 /* page can be read */
		41	#define PROT_WRITE 0x2 /* page can be written */
		42	#define MAP_SHARED 0x01 /* Share changes */
		43
2344	Serge	44	#undef mb
		45	#undef rmb
		46	#undef wmb
		47	#define mb() asm volatile("mfence")
		48	#define rmb() asm volatile ("lfence")
		49	#define wmb() asm volatile ("sfence")
		50
3263	Serge	51	unsigned long vm_mmap(struct file *file, unsigned long addr,
		52	unsigned long len, unsigned long prot,
		53	unsigned long flag, unsigned long offset);
		54
2344	Serge	55	static inline void clflush(volatile void *__p)
		56	{
		57	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		58	}
		59
2332	Serge	60	#define MAX_ERRNO 4095
		61
		62	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		63
		64
2326	Serge	65	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		66	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		67	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		68	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		69
2332	Serge	70	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		71	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		72	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		73	unsigned alignment,
3031	serge	74	bool map_and_fenceable,
		75	bool nonblocking);
2332	Serge	76	static int i915_gem_phys_pwrite(struct drm_device *dev,
		77	struct drm_i915_gem_object *obj,
		78	struct drm_i915_gem_pwrite *args,
		79	struct drm_file *file);
2326	Serge	80
3031	serge	81	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		82	struct drm_i915_gem_object *obj);
		83	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		84	struct drm_i915_fence_reg *fence,
		85	bool enable);
2332	Serge	86
3031	serge	87	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		88	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		89	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		90
		91	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		92	{
		93	if (obj->tiling_mode)
		94	i915_gem_release_mmap(obj);
		95
		96	/* As we do not have an associated fence register, we will force
		97	* a tiling change if we ever need to acquire one.
		98	*/
		99	obj->fence_dirty = false;
		100	obj->fence_reg = I915_FENCE_REG_NONE;
		101	}
		102
2332	Serge	103	/* some bookkeeping */
		104	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		105	size_t size)
		106	{
		107	dev_priv->mm.object_count++;
		108	dev_priv->mm.object_memory += size;
		109	}
		110
		111	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		112	size_t size)
		113	{
		114	dev_priv->mm.object_count--;
		115	dev_priv->mm.object_memory -= size;
		116	}
		117
		118	static int
		119	i915_gem_wait_for_error(struct drm_device *dev)
		120	{
		121	struct drm_i915_private *dev_priv = dev->dev_private;
		122	struct completion *x = &dev_priv->error_completion;
		123	unsigned long flags;
		124	int ret;
		125
		126	if (!atomic_read(&dev_priv->mm.wedged))
		127	return 0;
3255	Serge	128	#if 0
3031	serge	129	/*
		130	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		131	* userspace. If it takes that long something really bad is going on and
		132	* we should simply try to bail out and fail as gracefully as possible.
		133	*/
		134	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		135	if (ret == 0) {
		136	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		137	return -EIO;
		138	} else if (ret < 0) {
2332	Serge	139	return ret;
3031	serge	140	}
2332	Serge	141
		142	if (atomic_read(&dev_priv->mm.wedged)) {
		143	/* GPU is hung, bump the completion count to account for
		144	* the token we just consumed so that we never hit zero and
		145	* end up waiting upon a subsequent completion event that
		146	* will never happen.
		147	*/
		148	spin_lock_irqsave(&x->wait.lock, flags);
		149	x->done++;
		150	spin_unlock_irqrestore(&x->wait.lock, flags);
		151	}
3255	Serge	152	#endif
		153
2332	Serge	154	return 0;
		155	}
		156
		157	int i915_mutex_lock_interruptible(struct drm_device *dev)
		158	{
		159	int ret;
		160
		161	ret = i915_gem_wait_for_error(dev);
		162	if (ret)
		163	return ret;
		164
3255	Serge	165	mutex_lock(&dev->struct_mutex);
2332	Serge	166
		167	WARN_ON(i915_verify_lists(dev));
		168	return 0;
		169	}
		170
		171	static inline bool
		172	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		173	{
3031	serge	174	return obj->gtt_space && !obj->active;
2332	Serge	175	}
		176
		177
		178	#if 0
		179
		180	int
		181	i915_gem_init_ioctl(struct drm_device dev, void data,
		182	struct drm_file *file)
		183	{
		184	struct drm_i915_gem_init *args = data;
		185
3031	serge	186	if (drm_core_check_feature(dev, DRIVER_MODESET))
		187	return -ENODEV;
		188
2332	Serge	189	if (args->gtt_start >= args->gtt_end \|\|
		190	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		191	return -EINVAL;
		192
3031	serge	193	/* GEM with user mode setting was never supported on ilk and later. */
		194	if (INTEL_INFO(dev)->gen >= 5)
		195	return -ENODEV;
		196
2332	Serge	197	mutex_lock(&dev->struct_mutex);
3031	serge	198	i915_gem_init_global_gtt(dev, args->gtt_start,
		199	args->gtt_end, args->gtt_end);
2332	Serge	200	mutex_unlock(&dev->struct_mutex);
		201
		202	return 0;
		203	}
2351	Serge	204	#endif
2332	Serge	205
		206	int
		207	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		208	struct drm_file *file)
		209	{
		210	struct drm_i915_private *dev_priv = dev->dev_private;
		211	struct drm_i915_gem_get_aperture *args = data;
		212	struct drm_i915_gem_object *obj;
		213	size_t pinned;
		214
		215	pinned = 0;
		216	mutex_lock(&dev->struct_mutex);
3031	serge	217	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		218	if (obj->pin_count)
2332	Serge	219	pinned += obj->gtt_space->size;
		220	mutex_unlock(&dev->struct_mutex);
		221
		222	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	223	args->aper_available_size = args->aper_size - pinned;
2332	Serge	224
		225	return 0;
		226	}
		227
3031	serge	228	static int
		229	i915_gem_create(struct drm_file *file,
2332	Serge	230	struct drm_device *dev,
		231	uint64_t size,
		232	uint32_t *handle_p)
		233	{
		234	struct drm_i915_gem_object *obj;
		235	int ret;
		236	u32 handle;
		237
		238	size = roundup(size, PAGE_SIZE);
2342	Serge	239	if (size == 0)
		240	return -EINVAL;
2332	Serge	241
		242	/* Allocate the new object */
		243	obj = i915_gem_alloc_object(dev, size);
		244	if (obj == NULL)
		245	return -ENOMEM;
		246
		247	ret = drm_gem_handle_create(file, &obj->base, &handle);
		248	if (ret) {
		249	drm_gem_object_release(&obj->base);
		250	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		251	kfree(obj);
		252	return ret;
		253	}
		254
		255	/* drop reference from allocate - handle holds it now */
		256	drm_gem_object_unreference(&obj->base);
2351	Serge	257	trace_i915_gem_object_create(obj);
2332	Serge	258
		259	*handle_p = handle;
		260	return 0;
		261	}
		262
		263	int
		264	i915_gem_dumb_create(struct drm_file *file,
		265	struct drm_device *dev,
		266	struct drm_mode_create_dumb *args)
		267	{
		268	/* have to work out size/pitch and return them */
		269	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		270	args->size = args->pitch * args->height;
		271	return i915_gem_create(file, dev,
		272	args->size, &args->handle);
		273	}
		274
		275	int i915_gem_dumb_destroy(struct drm_file *file,
		276	struct drm_device *dev,
		277	uint32_t handle)
		278	{
		279	return drm_gem_handle_delete(file, handle);
		280	}
		281
2326	Serge	282	/**
2332	Serge	283	* Creates a new mm object and returns a handle to it.
		284	*/
		285	int
		286	i915_gem_create_ioctl(struct drm_device dev, void data,
		287	struct drm_file *file)
		288	{
		289	struct drm_i915_gem_create *args = data;
3031	serge	290
2332	Serge	291	return i915_gem_create(file, dev,
		292	args->size, &args->handle);
		293	}
		294
		295	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		296	{
		297	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		298
		299	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		300	obj->tiling_mode != I915_TILING_NONE;
		301	}
3260	Serge	302	#if 0
2332	Serge	303
3031	serge	304	static inline int
		305	__copy_to_user_swizzled(char __user *cpu_vaddr,
		306	const char *gpu_vaddr, int gpu_offset,
2332	Serge	307	int length)
		308	{
3031	serge	309	int ret, cpu_offset = 0;
2332	Serge	310
3031	serge	311	while (length > 0) {
		312	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		313	int this_length = min(cacheline_end - gpu_offset, length);
		314	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	315
3031	serge	316	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		317	gpu_vaddr + swizzled_gpu_offset,
		318	this_length);
		319	if (ret)
		320	return ret + length;
2332	Serge	321
3031	serge	322	cpu_offset += this_length;
		323	gpu_offset += this_length;
		324	length -= this_length;
		325	}
		326
		327	return 0;
2332	Serge	328	}
		329
3031	serge	330	static inline int
		331	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		332	const char __user *cpu_vaddr,
		333	int length)
2332	Serge	334	{
3031	serge	335	int ret, cpu_offset = 0;
2332	Serge	336
		337	while (length > 0) {
		338	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		339	int this_length = min(cacheline_end - gpu_offset, length);
		340	int swizzled_gpu_offset = gpu_offset ^ 64;
		341
3031	serge	342	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	343	cpu_vaddr + cpu_offset,
		344	this_length);
3031	serge	345	if (ret)
		346	return ret + length;
		347
2332	Serge	348	cpu_offset += this_length;
		349	gpu_offset += this_length;
		350	length -= this_length;
		351	}
		352
3031	serge	353	return 0;
2332	Serge	354	}
		355
3031	serge	356	/* Per-page copy function for the shmem pread fastpath.
		357	* Flushes invalid cachelines before reading the target if
		358	* needs_clflush is set. */
2332	Serge	359	static int
3031	serge	360	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		361	char __user *user_data,
		362	bool page_do_bit17_swizzling, bool needs_clflush)
		363	{
		364	char *vaddr;
		365	int ret;
		366
		367	if (unlikely(page_do_bit17_swizzling))
		368	return -EINVAL;
		369
		370	vaddr = kmap_atomic(page);
		371	if (needs_clflush)
		372	drm_clflush_virt_range(vaddr + shmem_page_offset,
		373	page_length);
		374	ret = __copy_to_user_inatomic(user_data,
		375	vaddr + shmem_page_offset,
		376	page_length);
		377	kunmap_atomic(vaddr);
		378
		379	return ret ? -EFAULT : 0;
		380	}
		381
		382	static void
		383	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		384	bool swizzled)
		385	{
		386	if (unlikely(swizzled)) {
		387	unsigned long start = (unsigned long) addr;
		388	unsigned long end = (unsigned long) addr + length;
		389
		390	/* For swizzling simply ensure that we always flush both
		391	* channels. Lame, but simple and it works. Swizzled
		392	* pwrite/pread is far from a hotpath - current userspace
		393	* doesn't use it at all. */
		394	start = round_down(start, 128);
		395	end = round_up(end, 128);
		396
		397	drm_clflush_virt_range((void *)start, end - start);
		398	} else {
		399	drm_clflush_virt_range(addr, length);
		400	}
		401
		402	}
		403
		404	/* Only difference to the fast-path function is that this can handle bit17
		405	* and uses non-atomic copy and kmap functions. */
		406	static int
		407	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		408	char __user *user_data,
		409	bool page_do_bit17_swizzling, bool needs_clflush)
		410	{
		411	char *vaddr;
		412	int ret;
		413
		414	vaddr = kmap(page);
		415	if (needs_clflush)
		416	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		417	page_length,
		418	page_do_bit17_swizzling);
		419
		420	if (page_do_bit17_swizzling)
		421	ret = __copy_to_user_swizzled(user_data,
		422	vaddr, shmem_page_offset,
		423	page_length);
		424	else
		425	ret = __copy_to_user(user_data,
		426	vaddr + shmem_page_offset,
		427	page_length);
		428	kunmap(page);
		429
		430	return ret ? - EFAULT : 0;
		431	}
		432
		433	static int
		434	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	435	struct drm_i915_gem_object *obj,
		436	struct drm_i915_gem_pread *args,
		437	struct drm_file *file)
		438	{
3031	serge	439	char __user *user_data;
2332	Serge	440	ssize_t remain;
		441	loff_t offset;
3031	serge	442	int shmem_page_offset, page_length, ret = 0;
		443	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		444	int hit_slowpath = 0;
		445	int prefaulted = 0;
		446	int needs_clflush = 0;
		447	struct scatterlist *sg;
		448	int i;
2332	Serge	449
		450	user_data = (char __user *) (uintptr_t) args->data_ptr;
		451	remain = args->size;
		452
3031	serge	453	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		454
		455	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		456	/* If we're not in the cpu read domain, set ourself into the gtt
		457	* read domain and manually flush cachelines (if required). This
		458	* optimizes for the case when the gpu will dirty the data
		459	* anyway again before the next pread happens. */
		460	if (obj->cache_level == I915_CACHE_NONE)
		461	needs_clflush = 1;
		462	if (obj->gtt_space) {
		463	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		464	if (ret)
		465	return ret;
		466	}
		467	}
		468
		469	ret = i915_gem_object_get_pages(obj);
		470	if (ret)
		471	return ret;
		472
		473	i915_gem_object_pin_pages(obj);
		474
2332	Serge	475	offset = args->offset;
		476
3031	serge	477	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	478	struct page *page;
		479
3031	serge	480	if (i < offset >> PAGE_SHIFT)
		481	continue;
		482
		483	if (remain <= 0)
		484	break;
		485
2332	Serge	486	/* Operation in this page
		487	*
3031	serge	488	* shmem_page_offset = offset within page in shmem file
2332	Serge	489	* page_length = bytes to copy for this page
		490	*/
3031	serge	491	shmem_page_offset = offset_in_page(offset);
2332	Serge	492	page_length = remain;
3031	serge	493	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		494	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	495
3031	serge	496	page = sg_page(sg);
		497	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		498	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	499
3031	serge	500	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		501	user_data, page_do_bit17_swizzling,
		502	needs_clflush);
		503	if (ret == 0)
		504	goto next_page;
2332	Serge	505
3031	serge	506	hit_slowpath = 1;
		507	mutex_unlock(&dev->struct_mutex);
		508
		509	if (!prefaulted) {
		510	ret = fault_in_multipages_writeable(user_data, remain);
		511	/* Userspace is tricking us, but we've already clobbered
		512	* its pages with the prefault and promised to write the
		513	* data up to the first fault. Hence ignore any errors
		514	* and just continue. */
		515	(void)ret;
		516	prefaulted = 1;
		517	}
		518
		519	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		520	user_data, page_do_bit17_swizzling,
		521	needs_clflush);
		522
		523	mutex_lock(&dev->struct_mutex);
		524
		525	next_page:
2332	Serge	526	mark_page_accessed(page);
3031	serge	527
2332	Serge	528	if (ret)
3031	serge	529	goto out;
2332	Serge	530
		531	remain -= page_length;
		532	user_data += page_length;
		533	offset += page_length;
		534	}
		535
3031	serge	536	out:
		537	i915_gem_object_unpin_pages(obj);
		538
		539	if (hit_slowpath) {
		540	/* Fixup: Kill any reinstated backing storage pages */
		541	if (obj->madv == __I915_MADV_PURGED)
		542	i915_gem_object_truncate(obj);
		543	}
		544
		545	return ret;
2332	Serge	546	}
		547
		548	/**
3031	serge	549	* Reads data from the object referenced by handle.
		550	*
		551	* On error, the contents of *data are undefined.
2332	Serge	552	*/
3031	serge	553	int
		554	i915_gem_pread_ioctl(struct drm_device dev, void data,
		555	struct drm_file *file)
		556	{
		557	struct drm_i915_gem_pread *args = data;
		558	struct drm_i915_gem_object *obj;
		559	int ret = 0;
		560
		561	if (args->size == 0)
		562	return 0;
		563
		564	if (!access_ok(VERIFY_WRITE,
		565	(char __user *)(uintptr_t)args->data_ptr,
		566	args->size))
		567	return -EFAULT;
		568
		569	ret = i915_mutex_lock_interruptible(dev);
		570	if (ret)
		571	return ret;
		572
		573	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		574	if (&obj->base == NULL) {
		575	ret = -ENOENT;
		576	goto unlock;
		577	}
		578
		579	/* Bounds check source. */
		580	if (args->offset > obj->base.size \|\|
		581	args->size > obj->base.size - args->offset) {
		582	ret = -EINVAL;
		583	goto out;
		584	}
		585
		586	/* prime objects have no backing filp to GEM pread/pwrite
		587	* pages from.
		588	*/
		589	if (!obj->base.filp) {
		590	ret = -EINVAL;
		591	goto out;
		592	}
		593
		594	trace_i915_gem_object_pread(obj, args->offset, args->size);
		595
		596	ret = i915_gem_shmem_pread(dev, obj, args, file);
		597
		598	out:
		599	drm_gem_object_unreference(&obj->base);
		600	unlock:
		601	mutex_unlock(&dev->struct_mutex);
		602	return ret;
		603	}
		604
		605	/* This is the fast write path which cannot handle
		606	* page faults in the source data
		607	*/
		608
		609	static inline int
		610	fast_user_write(struct io_mapping *mapping,
		611	loff_t page_base, int page_offset,
		612	char __user *user_data,
		613	int length)
		614	{
		615	void __iomem *vaddr_atomic;
		616	void *vaddr;
		617	unsigned long unwritten;
		618
		619	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		620	/* We can use the cpu mem copy function because this is X86. */
		621	vaddr = (void __force*)vaddr_atomic + page_offset;
		622	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		623	user_data, length);
		624	io_mapping_unmap_atomic(vaddr_atomic);
		625	return unwritten;
		626	}
3260	Serge	627	#endif
3031	serge	628
3260	Serge	629	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	630	/**
		631	* This is the fast pwrite path, where we copy the data directly from the
		632	* user into the GTT, uncached.
		633	*/
2332	Serge	634	static int
3031	serge	635	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		636	struct drm_i915_gem_object *obj,
		637	struct drm_i915_gem_pwrite *args,
		638	struct drm_file *file)
2332	Serge	639	{
3031	serge	640	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	641	ssize_t remain;
3031	serge	642	loff_t offset, page_base;
		643	char __user *user_data;
		644	int page_offset, page_length, ret;
3260	Serge	645	char *vaddr;
2332	Serge	646
3031	serge	647	ret = i915_gem_object_pin(obj, 0, true, true);
		648	if (ret)
		649	goto out;
		650
		651	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		652	if (ret)
		653	goto out_unpin;
		654
		655	ret = i915_gem_object_put_fence(obj);
		656	if (ret)
		657	goto out_unpin;
		658
3260	Serge	659	vaddr = AllocKernelSpace(4096);
		660	if(vaddr == NULL)
		661	{
		662	ret = -ENOSPC;
		663	goto out_unpin;
		664	};
		665
3031	serge	666	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	667	remain = args->size;
		668
3031	serge	669	offset = obj->gtt_offset + args->offset;
2332	Serge	670
3031	serge	671	while (remain > 0) {
		672	/* Operation in this page
		673	*
		674	* page_base = page offset within aperture
		675	* page_offset = offset within page
		676	* page_length = bytes to copy for this page
		677	*/
		678	page_base = offset & PAGE_MASK;
		679	page_offset = offset_in_page(offset);
		680	page_length = remain;
		681	if ((page_offset + remain) > PAGE_SIZE)
		682	page_length = PAGE_SIZE - page_offset;
2332	Serge	683
3260	Serge	684	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	685
3260	Serge	686	memcpy(vaddr+page_offset, user_data, page_length);
		687
3031	serge	688	remain -= page_length;
		689	user_data += page_length;
		690	offset += page_length;
2332	Serge	691	}
		692
3260	Serge	693	FreeKernelSpace(vaddr);
		694
3031	serge	695	out_unpin:
		696	i915_gem_object_unpin(obj);
		697	out:
3260	Serge	698	printf("% s ret = %d\n", __FUNCTION__, ret);
		699
3031	serge	700	return ret;
		701	}
		702
		703	/* Per-page copy function for the shmem pwrite fastpath.
		704	* Flushes invalid cachelines before writing to the target if
		705	* needs_clflush_before is set and flushes out any written cachelines after
		706	* writing if needs_clflush is set. */
		707	static int
		708	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		709	char __user *user_data,
		710	bool page_do_bit17_swizzling,
		711	bool needs_clflush_before,
		712	bool needs_clflush_after)
		713	{
		714	char *vaddr;
3260	Serge	715	int ret = 0;
3031	serge	716
		717	if (unlikely(page_do_bit17_swizzling))
		718	return -EINVAL;
		719
3260	Serge	720	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	721	if (needs_clflush_before)
		722	drm_clflush_virt_range(vaddr + shmem_page_offset,
		723	page_length);
3260	Serge	724	memcpy(vaddr + shmem_page_offset,
3031	serge	725	user_data,
		726	page_length);
		727	if (needs_clflush_after)
		728	drm_clflush_virt_range(vaddr + shmem_page_offset,
		729	page_length);
3260	Serge	730	FreeKernelSpace(vaddr);
3031	serge	731
		732	return ret ? -EFAULT : 0;
		733	}
3260	Serge	734	#if 0
3031	serge	735
		736	/* Only difference to the fast-path function is that this can handle bit17
		737	* and uses non-atomic copy and kmap functions. */
		738	static int
		739	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		740	char __user *user_data,
		741	bool page_do_bit17_swizzling,
		742	bool needs_clflush_before,
		743	bool needs_clflush_after)
		744	{
		745	char *vaddr;
		746	int ret;
		747
		748	vaddr = kmap(page);
		749	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		750	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		751	page_length,
		752	page_do_bit17_swizzling);
		753	if (page_do_bit17_swizzling)
		754	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		755	user_data,
		756	page_length);
		757	else
		758	ret = __copy_from_user(vaddr + shmem_page_offset,
		759	user_data,
		760	page_length);
		761	if (needs_clflush_after)
		762	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		763	page_length,
		764	page_do_bit17_swizzling);
		765	kunmap(page);
		766
		767	return ret ? -EFAULT : 0;
		768	}
3260	Serge	769	#endif
3031	serge	770
3260	Serge	771
3031	serge	772	static int
		773	i915_gem_shmem_pwrite(struct drm_device *dev,
		774	struct drm_i915_gem_object *obj,
		775	struct drm_i915_gem_pwrite *args,
		776	struct drm_file *file)
		777	{
		778	ssize_t remain;
		779	loff_t offset;
		780	char __user *user_data;
		781	int shmem_page_offset, page_length, ret = 0;
		782	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		783	int hit_slowpath = 0;
		784	int needs_clflush_after = 0;
		785	int needs_clflush_before = 0;
		786	int i;
		787	struct scatterlist *sg;
		788
		789	user_data = (char __user *) (uintptr_t) args->data_ptr;
		790	remain = args->size;
		791
		792	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		793
		794	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		795	/* If we're not in the cpu write domain, set ourself into the gtt
		796	* write domain and manually flush cachelines (if required). This
		797	* optimizes for the case when the gpu will use the data
		798	* right away and we therefore have to clflush anyway. */
		799	if (obj->cache_level == I915_CACHE_NONE)
		800	needs_clflush_after = 1;
		801	if (obj->gtt_space) {
		802	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		803	if (ret)
		804	return ret;
		805	}
		806	}
		807	/* Same trick applies for invalidate partially written cachelines before
		808	* writing. */
		809	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		810	&& obj->cache_level == I915_CACHE_NONE)
		811	needs_clflush_before = 1;
		812
		813	ret = i915_gem_object_get_pages(obj);
2332	Serge	814	if (ret)
3031	serge	815	return ret;
2332	Serge	816
3031	serge	817	i915_gem_object_pin_pages(obj);
2332	Serge	818
		819	offset = args->offset;
3031	serge	820	obj->dirty = 1;
2332	Serge	821
3031	serge	822	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	823	struct page *page;
3031	serge	824	int partial_cacheline_write;
2332	Serge	825
3031	serge	826	if (i < offset >> PAGE_SHIFT)
		827	continue;
		828
		829	if (remain <= 0)
		830	break;
		831
2332	Serge	832	/* Operation in this page
		833	*
		834	* shmem_page_offset = offset within page in shmem file
		835	* page_length = bytes to copy for this page
		836	*/
		837	shmem_page_offset = offset_in_page(offset);
		838
		839	page_length = remain;
		840	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		841	page_length = PAGE_SIZE - shmem_page_offset;
		842
3031	serge	843	/* If we don't overwrite a cacheline completely we need to be
		844	* careful to have up-to-date data by first clflushing. Don't
		845	* overcomplicate things and flush the entire patch. */
		846	partial_cacheline_write = needs_clflush_before &&
		847	((shmem_page_offset \| page_length)
3260	Serge	848	& (x86_clflush_size - 1));
2332	Serge	849
3031	serge	850	page = sg_page(sg);
		851	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		852	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	853
3031	serge	854	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		855	user_data, page_do_bit17_swizzling,
		856	partial_cacheline_write,
		857	needs_clflush_after);
		858	if (ret == 0)
		859	goto next_page;
		860
		861	hit_slowpath = 1;
		862	mutex_unlock(&dev->struct_mutex);
3260	Serge	863	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	864
3260	Serge	865	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		866	// user_data, page_do_bit17_swizzling,
		867	// partial_cacheline_write,
		868	// needs_clflush_after);
		869
3031	serge	870	mutex_lock(&dev->struct_mutex);
		871
		872	next_page:
2332	Serge	873
3031	serge	874	if (ret)
		875	goto out;
		876
2332	Serge	877	remain -= page_length;
3031	serge	878	user_data += page_length;
2332	Serge	879	offset += page_length;
		880	}
		881
		882	out:
3031	serge	883	i915_gem_object_unpin_pages(obj);
		884
		885	if (hit_slowpath) {
		886	/* Fixup: Kill any reinstated backing storage pages */
		887	if (obj->madv == __I915_MADV_PURGED)
		888	i915_gem_object_truncate(obj);
		889	/* and flush dirty cachelines in case the object isn't in the cpu write
		890	* domain anymore. */
		891	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		892	i915_gem_clflush_object(obj);
3243	Serge	893	i915_gem_chipset_flush(dev);
3031	serge	894	}
2332	Serge	895	}
		896
3031	serge	897	if (needs_clflush_after)
3243	Serge	898	i915_gem_chipset_flush(dev);
3031	serge	899
2332	Serge	900	return ret;
		901	}
3031	serge	902
		903	/**
		904	* Writes data to the object referenced by handle.
		905	*
		906	* On error, the contents of the buffer that were to be modified are undefined.
		907	*/
		908	int
		909	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		910	struct drm_file *file)
		911	{
		912	struct drm_i915_gem_pwrite *args = data;
		913	struct drm_i915_gem_object *obj;
		914	int ret;
		915
		916	if (args->size == 0)
		917	return 0;
		918
		919	ret = i915_mutex_lock_interruptible(dev);
		920	if (ret)
		921	return ret;
		922
		923	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		924	if (&obj->base == NULL) {
		925	ret = -ENOENT;
		926	goto unlock;
		927	}
		928
		929	/* Bounds check destination. */
		930	if (args->offset > obj->base.size \|\|
		931	args->size > obj->base.size - args->offset) {
		932	ret = -EINVAL;
		933	goto out;
		934	}
		935
		936	/* prime objects have no backing filp to GEM pread/pwrite
		937	* pages from.
		938	*/
		939	if (!obj->base.filp) {
		940	ret = -EINVAL;
		941	goto out;
		942	}
		943
		944	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		945
		946	ret = -EFAULT;
		947	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		948	* it would end up going through the fenced access, and we'll get
		949	* different detiling behavior between reading and writing.
		950	* pread/pwrite currently are reading and writing from the CPU
		951	* perspective, requiring manual detiling by the client.
		952	*/
3260	Serge	953	// if (obj->phys_obj) {
		954	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		955	// goto out;
		956	// }
3031	serge	957
		958	if (obj->cache_level == I915_CACHE_NONE &&
		959	obj->tiling_mode == I915_TILING_NONE &&
		960	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		961	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		962	/* Note that the gtt paths might fail with non-page-backed user
		963	* pointers (e.g. gtt mappings when moving data between
		964	* textures). Fallback to the shmem path in that case. */
		965	}
		966
		967	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	968	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	969
		970	out:
		971	drm_gem_object_unreference(&obj->base);
		972	unlock:
		973	mutex_unlock(&dev->struct_mutex);
		974	return ret;
		975	}
		976
		977	int
		978	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		979	bool interruptible)
		980	{
		981	if (atomic_read(&dev_priv->mm.wedged)) {
		982	struct completion *x = &dev_priv->error_completion;
		983	bool recovery_complete;
		984	unsigned long flags;
2332	Serge	985
3031	serge	986	/* Give the error handler a chance to run. */
		987	spin_lock_irqsave(&x->wait.lock, flags);
		988	recovery_complete = x->done > 0;
		989	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	990
3031	serge	991	/* Non-interruptible callers can't handle -EAGAIN, hence return
		992	* -EIO unconditionally for these. */
		993	if (!interruptible)
		994	return -EIO;
2332	Serge	995
3031	serge	996	/* Recovery complete, but still wedged means reset failure. */
		997	if (recovery_complete)
		998	return -EIO;
2332	Serge	999
3031	serge	1000	return -EAGAIN;
		1001	}
2332	Serge	1002
3031	serge	1003	return 0;
		1004	}
2332	Serge	1005
3031	serge	1006	/*
		1007	* Compare seqno against outstanding lazy request. Emit a request if they are
		1008	* equal.
		1009	*/
		1010	static int
		1011	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1012	{
		1013	int ret;
2332	Serge	1014
3031	serge	1015	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1016
3031	serge	1017	ret = 0;
		1018	if (seqno == ring->outstanding_lazy_request)
		1019	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1020
3031	serge	1021	return ret;
		1022	}
2332	Serge	1023
3031	serge	1024	/**
		1025	* __wait_seqno - wait until execution of seqno has finished
		1026	* @ring: the ring expected to report seqno
		1027	* @seqno: duh!
		1028	* @interruptible: do an interruptible wait (normally yes)
		1029	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1030	*
		1031	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1032	* errno with remaining time filled in timeout argument.
		1033	*/
		1034	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1035	bool interruptible, struct timespec *timeout)
		1036	{
		1037	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1038	struct timespec before, now, wait_time={1,0};
		1039	unsigned long timeout_jiffies;
		1040	long end;
		1041	bool wait_forever = true;
		1042	int ret;
2332	Serge	1043
3031	serge	1044	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1045	return 0;
2332	Serge	1046
3031	serge	1047	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1048
3031	serge	1049	if (timeout != NULL) {
		1050	wait_time = *timeout;
		1051	wait_forever = false;
		1052	}
2332	Serge	1053
3031	serge	1054	// timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1055
3031	serge	1056	if (WARN_ON(!ring->irq_get(ring)))
		1057	return -ENODEV;
		1058	#if 0
2332	Serge	1059
3031	serge	1060	/* Record current time in case interrupted by signal, or wedged * */
		1061	getrawmonotonic(&before);
2332	Serge	1062
3031	serge	1063	#define EXIT_COND \
		1064	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1065	atomic_read(&dev_priv->mm.wedged))
		1066	do {
		1067	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1068	timeout_jiffies);
2332	Serge	1069
3031	serge	1070	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1071	if (ret)
		1072	end = ret;
		1073	} while (end == 0 && wait_forever);
2332	Serge	1074
3031	serge	1075	getrawmonotonic(&now);
2332	Serge	1076
3031	serge	1077	ring->irq_put(ring);
		1078	trace_i915_gem_request_wait_end(ring, seqno);
		1079	#undef EXIT_COND
2332	Serge	1080
3031	serge	1081	if (timeout) {
		1082	// struct timespec sleep_time = timespec_sub(now, before);
		1083	// timeout = timespec_sub(timeout, sleep_time);
		1084	}
2332	Serge	1085
3031	serge	1086	switch (end) {
		1087	case -EIO:
		1088	case -EAGAIN: /* Wedged */
		1089	case -ERESTARTSYS: /* Signal */
		1090	return (int)end;
		1091	case 0: /* Timeout */
		1092	// if (timeout)
		1093	// set_normalized_timespec(timeout, 0, 0);
		1094	return -ETIME;
		1095	default: /* Completed */
		1096	WARN_ON(end < 0); /* We're not aware of other errors */
		1097	return 0;
		1098	}
3260	Serge	1099
3031	serge	1100	#endif
2332	Serge	1101
3031	serge	1102	#define EXIT_COND \
		1103	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1104	atomic_read(&dev_priv->mm.wedged))
		1105	wait_event(ring->irq_queue, EXIT_COND);
		1106	#undef EXIT_COND
		1107	ring->irq_put(ring);
2332	Serge	1108
3031	serge	1109	return 0;
		1110	}
2332	Serge	1111
3031	serge	1112	/**
		1113	* Waits for a sequence number to be signaled, and cleans up the
		1114	* request and object lists appropriately for that event.
		1115	*/
		1116	int
		1117	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1118	{
		1119	struct drm_device *dev = ring->dev;
		1120	struct drm_i915_private *dev_priv = dev->dev_private;
		1121	bool interruptible = dev_priv->mm.interruptible;
		1122	int ret;
2332	Serge	1123
3031	serge	1124	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1125	BUG_ON(seqno == 0);
2332	Serge	1126
3031	serge	1127	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1128	if (ret)
		1129	return ret;
2332	Serge	1130
3031	serge	1131	ret = i915_gem_check_olr(ring, seqno);
		1132	if (ret)
		1133	return ret;
2332	Serge	1134
3031	serge	1135	return __wait_seqno(ring, seqno, interruptible, NULL);
		1136	}
2332	Serge	1137
3031	serge	1138	/**
		1139	* Ensures that all rendering to the object has completed and the object is
		1140	* safe to unbind from the GTT or access from the CPU.
		1141	*/
		1142	static __must_check int
		1143	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1144	bool readonly)
		1145	{
		1146	struct intel_ring_buffer *ring = obj->ring;
		1147	u32 seqno;
		1148	int ret;
2332	Serge	1149
3031	serge	1150	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1151	if (seqno == 0)
		1152	return 0;
2332	Serge	1153
3031	serge	1154	ret = i915_wait_seqno(ring, seqno);
		1155	if (ret)
		1156	return ret;
2332	Serge	1157
3031	serge	1158	i915_gem_retire_requests_ring(ring);
2332	Serge	1159
3031	serge	1160	/* Manually manage the write flush as we may have not yet
		1161	* retired the buffer.
		1162	*/
		1163	if (obj->last_write_seqno &&
		1164	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1165	obj->last_write_seqno = 0;
		1166	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1167	}
2332	Serge	1168
3031	serge	1169	return 0;
		1170	}
2332	Serge	1171
3260	Serge	1172	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1173	* as the object state may change during this call.
		1174	*/
		1175	static __must_check int
		1176	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1177	bool readonly)
		1178	{
		1179	struct drm_device *dev = obj->base.dev;
		1180	struct drm_i915_private *dev_priv = dev->dev_private;
		1181	struct intel_ring_buffer *ring = obj->ring;
		1182	u32 seqno;
		1183	int ret;
2332	Serge	1184
3260	Serge	1185	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1186	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1187
3260	Serge	1188	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1189	if (seqno == 0)
		1190	return 0;
2332	Serge	1191
3260	Serge	1192	ret = i915_gem_check_wedge(dev_priv, true);
		1193	if (ret)
		1194	return ret;
2332	Serge	1195
3260	Serge	1196	ret = i915_gem_check_olr(ring, seqno);
		1197	if (ret)
		1198	return ret;
2332	Serge	1199
3260	Serge	1200	mutex_unlock(&dev->struct_mutex);
		1201	ret = __wait_seqno(ring, seqno, true, NULL);
		1202	mutex_lock(&dev->struct_mutex);
2332	Serge	1203
3260	Serge	1204	i915_gem_retire_requests_ring(ring);
2332	Serge	1205
3260	Serge	1206	/* Manually manage the write flush as we may have not yet
		1207	* retired the buffer.
		1208	*/
		1209	if (obj->last_write_seqno &&
		1210	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1211	obj->last_write_seqno = 0;
		1212	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1213	}
2332	Serge	1214
3260	Serge	1215	return ret;
		1216	}
2332	Serge	1217
3260	Serge	1218	/**
		1219	* Called when user space prepares to use an object with the CPU, either
		1220	* through the mmap ioctl's mapping or a GTT mapping.
		1221	*/
		1222	int
		1223	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1224	struct drm_file *file)
		1225	{
		1226	struct drm_i915_gem_set_domain *args = data;
		1227	struct drm_i915_gem_object *obj;
		1228	uint32_t read_domains = args->read_domains;
		1229	uint32_t write_domain = args->write_domain;
		1230	int ret;
2332	Serge	1231
3260	Serge	1232	/* Only handle setting domains to types used by the CPU. */
		1233	if (write_domain & I915_GEM_GPU_DOMAINS)
		1234	return -EINVAL;
2332	Serge	1235
3260	Serge	1236	if (read_domains & I915_GEM_GPU_DOMAINS)
		1237	return -EINVAL;
2332	Serge	1238
3260	Serge	1239	/* Having something in the write domain implies it's in the read
		1240	* domain, and only that read domain. Enforce that in the request.
		1241	*/
		1242	if (write_domain != 0 && read_domains != write_domain)
		1243	return -EINVAL;
2332	Serge	1244
3260	Serge	1245	ret = i915_mutex_lock_interruptible(dev);
		1246	if (ret)
		1247	return ret;
2332	Serge	1248
3260	Serge	1249	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1250	if (&obj->base == NULL) {
		1251	ret = -ENOENT;
		1252	goto unlock;
		1253	}
2332	Serge	1254
3260	Serge	1255	/* Try to flush the object off the GPU without holding the lock.
		1256	* We will repeat the flush holding the lock in the normal manner
		1257	* to catch cases where we are gazumped.
		1258	*/
		1259	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1260	if (ret)
		1261	goto unref;
2332	Serge	1262
3260	Serge	1263	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1264	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1265
3260	Serge	1266	/* Silently promote "you're not bound, there was nothing to do"
		1267	* to success, since the client was just asking us to
		1268	* make sure everything was done.
		1269	*/
		1270	if (ret == -EINVAL)
		1271	ret = 0;
		1272	} else {
		1273	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1274	}
2332	Serge	1275
3260	Serge	1276	unref:
		1277	drm_gem_object_unreference(&obj->base);
		1278	unlock:
		1279	mutex_unlock(&dev->struct_mutex);
		1280	return ret;
		1281	}
2332	Serge	1282
		1283
		1284
		1285
		1286
		1287
3260	Serge	1288	/**
		1289	* Maps the contents of an object, returning the address it is mapped
		1290	* into.
		1291	*
		1292	* While the mapping holds a reference on the contents of the object, it doesn't
		1293	* imply a ref on the object itself.
		1294	*/
		1295	int
		1296	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1297	struct drm_file *file)
		1298	{
		1299	struct drm_i915_gem_mmap *args = data;
		1300	struct drm_gem_object *obj;
		1301	unsigned long addr = 0;
2332	Serge	1302
3260	Serge	1303	obj = drm_gem_object_lookup(dev, file, args->handle);
		1304	if (obj == NULL)
		1305	return -ENOENT;
		1306
3263	Serge	1307	dbgprintf("%s offset %lld size %lld\n",
		1308	__FUNCTION__, args->offset, args->size);
3260	Serge	1309	/* prime objects have no backing filp to GEM mmap
		1310	* pages from.
		1311	*/
		1312	if (!obj->filp) {
		1313	drm_gem_object_unreference_unlocked(obj);
		1314	return -EINVAL;
		1315	}
2332	Serge	1316
3263	Serge	1317	addr = vm_mmap(obj->filp, 0, args->size,
		1318	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1319	args->offset);
3260	Serge	1320	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1321	if (IS_ERR((void *)addr))
		1322	return addr;
2332	Serge	1323
3260	Serge	1324	args->addr_ptr = (uint64_t) addr;
2332	Serge	1325
3263	Serge	1326	return 0;
3260	Serge	1327	}
2332	Serge	1328
		1329
		1330
		1331
		1332
		1333
		1334
		1335
3031	serge	1336
		1337
		1338
		1339
		1340
		1341	/**
		1342	* i915_gem_release_mmap - remove physical page mappings
		1343	* @obj: obj in question
		1344	*
		1345	* Preserve the reservation of the mmapping with the DRM core code, but
		1346	* relinquish ownership of the pages back to the system.
		1347	*
		1348	* It is vital that we remove the page mapping if we have mapped a tiled
		1349	* object through the GTT and then lose the fence register due to
		1350	* resource pressure. Similarly if the object has been moved out of the
		1351	* aperture, than pages mapped into userspace must be revoked. Removing the
		1352	* mapping will then trigger a page fault on the next user access, allowing
		1353	* fixup by i915_gem_fault().
		1354	*/
		1355	void
		1356	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1357	{
		1358	if (!obj->fault_mappable)
		1359	return;
		1360
		1361	if (obj->base.dev->dev_mapping)
		1362	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1363	// (loff_t)obj->base.map_list.hash.key<
		1364	// obj->base.size, 1);
		1365
		1366	obj->fault_mappable = false;
		1367	}
		1368
2332	Serge	1369	static uint32_t
		1370	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1371	{
		1372	uint32_t gtt_size;
		1373
		1374	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1375	tiling_mode == I915_TILING_NONE)
		1376	return size;
		1377
		1378	/* Previous chips need a power-of-two fence region when tiling */
		1379	if (INTEL_INFO(dev)->gen == 3)
		1380	gtt_size = 1024*1024;
		1381	else
		1382	gtt_size = 512*1024;
		1383
		1384	while (gtt_size < size)
		1385	gtt_size <<= 1;
		1386
		1387	return gtt_size;
		1388	}
		1389
		1390	/**
		1391	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1392	* @obj: object to check
		1393	*
		1394	* Return the required GTT alignment for an object, taking into account
		1395	* potential fence register mapping.
		1396	*/
		1397	static uint32_t
		1398	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1399	uint32_t size,
		1400	int tiling_mode)
		1401	{
		1402	/*
		1403	* Minimum alignment is 4k (GTT page size), but might be greater
		1404	* if a fence register is needed for the object.
		1405	*/
		1406	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1407	tiling_mode == I915_TILING_NONE)
		1408	return 4096;
		1409
		1410	/*
		1411	* Previous chips need to be aligned to the size of the smallest
		1412	* fence register that can contain the object.
		1413	*/
		1414	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1415	}
		1416
		1417	/**
		1418	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1419	* unfenced object
		1420	* @dev: the device
		1421	* @size: size of the object
		1422	* @tiling_mode: tiling mode of the object
		1423	*
		1424	* Return the required GTT alignment for an object, only taking into account
		1425	* unfenced tiled surface requirements.
		1426	*/
		1427	uint32_t
		1428	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1429	uint32_t size,
		1430	int tiling_mode)
		1431	{
		1432	/*
		1433	* Minimum alignment is 4k (GTT page size) for sane hw.
		1434	*/
		1435	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1436	tiling_mode == I915_TILING_NONE)
		1437	return 4096;
		1438
		1439	/* Previous hardware however needs to be aligned to a power-of-two
		1440	* tile height. The simplest method for determining this is to reuse
		1441	* the power-of-tile object size.
		1442	*/
		1443	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1444	}
		1445
3031	serge	1446	/* Immediately discard the backing storage */
		1447	static void
		1448	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1449	{
		1450	// struct inode *inode;
2332	Serge	1451
3031	serge	1452	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1453
3263	Serge	1454	if (obj->base.filp == NULL)
		1455	return;
2332	Serge	1456
3031	serge	1457	/* Our goal here is to return as much of the memory as
		1458	* is possible back to the system as we are called from OOM.
		1459	* To do this we must instruct the shmfs to drop all of its
		1460	* backing pages, now.
		1461	*/
		1462	// inode = obj->base.filp->f_path.dentry->d_inode;
		1463	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1464
3031	serge	1465	obj->madv = __I915_MADV_PURGED;
		1466	}
2332	Serge	1467
3031	serge	1468	static inline int
		1469	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1470	{
		1471	return obj->madv == I915_MADV_DONTNEED;
		1472	}
2332	Serge	1473
3031	serge	1474	static void
		1475	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1476	{
3243	Serge	1477	int page_count = obj->base.size / PAGE_SIZE;
		1478	struct scatterlist *sg;
3031	serge	1479	int ret, i;
2332	Serge	1480
3031	serge	1481	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1482
3031	serge	1483	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1484	if (ret) {
		1485	/* In the event of a disaster, abandon all caches and
		1486	* hope for the best.
		1487	*/
		1488	WARN_ON(ret != -EIO);
		1489	i915_gem_clflush_object(obj);
		1490	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1491	}
2332	Serge	1492
3031	serge	1493	if (obj->madv == I915_MADV_DONTNEED)
		1494	obj->dirty = 0;
2332	Serge	1495
3243	Serge	1496	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1497	struct page *page = sg_page(sg);
2332	Serge	1498
3243	Serge	1499
		1500
3263	Serge	1501	// page_cache_release(page);
3243	Serge	1502	}
		1503	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3031	serge	1504	obj->dirty = 0;
3243	Serge	1505
		1506	sg_free_table(obj->pages);
		1507	kfree(obj->pages);
3031	serge	1508	}
2332	Serge	1509
3031	serge	1510	static int
		1511	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1512	{
		1513	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1514
3243	Serge	1515	if (obj->pages == NULL)
3031	serge	1516	return 0;
2332	Serge	1517
3031	serge	1518	BUG_ON(obj->gtt_space);
		1519
		1520	if (obj->pages_pin_count)
		1521	return -EBUSY;
		1522
3243	Serge	1523	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1524	* array, hence protect them from being reaped by removing them from gtt
		1525	* lists early. */
		1526	list_del(&obj->gtt_list);
		1527
3031	serge	1528	ops->put_pages(obj);
3243	Serge	1529	obj->pages = NULL;
3031	serge	1530
		1531	if (i915_gem_object_is_purgeable(obj))
		1532	i915_gem_object_truncate(obj);
		1533
		1534	return 0;
		1535	}
		1536
		1537
		1538
		1539
		1540
		1541
		1542
		1543
2332	Serge	1544	static int
3031	serge	1545	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1546	{
3260	Serge	1547	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1548	int page_count, i;
3260	Serge	1549	struct address_space *mapping;
3243	Serge	1550	struct sg_table *st;
		1551	struct scatterlist *sg;
		1552	struct page *page;
		1553	gfp_t gfp;
2332	Serge	1554
3243	Serge	1555	/* Assert that the object is not currently in any GPU domain. As it
		1556	* wasn't in the GTT, there shouldn't be any way it could have been in
		1557	* a GPU cache
2332	Serge	1558	*/
3243	Serge	1559	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1560	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1561
		1562	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1563	if (st == NULL)
		1564	return -ENOMEM;
		1565
2332	Serge	1566	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1567	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1568	sg_free_table(st);
		1569	kfree(st);
2332	Serge	1570	return -ENOMEM;
3243	Serge	1571	}
2332	Serge	1572
3243	Serge	1573	/* Get the list of pages out of our struct file. They'll be pinned
		1574	* at this point until we release them.
		1575	*
		1576	* Fail silently without starting the shrinker
		1577	*/
		1578	for_each_sg(st->sgl, sg, page_count, i) {
3260	Serge	1579	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1580	if (IS_ERR(page)) {
		1581	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1582	goto err_pages;
		1583
3260	Serge	1584	}
3243	Serge	1585	sg_set_page(sg, page, PAGE_SIZE, 0);
		1586	}
3031	serge	1587
3243	Serge	1588	obj->pages = st;
3031	serge	1589
3260	Serge	1590	DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1591
		1592	return 0;
		1593
		1594	err_pages:
3243	Serge	1595	for_each_sg(st->sgl, sg, i, page_count)
		1596	page_cache_release(sg_page(sg));
		1597	sg_free_table(st);
		1598	kfree(st);
		1599	return PTR_ERR(page);
2332	Serge	1600	}
		1601
3031	serge	1602	/* Ensure that the associated pages are gathered from the backing storage
		1603	* and pinned into our object. i915_gem_object_get_pages() may be called
		1604	* multiple times before they are released by a single call to
		1605	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1606	* either as a result of memory pressure (reaping pages under the shrinker)
		1607	* or as the object is itself released.
		1608	*/
		1609	int
		1610	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1611	{
3031	serge	1612	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1613	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1614	int ret;
2332	Serge	1615
3243	Serge	1616	if (obj->pages)
3031	serge	1617	return 0;
2332	Serge	1618
3031	serge	1619	BUG_ON(obj->pages_pin_count);
2332	Serge	1620
3031	serge	1621	ret = ops->get_pages(obj);
		1622	if (ret)
		1623	return ret;
2344	Serge	1624
3031	serge	1625	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1626	return 0;
2332	Serge	1627	}
		1628
		1629	void
		1630	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1631	struct intel_ring_buffer *ring)
2332	Serge	1632	{
		1633	struct drm_device *dev = obj->base.dev;
		1634	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1635	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1636
		1637	BUG_ON(ring == NULL);
		1638	obj->ring = ring;
		1639
		1640	/* Add a reference if we're newly entering the active list. */
		1641	if (!obj->active) {
2344	Serge	1642	drm_gem_object_reference(&obj->base);
2332	Serge	1643	obj->active = 1;
		1644	}
		1645
		1646	/* Move from whatever list we were on to the tail of execution. */
		1647	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1648	list_move_tail(&obj->ring_list, &ring->active_list);
		1649
3031	serge	1650	obj->last_read_seqno = seqno;
		1651
2332	Serge	1652	if (obj->fenced_gpu_access) {
3031	serge	1653	obj->last_fenced_seqno = seqno;
		1654
		1655	/* Bump MRU to take account of the delayed flush */
		1656	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1657	struct drm_i915_fence_reg *reg;
		1658
		1659	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1660	list_move_tail(®->lru_list,
		1661	&dev_priv->mm.fence_list);
		1662	}
2332	Serge	1663	}
		1664	}
		1665
2344	Serge	1666	static void
3031	serge	1667	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1668	{
		1669	struct drm_device *dev = obj->base.dev;
3031	serge	1670	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1671
3031	serge	1672	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1673	BUG_ON(!obj->active);
2332	Serge	1674
3031	serge	1675	if (obj->pin_count) /* are we a framebuffer? */
		1676	intel_mark_fb_idle(obj);
2344	Serge	1677
2352	Serge	1678	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1679
3031	serge	1680	list_del_init(&obj->ring_list);
2352	Serge	1681	obj->ring = NULL;
2344	Serge	1682
3031	serge	1683	obj->last_read_seqno = 0;
		1684	obj->last_write_seqno = 0;
		1685	obj->base.write_domain = 0;
		1686
		1687	obj->last_fenced_seqno = 0;
2352	Serge	1688	obj->fenced_gpu_access = false;
2344	Serge	1689
2352	Serge	1690	obj->active = 0;
		1691	drm_gem_object_unreference(&obj->base);
		1692
		1693	WARN_ON(i915_verify_lists(dev));
		1694	}
		1695
3243	Serge	1696	static int
		1697	i915_gem_handle_seqno_wrap(struct drm_device *dev)
2344	Serge	1698	{
3243	Serge	1699	struct drm_i915_private *dev_priv = dev->dev_private;
		1700	struct intel_ring_buffer *ring;
		1701	int ret, i, j;
2344	Serge	1702
3243	Serge	1703	/* The hardware uses various monotonic 32-bit counters, if we
		1704	* detect that they will wraparound we need to idle the GPU
		1705	* and reset those counters.
		1706	*/
		1707	ret = 0;
		1708	for_each_ring(ring, dev_priv, i) {
		1709	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1710	ret \|= ring->sync_seqno[j] != 0;
		1711	}
		1712	if (ret == 0)
		1713	return ret;
2344	Serge	1714
3243	Serge	1715	ret = i915_gpu_idle(dev);
		1716	if (ret)
		1717	return ret;
		1718
		1719	i915_gem_retire_requests(dev);
		1720	for_each_ring(ring, dev_priv, i) {
		1721	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1722	ring->sync_seqno[j] = 0;
		1723	}
		1724
		1725	return 0;
2344	Serge	1726	}
		1727
3243	Serge	1728	int
		1729	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1730	{
3243	Serge	1731	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1732
3243	Serge	1733	/* reserve 0 for non-seqno */
		1734	if (dev_priv->next_seqno == 0) {
		1735	int ret = i915_gem_handle_seqno_wrap(dev);
		1736	if (ret)
		1737	return ret;
		1738
		1739	dev_priv->next_seqno = 1;
		1740	}
		1741
		1742	*seqno = dev_priv->next_seqno++;
		1743	return 0;
2332	Serge	1744	}
		1745
2352	Serge	1746	int
		1747	i915_add_request(struct intel_ring_buffer *ring,
		1748	struct drm_file *file,
3031	serge	1749	u32 *out_seqno)
2352	Serge	1750	{
		1751	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1752	struct drm_i915_gem_request *request;
		1753	u32 request_ring_position;
2352	Serge	1754	int was_empty;
		1755	int ret;
2332	Serge	1756
3031	serge	1757	/*
		1758	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1759	* after having emitted the batchbuffer command. Hence we need to fix
		1760	* things up similar to emitting the lazy request. The difference here
		1761	* is that the flush _must_ happen before the next request, no matter
		1762	* what.
		1763	*/
		1764	ret = intel_ring_flush_all_caches(ring);
		1765	if (ret)
		1766	return ret;
2332	Serge	1767
3031	serge	1768	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1769	if (request == NULL)
		1770	return -ENOMEM;
		1771
		1772
		1773	/* Record the position of the start of the request so that
		1774	* should we detect the updated seqno part-way through the
		1775	* GPU processing the request, we never over-estimate the
		1776	* position of the head.
		1777	*/
		1778	request_ring_position = intel_ring_get_tail(ring);
		1779
3243	Serge	1780	ret = ring->add_request(ring);
3031	serge	1781	if (ret) {
		1782	kfree(request);
2352	Serge	1783	return ret;
3031	serge	1784	}
2332	Serge	1785
3243	Serge	1786	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1787	request->ring = ring;
3031	serge	1788	request->tail = request_ring_position;
		1789	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1790	was_empty = list_empty(&ring->request_list);
		1791	list_add_tail(&request->list, &ring->request_list);
3031	serge	1792	request->file_priv = NULL;
2332	Serge	1793
3263	Serge	1794	if (file) {
		1795	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1796
3263	Serge	1797	spin_lock(&file_priv->mm.lock);
		1798	request->file_priv = file_priv;
		1799	list_add_tail(&request->client_list,
		1800	&file_priv->mm.request_list);
		1801	spin_unlock(&file_priv->mm.lock);
		1802	}
		1803
		1804	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1805	ring->outstanding_lazy_request = 0;
2332	Serge	1806
2360	Serge	1807	if (!dev_priv->mm.suspended) {
		1808	if (i915_enable_hangcheck) {
2352	Serge	1809	// mod_timer(&dev_priv->hangcheck_timer,
		1810	// jiffies +
		1811	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1812	}
3031	serge	1813	if (was_empty) {
2360	Serge	1814	queue_delayed_work(dev_priv->wq,
		1815	&dev_priv->mm.retire_work, HZ);
3031	serge	1816	intel_mark_busy(dev_priv->dev);
		1817	}
2360	Serge	1818	}
3031	serge	1819
		1820	if (out_seqno)
3243	Serge	1821	*out_seqno = request->seqno;
2352	Serge	1822	return 0;
		1823	}
2332	Serge	1824
3263	Serge	1825	static inline void
		1826	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1827	{
		1828	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1829
3263	Serge	1830	if (!file_priv)
		1831	return;
2332	Serge	1832
3263	Serge	1833	spin_lock(&file_priv->mm.lock);
		1834	if (request->file_priv) {
		1835	list_del(&request->client_list);
		1836	request->file_priv = NULL;
		1837	}
		1838	spin_unlock(&file_priv->mm.lock);
		1839	}
2332	Serge	1840
3031	serge	1841	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1842	struct intel_ring_buffer *ring)
		1843	{
		1844	while (!list_empty(&ring->request_list)) {
		1845	struct drm_i915_gem_request *request;
2332	Serge	1846
3031	serge	1847	request = list_first_entry(&ring->request_list,
		1848	struct drm_i915_gem_request,
		1849	list);
2332	Serge	1850
3031	serge	1851	list_del(&request->list);
3263	Serge	1852	i915_gem_request_remove_from_client(request);
3031	serge	1853	kfree(request);
		1854	}
2332	Serge	1855
3031	serge	1856	while (!list_empty(&ring->active_list)) {
		1857	struct drm_i915_gem_object *obj;
2332	Serge	1858
3031	serge	1859	obj = list_first_entry(&ring->active_list,
		1860	struct drm_i915_gem_object,
		1861	ring_list);
2332	Serge	1862
3031	serge	1863	i915_gem_object_move_to_inactive(obj);
		1864	}
		1865	}
2332	Serge	1866
3031	serge	1867	static void i915_gem_reset_fences(struct drm_device *dev)
		1868	{
		1869	struct drm_i915_private *dev_priv = dev->dev_private;
		1870	int i;
2332	Serge	1871
3031	serge	1872	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1873	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1874
3031	serge	1875	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1876
3031	serge	1877	if (reg->obj)
		1878	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1879
3031	serge	1880	reg->pin_count = 0;
		1881	reg->obj = NULL;
		1882	INIT_LIST_HEAD(®->lru_list);
		1883	}
2360	Serge	1884
3031	serge	1885	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1886	}
2360	Serge	1887
3031	serge	1888	void i915_gem_reset(struct drm_device *dev)
		1889	{
		1890	struct drm_i915_private *dev_priv = dev->dev_private;
		1891	struct drm_i915_gem_object *obj;
		1892	struct intel_ring_buffer *ring;
		1893	int i;
2360	Serge	1894
3031	serge	1895	for_each_ring(ring, dev_priv, i)
		1896	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1897
3031	serge	1898	/* Move everything out of the GPU domains to ensure we do any
		1899	* necessary invalidation upon reuse.
		1900	*/
		1901	list_for_each_entry(obj,
		1902	&dev_priv->mm.inactive_list,
		1903	mm_list)
		1904	{
		1905	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1906	}
2360	Serge	1907
3031	serge	1908	/* The fence registers are invalidated so clear them out */
		1909	i915_gem_reset_fences(dev);
		1910	}
2360	Serge	1911
2352	Serge	1912	/**
		1913	* This function clears the request list as sequence numbers are passed.
		1914	*/
3031	serge	1915	void
2352	Serge	1916	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1917	{
		1918	uint32_t seqno;
2332	Serge	1919
3263	Serge	1920	ENTER();
		1921
2352	Serge	1922	if (list_empty(&ring->request_list))
		1923	return;
2332	Serge	1924
2352	Serge	1925	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1926
3031	serge	1927	seqno = ring->get_seqno(ring, true);
2332	Serge	1928
2352	Serge	1929	while (!list_empty(&ring->request_list)) {
		1930	struct drm_i915_gem_request *request;
2332	Serge	1931
2352	Serge	1932	request = list_first_entry(&ring->request_list,
		1933	struct drm_i915_gem_request,
		1934	list);
2332	Serge	1935
2352	Serge	1936	if (!i915_seqno_passed(seqno, request->seqno))
		1937	break;
2332	Serge	1938
2352	Serge	1939	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1940	/* We know the GPU must have read the request to have
		1941	* sent us the seqno + interrupt, so use the position
		1942	* of tail of the request to update the last known position
		1943	* of the GPU head.
		1944	*/
		1945	ring->last_retired_head = request->tail;
2332	Serge	1946
2352	Serge	1947	list_del(&request->list);
3263	Serge	1948	i915_gem_request_remove_from_client(request);
2352	Serge	1949	kfree(request);
		1950	}
2332	Serge	1951
2352	Serge	1952	/* Move any buffers on the active list that are no longer referenced
		1953	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1954	*/
		1955	while (!list_empty(&ring->active_list)) {
		1956	struct drm_i915_gem_object *obj;
2332	Serge	1957
2352	Serge	1958	obj = list_first_entry(&ring->active_list,
		1959	struct drm_i915_gem_object,
		1960	ring_list);
2332	Serge	1961
3031	serge	1962	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1963	break;
2332	Serge	1964
2352	Serge	1965	i915_gem_object_move_to_inactive(obj);
		1966	}
2332	Serge	1967
2352	Serge	1968	if (unlikely(ring->trace_irq_seqno &&
		1969	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1970	ring->irq_put(ring);
		1971	ring->trace_irq_seqno = 0;
		1972	}
2332	Serge	1973
2352	Serge	1974	WARN_ON(i915_verify_lists(ring->dev));
3263	Serge	1975	LEAVE();
2352	Serge	1976	}
2332	Serge	1977
2352	Serge	1978	void
		1979	i915_gem_retire_requests(struct drm_device *dev)
		1980	{
		1981	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1982	struct intel_ring_buffer *ring;
2352	Serge	1983	int i;
2332	Serge	1984
3031	serge	1985	for_each_ring(ring, dev_priv, i)
		1986	i915_gem_retire_requests_ring(ring);
2352	Serge	1987	}
		1988
2360	Serge	1989	static void
		1990	i915_gem_retire_work_handler(struct work_struct *work)
		1991	{
		1992	drm_i915_private_t *dev_priv;
		1993	struct drm_device *dev;
3031	serge	1994	struct intel_ring_buffer *ring;
2360	Serge	1995	bool idle;
		1996	int i;
2352	Serge	1997
3263	Serge	1998	ENTER();
		1999
2360	Serge	2000	dev_priv = container_of(work, drm_i915_private_t,
		2001	mm.retire_work.work);
		2002	dev = dev_priv->dev;
2352	Serge	2003
2360	Serge	2004	/* Come back later if the device is busy... */
		2005	if (!mutex_trylock(&dev->struct_mutex)) {
		2006	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	2007	return;
2360	Serge	2008	}
2352	Serge	2009
2360	Serge	2010	i915_gem_retire_requests(dev);
2352	Serge	2011
2360	Serge	2012	/* Send a periodic flush down the ring so we don't hold onto GEM
		2013	* objects indefinitely.
		2014	*/
		2015	idle = true;
3031	serge	2016	for_each_ring(ring, dev_priv, i) {
		2017	if (ring->gpu_caches_dirty)
		2018	i915_add_request(ring, NULL, NULL);
2352	Serge	2019
2360	Serge	2020	idle &= list_empty(&ring->request_list);
		2021	}
2352	Serge	2022
2360	Serge	2023	if (!dev_priv->mm.suspended && !idle)
		2024	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	2025	if (idle)
		2026	intel_mark_idle(dev);
2360	Serge	2027
		2028	mutex_unlock(&dev->struct_mutex);
3263	Serge	2029
		2030	LEAVE();
2360	Serge	2031	}
		2032
2344	Serge	2033	/**
3031	serge	2034	* Ensures that an object will eventually get non-busy by flushing any required
		2035	* write domains, emitting any outstanding lazy request and retiring and
		2036	* completed requests.
2352	Serge	2037	*/
3031	serge	2038	static int
		2039	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2040	{
3031	serge	2041	int ret;
2352	Serge	2042
3031	serge	2043	if (obj->active) {
		2044	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2045	if (ret)
		2046	return ret;
2352	Serge	2047
3031	serge	2048	i915_gem_retire_requests_ring(obj->ring);
		2049	}
2352	Serge	2050
3031	serge	2051	return 0;
		2052	}
2352	Serge	2053
3243	Serge	2054	/**
		2055	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2056	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2057	*
		2058	* Returns 0 if successful, else an error is returned with the remaining time in
		2059	* the timeout parameter.
		2060	* -ETIME: object is still busy after timeout
		2061	* -ERESTARTSYS: signal interrupted the wait
		2062	* -ENONENT: object doesn't exist
		2063	* Also possible, but rare:
		2064	* -EAGAIN: GPU wedged
		2065	* -ENOMEM: damn
		2066	* -ENODEV: Internal IRQ fail
		2067	* -E?: The add request failed
		2068	*
		2069	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2070	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2071	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2072	* without holding struct_mutex the object may become re-busied before this
		2073	* function completes. A similar but shorter * race condition exists in the busy
		2074	* ioctl
		2075	*/
2352	Serge	2076
		2077
		2078
		2079
		2080
		2081
		2082
		2083
		2084
3243	Serge	2085
		2086
		2087
2352	Serge	2088	/**
3031	serge	2089	* i915_gem_object_sync - sync an object to a ring.
		2090	*
		2091	* @obj: object which may be in use on another ring.
		2092	* @to: ring we wish to use the object on. May be NULL.
		2093	*
		2094	* This code is meant to abstract object synchronization with the GPU.
		2095	* Calling with NULL implies synchronizing the object with the CPU
		2096	* rather than a particular GPU ring.
		2097	*
		2098	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2099	*/
		2100	int
3031	serge	2101	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2102	struct intel_ring_buffer *to)
2344	Serge	2103	{
3031	serge	2104	struct intel_ring_buffer *from = obj->ring;
		2105	u32 seqno;
		2106	int ret, idx;
2332	Serge	2107
3031	serge	2108	if (from == NULL \|\| to == from)
		2109	return 0;
2332	Serge	2110
3031	serge	2111	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2112	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2113
3031	serge	2114	idx = intel_ring_sync_index(from, to);
		2115
		2116	seqno = obj->last_read_seqno;
		2117	if (seqno <= from->sync_seqno[idx])
		2118	return 0;
		2119
		2120	ret = i915_gem_check_olr(obj->ring, seqno);
		2121	if (ret)
		2122	return ret;
		2123
		2124	ret = to->sync_to(to, from, seqno);
		2125	if (!ret)
3243	Serge	2126	/* We use last_read_seqno because sync_to()
		2127	* might have just caused seqno wrap under
		2128	* the radar.
		2129	*/
		2130	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2131
		2132	return ret;
2344	Serge	2133	}
2332	Serge	2134
2344	Serge	2135	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2136	{
		2137	u32 old_write_domain, old_read_domains;
2332	Serge	2138
2344	Serge	2139	/* Act a barrier for all accesses through the GTT */
		2140	mb();
2332	Serge	2141
2344	Serge	2142	/* Force a pagefault for domain tracking on next user access */
		2143	// i915_gem_release_mmap(obj);
2332	Serge	2144
2344	Serge	2145	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2146	return;
2332	Serge	2147
2344	Serge	2148	old_read_domains = obj->base.read_domains;
		2149	old_write_domain = obj->base.write_domain;
2351	Serge	2150
2344	Serge	2151	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2152	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2153
2351	Serge	2154	trace_i915_gem_object_change_domain(obj,
		2155	old_read_domains,
		2156	old_write_domain);
2344	Serge	2157	}
2332	Serge	2158
2344	Serge	2159	/**
		2160	* Unbinds an object from the GTT aperture.
		2161	*/
		2162	int
		2163	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2164	{
3031	serge	2165	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	2166	int ret = 0;
2332	Serge	2167
3263	Serge	2168	if(obj == get_fb_obj())
		2169	return 0;
		2170
2344	Serge	2171	if (obj->gtt_space == NULL)
		2172	return 0;
2332	Serge	2173
3031	serge	2174	if (obj->pin_count)
		2175	return -EBUSY;
2332	Serge	2176
3243	Serge	2177	BUG_ON(obj->pages == NULL);
3031	serge	2178
2344	Serge	2179	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2180	if (ret)
2344	Serge	2181	return ret;
		2182	/* Continue on if we fail due to EIO, the GPU is hung so we
		2183	* should be safe and we need to cleanup or else we might
		2184	* cause memory corruption through use-after-free.
		2185	*/
2332	Serge	2186
2344	Serge	2187	i915_gem_object_finish_gtt(obj);
2332	Serge	2188
2344	Serge	2189	/* release the fence reg _after_ flushing */
		2190	ret = i915_gem_object_put_fence(obj);
3031	serge	2191	if (ret)
2344	Serge	2192	return ret;
2332	Serge	2193
2351	Serge	2194	trace_i915_gem_object_unbind(obj);
2332	Serge	2195
3031	serge	2196	if (obj->has_global_gtt_mapping)
3243	Serge	2197	i915_gem_gtt_unbind_object(obj);
3031	serge	2198	if (obj->has_aliasing_ppgtt_mapping) {
		2199	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2200	obj->has_aliasing_ppgtt_mapping = 0;
		2201	}
		2202	i915_gem_gtt_finish_object(obj);
2332	Serge	2203
3031	serge	2204	list_del(&obj->mm_list);
		2205	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2206	/* Avoid an unnecessary call to unbind on rebind. */
		2207	obj->map_and_fenceable = true;
2332	Serge	2208
2344	Serge	2209	drm_mm_put_block(obj->gtt_space);
		2210	obj->gtt_space = NULL;
		2211	obj->gtt_offset = 0;
2332	Serge	2212
2344	Serge	2213	return 0;
		2214	}
2332	Serge	2215
3031	serge	2216	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2217	{
		2218	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2219	struct intel_ring_buffer *ring;
2344	Serge	2220	int ret, i;
2332	Serge	2221
2344	Serge	2222	/* Flush everything onto the inactive list. */
3031	serge	2223	for_each_ring(ring, dev_priv, i) {
		2224	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2225	if (ret)
		2226	return ret;
3031	serge	2227
3243	Serge	2228	ret = intel_ring_idle(ring);
3031	serge	2229	if (ret)
		2230	return ret;
2344	Serge	2231	}
2332	Serge	2232
2344	Serge	2233	return 0;
		2234	}
2332	Serge	2235
3031	serge	2236	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2237	struct drm_i915_gem_object *obj)
		2238	{
		2239	drm_i915_private_t *dev_priv = dev->dev_private;
		2240	uint64_t val;
2332	Serge	2241
3031	serge	2242	if (obj) {
		2243	u32 size = obj->gtt_space->size;
2332	Serge	2244
3031	serge	2245	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2246	0xfffff000) << 32;
		2247	val \|= obj->gtt_offset & 0xfffff000;
		2248	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2249	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2250
3031	serge	2251	if (obj->tiling_mode == I915_TILING_Y)
		2252	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2253	val \|= I965_FENCE_REG_VALID;
		2254	} else
		2255	val = 0;
2332	Serge	2256
3031	serge	2257	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2258	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2259	}
2332	Serge	2260
3031	serge	2261	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2262	struct drm_i915_gem_object *obj)
		2263	{
		2264	drm_i915_private_t *dev_priv = dev->dev_private;
		2265	uint64_t val;
2332	Serge	2266
3031	serge	2267	if (obj) {
		2268	u32 size = obj->gtt_space->size;
2332	Serge	2269
3031	serge	2270	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2271	0xfffff000) << 32;
		2272	val \|= obj->gtt_offset & 0xfffff000;
		2273	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2274	if (obj->tiling_mode == I915_TILING_Y)
		2275	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2276	val \|= I965_FENCE_REG_VALID;
		2277	} else
		2278	val = 0;
2332	Serge	2279
3031	serge	2280	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2281	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2282	}
2332	Serge	2283
3031	serge	2284	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2285	struct drm_i915_gem_object *obj)
		2286	{
		2287	drm_i915_private_t *dev_priv = dev->dev_private;
		2288	u32 val;
2332	Serge	2289
3031	serge	2290	if (obj) {
		2291	u32 size = obj->gtt_space->size;
		2292	int pitch_val;
		2293	int tile_width;
2332	Serge	2294
3031	serge	2295	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2296	(size & -size) != size \|\|
		2297	(obj->gtt_offset & (size - 1)),
		2298	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2299	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2300
3031	serge	2301	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2302	tile_width = 128;
		2303	else
		2304	tile_width = 512;
2332	Serge	2305
3031	serge	2306	/* Note: pitch better be a power of two tile widths */
		2307	pitch_val = obj->stride / tile_width;
		2308	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2309
3031	serge	2310	val = obj->gtt_offset;
		2311	if (obj->tiling_mode == I915_TILING_Y)
		2312	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2313	val \|= I915_FENCE_SIZE_BITS(size);
		2314	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2315	val \|= I830_FENCE_REG_VALID;
		2316	} else
		2317	val = 0;
2332	Serge	2318
3031	serge	2319	if (reg < 8)
		2320	reg = FENCE_REG_830_0 + reg * 4;
		2321	else
		2322	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2323
3031	serge	2324	I915_WRITE(reg, val);
		2325	POSTING_READ(reg);
		2326	}
2332	Serge	2327
3031	serge	2328	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2329	struct drm_i915_gem_object *obj)
		2330	{
		2331	drm_i915_private_t *dev_priv = dev->dev_private;
		2332	uint32_t val;
2344	Serge	2333
3031	serge	2334	if (obj) {
		2335	u32 size = obj->gtt_space->size;
		2336	uint32_t pitch_val;
2344	Serge	2337
3031	serge	2338	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2339	(size & -size) != size \|\|
		2340	(obj->gtt_offset & (size - 1)),
		2341	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2342	obj->gtt_offset, size);
2344	Serge	2343
3031	serge	2344	pitch_val = obj->stride / 128;
		2345	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2346
3031	serge	2347	val = obj->gtt_offset;
		2348	if (obj->tiling_mode == I915_TILING_Y)
		2349	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2350	val \|= I830_FENCE_SIZE_BITS(size);
		2351	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2352	val \|= I830_FENCE_REG_VALID;
		2353	} else
		2354	val = 0;
		2355
		2356	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2357	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2358	}
		2359
		2360	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2361	struct drm_i915_gem_object *obj)
2332	Serge	2362	{
3031	serge	2363	switch (INTEL_INFO(dev)->gen) {
		2364	case 7:
		2365	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2366	case 5:
		2367	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2368	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2369	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2370	default: break;
		2371	}
2344	Serge	2372	}
		2373
3031	serge	2374	static inline int fence_number(struct drm_i915_private *dev_priv,
		2375	struct drm_i915_fence_reg *fence)
2344	Serge	2376	{
3031	serge	2377	return fence - dev_priv->fence_regs;
		2378	}
2332	Serge	2379
3031	serge	2380	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2381	struct drm_i915_fence_reg *fence,
		2382	bool enable)
		2383	{
		2384	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2385	int reg = fence_number(dev_priv, fence);
2332	Serge	2386
3031	serge	2387	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2388
		2389	if (enable) {
		2390	obj->fence_reg = reg;
		2391	fence->obj = obj;
		2392	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2393	} else {
		2394	obj->fence_reg = I915_FENCE_REG_NONE;
		2395	fence->obj = NULL;
		2396	list_del_init(&fence->lru_list);
2344	Serge	2397	}
3031	serge	2398	}
2344	Serge	2399
3031	serge	2400	static int
		2401	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2402	{
		2403	if (obj->last_fenced_seqno) {
		2404	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2405	if (ret)
		2406	return ret;
2344	Serge	2407
		2408	obj->last_fenced_seqno = 0;
		2409	}
		2410
		2411	/* Ensure that all CPU reads are completed before installing a fence
		2412	* and all writes before removing the fence.
2332	Serge	2413	*/
2344	Serge	2414	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2415	mb();
2332	Serge	2416
3031	serge	2417	obj->fenced_gpu_access = false;
2332	Serge	2418	return 0;
		2419	}
		2420
		2421	int
2344	Serge	2422	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2423	{
3031	serge	2424	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2425	int ret;
		2426
3031	serge	2427	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2428	if (ret)
		2429	return ret;
		2430
3031	serge	2431	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2432	return 0;
2332	Serge	2433
3031	serge	2434	i915_gem_object_update_fence(obj,
		2435	&dev_priv->fence_regs[obj->fence_reg],
		2436	false);
		2437	i915_gem_object_fence_lost(obj);
2344	Serge	2438
2332	Serge	2439	return 0;
		2440	}
		2441
3031	serge	2442	static struct drm_i915_fence_reg *
		2443	i915_find_fence_reg(struct drm_device *dev)
		2444	{
		2445	struct drm_i915_private *dev_priv = dev->dev_private;
		2446	struct drm_i915_fence_reg reg, avail;
		2447	int i;
2332	Serge	2448
3031	serge	2449	/* First try to find a free reg */
		2450	avail = NULL;
		2451	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2452	reg = &dev_priv->fence_regs[i];
		2453	if (!reg->obj)
		2454	return reg;
2332	Serge	2455
3031	serge	2456	if (!reg->pin_count)
		2457	avail = reg;
		2458	}
2332	Serge	2459
3031	serge	2460	if (avail == NULL)
		2461	return NULL;
2332	Serge	2462
3031	serge	2463	/* None available, try to steal one or wait for a user to finish */
		2464	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2465	if (reg->pin_count)
		2466	continue;
2332	Serge	2467
3031	serge	2468	return reg;
		2469	}
2332	Serge	2470
3031	serge	2471	return NULL;
		2472	}
2332	Serge	2473
3031	serge	2474	/**
		2475	* i915_gem_object_get_fence - set up fencing for an object
		2476	* @obj: object to map through a fence reg
		2477	*
		2478	* When mapping objects through the GTT, userspace wants to be able to write
		2479	* to them without having to worry about swizzling if the object is tiled.
		2480	* This function walks the fence regs looking for a free one for @obj,
		2481	* stealing one if it can't find any.
		2482	*
		2483	* It then sets up the reg based on the object's properties: address, pitch
		2484	* and tiling format.
		2485	*
		2486	* For an untiled surface, this removes any existing fence.
		2487	*/
		2488	int
		2489	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2490	{
		2491	struct drm_device *dev = obj->base.dev;
		2492	struct drm_i915_private *dev_priv = dev->dev_private;
		2493	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2494	struct drm_i915_fence_reg *reg;
		2495	int ret;
2332	Serge	2496
3031	serge	2497	/* Have we updated the tiling parameters upon the object and so
		2498	* will need to serialise the write to the associated fence register?
		2499	*/
		2500	if (obj->fence_dirty) {
		2501	ret = i915_gem_object_flush_fence(obj);
		2502	if (ret)
		2503	return ret;
		2504	}
2332	Serge	2505
3031	serge	2506	/* Just update our place in the LRU if our fence is getting reused. */
		2507	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2508	reg = &dev_priv->fence_regs[obj->fence_reg];
		2509	if (!obj->fence_dirty) {
		2510	list_move_tail(®->lru_list,
		2511	&dev_priv->mm.fence_list);
		2512	return 0;
		2513	}
		2514	} else if (enable) {
		2515	reg = i915_find_fence_reg(dev);
		2516	if (reg == NULL)
		2517	return -EDEADLK;
2332	Serge	2518
3031	serge	2519	if (reg->obj) {
		2520	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2521
3031	serge	2522	ret = i915_gem_object_flush_fence(old);
		2523	if (ret)
		2524	return ret;
2332	Serge	2525
3031	serge	2526	i915_gem_object_fence_lost(old);
		2527	}
		2528	} else
		2529	return 0;
2332	Serge	2530
3031	serge	2531	i915_gem_object_update_fence(obj, reg, enable);
		2532	obj->fence_dirty = false;
2332	Serge	2533
3031	serge	2534	return 0;
		2535	}
2332	Serge	2536
3031	serge	2537	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2538	struct drm_mm_node *gtt_space,
		2539	unsigned long cache_level)
		2540	{
		2541	struct drm_mm_node *other;
2332	Serge	2542
3031	serge	2543	/* On non-LLC machines we have to be careful when putting differing
		2544	* types of snoopable memory together to avoid the prefetcher
		2545	* crossing memory domains and dieing.
		2546	*/
		2547	if (HAS_LLC(dev))
		2548	return true;
2332	Serge	2549
3031	serge	2550	if (gtt_space == NULL)
		2551	return true;
2332	Serge	2552
3031	serge	2553	if (list_empty(>t_space->node_list))
		2554	return true;
2332	Serge	2555
3031	serge	2556	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2557	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2558	return false;
2344	Serge	2559
3031	serge	2560	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2561	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2562	return false;
2344	Serge	2563
3031	serge	2564	return true;
		2565	}
2344	Serge	2566
3031	serge	2567	static void i915_gem_verify_gtt(struct drm_device *dev)
		2568	{
		2569	#if WATCH_GTT
		2570	struct drm_i915_private *dev_priv = dev->dev_private;
		2571	struct drm_i915_gem_object *obj;
		2572	int err = 0;
2344	Serge	2573
3031	serge	2574	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2575	if (obj->gtt_space == NULL) {
		2576	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2577	err++;
		2578	continue;
		2579	}
2344	Serge	2580
3031	serge	2581	if (obj->cache_level != obj->gtt_space->color) {
		2582	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2583	obj->gtt_space->start,
		2584	obj->gtt_space->start + obj->gtt_space->size,
		2585	obj->cache_level,
		2586	obj->gtt_space->color);
		2587	err++;
		2588	continue;
		2589	}
2344	Serge	2590
3031	serge	2591	if (!i915_gem_valid_gtt_space(dev,
		2592	obj->gtt_space,
		2593	obj->cache_level)) {
		2594	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2595	obj->gtt_space->start,
		2596	obj->gtt_space->start + obj->gtt_space->size,
		2597	obj->cache_level);
		2598	err++;
		2599	continue;
		2600	}
		2601	}
2344	Serge	2602
3031	serge	2603	WARN_ON(err);
		2604	#endif
2326	Serge	2605	}
		2606
2332	Serge	2607	/**
		2608	* Finds free space in the GTT aperture and binds the object there.
		2609	*/
		2610	static int
		2611	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2612	unsigned alignment,
3031	serge	2613	bool map_and_fenceable,
		2614	bool nonblocking)
2332	Serge	2615	{
		2616	struct drm_device *dev = obj->base.dev;
		2617	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2618	struct drm_mm_node *node;
2332	Serge	2619	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2620	bool mappable, fenceable;
		2621	int ret;
2326	Serge	2622
2332	Serge	2623	if (obj->madv != I915_MADV_WILLNEED) {
3243	Serge	2624	DRM_ERROR("Attempting to bind a purgeable object\n");
2332	Serge	2625	return -EINVAL;
		2626	}
		2627
		2628	fence_size = i915_gem_get_gtt_size(dev,
		2629	obj->base.size,
		2630	obj->tiling_mode);
		2631	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2632	obj->base.size,
		2633	obj->tiling_mode);
		2634	unfenced_alignment =
		2635	i915_gem_get_unfenced_gtt_alignment(dev,
		2636	obj->base.size,
		2637	obj->tiling_mode);
		2638
		2639	if (alignment == 0)
		2640	alignment = map_and_fenceable ? fence_alignment :
		2641	unfenced_alignment;
		2642	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2643	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2644	return -EINVAL;
		2645	}
		2646
		2647	size = map_and_fenceable ? fence_size : obj->base.size;
		2648
		2649	/* If the object is bigger than the entire aperture, reject it early
		2650	* before evicting everything in a vain attempt to find space.
		2651	*/
		2652	if (obj->base.size >
		2653	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2654	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2655	return -E2BIG;
		2656	}
		2657
3031	serge	2658	ret = i915_gem_object_get_pages(obj);
		2659	if (ret)
		2660	return ret;
		2661
3243	Serge	2662	i915_gem_object_pin_pages(obj);
		2663
		2664	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2665	if (node == NULL) {
		2666	i915_gem_object_unpin_pages(obj);
		2667	return -ENOMEM;
		2668	}
		2669
2332	Serge	2670	search_free:
		2671	if (map_and_fenceable)
3243	Serge	2672	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2673	size, alignment, obj->cache_level,
3243	Serge	2674	0, dev_priv->mm.gtt_mappable_end);
2332	Serge	2675	else
3243	Serge	2676	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2677	size, alignment, obj->cache_level);
		2678	if (ret) {
2332	Serge	2679
3243	Serge	2680	i915_gem_object_unpin_pages(obj);
		2681	kfree(node);
2332	Serge	2682	return ret;
		2683	}
3243	Serge	2684	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2685	i915_gem_object_unpin_pages(obj);
		2686	drm_mm_put_block(node);
3031	serge	2687	return -EINVAL;
		2688	}
2332	Serge	2689
3031	serge	2690	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2691	if (ret) {
3243	Serge	2692	i915_gem_object_unpin_pages(obj);
		2693	drm_mm_put_block(node);
2332	Serge	2694	return ret;
		2695	}
		2696
3031	serge	2697	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2698	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2699
3243	Serge	2700	obj->gtt_space = node;
		2701	obj->gtt_offset = node->start;
2332	Serge	2702
		2703	fenceable =
3243	Serge	2704	node->size == fence_size &&
		2705	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2706
		2707	mappable =
		2708	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2709
		2710	obj->map_and_fenceable = mappable && fenceable;
		2711
3243	Serge	2712	i915_gem_object_unpin_pages(obj);
2351	Serge	2713	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2714	i915_gem_verify_gtt(dev);
2332	Serge	2715	return 0;
		2716	}
		2717
		2718	void
		2719	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2720	{
		2721	/* If we don't have a page list set up, then we're not pinned
		2722	* to GPU, and we can ignore the cache flush because it'll happen
		2723	* again at bind time.
		2724	*/
3243	Serge	2725	if (obj->pages == NULL)
2332	Serge	2726	return;
		2727
		2728	/* If the GPU is snooping the contents of the CPU cache,
		2729	* we do not need to manually clear the CPU cache lines. However,
		2730	* the caches are only snooped when the render cache is
		2731	* flushed/invalidated. As we always have to emit invalidations
		2732	* and flushes when moving into and out of the RENDER domain, correct
		2733	* snooping behaviour occurs naturally as the result of our domain
		2734	* tracking.
		2735	*/
		2736	if (obj->cache_level != I915_CACHE_NONE)
		2737	return;
3243	Serge	2738	#if 0
2344	Serge	2739	if(obj->mapped != NULL)
		2740	{
		2741	uint8_t *page_virtual;
		2742	unsigned int i;
2332	Serge	2743
2344	Serge	2744	page_virtual = obj->mapped;
		2745	asm volatile("mfence");
		2746	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2747	clflush(page_virtual + i);
		2748	asm volatile("mfence");
		2749	}
		2750	else
		2751	{
		2752	uint8_t *page_virtual;
		2753	unsigned int i;
		2754	page_virtual = AllocKernelSpace(obj->base.size);
		2755	if(page_virtual != NULL)
		2756	{
3031	serge	2757	dma_addr_t src, dst;
2344	Serge	2758	u32 count;
		2759
		2760	#define page_tabs 0xFDC00000 /* really dirty hack */
		2761
3031	serge	2762	src = obj->pages.page;
		2763	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2764	count = obj->base.size/4096;
		2765
		2766	while(count--)
		2767	{
		2768	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2769	};
		2770
		2771	asm volatile("mfence");
		2772	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2773	clflush(page_virtual + i);
		2774	asm volatile("mfence");
		2775	FreeKernelSpace(page_virtual);
		2776	}
		2777	else
		2778	{
		2779	asm volatile (
		2780	"mfence \n"
		2781	"wbinvd \n" /* this is really ugly */
		2782	"mfence");
		2783	}
		2784	}
3243	Serge	2785	#endif
		2786
2332	Serge	2787	}
		2788
2344	Serge	2789	/** Flushes the GTT write domain for the object if it's dirty. */
		2790	static void
		2791	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2792	{
		2793	uint32_t old_write_domain;
2332	Serge	2794
2344	Serge	2795	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2796	return;
2332	Serge	2797
2344	Serge	2798	/* No actual flushing is required for the GTT write domain. Writes
		2799	* to it immediately go to main memory as far as we know, so there's
		2800	* no chipset flush. It also doesn't land in render cache.
		2801	*
		2802	* However, we do have to enforce the order so that all writes through
		2803	* the GTT land before any writes to the device, such as updates to
		2804	* the GATT itself.
		2805	*/
		2806	wmb();
2332	Serge	2807
2344	Serge	2808	old_write_domain = obj->base.write_domain;
		2809	obj->base.write_domain = 0;
2332	Serge	2810
2351	Serge	2811	trace_i915_gem_object_change_domain(obj,
		2812	obj->base.read_domains,
		2813	old_write_domain);
2344	Serge	2814	}
2332	Serge	2815
		2816	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2817	static void
2332	Serge	2818	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2819	{
		2820	uint32_t old_write_domain;
		2821
		2822	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2823	return;
		2824
		2825	i915_gem_clflush_object(obj);
3243	Serge	2826	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2827	old_write_domain = obj->base.write_domain;
		2828	obj->base.write_domain = 0;
		2829
2351	Serge	2830	trace_i915_gem_object_change_domain(obj,
		2831	obj->base.read_domains,
		2832	old_write_domain);
2332	Serge	2833	}
		2834
		2835	/**
		2836	* Moves a single object to the GTT read, and possibly write domain.
		2837	*
		2838	* This function returns when the move is complete, including waiting on
		2839	* flushes to occur.
		2840	*/
		2841	int
		2842	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2843	{
3031	serge	2844	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2845	uint32_t old_write_domain, old_read_domains;
		2846	int ret;
		2847
		2848	/* Not valid to be called on unbound objects. */
		2849	if (obj->gtt_space == NULL)
		2850	return -EINVAL;
		2851
		2852	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2853	return 0;
		2854
3031	serge	2855	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2856	if (ret)
		2857	return ret;
		2858
		2859	i915_gem_object_flush_cpu_write_domain(obj);
		2860
		2861	old_write_domain = obj->base.write_domain;
		2862	old_read_domains = obj->base.read_domains;
		2863
		2864	/* It should now be out of any other write domains, and we can update
		2865	* the domain values for our changes.
		2866	*/
		2867	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2868	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2869	if (write) {
		2870	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2871	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2872	obj->dirty = 1;
		2873	}
		2874
2351	Serge	2875	trace_i915_gem_object_change_domain(obj,
		2876	old_read_domains,
		2877	old_write_domain);
		2878
3031	serge	2879	/* And bump the LRU for this access */
		2880	if (i915_gem_object_is_inactive(obj))
		2881	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2882
2332	Serge	2883	return 0;
		2884	}
		2885
2335	Serge	2886	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2887	enum i915_cache_level cache_level)
		2888	{
3031	serge	2889	struct drm_device *dev = obj->base.dev;
		2890	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2891	int ret;
2332	Serge	2892
2335	Serge	2893	if (obj->cache_level == cache_level)
		2894	return 0;
2332	Serge	2895
2335	Serge	2896	if (obj->pin_count) {
		2897	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2898	return -EBUSY;
		2899	}
2332	Serge	2900
3031	serge	2901	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2902	ret = i915_gem_object_unbind(obj);
		2903	if (ret)
		2904	return ret;
		2905	}
		2906
2335	Serge	2907	if (obj->gtt_space) {
		2908	ret = i915_gem_object_finish_gpu(obj);
		2909	if (ret)
		2910	return ret;
2332	Serge	2911
2335	Serge	2912	i915_gem_object_finish_gtt(obj);
2332	Serge	2913
2335	Serge	2914	/* Before SandyBridge, you could not use tiling or fence
		2915	* registers with snooped memory, so relinquish any fences
		2916	* currently pointing to our region in the aperture.
		2917	*/
3031	serge	2918	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2919	ret = i915_gem_object_put_fence(obj);
		2920	if (ret)
		2921	return ret;
		2922	}
2332	Serge	2923
3031	serge	2924	if (obj->has_global_gtt_mapping)
		2925	i915_gem_gtt_bind_object(obj, cache_level);
		2926	if (obj->has_aliasing_ppgtt_mapping)
		2927	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2928	obj, cache_level);
		2929
		2930	obj->gtt_space->color = cache_level;
2335	Serge	2931	}
2332	Serge	2932
2335	Serge	2933	if (cache_level == I915_CACHE_NONE) {
		2934	u32 old_read_domains, old_write_domain;
2332	Serge	2935
2335	Serge	2936	/* If we're coming from LLC cached, then we haven't
		2937	* actually been tracking whether the data is in the
		2938	* CPU cache or not, since we only allow one bit set
		2939	* in obj->write_domain and have been skipping the clflushes.
		2940	* Just set it to the CPU cache for now.
		2941	*/
		2942	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2943	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2944
2335	Serge	2945	old_read_domains = obj->base.read_domains;
		2946	old_write_domain = obj->base.write_domain;
2332	Serge	2947
2335	Serge	2948	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2949	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2950
2351	Serge	2951	trace_i915_gem_object_change_domain(obj,
		2952	old_read_domains,
		2953	old_write_domain);
2344	Serge	2954	}
2332	Serge	2955
2335	Serge	2956	obj->cache_level = cache_level;
3031	serge	2957	i915_gem_verify_gtt(dev);
2335	Serge	2958	return 0;
		2959	}
2332	Serge	2960
3260	Serge	2961	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		2962	struct drm_file *file)
		2963	{
		2964	struct drm_i915_gem_caching *args = data;
		2965	struct drm_i915_gem_object *obj;
		2966	int ret;
		2967
		2968	ret = i915_mutex_lock_interruptible(dev);
		2969	if (ret)
		2970	return ret;
		2971
		2972	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2973	if (&obj->base == NULL) {
		2974	ret = -ENOENT;
		2975	goto unlock;
		2976	}
		2977
		2978	args->caching = obj->cache_level != I915_CACHE_NONE;
		2979
		2980	drm_gem_object_unreference(&obj->base);
		2981	unlock:
		2982	mutex_unlock(&dev->struct_mutex);
		2983	return ret;
		2984	}
		2985
		2986	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		2987	struct drm_file *file)
		2988	{
		2989	struct drm_i915_gem_caching *args = data;
		2990	struct drm_i915_gem_object *obj;
		2991	enum i915_cache_level level;
		2992	int ret;
		2993
		2994	switch (args->caching) {
		2995	case I915_CACHING_NONE:
		2996	level = I915_CACHE_NONE;
		2997	break;
		2998	case I915_CACHING_CACHED:
		2999	level = I915_CACHE_LLC;
		3000	break;
		3001	default:
		3002	return -EINVAL;
		3003	}
		3004
		3005	ret = i915_mutex_lock_interruptible(dev);
		3006	if (ret)
		3007	return ret;
		3008
		3009	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3010	if (&obj->base == NULL) {
		3011	ret = -ENOENT;
		3012	goto unlock;
		3013	}
		3014
		3015	ret = i915_gem_object_set_cache_level(obj, level);
		3016
		3017	drm_gem_object_unreference(&obj->base);
		3018	unlock:
		3019	mutex_unlock(&dev->struct_mutex);
		3020	return ret;
		3021	}
		3022
2335	Serge	3023	/*
		3024	* Prepare buffer for display plane (scanout, cursors, etc).
		3025	* Can be called from an uninterruptible phase (modesetting) and allows
		3026	* any flushes to be pipelined (for pageflips).
		3027	*/
		3028	int
		3029	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3030	u32 alignment,
		3031	struct intel_ring_buffer *pipelined)
		3032	{
		3033	u32 old_read_domains, old_write_domain;
		3034	int ret;
2332	Serge	3035
3031	serge	3036	if (pipelined != obj->ring) {
		3037	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3038	if (ret)
		3039	return ret;
		3040	}
2332	Serge	3041
2335	Serge	3042	/* The display engine is not coherent with the LLC cache on gen6. As
		3043	* a result, we make sure that the pinning that is about to occur is
		3044	* done with uncached PTEs. This is lowest common denominator for all
		3045	* chipsets.
		3046	*
		3047	* However for gen6+, we could do better by using the GFDT bit instead
		3048	* of uncaching, which would allow us to flush all the LLC-cached data
		3049	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3050	*/
2360	Serge	3051	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3052	if (ret)
		3053	return ret;
2332	Serge	3054
2335	Serge	3055	/* As the user may map the buffer once pinned in the display plane
		3056	* (e.g. libkms for the bootup splash), we have to ensure that we
		3057	* always use map_and_fenceable for all scanout buffers.
		3058	*/
3031	serge	3059	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3060	if (ret)
		3061	return ret;
2332	Serge	3062
2335	Serge	3063	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3064
2335	Serge	3065	old_write_domain = obj->base.write_domain;
		3066	old_read_domains = obj->base.read_domains;
2332	Serge	3067
2335	Serge	3068	/* It should now be out of any other write domains, and we can update
		3069	* the domain values for our changes.
		3070	*/
3031	serge	3071	obj->base.write_domain = 0;
2335	Serge	3072	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3073
2351	Serge	3074	trace_i915_gem_object_change_domain(obj,
		3075	old_read_domains,
		3076	old_write_domain);
2332	Serge	3077
2335	Serge	3078	return 0;
		3079	}
2332	Serge	3080
2344	Serge	3081	int
		3082	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3083	{
		3084	int ret;
2332	Serge	3085
2344	Serge	3086	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3087	return 0;
2332	Serge	3088
3031	serge	3089	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3090	if (ret)
		3091	return ret;
2332	Serge	3092
2344	Serge	3093	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3094	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3095	return 0;
2344	Serge	3096	}
2332	Serge	3097
2344	Serge	3098	/**
		3099	* Moves a single object to the CPU read, and possibly write domain.
		3100	*
		3101	* This function returns when the move is complete, including waiting on
		3102	* flushes to occur.
		3103	*/
3031	serge	3104	int
2344	Serge	3105	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3106	{
		3107	uint32_t old_write_domain, old_read_domains;
		3108	int ret;
2332	Serge	3109
2344	Serge	3110	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3111	return 0;
2332	Serge	3112
3031	serge	3113	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3114	if (ret)
		3115	return ret;
2332	Serge	3116
2344	Serge	3117	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3118
2344	Serge	3119	old_write_domain = obj->base.write_domain;
		3120	old_read_domains = obj->base.read_domains;
2332	Serge	3121
2344	Serge	3122	/* Flush the CPU cache if it's still invalid. */
		3123	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3124	i915_gem_clflush_object(obj);
2332	Serge	3125
2344	Serge	3126	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3127	}
2332	Serge	3128
2344	Serge	3129	/* It should now be out of any other write domains, and we can update
		3130	* the domain values for our changes.
		3131	*/
		3132	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3133
2344	Serge	3134	/* If we're writing through the CPU, then the GPU read domains will
		3135	* need to be invalidated at next use.
		3136	*/
		3137	if (write) {
		3138	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3139	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3140	}
2332	Serge	3141
2351	Serge	3142	trace_i915_gem_object_change_domain(obj,
		3143	old_read_domains,
		3144	old_write_domain);
2332	Serge	3145
2344	Serge	3146	return 0;
		3147	}
2332	Serge	3148
3031	serge	3149	/* Throttle our rendering by waiting until the ring has completed our requests
		3150	* emitted over 20 msec ago.
2344	Serge	3151	*
3031	serge	3152	* Note that if we were to use the current jiffies each time around the loop,
		3153	* we wouldn't escape the function with any frames outstanding if the time to
		3154	* render a frame was over 20ms.
		3155	*
		3156	* This should get us reasonable parallelism between CPU and GPU but also
		3157	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3158	*/
3031	serge	3159	static int
		3160	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3161	{
3031	serge	3162	struct drm_i915_private *dev_priv = dev->dev_private;
		3163	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3164	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3165	struct drm_i915_gem_request *request;
		3166	struct intel_ring_buffer *ring = NULL;
		3167	u32 seqno = 0;
		3168	int ret;
2332	Serge	3169
3031	serge	3170	if (atomic_read(&dev_priv->mm.wedged))
		3171	return -EIO;
2332	Serge	3172
3031	serge	3173	spin_lock(&file_priv->mm.lock);
		3174	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3175	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3176	break;
2332	Serge	3177
3031	serge	3178	ring = request->ring;
		3179	seqno = request->seqno;
		3180	}
		3181	spin_unlock(&file_priv->mm.lock);
2332	Serge	3182
3031	serge	3183	if (seqno == 0)
		3184	return 0;
2332	Serge	3185
3031	serge	3186	ret = __wait_seqno(ring, seqno, true, NULL);
		3187	if (ret == 0)
		3188	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3189
3031	serge	3190	return ret;
2352	Serge	3191	}
2332	Serge	3192
		3193	int
		3194	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3195	uint32_t alignment,
3031	serge	3196	bool map_and_fenceable,
		3197	bool nonblocking)
2332	Serge	3198	{
		3199	int ret;
		3200
3031	serge	3201	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3202	return -EBUSY;
2332	Serge	3203
		3204	if (obj->gtt_space != NULL) {
		3205	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3206	(map_and_fenceable && !obj->map_and_fenceable)) {
		3207	WARN(obj->pin_count,
		3208	"bo is already pinned with incorrect alignment:"
		3209	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3210	" obj->map_and_fenceable=%d\n",
		3211	obj->gtt_offset, alignment,
		3212	map_and_fenceable,
		3213	obj->map_and_fenceable);
		3214	ret = i915_gem_object_unbind(obj);
		3215	if (ret)
		3216	return ret;
		3217	}
		3218	}
		3219
		3220	if (obj->gtt_space == NULL) {
3243	Serge	3221	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3222
2332	Serge	3223	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3224	map_and_fenceable,
		3225	nonblocking);
2332	Serge	3226	if (ret)
		3227	return ret;
3243	Serge	3228
		3229	if (!dev_priv->mm.aliasing_ppgtt)
		3230	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3231	}
		3232
3031	serge	3233	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3234	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3235
		3236	obj->pin_count++;
2332	Serge	3237	obj->pin_mappable \|= map_and_fenceable;
		3238
		3239	return 0;
		3240	}
		3241
2344	Serge	3242	void
		3243	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3244	{
		3245	BUG_ON(obj->pin_count == 0);
		3246	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3247
3031	serge	3248	if (--obj->pin_count == 0)
2344	Serge	3249	obj->pin_mappable = false;
		3250	}
2332	Serge	3251
3031	serge	3252	int
		3253	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3254	struct drm_file *file)
		3255	{
		3256	struct drm_i915_gem_pin *args = data;
		3257	struct drm_i915_gem_object *obj;
		3258	int ret;
2332	Serge	3259
3031	serge	3260	ret = i915_mutex_lock_interruptible(dev);
		3261	if (ret)
		3262	return ret;
2332	Serge	3263
3031	serge	3264	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3265	if (&obj->base == NULL) {
		3266	ret = -ENOENT;
		3267	goto unlock;
		3268	}
2332	Serge	3269
3031	serge	3270	if (obj->madv != I915_MADV_WILLNEED) {
		3271	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3272	ret = -EINVAL;
		3273	goto out;
		3274	}
2332	Serge	3275
3031	serge	3276	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3277	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3278	args->handle);
		3279	ret = -EINVAL;
		3280	goto out;
		3281	}
2332	Serge	3282
3243	Serge	3283	if (obj->user_pin_count == 0) {
3031	serge	3284	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3285	if (ret)
		3286	goto out;
		3287	}
2332	Serge	3288
3243	Serge	3289	obj->user_pin_count++;
		3290	obj->pin_filp = file;
		3291
3031	serge	3292	/* XXX - flush the CPU caches for pinned objects
		3293	* as the X server doesn't manage domains yet
		3294	*/
		3295	i915_gem_object_flush_cpu_write_domain(obj);
		3296	args->offset = obj->gtt_offset;
		3297	out:
		3298	drm_gem_object_unreference(&obj->base);
		3299	unlock:
		3300	mutex_unlock(&dev->struct_mutex);
		3301	return ret;
		3302	}
2332	Serge	3303
3255	Serge	3304	#if 0
		3305
3031	serge	3306	int
		3307	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3308	struct drm_file *file)
		3309	{
		3310	struct drm_i915_gem_pin *args = data;
		3311	struct drm_i915_gem_object *obj;
		3312	int ret;
2332	Serge	3313
3031	serge	3314	ret = i915_mutex_lock_interruptible(dev);
		3315	if (ret)
		3316	return ret;
2332	Serge	3317
3031	serge	3318	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3319	if (&obj->base == NULL) {
		3320	ret = -ENOENT;
		3321	goto unlock;
		3322	}
2332	Serge	3323
3031	serge	3324	if (obj->pin_filp != file) {
		3325	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3326	args->handle);
		3327	ret = -EINVAL;
		3328	goto out;
		3329	}
		3330	obj->user_pin_count--;
		3331	if (obj->user_pin_count == 0) {
		3332	obj->pin_filp = NULL;
		3333	i915_gem_object_unpin(obj);
		3334	}
2332	Serge	3335
3031	serge	3336	out:
		3337	drm_gem_object_unreference(&obj->base);
		3338	unlock:
		3339	mutex_unlock(&dev->struct_mutex);
		3340	return ret;
		3341	}
2332	Serge	3342
3260	Serge	3343	#endif
		3344
3031	serge	3345	int
		3346	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3347	struct drm_file *file)
		3348	{
		3349	struct drm_i915_gem_busy *args = data;
		3350	struct drm_i915_gem_object *obj;
		3351	int ret;
2332	Serge	3352
3031	serge	3353	ret = i915_mutex_lock_interruptible(dev);
		3354	if (ret)
		3355	return ret;
2332	Serge	3356
3031	serge	3357	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3358	if (&obj->base == NULL) {
		3359	ret = -ENOENT;
		3360	goto unlock;
		3361	}
2332	Serge	3362
3031	serge	3363	/* Count all active objects as busy, even if they are currently not used
		3364	* by the gpu. Users of this interface expect objects to eventually
		3365	* become non-busy without any further actions, therefore emit any
		3366	* necessary flushes here.
		3367	*/
		3368	ret = i915_gem_object_flush_active(obj);
2332	Serge	3369
3031	serge	3370	args->busy = obj->active;
		3371	if (obj->ring) {
		3372	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3373	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3374	}
2332	Serge	3375
3031	serge	3376	drm_gem_object_unreference(&obj->base);
		3377	unlock:
		3378	mutex_unlock(&dev->struct_mutex);
		3379	return ret;
		3380	}
2332	Serge	3381
3031	serge	3382	int
		3383	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3384	struct drm_file *file_priv)
		3385	{
		3386	return i915_gem_ring_throttle(dev, file_priv);
		3387	}
2332	Serge	3388
3263	Serge	3389	#if 0
		3390
3031	serge	3391	int
		3392	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3393	struct drm_file *file_priv)
		3394	{
		3395	struct drm_i915_gem_madvise *args = data;
		3396	struct drm_i915_gem_object *obj;
		3397	int ret;
2332	Serge	3398
3031	serge	3399	switch (args->madv) {
		3400	case I915_MADV_DONTNEED:
		3401	case I915_MADV_WILLNEED:
		3402	break;
		3403	default:
		3404	return -EINVAL;
		3405	}
2332	Serge	3406
3031	serge	3407	ret = i915_mutex_lock_interruptible(dev);
		3408	if (ret)
		3409	return ret;
2332	Serge	3410
3031	serge	3411	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3412	if (&obj->base == NULL) {
		3413	ret = -ENOENT;
		3414	goto unlock;
		3415	}
2332	Serge	3416
3031	serge	3417	if (obj->pin_count) {
		3418	ret = -EINVAL;
		3419	goto out;
		3420	}
2332	Serge	3421
3031	serge	3422	if (obj->madv != __I915_MADV_PURGED)
		3423	obj->madv = args->madv;
2332	Serge	3424
3031	serge	3425	/* if the object is no longer attached, discard its backing storage */
		3426	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3427	i915_gem_object_truncate(obj);
2332	Serge	3428
3031	serge	3429	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3430
3031	serge	3431	out:
		3432	drm_gem_object_unreference(&obj->base);
		3433	unlock:
		3434	mutex_unlock(&dev->struct_mutex);
		3435	return ret;
		3436	}
		3437	#endif
2332	Serge	3438
3031	serge	3439	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3440	const struct drm_i915_gem_object_ops *ops)
		3441	{
		3442	INIT_LIST_HEAD(&obj->mm_list);
		3443	INIT_LIST_HEAD(&obj->gtt_list);
		3444	INIT_LIST_HEAD(&obj->ring_list);
		3445	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3446
3031	serge	3447	obj->ops = ops;
		3448
		3449	obj->fence_reg = I915_FENCE_REG_NONE;
		3450	obj->madv = I915_MADV_WILLNEED;
		3451	/* Avoid an unnecessary call to unbind on the first bind. */
		3452	obj->map_and_fenceable = true;
		3453
		3454	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3455	}
		3456
		3457	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3458	.get_pages = i915_gem_object_get_pages_gtt,
		3459	.put_pages = i915_gem_object_put_pages_gtt,
		3460	};
		3461
2332	Serge	3462	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3463	size_t size)
		3464	{
		3465	struct drm_i915_gem_object *obj;
3031	serge	3466	struct address_space *mapping;
		3467	u32 mask;
2340	Serge	3468
2332	Serge	3469	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3470	if (obj == NULL)
		3471	return NULL;
		3472
		3473	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3474	kfree(obj);
		3475	return NULL;
		3476	}
		3477
		3478
3031	serge	3479	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3480
		3481	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3482	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3483
3031	serge	3484	if (HAS_LLC(dev)) {
		3485	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3486	* cache) for about a 10% performance improvement
		3487	* compared to uncached. Graphics requests other than
		3488	* display scanout are coherent with the CPU in
		3489	* accessing this cache. This means in this mode we
		3490	* don't need to clflush on the CPU side, and on the
		3491	* GPU side we only need to flush internal caches to
		3492	* get data visible to the CPU.
		3493	*
		3494	* However, we maintain the display planes as UC, and so
		3495	* need to rebind when first used as such.
		3496	*/
		3497	obj->cache_level = I915_CACHE_LLC;
		3498	} else
		3499	obj->cache_level = I915_CACHE_NONE;
		3500
		3501	return obj;
		3502	}
		3503
2344	Serge	3504	int i915_gem_init_object(struct drm_gem_object *obj)
		3505	{
		3506	BUG();
2332	Serge	3507
2344	Serge	3508	return 0;
		3509	}
2332	Serge	3510
3031	serge	3511	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3512	{
3031	serge	3513	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3514	struct drm_device *dev = obj->base.dev;
		3515	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3516
3031	serge	3517	trace_i915_gem_object_destroy(obj);
		3518
		3519	// if (obj->phys_obj)
		3520	// i915_gem_detach_phys_object(dev, obj);
		3521
		3522	obj->pin_count = 0;
		3523	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3524	bool was_interruptible;
		3525
		3526	was_interruptible = dev_priv->mm.interruptible;
		3527	dev_priv->mm.interruptible = false;
		3528
		3529	WARN_ON(i915_gem_object_unbind(obj));
		3530
		3531	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3532	}
2332	Serge	3533
3031	serge	3534	obj->pages_pin_count = 0;
		3535	i915_gem_object_put_pages(obj);
		3536	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3537
3243	Serge	3538	BUG_ON(obj->pages);
2332	Serge	3539
3031	serge	3540	// if (obj->base.import_attach)
		3541	// drm_prime_gem_destroy(&obj->base, NULL);
		3542
2344	Serge	3543	drm_gem_object_release(&obj->base);
		3544	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3545
2344	Serge	3546	kfree(obj->bit_17);
		3547	kfree(obj);
		3548	}
2332	Serge	3549
3031	serge	3550	#if 0
		3551	int
		3552	i915_gem_idle(struct drm_device *dev)
2344	Serge	3553	{
3031	serge	3554	drm_i915_private_t *dev_priv = dev->dev_private;
		3555	int ret;
2332	Serge	3556
3031	serge	3557	mutex_lock(&dev->struct_mutex);
2332	Serge	3558
3031	serge	3559	if (dev_priv->mm.suspended) {
		3560	mutex_unlock(&dev->struct_mutex);
		3561	return 0;
		3562	}
2332	Serge	3563
3031	serge	3564	ret = i915_gpu_idle(dev);
		3565	if (ret) {
		3566	mutex_unlock(&dev->struct_mutex);
		3567	return ret;
		3568	}
		3569	i915_gem_retire_requests(dev);
		3570
		3571	i915_gem_reset_fences(dev);
		3572
		3573	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3574	* We need to replace this with a semaphore, or something.
		3575	* And not confound mm.suspended!
		3576	*/
		3577	dev_priv->mm.suspended = 1;
		3578	del_timer_sync(&dev_priv->hangcheck_timer);
		3579
		3580	i915_kernel_lost_context(dev);
		3581	i915_gem_cleanup_ringbuffer(dev);
		3582
		3583	mutex_unlock(&dev->struct_mutex);
		3584
		3585	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	3586	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	3587
		3588	return 0;
2344	Serge	3589	}
3031	serge	3590	#endif
2332	Serge	3591
3031	serge	3592	void i915_gem_l3_remap(struct drm_device *dev)
		3593	{
		3594	drm_i915_private_t *dev_priv = dev->dev_private;
		3595	u32 misccpctl;
		3596	int i;
2332	Serge	3597
3031	serge	3598	if (!IS_IVYBRIDGE(dev))
		3599	return;
2332	Serge	3600
3243	Serge	3601	if (!dev_priv->l3_parity.remap_info)
3031	serge	3602	return;
2332	Serge	3603
3031	serge	3604	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3605	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3606	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3607
3031	serge	3608	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3609	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3610	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3611	DRM_DEBUG("0x%x was already programmed to %x\n",
		3612	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3613	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3614	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3615	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3616	}
2332	Serge	3617
3031	serge	3618	/* Make sure all the writes land before disabling dop clock gating */
		3619	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3620
3031	serge	3621	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3622	}
2332	Serge	3623
3031	serge	3624	void i915_gem_init_swizzling(struct drm_device *dev)
		3625	{
		3626	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3627
3031	serge	3628	if (INTEL_INFO(dev)->gen < 5 \|\|
		3629	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3630	return;
2332	Serge	3631
3031	serge	3632	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3633	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3634
3031	serge	3635	if (IS_GEN5(dev))
		3636	return;
2344	Serge	3637
3031	serge	3638	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3639	if (IS_GEN6(dev))
		3640	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3641	else
		3642	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3643	}
		3644
		3645	static bool
		3646	intel_enable_blt(struct drm_device *dev)
		3647	{
		3648	if (!HAS_BLT(dev))
		3649	return false;
		3650
		3651	/* The blitter was dysfunctional on early prototypes */
		3652	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3653	DRM_INFO("BLT not supported on this pre-production hardware;"
		3654	" graphics performance will be degraded.\n");
		3655	return false;
		3656	}
		3657
		3658	return true;
		3659	}
		3660
2332	Serge	3661	int
3031	serge	3662	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3663	{
		3664	drm_i915_private_t *dev_priv = dev->dev_private;
		3665	int ret;
2351	Serge	3666
3243	Serge	3667	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
3031	serge	3668	return -EIO;
		3669
		3670	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3671	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3672
		3673	i915_gem_l3_remap(dev);
		3674
		3675	i915_gem_init_swizzling(dev);
		3676
2332	Serge	3677	ret = intel_init_render_ring_buffer(dev);
		3678	if (ret)
		3679	return ret;
		3680
		3681	if (HAS_BSD(dev)) {
		3682	ret = intel_init_bsd_ring_buffer(dev);
		3683	if (ret)
		3684	goto cleanup_render_ring;
		3685	}
		3686
3031	serge	3687	if (intel_enable_blt(dev)) {
2332	Serge	3688	ret = intel_init_blt_ring_buffer(dev);
		3689	if (ret)
		3690	goto cleanup_bsd_ring;
		3691	}
		3692
		3693	dev_priv->next_seqno = 1;
2351	Serge	3694
3031	serge	3695	/*
		3696	* XXX: There was some w/a described somewhere suggesting loading
		3697	* contexts before PPGTT.
		3698	*/
		3699	i915_gem_context_init(dev);
		3700	i915_gem_init_ppgtt(dev);
		3701
2332	Serge	3702	return 0;
		3703
		3704	cleanup_bsd_ring:
		3705	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3706	cleanup_render_ring:
		3707	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3708	return ret;
		3709	}
		3710
3031	serge	3711	static bool
		3712	intel_enable_ppgtt(struct drm_device *dev)
		3713	{
		3714	if (i915_enable_ppgtt >= 0)
		3715	return i915_enable_ppgtt;
		3716
		3717	#ifdef CONFIG_INTEL_IOMMU
		3718	/* Disable ppgtt on SNB if VT-d is on. */
		3719	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3720	return false;
		3721	#endif
		3722
		3723	return true;
		3724	}
		3725
		3726	#define LFB_SIZE 0xC00000
		3727
		3728	int i915_gem_init(struct drm_device *dev)
		3729	{
		3730	struct drm_i915_private *dev_priv = dev->dev_private;
		3731	unsigned long gtt_size, mappable_size;
		3732	int ret;
		3733
		3734	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3735	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3736
		3737	mutex_lock(&dev->struct_mutex);
		3738	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3739	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3740	* aperture accordingly when using aliasing ppgtt. */
		3741	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3742
		3743	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3744
		3745	ret = i915_gem_init_aliasing_ppgtt(dev);
		3746	if (ret) {
		3747	mutex_unlock(&dev->struct_mutex);
		3748	return ret;
		3749	}
		3750	} else {
		3751	/* Let GEM Manage all of the aperture.
		3752	*
		3753	* However, leave one page at the end still bound to the scratch
		3754	* page. There are a number of places where the hardware
		3755	* apparently prefetches past the end of the object, and we've
		3756	* seen multiple hangs with the GPU head pointer stuck in a
		3757	* batchbuffer bound at the last page of the aperture. One page
		3758	* should be enough to keep any prefetching inside of the
		3759	* aperture.
		3760	*/
		3761	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3762	}
		3763
		3764	ret = i915_gem_init_hw(dev);
		3765	mutex_unlock(&dev->struct_mutex);
		3766	if (ret) {
		3767	i915_gem_cleanup_aliasing_ppgtt(dev);
		3768	return ret;
		3769	}
		3770
		3771	return 0;
		3772	}
		3773
2332	Serge	3774	void
		3775	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3776	{
		3777	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3778	struct intel_ring_buffer *ring;
2332	Serge	3779	int i;
		3780
3031	serge	3781	for_each_ring(ring, dev_priv, i)
		3782	intel_cleanup_ring_buffer(ring);
2332	Serge	3783	}
		3784
3031	serge	3785	#if 0
		3786
2332	Serge	3787	int
		3788	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3789	struct drm_file *file_priv)
		3790	{
		3791	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3792	int ret;
2332	Serge	3793
		3794	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3795	return 0;
		3796
		3797	if (atomic_read(&dev_priv->mm.wedged)) {
		3798	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3799	atomic_set(&dev_priv->mm.wedged, 0);
		3800	}
		3801
		3802	mutex_lock(&dev->struct_mutex);
		3803	dev_priv->mm.suspended = 0;
		3804
3031	serge	3805	ret = i915_gem_init_hw(dev);
2332	Serge	3806	if (ret != 0) {
		3807	mutex_unlock(&dev->struct_mutex);
		3808	return ret;
		3809	}
		3810
		3811	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3812	mutex_unlock(&dev->struct_mutex);
		3813
		3814	ret = drm_irq_install(dev);
		3815	if (ret)
		3816	goto cleanup_ringbuffer;
		3817
		3818	return 0;
		3819
		3820	cleanup_ringbuffer:
		3821	mutex_lock(&dev->struct_mutex);
		3822	i915_gem_cleanup_ringbuffer(dev);
		3823	dev_priv->mm.suspended = 1;
		3824	mutex_unlock(&dev->struct_mutex);
		3825
		3826	return ret;
		3827	}
		3828
		3829	int
		3830	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3831	struct drm_file *file_priv)
		3832	{
		3833	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3834	return 0;
		3835
		3836	drm_irq_uninstall(dev);
		3837	return i915_gem_idle(dev);
		3838	}
		3839
		3840	void
		3841	i915_gem_lastclose(struct drm_device *dev)
		3842	{
		3843	int ret;
		3844
		3845	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3846	return;
		3847
		3848	ret = i915_gem_idle(dev);
		3849	if (ret)
		3850	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3851	}
		3852	#endif
		3853
		3854	static void
2326	Serge	3855	init_ring_lists(struct intel_ring_buffer *ring)
		3856	{
		3857	INIT_LIST_HEAD(&ring->active_list);
		3858	INIT_LIST_HEAD(&ring->request_list);
		3859	}
		3860
		3861	void
		3862	i915_gem_load(struct drm_device *dev)
		3863	{
		3864	int i;
		3865	drm_i915_private_t *dev_priv = dev->dev_private;
		3866
		3867	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3868	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3869	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3870	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3871	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3872	for (i = 0; i < I915_NUM_RINGS; i++)
		3873	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3874	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3875	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3876	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3877	i915_gem_retire_work_handler);
2326	Serge	3878
		3879	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3880	if (IS_GEN3(dev)) {
3031	serge	3881	I915_WRITE(MI_ARB_STATE,
		3882	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3883	}
		3884
		3885	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3886
		3887	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3888	dev_priv->num_fence_regs = 16;
		3889	else
		3890	dev_priv->num_fence_regs = 8;
		3891
		3892	/* Initialize fence registers to zero */
3031	serge	3893	i915_gem_reset_fences(dev);
2326	Serge	3894
		3895	i915_gem_detect_bit_6_swizzle(dev);
		3896
		3897	dev_priv->mm.interruptible = true;
		3898
		3899	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3900	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3901	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3902	}
		3903
		3904

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 3482 – Rev 3263