WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
3263	Serge	40	#define PROT_READ 0x1 /* page can be read */
		41	#define PROT_WRITE 0x2 /* page can be written */
		42	#define MAP_SHARED 0x01 /* Share changes */
		43
2344	Serge	44	#undef mb
		45	#undef rmb
		46	#undef wmb
		47	#define mb() asm volatile("mfence")
		48	#define rmb() asm volatile ("lfence")
		49	#define wmb() asm volatile ("sfence")
		50
3266	Serge	51	struct drm_i915_gem_object *get_fb_obj();
		52
3263	Serge	53	unsigned long vm_mmap(struct file *file, unsigned long addr,
		54	unsigned long len, unsigned long prot,
		55	unsigned long flag, unsigned long offset);
		56
2344	Serge	57	static inline void clflush(volatile void *__p)
		58	{
		59	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		60	}
		61
2332	Serge	62	#define MAX_ERRNO 4095
		63
		64	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		65
		66
2326	Serge	67	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		68	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		69	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		70	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		71
2332	Serge	72	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		73	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		74	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		75	unsigned alignment,
3031	serge	76	bool map_and_fenceable,
		77	bool nonblocking);
2332	Serge	78	static int i915_gem_phys_pwrite(struct drm_device *dev,
		79	struct drm_i915_gem_object *obj,
		80	struct drm_i915_gem_pwrite *args,
		81	struct drm_file *file);
2326	Serge	82
3031	serge	83	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		84	struct drm_i915_gem_object *obj);
		85	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		86	struct drm_i915_fence_reg *fence,
		87	bool enable);
2332	Serge	88
3031	serge	89	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		90	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		91	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		92
		93	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		94	{
		95	if (obj->tiling_mode)
		96	i915_gem_release_mmap(obj);
		97
		98	/* As we do not have an associated fence register, we will force
		99	* a tiling change if we ever need to acquire one.
		100	*/
		101	obj->fence_dirty = false;
		102	obj->fence_reg = I915_FENCE_REG_NONE;
		103	}
		104
2332	Serge	105	/* some bookkeeping */
		106	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		107	size_t size)
		108	{
		109	dev_priv->mm.object_count++;
		110	dev_priv->mm.object_memory += size;
		111	}
		112
		113	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		114	size_t size)
		115	{
		116	dev_priv->mm.object_count--;
		117	dev_priv->mm.object_memory -= size;
		118	}
		119
		120	static int
		121	i915_gem_wait_for_error(struct drm_device *dev)
		122	{
		123	struct drm_i915_private *dev_priv = dev->dev_private;
		124	struct completion *x = &dev_priv->error_completion;
		125	unsigned long flags;
		126	int ret;
		127
		128	if (!atomic_read(&dev_priv->mm.wedged))
		129	return 0;
3255	Serge	130	#if 0
3031	serge	131	/*
		132	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		133	* userspace. If it takes that long something really bad is going on and
		134	* we should simply try to bail out and fail as gracefully as possible.
		135	*/
		136	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		137	if (ret == 0) {
		138	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		139	return -EIO;
		140	} else if (ret < 0) {
2332	Serge	141	return ret;
3031	serge	142	}
2332	Serge	143
		144	if (atomic_read(&dev_priv->mm.wedged)) {
		145	/* GPU is hung, bump the completion count to account for
		146	* the token we just consumed so that we never hit zero and
		147	* end up waiting upon a subsequent completion event that
		148	* will never happen.
		149	*/
		150	spin_lock_irqsave(&x->wait.lock, flags);
		151	x->done++;
		152	spin_unlock_irqrestore(&x->wait.lock, flags);
		153	}
3255	Serge	154	#endif
		155
2332	Serge	156	return 0;
		157	}
		158
		159	int i915_mutex_lock_interruptible(struct drm_device *dev)
		160	{
		161	int ret;
		162
		163	ret = i915_gem_wait_for_error(dev);
		164	if (ret)
		165	return ret;
		166
3255	Serge	167	mutex_lock(&dev->struct_mutex);
2332	Serge	168
		169	WARN_ON(i915_verify_lists(dev));
		170	return 0;
		171	}
		172
		173	static inline bool
		174	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		175	{
3031	serge	176	return obj->gtt_space && !obj->active;
2332	Serge	177	}
		178
		179
		180	#if 0
		181
		182	int
		183	i915_gem_init_ioctl(struct drm_device dev, void data,
		184	struct drm_file *file)
		185	{
		186	struct drm_i915_gem_init *args = data;
		187
3031	serge	188	if (drm_core_check_feature(dev, DRIVER_MODESET))
		189	return -ENODEV;
		190
2332	Serge	191	if (args->gtt_start >= args->gtt_end \|\|
		192	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		193	return -EINVAL;
		194
3031	serge	195	/* GEM with user mode setting was never supported on ilk and later. */
		196	if (INTEL_INFO(dev)->gen >= 5)
		197	return -ENODEV;
		198
2332	Serge	199	mutex_lock(&dev->struct_mutex);
3031	serge	200	i915_gem_init_global_gtt(dev, args->gtt_start,
		201	args->gtt_end, args->gtt_end);
2332	Serge	202	mutex_unlock(&dev->struct_mutex);
		203
		204	return 0;
		205	}
2351	Serge	206	#endif
2332	Serge	207
		208	int
		209	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		210	struct drm_file *file)
		211	{
		212	struct drm_i915_private *dev_priv = dev->dev_private;
		213	struct drm_i915_gem_get_aperture *args = data;
		214	struct drm_i915_gem_object *obj;
		215	size_t pinned;
		216
		217	pinned = 0;
		218	mutex_lock(&dev->struct_mutex);
3031	serge	219	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		220	if (obj->pin_count)
2332	Serge	221	pinned += obj->gtt_space->size;
		222	mutex_unlock(&dev->struct_mutex);
		223
		224	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	225	args->aper_available_size = args->aper_size - pinned;
2332	Serge	226
		227	return 0;
		228	}
		229
3031	serge	230	static int
		231	i915_gem_create(struct drm_file *file,
2332	Serge	232	struct drm_device *dev,
		233	uint64_t size,
		234	uint32_t *handle_p)
		235	{
		236	struct drm_i915_gem_object *obj;
		237	int ret;
		238	u32 handle;
		239
		240	size = roundup(size, PAGE_SIZE);
2342	Serge	241	if (size == 0)
		242	return -EINVAL;
2332	Serge	243
		244	/* Allocate the new object */
		245	obj = i915_gem_alloc_object(dev, size);
		246	if (obj == NULL)
		247	return -ENOMEM;
		248
		249	ret = drm_gem_handle_create(file, &obj->base, &handle);
		250	if (ret) {
		251	drm_gem_object_release(&obj->base);
		252	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		253	kfree(obj);
		254	return ret;
		255	}
		256
		257	/* drop reference from allocate - handle holds it now */
		258	drm_gem_object_unreference(&obj->base);
2351	Serge	259	trace_i915_gem_object_create(obj);
2332	Serge	260
		261	*handle_p = handle;
		262	return 0;
		263	}
		264
		265	int
		266	i915_gem_dumb_create(struct drm_file *file,
		267	struct drm_device *dev,
		268	struct drm_mode_create_dumb *args)
		269	{
		270	/* have to work out size/pitch and return them */
		271	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		272	args->size = args->pitch * args->height;
		273	return i915_gem_create(file, dev,
		274	args->size, &args->handle);
		275	}
		276
		277	int i915_gem_dumb_destroy(struct drm_file *file,
		278	struct drm_device *dev,
		279	uint32_t handle)
		280	{
		281	return drm_gem_handle_delete(file, handle);
		282	}
		283
2326	Serge	284	/**
2332	Serge	285	* Creates a new mm object and returns a handle to it.
		286	*/
		287	int
		288	i915_gem_create_ioctl(struct drm_device dev, void data,
		289	struct drm_file *file)
		290	{
		291	struct drm_i915_gem_create *args = data;
3031	serge	292
2332	Serge	293	return i915_gem_create(file, dev,
		294	args->size, &args->handle);
		295	}
		296
		297	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		298	{
		299	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		300
		301	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		302	obj->tiling_mode != I915_TILING_NONE;
		303	}
3260	Serge	304	#if 0
2332	Serge	305
3031	serge	306	static inline int
		307	__copy_to_user_swizzled(char __user *cpu_vaddr,
		308	const char *gpu_vaddr, int gpu_offset,
2332	Serge	309	int length)
		310	{
3031	serge	311	int ret, cpu_offset = 0;
2332	Serge	312
3031	serge	313	while (length > 0) {
		314	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		315	int this_length = min(cacheline_end - gpu_offset, length);
		316	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	317
3031	serge	318	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		319	gpu_vaddr + swizzled_gpu_offset,
		320	this_length);
		321	if (ret)
		322	return ret + length;
2332	Serge	323
3031	serge	324	cpu_offset += this_length;
		325	gpu_offset += this_length;
		326	length -= this_length;
		327	}
		328
		329	return 0;
2332	Serge	330	}
		331
3031	serge	332	static inline int
		333	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		334	const char __user *cpu_vaddr,
		335	int length)
2332	Serge	336	{
3031	serge	337	int ret, cpu_offset = 0;
2332	Serge	338
		339	while (length > 0) {
		340	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		341	int this_length = min(cacheline_end - gpu_offset, length);
		342	int swizzled_gpu_offset = gpu_offset ^ 64;
		343
3031	serge	344	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	345	cpu_vaddr + cpu_offset,
		346	this_length);
3031	serge	347	if (ret)
		348	return ret + length;
		349
2332	Serge	350	cpu_offset += this_length;
		351	gpu_offset += this_length;
		352	length -= this_length;
		353	}
		354
3031	serge	355	return 0;
2332	Serge	356	}
		357
3031	serge	358	/* Per-page copy function for the shmem pread fastpath.
		359	* Flushes invalid cachelines before reading the target if
		360	* needs_clflush is set. */
2332	Serge	361	static int
3031	serge	362	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		363	char __user *user_data,
		364	bool page_do_bit17_swizzling, bool needs_clflush)
		365	{
		366	char *vaddr;
		367	int ret;
		368
		369	if (unlikely(page_do_bit17_swizzling))
		370	return -EINVAL;
		371
		372	vaddr = kmap_atomic(page);
		373	if (needs_clflush)
		374	drm_clflush_virt_range(vaddr + shmem_page_offset,
		375	page_length);
		376	ret = __copy_to_user_inatomic(user_data,
		377	vaddr + shmem_page_offset,
		378	page_length);
		379	kunmap_atomic(vaddr);
		380
		381	return ret ? -EFAULT : 0;
		382	}
		383
		384	static void
		385	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		386	bool swizzled)
		387	{
		388	if (unlikely(swizzled)) {
		389	unsigned long start = (unsigned long) addr;
		390	unsigned long end = (unsigned long) addr + length;
		391
		392	/* For swizzling simply ensure that we always flush both
		393	* channels. Lame, but simple and it works. Swizzled
		394	* pwrite/pread is far from a hotpath - current userspace
		395	* doesn't use it at all. */
		396	start = round_down(start, 128);
		397	end = round_up(end, 128);
		398
		399	drm_clflush_virt_range((void *)start, end - start);
		400	} else {
		401	drm_clflush_virt_range(addr, length);
		402	}
		403
		404	}
		405
		406	/* Only difference to the fast-path function is that this can handle bit17
		407	* and uses non-atomic copy and kmap functions. */
		408	static int
		409	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		410	char __user *user_data,
		411	bool page_do_bit17_swizzling, bool needs_clflush)
		412	{
		413	char *vaddr;
		414	int ret;
		415
		416	vaddr = kmap(page);
		417	if (needs_clflush)
		418	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		419	page_length,
		420	page_do_bit17_swizzling);
		421
		422	if (page_do_bit17_swizzling)
		423	ret = __copy_to_user_swizzled(user_data,
		424	vaddr, shmem_page_offset,
		425	page_length);
		426	else
		427	ret = __copy_to_user(user_data,
		428	vaddr + shmem_page_offset,
		429	page_length);
		430	kunmap(page);
		431
		432	return ret ? - EFAULT : 0;
		433	}
		434
		435	static int
		436	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	437	struct drm_i915_gem_object *obj,
		438	struct drm_i915_gem_pread *args,
		439	struct drm_file *file)
		440	{
3031	serge	441	char __user *user_data;
2332	Serge	442	ssize_t remain;
		443	loff_t offset;
3031	serge	444	int shmem_page_offset, page_length, ret = 0;
		445	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		446	int hit_slowpath = 0;
		447	int prefaulted = 0;
		448	int needs_clflush = 0;
		449	struct scatterlist *sg;
		450	int i;
2332	Serge	451
		452	user_data = (char __user *) (uintptr_t) args->data_ptr;
		453	remain = args->size;
		454
3031	serge	455	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		456
		457	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		458	/* If we're not in the cpu read domain, set ourself into the gtt
		459	* read domain and manually flush cachelines (if required). This
		460	* optimizes for the case when the gpu will dirty the data
		461	* anyway again before the next pread happens. */
		462	if (obj->cache_level == I915_CACHE_NONE)
		463	needs_clflush = 1;
		464	if (obj->gtt_space) {
		465	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		466	if (ret)
		467	return ret;
		468	}
		469	}
		470
		471	ret = i915_gem_object_get_pages(obj);
		472	if (ret)
		473	return ret;
		474
		475	i915_gem_object_pin_pages(obj);
		476
2332	Serge	477	offset = args->offset;
		478
3031	serge	479	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	480	struct page *page;
		481
3031	serge	482	if (i < offset >> PAGE_SHIFT)
		483	continue;
		484
		485	if (remain <= 0)
		486	break;
		487
2332	Serge	488	/* Operation in this page
		489	*
3031	serge	490	* shmem_page_offset = offset within page in shmem file
2332	Serge	491	* page_length = bytes to copy for this page
		492	*/
3031	serge	493	shmem_page_offset = offset_in_page(offset);
2332	Serge	494	page_length = remain;
3031	serge	495	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		496	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	497
3031	serge	498	page = sg_page(sg);
		499	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		500	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	501
3031	serge	502	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		503	user_data, page_do_bit17_swizzling,
		504	needs_clflush);
		505	if (ret == 0)
		506	goto next_page;
2332	Serge	507
3031	serge	508	hit_slowpath = 1;
		509	mutex_unlock(&dev->struct_mutex);
		510
		511	if (!prefaulted) {
		512	ret = fault_in_multipages_writeable(user_data, remain);
		513	/* Userspace is tricking us, but we've already clobbered
		514	* its pages with the prefault and promised to write the
		515	* data up to the first fault. Hence ignore any errors
		516	* and just continue. */
		517	(void)ret;
		518	prefaulted = 1;
		519	}
		520
		521	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		522	user_data, page_do_bit17_swizzling,
		523	needs_clflush);
		524
		525	mutex_lock(&dev->struct_mutex);
		526
		527	next_page:
2332	Serge	528	mark_page_accessed(page);
3031	serge	529
2332	Serge	530	if (ret)
3031	serge	531	goto out;
2332	Serge	532
		533	remain -= page_length;
		534	user_data += page_length;
		535	offset += page_length;
		536	}
		537
3031	serge	538	out:
		539	i915_gem_object_unpin_pages(obj);
		540
		541	if (hit_slowpath) {
		542	/* Fixup: Kill any reinstated backing storage pages */
		543	if (obj->madv == __I915_MADV_PURGED)
		544	i915_gem_object_truncate(obj);
		545	}
		546
		547	return ret;
2332	Serge	548	}
		549
		550	/**
3031	serge	551	* Reads data from the object referenced by handle.
		552	*
		553	* On error, the contents of *data are undefined.
2332	Serge	554	*/
3031	serge	555	int
		556	i915_gem_pread_ioctl(struct drm_device dev, void data,
		557	struct drm_file *file)
		558	{
		559	struct drm_i915_gem_pread *args = data;
		560	struct drm_i915_gem_object *obj;
		561	int ret = 0;
		562
		563	if (args->size == 0)
		564	return 0;
		565
		566	if (!access_ok(VERIFY_WRITE,
		567	(char __user *)(uintptr_t)args->data_ptr,
		568	args->size))
		569	return -EFAULT;
		570
		571	ret = i915_mutex_lock_interruptible(dev);
		572	if (ret)
		573	return ret;
		574
		575	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		576	if (&obj->base == NULL) {
		577	ret = -ENOENT;
		578	goto unlock;
		579	}
		580
		581	/* Bounds check source. */
		582	if (args->offset > obj->base.size \|\|
		583	args->size > obj->base.size - args->offset) {
		584	ret = -EINVAL;
		585	goto out;
		586	}
		587
		588	/* prime objects have no backing filp to GEM pread/pwrite
		589	* pages from.
		590	*/
		591	if (!obj->base.filp) {
		592	ret = -EINVAL;
		593	goto out;
		594	}
		595
		596	trace_i915_gem_object_pread(obj, args->offset, args->size);
		597
		598	ret = i915_gem_shmem_pread(dev, obj, args, file);
		599
		600	out:
		601	drm_gem_object_unreference(&obj->base);
		602	unlock:
		603	mutex_unlock(&dev->struct_mutex);
		604	return ret;
		605	}
		606
		607	/* This is the fast write path which cannot handle
		608	* page faults in the source data
		609	*/
		610
		611	static inline int
		612	fast_user_write(struct io_mapping *mapping,
		613	loff_t page_base, int page_offset,
		614	char __user *user_data,
		615	int length)
		616	{
		617	void __iomem *vaddr_atomic;
		618	void *vaddr;
		619	unsigned long unwritten;
		620
		621	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		622	/* We can use the cpu mem copy function because this is X86. */
		623	vaddr = (void __force*)vaddr_atomic + page_offset;
		624	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		625	user_data, length);
		626	io_mapping_unmap_atomic(vaddr_atomic);
		627	return unwritten;
		628	}
3260	Serge	629	#endif
3031	serge	630
3260	Serge	631	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	632	/**
		633	* This is the fast pwrite path, where we copy the data directly from the
		634	* user into the GTT, uncached.
		635	*/
2332	Serge	636	static int
3031	serge	637	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		638	struct drm_i915_gem_object *obj,
		639	struct drm_i915_gem_pwrite *args,
		640	struct drm_file *file)
2332	Serge	641	{
3031	serge	642	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	643	ssize_t remain;
3031	serge	644	loff_t offset, page_base;
		645	char __user *user_data;
		646	int page_offset, page_length, ret;
3260	Serge	647	char *vaddr;
2332	Serge	648
3031	serge	649	ret = i915_gem_object_pin(obj, 0, true, true);
		650	if (ret)
		651	goto out;
		652
		653	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		654	if (ret)
		655	goto out_unpin;
		656
		657	ret = i915_gem_object_put_fence(obj);
		658	if (ret)
		659	goto out_unpin;
		660
3260	Serge	661	vaddr = AllocKernelSpace(4096);
		662	if(vaddr == NULL)
		663	{
		664	ret = -ENOSPC;
		665	goto out_unpin;
		666	};
		667
3031	serge	668	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	669	remain = args->size;
		670
3031	serge	671	offset = obj->gtt_offset + args->offset;
2332	Serge	672
3031	serge	673	while (remain > 0) {
		674	/* Operation in this page
		675	*
		676	* page_base = page offset within aperture
		677	* page_offset = offset within page
		678	* page_length = bytes to copy for this page
		679	*/
		680	page_base = offset & PAGE_MASK;
		681	page_offset = offset_in_page(offset);
		682	page_length = remain;
		683	if ((page_offset + remain) > PAGE_SIZE)
		684	page_length = PAGE_SIZE - page_offset;
2332	Serge	685
3260	Serge	686	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	687
3260	Serge	688	memcpy(vaddr+page_offset, user_data, page_length);
		689
3031	serge	690	remain -= page_length;
		691	user_data += page_length;
		692	offset += page_length;
2332	Serge	693	}
		694
3260	Serge	695	FreeKernelSpace(vaddr);
		696
3031	serge	697	out_unpin:
		698	i915_gem_object_unpin(obj);
		699	out:
3260	Serge	700	printf("% s ret = %d\n", __FUNCTION__, ret);
		701
3031	serge	702	return ret;
		703	}
		704
		705	/* Per-page copy function for the shmem pwrite fastpath.
		706	* Flushes invalid cachelines before writing to the target if
		707	* needs_clflush_before is set and flushes out any written cachelines after
		708	* writing if needs_clflush is set. */
		709	static int
		710	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		711	char __user *user_data,
		712	bool page_do_bit17_swizzling,
		713	bool needs_clflush_before,
		714	bool needs_clflush_after)
		715	{
		716	char *vaddr;
3260	Serge	717	int ret = 0;
3031	serge	718
		719	if (unlikely(page_do_bit17_swizzling))
		720	return -EINVAL;
		721
3260	Serge	722	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	723	if (needs_clflush_before)
		724	drm_clflush_virt_range(vaddr + shmem_page_offset,
		725	page_length);
3260	Serge	726	memcpy(vaddr + shmem_page_offset,
3031	serge	727	user_data,
		728	page_length);
		729	if (needs_clflush_after)
		730	drm_clflush_virt_range(vaddr + shmem_page_offset,
		731	page_length);
3260	Serge	732	FreeKernelSpace(vaddr);
3031	serge	733
		734	return ret ? -EFAULT : 0;
		735	}
3260	Serge	736	#if 0
3031	serge	737
		738	/* Only difference to the fast-path function is that this can handle bit17
		739	* and uses non-atomic copy and kmap functions. */
		740	static int
		741	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		742	char __user *user_data,
		743	bool page_do_bit17_swizzling,
		744	bool needs_clflush_before,
		745	bool needs_clflush_after)
		746	{
		747	char *vaddr;
		748	int ret;
		749
		750	vaddr = kmap(page);
		751	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		752	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		753	page_length,
		754	page_do_bit17_swizzling);
		755	if (page_do_bit17_swizzling)
		756	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		757	user_data,
		758	page_length);
		759	else
		760	ret = __copy_from_user(vaddr + shmem_page_offset,
		761	user_data,
		762	page_length);
		763	if (needs_clflush_after)
		764	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		765	page_length,
		766	page_do_bit17_swizzling);
		767	kunmap(page);
		768
		769	return ret ? -EFAULT : 0;
		770	}
3260	Serge	771	#endif
3031	serge	772
3260	Serge	773
3031	serge	774	static int
		775	i915_gem_shmem_pwrite(struct drm_device *dev,
		776	struct drm_i915_gem_object *obj,
		777	struct drm_i915_gem_pwrite *args,
		778	struct drm_file *file)
		779	{
		780	ssize_t remain;
		781	loff_t offset;
		782	char __user *user_data;
		783	int shmem_page_offset, page_length, ret = 0;
		784	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		785	int hit_slowpath = 0;
		786	int needs_clflush_after = 0;
		787	int needs_clflush_before = 0;
		788	int i;
		789	struct scatterlist *sg;
		790
		791	user_data = (char __user *) (uintptr_t) args->data_ptr;
		792	remain = args->size;
		793
		794	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		795
		796	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		797	/* If we're not in the cpu write domain, set ourself into the gtt
		798	* write domain and manually flush cachelines (if required). This
		799	* optimizes for the case when the gpu will use the data
		800	* right away and we therefore have to clflush anyway. */
		801	if (obj->cache_level == I915_CACHE_NONE)
		802	needs_clflush_after = 1;
		803	if (obj->gtt_space) {
		804	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		805	if (ret)
		806	return ret;
		807	}
		808	}
		809	/* Same trick applies for invalidate partially written cachelines before
		810	* writing. */
		811	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		812	&& obj->cache_level == I915_CACHE_NONE)
		813	needs_clflush_before = 1;
		814
		815	ret = i915_gem_object_get_pages(obj);
2332	Serge	816	if (ret)
3031	serge	817	return ret;
2332	Serge	818
3031	serge	819	i915_gem_object_pin_pages(obj);
2332	Serge	820
		821	offset = args->offset;
3031	serge	822	obj->dirty = 1;
2332	Serge	823
3031	serge	824	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	825	struct page *page;
3031	serge	826	int partial_cacheline_write;
2332	Serge	827
3031	serge	828	if (i < offset >> PAGE_SHIFT)
		829	continue;
		830
		831	if (remain <= 0)
		832	break;
		833
2332	Serge	834	/* Operation in this page
		835	*
		836	* shmem_page_offset = offset within page in shmem file
		837	* page_length = bytes to copy for this page
		838	*/
		839	shmem_page_offset = offset_in_page(offset);
		840
		841	page_length = remain;
		842	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		843	page_length = PAGE_SIZE - shmem_page_offset;
		844
3031	serge	845	/* If we don't overwrite a cacheline completely we need to be
		846	* careful to have up-to-date data by first clflushing. Don't
		847	* overcomplicate things and flush the entire patch. */
		848	partial_cacheline_write = needs_clflush_before &&
		849	((shmem_page_offset \| page_length)
3260	Serge	850	& (x86_clflush_size - 1));
2332	Serge	851
3031	serge	852	page = sg_page(sg);
		853	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		854	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	855
3031	serge	856	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		857	user_data, page_do_bit17_swizzling,
		858	partial_cacheline_write,
		859	needs_clflush_after);
		860	if (ret == 0)
		861	goto next_page;
		862
		863	hit_slowpath = 1;
		864	mutex_unlock(&dev->struct_mutex);
3260	Serge	865	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	866
3260	Serge	867	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		868	// user_data, page_do_bit17_swizzling,
		869	// partial_cacheline_write,
		870	// needs_clflush_after);
		871
3031	serge	872	mutex_lock(&dev->struct_mutex);
		873
		874	next_page:
2332	Serge	875
3031	serge	876	if (ret)
		877	goto out;
		878
2332	Serge	879	remain -= page_length;
3031	serge	880	user_data += page_length;
2332	Serge	881	offset += page_length;
		882	}
		883
		884	out:
3031	serge	885	i915_gem_object_unpin_pages(obj);
		886
		887	if (hit_slowpath) {
		888	/* Fixup: Kill any reinstated backing storage pages */
		889	if (obj->madv == __I915_MADV_PURGED)
		890	i915_gem_object_truncate(obj);
		891	/* and flush dirty cachelines in case the object isn't in the cpu write
		892	* domain anymore. */
		893	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		894	i915_gem_clflush_object(obj);
3243	Serge	895	i915_gem_chipset_flush(dev);
3031	serge	896	}
2332	Serge	897	}
		898
3031	serge	899	if (needs_clflush_after)
3243	Serge	900	i915_gem_chipset_flush(dev);
3031	serge	901
2332	Serge	902	return ret;
		903	}
3031	serge	904
		905	/**
		906	* Writes data to the object referenced by handle.
		907	*
		908	* On error, the contents of the buffer that were to be modified are undefined.
		909	*/
		910	int
		911	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		912	struct drm_file *file)
		913	{
		914	struct drm_i915_gem_pwrite *args = data;
		915	struct drm_i915_gem_object *obj;
		916	int ret;
		917
		918	if (args->size == 0)
		919	return 0;
		920
		921	ret = i915_mutex_lock_interruptible(dev);
		922	if (ret)
		923	return ret;
		924
		925	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		926	if (&obj->base == NULL) {
		927	ret = -ENOENT;
		928	goto unlock;
		929	}
		930
		931	/* Bounds check destination. */
		932	if (args->offset > obj->base.size \|\|
		933	args->size > obj->base.size - args->offset) {
		934	ret = -EINVAL;
		935	goto out;
		936	}
		937
		938	/* prime objects have no backing filp to GEM pread/pwrite
		939	* pages from.
		940	*/
		941	if (!obj->base.filp) {
		942	ret = -EINVAL;
		943	goto out;
		944	}
		945
		946	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		947
		948	ret = -EFAULT;
		949	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		950	* it would end up going through the fenced access, and we'll get
		951	* different detiling behavior between reading and writing.
		952	* pread/pwrite currently are reading and writing from the CPU
		953	* perspective, requiring manual detiling by the client.
		954	*/
3260	Serge	955	// if (obj->phys_obj) {
		956	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		957	// goto out;
		958	// }
3031	serge	959
		960	if (obj->cache_level == I915_CACHE_NONE &&
		961	obj->tiling_mode == I915_TILING_NONE &&
		962	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		963	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		964	/* Note that the gtt paths might fail with non-page-backed user
		965	* pointers (e.g. gtt mappings when moving data between
		966	* textures). Fallback to the shmem path in that case. */
		967	}
		968
		969	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	970	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	971
		972	out:
		973	drm_gem_object_unreference(&obj->base);
		974	unlock:
		975	mutex_unlock(&dev->struct_mutex);
		976	return ret;
		977	}
		978
		979	int
		980	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		981	bool interruptible)
		982	{
		983	if (atomic_read(&dev_priv->mm.wedged)) {
		984	struct completion *x = &dev_priv->error_completion;
		985	bool recovery_complete;
		986	unsigned long flags;
2332	Serge	987
3031	serge	988	/* Give the error handler a chance to run. */
		989	spin_lock_irqsave(&x->wait.lock, flags);
		990	recovery_complete = x->done > 0;
		991	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	992
3031	serge	993	/* Non-interruptible callers can't handle -EAGAIN, hence return
		994	* -EIO unconditionally for these. */
		995	if (!interruptible)
		996	return -EIO;
2332	Serge	997
3031	serge	998	/* Recovery complete, but still wedged means reset failure. */
		999	if (recovery_complete)
		1000	return -EIO;
2332	Serge	1001
3031	serge	1002	return -EAGAIN;
		1003	}
2332	Serge	1004
3031	serge	1005	return 0;
		1006	}
2332	Serge	1007
3031	serge	1008	/*
		1009	* Compare seqno against outstanding lazy request. Emit a request if they are
		1010	* equal.
		1011	*/
		1012	static int
		1013	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1014	{
		1015	int ret;
2332	Serge	1016
3031	serge	1017	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1018
3031	serge	1019	ret = 0;
		1020	if (seqno == ring->outstanding_lazy_request)
		1021	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1022
3031	serge	1023	return ret;
		1024	}
2332	Serge	1025
3031	serge	1026	/**
		1027	* __wait_seqno - wait until execution of seqno has finished
		1028	* @ring: the ring expected to report seqno
		1029	* @seqno: duh!
		1030	* @interruptible: do an interruptible wait (normally yes)
		1031	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1032	*
		1033	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1034	* errno with remaining time filled in timeout argument.
		1035	*/
		1036	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1037	bool interruptible, struct timespec *timeout)
		1038	{
		1039	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1040	struct timespec before, now, wait_time={1,0};
		1041	unsigned long timeout_jiffies;
		1042	long end;
		1043	bool wait_forever = true;
		1044	int ret;
2332	Serge	1045
3031	serge	1046	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1047	return 0;
2332	Serge	1048
3031	serge	1049	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1050
3031	serge	1051	if (timeout != NULL) {
		1052	wait_time = *timeout;
		1053	wait_forever = false;
		1054	}
2332	Serge	1055
3266	Serge	1056	timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1057
3031	serge	1058	if (WARN_ON(!ring->irq_get(ring)))
		1059	return -ENODEV;
2332	Serge	1060
3031	serge	1061	/* Record current time in case interrupted by signal, or wedged * */
		1062	getrawmonotonic(&before);
2332	Serge	1063
3031	serge	1064	#define EXIT_COND \
		1065	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1066	atomic_read(&dev_priv->mm.wedged))
		1067	do {
3266	Serge	1068	if (interruptible)
		1069	end = wait_event_interruptible_timeout(ring->irq_queue,
		1070	EXIT_COND,
		1071	timeout_jiffies);
		1072	else
3031	serge	1073	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1074	timeout_jiffies);
2332	Serge	1075
3031	serge	1076	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1077	if (ret)
		1078	end = ret;
		1079	} while (end == 0 && wait_forever);
2332	Serge	1080
3031	serge	1081	getrawmonotonic(&now);
2332	Serge	1082
3031	serge	1083	ring->irq_put(ring);
		1084	trace_i915_gem_request_wait_end(ring, seqno);
		1085	#undef EXIT_COND
2332	Serge	1086
3031	serge	1087	if (timeout) {
		1088	// struct timespec sleep_time = timespec_sub(now, before);
		1089	// timeout = timespec_sub(timeout, sleep_time);
		1090	}
2332	Serge	1091
3031	serge	1092	switch (end) {
		1093	case -EIO:
		1094	case -EAGAIN: /* Wedged */
		1095	case -ERESTARTSYS: /* Signal */
		1096	return (int)end;
		1097	case 0: /* Timeout */
3266	Serge	1098	if (timeout)
		1099	set_normalized_timespec(timeout, 0, 0);
3031	serge	1100	return -ETIME;
		1101	default: /* Completed */
		1102	WARN_ON(end < 0); /* We're not aware of other errors */
		1103	return 0;
		1104	}
		1105	}
2332	Serge	1106
3031	serge	1107	/**
		1108	* Waits for a sequence number to be signaled, and cleans up the
		1109	* request and object lists appropriately for that event.
		1110	*/
		1111	int
		1112	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1113	{
		1114	struct drm_device *dev = ring->dev;
		1115	struct drm_i915_private *dev_priv = dev->dev_private;
		1116	bool interruptible = dev_priv->mm.interruptible;
		1117	int ret;
2332	Serge	1118
3031	serge	1119	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1120	BUG_ON(seqno == 0);
2332	Serge	1121
3031	serge	1122	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1123	if (ret)
		1124	return ret;
2332	Serge	1125
3031	serge	1126	ret = i915_gem_check_olr(ring, seqno);
		1127	if (ret)
		1128	return ret;
2332	Serge	1129
3031	serge	1130	return __wait_seqno(ring, seqno, interruptible, NULL);
		1131	}
2332	Serge	1132
3031	serge	1133	/**
		1134	* Ensures that all rendering to the object has completed and the object is
		1135	* safe to unbind from the GTT or access from the CPU.
		1136	*/
		1137	static __must_check int
		1138	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1139	bool readonly)
		1140	{
		1141	struct intel_ring_buffer *ring = obj->ring;
		1142	u32 seqno;
		1143	int ret;
2332	Serge	1144
3031	serge	1145	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1146	if (seqno == 0)
		1147	return 0;
2332	Serge	1148
3031	serge	1149	ret = i915_wait_seqno(ring, seqno);
		1150	if (ret)
		1151	return ret;
2332	Serge	1152
3031	serge	1153	i915_gem_retire_requests_ring(ring);
2332	Serge	1154
3031	serge	1155	/* Manually manage the write flush as we may have not yet
		1156	* retired the buffer.
		1157	*/
		1158	if (obj->last_write_seqno &&
		1159	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1160	obj->last_write_seqno = 0;
		1161	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1162	}
2332	Serge	1163
3031	serge	1164	return 0;
		1165	}
2332	Serge	1166
3260	Serge	1167	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1168	* as the object state may change during this call.
		1169	*/
		1170	static __must_check int
		1171	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1172	bool readonly)
		1173	{
		1174	struct drm_device *dev = obj->base.dev;
		1175	struct drm_i915_private *dev_priv = dev->dev_private;
		1176	struct intel_ring_buffer *ring = obj->ring;
		1177	u32 seqno;
		1178	int ret;
2332	Serge	1179
3260	Serge	1180	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1181	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1182
3260	Serge	1183	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1184	if (seqno == 0)
		1185	return 0;
2332	Serge	1186
3260	Serge	1187	ret = i915_gem_check_wedge(dev_priv, true);
		1188	if (ret)
		1189	return ret;
2332	Serge	1190
3260	Serge	1191	ret = i915_gem_check_olr(ring, seqno);
		1192	if (ret)
		1193	return ret;
2332	Serge	1194
3260	Serge	1195	mutex_unlock(&dev->struct_mutex);
		1196	ret = __wait_seqno(ring, seqno, true, NULL);
		1197	mutex_lock(&dev->struct_mutex);
2332	Serge	1198
3260	Serge	1199	i915_gem_retire_requests_ring(ring);
2332	Serge	1200
3260	Serge	1201	/* Manually manage the write flush as we may have not yet
		1202	* retired the buffer.
		1203	*/
		1204	if (obj->last_write_seqno &&
		1205	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1206	obj->last_write_seqno = 0;
		1207	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1208	}
2332	Serge	1209
3260	Serge	1210	return ret;
		1211	}
2332	Serge	1212
3260	Serge	1213	/**
		1214	* Called when user space prepares to use an object with the CPU, either
		1215	* through the mmap ioctl's mapping or a GTT mapping.
		1216	*/
		1217	int
		1218	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1219	struct drm_file *file)
		1220	{
		1221	struct drm_i915_gem_set_domain *args = data;
		1222	struct drm_i915_gem_object *obj;
		1223	uint32_t read_domains = args->read_domains;
		1224	uint32_t write_domain = args->write_domain;
		1225	int ret;
2332	Serge	1226
3260	Serge	1227	/* Only handle setting domains to types used by the CPU. */
		1228	if (write_domain & I915_GEM_GPU_DOMAINS)
		1229	return -EINVAL;
2332	Serge	1230
3260	Serge	1231	if (read_domains & I915_GEM_GPU_DOMAINS)
		1232	return -EINVAL;
2332	Serge	1233
3260	Serge	1234	/* Having something in the write domain implies it's in the read
		1235	* domain, and only that read domain. Enforce that in the request.
		1236	*/
		1237	if (write_domain != 0 && read_domains != write_domain)
		1238	return -EINVAL;
2332	Serge	1239
3260	Serge	1240	ret = i915_mutex_lock_interruptible(dev);
		1241	if (ret)
		1242	return ret;
2332	Serge	1243
3260	Serge	1244	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1245	if (&obj->base == NULL) {
		1246	ret = -ENOENT;
		1247	goto unlock;
		1248	}
2332	Serge	1249
3260	Serge	1250	/* Try to flush the object off the GPU without holding the lock.
		1251	* We will repeat the flush holding the lock in the normal manner
		1252	* to catch cases where we are gazumped.
		1253	*/
		1254	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1255	if (ret)
		1256	goto unref;
2332	Serge	1257
3260	Serge	1258	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1259	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1260
3260	Serge	1261	/* Silently promote "you're not bound, there was nothing to do"
		1262	* to success, since the client was just asking us to
		1263	* make sure everything was done.
		1264	*/
		1265	if (ret == -EINVAL)
		1266	ret = 0;
		1267	} else {
		1268	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1269	}
2332	Serge	1270
3260	Serge	1271	unref:
		1272	drm_gem_object_unreference(&obj->base);
		1273	unlock:
		1274	mutex_unlock(&dev->struct_mutex);
		1275	return ret;
		1276	}
2332	Serge	1277
		1278
		1279
		1280
		1281
		1282
3260	Serge	1283	/**
		1284	* Maps the contents of an object, returning the address it is mapped
		1285	* into.
		1286	*
		1287	* While the mapping holds a reference on the contents of the object, it doesn't
		1288	* imply a ref on the object itself.
		1289	*/
		1290	int
		1291	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1292	struct drm_file *file)
		1293	{
		1294	struct drm_i915_gem_mmap *args = data;
		1295	struct drm_gem_object *obj;
		1296	unsigned long addr = 0;
2332	Serge	1297
3260	Serge	1298	obj = drm_gem_object_lookup(dev, file, args->handle);
		1299	if (obj == NULL)
		1300	return -ENOENT;
		1301
3263	Serge	1302	dbgprintf("%s offset %lld size %lld\n",
		1303	__FUNCTION__, args->offset, args->size);
3260	Serge	1304	/* prime objects have no backing filp to GEM mmap
		1305	* pages from.
		1306	*/
		1307	if (!obj->filp) {
		1308	drm_gem_object_unreference_unlocked(obj);
		1309	return -EINVAL;
		1310	}
2332	Serge	1311
3263	Serge	1312	addr = vm_mmap(obj->filp, 0, args->size,
		1313	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1314	args->offset);
3260	Serge	1315	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1316	if (IS_ERR((void *)addr))
		1317	return addr;
2332	Serge	1318
3260	Serge	1319	args->addr_ptr = (uint64_t) addr;
2332	Serge	1320
3263	Serge	1321	return 0;
3260	Serge	1322	}
2332	Serge	1323
		1324
		1325
		1326
		1327
		1328
		1329
		1330
3031	serge	1331
		1332
		1333
		1334
		1335
		1336	/**
		1337	* i915_gem_release_mmap - remove physical page mappings
		1338	* @obj: obj in question
		1339	*
		1340	* Preserve the reservation of the mmapping with the DRM core code, but
		1341	* relinquish ownership of the pages back to the system.
		1342	*
		1343	* It is vital that we remove the page mapping if we have mapped a tiled
		1344	* object through the GTT and then lose the fence register due to
		1345	* resource pressure. Similarly if the object has been moved out of the
		1346	* aperture, than pages mapped into userspace must be revoked. Removing the
		1347	* mapping will then trigger a page fault on the next user access, allowing
		1348	* fixup by i915_gem_fault().
		1349	*/
		1350	void
		1351	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1352	{
		1353	if (!obj->fault_mappable)
		1354	return;
		1355
		1356	if (obj->base.dev->dev_mapping)
		1357	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1358	// (loff_t)obj->base.map_list.hash.key<
		1359	// obj->base.size, 1);
		1360
		1361	obj->fault_mappable = false;
		1362	}
		1363
2332	Serge	1364	static uint32_t
		1365	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1366	{
		1367	uint32_t gtt_size;
		1368
		1369	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1370	tiling_mode == I915_TILING_NONE)
		1371	return size;
		1372
		1373	/* Previous chips need a power-of-two fence region when tiling */
		1374	if (INTEL_INFO(dev)->gen == 3)
		1375	gtt_size = 1024*1024;
		1376	else
		1377	gtt_size = 512*1024;
		1378
		1379	while (gtt_size < size)
		1380	gtt_size <<= 1;
		1381
		1382	return gtt_size;
		1383	}
		1384
		1385	/**
		1386	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1387	* @obj: object to check
		1388	*
		1389	* Return the required GTT alignment for an object, taking into account
		1390	* potential fence register mapping.
		1391	*/
		1392	static uint32_t
		1393	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1394	uint32_t size,
		1395	int tiling_mode)
		1396	{
		1397	/*
		1398	* Minimum alignment is 4k (GTT page size), but might be greater
		1399	* if a fence register is needed for the object.
		1400	*/
		1401	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1402	tiling_mode == I915_TILING_NONE)
		1403	return 4096;
		1404
		1405	/*
		1406	* Previous chips need to be aligned to the size of the smallest
		1407	* fence register that can contain the object.
		1408	*/
		1409	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1410	}
		1411
		1412	/**
		1413	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1414	* unfenced object
		1415	* @dev: the device
		1416	* @size: size of the object
		1417	* @tiling_mode: tiling mode of the object
		1418	*
		1419	* Return the required GTT alignment for an object, only taking into account
		1420	* unfenced tiled surface requirements.
		1421	*/
		1422	uint32_t
		1423	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1424	uint32_t size,
		1425	int tiling_mode)
		1426	{
		1427	/*
		1428	* Minimum alignment is 4k (GTT page size) for sane hw.
		1429	*/
		1430	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1431	tiling_mode == I915_TILING_NONE)
		1432	return 4096;
		1433
		1434	/* Previous hardware however needs to be aligned to a power-of-two
		1435	* tile height. The simplest method for determining this is to reuse
		1436	* the power-of-tile object size.
		1437	*/
		1438	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1439	}
		1440
3031	serge	1441	/* Immediately discard the backing storage */
		1442	static void
		1443	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1444	{
		1445	// struct inode *inode;
2332	Serge	1446
3031	serge	1447	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1448
3263	Serge	1449	if (obj->base.filp == NULL)
		1450	return;
2332	Serge	1451
3031	serge	1452	/* Our goal here is to return as much of the memory as
		1453	* is possible back to the system as we are called from OOM.
		1454	* To do this we must instruct the shmfs to drop all of its
		1455	* backing pages, now.
		1456	*/
		1457	// inode = obj->base.filp->f_path.dentry->d_inode;
		1458	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1459
3031	serge	1460	obj->madv = __I915_MADV_PURGED;
		1461	}
2332	Serge	1462
3031	serge	1463	static inline int
		1464	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1465	{
		1466	return obj->madv == I915_MADV_DONTNEED;
		1467	}
2332	Serge	1468
3031	serge	1469	static void
		1470	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1471	{
3243	Serge	1472	int page_count = obj->base.size / PAGE_SIZE;
		1473	struct scatterlist *sg;
3031	serge	1474	int ret, i;
2332	Serge	1475
3031	serge	1476	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1477
3031	serge	1478	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1479	if (ret) {
		1480	/* In the event of a disaster, abandon all caches and
		1481	* hope for the best.
		1482	*/
		1483	WARN_ON(ret != -EIO);
		1484	i915_gem_clflush_object(obj);
		1485	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1486	}
2332	Serge	1487
3031	serge	1488	if (obj->madv == I915_MADV_DONTNEED)
		1489	obj->dirty = 0;
2332	Serge	1490
3243	Serge	1491	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1492	struct page *page = sg_page(sg);
2332	Serge	1493
3243	Serge	1494
		1495
3263	Serge	1496	// page_cache_release(page);
3243	Serge	1497	}
		1498	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3031	serge	1499	obj->dirty = 0;
3243	Serge	1500
		1501	sg_free_table(obj->pages);
		1502	kfree(obj->pages);
3031	serge	1503	}
2332	Serge	1504
3031	serge	1505	static int
		1506	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1507	{
		1508	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1509
3243	Serge	1510	if (obj->pages == NULL)
3031	serge	1511	return 0;
2332	Serge	1512
3031	serge	1513	BUG_ON(obj->gtt_space);
		1514
		1515	if (obj->pages_pin_count)
		1516	return -EBUSY;
		1517
3243	Serge	1518	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1519	* array, hence protect them from being reaped by removing them from gtt
		1520	* lists early. */
		1521	list_del(&obj->gtt_list);
		1522
3031	serge	1523	ops->put_pages(obj);
3243	Serge	1524	obj->pages = NULL;
3031	serge	1525
		1526	if (i915_gem_object_is_purgeable(obj))
		1527	i915_gem_object_truncate(obj);
		1528
		1529	return 0;
		1530	}
		1531
		1532
		1533
		1534
		1535
		1536
		1537
		1538
2332	Serge	1539	static int
3031	serge	1540	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1541	{
3260	Serge	1542	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1543	int page_count, i;
3260	Serge	1544	struct address_space *mapping;
3243	Serge	1545	struct sg_table *st;
		1546	struct scatterlist *sg;
		1547	struct page *page;
		1548	gfp_t gfp;
2332	Serge	1549
3243	Serge	1550	/* Assert that the object is not currently in any GPU domain. As it
		1551	* wasn't in the GTT, there shouldn't be any way it could have been in
		1552	* a GPU cache
2332	Serge	1553	*/
3243	Serge	1554	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1555	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1556
		1557	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1558	if (st == NULL)
		1559	return -ENOMEM;
		1560
2332	Serge	1561	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1562	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1563	sg_free_table(st);
		1564	kfree(st);
2332	Serge	1565	return -ENOMEM;
3243	Serge	1566	}
2332	Serge	1567
3243	Serge	1568	/* Get the list of pages out of our struct file. They'll be pinned
		1569	* at this point until we release them.
		1570	*
		1571	* Fail silently without starting the shrinker
		1572	*/
		1573	for_each_sg(st->sgl, sg, page_count, i) {
3260	Serge	1574	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1575	if (IS_ERR(page)) {
		1576	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1577	goto err_pages;
		1578
3260	Serge	1579	}
3243	Serge	1580	sg_set_page(sg, page, PAGE_SIZE, 0);
		1581	}
3031	serge	1582
3243	Serge	1583	obj->pages = st;
3031	serge	1584
3260	Serge	1585	DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1586
		1587	return 0;
		1588
		1589	err_pages:
3243	Serge	1590	for_each_sg(st->sgl, sg, i, page_count)
		1591	page_cache_release(sg_page(sg));
		1592	sg_free_table(st);
		1593	kfree(st);
		1594	return PTR_ERR(page);
2332	Serge	1595	}
		1596
3031	serge	1597	/* Ensure that the associated pages are gathered from the backing storage
		1598	* and pinned into our object. i915_gem_object_get_pages() may be called
		1599	* multiple times before they are released by a single call to
		1600	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1601	* either as a result of memory pressure (reaping pages under the shrinker)
		1602	* or as the object is itself released.
		1603	*/
		1604	int
		1605	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1606	{
3031	serge	1607	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1608	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1609	int ret;
2332	Serge	1610
3243	Serge	1611	if (obj->pages)
3031	serge	1612	return 0;
2332	Serge	1613
3031	serge	1614	BUG_ON(obj->pages_pin_count);
2332	Serge	1615
3031	serge	1616	ret = ops->get_pages(obj);
		1617	if (ret)
		1618	return ret;
2344	Serge	1619
3031	serge	1620	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1621	return 0;
2332	Serge	1622	}
		1623
		1624	void
		1625	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1626	struct intel_ring_buffer *ring)
2332	Serge	1627	{
		1628	struct drm_device *dev = obj->base.dev;
		1629	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1630	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1631
		1632	BUG_ON(ring == NULL);
		1633	obj->ring = ring;
		1634
		1635	/* Add a reference if we're newly entering the active list. */
		1636	if (!obj->active) {
2344	Serge	1637	drm_gem_object_reference(&obj->base);
2332	Serge	1638	obj->active = 1;
		1639	}
		1640
		1641	/* Move from whatever list we were on to the tail of execution. */
		1642	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1643	list_move_tail(&obj->ring_list, &ring->active_list);
		1644
3031	serge	1645	obj->last_read_seqno = seqno;
		1646
2332	Serge	1647	if (obj->fenced_gpu_access) {
3031	serge	1648	obj->last_fenced_seqno = seqno;
		1649
		1650	/* Bump MRU to take account of the delayed flush */
		1651	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1652	struct drm_i915_fence_reg *reg;
		1653
		1654	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1655	list_move_tail(®->lru_list,
		1656	&dev_priv->mm.fence_list);
		1657	}
2332	Serge	1658	}
		1659	}
		1660
2344	Serge	1661	static void
3031	serge	1662	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1663	{
		1664	struct drm_device *dev = obj->base.dev;
3031	serge	1665	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1666
3031	serge	1667	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1668	BUG_ON(!obj->active);
2332	Serge	1669
3031	serge	1670	if (obj->pin_count) /* are we a framebuffer? */
		1671	intel_mark_fb_idle(obj);
2344	Serge	1672
2352	Serge	1673	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1674
3031	serge	1675	list_del_init(&obj->ring_list);
2352	Serge	1676	obj->ring = NULL;
2344	Serge	1677
3031	serge	1678	obj->last_read_seqno = 0;
		1679	obj->last_write_seqno = 0;
		1680	obj->base.write_domain = 0;
		1681
		1682	obj->last_fenced_seqno = 0;
2352	Serge	1683	obj->fenced_gpu_access = false;
2344	Serge	1684
2352	Serge	1685	obj->active = 0;
		1686	drm_gem_object_unreference(&obj->base);
		1687
		1688	WARN_ON(i915_verify_lists(dev));
		1689	}
		1690
3243	Serge	1691	static int
		1692	i915_gem_handle_seqno_wrap(struct drm_device *dev)
2344	Serge	1693	{
3243	Serge	1694	struct drm_i915_private *dev_priv = dev->dev_private;
		1695	struct intel_ring_buffer *ring;
		1696	int ret, i, j;
2344	Serge	1697
3243	Serge	1698	/* The hardware uses various monotonic 32-bit counters, if we
		1699	* detect that they will wraparound we need to idle the GPU
		1700	* and reset those counters.
		1701	*/
		1702	ret = 0;
		1703	for_each_ring(ring, dev_priv, i) {
		1704	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1705	ret \|= ring->sync_seqno[j] != 0;
		1706	}
		1707	if (ret == 0)
		1708	return ret;
2344	Serge	1709
3243	Serge	1710	ret = i915_gpu_idle(dev);
		1711	if (ret)
		1712	return ret;
		1713
		1714	i915_gem_retire_requests(dev);
		1715	for_each_ring(ring, dev_priv, i) {
		1716	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1717	ring->sync_seqno[j] = 0;
		1718	}
		1719
		1720	return 0;
2344	Serge	1721	}
		1722
3243	Serge	1723	int
		1724	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1725	{
3243	Serge	1726	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1727
3243	Serge	1728	/* reserve 0 for non-seqno */
		1729	if (dev_priv->next_seqno == 0) {
		1730	int ret = i915_gem_handle_seqno_wrap(dev);
		1731	if (ret)
		1732	return ret;
		1733
		1734	dev_priv->next_seqno = 1;
		1735	}
		1736
		1737	*seqno = dev_priv->next_seqno++;
		1738	return 0;
2332	Serge	1739	}
		1740
2352	Serge	1741	int
		1742	i915_add_request(struct intel_ring_buffer *ring,
		1743	struct drm_file *file,
3031	serge	1744	u32 *out_seqno)
2352	Serge	1745	{
		1746	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1747	struct drm_i915_gem_request *request;
		1748	u32 request_ring_position;
2352	Serge	1749	int was_empty;
		1750	int ret;
2332	Serge	1751
3031	serge	1752	/*
		1753	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1754	* after having emitted the batchbuffer command. Hence we need to fix
		1755	* things up similar to emitting the lazy request. The difference here
		1756	* is that the flush _must_ happen before the next request, no matter
		1757	* what.
		1758	*/
		1759	ret = intel_ring_flush_all_caches(ring);
		1760	if (ret)
		1761	return ret;
2332	Serge	1762
3031	serge	1763	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1764	if (request == NULL)
		1765	return -ENOMEM;
		1766
		1767
		1768	/* Record the position of the start of the request so that
		1769	* should we detect the updated seqno part-way through the
		1770	* GPU processing the request, we never over-estimate the
		1771	* position of the head.
		1772	*/
		1773	request_ring_position = intel_ring_get_tail(ring);
		1774
3243	Serge	1775	ret = ring->add_request(ring);
3031	serge	1776	if (ret) {
		1777	kfree(request);
2352	Serge	1778	return ret;
3031	serge	1779	}
2332	Serge	1780
3243	Serge	1781	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1782	request->ring = ring;
3031	serge	1783	request->tail = request_ring_position;
		1784	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1785	was_empty = list_empty(&ring->request_list);
		1786	list_add_tail(&request->list, &ring->request_list);
3031	serge	1787	request->file_priv = NULL;
2332	Serge	1788
3263	Serge	1789	if (file) {
		1790	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1791
3263	Serge	1792	spin_lock(&file_priv->mm.lock);
		1793	request->file_priv = file_priv;
		1794	list_add_tail(&request->client_list,
		1795	&file_priv->mm.request_list);
		1796	spin_unlock(&file_priv->mm.lock);
		1797	}
		1798
		1799	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1800	ring->outstanding_lazy_request = 0;
2332	Serge	1801
2360	Serge	1802	if (!dev_priv->mm.suspended) {
		1803	if (i915_enable_hangcheck) {
2352	Serge	1804	// mod_timer(&dev_priv->hangcheck_timer,
		1805	// jiffies +
		1806	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1807	}
3031	serge	1808	if (was_empty) {
2360	Serge	1809	queue_delayed_work(dev_priv->wq,
		1810	&dev_priv->mm.retire_work, HZ);
3031	serge	1811	intel_mark_busy(dev_priv->dev);
		1812	}
2360	Serge	1813	}
3031	serge	1814
		1815	if (out_seqno)
3243	Serge	1816	*out_seqno = request->seqno;
2352	Serge	1817	return 0;
		1818	}
2332	Serge	1819
3263	Serge	1820	static inline void
		1821	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1822	{
		1823	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1824
3263	Serge	1825	if (!file_priv)
		1826	return;
2332	Serge	1827
3263	Serge	1828	spin_lock(&file_priv->mm.lock);
		1829	if (request->file_priv) {
		1830	list_del(&request->client_list);
		1831	request->file_priv = NULL;
		1832	}
		1833	spin_unlock(&file_priv->mm.lock);
		1834	}
2332	Serge	1835
3031	serge	1836	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1837	struct intel_ring_buffer *ring)
		1838	{
		1839	while (!list_empty(&ring->request_list)) {
		1840	struct drm_i915_gem_request *request;
2332	Serge	1841
3031	serge	1842	request = list_first_entry(&ring->request_list,
		1843	struct drm_i915_gem_request,
		1844	list);
2332	Serge	1845
3031	serge	1846	list_del(&request->list);
3263	Serge	1847	i915_gem_request_remove_from_client(request);
3031	serge	1848	kfree(request);
		1849	}
2332	Serge	1850
3031	serge	1851	while (!list_empty(&ring->active_list)) {
		1852	struct drm_i915_gem_object *obj;
2332	Serge	1853
3031	serge	1854	obj = list_first_entry(&ring->active_list,
		1855	struct drm_i915_gem_object,
		1856	ring_list);
2332	Serge	1857
3031	serge	1858	i915_gem_object_move_to_inactive(obj);
		1859	}
		1860	}
2332	Serge	1861
3031	serge	1862	static void i915_gem_reset_fences(struct drm_device *dev)
		1863	{
		1864	struct drm_i915_private *dev_priv = dev->dev_private;
		1865	int i;
2332	Serge	1866
3031	serge	1867	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1868	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1869
3031	serge	1870	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1871
3031	serge	1872	if (reg->obj)
		1873	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1874
3031	serge	1875	reg->pin_count = 0;
		1876	reg->obj = NULL;
		1877	INIT_LIST_HEAD(®->lru_list);
		1878	}
2360	Serge	1879
3031	serge	1880	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1881	}
2360	Serge	1882
3031	serge	1883	void i915_gem_reset(struct drm_device *dev)
		1884	{
		1885	struct drm_i915_private *dev_priv = dev->dev_private;
		1886	struct drm_i915_gem_object *obj;
		1887	struct intel_ring_buffer *ring;
		1888	int i;
2360	Serge	1889
3031	serge	1890	for_each_ring(ring, dev_priv, i)
		1891	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1892
3031	serge	1893	/* Move everything out of the GPU domains to ensure we do any
		1894	* necessary invalidation upon reuse.
		1895	*/
		1896	list_for_each_entry(obj,
		1897	&dev_priv->mm.inactive_list,
		1898	mm_list)
		1899	{
		1900	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1901	}
2360	Serge	1902
3031	serge	1903	/* The fence registers are invalidated so clear them out */
		1904	i915_gem_reset_fences(dev);
		1905	}
2360	Serge	1906
2352	Serge	1907	/**
		1908	* This function clears the request list as sequence numbers are passed.
		1909	*/
3031	serge	1910	void
2352	Serge	1911	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1912	{
		1913	uint32_t seqno;
2332	Serge	1914
2352	Serge	1915	if (list_empty(&ring->request_list))
		1916	return;
2332	Serge	1917
2352	Serge	1918	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1919
3031	serge	1920	seqno = ring->get_seqno(ring, true);
2332	Serge	1921
2352	Serge	1922	while (!list_empty(&ring->request_list)) {
		1923	struct drm_i915_gem_request *request;
2332	Serge	1924
2352	Serge	1925	request = list_first_entry(&ring->request_list,
		1926	struct drm_i915_gem_request,
		1927	list);
2332	Serge	1928
2352	Serge	1929	if (!i915_seqno_passed(seqno, request->seqno))
		1930	break;
2332	Serge	1931
2352	Serge	1932	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1933	/* We know the GPU must have read the request to have
		1934	* sent us the seqno + interrupt, so use the position
		1935	* of tail of the request to update the last known position
		1936	* of the GPU head.
		1937	*/
		1938	ring->last_retired_head = request->tail;
2332	Serge	1939
2352	Serge	1940	list_del(&request->list);
3263	Serge	1941	i915_gem_request_remove_from_client(request);
2352	Serge	1942	kfree(request);
		1943	}
2332	Serge	1944
2352	Serge	1945	/* Move any buffers on the active list that are no longer referenced
		1946	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1947	*/
		1948	while (!list_empty(&ring->active_list)) {
		1949	struct drm_i915_gem_object *obj;
2332	Serge	1950
2352	Serge	1951	obj = list_first_entry(&ring->active_list,
		1952	struct drm_i915_gem_object,
		1953	ring_list);
2332	Serge	1954
3031	serge	1955	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1956	break;
2332	Serge	1957
2352	Serge	1958	i915_gem_object_move_to_inactive(obj);
		1959	}
2332	Serge	1960
2352	Serge	1961	if (unlikely(ring->trace_irq_seqno &&
		1962	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1963	ring->irq_put(ring);
		1964	ring->trace_irq_seqno = 0;
		1965	}
2332	Serge	1966
2352	Serge	1967	WARN_ON(i915_verify_lists(ring->dev));
		1968	}
2332	Serge	1969
2352	Serge	1970	void
		1971	i915_gem_retire_requests(struct drm_device *dev)
		1972	{
		1973	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1974	struct intel_ring_buffer *ring;
2352	Serge	1975	int i;
2332	Serge	1976
3031	serge	1977	for_each_ring(ring, dev_priv, i)
		1978	i915_gem_retire_requests_ring(ring);
2352	Serge	1979	}
		1980
2360	Serge	1981	static void
		1982	i915_gem_retire_work_handler(struct work_struct *work)
		1983	{
		1984	drm_i915_private_t *dev_priv;
		1985	struct drm_device *dev;
3031	serge	1986	struct intel_ring_buffer *ring;
2360	Serge	1987	bool idle;
		1988	int i;
2352	Serge	1989
2360	Serge	1990	dev_priv = container_of(work, drm_i915_private_t,
		1991	mm.retire_work.work);
		1992	dev = dev_priv->dev;
2352	Serge	1993
2360	Serge	1994	/* Come back later if the device is busy... */
		1995	if (!mutex_trylock(&dev->struct_mutex)) {
		1996	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	1997	return;
2360	Serge	1998	}
2352	Serge	1999
2360	Serge	2000	i915_gem_retire_requests(dev);
2352	Serge	2001
2360	Serge	2002	/* Send a periodic flush down the ring so we don't hold onto GEM
		2003	* objects indefinitely.
		2004	*/
		2005	idle = true;
3031	serge	2006	for_each_ring(ring, dev_priv, i) {
		2007	if (ring->gpu_caches_dirty)
		2008	i915_add_request(ring, NULL, NULL);
2352	Serge	2009
2360	Serge	2010	idle &= list_empty(&ring->request_list);
		2011	}
2352	Serge	2012
2360	Serge	2013	if (!dev_priv->mm.suspended && !idle)
		2014	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	2015	if (idle)
		2016	intel_mark_idle(dev);
2360	Serge	2017
		2018	mutex_unlock(&dev->struct_mutex);
		2019	}
		2020
2344	Serge	2021	/**
3031	serge	2022	* Ensures that an object will eventually get non-busy by flushing any required
		2023	* write domains, emitting any outstanding lazy request and retiring and
		2024	* completed requests.
2352	Serge	2025	*/
3031	serge	2026	static int
		2027	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2028	{
3031	serge	2029	int ret;
2352	Serge	2030
3031	serge	2031	if (obj->active) {
		2032	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2033	if (ret)
		2034	return ret;
2352	Serge	2035
3031	serge	2036	i915_gem_retire_requests_ring(obj->ring);
		2037	}
2352	Serge	2038
3031	serge	2039	return 0;
		2040	}
2352	Serge	2041
3243	Serge	2042	/**
		2043	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2044	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2045	*
		2046	* Returns 0 if successful, else an error is returned with the remaining time in
		2047	* the timeout parameter.
		2048	* -ETIME: object is still busy after timeout
		2049	* -ERESTARTSYS: signal interrupted the wait
		2050	* -ENONENT: object doesn't exist
		2051	* Also possible, but rare:
		2052	* -EAGAIN: GPU wedged
		2053	* -ENOMEM: damn
		2054	* -ENODEV: Internal IRQ fail
		2055	* -E?: The add request failed
		2056	*
		2057	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2058	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2059	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2060	* without holding struct_mutex the object may become re-busied before this
		2061	* function completes. A similar but shorter * race condition exists in the busy
		2062	* ioctl
		2063	*/
2352	Serge	2064
		2065
		2066
		2067
		2068
		2069
		2070
		2071
		2072
3243	Serge	2073
		2074
		2075
2352	Serge	2076	/**
3031	serge	2077	* i915_gem_object_sync - sync an object to a ring.
		2078	*
		2079	* @obj: object which may be in use on another ring.
		2080	* @to: ring we wish to use the object on. May be NULL.
		2081	*
		2082	* This code is meant to abstract object synchronization with the GPU.
		2083	* Calling with NULL implies synchronizing the object with the CPU
		2084	* rather than a particular GPU ring.
		2085	*
		2086	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2087	*/
		2088	int
3031	serge	2089	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2090	struct intel_ring_buffer *to)
2344	Serge	2091	{
3031	serge	2092	struct intel_ring_buffer *from = obj->ring;
		2093	u32 seqno;
		2094	int ret, idx;
2332	Serge	2095
3031	serge	2096	if (from == NULL \|\| to == from)
		2097	return 0;
2332	Serge	2098
3031	serge	2099	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2100	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2101
3031	serge	2102	idx = intel_ring_sync_index(from, to);
		2103
		2104	seqno = obj->last_read_seqno;
		2105	if (seqno <= from->sync_seqno[idx])
		2106	return 0;
		2107
		2108	ret = i915_gem_check_olr(obj->ring, seqno);
		2109	if (ret)
		2110	return ret;
		2111
		2112	ret = to->sync_to(to, from, seqno);
		2113	if (!ret)
3243	Serge	2114	/* We use last_read_seqno because sync_to()
		2115	* might have just caused seqno wrap under
		2116	* the radar.
		2117	*/
		2118	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2119
		2120	return ret;
2344	Serge	2121	}
2332	Serge	2122
2344	Serge	2123	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2124	{
		2125	u32 old_write_domain, old_read_domains;
2332	Serge	2126
2344	Serge	2127	/* Act a barrier for all accesses through the GTT */
		2128	mb();
2332	Serge	2129
2344	Serge	2130	/* Force a pagefault for domain tracking on next user access */
		2131	// i915_gem_release_mmap(obj);
2332	Serge	2132
2344	Serge	2133	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2134	return;
2332	Serge	2135
2344	Serge	2136	old_read_domains = obj->base.read_domains;
		2137	old_write_domain = obj->base.write_domain;
2351	Serge	2138
2344	Serge	2139	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2140	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2141
2351	Serge	2142	trace_i915_gem_object_change_domain(obj,
		2143	old_read_domains,
		2144	old_write_domain);
2344	Serge	2145	}
2332	Serge	2146
2344	Serge	2147	/**
		2148	* Unbinds an object from the GTT aperture.
		2149	*/
		2150	int
		2151	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2152	{
3031	serge	2153	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	2154	int ret = 0;
2332	Serge	2155
3263	Serge	2156	if(obj == get_fb_obj())
		2157	return 0;
		2158
2344	Serge	2159	if (obj->gtt_space == NULL)
		2160	return 0;
2332	Serge	2161
3031	serge	2162	if (obj->pin_count)
		2163	return -EBUSY;
2332	Serge	2164
3243	Serge	2165	BUG_ON(obj->pages == NULL);
3031	serge	2166
2344	Serge	2167	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2168	if (ret)
2344	Serge	2169	return ret;
		2170	/* Continue on if we fail due to EIO, the GPU is hung so we
		2171	* should be safe and we need to cleanup or else we might
		2172	* cause memory corruption through use-after-free.
		2173	*/
2332	Serge	2174
2344	Serge	2175	i915_gem_object_finish_gtt(obj);
2332	Serge	2176
2344	Serge	2177	/* release the fence reg _after_ flushing */
		2178	ret = i915_gem_object_put_fence(obj);
3031	serge	2179	if (ret)
2344	Serge	2180	return ret;
2332	Serge	2181
2351	Serge	2182	trace_i915_gem_object_unbind(obj);
2332	Serge	2183
3031	serge	2184	if (obj->has_global_gtt_mapping)
3243	Serge	2185	i915_gem_gtt_unbind_object(obj);
3031	serge	2186	if (obj->has_aliasing_ppgtt_mapping) {
		2187	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2188	obj->has_aliasing_ppgtt_mapping = 0;
		2189	}
		2190	i915_gem_gtt_finish_object(obj);
2332	Serge	2191
3031	serge	2192	list_del(&obj->mm_list);
		2193	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2194	/* Avoid an unnecessary call to unbind on rebind. */
		2195	obj->map_and_fenceable = true;
2332	Serge	2196
2344	Serge	2197	drm_mm_put_block(obj->gtt_space);
		2198	obj->gtt_space = NULL;
		2199	obj->gtt_offset = 0;
2332	Serge	2200
2344	Serge	2201	return 0;
		2202	}
2332	Serge	2203
3031	serge	2204	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2205	{
		2206	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2207	struct intel_ring_buffer *ring;
2344	Serge	2208	int ret, i;
2332	Serge	2209
2344	Serge	2210	/* Flush everything onto the inactive list. */
3031	serge	2211	for_each_ring(ring, dev_priv, i) {
		2212	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2213	if (ret)
		2214	return ret;
3031	serge	2215
3243	Serge	2216	ret = intel_ring_idle(ring);
3031	serge	2217	if (ret)
		2218	return ret;
2344	Serge	2219	}
2332	Serge	2220
2344	Serge	2221	return 0;
		2222	}
2332	Serge	2223
3031	serge	2224	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2225	struct drm_i915_gem_object *obj)
		2226	{
		2227	drm_i915_private_t *dev_priv = dev->dev_private;
		2228	uint64_t val;
2332	Serge	2229
3031	serge	2230	if (obj) {
		2231	u32 size = obj->gtt_space->size;
2332	Serge	2232
3031	serge	2233	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2234	0xfffff000) << 32;
		2235	val \|= obj->gtt_offset & 0xfffff000;
		2236	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2237	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2238
3031	serge	2239	if (obj->tiling_mode == I915_TILING_Y)
		2240	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2241	val \|= I965_FENCE_REG_VALID;
		2242	} else
		2243	val = 0;
2332	Serge	2244
3031	serge	2245	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2246	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2247	}
2332	Serge	2248
3031	serge	2249	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2250	struct drm_i915_gem_object *obj)
		2251	{
		2252	drm_i915_private_t *dev_priv = dev->dev_private;
		2253	uint64_t val;
2332	Serge	2254
3031	serge	2255	if (obj) {
		2256	u32 size = obj->gtt_space->size;
2332	Serge	2257
3031	serge	2258	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2259	0xfffff000) << 32;
		2260	val \|= obj->gtt_offset & 0xfffff000;
		2261	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2262	if (obj->tiling_mode == I915_TILING_Y)
		2263	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2264	val \|= I965_FENCE_REG_VALID;
		2265	} else
		2266	val = 0;
2332	Serge	2267
3031	serge	2268	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2269	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2270	}
2332	Serge	2271
3031	serge	2272	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2273	struct drm_i915_gem_object *obj)
		2274	{
		2275	drm_i915_private_t *dev_priv = dev->dev_private;
		2276	u32 val;
2332	Serge	2277
3031	serge	2278	if (obj) {
		2279	u32 size = obj->gtt_space->size;
		2280	int pitch_val;
		2281	int tile_width;
2332	Serge	2282
3031	serge	2283	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2284	(size & -size) != size \|\|
		2285	(obj->gtt_offset & (size - 1)),
		2286	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2287	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2288
3031	serge	2289	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2290	tile_width = 128;
		2291	else
		2292	tile_width = 512;
2332	Serge	2293
3031	serge	2294	/* Note: pitch better be a power of two tile widths */
		2295	pitch_val = obj->stride / tile_width;
		2296	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2297
3031	serge	2298	val = obj->gtt_offset;
		2299	if (obj->tiling_mode == I915_TILING_Y)
		2300	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2301	val \|= I915_FENCE_SIZE_BITS(size);
		2302	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2303	val \|= I830_FENCE_REG_VALID;
		2304	} else
		2305	val = 0;
2332	Serge	2306
3031	serge	2307	if (reg < 8)
		2308	reg = FENCE_REG_830_0 + reg * 4;
		2309	else
		2310	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2311
3031	serge	2312	I915_WRITE(reg, val);
		2313	POSTING_READ(reg);
		2314	}
2332	Serge	2315
3031	serge	2316	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2317	struct drm_i915_gem_object *obj)
		2318	{
		2319	drm_i915_private_t *dev_priv = dev->dev_private;
		2320	uint32_t val;
2344	Serge	2321
3031	serge	2322	if (obj) {
		2323	u32 size = obj->gtt_space->size;
		2324	uint32_t pitch_val;
2344	Serge	2325
3031	serge	2326	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2327	(size & -size) != size \|\|
		2328	(obj->gtt_offset & (size - 1)),
		2329	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2330	obj->gtt_offset, size);
2344	Serge	2331
3031	serge	2332	pitch_val = obj->stride / 128;
		2333	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2334
3031	serge	2335	val = obj->gtt_offset;
		2336	if (obj->tiling_mode == I915_TILING_Y)
		2337	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2338	val \|= I830_FENCE_SIZE_BITS(size);
		2339	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2340	val \|= I830_FENCE_REG_VALID;
		2341	} else
		2342	val = 0;
		2343
		2344	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2345	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2346	}
		2347
		2348	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2349	struct drm_i915_gem_object *obj)
2332	Serge	2350	{
3031	serge	2351	switch (INTEL_INFO(dev)->gen) {
		2352	case 7:
		2353	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2354	case 5:
		2355	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2356	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2357	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2358	default: break;
		2359	}
2344	Serge	2360	}
		2361
3031	serge	2362	static inline int fence_number(struct drm_i915_private *dev_priv,
		2363	struct drm_i915_fence_reg *fence)
2344	Serge	2364	{
3031	serge	2365	return fence - dev_priv->fence_regs;
		2366	}
2332	Serge	2367
3031	serge	2368	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2369	struct drm_i915_fence_reg *fence,
		2370	bool enable)
		2371	{
		2372	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2373	int reg = fence_number(dev_priv, fence);
2332	Serge	2374
3031	serge	2375	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2376
		2377	if (enable) {
		2378	obj->fence_reg = reg;
		2379	fence->obj = obj;
		2380	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2381	} else {
		2382	obj->fence_reg = I915_FENCE_REG_NONE;
		2383	fence->obj = NULL;
		2384	list_del_init(&fence->lru_list);
2344	Serge	2385	}
3031	serge	2386	}
2344	Serge	2387
3031	serge	2388	static int
		2389	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2390	{
		2391	if (obj->last_fenced_seqno) {
		2392	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2393	if (ret)
		2394	return ret;
2344	Serge	2395
		2396	obj->last_fenced_seqno = 0;
		2397	}
		2398
		2399	/* Ensure that all CPU reads are completed before installing a fence
		2400	* and all writes before removing the fence.
2332	Serge	2401	*/
2344	Serge	2402	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2403	mb();
2332	Serge	2404
3031	serge	2405	obj->fenced_gpu_access = false;
2332	Serge	2406	return 0;
		2407	}
		2408
		2409	int
2344	Serge	2410	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2411	{
3031	serge	2412	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2413	int ret;
		2414
3031	serge	2415	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2416	if (ret)
		2417	return ret;
		2418
3031	serge	2419	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2420	return 0;
2332	Serge	2421
3031	serge	2422	i915_gem_object_update_fence(obj,
		2423	&dev_priv->fence_regs[obj->fence_reg],
		2424	false);
		2425	i915_gem_object_fence_lost(obj);
2344	Serge	2426
2332	Serge	2427	return 0;
		2428	}
		2429
3031	serge	2430	static struct drm_i915_fence_reg *
		2431	i915_find_fence_reg(struct drm_device *dev)
		2432	{
		2433	struct drm_i915_private *dev_priv = dev->dev_private;
		2434	struct drm_i915_fence_reg reg, avail;
		2435	int i;
2332	Serge	2436
3031	serge	2437	/* First try to find a free reg */
		2438	avail = NULL;
		2439	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2440	reg = &dev_priv->fence_regs[i];
		2441	if (!reg->obj)
		2442	return reg;
2332	Serge	2443
3031	serge	2444	if (!reg->pin_count)
		2445	avail = reg;
		2446	}
2332	Serge	2447
3031	serge	2448	if (avail == NULL)
		2449	return NULL;
2332	Serge	2450
3031	serge	2451	/* None available, try to steal one or wait for a user to finish */
		2452	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2453	if (reg->pin_count)
		2454	continue;
2332	Serge	2455
3031	serge	2456	return reg;
		2457	}
2332	Serge	2458
3031	serge	2459	return NULL;
		2460	}
2332	Serge	2461
3031	serge	2462	/**
		2463	* i915_gem_object_get_fence - set up fencing for an object
		2464	* @obj: object to map through a fence reg
		2465	*
		2466	* When mapping objects through the GTT, userspace wants to be able to write
		2467	* to them without having to worry about swizzling if the object is tiled.
		2468	* This function walks the fence regs looking for a free one for @obj,
		2469	* stealing one if it can't find any.
		2470	*
		2471	* It then sets up the reg based on the object's properties: address, pitch
		2472	* and tiling format.
		2473	*
		2474	* For an untiled surface, this removes any existing fence.
		2475	*/
		2476	int
		2477	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2478	{
		2479	struct drm_device *dev = obj->base.dev;
		2480	struct drm_i915_private *dev_priv = dev->dev_private;
		2481	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2482	struct drm_i915_fence_reg *reg;
		2483	int ret;
2332	Serge	2484
3031	serge	2485	/* Have we updated the tiling parameters upon the object and so
		2486	* will need to serialise the write to the associated fence register?
		2487	*/
		2488	if (obj->fence_dirty) {
		2489	ret = i915_gem_object_flush_fence(obj);
		2490	if (ret)
		2491	return ret;
		2492	}
2332	Serge	2493
3031	serge	2494	/* Just update our place in the LRU if our fence is getting reused. */
		2495	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2496	reg = &dev_priv->fence_regs[obj->fence_reg];
		2497	if (!obj->fence_dirty) {
		2498	list_move_tail(®->lru_list,
		2499	&dev_priv->mm.fence_list);
		2500	return 0;
		2501	}
		2502	} else if (enable) {
		2503	reg = i915_find_fence_reg(dev);
		2504	if (reg == NULL)
		2505	return -EDEADLK;
2332	Serge	2506
3031	serge	2507	if (reg->obj) {
		2508	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2509
3031	serge	2510	ret = i915_gem_object_flush_fence(old);
		2511	if (ret)
		2512	return ret;
2332	Serge	2513
3031	serge	2514	i915_gem_object_fence_lost(old);
		2515	}
		2516	} else
		2517	return 0;
2332	Serge	2518
3031	serge	2519	i915_gem_object_update_fence(obj, reg, enable);
		2520	obj->fence_dirty = false;
2332	Serge	2521
3031	serge	2522	return 0;
		2523	}
2332	Serge	2524
3031	serge	2525	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2526	struct drm_mm_node *gtt_space,
		2527	unsigned long cache_level)
		2528	{
		2529	struct drm_mm_node *other;
2332	Serge	2530
3031	serge	2531	/* On non-LLC machines we have to be careful when putting differing
		2532	* types of snoopable memory together to avoid the prefetcher
		2533	* crossing memory domains and dieing.
		2534	*/
		2535	if (HAS_LLC(dev))
		2536	return true;
2332	Serge	2537
3031	serge	2538	if (gtt_space == NULL)
		2539	return true;
2332	Serge	2540
3031	serge	2541	if (list_empty(>t_space->node_list))
		2542	return true;
2332	Serge	2543
3031	serge	2544	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2545	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2546	return false;
2344	Serge	2547
3031	serge	2548	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2549	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2550	return false;
2344	Serge	2551
3031	serge	2552	return true;
		2553	}
2344	Serge	2554
3031	serge	2555	static void i915_gem_verify_gtt(struct drm_device *dev)
		2556	{
		2557	#if WATCH_GTT
		2558	struct drm_i915_private *dev_priv = dev->dev_private;
		2559	struct drm_i915_gem_object *obj;
		2560	int err = 0;
2344	Serge	2561
3031	serge	2562	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2563	if (obj->gtt_space == NULL) {
		2564	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2565	err++;
		2566	continue;
		2567	}
2344	Serge	2568
3031	serge	2569	if (obj->cache_level != obj->gtt_space->color) {
		2570	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2571	obj->gtt_space->start,
		2572	obj->gtt_space->start + obj->gtt_space->size,
		2573	obj->cache_level,
		2574	obj->gtt_space->color);
		2575	err++;
		2576	continue;
		2577	}
2344	Serge	2578
3031	serge	2579	if (!i915_gem_valid_gtt_space(dev,
		2580	obj->gtt_space,
		2581	obj->cache_level)) {
		2582	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2583	obj->gtt_space->start,
		2584	obj->gtt_space->start + obj->gtt_space->size,
		2585	obj->cache_level);
		2586	err++;
		2587	continue;
		2588	}
		2589	}
2344	Serge	2590
3031	serge	2591	WARN_ON(err);
		2592	#endif
2326	Serge	2593	}
		2594
2332	Serge	2595	/**
		2596	* Finds free space in the GTT aperture and binds the object there.
		2597	*/
		2598	static int
		2599	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2600	unsigned alignment,
3031	serge	2601	bool map_and_fenceable,
		2602	bool nonblocking)
2332	Serge	2603	{
		2604	struct drm_device *dev = obj->base.dev;
		2605	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2606	struct drm_mm_node *node;
2332	Serge	2607	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2608	bool mappable, fenceable;
		2609	int ret;
2326	Serge	2610
2332	Serge	2611	if (obj->madv != I915_MADV_WILLNEED) {
3243	Serge	2612	DRM_ERROR("Attempting to bind a purgeable object\n");
2332	Serge	2613	return -EINVAL;
		2614	}
		2615
		2616	fence_size = i915_gem_get_gtt_size(dev,
		2617	obj->base.size,
		2618	obj->tiling_mode);
		2619	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2620	obj->base.size,
		2621	obj->tiling_mode);
		2622	unfenced_alignment =
		2623	i915_gem_get_unfenced_gtt_alignment(dev,
		2624	obj->base.size,
		2625	obj->tiling_mode);
		2626
		2627	if (alignment == 0)
		2628	alignment = map_and_fenceable ? fence_alignment :
		2629	unfenced_alignment;
		2630	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2631	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2632	return -EINVAL;
		2633	}
		2634
		2635	size = map_and_fenceable ? fence_size : obj->base.size;
		2636
		2637	/* If the object is bigger than the entire aperture, reject it early
		2638	* before evicting everything in a vain attempt to find space.
		2639	*/
		2640	if (obj->base.size >
		2641	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2642	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2643	return -E2BIG;
		2644	}
		2645
3031	serge	2646	ret = i915_gem_object_get_pages(obj);
		2647	if (ret)
		2648	return ret;
		2649
3243	Serge	2650	i915_gem_object_pin_pages(obj);
		2651
		2652	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2653	if (node == NULL) {
		2654	i915_gem_object_unpin_pages(obj);
		2655	return -ENOMEM;
		2656	}
		2657
2332	Serge	2658	search_free:
		2659	if (map_and_fenceable)
3243	Serge	2660	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2661	size, alignment, obj->cache_level,
3243	Serge	2662	0, dev_priv->mm.gtt_mappable_end);
2332	Serge	2663	else
3243	Serge	2664	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2665	size, alignment, obj->cache_level);
		2666	if (ret) {
2332	Serge	2667
3243	Serge	2668	i915_gem_object_unpin_pages(obj);
		2669	kfree(node);
2332	Serge	2670	return ret;
		2671	}
3243	Serge	2672	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2673	i915_gem_object_unpin_pages(obj);
		2674	drm_mm_put_block(node);
3031	serge	2675	return -EINVAL;
		2676	}
2332	Serge	2677
3031	serge	2678	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2679	if (ret) {
3243	Serge	2680	i915_gem_object_unpin_pages(obj);
		2681	drm_mm_put_block(node);
2332	Serge	2682	return ret;
		2683	}
		2684
3031	serge	2685	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2686	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2687
3243	Serge	2688	obj->gtt_space = node;
		2689	obj->gtt_offset = node->start;
2332	Serge	2690
		2691	fenceable =
3243	Serge	2692	node->size == fence_size &&
		2693	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2694
		2695	mappable =
		2696	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2697
		2698	obj->map_and_fenceable = mappable && fenceable;
		2699
3243	Serge	2700	i915_gem_object_unpin_pages(obj);
2351	Serge	2701	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2702	i915_gem_verify_gtt(dev);
2332	Serge	2703	return 0;
		2704	}
		2705
		2706	void
		2707	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2708	{
		2709	/* If we don't have a page list set up, then we're not pinned
		2710	* to GPU, and we can ignore the cache flush because it'll happen
		2711	* again at bind time.
		2712	*/
3243	Serge	2713	if (obj->pages == NULL)
2332	Serge	2714	return;
		2715
		2716	/* If the GPU is snooping the contents of the CPU cache,
		2717	* we do not need to manually clear the CPU cache lines. However,
		2718	* the caches are only snooped when the render cache is
		2719	* flushed/invalidated. As we always have to emit invalidations
		2720	* and flushes when moving into and out of the RENDER domain, correct
		2721	* snooping behaviour occurs naturally as the result of our domain
		2722	* tracking.
		2723	*/
		2724	if (obj->cache_level != I915_CACHE_NONE)
		2725	return;
3243	Serge	2726	#if 0
2344	Serge	2727	if(obj->mapped != NULL)
		2728	{
		2729	uint8_t *page_virtual;
		2730	unsigned int i;
2332	Serge	2731
2344	Serge	2732	page_virtual = obj->mapped;
		2733	asm volatile("mfence");
		2734	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2735	clflush(page_virtual + i);
		2736	asm volatile("mfence");
		2737	}
		2738	else
		2739	{
		2740	uint8_t *page_virtual;
		2741	unsigned int i;
		2742	page_virtual = AllocKernelSpace(obj->base.size);
		2743	if(page_virtual != NULL)
		2744	{
3031	serge	2745	dma_addr_t src, dst;
2344	Serge	2746	u32 count;
		2747
		2748	#define page_tabs 0xFDC00000 /* really dirty hack */
		2749
3031	serge	2750	src = obj->pages.page;
		2751	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2752	count = obj->base.size/4096;
		2753
		2754	while(count--)
		2755	{
		2756	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2757	};
		2758
		2759	asm volatile("mfence");
		2760	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2761	clflush(page_virtual + i);
		2762	asm volatile("mfence");
		2763	FreeKernelSpace(page_virtual);
		2764	}
		2765	else
		2766	{
		2767	asm volatile (
		2768	"mfence \n"
		2769	"wbinvd \n" /* this is really ugly */
		2770	"mfence");
		2771	}
		2772	}
3243	Serge	2773	#endif
		2774
2332	Serge	2775	}
		2776
2344	Serge	2777	/** Flushes the GTT write domain for the object if it's dirty. */
		2778	static void
		2779	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2780	{
		2781	uint32_t old_write_domain;
2332	Serge	2782
2344	Serge	2783	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2784	return;
2332	Serge	2785
2344	Serge	2786	/* No actual flushing is required for the GTT write domain. Writes
		2787	* to it immediately go to main memory as far as we know, so there's
		2788	* no chipset flush. It also doesn't land in render cache.
		2789	*
		2790	* However, we do have to enforce the order so that all writes through
		2791	* the GTT land before any writes to the device, such as updates to
		2792	* the GATT itself.
		2793	*/
		2794	wmb();
2332	Serge	2795
2344	Serge	2796	old_write_domain = obj->base.write_domain;
		2797	obj->base.write_domain = 0;
2332	Serge	2798
2351	Serge	2799	trace_i915_gem_object_change_domain(obj,
		2800	obj->base.read_domains,
		2801	old_write_domain);
2344	Serge	2802	}
2332	Serge	2803
		2804	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2805	static void
2332	Serge	2806	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2807	{
		2808	uint32_t old_write_domain;
		2809
		2810	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2811	return;
		2812
		2813	i915_gem_clflush_object(obj);
3243	Serge	2814	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2815	old_write_domain = obj->base.write_domain;
		2816	obj->base.write_domain = 0;
		2817
2351	Serge	2818	trace_i915_gem_object_change_domain(obj,
		2819	obj->base.read_domains,
		2820	old_write_domain);
2332	Serge	2821	}
		2822
		2823	/**
		2824	* Moves a single object to the GTT read, and possibly write domain.
		2825	*
		2826	* This function returns when the move is complete, including waiting on
		2827	* flushes to occur.
		2828	*/
		2829	int
		2830	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2831	{
3031	serge	2832	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2833	uint32_t old_write_domain, old_read_domains;
		2834	int ret;
		2835
		2836	/* Not valid to be called on unbound objects. */
		2837	if (obj->gtt_space == NULL)
		2838	return -EINVAL;
		2839
		2840	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2841	return 0;
		2842
3031	serge	2843	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2844	if (ret)
		2845	return ret;
		2846
		2847	i915_gem_object_flush_cpu_write_domain(obj);
		2848
		2849	old_write_domain = obj->base.write_domain;
		2850	old_read_domains = obj->base.read_domains;
		2851
		2852	/* It should now be out of any other write domains, and we can update
		2853	* the domain values for our changes.
		2854	*/
		2855	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2856	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2857	if (write) {
		2858	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2859	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2860	obj->dirty = 1;
		2861	}
		2862
2351	Serge	2863	trace_i915_gem_object_change_domain(obj,
		2864	old_read_domains,
		2865	old_write_domain);
		2866
3031	serge	2867	/* And bump the LRU for this access */
		2868	if (i915_gem_object_is_inactive(obj))
		2869	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2870
2332	Serge	2871	return 0;
		2872	}
		2873
2335	Serge	2874	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2875	enum i915_cache_level cache_level)
		2876	{
3031	serge	2877	struct drm_device *dev = obj->base.dev;
		2878	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2879	int ret;
2332	Serge	2880
2335	Serge	2881	if (obj->cache_level == cache_level)
		2882	return 0;
2332	Serge	2883
2335	Serge	2884	if (obj->pin_count) {
		2885	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2886	return -EBUSY;
		2887	}
2332	Serge	2888
3031	serge	2889	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2890	ret = i915_gem_object_unbind(obj);
		2891	if (ret)
		2892	return ret;
		2893	}
		2894
2335	Serge	2895	if (obj->gtt_space) {
		2896	ret = i915_gem_object_finish_gpu(obj);
		2897	if (ret)
		2898	return ret;
2332	Serge	2899
2335	Serge	2900	i915_gem_object_finish_gtt(obj);
2332	Serge	2901
2335	Serge	2902	/* Before SandyBridge, you could not use tiling or fence
		2903	* registers with snooped memory, so relinquish any fences
		2904	* currently pointing to our region in the aperture.
		2905	*/
3031	serge	2906	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2907	ret = i915_gem_object_put_fence(obj);
		2908	if (ret)
		2909	return ret;
		2910	}
2332	Serge	2911
3031	serge	2912	if (obj->has_global_gtt_mapping)
		2913	i915_gem_gtt_bind_object(obj, cache_level);
		2914	if (obj->has_aliasing_ppgtt_mapping)
		2915	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2916	obj, cache_level);
		2917
		2918	obj->gtt_space->color = cache_level;
2335	Serge	2919	}
2332	Serge	2920
2335	Serge	2921	if (cache_level == I915_CACHE_NONE) {
		2922	u32 old_read_domains, old_write_domain;
2332	Serge	2923
2335	Serge	2924	/* If we're coming from LLC cached, then we haven't
		2925	* actually been tracking whether the data is in the
		2926	* CPU cache or not, since we only allow one bit set
		2927	* in obj->write_domain and have been skipping the clflushes.
		2928	* Just set it to the CPU cache for now.
		2929	*/
		2930	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2931	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2932
2335	Serge	2933	old_read_domains = obj->base.read_domains;
		2934	old_write_domain = obj->base.write_domain;
2332	Serge	2935
2335	Serge	2936	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2937	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2938
2351	Serge	2939	trace_i915_gem_object_change_domain(obj,
		2940	old_read_domains,
		2941	old_write_domain);
2344	Serge	2942	}
2332	Serge	2943
2335	Serge	2944	obj->cache_level = cache_level;
3031	serge	2945	i915_gem_verify_gtt(dev);
2335	Serge	2946	return 0;
		2947	}
2332	Serge	2948
3260	Serge	2949	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		2950	struct drm_file *file)
		2951	{
		2952	struct drm_i915_gem_caching *args = data;
		2953	struct drm_i915_gem_object *obj;
		2954	int ret;
		2955
		2956	ret = i915_mutex_lock_interruptible(dev);
		2957	if (ret)
		2958	return ret;
		2959
		2960	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2961	if (&obj->base == NULL) {
		2962	ret = -ENOENT;
		2963	goto unlock;
		2964	}
		2965
		2966	args->caching = obj->cache_level != I915_CACHE_NONE;
		2967
		2968	drm_gem_object_unreference(&obj->base);
		2969	unlock:
		2970	mutex_unlock(&dev->struct_mutex);
		2971	return ret;
		2972	}
		2973
		2974	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		2975	struct drm_file *file)
		2976	{
		2977	struct drm_i915_gem_caching *args = data;
		2978	struct drm_i915_gem_object *obj;
		2979	enum i915_cache_level level;
		2980	int ret;
		2981
		2982	switch (args->caching) {
		2983	case I915_CACHING_NONE:
		2984	level = I915_CACHE_NONE;
		2985	break;
		2986	case I915_CACHING_CACHED:
		2987	level = I915_CACHE_LLC;
		2988	break;
		2989	default:
		2990	return -EINVAL;
		2991	}
		2992
		2993	ret = i915_mutex_lock_interruptible(dev);
		2994	if (ret)
		2995	return ret;
		2996
		2997	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2998	if (&obj->base == NULL) {
		2999	ret = -ENOENT;
		3000	goto unlock;
		3001	}
		3002
		3003	ret = i915_gem_object_set_cache_level(obj, level);
		3004
		3005	drm_gem_object_unreference(&obj->base);
		3006	unlock:
		3007	mutex_unlock(&dev->struct_mutex);
		3008	return ret;
		3009	}
		3010
2335	Serge	3011	/*
		3012	* Prepare buffer for display plane (scanout, cursors, etc).
		3013	* Can be called from an uninterruptible phase (modesetting) and allows
		3014	* any flushes to be pipelined (for pageflips).
		3015	*/
		3016	int
		3017	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3018	u32 alignment,
		3019	struct intel_ring_buffer *pipelined)
		3020	{
		3021	u32 old_read_domains, old_write_domain;
		3022	int ret;
2332	Serge	3023
3031	serge	3024	if (pipelined != obj->ring) {
		3025	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3026	if (ret)
		3027	return ret;
		3028	}
2332	Serge	3029
2335	Serge	3030	/* The display engine is not coherent with the LLC cache on gen6. As
		3031	* a result, we make sure that the pinning that is about to occur is
		3032	* done with uncached PTEs. This is lowest common denominator for all
		3033	* chipsets.
		3034	*
		3035	* However for gen6+, we could do better by using the GFDT bit instead
		3036	* of uncaching, which would allow us to flush all the LLC-cached data
		3037	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3038	*/
2360	Serge	3039	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3040	if (ret)
		3041	return ret;
2332	Serge	3042
2335	Serge	3043	/* As the user may map the buffer once pinned in the display plane
		3044	* (e.g. libkms for the bootup splash), we have to ensure that we
		3045	* always use map_and_fenceable for all scanout buffers.
		3046	*/
3031	serge	3047	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3048	if (ret)
		3049	return ret;
2332	Serge	3050
2335	Serge	3051	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3052
2335	Serge	3053	old_write_domain = obj->base.write_domain;
		3054	old_read_domains = obj->base.read_domains;
2332	Serge	3055
2335	Serge	3056	/* It should now be out of any other write domains, and we can update
		3057	* the domain values for our changes.
		3058	*/
3031	serge	3059	obj->base.write_domain = 0;
2335	Serge	3060	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3061
2351	Serge	3062	trace_i915_gem_object_change_domain(obj,
		3063	old_read_domains,
		3064	old_write_domain);
2332	Serge	3065
2335	Serge	3066	return 0;
		3067	}
2332	Serge	3068
2344	Serge	3069	int
		3070	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3071	{
		3072	int ret;
2332	Serge	3073
2344	Serge	3074	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3075	return 0;
2332	Serge	3076
3031	serge	3077	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3078	if (ret)
		3079	return ret;
2332	Serge	3080
2344	Serge	3081	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3082	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3083	return 0;
2344	Serge	3084	}
2332	Serge	3085
2344	Serge	3086	/**
		3087	* Moves a single object to the CPU read, and possibly write domain.
		3088	*
		3089	* This function returns when the move is complete, including waiting on
		3090	* flushes to occur.
		3091	*/
3031	serge	3092	int
2344	Serge	3093	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3094	{
		3095	uint32_t old_write_domain, old_read_domains;
		3096	int ret;
2332	Serge	3097
2344	Serge	3098	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3099	return 0;
2332	Serge	3100
3031	serge	3101	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3102	if (ret)
		3103	return ret;
2332	Serge	3104
2344	Serge	3105	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3106
2344	Serge	3107	old_write_domain = obj->base.write_domain;
		3108	old_read_domains = obj->base.read_domains;
2332	Serge	3109
2344	Serge	3110	/* Flush the CPU cache if it's still invalid. */
		3111	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3112	i915_gem_clflush_object(obj);
2332	Serge	3113
2344	Serge	3114	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3115	}
2332	Serge	3116
2344	Serge	3117	/* It should now be out of any other write domains, and we can update
		3118	* the domain values for our changes.
		3119	*/
		3120	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3121
2344	Serge	3122	/* If we're writing through the CPU, then the GPU read domains will
		3123	* need to be invalidated at next use.
		3124	*/
		3125	if (write) {
		3126	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3127	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3128	}
2332	Serge	3129
2351	Serge	3130	trace_i915_gem_object_change_domain(obj,
		3131	old_read_domains,
		3132	old_write_domain);
2332	Serge	3133
2344	Serge	3134	return 0;
		3135	}
2332	Serge	3136
3031	serge	3137	/* Throttle our rendering by waiting until the ring has completed our requests
		3138	* emitted over 20 msec ago.
2344	Serge	3139	*
3031	serge	3140	* Note that if we were to use the current jiffies each time around the loop,
		3141	* we wouldn't escape the function with any frames outstanding if the time to
		3142	* render a frame was over 20ms.
		3143	*
		3144	* This should get us reasonable parallelism between CPU and GPU but also
		3145	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3146	*/
3031	serge	3147	static int
		3148	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3149	{
3031	serge	3150	struct drm_i915_private *dev_priv = dev->dev_private;
		3151	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3152	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3153	struct drm_i915_gem_request *request;
		3154	struct intel_ring_buffer *ring = NULL;
		3155	u32 seqno = 0;
		3156	int ret;
2332	Serge	3157
3031	serge	3158	if (atomic_read(&dev_priv->mm.wedged))
		3159	return -EIO;
2332	Serge	3160
3031	serge	3161	spin_lock(&file_priv->mm.lock);
		3162	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3163	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3164	break;
2332	Serge	3165
3031	serge	3166	ring = request->ring;
		3167	seqno = request->seqno;
		3168	}
		3169	spin_unlock(&file_priv->mm.lock);
2332	Serge	3170
3031	serge	3171	if (seqno == 0)
		3172	return 0;
2332	Serge	3173
3031	serge	3174	ret = __wait_seqno(ring, seqno, true, NULL);
		3175	if (ret == 0)
		3176	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3177
3031	serge	3178	return ret;
2352	Serge	3179	}
2332	Serge	3180
		3181	int
		3182	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3183	uint32_t alignment,
3031	serge	3184	bool map_and_fenceable,
		3185	bool nonblocking)
2332	Serge	3186	{
		3187	int ret;
		3188
3031	serge	3189	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3190	return -EBUSY;
2332	Serge	3191
		3192	if (obj->gtt_space != NULL) {
		3193	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3194	(map_and_fenceable && !obj->map_and_fenceable)) {
		3195	WARN(obj->pin_count,
		3196	"bo is already pinned with incorrect alignment:"
		3197	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3198	" obj->map_and_fenceable=%d\n",
		3199	obj->gtt_offset, alignment,
		3200	map_and_fenceable,
		3201	obj->map_and_fenceable);
		3202	ret = i915_gem_object_unbind(obj);
		3203	if (ret)
		3204	return ret;
		3205	}
		3206	}
		3207
		3208	if (obj->gtt_space == NULL) {
3243	Serge	3209	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3210
2332	Serge	3211	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3212	map_and_fenceable,
		3213	nonblocking);
2332	Serge	3214	if (ret)
		3215	return ret;
3243	Serge	3216
		3217	if (!dev_priv->mm.aliasing_ppgtt)
		3218	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3219	}
		3220
3031	serge	3221	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3222	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3223
		3224	obj->pin_count++;
2332	Serge	3225	obj->pin_mappable \|= map_and_fenceable;
		3226
		3227	return 0;
		3228	}
		3229
2344	Serge	3230	void
		3231	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3232	{
		3233	BUG_ON(obj->pin_count == 0);
		3234	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3235
3031	serge	3236	if (--obj->pin_count == 0)
2344	Serge	3237	obj->pin_mappable = false;
		3238	}
2332	Serge	3239
3031	serge	3240	int
		3241	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3242	struct drm_file *file)
		3243	{
		3244	struct drm_i915_gem_pin *args = data;
		3245	struct drm_i915_gem_object *obj;
		3246	int ret;
2332	Serge	3247
3031	serge	3248	ret = i915_mutex_lock_interruptible(dev);
		3249	if (ret)
		3250	return ret;
2332	Serge	3251
3031	serge	3252	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3253	if (&obj->base == NULL) {
		3254	ret = -ENOENT;
		3255	goto unlock;
		3256	}
2332	Serge	3257
3031	serge	3258	if (obj->madv != I915_MADV_WILLNEED) {
		3259	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3260	ret = -EINVAL;
		3261	goto out;
		3262	}
2332	Serge	3263
3031	serge	3264	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3265	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3266	args->handle);
		3267	ret = -EINVAL;
		3268	goto out;
		3269	}
2332	Serge	3270
3243	Serge	3271	if (obj->user_pin_count == 0) {
3031	serge	3272	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3273	if (ret)
		3274	goto out;
		3275	}
2332	Serge	3276
3243	Serge	3277	obj->user_pin_count++;
		3278	obj->pin_filp = file;
		3279
3031	serge	3280	/* XXX - flush the CPU caches for pinned objects
		3281	* as the X server doesn't manage domains yet
		3282	*/
		3283	i915_gem_object_flush_cpu_write_domain(obj);
		3284	args->offset = obj->gtt_offset;
		3285	out:
		3286	drm_gem_object_unreference(&obj->base);
		3287	unlock:
		3288	mutex_unlock(&dev->struct_mutex);
		3289	return ret;
		3290	}
2332	Serge	3291
3255	Serge	3292	#if 0
		3293
3031	serge	3294	int
		3295	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3296	struct drm_file *file)
		3297	{
		3298	struct drm_i915_gem_pin *args = data;
		3299	struct drm_i915_gem_object *obj;
		3300	int ret;
2332	Serge	3301
3031	serge	3302	ret = i915_mutex_lock_interruptible(dev);
		3303	if (ret)
		3304	return ret;
2332	Serge	3305
3031	serge	3306	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3307	if (&obj->base == NULL) {
		3308	ret = -ENOENT;
		3309	goto unlock;
		3310	}
2332	Serge	3311
3031	serge	3312	if (obj->pin_filp != file) {
		3313	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3314	args->handle);
		3315	ret = -EINVAL;
		3316	goto out;
		3317	}
		3318	obj->user_pin_count--;
		3319	if (obj->user_pin_count == 0) {
		3320	obj->pin_filp = NULL;
		3321	i915_gem_object_unpin(obj);
		3322	}
2332	Serge	3323
3031	serge	3324	out:
		3325	drm_gem_object_unreference(&obj->base);
		3326	unlock:
		3327	mutex_unlock(&dev->struct_mutex);
		3328	return ret;
		3329	}
2332	Serge	3330
3260	Serge	3331	#endif
		3332
3031	serge	3333	int
		3334	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3335	struct drm_file *file)
		3336	{
		3337	struct drm_i915_gem_busy *args = data;
		3338	struct drm_i915_gem_object *obj;
		3339	int ret;
2332	Serge	3340
3031	serge	3341	ret = i915_mutex_lock_interruptible(dev);
		3342	if (ret)
		3343	return ret;
2332	Serge	3344
3031	serge	3345	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3346	if (&obj->base == NULL) {
		3347	ret = -ENOENT;
		3348	goto unlock;
		3349	}
2332	Serge	3350
3031	serge	3351	/* Count all active objects as busy, even if they are currently not used
		3352	* by the gpu. Users of this interface expect objects to eventually
		3353	* become non-busy without any further actions, therefore emit any
		3354	* necessary flushes here.
		3355	*/
		3356	ret = i915_gem_object_flush_active(obj);
2332	Serge	3357
3031	serge	3358	args->busy = obj->active;
		3359	if (obj->ring) {
		3360	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3361	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3362	}
2332	Serge	3363
3031	serge	3364	drm_gem_object_unreference(&obj->base);
		3365	unlock:
		3366	mutex_unlock(&dev->struct_mutex);
		3367	return ret;
		3368	}
2332	Serge	3369
3031	serge	3370	int
		3371	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3372	struct drm_file *file_priv)
		3373	{
		3374	return i915_gem_ring_throttle(dev, file_priv);
		3375	}
2332	Serge	3376
3263	Serge	3377	#if 0
		3378
3031	serge	3379	int
		3380	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3381	struct drm_file *file_priv)
		3382	{
		3383	struct drm_i915_gem_madvise *args = data;
		3384	struct drm_i915_gem_object *obj;
		3385	int ret;
2332	Serge	3386
3031	serge	3387	switch (args->madv) {
		3388	case I915_MADV_DONTNEED:
		3389	case I915_MADV_WILLNEED:
		3390	break;
		3391	default:
		3392	return -EINVAL;
		3393	}
2332	Serge	3394
3031	serge	3395	ret = i915_mutex_lock_interruptible(dev);
		3396	if (ret)
		3397	return ret;
2332	Serge	3398
3031	serge	3399	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3400	if (&obj->base == NULL) {
		3401	ret = -ENOENT;
		3402	goto unlock;
		3403	}
2332	Serge	3404
3031	serge	3405	if (obj->pin_count) {
		3406	ret = -EINVAL;
		3407	goto out;
		3408	}
2332	Serge	3409
3031	serge	3410	if (obj->madv != __I915_MADV_PURGED)
		3411	obj->madv = args->madv;
2332	Serge	3412
3031	serge	3413	/* if the object is no longer attached, discard its backing storage */
		3414	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3415	i915_gem_object_truncate(obj);
2332	Serge	3416
3031	serge	3417	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3418
3031	serge	3419	out:
		3420	drm_gem_object_unreference(&obj->base);
		3421	unlock:
		3422	mutex_unlock(&dev->struct_mutex);
		3423	return ret;
		3424	}
		3425	#endif
2332	Serge	3426
3031	serge	3427	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3428	const struct drm_i915_gem_object_ops *ops)
		3429	{
		3430	INIT_LIST_HEAD(&obj->mm_list);
		3431	INIT_LIST_HEAD(&obj->gtt_list);
		3432	INIT_LIST_HEAD(&obj->ring_list);
		3433	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3434
3031	serge	3435	obj->ops = ops;
		3436
		3437	obj->fence_reg = I915_FENCE_REG_NONE;
		3438	obj->madv = I915_MADV_WILLNEED;
		3439	/* Avoid an unnecessary call to unbind on the first bind. */
		3440	obj->map_and_fenceable = true;
		3441
		3442	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3443	}
		3444
		3445	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3446	.get_pages = i915_gem_object_get_pages_gtt,
		3447	.put_pages = i915_gem_object_put_pages_gtt,
		3448	};
		3449
2332	Serge	3450	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3451	size_t size)
		3452	{
		3453	struct drm_i915_gem_object *obj;
3031	serge	3454	struct address_space *mapping;
		3455	u32 mask;
2340	Serge	3456
2332	Serge	3457	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3458	if (obj == NULL)
		3459	return NULL;
		3460
		3461	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3462	kfree(obj);
		3463	return NULL;
		3464	}
		3465
		3466
3031	serge	3467	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3468
		3469	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3470	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3471
3031	serge	3472	if (HAS_LLC(dev)) {
		3473	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3474	* cache) for about a 10% performance improvement
		3475	* compared to uncached. Graphics requests other than
		3476	* display scanout are coherent with the CPU in
		3477	* accessing this cache. This means in this mode we
		3478	* don't need to clflush on the CPU side, and on the
		3479	* GPU side we only need to flush internal caches to
		3480	* get data visible to the CPU.
		3481	*
		3482	* However, we maintain the display planes as UC, and so
		3483	* need to rebind when first used as such.
		3484	*/
		3485	obj->cache_level = I915_CACHE_LLC;
		3486	} else
		3487	obj->cache_level = I915_CACHE_NONE;
		3488
		3489	return obj;
		3490	}
		3491
2344	Serge	3492	int i915_gem_init_object(struct drm_gem_object *obj)
		3493	{
		3494	BUG();
2332	Serge	3495
2344	Serge	3496	return 0;
		3497	}
2332	Serge	3498
3031	serge	3499	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3500	{
3031	serge	3501	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3502	struct drm_device *dev = obj->base.dev;
		3503	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3504
3031	serge	3505	trace_i915_gem_object_destroy(obj);
		3506
		3507	// if (obj->phys_obj)
		3508	// i915_gem_detach_phys_object(dev, obj);
		3509
		3510	obj->pin_count = 0;
		3511	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3512	bool was_interruptible;
		3513
		3514	was_interruptible = dev_priv->mm.interruptible;
		3515	dev_priv->mm.interruptible = false;
		3516
		3517	WARN_ON(i915_gem_object_unbind(obj));
		3518
		3519	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3520	}
2332	Serge	3521
3031	serge	3522	obj->pages_pin_count = 0;
		3523	i915_gem_object_put_pages(obj);
		3524	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3525
3243	Serge	3526	BUG_ON(obj->pages);
2332	Serge	3527
3031	serge	3528	// if (obj->base.import_attach)
		3529	// drm_prime_gem_destroy(&obj->base, NULL);
		3530
2344	Serge	3531	drm_gem_object_release(&obj->base);
		3532	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3533
2344	Serge	3534	kfree(obj->bit_17);
		3535	kfree(obj);
		3536	}
2332	Serge	3537
3031	serge	3538	#if 0
		3539	int
		3540	i915_gem_idle(struct drm_device *dev)
2344	Serge	3541	{
3031	serge	3542	drm_i915_private_t *dev_priv = dev->dev_private;
		3543	int ret;
2332	Serge	3544
3031	serge	3545	mutex_lock(&dev->struct_mutex);
2332	Serge	3546
3031	serge	3547	if (dev_priv->mm.suspended) {
		3548	mutex_unlock(&dev->struct_mutex);
		3549	return 0;
		3550	}
2332	Serge	3551
3031	serge	3552	ret = i915_gpu_idle(dev);
		3553	if (ret) {
		3554	mutex_unlock(&dev->struct_mutex);
		3555	return ret;
		3556	}
		3557	i915_gem_retire_requests(dev);
		3558
		3559	i915_gem_reset_fences(dev);
		3560
		3561	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3562	* We need to replace this with a semaphore, or something.
		3563	* And not confound mm.suspended!
		3564	*/
		3565	dev_priv->mm.suspended = 1;
		3566	del_timer_sync(&dev_priv->hangcheck_timer);
		3567
		3568	i915_kernel_lost_context(dev);
		3569	i915_gem_cleanup_ringbuffer(dev);
		3570
		3571	mutex_unlock(&dev->struct_mutex);
		3572
		3573	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	3574	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	3575
		3576	return 0;
2344	Serge	3577	}
3031	serge	3578	#endif
2332	Serge	3579
3031	serge	3580	void i915_gem_l3_remap(struct drm_device *dev)
		3581	{
		3582	drm_i915_private_t *dev_priv = dev->dev_private;
		3583	u32 misccpctl;
		3584	int i;
2332	Serge	3585
3031	serge	3586	if (!IS_IVYBRIDGE(dev))
		3587	return;
2332	Serge	3588
3243	Serge	3589	if (!dev_priv->l3_parity.remap_info)
3031	serge	3590	return;
2332	Serge	3591
3031	serge	3592	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3593	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3594	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3595
3031	serge	3596	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3597	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3598	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3599	DRM_DEBUG("0x%x was already programmed to %x\n",
		3600	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3601	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3602	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3603	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3604	}
2332	Serge	3605
3031	serge	3606	/* Make sure all the writes land before disabling dop clock gating */
		3607	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3608
3031	serge	3609	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3610	}
2332	Serge	3611
3031	serge	3612	void i915_gem_init_swizzling(struct drm_device *dev)
		3613	{
		3614	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3615
3031	serge	3616	if (INTEL_INFO(dev)->gen < 5 \|\|
		3617	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3618	return;
2332	Serge	3619
3031	serge	3620	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3621	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3622
3031	serge	3623	if (IS_GEN5(dev))
		3624	return;
2344	Serge	3625
3031	serge	3626	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3627	if (IS_GEN6(dev))
		3628	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3629	else
		3630	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3631	}
		3632
		3633	static bool
		3634	intel_enable_blt(struct drm_device *dev)
		3635	{
		3636	if (!HAS_BLT(dev))
		3637	return false;
		3638
		3639	/* The blitter was dysfunctional on early prototypes */
		3640	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3641	DRM_INFO("BLT not supported on this pre-production hardware;"
		3642	" graphics performance will be degraded.\n");
		3643	return false;
		3644	}
		3645
		3646	return true;
		3647	}
		3648
2332	Serge	3649	int
3031	serge	3650	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3651	{
		3652	drm_i915_private_t *dev_priv = dev->dev_private;
		3653	int ret;
2351	Serge	3654
3243	Serge	3655	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
3031	serge	3656	return -EIO;
		3657
		3658	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3659	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3660
		3661	i915_gem_l3_remap(dev);
		3662
		3663	i915_gem_init_swizzling(dev);
		3664
2332	Serge	3665	ret = intel_init_render_ring_buffer(dev);
		3666	if (ret)
		3667	return ret;
		3668
		3669	if (HAS_BSD(dev)) {
		3670	ret = intel_init_bsd_ring_buffer(dev);
		3671	if (ret)
		3672	goto cleanup_render_ring;
		3673	}
		3674
3031	serge	3675	if (intel_enable_blt(dev)) {
2332	Serge	3676	ret = intel_init_blt_ring_buffer(dev);
		3677	if (ret)
		3678	goto cleanup_bsd_ring;
		3679	}
		3680
		3681	dev_priv->next_seqno = 1;
2351	Serge	3682
3031	serge	3683	/*
		3684	* XXX: There was some w/a described somewhere suggesting loading
		3685	* contexts before PPGTT.
		3686	*/
		3687	i915_gem_context_init(dev);
		3688	i915_gem_init_ppgtt(dev);
		3689
2332	Serge	3690	return 0;
		3691
		3692	cleanup_bsd_ring:
		3693	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3694	cleanup_render_ring:
		3695	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3696	return ret;
		3697	}
		3698
3031	serge	3699	static bool
		3700	intel_enable_ppgtt(struct drm_device *dev)
		3701	{
		3702	if (i915_enable_ppgtt >= 0)
		3703	return i915_enable_ppgtt;
		3704
		3705	#ifdef CONFIG_INTEL_IOMMU
		3706	/* Disable ppgtt on SNB if VT-d is on. */
		3707	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3708	return false;
		3709	#endif
		3710
		3711	return true;
		3712	}
		3713
		3714	#define LFB_SIZE 0xC00000
		3715
		3716	int i915_gem_init(struct drm_device *dev)
		3717	{
		3718	struct drm_i915_private *dev_priv = dev->dev_private;
		3719	unsigned long gtt_size, mappable_size;
		3720	int ret;
		3721
		3722	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3723	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3724
		3725	mutex_lock(&dev->struct_mutex);
		3726	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3727	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3728	* aperture accordingly when using aliasing ppgtt. */
		3729	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3730
		3731	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3732
		3733	ret = i915_gem_init_aliasing_ppgtt(dev);
		3734	if (ret) {
		3735	mutex_unlock(&dev->struct_mutex);
		3736	return ret;
		3737	}
		3738	} else {
		3739	/* Let GEM Manage all of the aperture.
		3740	*
		3741	* However, leave one page at the end still bound to the scratch
		3742	* page. There are a number of places where the hardware
		3743	* apparently prefetches past the end of the object, and we've
		3744	* seen multiple hangs with the GPU head pointer stuck in a
		3745	* batchbuffer bound at the last page of the aperture. One page
		3746	* should be enough to keep any prefetching inside of the
		3747	* aperture.
		3748	*/
		3749	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3750	}
		3751
		3752	ret = i915_gem_init_hw(dev);
		3753	mutex_unlock(&dev->struct_mutex);
		3754	if (ret) {
		3755	i915_gem_cleanup_aliasing_ppgtt(dev);
		3756	return ret;
		3757	}
		3758
		3759	return 0;
		3760	}
		3761
2332	Serge	3762	void
		3763	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3764	{
		3765	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3766	struct intel_ring_buffer *ring;
2332	Serge	3767	int i;
		3768
3031	serge	3769	for_each_ring(ring, dev_priv, i)
		3770	intel_cleanup_ring_buffer(ring);
2332	Serge	3771	}
		3772
3031	serge	3773	#if 0
		3774
2332	Serge	3775	int
		3776	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3777	struct drm_file *file_priv)
		3778	{
		3779	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3780	int ret;
2332	Serge	3781
		3782	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3783	return 0;
		3784
		3785	if (atomic_read(&dev_priv->mm.wedged)) {
		3786	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3787	atomic_set(&dev_priv->mm.wedged, 0);
		3788	}
		3789
		3790	mutex_lock(&dev->struct_mutex);
		3791	dev_priv->mm.suspended = 0;
		3792
3031	serge	3793	ret = i915_gem_init_hw(dev);
2332	Serge	3794	if (ret != 0) {
		3795	mutex_unlock(&dev->struct_mutex);
		3796	return ret;
		3797	}
		3798
		3799	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3800	mutex_unlock(&dev->struct_mutex);
		3801
		3802	ret = drm_irq_install(dev);
		3803	if (ret)
		3804	goto cleanup_ringbuffer;
		3805
		3806	return 0;
		3807
		3808	cleanup_ringbuffer:
		3809	mutex_lock(&dev->struct_mutex);
		3810	i915_gem_cleanup_ringbuffer(dev);
		3811	dev_priv->mm.suspended = 1;
		3812	mutex_unlock(&dev->struct_mutex);
		3813
		3814	return ret;
		3815	}
		3816
		3817	int
		3818	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3819	struct drm_file *file_priv)
		3820	{
		3821	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3822	return 0;
		3823
		3824	drm_irq_uninstall(dev);
		3825	return i915_gem_idle(dev);
		3826	}
		3827
		3828	void
		3829	i915_gem_lastclose(struct drm_device *dev)
		3830	{
		3831	int ret;
		3832
		3833	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3834	return;
		3835
		3836	ret = i915_gem_idle(dev);
		3837	if (ret)
		3838	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3839	}
		3840	#endif
		3841
		3842	static void
2326	Serge	3843	init_ring_lists(struct intel_ring_buffer *ring)
		3844	{
		3845	INIT_LIST_HEAD(&ring->active_list);
		3846	INIT_LIST_HEAD(&ring->request_list);
		3847	}
		3848
		3849	void
		3850	i915_gem_load(struct drm_device *dev)
		3851	{
		3852	int i;
		3853	drm_i915_private_t *dev_priv = dev->dev_private;
		3854
		3855	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3856	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3857	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3858	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3859	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3860	for (i = 0; i < I915_NUM_RINGS; i++)
		3861	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3862	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3863	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3864	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3865	i915_gem_retire_work_handler);
2326	Serge	3866
		3867	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3868	if (IS_GEN3(dev)) {
3031	serge	3869	I915_WRITE(MI_ARB_STATE,
		3870	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3871	}
		3872
		3873	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3874
		3875	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3876	dev_priv->num_fence_regs = 16;
		3877	else
		3878	dev_priv->num_fence_regs = 8;
		3879
		3880	/* Initialize fence registers to zero */
3031	serge	3881	i915_gem_reset_fences(dev);
2326	Serge	3882
		3883	i915_gem_detect_bit_6_swizzle(dev);
		3884
		3885	dev_priv->mm.interruptible = true;
		3886
		3887	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3888	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3889	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3890	}
		3891
		3892

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3266