WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
		33	//#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
2344	Serge	40	#undef mb
		41	#undef rmb
		42	#undef wmb
		43	#define mb() asm volatile("mfence")
		44	#define rmb() asm volatile ("lfence")
		45	#define wmb() asm volatile ("sfence")
		46
		47	static inline void clflush(volatile void *__p)
		48	{
		49	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		50	}
		51
2332	Serge	52	#define MAX_ERRNO 4095
		53
		54	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		55
		56	static inline long IS_ERR(const void *ptr)
		57	{
		58	return IS_ERR_VALUE((unsigned long)ptr);
		59	}
		60
		61	static inline void *ERR_PTR(long error)
		62	{
		63	return (void *) error;
		64	}
		65
		66	static inline long PTR_ERR(const void *ptr)
		67	{
		68	return (long) ptr;
		69	}
		70
2344	Serge	71	void
		72	drm_gem_object_free(struct kref *kref)
		73	{
		74	struct drm_gem_object obj = (struct drm_gem_object ) kref;
		75	struct drm_device *dev = obj->dev;
2332	Serge	76
2344	Serge	77	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		78
		79	i915_gem_free_object(obj);
		80	}
		81
2332	Serge	82	/**
		83	* Initialize an already allocated GEM object of the specified size with
		84	* shmfs backing store.
		85	*/
		86	int drm_gem_object_init(struct drm_device *dev,
		87	struct drm_gem_object *obj, size_t size)
		88	{
		89	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		90
		91	obj->dev = dev;
2344	Serge	92	kref_init(&obj->refcount);
2332	Serge	93	atomic_set(&obj->handle_count, 0);
		94	obj->size = size;
		95
		96	return 0;
		97	}
		98
2344	Serge	99	void
		100	drm_gem_object_release(struct drm_gem_object *obj)
		101	{ }
2332	Serge	102
		103
2326	Serge	104	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		105	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		106	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		107	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		108
2332	Serge	109	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		110	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		111	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		112	unsigned alignment,
3031	serge	113	bool map_and_fenceable,
		114	bool nonblocking);
2332	Serge	115	static int i915_gem_phys_pwrite(struct drm_device *dev,
		116	struct drm_i915_gem_object *obj,
		117	struct drm_i915_gem_pwrite *args,
		118	struct drm_file *file);
2326	Serge	119
3031	serge	120	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		121	struct drm_i915_gem_object *obj);
		122	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		123	struct drm_i915_fence_reg *fence,
		124	bool enable);
2332	Serge	125
3031	serge	126	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		127	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		128	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		129
		130	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		131	{
		132	if (obj->tiling_mode)
		133	i915_gem_release_mmap(obj);
		134
		135	/* As we do not have an associated fence register, we will force
		136	* a tiling change if we ever need to acquire one.
		137	*/
		138	obj->fence_dirty = false;
		139	obj->fence_reg = I915_FENCE_REG_NONE;
		140	}
		141
2332	Serge	142	/* some bookkeeping */
		143	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		144	size_t size)
		145	{
		146	dev_priv->mm.object_count++;
		147	dev_priv->mm.object_memory += size;
		148	}
		149
		150	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		151	size_t size)
		152	{
		153	dev_priv->mm.object_count--;
		154	dev_priv->mm.object_memory -= size;
		155	}
		156
		157	#if 0
		158
		159	static int
		160	i915_gem_wait_for_error(struct drm_device *dev)
		161	{
		162	struct drm_i915_private *dev_priv = dev->dev_private;
		163	struct completion *x = &dev_priv->error_completion;
		164	unsigned long flags;
		165	int ret;
		166
		167	if (!atomic_read(&dev_priv->mm.wedged))
		168	return 0;
		169
3031	serge	170	/*
		171	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		172	* userspace. If it takes that long something really bad is going on and
		173	* we should simply try to bail out and fail as gracefully as possible.
		174	*/
		175	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		176	if (ret == 0) {
		177	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		178	return -EIO;
		179	} else if (ret < 0) {
2332	Serge	180	return ret;
3031	serge	181	}
2332	Serge	182
		183	if (atomic_read(&dev_priv->mm.wedged)) {
		184	/* GPU is hung, bump the completion count to account for
		185	* the token we just consumed so that we never hit zero and
		186	* end up waiting upon a subsequent completion event that
		187	* will never happen.
		188	*/
		189	spin_lock_irqsave(&x->wait.lock, flags);
		190	x->done++;
		191	spin_unlock_irqrestore(&x->wait.lock, flags);
		192	}
		193	return 0;
		194	}
		195
		196	int i915_mutex_lock_interruptible(struct drm_device *dev)
		197	{
		198	int ret;
		199
		200	ret = i915_gem_wait_for_error(dev);
		201	if (ret)
		202	return ret;
		203
		204	ret = mutex_lock_interruptible(&dev->struct_mutex);
		205	if (ret)
		206	return ret;
		207
		208	WARN_ON(i915_verify_lists(dev));
		209	return 0;
		210	}
2352	Serge	211	#endif
2332	Serge	212
		213	static inline bool
		214	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		215	{
3031	serge	216	return obj->gtt_space && !obj->active;
2332	Serge	217	}
		218
		219
		220	#if 0
		221
		222	int
		223	i915_gem_init_ioctl(struct drm_device dev, void data,
		224	struct drm_file *file)
		225	{
		226	struct drm_i915_gem_init *args = data;
		227
3031	serge	228	if (drm_core_check_feature(dev, DRIVER_MODESET))
		229	return -ENODEV;
		230
2332	Serge	231	if (args->gtt_start >= args->gtt_end \|\|
		232	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		233	return -EINVAL;
		234
3031	serge	235	/* GEM with user mode setting was never supported on ilk and later. */
		236	if (INTEL_INFO(dev)->gen >= 5)
		237	return -ENODEV;
		238
2332	Serge	239	mutex_lock(&dev->struct_mutex);
3031	serge	240	i915_gem_init_global_gtt(dev, args->gtt_start,
		241	args->gtt_end, args->gtt_end);
2332	Serge	242	mutex_unlock(&dev->struct_mutex);
		243
		244	return 0;
		245	}
2351	Serge	246	#endif
2332	Serge	247
		248	int
		249	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		250	struct drm_file *file)
		251	{
		252	struct drm_i915_private *dev_priv = dev->dev_private;
		253	struct drm_i915_gem_get_aperture *args = data;
		254	struct drm_i915_gem_object *obj;
		255	size_t pinned;
		256
		257	pinned = 0;
		258	mutex_lock(&dev->struct_mutex);
3031	serge	259	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		260	if (obj->pin_count)
2332	Serge	261	pinned += obj->gtt_space->size;
		262	mutex_unlock(&dev->struct_mutex);
		263
		264	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	265	args->aper_available_size = args->aper_size - pinned;
2332	Serge	266
		267	return 0;
		268	}
		269
2351	Serge	270	#if 0
3031	serge	271	static int
		272	i915_gem_create(struct drm_file *file,
2332	Serge	273	struct drm_device *dev,
		274	uint64_t size,
		275	uint32_t *handle_p)
		276	{
		277	struct drm_i915_gem_object *obj;
		278	int ret;
		279	u32 handle;
		280
		281	size = roundup(size, PAGE_SIZE);
2342	Serge	282	if (size == 0)
		283	return -EINVAL;
2332	Serge	284
		285	/* Allocate the new object */
		286	obj = i915_gem_alloc_object(dev, size);
		287	if (obj == NULL)
		288	return -ENOMEM;
		289
		290	ret = drm_gem_handle_create(file, &obj->base, &handle);
		291	if (ret) {
		292	drm_gem_object_release(&obj->base);
		293	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		294	kfree(obj);
		295	return ret;
		296	}
		297
		298	/* drop reference from allocate - handle holds it now */
		299	drm_gem_object_unreference(&obj->base);
2351	Serge	300	trace_i915_gem_object_create(obj);
2332	Serge	301
		302	*handle_p = handle;
		303	return 0;
		304	}
		305
		306	int
		307	i915_gem_dumb_create(struct drm_file *file,
		308	struct drm_device *dev,
		309	struct drm_mode_create_dumb *args)
		310	{
		311	/* have to work out size/pitch and return them */
		312	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		313	args->size = args->pitch * args->height;
		314	return i915_gem_create(file, dev,
		315	args->size, &args->handle);
		316	}
		317
		318	int i915_gem_dumb_destroy(struct drm_file *file,
		319	struct drm_device *dev,
		320	uint32_t handle)
		321	{
		322	return drm_gem_handle_delete(file, handle);
		323	}
		324
2326	Serge	325	/**
2332	Serge	326	* Creates a new mm object and returns a handle to it.
		327	*/
		328	int
		329	i915_gem_create_ioctl(struct drm_device dev, void data,
		330	struct drm_file *file)
		331	{
		332	struct drm_i915_gem_create *args = data;
3031	serge	333
2332	Serge	334	return i915_gem_create(file, dev,
		335	args->size, &args->handle);
		336	}
		337
		338	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		339	{
		340	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		341
		342	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		343	obj->tiling_mode != I915_TILING_NONE;
		344	}
		345
3031	serge	346	static inline int
		347	__copy_to_user_swizzled(char __user *cpu_vaddr,
		348	const char *gpu_vaddr, int gpu_offset,
2332	Serge	349	int length)
		350	{
3031	serge	351	int ret, cpu_offset = 0;
2332	Serge	352
3031	serge	353	while (length > 0) {
		354	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		355	int this_length = min(cacheline_end - gpu_offset, length);
		356	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	357
3031	serge	358	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		359	gpu_vaddr + swizzled_gpu_offset,
		360	this_length);
		361	if (ret)
		362	return ret + length;
2332	Serge	363
3031	serge	364	cpu_offset += this_length;
		365	gpu_offset += this_length;
		366	length -= this_length;
		367	}
		368
		369	return 0;
2332	Serge	370	}
		371
3031	serge	372	static inline int
		373	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		374	const char __user *cpu_vaddr,
		375	int length)
2332	Serge	376	{
3031	serge	377	int ret, cpu_offset = 0;
2332	Serge	378
		379	while (length > 0) {
		380	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		381	int this_length = min(cacheline_end - gpu_offset, length);
		382	int swizzled_gpu_offset = gpu_offset ^ 64;
		383
3031	serge	384	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	385	cpu_vaddr + cpu_offset,
		386	this_length);
3031	serge	387	if (ret)
		388	return ret + length;
		389
2332	Serge	390	cpu_offset += this_length;
		391	gpu_offset += this_length;
		392	length -= this_length;
		393	}
		394
3031	serge	395	return 0;
2332	Serge	396	}
		397
3031	serge	398	/* Per-page copy function for the shmem pread fastpath.
		399	* Flushes invalid cachelines before reading the target if
		400	* needs_clflush is set. */
2332	Serge	401	static int
3031	serge	402	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		403	char __user *user_data,
		404	bool page_do_bit17_swizzling, bool needs_clflush)
		405	{
		406	char *vaddr;
		407	int ret;
		408
		409	if (unlikely(page_do_bit17_swizzling))
		410	return -EINVAL;
		411
		412	vaddr = kmap_atomic(page);
		413	if (needs_clflush)
		414	drm_clflush_virt_range(vaddr + shmem_page_offset,
		415	page_length);
		416	ret = __copy_to_user_inatomic(user_data,
		417	vaddr + shmem_page_offset,
		418	page_length);
		419	kunmap_atomic(vaddr);
		420
		421	return ret ? -EFAULT : 0;
		422	}
		423
		424	static void
		425	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		426	bool swizzled)
		427	{
		428	if (unlikely(swizzled)) {
		429	unsigned long start = (unsigned long) addr;
		430	unsigned long end = (unsigned long) addr + length;
		431
		432	/* For swizzling simply ensure that we always flush both
		433	* channels. Lame, but simple and it works. Swizzled
		434	* pwrite/pread is far from a hotpath - current userspace
		435	* doesn't use it at all. */
		436	start = round_down(start, 128);
		437	end = round_up(end, 128);
		438
		439	drm_clflush_virt_range((void *)start, end - start);
		440	} else {
		441	drm_clflush_virt_range(addr, length);
		442	}
		443
		444	}
		445
		446	/* Only difference to the fast-path function is that this can handle bit17
		447	* and uses non-atomic copy and kmap functions. */
		448	static int
		449	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		450	char __user *user_data,
		451	bool page_do_bit17_swizzling, bool needs_clflush)
		452	{
		453	char *vaddr;
		454	int ret;
		455
		456	vaddr = kmap(page);
		457	if (needs_clflush)
		458	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		459	page_length,
		460	page_do_bit17_swizzling);
		461
		462	if (page_do_bit17_swizzling)
		463	ret = __copy_to_user_swizzled(user_data,
		464	vaddr, shmem_page_offset,
		465	page_length);
		466	else
		467	ret = __copy_to_user(user_data,
		468	vaddr + shmem_page_offset,
		469	page_length);
		470	kunmap(page);
		471
		472	return ret ? - EFAULT : 0;
		473	}
		474
		475	static int
		476	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	477	struct drm_i915_gem_object *obj,
		478	struct drm_i915_gem_pread *args,
		479	struct drm_file *file)
		480	{
3031	serge	481	char __user *user_data;
2332	Serge	482	ssize_t remain;
		483	loff_t offset;
3031	serge	484	int shmem_page_offset, page_length, ret = 0;
		485	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		486	int hit_slowpath = 0;
		487	int prefaulted = 0;
		488	int needs_clflush = 0;
		489	struct scatterlist *sg;
		490	int i;
2332	Serge	491
		492	user_data = (char __user *) (uintptr_t) args->data_ptr;
		493	remain = args->size;
		494
3031	serge	495	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		496
		497	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		498	/* If we're not in the cpu read domain, set ourself into the gtt
		499	* read domain and manually flush cachelines (if required). This
		500	* optimizes for the case when the gpu will dirty the data
		501	* anyway again before the next pread happens. */
		502	if (obj->cache_level == I915_CACHE_NONE)
		503	needs_clflush = 1;
		504	if (obj->gtt_space) {
		505	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		506	if (ret)
		507	return ret;
		508	}
		509	}
		510
		511	ret = i915_gem_object_get_pages(obj);
		512	if (ret)
		513	return ret;
		514
		515	i915_gem_object_pin_pages(obj);
		516
2332	Serge	517	offset = args->offset;
		518
3031	serge	519	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	520	struct page *page;
		521
3031	serge	522	if (i < offset >> PAGE_SHIFT)
		523	continue;
		524
		525	if (remain <= 0)
		526	break;
		527
2332	Serge	528	/* Operation in this page
		529	*
3031	serge	530	* shmem_page_offset = offset within page in shmem file
2332	Serge	531	* page_length = bytes to copy for this page
		532	*/
3031	serge	533	shmem_page_offset = offset_in_page(offset);
2332	Serge	534	page_length = remain;
3031	serge	535	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		536	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	537
3031	serge	538	page = sg_page(sg);
		539	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		540	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	541
3031	serge	542	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		543	user_data, page_do_bit17_swizzling,
		544	needs_clflush);
		545	if (ret == 0)
		546	goto next_page;
2332	Serge	547
3031	serge	548	hit_slowpath = 1;
		549	mutex_unlock(&dev->struct_mutex);
		550
		551	if (!prefaulted) {
		552	ret = fault_in_multipages_writeable(user_data, remain);
		553	/* Userspace is tricking us, but we've already clobbered
		554	* its pages with the prefault and promised to write the
		555	* data up to the first fault. Hence ignore any errors
		556	* and just continue. */
		557	(void)ret;
		558	prefaulted = 1;
		559	}
		560
		561	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		562	user_data, page_do_bit17_swizzling,
		563	needs_clflush);
		564
		565	mutex_lock(&dev->struct_mutex);
		566
		567	next_page:
2332	Serge	568	mark_page_accessed(page);
3031	serge	569
2332	Serge	570	if (ret)
3031	serge	571	goto out;
2332	Serge	572
		573	remain -= page_length;
		574	user_data += page_length;
		575	offset += page_length;
		576	}
		577
3031	serge	578	out:
		579	i915_gem_object_unpin_pages(obj);
		580
		581	if (hit_slowpath) {
		582	/* Fixup: Kill any reinstated backing storage pages */
		583	if (obj->madv == __I915_MADV_PURGED)
		584	i915_gem_object_truncate(obj);
		585	}
		586
		587	return ret;
2332	Serge	588	}
		589
		590	/**
3031	serge	591	* Reads data from the object referenced by handle.
		592	*
		593	* On error, the contents of *data are undefined.
2332	Serge	594	*/
3031	serge	595	int
		596	i915_gem_pread_ioctl(struct drm_device dev, void data,
		597	struct drm_file *file)
		598	{
		599	struct drm_i915_gem_pread *args = data;
		600	struct drm_i915_gem_object *obj;
		601	int ret = 0;
		602
		603	if (args->size == 0)
		604	return 0;
		605
		606	if (!access_ok(VERIFY_WRITE,
		607	(char __user *)(uintptr_t)args->data_ptr,
		608	args->size))
		609	return -EFAULT;
		610
		611	ret = i915_mutex_lock_interruptible(dev);
		612	if (ret)
		613	return ret;
		614
		615	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		616	if (&obj->base == NULL) {
		617	ret = -ENOENT;
		618	goto unlock;
		619	}
		620
		621	/* Bounds check source. */
		622	if (args->offset > obj->base.size \|\|
		623	args->size > obj->base.size - args->offset) {
		624	ret = -EINVAL;
		625	goto out;
		626	}
		627
		628	/* prime objects have no backing filp to GEM pread/pwrite
		629	* pages from.
		630	*/
		631	if (!obj->base.filp) {
		632	ret = -EINVAL;
		633	goto out;
		634	}
		635
		636	trace_i915_gem_object_pread(obj, args->offset, args->size);
		637
		638	ret = i915_gem_shmem_pread(dev, obj, args, file);
		639
		640	out:
		641	drm_gem_object_unreference(&obj->base);
		642	unlock:
		643	mutex_unlock(&dev->struct_mutex);
		644	return ret;
		645	}
		646
		647	/* This is the fast write path which cannot handle
		648	* page faults in the source data
		649	*/
		650
		651	static inline int
		652	fast_user_write(struct io_mapping *mapping,
		653	loff_t page_base, int page_offset,
		654	char __user *user_data,
		655	int length)
		656	{
		657	void __iomem *vaddr_atomic;
		658	void *vaddr;
		659	unsigned long unwritten;
		660
		661	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		662	/* We can use the cpu mem copy function because this is X86. */
		663	vaddr = (void __force*)vaddr_atomic + page_offset;
		664	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		665	user_data, length);
		666	io_mapping_unmap_atomic(vaddr_atomic);
		667	return unwritten;
		668	}
		669
		670	/**
		671	* This is the fast pwrite path, where we copy the data directly from the
		672	* user into the GTT, uncached.
		673	*/
2332	Serge	674	static int
3031	serge	675	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		676	struct drm_i915_gem_object *obj,
		677	struct drm_i915_gem_pwrite *args,
		678	struct drm_file *file)
2332	Serge	679	{
3031	serge	680	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	681	ssize_t remain;
3031	serge	682	loff_t offset, page_base;
		683	char __user *user_data;
		684	int page_offset, page_length, ret;
2332	Serge	685
3031	serge	686	ret = i915_gem_object_pin(obj, 0, true, true);
		687	if (ret)
		688	goto out;
		689
		690	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		691	if (ret)
		692	goto out_unpin;
		693
		694	ret = i915_gem_object_put_fence(obj);
		695	if (ret)
		696	goto out_unpin;
		697
		698	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	699	remain = args->size;
		700
3031	serge	701	offset = obj->gtt_offset + args->offset;
2332	Serge	702
3031	serge	703	while (remain > 0) {
		704	/* Operation in this page
		705	*
		706	* page_base = page offset within aperture
		707	* page_offset = offset within page
		708	* page_length = bytes to copy for this page
		709	*/
		710	page_base = offset & PAGE_MASK;
		711	page_offset = offset_in_page(offset);
		712	page_length = remain;
		713	if ((page_offset + remain) > PAGE_SIZE)
		714	page_length = PAGE_SIZE - page_offset;
2332	Serge	715
3031	serge	716	/* If we get a fault while copying data, then (presumably) our
		717	* source page isn't available. Return the error and we'll
		718	* retry in the slow path.
		719	*/
		720	if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
		721	page_offset, user_data, page_length)) {
		722	ret = -EFAULT;
		723	goto out_unpin;
		724	}
		725
		726	remain -= page_length;
		727	user_data += page_length;
		728	offset += page_length;
2332	Serge	729	}
		730
3031	serge	731	out_unpin:
		732	i915_gem_object_unpin(obj);
		733	out:
		734	return ret;
		735	}
		736
		737	/* Per-page copy function for the shmem pwrite fastpath.
		738	* Flushes invalid cachelines before writing to the target if
		739	* needs_clflush_before is set and flushes out any written cachelines after
		740	* writing if needs_clflush is set. */
		741	static int
		742	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		743	char __user *user_data,
		744	bool page_do_bit17_swizzling,
		745	bool needs_clflush_before,
		746	bool needs_clflush_after)
		747	{
		748	char *vaddr;
		749	int ret;
		750
		751	if (unlikely(page_do_bit17_swizzling))
		752	return -EINVAL;
		753
		754	vaddr = kmap_atomic(page);
		755	if (needs_clflush_before)
		756	drm_clflush_virt_range(vaddr + shmem_page_offset,
		757	page_length);
		758	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
		759	user_data,
		760	page_length);
		761	if (needs_clflush_after)
		762	drm_clflush_virt_range(vaddr + shmem_page_offset,
		763	page_length);
		764	kunmap_atomic(vaddr);
		765
		766	return ret ? -EFAULT : 0;
		767	}
		768
		769	/* Only difference to the fast-path function is that this can handle bit17
		770	* and uses non-atomic copy and kmap functions. */
		771	static int
		772	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		773	char __user *user_data,
		774	bool page_do_bit17_swizzling,
		775	bool needs_clflush_before,
		776	bool needs_clflush_after)
		777	{
		778	char *vaddr;
		779	int ret;
		780
		781	vaddr = kmap(page);
		782	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		783	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		784	page_length,
		785	page_do_bit17_swizzling);
		786	if (page_do_bit17_swizzling)
		787	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		788	user_data,
		789	page_length);
		790	else
		791	ret = __copy_from_user(vaddr + shmem_page_offset,
		792	user_data,
		793	page_length);
		794	if (needs_clflush_after)
		795	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		796	page_length,
		797	page_do_bit17_swizzling);
		798	kunmap(page);
		799
		800	return ret ? -EFAULT : 0;
		801	}
		802
		803	static int
		804	i915_gem_shmem_pwrite(struct drm_device *dev,
		805	struct drm_i915_gem_object *obj,
		806	struct drm_i915_gem_pwrite *args,
		807	struct drm_file *file)
		808	{
		809	ssize_t remain;
		810	loff_t offset;
		811	char __user *user_data;
		812	int shmem_page_offset, page_length, ret = 0;
		813	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		814	int hit_slowpath = 0;
		815	int needs_clflush_after = 0;
		816	int needs_clflush_before = 0;
		817	int i;
		818	struct scatterlist *sg;
		819
		820	user_data = (char __user *) (uintptr_t) args->data_ptr;
		821	remain = args->size;
		822
		823	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		824
		825	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		826	/* If we're not in the cpu write domain, set ourself into the gtt
		827	* write domain and manually flush cachelines (if required). This
		828	* optimizes for the case when the gpu will use the data
		829	* right away and we therefore have to clflush anyway. */
		830	if (obj->cache_level == I915_CACHE_NONE)
		831	needs_clflush_after = 1;
		832	if (obj->gtt_space) {
		833	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		834	if (ret)
		835	return ret;
		836	}
		837	}
		838	/* Same trick applies for invalidate partially written cachelines before
		839	* writing. */
		840	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		841	&& obj->cache_level == I915_CACHE_NONE)
		842	needs_clflush_before = 1;
		843
		844	ret = i915_gem_object_get_pages(obj);
2332	Serge	845	if (ret)
3031	serge	846	return ret;
2332	Serge	847
3031	serge	848	i915_gem_object_pin_pages(obj);
2332	Serge	849
		850	offset = args->offset;
3031	serge	851	obj->dirty = 1;
2332	Serge	852
3031	serge	853	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	854	struct page *page;
3031	serge	855	int partial_cacheline_write;
2332	Serge	856
3031	serge	857	if (i < offset >> PAGE_SHIFT)
		858	continue;
		859
		860	if (remain <= 0)
		861	break;
		862
2332	Serge	863	/* Operation in this page
		864	*
		865	* shmem_page_offset = offset within page in shmem file
		866	* page_length = bytes to copy for this page
		867	*/
		868	shmem_page_offset = offset_in_page(offset);
		869
		870	page_length = remain;
		871	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		872	page_length = PAGE_SIZE - shmem_page_offset;
		873
3031	serge	874	/* If we don't overwrite a cacheline completely we need to be
		875	* careful to have up-to-date data by first clflushing. Don't
		876	* overcomplicate things and flush the entire patch. */
		877	partial_cacheline_write = needs_clflush_before &&
		878	((shmem_page_offset \| page_length)
		879	& (boot_cpu_data.x86_clflush_size - 1));
2332	Serge	880
3031	serge	881	page = sg_page(sg);
		882	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		883	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	884
3031	serge	885	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		886	user_data, page_do_bit17_swizzling,
		887	partial_cacheline_write,
		888	needs_clflush_after);
		889	if (ret == 0)
		890	goto next_page;
		891
		892	hit_slowpath = 1;
		893	mutex_unlock(&dev->struct_mutex);
		894	ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		895	user_data, page_do_bit17_swizzling,
		896	partial_cacheline_write,
		897	needs_clflush_after);
		898
		899	mutex_lock(&dev->struct_mutex);
		900
		901	next_page:
		902	set_page_dirty(page);
2332	Serge	903	mark_page_accessed(page);
		904
3031	serge	905	if (ret)
		906	goto out;
		907
2332	Serge	908	remain -= page_length;
3031	serge	909	user_data += page_length;
2332	Serge	910	offset += page_length;
		911	}
		912
		913	out:
3031	serge	914	i915_gem_object_unpin_pages(obj);
		915
		916	if (hit_slowpath) {
		917	/* Fixup: Kill any reinstated backing storage pages */
		918	if (obj->madv == __I915_MADV_PURGED)
		919	i915_gem_object_truncate(obj);
		920	/* and flush dirty cachelines in case the object isn't in the cpu write
		921	* domain anymore. */
		922	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		923	i915_gem_clflush_object(obj);
		924	intel_gtt_chipset_flush();
		925	}
2332	Serge	926	}
		927
3031	serge	928	if (needs_clflush_after)
		929	intel_gtt_chipset_flush();
		930
2332	Serge	931	return ret;
		932	}
3031	serge	933
		934	/**
		935	* Writes data to the object referenced by handle.
		936	*
		937	* On error, the contents of the buffer that were to be modified are undefined.
		938	*/
		939	int
		940	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		941	struct drm_file *file)
		942	{
		943	struct drm_i915_gem_pwrite *args = data;
		944	struct drm_i915_gem_object *obj;
		945	int ret;
		946
		947	if (args->size == 0)
		948	return 0;
		949
		950	if (!access_ok(VERIFY_READ,
		951	(char __user *)(uintptr_t)args->data_ptr,
		952	args->size))
		953	return -EFAULT;
		954
		955	ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
		956	args->size);
		957	if (ret)
		958	return -EFAULT;
		959
		960	ret = i915_mutex_lock_interruptible(dev);
		961	if (ret)
		962	return ret;
		963
		964	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		965	if (&obj->base == NULL) {
		966	ret = -ENOENT;
		967	goto unlock;
		968	}
		969
		970	/* Bounds check destination. */
		971	if (args->offset > obj->base.size \|\|
		972	args->size > obj->base.size - args->offset) {
		973	ret = -EINVAL;
		974	goto out;
		975	}
		976
		977	/* prime objects have no backing filp to GEM pread/pwrite
		978	* pages from.
		979	*/
		980	if (!obj->base.filp) {
		981	ret = -EINVAL;
		982	goto out;
		983	}
		984
		985	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		986
		987	ret = -EFAULT;
		988	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		989	* it would end up going through the fenced access, and we'll get
		990	* different detiling behavior between reading and writing.
		991	* pread/pwrite currently are reading and writing from the CPU
		992	* perspective, requiring manual detiling by the client.
		993	*/
		994	if (obj->phys_obj) {
		995	ret = i915_gem_phys_pwrite(dev, obj, args, file);
		996	goto out;
		997	}
		998
		999	if (obj->cache_level == I915_CACHE_NONE &&
		1000	obj->tiling_mode == I915_TILING_NONE &&
		1001	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		1002	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		1003	/* Note that the gtt paths might fail with non-page-backed user
		1004	* pointers (e.g. gtt mappings when moving data between
		1005	* textures). Fallback to the shmem path in that case. */
		1006	}
		1007
		1008	if (ret == -EFAULT \|\| ret == -ENOSPC)
		1009	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
		1010
		1011	out:
		1012	drm_gem_object_unreference(&obj->base);
		1013	unlock:
		1014	mutex_unlock(&dev->struct_mutex);
		1015	return ret;
		1016	}
		1017
2332	Serge	1018	#endif
		1019
3031	serge	1020	int
		1021	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		1022	bool interruptible)
		1023	{
		1024	if (atomic_read(&dev_priv->mm.wedged)) {
		1025	struct completion *x = &dev_priv->error_completion;
		1026	bool recovery_complete;
		1027	unsigned long flags;
2332	Serge	1028
3031	serge	1029	/* Give the error handler a chance to run. */
		1030	spin_lock_irqsave(&x->wait.lock, flags);
		1031	recovery_complete = x->done > 0;
		1032	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	1033
3031	serge	1034	/* Non-interruptible callers can't handle -EAGAIN, hence return
		1035	* -EIO unconditionally for these. */
		1036	if (!interruptible)
		1037	return -EIO;
2332	Serge	1038
3031	serge	1039	/* Recovery complete, but still wedged means reset failure. */
		1040	if (recovery_complete)
		1041	return -EIO;
2332	Serge	1042
3031	serge	1043	return -EAGAIN;
		1044	}
2332	Serge	1045
3031	serge	1046	return 0;
		1047	}
2332	Serge	1048
3031	serge	1049	/*
		1050	* Compare seqno against outstanding lazy request. Emit a request if they are
		1051	* equal.
		1052	*/
		1053	static int
		1054	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1055	{
		1056	int ret;
2332	Serge	1057
3031	serge	1058	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1059
3031	serge	1060	ret = 0;
		1061	if (seqno == ring->outstanding_lazy_request)
		1062	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1063
3031	serge	1064	return ret;
		1065	}
2332	Serge	1066
3031	serge	1067	/**
		1068	* __wait_seqno - wait until execution of seqno has finished
		1069	* @ring: the ring expected to report seqno
		1070	* @seqno: duh!
		1071	* @interruptible: do an interruptible wait (normally yes)
		1072	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1073	*
		1074	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1075	* errno with remaining time filled in timeout argument.
		1076	*/
		1077	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1078	bool interruptible, struct timespec *timeout)
		1079	{
		1080	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1081	struct timespec before, now, wait_time={1,0};
		1082	unsigned long timeout_jiffies;
		1083	long end;
		1084	bool wait_forever = true;
		1085	int ret;
2332	Serge	1086
3031	serge	1087	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1088	return 0;
2332	Serge	1089
3031	serge	1090	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1091
3031	serge	1092	if (timeout != NULL) {
		1093	wait_time = *timeout;
		1094	wait_forever = false;
		1095	}
2332	Serge	1096
3031	serge	1097	// timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1098
3031	serge	1099	if (WARN_ON(!ring->irq_get(ring)))
		1100	return -ENODEV;
		1101	#if 0
2332	Serge	1102
3031	serge	1103	/* Record current time in case interrupted by signal, or wedged * */
		1104	getrawmonotonic(&before);
2332	Serge	1105
3031	serge	1106	#define EXIT_COND \
		1107	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1108	atomic_read(&dev_priv->mm.wedged))
		1109	do {
		1110	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1111	timeout_jiffies);
2332	Serge	1112
3031	serge	1113	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1114	if (ret)
		1115	end = ret;
		1116	} while (end == 0 && wait_forever);
2332	Serge	1117
3031	serge	1118	getrawmonotonic(&now);
2332	Serge	1119
3031	serge	1120	ring->irq_put(ring);
		1121	trace_i915_gem_request_wait_end(ring, seqno);
		1122	#undef EXIT_COND
2332	Serge	1123
3031	serge	1124	if (timeout) {
		1125	// struct timespec sleep_time = timespec_sub(now, before);
		1126	// timeout = timespec_sub(timeout, sleep_time);
		1127	}
2332	Serge	1128
3031	serge	1129	switch (end) {
		1130	case -EIO:
		1131	case -EAGAIN: /* Wedged */
		1132	case -ERESTARTSYS: /* Signal */
		1133	return (int)end;
		1134	case 0: /* Timeout */
		1135	// if (timeout)
		1136	// set_normalized_timespec(timeout, 0, 0);
		1137	return -ETIME;
		1138	default: /* Completed */
		1139	WARN_ON(end < 0); /* We're not aware of other errors */
		1140	return 0;
		1141	}
		1142	#endif
2332	Serge	1143
3031	serge	1144	#define EXIT_COND \
		1145	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1146	atomic_read(&dev_priv->mm.wedged))
		1147	wait_event(ring->irq_queue, EXIT_COND);
		1148	#undef EXIT_COND
		1149	ring->irq_put(ring);
2332	Serge	1150
3031	serge	1151	return 0;
		1152	}
2332	Serge	1153
3031	serge	1154	/**
		1155	* Waits for a sequence number to be signaled, and cleans up the
		1156	* request and object lists appropriately for that event.
		1157	*/
		1158	int
		1159	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1160	{
		1161	struct drm_device *dev = ring->dev;
		1162	struct drm_i915_private *dev_priv = dev->dev_private;
		1163	bool interruptible = dev_priv->mm.interruptible;
		1164	int ret;
2332	Serge	1165
3031	serge	1166	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1167	BUG_ON(seqno == 0);
2332	Serge	1168
3031	serge	1169	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1170	if (ret)
		1171	return ret;
2332	Serge	1172
3031	serge	1173	ret = i915_gem_check_olr(ring, seqno);
		1174	if (ret)
		1175	return ret;
2332	Serge	1176
3031	serge	1177	return __wait_seqno(ring, seqno, interruptible, NULL);
		1178	}
2332	Serge	1179
3031	serge	1180	/**
		1181	* Ensures that all rendering to the object has completed and the object is
		1182	* safe to unbind from the GTT or access from the CPU.
		1183	*/
		1184	static __must_check int
		1185	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1186	bool readonly)
		1187	{
		1188	struct intel_ring_buffer *ring = obj->ring;
		1189	u32 seqno;
		1190	int ret;
2332	Serge	1191
3031	serge	1192	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1193	if (seqno == 0)
		1194	return 0;
2332	Serge	1195
3031	serge	1196	ret = i915_wait_seqno(ring, seqno);
		1197	if (ret)
		1198	return ret;
2332	Serge	1199
3031	serge	1200	i915_gem_retire_requests_ring(ring);
2332	Serge	1201
3031	serge	1202	/* Manually manage the write flush as we may have not yet
		1203	* retired the buffer.
		1204	*/
		1205	if (obj->last_write_seqno &&
		1206	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1207	obj->last_write_seqno = 0;
		1208	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1209	}
2332	Serge	1210
3031	serge	1211	return 0;
		1212	}
2332	Serge	1213
		1214
		1215
		1216
		1217
		1218
		1219
		1220
		1221
		1222
		1223
		1224
		1225
		1226
		1227
		1228
		1229
		1230
		1231
		1232
		1233
		1234
		1235
		1236
		1237
		1238
		1239
		1240
		1241
		1242
		1243
		1244
		1245
		1246
		1247
		1248
		1249
3031	serge	1250
		1251
		1252
		1253
		1254
		1255
		1256	/**
		1257	* i915_gem_release_mmap - remove physical page mappings
		1258	* @obj: obj in question
		1259	*
		1260	* Preserve the reservation of the mmapping with the DRM core code, but
		1261	* relinquish ownership of the pages back to the system.
		1262	*
		1263	* It is vital that we remove the page mapping if we have mapped a tiled
		1264	* object through the GTT and then lose the fence register due to
		1265	* resource pressure. Similarly if the object has been moved out of the
		1266	* aperture, than pages mapped into userspace must be revoked. Removing the
		1267	* mapping will then trigger a page fault on the next user access, allowing
		1268	* fixup by i915_gem_fault().
		1269	*/
		1270	void
		1271	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1272	{
		1273	if (!obj->fault_mappable)
		1274	return;
		1275
		1276	if (obj->base.dev->dev_mapping)
		1277	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1278	// (loff_t)obj->base.map_list.hash.key<
		1279	// obj->base.size, 1);
		1280
		1281	obj->fault_mappable = false;
		1282	}
		1283
2332	Serge	1284	static uint32_t
		1285	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1286	{
		1287	uint32_t gtt_size;
		1288
		1289	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1290	tiling_mode == I915_TILING_NONE)
		1291	return size;
		1292
		1293	/* Previous chips need a power-of-two fence region when tiling */
		1294	if (INTEL_INFO(dev)->gen == 3)
		1295	gtt_size = 1024*1024;
		1296	else
		1297	gtt_size = 512*1024;
		1298
		1299	while (gtt_size < size)
		1300	gtt_size <<= 1;
		1301
		1302	return gtt_size;
		1303	}
		1304
		1305	/**
		1306	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1307	* @obj: object to check
		1308	*
		1309	* Return the required GTT alignment for an object, taking into account
		1310	* potential fence register mapping.
		1311	*/
		1312	static uint32_t
		1313	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1314	uint32_t size,
		1315	int tiling_mode)
		1316	{
		1317	/*
		1318	* Minimum alignment is 4k (GTT page size), but might be greater
		1319	* if a fence register is needed for the object.
		1320	*/
		1321	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1322	tiling_mode == I915_TILING_NONE)
		1323	return 4096;
		1324
		1325	/*
		1326	* Previous chips need to be aligned to the size of the smallest
		1327	* fence register that can contain the object.
		1328	*/
		1329	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1330	}
		1331
		1332	/**
		1333	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1334	* unfenced object
		1335	* @dev: the device
		1336	* @size: size of the object
		1337	* @tiling_mode: tiling mode of the object
		1338	*
		1339	* Return the required GTT alignment for an object, only taking into account
		1340	* unfenced tiled surface requirements.
		1341	*/
		1342	uint32_t
		1343	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1344	uint32_t size,
		1345	int tiling_mode)
		1346	{
		1347	/*
		1348	* Minimum alignment is 4k (GTT page size) for sane hw.
		1349	*/
		1350	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1351	tiling_mode == I915_TILING_NONE)
		1352	return 4096;
		1353
		1354	/* Previous hardware however needs to be aligned to a power-of-two
		1355	* tile height. The simplest method for determining this is to reuse
		1356	* the power-of-tile object size.
		1357	*/
		1358	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1359	}
		1360
3031	serge	1361	/* Immediately discard the backing storage */
		1362	static void
		1363	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1364	{
		1365	// struct inode *inode;
2332	Serge	1366
3031	serge	1367	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1368
3031	serge	1369	// if (obj->base.filp == NULL)
		1370	// return;
2332	Serge	1371
3031	serge	1372	/* Our goal here is to return as much of the memory as
		1373	* is possible back to the system as we are called from OOM.
		1374	* To do this we must instruct the shmfs to drop all of its
		1375	* backing pages, now.
		1376	*/
		1377	// inode = obj->base.filp->f_path.dentry->d_inode;
		1378	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1379
3031	serge	1380	obj->madv = __I915_MADV_PURGED;
		1381	}
2332	Serge	1382
3031	serge	1383	static inline int
		1384	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1385	{
		1386	return obj->madv == I915_MADV_DONTNEED;
		1387	}
2332	Serge	1388
3031	serge	1389	static void
		1390	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1391	{
		1392	int ret, i;
2332	Serge	1393
3031	serge	1394	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1395
3031	serge	1396	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1397	if (ret) {
		1398	/* In the event of a disaster, abandon all caches and
		1399	* hope for the best.
		1400	*/
		1401	WARN_ON(ret != -EIO);
		1402	i915_gem_clflush_object(obj);
		1403	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1404	}
2332	Serge	1405
3031	serge	1406	if (obj->madv == I915_MADV_DONTNEED)
		1407	obj->dirty = 0;
2332	Serge	1408
3031	serge	1409	for (i = 0; i < obj->pages.nents; i++)
		1410	FreePage(obj->pages.page[i]);
2332	Serge	1411
3039	serge	1412	DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, obj->pages.nents);
3031	serge	1413	obj->dirty = 0;
		1414	kfree(obj->pages.page);
		1415	}
2332	Serge	1416
3031	serge	1417	static int
		1418	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1419	{
		1420	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1421
3039	serge	1422	// printf("page %x pin count %d\n",
		1423	// obj->pages.page, obj->pages_pin_count );
		1424
3031	serge	1425	if (obj->pages.page == NULL)
		1426	return 0;
2332	Serge	1427
3031	serge	1428	BUG_ON(obj->gtt_space);
		1429
		1430	if (obj->pages_pin_count)
		1431	return -EBUSY;
		1432
		1433	ops->put_pages(obj);
		1434	obj->pages.page = NULL;
		1435
		1436	list_del(&obj->gtt_list);
		1437	if (i915_gem_object_is_purgeable(obj))
		1438	i915_gem_object_truncate(obj);
		1439
		1440	return 0;
		1441	}
		1442
		1443
		1444
		1445
		1446
		1447
		1448
		1449
2332	Serge	1450	static int
3031	serge	1451	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1452	{
3031	serge	1453	dma_addr_t page;
2332	Serge	1454	int page_count, i;
		1455
		1456	/* Get the list of pages out of our struct file. They'll be pinned
		1457	* at this point until we release them.
		1458	*/
		1459	page_count = obj->base.size / PAGE_SIZE;
3031	serge	1460	BUG_ON(obj->pages.page != NULL);
		1461	obj->pages.page = malloc(page_count * sizeof(dma_addr_t));
		1462	if (obj->pages.page == NULL)
2332	Serge	1463	return -ENOMEM;
		1464
		1465	for (i = 0; i < page_count; i++) {
3031	serge	1466	page = AllocPage(); // oh-oh
		1467	if ( page == 0 )
2332	Serge	1468	goto err_pages;
		1469
3031	serge	1470	obj->pages.page[i] = page;
		1471	};
3037	serge	1472	DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
3031	serge	1473	obj->pages.nents = page_count;
		1474
		1475
2332	Serge	1476	// if (obj->tiling_mode != I915_TILING_NONE)
		1477	// i915_gem_object_do_bit_17_swizzle(obj);
		1478
		1479	return 0;
		1480
		1481	err_pages:
2344	Serge	1482	while (i--)
3031	serge	1483	FreePage(obj->pages.page[i]);
2332	Serge	1484
3031	serge	1485	free(obj->pages.page);
		1486	obj->pages.page = NULL;
		1487	obj->pages.nents = 0;
		1488
		1489	return -ENOMEM;
2332	Serge	1490	}
		1491
3031	serge	1492	/* Ensure that the associated pages are gathered from the backing storage
		1493	* and pinned into our object. i915_gem_object_get_pages() may be called
		1494	* multiple times before they are released by a single call to
		1495	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1496	* either as a result of memory pressure (reaping pages under the shrinker)
		1497	* or as the object is itself released.
		1498	*/
		1499	int
		1500	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1501	{
3031	serge	1502	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1503	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1504	int ret;
2332	Serge	1505
3031	serge	1506	if (obj->pages.page)
		1507	return 0;
2332	Serge	1508
3031	serge	1509	BUG_ON(obj->pages_pin_count);
2332	Serge	1510
3031	serge	1511	ret = ops->get_pages(obj);
		1512	if (ret)
		1513	return ret;
2344	Serge	1514
3031	serge	1515	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
		1516	return 0;
2332	Serge	1517	}
		1518
		1519	void
		1520	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
		1521	struct intel_ring_buffer *ring,
		1522	u32 seqno)
		1523	{
		1524	struct drm_device *dev = obj->base.dev;
		1525	struct drm_i915_private *dev_priv = dev->dev_private;
		1526
		1527	BUG_ON(ring == NULL);
		1528	obj->ring = ring;
		1529
		1530	/* Add a reference if we're newly entering the active list. */
		1531	if (!obj->active) {
2344	Serge	1532	drm_gem_object_reference(&obj->base);
2332	Serge	1533	obj->active = 1;
		1534	}
		1535
		1536	/* Move from whatever list we were on to the tail of execution. */
		1537	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1538	list_move_tail(&obj->ring_list, &ring->active_list);
		1539
3031	serge	1540	obj->last_read_seqno = seqno;
		1541
2332	Serge	1542	if (obj->fenced_gpu_access) {
3031	serge	1543	obj->last_fenced_seqno = seqno;
		1544
		1545	/* Bump MRU to take account of the delayed flush */
		1546	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1547	struct drm_i915_fence_reg *reg;
		1548
		1549	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1550	list_move_tail(®->lru_list,
		1551	&dev_priv->mm.fence_list);
		1552	}
2332	Serge	1553	}
		1554	}
		1555
2344	Serge	1556	static void
3031	serge	1557	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1558	{
		1559	struct drm_device *dev = obj->base.dev;
3031	serge	1560	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1561
3031	serge	1562	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1563	BUG_ON(!obj->active);
2332	Serge	1564
3031	serge	1565	if (obj->pin_count) /* are we a framebuffer? */
		1566	intel_mark_fb_idle(obj);
2344	Serge	1567
2352	Serge	1568	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1569
3031	serge	1570	list_del_init(&obj->ring_list);
2352	Serge	1571	obj->ring = NULL;
2344	Serge	1572
3031	serge	1573	obj->last_read_seqno = 0;
		1574	obj->last_write_seqno = 0;
		1575	obj->base.write_domain = 0;
		1576
		1577	obj->last_fenced_seqno = 0;
2352	Serge	1578	obj->fenced_gpu_access = false;
2344	Serge	1579
2352	Serge	1580	obj->active = 0;
		1581	drm_gem_object_unreference(&obj->base);
		1582
		1583	WARN_ON(i915_verify_lists(dev));
		1584	}
		1585
3031	serge	1586	static u32
		1587	i915_gem_get_seqno(struct drm_device *dev)
2344	Serge	1588	{
3031	serge	1589	drm_i915_private_t *dev_priv = dev->dev_private;
		1590	u32 seqno = dev_priv->next_seqno;
2344	Serge	1591
3031	serge	1592	/* reserve 0 for non-seqno */
		1593	if (++dev_priv->next_seqno == 0)
		1594	dev_priv->next_seqno = 1;
2344	Serge	1595
3031	serge	1596	return seqno;
2344	Serge	1597	}
		1598
3031	serge	1599	u32
		1600	i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
2344	Serge	1601	{
3031	serge	1602	if (ring->outstanding_lazy_request == 0)
		1603	ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
2344	Serge	1604
3031	serge	1605	return ring->outstanding_lazy_request;
2332	Serge	1606	}
		1607
2352	Serge	1608	int
		1609	i915_add_request(struct intel_ring_buffer *ring,
		1610	struct drm_file *file,
3031	serge	1611	u32 *out_seqno)
2352	Serge	1612	{
		1613	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1614	struct drm_i915_gem_request *request;
		1615	u32 request_ring_position;
		1616	u32 seqno;
2352	Serge	1617	int was_empty;
		1618	int ret;
2332	Serge	1619
3031	serge	1620	/*
		1621	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1622	* after having emitted the batchbuffer command. Hence we need to fix
		1623	* things up similar to emitting the lazy request. The difference here
		1624	* is that the flush _must_ happen before the next request, no matter
		1625	* what.
		1626	*/
		1627	ret = intel_ring_flush_all_caches(ring);
		1628	if (ret)
		1629	return ret;
2332	Serge	1630
3031	serge	1631	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1632	if (request == NULL)
		1633	return -ENOMEM;
		1634
		1635	seqno = i915_gem_next_request_seqno(ring);
		1636
		1637	/* Record the position of the start of the request so that
		1638	* should we detect the updated seqno part-way through the
		1639	* GPU processing the request, we never over-estimate the
		1640	* position of the head.
		1641	*/
		1642	request_ring_position = intel_ring_get_tail(ring);
		1643
2352	Serge	1644	ret = ring->add_request(ring, &seqno);
3031	serge	1645	if (ret) {
		1646	kfree(request);
2352	Serge	1647	return ret;
3031	serge	1648	}
2332	Serge	1649
2352	Serge	1650	trace_i915_gem_request_add(ring, seqno);
2332	Serge	1651
2352	Serge	1652	request->seqno = seqno;
		1653	request->ring = ring;
3031	serge	1654	request->tail = request_ring_position;
		1655	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1656	was_empty = list_empty(&ring->request_list);
		1657	list_add_tail(&request->list, &ring->request_list);
3031	serge	1658	request->file_priv = NULL;
2332	Serge	1659
		1660
3031	serge	1661	ring->outstanding_lazy_request = 0;
2332	Serge	1662
2360	Serge	1663	if (!dev_priv->mm.suspended) {
		1664	if (i915_enable_hangcheck) {
2352	Serge	1665	// mod_timer(&dev_priv->hangcheck_timer,
		1666	// jiffies +
		1667	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1668	}
3031	serge	1669	if (was_empty) {
2360	Serge	1670	queue_delayed_work(dev_priv->wq,
		1671	&dev_priv->mm.retire_work, HZ);
3031	serge	1672	intel_mark_busy(dev_priv->dev);
		1673	}
2360	Serge	1674	}
3031	serge	1675
		1676	if (out_seqno)
		1677	*out_seqno = seqno;
2352	Serge	1678	return 0;
		1679	}
2332	Serge	1680
		1681
		1682
		1683
3031	serge	1684	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1685	struct intel_ring_buffer *ring)
		1686	{
		1687	while (!list_empty(&ring->request_list)) {
		1688	struct drm_i915_gem_request *request;
2332	Serge	1689
3031	serge	1690	request = list_first_entry(&ring->request_list,
		1691	struct drm_i915_gem_request,
		1692	list);
2332	Serge	1693
3031	serge	1694	list_del(&request->list);
		1695	// i915_gem_request_remove_from_client(request);
		1696	kfree(request);
		1697	}
2332	Serge	1698
3031	serge	1699	while (!list_empty(&ring->active_list)) {
		1700	struct drm_i915_gem_object *obj;
2332	Serge	1701
3031	serge	1702	obj = list_first_entry(&ring->active_list,
		1703	struct drm_i915_gem_object,
		1704	ring_list);
2332	Serge	1705
3031	serge	1706	i915_gem_object_move_to_inactive(obj);
		1707	}
		1708	}
2332	Serge	1709
3031	serge	1710	static void i915_gem_reset_fences(struct drm_device *dev)
		1711	{
		1712	struct drm_i915_private *dev_priv = dev->dev_private;
		1713	int i;
2332	Serge	1714
3031	serge	1715	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1716	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1717
3031	serge	1718	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1719
3031	serge	1720	if (reg->obj)
		1721	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1722
3031	serge	1723	reg->pin_count = 0;
		1724	reg->obj = NULL;
		1725	INIT_LIST_HEAD(®->lru_list);
		1726	}
2360	Serge	1727
3031	serge	1728	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1729	}
2360	Serge	1730
3031	serge	1731	void i915_gem_reset(struct drm_device *dev)
		1732	{
		1733	struct drm_i915_private *dev_priv = dev->dev_private;
		1734	struct drm_i915_gem_object *obj;
		1735	struct intel_ring_buffer *ring;
		1736	int i;
2360	Serge	1737
3031	serge	1738	for_each_ring(ring, dev_priv, i)
		1739	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1740
3031	serge	1741	/* Move everything out of the GPU domains to ensure we do any
		1742	* necessary invalidation upon reuse.
		1743	*/
		1744	list_for_each_entry(obj,
		1745	&dev_priv->mm.inactive_list,
		1746	mm_list)
		1747	{
		1748	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1749	}
2360	Serge	1750
3031	serge	1751	/* The fence registers are invalidated so clear them out */
		1752	i915_gem_reset_fences(dev);
		1753	}
2360	Serge	1754
2352	Serge	1755	/**
		1756	* This function clears the request list as sequence numbers are passed.
		1757	*/
3031	serge	1758	void
2352	Serge	1759	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1760	{
		1761	uint32_t seqno;
		1762	int i;
2332	Serge	1763
2352	Serge	1764	if (list_empty(&ring->request_list))
		1765	return;
2332	Serge	1766
2352	Serge	1767	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1768
3031	serge	1769	seqno = ring->get_seqno(ring, true);
2332	Serge	1770
2352	Serge	1771	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
		1772	if (seqno >= ring->sync_seqno[i])
		1773	ring->sync_seqno[i] = 0;
2332	Serge	1774
2352	Serge	1775	while (!list_empty(&ring->request_list)) {
		1776	struct drm_i915_gem_request *request;
2332	Serge	1777
2352	Serge	1778	request = list_first_entry(&ring->request_list,
		1779	struct drm_i915_gem_request,
		1780	list);
2332	Serge	1781
2352	Serge	1782	if (!i915_seqno_passed(seqno, request->seqno))
		1783	break;
2332	Serge	1784
2352	Serge	1785	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1786	/* We know the GPU must have read the request to have
		1787	* sent us the seqno + interrupt, so use the position
		1788	* of tail of the request to update the last known position
		1789	* of the GPU head.
		1790	*/
		1791	ring->last_retired_head = request->tail;
2332	Serge	1792
2352	Serge	1793	list_del(&request->list);
		1794	kfree(request);
		1795	}
2332	Serge	1796
2352	Serge	1797	/* Move any buffers on the active list that are no longer referenced
		1798	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1799	*/
		1800	while (!list_empty(&ring->active_list)) {
		1801	struct drm_i915_gem_object *obj;
2332	Serge	1802
2352	Serge	1803	obj = list_first_entry(&ring->active_list,
		1804	struct drm_i915_gem_object,
		1805	ring_list);
2332	Serge	1806
3031	serge	1807	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1808	break;
2332	Serge	1809
2352	Serge	1810	i915_gem_object_move_to_inactive(obj);
		1811	}
2332	Serge	1812
2352	Serge	1813	if (unlikely(ring->trace_irq_seqno &&
		1814	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1815	ring->irq_put(ring);
		1816	ring->trace_irq_seqno = 0;
		1817	}
2332	Serge	1818
2352	Serge	1819	WARN_ON(i915_verify_lists(ring->dev));
		1820	}
2332	Serge	1821
2352	Serge	1822	void
		1823	i915_gem_retire_requests(struct drm_device *dev)
		1824	{
		1825	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1826	struct intel_ring_buffer *ring;
2352	Serge	1827	int i;
2332	Serge	1828
3031	serge	1829	for_each_ring(ring, dev_priv, i)
		1830	i915_gem_retire_requests_ring(ring);
2352	Serge	1831	}
		1832
2360	Serge	1833	static void
		1834	i915_gem_retire_work_handler(struct work_struct *work)
		1835	{
		1836	drm_i915_private_t *dev_priv;
		1837	struct drm_device *dev;
3031	serge	1838	struct intel_ring_buffer *ring;
2360	Serge	1839	bool idle;
		1840	int i;
2352	Serge	1841
2360	Serge	1842	dev_priv = container_of(work, drm_i915_private_t,
		1843	mm.retire_work.work);
		1844	dev = dev_priv->dev;
2352	Serge	1845
2360	Serge	1846	/* Come back later if the device is busy... */
		1847	if (!mutex_trylock(&dev->struct_mutex)) {
		1848	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
		1849	return;
		1850	}
2352	Serge	1851
2360	Serge	1852	i915_gem_retire_requests(dev);
2352	Serge	1853
2360	Serge	1854	/* Send a periodic flush down the ring so we don't hold onto GEM
		1855	* objects indefinitely.
		1856	*/
		1857	idle = true;
3031	serge	1858	for_each_ring(ring, dev_priv, i) {
		1859	if (ring->gpu_caches_dirty)
		1860	i915_add_request(ring, NULL, NULL);
2352	Serge	1861
2360	Serge	1862	idle &= list_empty(&ring->request_list);
		1863	}
2352	Serge	1864
2360	Serge	1865	if (!dev_priv->mm.suspended && !idle)
		1866	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	1867	if (idle)
		1868	intel_mark_idle(dev);
2360	Serge	1869
		1870	mutex_unlock(&dev->struct_mutex);
		1871	}
		1872
2344	Serge	1873	/**
3031	serge	1874	* Ensures that an object will eventually get non-busy by flushing any required
		1875	* write domains, emitting any outstanding lazy request and retiring and
		1876	* completed requests.
2352	Serge	1877	*/
3031	serge	1878	static int
		1879	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	1880	{
3031	serge	1881	int ret;
2352	Serge	1882
3031	serge	1883	if (obj->active) {
		1884	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		1885	if (ret)
		1886	return ret;
2352	Serge	1887
3031	serge	1888	i915_gem_retire_requests_ring(obj->ring);
		1889	}
2352	Serge	1890
3031	serge	1891	return 0;
		1892	}
2352	Serge	1893
		1894
		1895
		1896
		1897
		1898
		1899
		1900
		1901
		1902
		1903	/**
3031	serge	1904	* i915_gem_object_sync - sync an object to a ring.
		1905	*
		1906	* @obj: object which may be in use on another ring.
		1907	* @to: ring we wish to use the object on. May be NULL.
		1908	*
		1909	* This code is meant to abstract object synchronization with the GPU.
		1910	* Calling with NULL implies synchronizing the object with the CPU
		1911	* rather than a particular GPU ring.
		1912	*
		1913	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	1914	*/
		1915	int
3031	serge	1916	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		1917	struct intel_ring_buffer *to)
2344	Serge	1918	{
3031	serge	1919	struct intel_ring_buffer *from = obj->ring;
		1920	u32 seqno;
		1921	int ret, idx;
2332	Serge	1922
3031	serge	1923	if (from == NULL \|\| to == from)
		1924	return 0;
2332	Serge	1925
3031	serge	1926	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		1927	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	1928
3031	serge	1929	idx = intel_ring_sync_index(from, to);
		1930
		1931	seqno = obj->last_read_seqno;
		1932	if (seqno <= from->sync_seqno[idx])
		1933	return 0;
		1934
		1935	ret = i915_gem_check_olr(obj->ring, seqno);
		1936	if (ret)
		1937	return ret;
		1938
		1939	ret = to->sync_to(to, from, seqno);
		1940	if (!ret)
		1941	from->sync_seqno[idx] = seqno;
		1942
		1943	return ret;
2344	Serge	1944	}
2332	Serge	1945
2344	Serge	1946	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		1947	{
		1948	u32 old_write_domain, old_read_domains;
2332	Serge	1949
2344	Serge	1950	/* Act a barrier for all accesses through the GTT */
		1951	mb();
2332	Serge	1952
2344	Serge	1953	/* Force a pagefault for domain tracking on next user access */
		1954	// i915_gem_release_mmap(obj);
2332	Serge	1955
2344	Serge	1956	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		1957	return;
2332	Serge	1958
2344	Serge	1959	old_read_domains = obj->base.read_domains;
		1960	old_write_domain = obj->base.write_domain;
2351	Serge	1961
2344	Serge	1962	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		1963	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	1964
2351	Serge	1965	trace_i915_gem_object_change_domain(obj,
		1966	old_read_domains,
		1967	old_write_domain);
2344	Serge	1968	}
2332	Serge	1969
2344	Serge	1970	/**
		1971	* Unbinds an object from the GTT aperture.
		1972	*/
		1973	int
		1974	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		1975	{
3031	serge	1976	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	1977	int ret = 0;
2332	Serge	1978
2344	Serge	1979	if (obj->gtt_space == NULL)
		1980	return 0;
2332	Serge	1981
3031	serge	1982	if (obj->pin_count)
		1983	return -EBUSY;
2332	Serge	1984
3031	serge	1985	BUG_ON(obj->pages.page == NULL);
		1986
2344	Serge	1987	ret = i915_gem_object_finish_gpu(obj);
3031	serge	1988	if (ret)
2344	Serge	1989	return ret;
		1990	/* Continue on if we fail due to EIO, the GPU is hung so we
		1991	* should be safe and we need to cleanup or else we might
		1992	* cause memory corruption through use-after-free.
		1993	*/
2332	Serge	1994
2344	Serge	1995	i915_gem_object_finish_gtt(obj);
2332	Serge	1996
2344	Serge	1997	/* release the fence reg _after_ flushing */
		1998	ret = i915_gem_object_put_fence(obj);
3031	serge	1999	if (ret)
2344	Serge	2000	return ret;
2332	Serge	2001
2351	Serge	2002	trace_i915_gem_object_unbind(obj);
2332	Serge	2003
3031	serge	2004	if (obj->has_global_gtt_mapping)
2344	Serge	2005	i915_gem_gtt_unbind_object(obj);
3031	serge	2006	if (obj->has_aliasing_ppgtt_mapping) {
		2007	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2008	obj->has_aliasing_ppgtt_mapping = 0;
		2009	}
		2010	i915_gem_gtt_finish_object(obj);
2332	Serge	2011
3031	serge	2012	list_del(&obj->mm_list);
		2013	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2014	/* Avoid an unnecessary call to unbind on rebind. */
		2015	obj->map_and_fenceable = true;
2332	Serge	2016
2344	Serge	2017	drm_mm_put_block(obj->gtt_space);
		2018	obj->gtt_space = NULL;
		2019	obj->gtt_offset = 0;
2332	Serge	2020
2344	Serge	2021	return 0;
		2022	}
2332	Serge	2023
2344	Serge	2024	static int i915_ring_idle(struct intel_ring_buffer *ring)
		2025	{
3031	serge	2026	if (list_empty(&ring->active_list))
2344	Serge	2027	return 0;
2332	Serge	2028
3031	serge	2029	return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
2344	Serge	2030	}
2332	Serge	2031
3031	serge	2032	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2033	{
		2034	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2035	struct intel_ring_buffer *ring;
2344	Serge	2036	int ret, i;
2332	Serge	2037
2344	Serge	2038	/* Flush everything onto the inactive list. */
3031	serge	2039	for_each_ring(ring, dev_priv, i) {
		2040	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2041	if (ret)
		2042	return ret;
3031	serge	2043
		2044	ret = i915_ring_idle(ring);
		2045	if (ret)
		2046	return ret;
2344	Serge	2047	}
2332	Serge	2048
2344	Serge	2049	return 0;
		2050	}
2332	Serge	2051
3031	serge	2052	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2053	struct drm_i915_gem_object *obj)
		2054	{
		2055	drm_i915_private_t *dev_priv = dev->dev_private;
		2056	uint64_t val;
2332	Serge	2057
3031	serge	2058	if (obj) {
		2059	u32 size = obj->gtt_space->size;
2332	Serge	2060
3031	serge	2061	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2062	0xfffff000) << 32;
		2063	val \|= obj->gtt_offset & 0xfffff000;
		2064	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2065	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2066
3031	serge	2067	if (obj->tiling_mode == I915_TILING_Y)
		2068	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2069	val \|= I965_FENCE_REG_VALID;
		2070	} else
		2071	val = 0;
2332	Serge	2072
3031	serge	2073	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2074	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2075	}
2332	Serge	2076
3031	serge	2077	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2078	struct drm_i915_gem_object *obj)
		2079	{
		2080	drm_i915_private_t *dev_priv = dev->dev_private;
		2081	uint64_t val;
2332	Serge	2082
3031	serge	2083	if (obj) {
		2084	u32 size = obj->gtt_space->size;
2332	Serge	2085
3031	serge	2086	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2087	0xfffff000) << 32;
		2088	val \|= obj->gtt_offset & 0xfffff000;
		2089	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2090	if (obj->tiling_mode == I915_TILING_Y)
		2091	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2092	val \|= I965_FENCE_REG_VALID;
		2093	} else
		2094	val = 0;
2332	Serge	2095
3031	serge	2096	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2097	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2098	}
2332	Serge	2099
3031	serge	2100	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2101	struct drm_i915_gem_object *obj)
		2102	{
		2103	drm_i915_private_t *dev_priv = dev->dev_private;
		2104	u32 val;
2332	Serge	2105
3031	serge	2106	if (obj) {
		2107	u32 size = obj->gtt_space->size;
		2108	int pitch_val;
		2109	int tile_width;
2332	Serge	2110
3031	serge	2111	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2112	(size & -size) != size \|\|
		2113	(obj->gtt_offset & (size - 1)),
		2114	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2115	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2116
3031	serge	2117	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2118	tile_width = 128;
		2119	else
		2120	tile_width = 512;
2332	Serge	2121
3031	serge	2122	/* Note: pitch better be a power of two tile widths */
		2123	pitch_val = obj->stride / tile_width;
		2124	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2125
3031	serge	2126	val = obj->gtt_offset;
		2127	if (obj->tiling_mode == I915_TILING_Y)
		2128	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2129	val \|= I915_FENCE_SIZE_BITS(size);
		2130	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2131	val \|= I830_FENCE_REG_VALID;
		2132	} else
		2133	val = 0;
2332	Serge	2134
3031	serge	2135	if (reg < 8)
		2136	reg = FENCE_REG_830_0 + reg * 4;
		2137	else
		2138	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2139
3031	serge	2140	I915_WRITE(reg, val);
		2141	POSTING_READ(reg);
		2142	}
2332	Serge	2143
3031	serge	2144	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2145	struct drm_i915_gem_object *obj)
		2146	{
		2147	drm_i915_private_t *dev_priv = dev->dev_private;
		2148	uint32_t val;
2344	Serge	2149
3031	serge	2150	if (obj) {
		2151	u32 size = obj->gtt_space->size;
		2152	uint32_t pitch_val;
2344	Serge	2153
3031	serge	2154	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2155	(size & -size) != size \|\|
		2156	(obj->gtt_offset & (size - 1)),
		2157	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2158	obj->gtt_offset, size);
2344	Serge	2159
3031	serge	2160	pitch_val = obj->stride / 128;
		2161	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2162
3031	serge	2163	val = obj->gtt_offset;
		2164	if (obj->tiling_mode == I915_TILING_Y)
		2165	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2166	val \|= I830_FENCE_SIZE_BITS(size);
		2167	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2168	val \|= I830_FENCE_REG_VALID;
		2169	} else
		2170	val = 0;
		2171
		2172	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2173	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2174	}
		2175
		2176	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2177	struct drm_i915_gem_object *obj)
2332	Serge	2178	{
3031	serge	2179	switch (INTEL_INFO(dev)->gen) {
		2180	case 7:
		2181	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2182	case 5:
		2183	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2184	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2185	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2186	default: break;
		2187	}
2344	Serge	2188	}
		2189
3031	serge	2190	static inline int fence_number(struct drm_i915_private *dev_priv,
		2191	struct drm_i915_fence_reg *fence)
2344	Serge	2192	{
3031	serge	2193	return fence - dev_priv->fence_regs;
		2194	}
2332	Serge	2195
3031	serge	2196	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2197	struct drm_i915_fence_reg *fence,
		2198	bool enable)
		2199	{
		2200	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2201	int reg = fence_number(dev_priv, fence);
2332	Serge	2202
3031	serge	2203	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2204
		2205	if (enable) {
		2206	obj->fence_reg = reg;
		2207	fence->obj = obj;
		2208	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2209	} else {
		2210	obj->fence_reg = I915_FENCE_REG_NONE;
		2211	fence->obj = NULL;
		2212	list_del_init(&fence->lru_list);
2344	Serge	2213	}
3031	serge	2214	}
2344	Serge	2215
3031	serge	2216	static int
		2217	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2218	{
		2219	if (obj->last_fenced_seqno) {
		2220	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2221	if (ret)
		2222	return ret;
2344	Serge	2223
		2224	obj->last_fenced_seqno = 0;
		2225	}
		2226
		2227	/* Ensure that all CPU reads are completed before installing a fence
		2228	* and all writes before removing the fence.
2332	Serge	2229	*/
2344	Serge	2230	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2231	mb();
2332	Serge	2232
3031	serge	2233	obj->fenced_gpu_access = false;
2332	Serge	2234	return 0;
		2235	}
		2236
		2237	int
2344	Serge	2238	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2239	{
3031	serge	2240	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2241	int ret;
		2242
3031	serge	2243	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2244	if (ret)
		2245	return ret;
		2246
3031	serge	2247	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2248	return 0;
2332	Serge	2249
3031	serge	2250	i915_gem_object_update_fence(obj,
		2251	&dev_priv->fence_regs[obj->fence_reg],
		2252	false);
		2253	i915_gem_object_fence_lost(obj);
2344	Serge	2254
2332	Serge	2255	return 0;
		2256	}
		2257
3031	serge	2258	static struct drm_i915_fence_reg *
		2259	i915_find_fence_reg(struct drm_device *dev)
		2260	{
		2261	struct drm_i915_private *dev_priv = dev->dev_private;
		2262	struct drm_i915_fence_reg reg, avail;
		2263	int i;
2332	Serge	2264
3031	serge	2265	/* First try to find a free reg */
		2266	avail = NULL;
		2267	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2268	reg = &dev_priv->fence_regs[i];
		2269	if (!reg->obj)
		2270	return reg;
2332	Serge	2271
3031	serge	2272	if (!reg->pin_count)
		2273	avail = reg;
		2274	}
2332	Serge	2275
3031	serge	2276	if (avail == NULL)
		2277	return NULL;
2332	Serge	2278
3031	serge	2279	/* None available, try to steal one or wait for a user to finish */
		2280	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2281	if (reg->pin_count)
		2282	continue;
2332	Serge	2283
3031	serge	2284	return reg;
		2285	}
2332	Serge	2286
3031	serge	2287	return NULL;
		2288	}
2332	Serge	2289
3031	serge	2290	/**
		2291	* i915_gem_object_get_fence - set up fencing for an object
		2292	* @obj: object to map through a fence reg
		2293	*
		2294	* When mapping objects through the GTT, userspace wants to be able to write
		2295	* to them without having to worry about swizzling if the object is tiled.
		2296	* This function walks the fence regs looking for a free one for @obj,
		2297	* stealing one if it can't find any.
		2298	*
		2299	* It then sets up the reg based on the object's properties: address, pitch
		2300	* and tiling format.
		2301	*
		2302	* For an untiled surface, this removes any existing fence.
		2303	*/
		2304	int
		2305	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2306	{
		2307	struct drm_device *dev = obj->base.dev;
		2308	struct drm_i915_private *dev_priv = dev->dev_private;
		2309	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2310	struct drm_i915_fence_reg *reg;
		2311	int ret;
2332	Serge	2312
3031	serge	2313	/* Have we updated the tiling parameters upon the object and so
		2314	* will need to serialise the write to the associated fence register?
		2315	*/
		2316	if (obj->fence_dirty) {
		2317	ret = i915_gem_object_flush_fence(obj);
		2318	if (ret)
		2319	return ret;
		2320	}
2332	Serge	2321
3031	serge	2322	/* Just update our place in the LRU if our fence is getting reused. */
		2323	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2324	reg = &dev_priv->fence_regs[obj->fence_reg];
		2325	if (!obj->fence_dirty) {
		2326	list_move_tail(®->lru_list,
		2327	&dev_priv->mm.fence_list);
		2328	return 0;
		2329	}
		2330	} else if (enable) {
		2331	reg = i915_find_fence_reg(dev);
		2332	if (reg == NULL)
		2333	return -EDEADLK;
2332	Serge	2334
3031	serge	2335	if (reg->obj) {
		2336	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2337
3031	serge	2338	ret = i915_gem_object_flush_fence(old);
		2339	if (ret)
		2340	return ret;
2332	Serge	2341
3031	serge	2342	i915_gem_object_fence_lost(old);
		2343	}
		2344	} else
		2345	return 0;
2332	Serge	2346
3031	serge	2347	i915_gem_object_update_fence(obj, reg, enable);
		2348	obj->fence_dirty = false;
2332	Serge	2349
3031	serge	2350	return 0;
		2351	}
2332	Serge	2352
3031	serge	2353	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2354	struct drm_mm_node *gtt_space,
		2355	unsigned long cache_level)
		2356	{
		2357	struct drm_mm_node *other;
2332	Serge	2358
3031	serge	2359	/* On non-LLC machines we have to be careful when putting differing
		2360	* types of snoopable memory together to avoid the prefetcher
		2361	* crossing memory domains and dieing.
		2362	*/
		2363	if (HAS_LLC(dev))
		2364	return true;
2332	Serge	2365
3031	serge	2366	if (gtt_space == NULL)
		2367	return true;
2332	Serge	2368
3031	serge	2369	if (list_empty(>t_space->node_list))
		2370	return true;
2332	Serge	2371
3031	serge	2372	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2373	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2374	return false;
2344	Serge	2375
3031	serge	2376	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2377	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2378	return false;
2344	Serge	2379
3031	serge	2380	return true;
		2381	}
2344	Serge	2382
3031	serge	2383	static void i915_gem_verify_gtt(struct drm_device *dev)
		2384	{
		2385	#if WATCH_GTT
		2386	struct drm_i915_private *dev_priv = dev->dev_private;
		2387	struct drm_i915_gem_object *obj;
		2388	int err = 0;
2344	Serge	2389
3031	serge	2390	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2391	if (obj->gtt_space == NULL) {
		2392	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2393	err++;
		2394	continue;
		2395	}
2344	Serge	2396
3031	serge	2397	if (obj->cache_level != obj->gtt_space->color) {
		2398	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2399	obj->gtt_space->start,
		2400	obj->gtt_space->start + obj->gtt_space->size,
		2401	obj->cache_level,
		2402	obj->gtt_space->color);
		2403	err++;
		2404	continue;
		2405	}
2344	Serge	2406
3031	serge	2407	if (!i915_gem_valid_gtt_space(dev,
		2408	obj->gtt_space,
		2409	obj->cache_level)) {
		2410	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2411	obj->gtt_space->start,
		2412	obj->gtt_space->start + obj->gtt_space->size,
		2413	obj->cache_level);
		2414	err++;
		2415	continue;
		2416	}
		2417	}
2344	Serge	2418
3031	serge	2419	WARN_ON(err);
		2420	#endif
2326	Serge	2421	}
		2422
2332	Serge	2423	/**
		2424	* Finds free space in the GTT aperture and binds the object there.
		2425	*/
		2426	static int
		2427	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2428	unsigned alignment,
3031	serge	2429	bool map_and_fenceable,
		2430	bool nonblocking)
2332	Serge	2431	{
		2432	struct drm_device *dev = obj->base.dev;
		2433	drm_i915_private_t *dev_priv = dev->dev_private;
		2434	struct drm_mm_node *free_space;
		2435	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2436	bool mappable, fenceable;
		2437	int ret;
2326	Serge	2438
2332	Serge	2439	if (obj->madv != I915_MADV_WILLNEED) {
		2440	DRM_ERROR("Attempting to bind a purgeable object\n");
		2441	return -EINVAL;
		2442	}
		2443
		2444	fence_size = i915_gem_get_gtt_size(dev,
		2445	obj->base.size,
		2446	obj->tiling_mode);
		2447	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2448	obj->base.size,
		2449	obj->tiling_mode);
		2450	unfenced_alignment =
		2451	i915_gem_get_unfenced_gtt_alignment(dev,
		2452	obj->base.size,
		2453	obj->tiling_mode);
		2454
		2455	if (alignment == 0)
		2456	alignment = map_and_fenceable ? fence_alignment :
		2457	unfenced_alignment;
		2458	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2459	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2460	return -EINVAL;
		2461	}
		2462
		2463	size = map_and_fenceable ? fence_size : obj->base.size;
		2464
		2465	/* If the object is bigger than the entire aperture, reject it early
		2466	* before evicting everything in a vain attempt to find space.
		2467	*/
		2468	if (obj->base.size >
		2469	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2470	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2471	return -E2BIG;
		2472	}
		2473
3031	serge	2474	ret = i915_gem_object_get_pages(obj);
		2475	if (ret)
		2476	return ret;
		2477
2332	Serge	2478	search_free:
		2479	if (map_and_fenceable)
		2480	free_space =
3031	serge	2481	drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
		2482	size, alignment, obj->cache_level,
		2483	0, dev_priv->mm.gtt_mappable_end,
		2484	false);
2332	Serge	2485	else
3031	serge	2486	free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
		2487	size, alignment, obj->cache_level,
		2488	false);
2332	Serge	2489
		2490	if (free_space != NULL) {
		2491	if (map_and_fenceable)
		2492	obj->gtt_space =
		2493	drm_mm_get_block_range_generic(free_space,
3031	serge	2494	size, alignment, obj->cache_level,
		2495	0, dev_priv->mm.gtt_mappable_end,
		2496	false);
2332	Serge	2497	else
		2498	obj->gtt_space =
3031	serge	2499	drm_mm_get_block_generic(free_space,
		2500	size, alignment, obj->cache_level,
		2501	false);
2332	Serge	2502	}
		2503	if (obj->gtt_space == NULL) {
		2504	ret = 1; //i915_gem_evict_something(dev, size, alignment,
		2505	// map_and_fenceable);
		2506	if (ret)
		2507	return ret;
		2508
		2509	goto search_free;
		2510	}
3031	serge	2511	if (WARN_ON(!i915_gem_valid_gtt_space(dev,
		2512	obj->gtt_space,
		2513	obj->cache_level))) {
2332	Serge	2514	drm_mm_put_block(obj->gtt_space);
		2515	obj->gtt_space = NULL;
3031	serge	2516	return -EINVAL;
		2517	}
2332	Serge	2518
		2519
3031	serge	2520	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2521	if (ret) {
		2522	drm_mm_put_block(obj->gtt_space);
		2523	obj->gtt_space = NULL;
		2524	return ret;
		2525	}
		2526
3031	serge	2527	if (!dev_priv->mm.aliasing_ppgtt)
		2528	i915_gem_gtt_bind_object(obj, obj->cache_level);
		2529
		2530	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2531	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2532
		2533	obj->gtt_offset = obj->gtt_space->start;
		2534
		2535	fenceable =
		2536	obj->gtt_space->size == fence_size &&
2342	Serge	2537	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2332	Serge	2538
		2539	mappable =
		2540	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2541
		2542	obj->map_and_fenceable = mappable && fenceable;
		2543
2351	Serge	2544	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2545	i915_gem_verify_gtt(dev);
2332	Serge	2546	return 0;
		2547	}
		2548
		2549	void
		2550	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2551	{
		2552	/* If we don't have a page list set up, then we're not pinned
		2553	* to GPU, and we can ignore the cache flush because it'll happen
		2554	* again at bind time.
		2555	*/
3031	serge	2556	if (obj->pages.page == NULL)
2332	Serge	2557	return;
		2558
		2559	/* If the GPU is snooping the contents of the CPU cache,
		2560	* we do not need to manually clear the CPU cache lines. However,
		2561	* the caches are only snooped when the render cache is
		2562	* flushed/invalidated. As we always have to emit invalidations
		2563	* and flushes when moving into and out of the RENDER domain, correct
		2564	* snooping behaviour occurs naturally as the result of our domain
		2565	* tracking.
		2566	*/
		2567	if (obj->cache_level != I915_CACHE_NONE)
		2568	return;
		2569
2344	Serge	2570	if(obj->mapped != NULL)
		2571	{
		2572	uint8_t *page_virtual;
		2573	unsigned int i;
2332	Serge	2574
2344	Serge	2575	page_virtual = obj->mapped;
		2576	asm volatile("mfence");
		2577	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2578	clflush(page_virtual + i);
		2579	asm volatile("mfence");
		2580	}
		2581	else
		2582	{
		2583	uint8_t *page_virtual;
		2584	unsigned int i;
		2585	page_virtual = AllocKernelSpace(obj->base.size);
		2586	if(page_virtual != NULL)
		2587	{
3031	serge	2588	dma_addr_t src, dst;
2344	Serge	2589	u32 count;
		2590
		2591	#define page_tabs 0xFDC00000 /* really dirty hack */
		2592
3031	serge	2593	src = obj->pages.page;
		2594	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2595	count = obj->base.size/4096;
		2596
		2597	while(count--)
		2598	{
		2599	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2600	};
		2601
		2602	asm volatile("mfence");
		2603	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2604	clflush(page_virtual + i);
		2605	asm volatile("mfence");
		2606	FreeKernelSpace(page_virtual);
		2607	}
		2608	else
		2609	{
		2610	asm volatile (
		2611	"mfence \n"
		2612	"wbinvd \n" /* this is really ugly */
		2613	"mfence");
		2614	}
		2615	}
2332	Serge	2616	}
		2617
2344	Serge	2618	/** Flushes the GTT write domain for the object if it's dirty. */
		2619	static void
		2620	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2621	{
		2622	uint32_t old_write_domain;
2332	Serge	2623
2344	Serge	2624	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2625	return;
2332	Serge	2626
2344	Serge	2627	/* No actual flushing is required for the GTT write domain. Writes
		2628	* to it immediately go to main memory as far as we know, so there's
		2629	* no chipset flush. It also doesn't land in render cache.
		2630	*
		2631	* However, we do have to enforce the order so that all writes through
		2632	* the GTT land before any writes to the device, such as updates to
		2633	* the GATT itself.
		2634	*/
		2635	wmb();
2332	Serge	2636
2344	Serge	2637	old_write_domain = obj->base.write_domain;
		2638	obj->base.write_domain = 0;
2332	Serge	2639
2351	Serge	2640	trace_i915_gem_object_change_domain(obj,
		2641	obj->base.read_domains,
		2642	old_write_domain);
2344	Serge	2643	}
2332	Serge	2644
		2645	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2646	static void
2332	Serge	2647	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2648	{
		2649	uint32_t old_write_domain;
		2650
		2651	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2652	return;
		2653
		2654	i915_gem_clflush_object(obj);
		2655	intel_gtt_chipset_flush();
		2656	old_write_domain = obj->base.write_domain;
		2657	obj->base.write_domain = 0;
		2658
2351	Serge	2659	trace_i915_gem_object_change_domain(obj,
		2660	obj->base.read_domains,
		2661	old_write_domain);
2332	Serge	2662	}
		2663
		2664	/**
		2665	* Moves a single object to the GTT read, and possibly write domain.
		2666	*
		2667	* This function returns when the move is complete, including waiting on
		2668	* flushes to occur.
		2669	*/
		2670	int
		2671	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2672	{
3031	serge	2673	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2674	uint32_t old_write_domain, old_read_domains;
		2675	int ret;
		2676
		2677	/* Not valid to be called on unbound objects. */
		2678	if (obj->gtt_space == NULL)
		2679	return -EINVAL;
		2680
		2681	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2682	return 0;
		2683
3031	serge	2684	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2685	if (ret)
		2686	return ret;
		2687
		2688	i915_gem_object_flush_cpu_write_domain(obj);
		2689
		2690	old_write_domain = obj->base.write_domain;
		2691	old_read_domains = obj->base.read_domains;
		2692
		2693	/* It should now be out of any other write domains, and we can update
		2694	* the domain values for our changes.
		2695	*/
		2696	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2697	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2698	if (write) {
		2699	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2700	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2701	obj->dirty = 1;
		2702	}
		2703
2351	Serge	2704	trace_i915_gem_object_change_domain(obj,
		2705	old_read_domains,
		2706	old_write_domain);
		2707
3031	serge	2708	/* And bump the LRU for this access */
		2709	if (i915_gem_object_is_inactive(obj))
		2710	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2711
2332	Serge	2712	return 0;
		2713	}
		2714
2335	Serge	2715	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2716	enum i915_cache_level cache_level)
		2717	{
3031	serge	2718	struct drm_device *dev = obj->base.dev;
		2719	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2720	int ret;
2332	Serge	2721
2335	Serge	2722	if (obj->cache_level == cache_level)
		2723	return 0;
2332	Serge	2724
2335	Serge	2725	if (obj->pin_count) {
		2726	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2727	return -EBUSY;
		2728	}
2332	Serge	2729
3031	serge	2730	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2731	ret = i915_gem_object_unbind(obj);
		2732	if (ret)
		2733	return ret;
		2734	}
		2735
2335	Serge	2736	if (obj->gtt_space) {
		2737	ret = i915_gem_object_finish_gpu(obj);
		2738	if (ret)
		2739	return ret;
2332	Serge	2740
2335	Serge	2741	i915_gem_object_finish_gtt(obj);
2332	Serge	2742
2335	Serge	2743	/* Before SandyBridge, you could not use tiling or fence
		2744	* registers with snooped memory, so relinquish any fences
		2745	* currently pointing to our region in the aperture.
		2746	*/
3031	serge	2747	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2748	ret = i915_gem_object_put_fence(obj);
		2749	if (ret)
		2750	return ret;
		2751	}
2332	Serge	2752
3031	serge	2753	if (obj->has_global_gtt_mapping)
		2754	i915_gem_gtt_bind_object(obj, cache_level);
		2755	if (obj->has_aliasing_ppgtt_mapping)
		2756	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2757	obj, cache_level);
		2758
		2759	obj->gtt_space->color = cache_level;
2335	Serge	2760	}
2332	Serge	2761
2335	Serge	2762	if (cache_level == I915_CACHE_NONE) {
		2763	u32 old_read_domains, old_write_domain;
2332	Serge	2764
2335	Serge	2765	/* If we're coming from LLC cached, then we haven't
		2766	* actually been tracking whether the data is in the
		2767	* CPU cache or not, since we only allow one bit set
		2768	* in obj->write_domain and have been skipping the clflushes.
		2769	* Just set it to the CPU cache for now.
		2770	*/
		2771	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2772	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2773
2335	Serge	2774	old_read_domains = obj->base.read_domains;
		2775	old_write_domain = obj->base.write_domain;
2332	Serge	2776
2335	Serge	2777	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2778	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2779
2351	Serge	2780	trace_i915_gem_object_change_domain(obj,
		2781	old_read_domains,
		2782	old_write_domain);
2344	Serge	2783	}
2332	Serge	2784
2335	Serge	2785	obj->cache_level = cache_level;
3031	serge	2786	i915_gem_verify_gtt(dev);
2335	Serge	2787	return 0;
		2788	}
2332	Serge	2789
2335	Serge	2790	/*
		2791	* Prepare buffer for display plane (scanout, cursors, etc).
		2792	* Can be called from an uninterruptible phase (modesetting) and allows
		2793	* any flushes to be pipelined (for pageflips).
		2794	*/
		2795	int
		2796	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		2797	u32 alignment,
		2798	struct intel_ring_buffer *pipelined)
		2799	{
		2800	u32 old_read_domains, old_write_domain;
		2801	int ret;
2332	Serge	2802
3031	serge	2803	if (pipelined != obj->ring) {
		2804	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	2805	if (ret)
		2806	return ret;
		2807	}
2332	Serge	2808
2335	Serge	2809	/* The display engine is not coherent with the LLC cache on gen6. As
		2810	* a result, we make sure that the pinning that is about to occur is
		2811	* done with uncached PTEs. This is lowest common denominator for all
		2812	* chipsets.
		2813	*
		2814	* However for gen6+, we could do better by using the GFDT bit instead
		2815	* of uncaching, which would allow us to flush all the LLC-cached data
		2816	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		2817	*/
2360	Serge	2818	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		2819	if (ret)
		2820	return ret;
2332	Serge	2821
2335	Serge	2822	/* As the user may map the buffer once pinned in the display plane
		2823	* (e.g. libkms for the bootup splash), we have to ensure that we
		2824	* always use map_and_fenceable for all scanout buffers.
		2825	*/
3031	serge	2826	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	2827	if (ret)
		2828	return ret;
2332	Serge	2829
2335	Serge	2830	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	2831
2335	Serge	2832	old_write_domain = obj->base.write_domain;
		2833	old_read_domains = obj->base.read_domains;
2332	Serge	2834
2335	Serge	2835	/* It should now be out of any other write domains, and we can update
		2836	* the domain values for our changes.
		2837	*/
3031	serge	2838	obj->base.write_domain = 0;
2335	Serge	2839	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	2840
2351	Serge	2841	trace_i915_gem_object_change_domain(obj,
		2842	old_read_domains,
		2843	old_write_domain);
2332	Serge	2844
2335	Serge	2845	return 0;
		2846	}
2332	Serge	2847
2344	Serge	2848	int
		2849	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		2850	{
		2851	int ret;
2332	Serge	2852
2344	Serge	2853	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		2854	return 0;
2332	Serge	2855
3031	serge	2856	ret = i915_gem_object_wait_rendering(obj, false);
2344	Serge	2857	if (ret)
		2858	return ret;
2332	Serge	2859
2344	Serge	2860	/* Ensure that we invalidate the GPU's caches and TLBs. */
		2861	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	2862	return 0;
2344	Serge	2863	}
2332	Serge	2864
2344	Serge	2865	/**
		2866	* Moves a single object to the CPU read, and possibly write domain.
		2867	*
		2868	* This function returns when the move is complete, including waiting on
		2869	* flushes to occur.
		2870	*/
3031	serge	2871	int
2344	Serge	2872	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		2873	{
		2874	uint32_t old_write_domain, old_read_domains;
		2875	int ret;
2332	Serge	2876
2344	Serge	2877	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		2878	return 0;
2332	Serge	2879
3031	serge	2880	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	2881	if (ret)
		2882	return ret;
2332	Serge	2883
2344	Serge	2884	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	2885
2344	Serge	2886	old_write_domain = obj->base.write_domain;
		2887	old_read_domains = obj->base.read_domains;
2332	Serge	2888
2344	Serge	2889	/* Flush the CPU cache if it's still invalid. */
		2890	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		2891	i915_gem_clflush_object(obj);
2332	Serge	2892
2344	Serge	2893	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		2894	}
2332	Serge	2895
2344	Serge	2896	/* It should now be out of any other write domains, and we can update
		2897	* the domain values for our changes.
		2898	*/
		2899	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	2900
2344	Serge	2901	/* If we're writing through the CPU, then the GPU read domains will
		2902	* need to be invalidated at next use.
		2903	*/
		2904	if (write) {
		2905	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2906	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2907	}
2332	Serge	2908
2351	Serge	2909	trace_i915_gem_object_change_domain(obj,
		2910	old_read_domains,
		2911	old_write_domain);
2332	Serge	2912
2344	Serge	2913	return 0;
		2914	}
2332	Serge	2915
3031	serge	2916	#if 0
		2917	/* Throttle our rendering by waiting until the ring has completed our requests
		2918	* emitted over 20 msec ago.
2344	Serge	2919	*
3031	serge	2920	* Note that if we were to use the current jiffies each time around the loop,
		2921	* we wouldn't escape the function with any frames outstanding if the time to
		2922	* render a frame was over 20ms.
		2923	*
		2924	* This should get us reasonable parallelism between CPU and GPU but also
		2925	* relatively low latency when blocking on a particular request to finish.
2344	Serge	2926	*/
3031	serge	2927	static int
		2928	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	2929	{
3031	serge	2930	struct drm_i915_private *dev_priv = dev->dev_private;
		2931	struct drm_i915_file_private *file_priv = file->driver_priv;
		2932	unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20);
		2933	struct drm_i915_gem_request *request;
		2934	struct intel_ring_buffer *ring = NULL;
		2935	u32 seqno = 0;
		2936	int ret;
2332	Serge	2937
3031	serge	2938	if (atomic_read(&dev_priv->mm.wedged))
		2939	return -EIO;
2332	Serge	2940
3031	serge	2941	spin_lock(&file_priv->mm.lock);
		2942	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		2943	if (time_after_eq(request->emitted_jiffies, recent_enough))
		2944	break;
2332	Serge	2945
3031	serge	2946	ring = request->ring;
		2947	seqno = request->seqno;
		2948	}
		2949	spin_unlock(&file_priv->mm.lock);
2332	Serge	2950
3031	serge	2951	if (seqno == 0)
		2952	return 0;
2332	Serge	2953
3031	serge	2954	ret = __wait_seqno(ring, seqno, true, NULL);
		2955	if (ret == 0)
		2956	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	2957
3031	serge	2958	return ret;
2352	Serge	2959	}
3031	serge	2960	#endif
2332	Serge	2961
		2962	int
		2963	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		2964	uint32_t alignment,
3031	serge	2965	bool map_and_fenceable,
		2966	bool nonblocking)
2332	Serge	2967	{
		2968	int ret;
		2969
3031	serge	2970	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		2971	return -EBUSY;
2332	Serge	2972
		2973	#if 0
		2974	if (obj->gtt_space != NULL) {
		2975	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		2976	(map_and_fenceable && !obj->map_and_fenceable)) {
		2977	WARN(obj->pin_count,
		2978	"bo is already pinned with incorrect alignment:"
		2979	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		2980	" obj->map_and_fenceable=%d\n",
		2981	obj->gtt_offset, alignment,
		2982	map_and_fenceable,
		2983	obj->map_and_fenceable);
		2984	ret = i915_gem_object_unbind(obj);
		2985	if (ret)
		2986	return ret;
		2987	}
		2988	}
		2989	#endif
		2990
		2991	if (obj->gtt_space == NULL) {
		2992	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	2993	map_and_fenceable,
		2994	nonblocking);
2332	Serge	2995	if (ret)
		2996	return ret;
		2997	}
		2998
3031	serge	2999	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3000	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3001
		3002	obj->pin_count++;
2332	Serge	3003	obj->pin_mappable \|= map_and_fenceable;
		3004
		3005	return 0;
		3006	}
		3007
2344	Serge	3008	void
		3009	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3010	{
		3011	BUG_ON(obj->pin_count == 0);
		3012	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3013
3031	serge	3014	if (--obj->pin_count == 0)
2344	Serge	3015	obj->pin_mappable = false;
		3016	}
2332	Serge	3017
3031	serge	3018	#if 0
		3019	int
		3020	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3021	struct drm_file *file)
		3022	{
		3023	struct drm_i915_gem_pin *args = data;
		3024	struct drm_i915_gem_object *obj;
		3025	int ret;
2332	Serge	3026
3031	serge	3027	ret = i915_mutex_lock_interruptible(dev);
		3028	if (ret)
		3029	return ret;
2332	Serge	3030
3031	serge	3031	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3032	if (&obj->base == NULL) {
		3033	ret = -ENOENT;
		3034	goto unlock;
		3035	}
2332	Serge	3036
3031	serge	3037	if (obj->madv != I915_MADV_WILLNEED) {
		3038	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3039	ret = -EINVAL;
		3040	goto out;
		3041	}
2332	Serge	3042
3031	serge	3043	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3044	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3045	args->handle);
		3046	ret = -EINVAL;
		3047	goto out;
		3048	}
2332	Serge	3049
3031	serge	3050	obj->user_pin_count++;
		3051	obj->pin_filp = file;
		3052	if (obj->user_pin_count == 1) {
		3053	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3054	if (ret)
		3055	goto out;
		3056	}
2332	Serge	3057
3031	serge	3058	/* XXX - flush the CPU caches for pinned objects
		3059	* as the X server doesn't manage domains yet
		3060	*/
		3061	i915_gem_object_flush_cpu_write_domain(obj);
		3062	args->offset = obj->gtt_offset;
		3063	out:
		3064	drm_gem_object_unreference(&obj->base);
		3065	unlock:
		3066	mutex_unlock(&dev->struct_mutex);
		3067	return ret;
		3068	}
2332	Serge	3069
3031	serge	3070	int
		3071	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3072	struct drm_file *file)
		3073	{
		3074	struct drm_i915_gem_pin *args = data;
		3075	struct drm_i915_gem_object *obj;
		3076	int ret;
2332	Serge	3077
3031	serge	3078	ret = i915_mutex_lock_interruptible(dev);
		3079	if (ret)
		3080	return ret;
2332	Serge	3081
3031	serge	3082	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3083	if (&obj->base == NULL) {
		3084	ret = -ENOENT;
		3085	goto unlock;
		3086	}
2332	Serge	3087
3031	serge	3088	if (obj->pin_filp != file) {
		3089	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3090	args->handle);
		3091	ret = -EINVAL;
		3092	goto out;
		3093	}
		3094	obj->user_pin_count--;
		3095	if (obj->user_pin_count == 0) {
		3096	obj->pin_filp = NULL;
		3097	i915_gem_object_unpin(obj);
		3098	}
2332	Serge	3099
3031	serge	3100	out:
		3101	drm_gem_object_unreference(&obj->base);
		3102	unlock:
		3103	mutex_unlock(&dev->struct_mutex);
		3104	return ret;
		3105	}
2332	Serge	3106
3031	serge	3107	int
		3108	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3109	struct drm_file *file)
		3110	{
		3111	struct drm_i915_gem_busy *args = data;
		3112	struct drm_i915_gem_object *obj;
		3113	int ret;
2332	Serge	3114
3031	serge	3115	ret = i915_mutex_lock_interruptible(dev);
		3116	if (ret)
		3117	return ret;
2332	Serge	3118
3031	serge	3119	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3120	if (&obj->base == NULL) {
		3121	ret = -ENOENT;
		3122	goto unlock;
		3123	}
2332	Serge	3124
3031	serge	3125	/* Count all active objects as busy, even if they are currently not used
		3126	* by the gpu. Users of this interface expect objects to eventually
		3127	* become non-busy without any further actions, therefore emit any
		3128	* necessary flushes here.
		3129	*/
		3130	ret = i915_gem_object_flush_active(obj);
2332	Serge	3131
3031	serge	3132	args->busy = obj->active;
		3133	if (obj->ring) {
		3134	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3135	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3136	}
2332	Serge	3137
3031	serge	3138	drm_gem_object_unreference(&obj->base);
		3139	unlock:
		3140	mutex_unlock(&dev->struct_mutex);
		3141	return ret;
		3142	}
2332	Serge	3143
3031	serge	3144	int
		3145	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3146	struct drm_file *file_priv)
		3147	{
		3148	return i915_gem_ring_throttle(dev, file_priv);
		3149	}
2332	Serge	3150
3031	serge	3151	int
		3152	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3153	struct drm_file *file_priv)
		3154	{
		3155	struct drm_i915_gem_madvise *args = data;
		3156	struct drm_i915_gem_object *obj;
		3157	int ret;
2332	Serge	3158
3031	serge	3159	switch (args->madv) {
		3160	case I915_MADV_DONTNEED:
		3161	case I915_MADV_WILLNEED:
		3162	break;
		3163	default:
		3164	return -EINVAL;
		3165	}
2332	Serge	3166
3031	serge	3167	ret = i915_mutex_lock_interruptible(dev);
		3168	if (ret)
		3169	return ret;
2332	Serge	3170
3031	serge	3171	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3172	if (&obj->base == NULL) {
		3173	ret = -ENOENT;
		3174	goto unlock;
		3175	}
2332	Serge	3176
3031	serge	3177	if (obj->pin_count) {
		3178	ret = -EINVAL;
		3179	goto out;
		3180	}
2332	Serge	3181
3031	serge	3182	if (obj->madv != __I915_MADV_PURGED)
		3183	obj->madv = args->madv;
2332	Serge	3184
3031	serge	3185	/* if the object is no longer attached, discard its backing storage */
		3186	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3187	i915_gem_object_truncate(obj);
2332	Serge	3188
3031	serge	3189	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3190
3031	serge	3191	out:
		3192	drm_gem_object_unreference(&obj->base);
		3193	unlock:
		3194	mutex_unlock(&dev->struct_mutex);
		3195	return ret;
		3196	}
		3197	#endif
2332	Serge	3198
3031	serge	3199	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3200	const struct drm_i915_gem_object_ops *ops)
		3201	{
		3202	INIT_LIST_HEAD(&obj->mm_list);
		3203	INIT_LIST_HEAD(&obj->gtt_list);
		3204	INIT_LIST_HEAD(&obj->ring_list);
		3205	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3206
3031	serge	3207	obj->ops = ops;
		3208
		3209	obj->fence_reg = I915_FENCE_REG_NONE;
		3210	obj->madv = I915_MADV_WILLNEED;
		3211	/* Avoid an unnecessary call to unbind on the first bind. */
		3212	obj->map_and_fenceable = true;
		3213
		3214	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3215	}
		3216
		3217	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3218	.get_pages = i915_gem_object_get_pages_gtt,
		3219	.put_pages = i915_gem_object_put_pages_gtt,
		3220	};
		3221
2332	Serge	3222	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3223	size_t size)
		3224	{
		3225	struct drm_i915_gem_object *obj;
3031	serge	3226	struct address_space *mapping;
		3227	u32 mask;
2340	Serge	3228
2332	Serge	3229	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3230	if (obj == NULL)
		3231	return NULL;
		3232
		3233	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3234	kfree(obj);
		3235	return NULL;
		3236	}
		3237
		3238
3031	serge	3239	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3240
		3241	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3242	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3243
3031	serge	3244	if (HAS_LLC(dev)) {
		3245	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3246	* cache) for about a 10% performance improvement
		3247	* compared to uncached. Graphics requests other than
		3248	* display scanout are coherent with the CPU in
		3249	* accessing this cache. This means in this mode we
		3250	* don't need to clflush on the CPU side, and on the
		3251	* GPU side we only need to flush internal caches to
		3252	* get data visible to the CPU.
		3253	*
		3254	* However, we maintain the display planes as UC, and so
		3255	* need to rebind when first used as such.
		3256	*/
		3257	obj->cache_level = I915_CACHE_LLC;
		3258	} else
		3259	obj->cache_level = I915_CACHE_NONE;
		3260
		3261	return obj;
		3262	}
		3263
2344	Serge	3264	int i915_gem_init_object(struct drm_gem_object *obj)
		3265	{
		3266	BUG();
2332	Serge	3267
2344	Serge	3268	return 0;
		3269	}
2332	Serge	3270
3031	serge	3271	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3272	{
3031	serge	3273	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3274	struct drm_device *dev = obj->base.dev;
		3275	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3276
3031	serge	3277	trace_i915_gem_object_destroy(obj);
		3278
		3279	// if (obj->phys_obj)
		3280	// i915_gem_detach_phys_object(dev, obj);
		3281
		3282	obj->pin_count = 0;
		3283	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3284	bool was_interruptible;
		3285
		3286	was_interruptible = dev_priv->mm.interruptible;
		3287	dev_priv->mm.interruptible = false;
		3288
		3289	WARN_ON(i915_gem_object_unbind(obj));
		3290
		3291	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3292	}
2332	Serge	3293
3031	serge	3294	obj->pages_pin_count = 0;
		3295	i915_gem_object_put_pages(obj);
		3296	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3297
3031	serge	3298	BUG_ON(obj->pages.page);
2332	Serge	3299
3031	serge	3300	// if (obj->base.import_attach)
		3301	// drm_prime_gem_destroy(&obj->base, NULL);
		3302
2344	Serge	3303	drm_gem_object_release(&obj->base);
		3304	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3305
2344	Serge	3306	kfree(obj->bit_17);
		3307	kfree(obj);
		3308	}
2332	Serge	3309
3031	serge	3310	#if 0
		3311	int
		3312	i915_gem_idle(struct drm_device *dev)
2344	Serge	3313	{
3031	serge	3314	drm_i915_private_t *dev_priv = dev->dev_private;
		3315	int ret;
2332	Serge	3316
3031	serge	3317	mutex_lock(&dev->struct_mutex);
2332	Serge	3318
3031	serge	3319	if (dev_priv->mm.suspended) {
		3320	mutex_unlock(&dev->struct_mutex);
		3321	return 0;
		3322	}
2332	Serge	3323
3031	serge	3324	ret = i915_gpu_idle(dev);
		3325	if (ret) {
		3326	mutex_unlock(&dev->struct_mutex);
		3327	return ret;
		3328	}
		3329	i915_gem_retire_requests(dev);
		3330
		3331	i915_gem_reset_fences(dev);
		3332
		3333	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3334	* We need to replace this with a semaphore, or something.
		3335	* And not confound mm.suspended!
		3336	*/
		3337	dev_priv->mm.suspended = 1;
		3338	del_timer_sync(&dev_priv->hangcheck_timer);
		3339
		3340	i915_kernel_lost_context(dev);
		3341	i915_gem_cleanup_ringbuffer(dev);
		3342
		3343	mutex_unlock(&dev->struct_mutex);
		3344
		3345	/* Cancel the retire work handler, which should be idle now. */
		3346	// cancel_delayed_work_sync(&dev_priv->mm.retire_work);
		3347
		3348	return 0;
2344	Serge	3349	}
3031	serge	3350	#endif
2332	Serge	3351
3031	serge	3352	void i915_gem_l3_remap(struct drm_device *dev)
		3353	{
		3354	drm_i915_private_t *dev_priv = dev->dev_private;
		3355	u32 misccpctl;
		3356	int i;
2332	Serge	3357
3031	serge	3358	if (!IS_IVYBRIDGE(dev))
		3359	return;
2332	Serge	3360
3031	serge	3361	if (!dev_priv->mm.l3_remap_info)
		3362	return;
2332	Serge	3363
3031	serge	3364	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3365	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3366	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3367
3031	serge	3368	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3369	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
		3370	if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
		3371	DRM_DEBUG("0x%x was already programmed to %x\n",
		3372	GEN7_L3LOG_BASE + i, remap);
		3373	if (remap && !dev_priv->mm.l3_remap_info[i/4])
		3374	DRM_DEBUG_DRIVER("Clearing remapped register\n");
		3375	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
		3376	}
2332	Serge	3377
3031	serge	3378	/* Make sure all the writes land before disabling dop clock gating */
		3379	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3380
3031	serge	3381	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3382	}
2332	Serge	3383
3031	serge	3384	void i915_gem_init_swizzling(struct drm_device *dev)
		3385	{
		3386	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3387
3031	serge	3388	if (INTEL_INFO(dev)->gen < 5 \|\|
		3389	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3390	return;
2332	Serge	3391
3031	serge	3392	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3393	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3394
3031	serge	3395	if (IS_GEN5(dev))
		3396	return;
2344	Serge	3397
3031	serge	3398	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3399	if (IS_GEN6(dev))
		3400	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3401	else
		3402	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3403	}
		3404
		3405	void i915_gem_init_ppgtt(struct drm_device *dev)
		3406	{
		3407	drm_i915_private_t *dev_priv = dev->dev_private;
		3408	uint32_t pd_offset;
		3409	struct intel_ring_buffer *ring;
		3410	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
		3411	uint32_t __iomem *pd_addr;
		3412	uint32_t pd_entry;
		3413	int i;
		3414
		3415	if (!dev_priv->mm.aliasing_ppgtt)
		3416	return;
		3417
		3418
		3419	pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
		3420	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		3421	dma_addr_t pt_addr;
		3422
		3423	if (dev_priv->mm.gtt->needs_dmar)
		3424	pt_addr = ppgtt->pt_dma_addr[i];
		3425	else
		3426	pt_addr = ppgtt->pt_pages[i];
		3427
		3428	pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
		3429	pd_entry \|= GEN6_PDE_VALID;
		3430
		3431	writel(pd_entry, pd_addr + i);
		3432	}
		3433	readl(pd_addr);
		3434
		3435	pd_offset = ppgtt->pd_offset;
		3436	pd_offset /= 64; /* in cachelines, */
		3437	pd_offset <<= 16;
		3438
		3439	if (INTEL_INFO(dev)->gen == 6) {
		3440	uint32_t ecochk, gab_ctl, ecobits;
		3441
		3442	ecobits = I915_READ(GAC_ECO_BITS);
		3443	I915_WRITE(GAC_ECO_BITS, ecobits \| ECOBITS_PPGTT_CACHE64B);
		3444
		3445	gab_ctl = I915_READ(GAB_CTL);
		3446	I915_WRITE(GAB_CTL, gab_ctl \| GAB_CTL_CONT_AFTER_PAGEFAULT);
		3447
		3448	ecochk = I915_READ(GAM_ECOCHK);
		3449	I915_WRITE(GAM_ECOCHK, ecochk \| ECOCHK_SNB_BIT \|
		3450	ECOCHK_PPGTT_CACHE64B);
		3451	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
		3452	} else if (INTEL_INFO(dev)->gen >= 7) {
		3453	I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
		3454	/* GFX_MODE is per-ring on gen7+ */
		3455	}
		3456
		3457	for_each_ring(ring, dev_priv, i) {
		3458	if (INTEL_INFO(dev)->gen >= 7)
		3459	I915_WRITE(RING_MODE_GEN7(ring),
		3460	_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
		3461
		3462	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
		3463	I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
		3464	}
		3465	}
		3466
		3467	static bool
		3468	intel_enable_blt(struct drm_device *dev)
		3469	{
		3470	if (!HAS_BLT(dev))
		3471	return false;
		3472
		3473	/* The blitter was dysfunctional on early prototypes */
		3474	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3475	DRM_INFO("BLT not supported on this pre-production hardware;"
		3476	" graphics performance will be degraded.\n");
		3477	return false;
		3478	}
		3479
		3480	return true;
		3481	}
		3482
2332	Serge	3483	int
3031	serge	3484	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3485	{
		3486	drm_i915_private_t *dev_priv = dev->dev_private;
		3487	int ret;
2351	Serge	3488
3031	serge	3489	if (!intel_enable_gtt())
		3490	return -EIO;
		3491
		3492	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3493	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3494
		3495	i915_gem_l3_remap(dev);
		3496
		3497	i915_gem_init_swizzling(dev);
		3498
2332	Serge	3499	ret = intel_init_render_ring_buffer(dev);
		3500	if (ret)
		3501	return ret;
		3502
		3503	if (HAS_BSD(dev)) {
		3504	ret = intel_init_bsd_ring_buffer(dev);
		3505	if (ret)
		3506	goto cleanup_render_ring;
		3507	}
		3508
3031	serge	3509	if (intel_enable_blt(dev)) {
2332	Serge	3510	ret = intel_init_blt_ring_buffer(dev);
		3511	if (ret)
		3512	goto cleanup_bsd_ring;
		3513	}
		3514
		3515	dev_priv->next_seqno = 1;
2351	Serge	3516
3031	serge	3517	/*
		3518	* XXX: There was some w/a described somewhere suggesting loading
		3519	* contexts before PPGTT.
		3520	*/
		3521	i915_gem_context_init(dev);
		3522	i915_gem_init_ppgtt(dev);
		3523
2332	Serge	3524	return 0;
		3525
		3526	cleanup_bsd_ring:
		3527	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3528	cleanup_render_ring:
		3529	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3530	return ret;
		3531	}
		3532
3031	serge	3533	static bool
		3534	intel_enable_ppgtt(struct drm_device *dev)
		3535	{
		3536	if (i915_enable_ppgtt >= 0)
		3537	return i915_enable_ppgtt;
		3538
		3539	#ifdef CONFIG_INTEL_IOMMU
		3540	/* Disable ppgtt on SNB if VT-d is on. */
		3541	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3542	return false;
		3543	#endif
		3544
		3545	return true;
		3546	}
		3547
		3548	#define LFB_SIZE 0xC00000
		3549
		3550	int i915_gem_init(struct drm_device *dev)
		3551	{
		3552	struct drm_i915_private *dev_priv = dev->dev_private;
		3553	unsigned long gtt_size, mappable_size;
		3554	int ret;
		3555
		3556	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3557	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3558
		3559	mutex_lock(&dev->struct_mutex);
		3560	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3561	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3562	* aperture accordingly when using aliasing ppgtt. */
		3563	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3564
		3565	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3566
		3567	ret = i915_gem_init_aliasing_ppgtt(dev);
		3568	if (ret) {
		3569	mutex_unlock(&dev->struct_mutex);
		3570	return ret;
		3571	}
		3572	} else {
		3573	/* Let GEM Manage all of the aperture.
		3574	*
		3575	* However, leave one page at the end still bound to the scratch
		3576	* page. There are a number of places where the hardware
		3577	* apparently prefetches past the end of the object, and we've
		3578	* seen multiple hangs with the GPU head pointer stuck in a
		3579	* batchbuffer bound at the last page of the aperture. One page
		3580	* should be enough to keep any prefetching inside of the
		3581	* aperture.
		3582	*/
		3583	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3584	}
		3585
		3586	ret = i915_gem_init_hw(dev);
		3587	mutex_unlock(&dev->struct_mutex);
		3588	if (ret) {
		3589	i915_gem_cleanup_aliasing_ppgtt(dev);
		3590	return ret;
		3591	}
		3592
		3593	return 0;
		3594	}
		3595
2332	Serge	3596	void
		3597	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3598	{
		3599	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3600	struct intel_ring_buffer *ring;
2332	Serge	3601	int i;
		3602
3031	serge	3603	for_each_ring(ring, dev_priv, i)
		3604	intel_cleanup_ring_buffer(ring);
2332	Serge	3605	}
		3606
3031	serge	3607	#if 0
		3608
2332	Serge	3609	int
		3610	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3611	struct drm_file *file_priv)
		3612	{
		3613	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3614	int ret;
2332	Serge	3615
		3616	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3617	return 0;
		3618
		3619	if (atomic_read(&dev_priv->mm.wedged)) {
		3620	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3621	atomic_set(&dev_priv->mm.wedged, 0);
		3622	}
		3623
		3624	mutex_lock(&dev->struct_mutex);
		3625	dev_priv->mm.suspended = 0;
		3626
3031	serge	3627	ret = i915_gem_init_hw(dev);
2332	Serge	3628	if (ret != 0) {
		3629	mutex_unlock(&dev->struct_mutex);
		3630	return ret;
		3631	}
		3632
		3633	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3634	mutex_unlock(&dev->struct_mutex);
		3635
		3636	ret = drm_irq_install(dev);
		3637	if (ret)
		3638	goto cleanup_ringbuffer;
		3639
		3640	return 0;
		3641
		3642	cleanup_ringbuffer:
		3643	mutex_lock(&dev->struct_mutex);
		3644	i915_gem_cleanup_ringbuffer(dev);
		3645	dev_priv->mm.suspended = 1;
		3646	mutex_unlock(&dev->struct_mutex);
		3647
		3648	return ret;
		3649	}
		3650
		3651	int
		3652	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3653	struct drm_file *file_priv)
		3654	{
		3655	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3656	return 0;
		3657
		3658	drm_irq_uninstall(dev);
		3659	return i915_gem_idle(dev);
		3660	}
		3661
		3662	void
		3663	i915_gem_lastclose(struct drm_device *dev)
		3664	{
		3665	int ret;
		3666
		3667	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3668	return;
		3669
		3670	ret = i915_gem_idle(dev);
		3671	if (ret)
		3672	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3673	}
		3674	#endif
		3675
		3676	static void
2326	Serge	3677	init_ring_lists(struct intel_ring_buffer *ring)
		3678	{
		3679	INIT_LIST_HEAD(&ring->active_list);
		3680	INIT_LIST_HEAD(&ring->request_list);
		3681	}
		3682
		3683	void
		3684	i915_gem_load(struct drm_device *dev)
		3685	{
		3686	int i;
		3687	drm_i915_private_t *dev_priv = dev->dev_private;
		3688
		3689	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3690	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3691	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3692	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3693	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3694	for (i = 0; i < I915_NUM_RINGS; i++)
		3695	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3696	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3697	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3698	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3699	i915_gem_retire_work_handler);
2326	Serge	3700
		3701	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3702	if (IS_GEN3(dev)) {
3031	serge	3703	I915_WRITE(MI_ARB_STATE,
		3704	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3705	}
		3706
		3707	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3708
		3709	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3710	dev_priv->num_fence_regs = 16;
		3711	else
		3712	dev_priv->num_fence_regs = 8;
		3713
		3714	/* Initialize fence registers to zero */
3031	serge	3715	i915_gem_reset_fences(dev);
2326	Serge	3716
		3717	i915_gem_detect_bit_6_swizzle(dev);
		3718
		3719	dev_priv->mm.interruptible = true;
		3720
		3721	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3722	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3723	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3724	}
		3725
		3726

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3039