WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
		33	//#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
2344	Serge	40	#undef mb
		41	#undef rmb
		42	#undef wmb
		43	#define mb() asm volatile("mfence")
		44	#define rmb() asm volatile ("lfence")
		45	#define wmb() asm volatile ("sfence")
		46
		47	static inline void clflush(volatile void *__p)
		48	{
		49	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		50	}
		51
2332	Serge	52	#define MAX_ERRNO 4095
		53
		54	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		55
		56	static inline long IS_ERR(const void *ptr)
		57	{
		58	return IS_ERR_VALUE((unsigned long)ptr);
		59	}
		60
		61	static inline void *ERR_PTR(long error)
		62	{
		63	return (void *) error;
		64	}
		65
		66	static inline long PTR_ERR(const void *ptr)
		67	{
		68	return (long) ptr;
		69	}
		70
2344	Serge	71	void
		72	drm_gem_object_free(struct kref *kref)
		73	{
		74	struct drm_gem_object obj = (struct drm_gem_object ) kref;
		75	struct drm_device *dev = obj->dev;
2332	Serge	76
2344	Serge	77	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		78
		79	i915_gem_free_object(obj);
		80	}
		81
2332	Serge	82	/**
		83	* Initialize an already allocated GEM object of the specified size with
		84	* shmfs backing store.
		85	*/
		86	int drm_gem_object_init(struct drm_device *dev,
		87	struct drm_gem_object *obj, size_t size)
		88	{
		89	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		90
		91	obj->dev = dev;
2344	Serge	92	kref_init(&obj->refcount);
2332	Serge	93	atomic_set(&obj->handle_count, 0);
		94	obj->size = size;
		95
		96	return 0;
		97	}
		98
2344	Serge	99	void
		100	drm_gem_object_release(struct drm_gem_object *obj)
		101	{ }
2332	Serge	102
		103
2326	Serge	104	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		105	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		106	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		107	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		108
2332	Serge	109	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		110	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		111	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		112	unsigned alignment,
3031	serge	113	bool map_and_fenceable,
		114	bool nonblocking);
2332	Serge	115	static int i915_gem_phys_pwrite(struct drm_device *dev,
		116	struct drm_i915_gem_object *obj,
		117	struct drm_i915_gem_pwrite *args,
		118	struct drm_file *file);
2326	Serge	119
3031	serge	120	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		121	struct drm_i915_gem_object *obj);
		122	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		123	struct drm_i915_fence_reg *fence,
		124	bool enable);
2332	Serge	125
3031	serge	126	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		127	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		128	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		129
		130	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		131	{
		132	if (obj->tiling_mode)
		133	i915_gem_release_mmap(obj);
		134
		135	/* As we do not have an associated fence register, we will force
		136	* a tiling change if we ever need to acquire one.
		137	*/
		138	obj->fence_dirty = false;
		139	obj->fence_reg = I915_FENCE_REG_NONE;
		140	}
		141
2332	Serge	142	/* some bookkeeping */
		143	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		144	size_t size)
		145	{
		146	dev_priv->mm.object_count++;
		147	dev_priv->mm.object_memory += size;
		148	}
		149
		150	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		151	size_t size)
		152	{
		153	dev_priv->mm.object_count--;
		154	dev_priv->mm.object_memory -= size;
		155	}
		156
		157	#if 0
		158
		159	static int
		160	i915_gem_wait_for_error(struct drm_device *dev)
		161	{
		162	struct drm_i915_private *dev_priv = dev->dev_private;
		163	struct completion *x = &dev_priv->error_completion;
		164	unsigned long flags;
		165	int ret;
		166
		167	if (!atomic_read(&dev_priv->mm.wedged))
		168	return 0;
		169
3031	serge	170	/*
		171	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		172	* userspace. If it takes that long something really bad is going on and
		173	* we should simply try to bail out and fail as gracefully as possible.
		174	*/
		175	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		176	if (ret == 0) {
		177	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		178	return -EIO;
		179	} else if (ret < 0) {
2332	Serge	180	return ret;
3031	serge	181	}
2332	Serge	182
		183	if (atomic_read(&dev_priv->mm.wedged)) {
		184	/* GPU is hung, bump the completion count to account for
		185	* the token we just consumed so that we never hit zero and
		186	* end up waiting upon a subsequent completion event that
		187	* will never happen.
		188	*/
		189	spin_lock_irqsave(&x->wait.lock, flags);
		190	x->done++;
		191	spin_unlock_irqrestore(&x->wait.lock, flags);
		192	}
		193	return 0;
		194	}
		195
		196	int i915_mutex_lock_interruptible(struct drm_device *dev)
		197	{
		198	int ret;
		199
		200	ret = i915_gem_wait_for_error(dev);
		201	if (ret)
		202	return ret;
		203
		204	ret = mutex_lock_interruptible(&dev->struct_mutex);
		205	if (ret)
		206	return ret;
		207
		208	WARN_ON(i915_verify_lists(dev));
		209	return 0;
		210	}
2352	Serge	211	#endif
2332	Serge	212
		213	static inline bool
		214	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		215	{
3031	serge	216	return obj->gtt_space && !obj->active;
2332	Serge	217	}
		218
		219
		220	#if 0
		221
		222	int
		223	i915_gem_init_ioctl(struct drm_device dev, void data,
		224	struct drm_file *file)
		225	{
		226	struct drm_i915_gem_init *args = data;
		227
3031	serge	228	if (drm_core_check_feature(dev, DRIVER_MODESET))
		229	return -ENODEV;
		230
2332	Serge	231	if (args->gtt_start >= args->gtt_end \|\|
		232	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		233	return -EINVAL;
		234
3031	serge	235	/* GEM with user mode setting was never supported on ilk and later. */
		236	if (INTEL_INFO(dev)->gen >= 5)
		237	return -ENODEV;
		238
2332	Serge	239	mutex_lock(&dev->struct_mutex);
3031	serge	240	i915_gem_init_global_gtt(dev, args->gtt_start,
		241	args->gtt_end, args->gtt_end);
2332	Serge	242	mutex_unlock(&dev->struct_mutex);
		243
		244	return 0;
		245	}
2351	Serge	246	#endif
2332	Serge	247
		248	int
		249	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		250	struct drm_file *file)
		251	{
		252	struct drm_i915_private *dev_priv = dev->dev_private;
		253	struct drm_i915_gem_get_aperture *args = data;
		254	struct drm_i915_gem_object *obj;
		255	size_t pinned;
		256
		257	pinned = 0;
		258	mutex_lock(&dev->struct_mutex);
3031	serge	259	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		260	if (obj->pin_count)
2332	Serge	261	pinned += obj->gtt_space->size;
		262	mutex_unlock(&dev->struct_mutex);
		263
		264	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	265	args->aper_available_size = args->aper_size - pinned;
2332	Serge	266
		267	return 0;
		268	}
		269
2351	Serge	270	#if 0
3031	serge	271	static int
		272	i915_gem_create(struct drm_file *file,
2332	Serge	273	struct drm_device *dev,
		274	uint64_t size,
		275	uint32_t *handle_p)
		276	{
		277	struct drm_i915_gem_object *obj;
		278	int ret;
		279	u32 handle;
		280
		281	size = roundup(size, PAGE_SIZE);
2342	Serge	282	if (size == 0)
		283	return -EINVAL;
2332	Serge	284
		285	/* Allocate the new object */
		286	obj = i915_gem_alloc_object(dev, size);
		287	if (obj == NULL)
		288	return -ENOMEM;
		289
		290	ret = drm_gem_handle_create(file, &obj->base, &handle);
		291	if (ret) {
		292	drm_gem_object_release(&obj->base);
		293	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		294	kfree(obj);
		295	return ret;
		296	}
		297
		298	/* drop reference from allocate - handle holds it now */
		299	drm_gem_object_unreference(&obj->base);
2351	Serge	300	trace_i915_gem_object_create(obj);
2332	Serge	301
		302	*handle_p = handle;
		303	return 0;
		304	}
		305
		306	int
		307	i915_gem_dumb_create(struct drm_file *file,
		308	struct drm_device *dev,
		309	struct drm_mode_create_dumb *args)
		310	{
		311	/* have to work out size/pitch and return them */
		312	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		313	args->size = args->pitch * args->height;
		314	return i915_gem_create(file, dev,
		315	args->size, &args->handle);
		316	}
		317
		318	int i915_gem_dumb_destroy(struct drm_file *file,
		319	struct drm_device *dev,
		320	uint32_t handle)
		321	{
		322	return drm_gem_handle_delete(file, handle);
		323	}
		324
2326	Serge	325	/**
2332	Serge	326	* Creates a new mm object and returns a handle to it.
		327	*/
		328	int
		329	i915_gem_create_ioctl(struct drm_device dev, void data,
		330	struct drm_file *file)
		331	{
		332	struct drm_i915_gem_create *args = data;
3031	serge	333
2332	Serge	334	return i915_gem_create(file, dev,
		335	args->size, &args->handle);
		336	}
		337
		338	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		339	{
		340	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		341
		342	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		343	obj->tiling_mode != I915_TILING_NONE;
		344	}
		345
3031	serge	346	static inline int
		347	__copy_to_user_swizzled(char __user *cpu_vaddr,
		348	const char *gpu_vaddr, int gpu_offset,
2332	Serge	349	int length)
		350	{
3031	serge	351	int ret, cpu_offset = 0;
2332	Serge	352
3031	serge	353	while (length > 0) {
		354	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		355	int this_length = min(cacheline_end - gpu_offset, length);
		356	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	357
3031	serge	358	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		359	gpu_vaddr + swizzled_gpu_offset,
		360	this_length);
		361	if (ret)
		362	return ret + length;
2332	Serge	363
3031	serge	364	cpu_offset += this_length;
		365	gpu_offset += this_length;
		366	length -= this_length;
		367	}
		368
		369	return 0;
2332	Serge	370	}
		371
3031	serge	372	static inline int
		373	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		374	const char __user *cpu_vaddr,
		375	int length)
2332	Serge	376	{
3031	serge	377	int ret, cpu_offset = 0;
2332	Serge	378
		379	while (length > 0) {
		380	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		381	int this_length = min(cacheline_end - gpu_offset, length);
		382	int swizzled_gpu_offset = gpu_offset ^ 64;
		383
3031	serge	384	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	385	cpu_vaddr + cpu_offset,
		386	this_length);
3031	serge	387	if (ret)
		388	return ret + length;
		389
2332	Serge	390	cpu_offset += this_length;
		391	gpu_offset += this_length;
		392	length -= this_length;
		393	}
		394
3031	serge	395	return 0;
2332	Serge	396	}
		397
3031	serge	398	/* Per-page copy function for the shmem pread fastpath.
		399	* Flushes invalid cachelines before reading the target if
		400	* needs_clflush is set. */
2332	Serge	401	static int
3031	serge	402	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		403	char __user *user_data,
		404	bool page_do_bit17_swizzling, bool needs_clflush)
		405	{
		406	char *vaddr;
		407	int ret;
		408
		409	if (unlikely(page_do_bit17_swizzling))
		410	return -EINVAL;
		411
		412	vaddr = kmap_atomic(page);
		413	if (needs_clflush)
		414	drm_clflush_virt_range(vaddr + shmem_page_offset,
		415	page_length);
		416	ret = __copy_to_user_inatomic(user_data,
		417	vaddr + shmem_page_offset,
		418	page_length);
		419	kunmap_atomic(vaddr);
		420
		421	return ret ? -EFAULT : 0;
		422	}
		423
		424	static void
		425	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		426	bool swizzled)
		427	{
		428	if (unlikely(swizzled)) {
		429	unsigned long start = (unsigned long) addr;
		430	unsigned long end = (unsigned long) addr + length;
		431
		432	/* For swizzling simply ensure that we always flush both
		433	* channels. Lame, but simple and it works. Swizzled
		434	* pwrite/pread is far from a hotpath - current userspace
		435	* doesn't use it at all. */
		436	start = round_down(start, 128);
		437	end = round_up(end, 128);
		438
		439	drm_clflush_virt_range((void *)start, end - start);
		440	} else {
		441	drm_clflush_virt_range(addr, length);
		442	}
		443
		444	}
		445
		446	/* Only difference to the fast-path function is that this can handle bit17
		447	* and uses non-atomic copy and kmap functions. */
		448	static int
		449	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		450	char __user *user_data,
		451	bool page_do_bit17_swizzling, bool needs_clflush)
		452	{
		453	char *vaddr;
		454	int ret;
		455
		456	vaddr = kmap(page);
		457	if (needs_clflush)
		458	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		459	page_length,
		460	page_do_bit17_swizzling);
		461
		462	if (page_do_bit17_swizzling)
		463	ret = __copy_to_user_swizzled(user_data,
		464	vaddr, shmem_page_offset,
		465	page_length);
		466	else
		467	ret = __copy_to_user(user_data,
		468	vaddr + shmem_page_offset,
		469	page_length);
		470	kunmap(page);
		471
		472	return ret ? - EFAULT : 0;
		473	}
		474
		475	static int
		476	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	477	struct drm_i915_gem_object *obj,
		478	struct drm_i915_gem_pread *args,
		479	struct drm_file *file)
		480	{
3031	serge	481	char __user *user_data;
2332	Serge	482	ssize_t remain;
		483	loff_t offset;
3031	serge	484	int shmem_page_offset, page_length, ret = 0;
		485	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		486	int hit_slowpath = 0;
		487	int prefaulted = 0;
		488	int needs_clflush = 0;
		489	struct scatterlist *sg;
		490	int i;
2332	Serge	491
		492	user_data = (char __user *) (uintptr_t) args->data_ptr;
		493	remain = args->size;
		494
3031	serge	495	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		496
		497	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		498	/* If we're not in the cpu read domain, set ourself into the gtt
		499	* read domain and manually flush cachelines (if required). This
		500	* optimizes for the case when the gpu will dirty the data
		501	* anyway again before the next pread happens. */
		502	if (obj->cache_level == I915_CACHE_NONE)
		503	needs_clflush = 1;
		504	if (obj->gtt_space) {
		505	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		506	if (ret)
		507	return ret;
		508	}
		509	}
		510
		511	ret = i915_gem_object_get_pages(obj);
		512	if (ret)
		513	return ret;
		514
		515	i915_gem_object_pin_pages(obj);
		516
2332	Serge	517	offset = args->offset;
		518
3031	serge	519	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	520	struct page *page;
		521
3031	serge	522	if (i < offset >> PAGE_SHIFT)
		523	continue;
		524
		525	if (remain <= 0)
		526	break;
		527
2332	Serge	528	/* Operation in this page
		529	*
3031	serge	530	* shmem_page_offset = offset within page in shmem file
2332	Serge	531	* page_length = bytes to copy for this page
		532	*/
3031	serge	533	shmem_page_offset = offset_in_page(offset);
2332	Serge	534	page_length = remain;
3031	serge	535	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		536	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	537
3031	serge	538	page = sg_page(sg);
		539	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		540	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	541
3031	serge	542	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		543	user_data, page_do_bit17_swizzling,
		544	needs_clflush);
		545	if (ret == 0)
		546	goto next_page;
2332	Serge	547
3031	serge	548	hit_slowpath = 1;
		549	mutex_unlock(&dev->struct_mutex);
		550
		551	if (!prefaulted) {
		552	ret = fault_in_multipages_writeable(user_data, remain);
		553	/* Userspace is tricking us, but we've already clobbered
		554	* its pages with the prefault and promised to write the
		555	* data up to the first fault. Hence ignore any errors
		556	* and just continue. */
		557	(void)ret;
		558	prefaulted = 1;
		559	}
		560
		561	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		562	user_data, page_do_bit17_swizzling,
		563	needs_clflush);
		564
		565	mutex_lock(&dev->struct_mutex);
		566
		567	next_page:
2332	Serge	568	mark_page_accessed(page);
3031	serge	569
2332	Serge	570	if (ret)
3031	serge	571	goto out;
2332	Serge	572
		573	remain -= page_length;
		574	user_data += page_length;
		575	offset += page_length;
		576	}
		577
3031	serge	578	out:
		579	i915_gem_object_unpin_pages(obj);
		580
		581	if (hit_slowpath) {
		582	/* Fixup: Kill any reinstated backing storage pages */
		583	if (obj->madv == __I915_MADV_PURGED)
		584	i915_gem_object_truncate(obj);
		585	}
		586
		587	return ret;
2332	Serge	588	}
		589
		590	/**
3031	serge	591	* Reads data from the object referenced by handle.
		592	*
		593	* On error, the contents of *data are undefined.
2332	Serge	594	*/
3031	serge	595	int
		596	i915_gem_pread_ioctl(struct drm_device dev, void data,
		597	struct drm_file *file)
		598	{
		599	struct drm_i915_gem_pread *args = data;
		600	struct drm_i915_gem_object *obj;
		601	int ret = 0;
		602
		603	if (args->size == 0)
		604	return 0;
		605
		606	if (!access_ok(VERIFY_WRITE,
		607	(char __user *)(uintptr_t)args->data_ptr,
		608	args->size))
		609	return -EFAULT;
		610
		611	ret = i915_mutex_lock_interruptible(dev);
		612	if (ret)
		613	return ret;
		614
		615	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		616	if (&obj->base == NULL) {
		617	ret = -ENOENT;
		618	goto unlock;
		619	}
		620
		621	/* Bounds check source. */
		622	if (args->offset > obj->base.size \|\|
		623	args->size > obj->base.size - args->offset) {
		624	ret = -EINVAL;
		625	goto out;
		626	}
		627
		628	/* prime objects have no backing filp to GEM pread/pwrite
		629	* pages from.
		630	*/
		631	if (!obj->base.filp) {
		632	ret = -EINVAL;
		633	goto out;
		634	}
		635
		636	trace_i915_gem_object_pread(obj, args->offset, args->size);
		637
		638	ret = i915_gem_shmem_pread(dev, obj, args, file);
		639
		640	out:
		641	drm_gem_object_unreference(&obj->base);
		642	unlock:
		643	mutex_unlock(&dev->struct_mutex);
		644	return ret;
		645	}
		646
		647	/* This is the fast write path which cannot handle
		648	* page faults in the source data
		649	*/
		650
		651	static inline int
		652	fast_user_write(struct io_mapping *mapping,
		653	loff_t page_base, int page_offset,
		654	char __user *user_data,
		655	int length)
		656	{
		657	void __iomem *vaddr_atomic;
		658	void *vaddr;
		659	unsigned long unwritten;
		660
		661	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		662	/* We can use the cpu mem copy function because this is X86. */
		663	vaddr = (void __force*)vaddr_atomic + page_offset;
		664	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		665	user_data, length);
		666	io_mapping_unmap_atomic(vaddr_atomic);
		667	return unwritten;
		668	}
		669
		670	/**
		671	* This is the fast pwrite path, where we copy the data directly from the
		672	* user into the GTT, uncached.
		673	*/
2332	Serge	674	static int
3031	serge	675	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		676	struct drm_i915_gem_object *obj,
		677	struct drm_i915_gem_pwrite *args,
		678	struct drm_file *file)
2332	Serge	679	{
3031	serge	680	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	681	ssize_t remain;
3031	serge	682	loff_t offset, page_base;
		683	char __user *user_data;
		684	int page_offset, page_length, ret;
2332	Serge	685
3031	serge	686	ret = i915_gem_object_pin(obj, 0, true, true);
		687	if (ret)
		688	goto out;
		689
		690	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		691	if (ret)
		692	goto out_unpin;
		693
		694	ret = i915_gem_object_put_fence(obj);
		695	if (ret)
		696	goto out_unpin;
		697
		698	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	699	remain = args->size;
		700
3031	serge	701	offset = obj->gtt_offset + args->offset;
2332	Serge	702
3031	serge	703	while (remain > 0) {
		704	/* Operation in this page
		705	*
		706	* page_base = page offset within aperture
		707	* page_offset = offset within page
		708	* page_length = bytes to copy for this page
		709	*/
		710	page_base = offset & PAGE_MASK;
		711	page_offset = offset_in_page(offset);
		712	page_length = remain;
		713	if ((page_offset + remain) > PAGE_SIZE)
		714	page_length = PAGE_SIZE - page_offset;
2332	Serge	715
3031	serge	716	/* If we get a fault while copying data, then (presumably) our
		717	* source page isn't available. Return the error and we'll
		718	* retry in the slow path.
		719	*/
		720	if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
		721	page_offset, user_data, page_length)) {
		722	ret = -EFAULT;
		723	goto out_unpin;
		724	}
		725
		726	remain -= page_length;
		727	user_data += page_length;
		728	offset += page_length;
2332	Serge	729	}
		730
3031	serge	731	out_unpin:
		732	i915_gem_object_unpin(obj);
		733	out:
		734	return ret;
		735	}
		736
		737	/* Per-page copy function for the shmem pwrite fastpath.
		738	* Flushes invalid cachelines before writing to the target if
		739	* needs_clflush_before is set and flushes out any written cachelines after
		740	* writing if needs_clflush is set. */
		741	static int
		742	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		743	char __user *user_data,
		744	bool page_do_bit17_swizzling,
		745	bool needs_clflush_before,
		746	bool needs_clflush_after)
		747	{
		748	char *vaddr;
		749	int ret;
		750
		751	if (unlikely(page_do_bit17_swizzling))
		752	return -EINVAL;
		753
		754	vaddr = kmap_atomic(page);
		755	if (needs_clflush_before)
		756	drm_clflush_virt_range(vaddr + shmem_page_offset,
		757	page_length);
		758	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
		759	user_data,
		760	page_length);
		761	if (needs_clflush_after)
		762	drm_clflush_virt_range(vaddr + shmem_page_offset,
		763	page_length);
		764	kunmap_atomic(vaddr);
		765
		766	return ret ? -EFAULT : 0;
		767	}
		768
		769	/* Only difference to the fast-path function is that this can handle bit17
		770	* and uses non-atomic copy and kmap functions. */
		771	static int
		772	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		773	char __user *user_data,
		774	bool page_do_bit17_swizzling,
		775	bool needs_clflush_before,
		776	bool needs_clflush_after)
		777	{
		778	char *vaddr;
		779	int ret;
		780
		781	vaddr = kmap(page);
		782	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		783	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		784	page_length,
		785	page_do_bit17_swizzling);
		786	if (page_do_bit17_swizzling)
		787	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		788	user_data,
		789	page_length);
		790	else
		791	ret = __copy_from_user(vaddr + shmem_page_offset,
		792	user_data,
		793	page_length);
		794	if (needs_clflush_after)
		795	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		796	page_length,
		797	page_do_bit17_swizzling);
		798	kunmap(page);
		799
		800	return ret ? -EFAULT : 0;
		801	}
		802
		803	static int
		804	i915_gem_shmem_pwrite(struct drm_device *dev,
		805	struct drm_i915_gem_object *obj,
		806	struct drm_i915_gem_pwrite *args,
		807	struct drm_file *file)
		808	{
		809	ssize_t remain;
		810	loff_t offset;
		811	char __user *user_data;
		812	int shmem_page_offset, page_length, ret = 0;
		813	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		814	int hit_slowpath = 0;
		815	int needs_clflush_after = 0;
		816	int needs_clflush_before = 0;
		817	int i;
		818	struct scatterlist *sg;
		819
		820	user_data = (char __user *) (uintptr_t) args->data_ptr;
		821	remain = args->size;
		822
		823	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		824
		825	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		826	/* If we're not in the cpu write domain, set ourself into the gtt
		827	* write domain and manually flush cachelines (if required). This
		828	* optimizes for the case when the gpu will use the data
		829	* right away and we therefore have to clflush anyway. */
		830	if (obj->cache_level == I915_CACHE_NONE)
		831	needs_clflush_after = 1;
		832	if (obj->gtt_space) {
		833	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		834	if (ret)
		835	return ret;
		836	}
		837	}
		838	/* Same trick applies for invalidate partially written cachelines before
		839	* writing. */
		840	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		841	&& obj->cache_level == I915_CACHE_NONE)
		842	needs_clflush_before = 1;
		843
		844	ret = i915_gem_object_get_pages(obj);
2332	Serge	845	if (ret)
3031	serge	846	return ret;
2332	Serge	847
3031	serge	848	i915_gem_object_pin_pages(obj);
2332	Serge	849
		850	offset = args->offset;
3031	serge	851	obj->dirty = 1;
2332	Serge	852
3031	serge	853	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	854	struct page *page;
3031	serge	855	int partial_cacheline_write;
2332	Serge	856
3031	serge	857	if (i < offset >> PAGE_SHIFT)
		858	continue;
		859
		860	if (remain <= 0)
		861	break;
		862
2332	Serge	863	/* Operation in this page
		864	*
		865	* shmem_page_offset = offset within page in shmem file
		866	* page_length = bytes to copy for this page
		867	*/
		868	shmem_page_offset = offset_in_page(offset);
		869
		870	page_length = remain;
		871	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		872	page_length = PAGE_SIZE - shmem_page_offset;
		873
3031	serge	874	/* If we don't overwrite a cacheline completely we need to be
		875	* careful to have up-to-date data by first clflushing. Don't
		876	* overcomplicate things and flush the entire patch. */
		877	partial_cacheline_write = needs_clflush_before &&
		878	((shmem_page_offset \| page_length)
		879	& (boot_cpu_data.x86_clflush_size - 1));
2332	Serge	880
3031	serge	881	page = sg_page(sg);
		882	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		883	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	884
3031	serge	885	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		886	user_data, page_do_bit17_swizzling,
		887	partial_cacheline_write,
		888	needs_clflush_after);
		889	if (ret == 0)
		890	goto next_page;
		891
		892	hit_slowpath = 1;
		893	mutex_unlock(&dev->struct_mutex);
		894	ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		895	user_data, page_do_bit17_swizzling,
		896	partial_cacheline_write,
		897	needs_clflush_after);
		898
		899	mutex_lock(&dev->struct_mutex);
		900
		901	next_page:
		902	set_page_dirty(page);
2332	Serge	903	mark_page_accessed(page);
		904
3031	serge	905	if (ret)
		906	goto out;
		907
2332	Serge	908	remain -= page_length;
3031	serge	909	user_data += page_length;
2332	Serge	910	offset += page_length;
		911	}
		912
		913	out:
3031	serge	914	i915_gem_object_unpin_pages(obj);
		915
		916	if (hit_slowpath) {
		917	/* Fixup: Kill any reinstated backing storage pages */
		918	if (obj->madv == __I915_MADV_PURGED)
		919	i915_gem_object_truncate(obj);
		920	/* and flush dirty cachelines in case the object isn't in the cpu write
		921	* domain anymore. */
		922	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		923	i915_gem_clflush_object(obj);
		924	intel_gtt_chipset_flush();
		925	}
2332	Serge	926	}
		927
3031	serge	928	if (needs_clflush_after)
		929	intel_gtt_chipset_flush();
		930
2332	Serge	931	return ret;
		932	}
3031	serge	933
		934	/**
		935	* Writes data to the object referenced by handle.
		936	*
		937	* On error, the contents of the buffer that were to be modified are undefined.
		938	*/
		939	int
		940	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		941	struct drm_file *file)
		942	{
		943	struct drm_i915_gem_pwrite *args = data;
		944	struct drm_i915_gem_object *obj;
		945	int ret;
		946
		947	if (args->size == 0)
		948	return 0;
		949
		950	if (!access_ok(VERIFY_READ,
		951	(char __user *)(uintptr_t)args->data_ptr,
		952	args->size))
		953	return -EFAULT;
		954
		955	ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
		956	args->size);
		957	if (ret)
		958	return -EFAULT;
		959
		960	ret = i915_mutex_lock_interruptible(dev);
		961	if (ret)
		962	return ret;
		963
		964	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		965	if (&obj->base == NULL) {
		966	ret = -ENOENT;
		967	goto unlock;
		968	}
		969
		970	/* Bounds check destination. */
		971	if (args->offset > obj->base.size \|\|
		972	args->size > obj->base.size - args->offset) {
		973	ret = -EINVAL;
		974	goto out;
		975	}
		976
		977	/* prime objects have no backing filp to GEM pread/pwrite
		978	* pages from.
		979	*/
		980	if (!obj->base.filp) {
		981	ret = -EINVAL;
		982	goto out;
		983	}
		984
		985	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		986
		987	ret = -EFAULT;
		988	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		989	* it would end up going through the fenced access, and we'll get
		990	* different detiling behavior between reading and writing.
		991	* pread/pwrite currently are reading and writing from the CPU
		992	* perspective, requiring manual detiling by the client.
		993	*/
		994	if (obj->phys_obj) {
		995	ret = i915_gem_phys_pwrite(dev, obj, args, file);
		996	goto out;
		997	}
		998
		999	if (obj->cache_level == I915_CACHE_NONE &&
		1000	obj->tiling_mode == I915_TILING_NONE &&
		1001	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		1002	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		1003	/* Note that the gtt paths might fail with non-page-backed user
		1004	* pointers (e.g. gtt mappings when moving data between
		1005	* textures). Fallback to the shmem path in that case. */
		1006	}
		1007
		1008	if (ret == -EFAULT \|\| ret == -ENOSPC)
		1009	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
		1010
		1011	out:
		1012	drm_gem_object_unreference(&obj->base);
		1013	unlock:
		1014	mutex_unlock(&dev->struct_mutex);
		1015	return ret;
		1016	}
		1017
2332	Serge	1018	#endif
		1019
3031	serge	1020	int
		1021	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		1022	bool interruptible)
		1023	{
		1024	if (atomic_read(&dev_priv->mm.wedged)) {
		1025	struct completion *x = &dev_priv->error_completion;
		1026	bool recovery_complete;
		1027	unsigned long flags;
2332	Serge	1028
3031	serge	1029	/* Give the error handler a chance to run. */
		1030	spin_lock_irqsave(&x->wait.lock, flags);
		1031	recovery_complete = x->done > 0;
		1032	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	1033
3031	serge	1034	/* Non-interruptible callers can't handle -EAGAIN, hence return
		1035	* -EIO unconditionally for these. */
		1036	if (!interruptible)
		1037	return -EIO;
2332	Serge	1038
3031	serge	1039	/* Recovery complete, but still wedged means reset failure. */
		1040	if (recovery_complete)
		1041	return -EIO;
2332	Serge	1042
3031	serge	1043	return -EAGAIN;
		1044	}
2332	Serge	1045
3031	serge	1046	return 0;
		1047	}
2332	Serge	1048
3031	serge	1049	/*
		1050	* Compare seqno against outstanding lazy request. Emit a request if they are
		1051	* equal.
		1052	*/
		1053	static int
		1054	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1055	{
		1056	int ret;
2332	Serge	1057
3031	serge	1058	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1059
3031	serge	1060	ret = 0;
		1061	if (seqno == ring->outstanding_lazy_request)
		1062	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1063
3031	serge	1064	return ret;
		1065	}
2332	Serge	1066
3031	serge	1067	/**
		1068	* __wait_seqno - wait until execution of seqno has finished
		1069	* @ring: the ring expected to report seqno
		1070	* @seqno: duh!
		1071	* @interruptible: do an interruptible wait (normally yes)
		1072	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1073	*
		1074	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1075	* errno with remaining time filled in timeout argument.
		1076	*/
		1077	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1078	bool interruptible, struct timespec *timeout)
		1079	{
		1080	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1081	struct timespec before, now, wait_time={1,0};
		1082	unsigned long timeout_jiffies;
		1083	long end;
		1084	bool wait_forever = true;
		1085	int ret;
2332	Serge	1086
3031	serge	1087	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1088	return 0;
2332	Serge	1089
3031	serge	1090	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1091
3031	serge	1092	if (timeout != NULL) {
		1093	wait_time = *timeout;
		1094	wait_forever = false;
		1095	}
2332	Serge	1096
3031	serge	1097	// timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1098
3031	serge	1099	if (WARN_ON(!ring->irq_get(ring)))
		1100	return -ENODEV;
		1101	#if 0
2332	Serge	1102
3031	serge	1103	/* Record current time in case interrupted by signal, or wedged * */
		1104	getrawmonotonic(&before);
2332	Serge	1105
3031	serge	1106	#define EXIT_COND \
		1107	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1108	atomic_read(&dev_priv->mm.wedged))
		1109	do {
		1110	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1111	timeout_jiffies);
2332	Serge	1112
3031	serge	1113	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1114	if (ret)
		1115	end = ret;
		1116	} while (end == 0 && wait_forever);
2332	Serge	1117
3031	serge	1118	getrawmonotonic(&now);
2332	Serge	1119
3031	serge	1120	ring->irq_put(ring);
		1121	trace_i915_gem_request_wait_end(ring, seqno);
		1122	#undef EXIT_COND
2332	Serge	1123
3031	serge	1124	if (timeout) {
		1125	// struct timespec sleep_time = timespec_sub(now, before);
		1126	// timeout = timespec_sub(timeout, sleep_time);
		1127	}
2332	Serge	1128
3031	serge	1129	switch (end) {
		1130	case -EIO:
		1131	case -EAGAIN: /* Wedged */
		1132	case -ERESTARTSYS: /* Signal */
		1133	return (int)end;
		1134	case 0: /* Timeout */
		1135	// if (timeout)
		1136	// set_normalized_timespec(timeout, 0, 0);
		1137	return -ETIME;
		1138	default: /* Completed */
		1139	WARN_ON(end < 0); /* We're not aware of other errors */
		1140	return 0;
		1141	}
		1142	#endif
2332	Serge	1143
3031	serge	1144	#define EXIT_COND \
		1145	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1146	atomic_read(&dev_priv->mm.wedged))
		1147	wait_event(ring->irq_queue, EXIT_COND);
		1148	#undef EXIT_COND
		1149	ring->irq_put(ring);
2332	Serge	1150
3031	serge	1151	return 0;
		1152	}
2332	Serge	1153
3031	serge	1154	/**
		1155	* Waits for a sequence number to be signaled, and cleans up the
		1156	* request and object lists appropriately for that event.
		1157	*/
		1158	int
		1159	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1160	{
		1161	struct drm_device *dev = ring->dev;
		1162	struct drm_i915_private *dev_priv = dev->dev_private;
		1163	bool interruptible = dev_priv->mm.interruptible;
		1164	int ret;
2332	Serge	1165
3031	serge	1166	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1167	BUG_ON(seqno == 0);
2332	Serge	1168
3031	serge	1169	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1170	if (ret)
		1171	return ret;
2332	Serge	1172
3031	serge	1173	ret = i915_gem_check_olr(ring, seqno);
		1174	if (ret)
		1175	return ret;
2332	Serge	1176
3031	serge	1177	return __wait_seqno(ring, seqno, interruptible, NULL);
		1178	}
2332	Serge	1179
3031	serge	1180	/**
		1181	* Ensures that all rendering to the object has completed and the object is
		1182	* safe to unbind from the GTT or access from the CPU.
		1183	*/
		1184	static __must_check int
		1185	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1186	bool readonly)
		1187	{
		1188	struct intel_ring_buffer *ring = obj->ring;
		1189	u32 seqno;
		1190	int ret;
2332	Serge	1191
3031	serge	1192	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1193	if (seqno == 0)
		1194	return 0;
2332	Serge	1195
3031	serge	1196	ret = i915_wait_seqno(ring, seqno);
		1197	if (ret)
		1198	return ret;
2332	Serge	1199
3031	serge	1200	i915_gem_retire_requests_ring(ring);
2332	Serge	1201
3031	serge	1202	/* Manually manage the write flush as we may have not yet
		1203	* retired the buffer.
		1204	*/
		1205	if (obj->last_write_seqno &&
		1206	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1207	obj->last_write_seqno = 0;
		1208	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1209	}
2332	Serge	1210
3031	serge	1211	return 0;
		1212	}
2332	Serge	1213
		1214
		1215
		1216
		1217
		1218
		1219
		1220
		1221
		1222
		1223
		1224
		1225
		1226
		1227
		1228
		1229
		1230
		1231
		1232
		1233
		1234
		1235
		1236
		1237
		1238
		1239
		1240
		1241
		1242
		1243
		1244
		1245
		1246
		1247
		1248
		1249
3031	serge	1250
		1251
		1252
		1253
		1254
		1255
		1256	/**
		1257	* i915_gem_release_mmap - remove physical page mappings
		1258	* @obj: obj in question
		1259	*
		1260	* Preserve the reservation of the mmapping with the DRM core code, but
		1261	* relinquish ownership of the pages back to the system.
		1262	*
		1263	* It is vital that we remove the page mapping if we have mapped a tiled
		1264	* object through the GTT and then lose the fence register due to
		1265	* resource pressure. Similarly if the object has been moved out of the
		1266	* aperture, than pages mapped into userspace must be revoked. Removing the
		1267	* mapping will then trigger a page fault on the next user access, allowing
		1268	* fixup by i915_gem_fault().
		1269	*/
		1270	void
		1271	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1272	{
		1273	if (!obj->fault_mappable)
		1274	return;
		1275
		1276	if (obj->base.dev->dev_mapping)
		1277	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1278	// (loff_t)obj->base.map_list.hash.key<
		1279	// obj->base.size, 1);
		1280
		1281	obj->fault_mappable = false;
		1282	}
		1283
2332	Serge	1284	static uint32_t
		1285	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1286	{
		1287	uint32_t gtt_size;
		1288
		1289	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1290	tiling_mode == I915_TILING_NONE)
		1291	return size;
		1292
		1293	/* Previous chips need a power-of-two fence region when tiling */
		1294	if (INTEL_INFO(dev)->gen == 3)
		1295	gtt_size = 1024*1024;
		1296	else
		1297	gtt_size = 512*1024;
		1298
		1299	while (gtt_size < size)
		1300	gtt_size <<= 1;
		1301
		1302	return gtt_size;
		1303	}
		1304
		1305	/**
		1306	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1307	* @obj: object to check
		1308	*
		1309	* Return the required GTT alignment for an object, taking into account
		1310	* potential fence register mapping.
		1311	*/
		1312	static uint32_t
		1313	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1314	uint32_t size,
		1315	int tiling_mode)
		1316	{
		1317	/*
		1318	* Minimum alignment is 4k (GTT page size), but might be greater
		1319	* if a fence register is needed for the object.
		1320	*/
		1321	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1322	tiling_mode == I915_TILING_NONE)
		1323	return 4096;
		1324
		1325	/*
		1326	* Previous chips need to be aligned to the size of the smallest
		1327	* fence register that can contain the object.
		1328	*/
		1329	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1330	}
		1331
		1332	/**
		1333	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1334	* unfenced object
		1335	* @dev: the device
		1336	* @size: size of the object
		1337	* @tiling_mode: tiling mode of the object
		1338	*
		1339	* Return the required GTT alignment for an object, only taking into account
		1340	* unfenced tiled surface requirements.
		1341	*/
		1342	uint32_t
		1343	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1344	uint32_t size,
		1345	int tiling_mode)
		1346	{
		1347	/*
		1348	* Minimum alignment is 4k (GTT page size) for sane hw.
		1349	*/
		1350	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1351	tiling_mode == I915_TILING_NONE)
		1352	return 4096;
		1353
		1354	/* Previous hardware however needs to be aligned to a power-of-two
		1355	* tile height. The simplest method for determining this is to reuse
		1356	* the power-of-tile object size.
		1357	*/
		1358	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1359	}
		1360
3031	serge	1361	/* Immediately discard the backing storage */
		1362	static void
		1363	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1364	{
		1365	// struct inode *inode;
2332	Serge	1366
3031	serge	1367	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1368
3031	serge	1369	// if (obj->base.filp == NULL)
		1370	// return;
2332	Serge	1371
3031	serge	1372	/* Our goal here is to return as much of the memory as
		1373	* is possible back to the system as we are called from OOM.
		1374	* To do this we must instruct the shmfs to drop all of its
		1375	* backing pages, now.
		1376	*/
		1377	// inode = obj->base.filp->f_path.dentry->d_inode;
		1378	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1379
3031	serge	1380	obj->madv = __I915_MADV_PURGED;
		1381	}
2332	Serge	1382
3031	serge	1383	static inline int
		1384	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1385	{
		1386	return obj->madv == I915_MADV_DONTNEED;
		1387	}
2332	Serge	1388
3031	serge	1389	static void
		1390	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1391	{
		1392	int ret, i;
2332	Serge	1393
3031	serge	1394	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1395
3031	serge	1396	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1397	if (ret) {
		1398	/* In the event of a disaster, abandon all caches and
		1399	* hope for the best.
		1400	*/
		1401	WARN_ON(ret != -EIO);
		1402	i915_gem_clflush_object(obj);
		1403	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1404	}
2332	Serge	1405
3031	serge	1406	if (obj->madv == I915_MADV_DONTNEED)
		1407	obj->dirty = 0;
2332	Serge	1408
3031	serge	1409	for (i = 0; i < obj->pages.nents; i++)
		1410	FreePage(obj->pages.page[i]);
2332	Serge	1411
3037	serge	1412	DRM_DEBUG_KMS("%s free %d pages\n", __FUNCTION__, obj->pages.nents);
3031	serge	1413	obj->dirty = 0;
		1414	kfree(obj->pages.page);
		1415	}
2332	Serge	1416
3031	serge	1417	static int
		1418	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1419	{
		1420	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1421
3031	serge	1422	if (obj->pages.page == NULL)
		1423	return 0;
2332	Serge	1424
3031	serge	1425	BUG_ON(obj->gtt_space);
		1426
		1427	if (obj->pages_pin_count)
		1428	return -EBUSY;
		1429
		1430	ops->put_pages(obj);
		1431	obj->pages.page = NULL;
		1432
		1433	list_del(&obj->gtt_list);
		1434	if (i915_gem_object_is_purgeable(obj))
		1435	i915_gem_object_truncate(obj);
		1436
		1437	return 0;
		1438	}
		1439
		1440
		1441
		1442
		1443
		1444
		1445
		1446
2332	Serge	1447	static int
3031	serge	1448	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1449	{
3031	serge	1450	dma_addr_t page;
2332	Serge	1451	int page_count, i;
		1452
		1453	/* Get the list of pages out of our struct file. They'll be pinned
		1454	* at this point until we release them.
		1455	*/
		1456	page_count = obj->base.size / PAGE_SIZE;
3031	serge	1457	BUG_ON(obj->pages.page != NULL);
		1458	obj->pages.page = malloc(page_count * sizeof(dma_addr_t));
		1459	if (obj->pages.page == NULL)
2332	Serge	1460	return -ENOMEM;
		1461
		1462	for (i = 0; i < page_count; i++) {
3031	serge	1463	page = AllocPage(); // oh-oh
		1464	if ( page == 0 )
2332	Serge	1465	goto err_pages;
		1466
3031	serge	1467	obj->pages.page[i] = page;
		1468	};
3037	serge	1469	DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
3031	serge	1470	obj->pages.nents = page_count;
		1471
		1472
2332	Serge	1473	// if (obj->tiling_mode != I915_TILING_NONE)
		1474	// i915_gem_object_do_bit_17_swizzle(obj);
		1475
		1476	return 0;
		1477
		1478	err_pages:
2344	Serge	1479	while (i--)
3031	serge	1480	FreePage(obj->pages.page[i]);
2332	Serge	1481
3031	serge	1482	free(obj->pages.page);
		1483	obj->pages.page = NULL;
		1484	obj->pages.nents = 0;
		1485
		1486	return -ENOMEM;
2332	Serge	1487	}
		1488
3031	serge	1489	/* Ensure that the associated pages are gathered from the backing storage
		1490	* and pinned into our object. i915_gem_object_get_pages() may be called
		1491	* multiple times before they are released by a single call to
		1492	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1493	* either as a result of memory pressure (reaping pages under the shrinker)
		1494	* or as the object is itself released.
		1495	*/
		1496	int
		1497	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1498	{
3031	serge	1499	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1500	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1501	int ret;
2332	Serge	1502
3031	serge	1503	if (obj->pages.page)
		1504	return 0;
2332	Serge	1505
3031	serge	1506	BUG_ON(obj->pages_pin_count);
2332	Serge	1507
3031	serge	1508	ret = ops->get_pages(obj);
		1509	if (ret)
		1510	return ret;
2344	Serge	1511
3031	serge	1512	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
		1513	return 0;
2332	Serge	1514	}
		1515
		1516	void
		1517	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
		1518	struct intel_ring_buffer *ring,
		1519	u32 seqno)
		1520	{
		1521	struct drm_device *dev = obj->base.dev;
		1522	struct drm_i915_private *dev_priv = dev->dev_private;
		1523
		1524	BUG_ON(ring == NULL);
		1525	obj->ring = ring;
		1526
		1527	/* Add a reference if we're newly entering the active list. */
		1528	if (!obj->active) {
2344	Serge	1529	drm_gem_object_reference(&obj->base);
2332	Serge	1530	obj->active = 1;
		1531	}
		1532
		1533	/* Move from whatever list we were on to the tail of execution. */
		1534	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1535	list_move_tail(&obj->ring_list, &ring->active_list);
		1536
3031	serge	1537	obj->last_read_seqno = seqno;
		1538
2332	Serge	1539	if (obj->fenced_gpu_access) {
3031	serge	1540	obj->last_fenced_seqno = seqno;
		1541
		1542	/* Bump MRU to take account of the delayed flush */
		1543	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1544	struct drm_i915_fence_reg *reg;
		1545
		1546	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1547	list_move_tail(®->lru_list,
		1548	&dev_priv->mm.fence_list);
		1549	}
2332	Serge	1550	}
		1551	}
		1552
2344	Serge	1553	static void
3031	serge	1554	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1555	{
		1556	struct drm_device *dev = obj->base.dev;
3031	serge	1557	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1558
3031	serge	1559	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1560	BUG_ON(!obj->active);
2332	Serge	1561
3031	serge	1562	if (obj->pin_count) /* are we a framebuffer? */
		1563	intel_mark_fb_idle(obj);
2344	Serge	1564
2352	Serge	1565	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1566
3031	serge	1567	list_del_init(&obj->ring_list);
2352	Serge	1568	obj->ring = NULL;
2344	Serge	1569
3031	serge	1570	obj->last_read_seqno = 0;
		1571	obj->last_write_seqno = 0;
		1572	obj->base.write_domain = 0;
		1573
		1574	obj->last_fenced_seqno = 0;
2352	Serge	1575	obj->fenced_gpu_access = false;
2344	Serge	1576
2352	Serge	1577	obj->active = 0;
		1578	drm_gem_object_unreference(&obj->base);
		1579
		1580	WARN_ON(i915_verify_lists(dev));
		1581	}
		1582
3031	serge	1583	static u32
		1584	i915_gem_get_seqno(struct drm_device *dev)
2344	Serge	1585	{
3031	serge	1586	drm_i915_private_t *dev_priv = dev->dev_private;
		1587	u32 seqno = dev_priv->next_seqno;
2344	Serge	1588
3031	serge	1589	/* reserve 0 for non-seqno */
		1590	if (++dev_priv->next_seqno == 0)
		1591	dev_priv->next_seqno = 1;
2344	Serge	1592
3031	serge	1593	return seqno;
2344	Serge	1594	}
		1595
3031	serge	1596	u32
		1597	i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
2344	Serge	1598	{
3031	serge	1599	if (ring->outstanding_lazy_request == 0)
		1600	ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
2344	Serge	1601
3031	serge	1602	return ring->outstanding_lazy_request;
2332	Serge	1603	}
		1604
2352	Serge	1605	int
		1606	i915_add_request(struct intel_ring_buffer *ring,
		1607	struct drm_file *file,
3031	serge	1608	u32 *out_seqno)
2352	Serge	1609	{
		1610	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1611	struct drm_i915_gem_request *request;
		1612	u32 request_ring_position;
		1613	u32 seqno;
2352	Serge	1614	int was_empty;
		1615	int ret;
2332	Serge	1616
3031	serge	1617	/*
		1618	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1619	* after having emitted the batchbuffer command. Hence we need to fix
		1620	* things up similar to emitting the lazy request. The difference here
		1621	* is that the flush _must_ happen before the next request, no matter
		1622	* what.
		1623	*/
		1624	ret = intel_ring_flush_all_caches(ring);
		1625	if (ret)
		1626	return ret;
2332	Serge	1627
3031	serge	1628	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1629	if (request == NULL)
		1630	return -ENOMEM;
		1631
		1632	seqno = i915_gem_next_request_seqno(ring);
		1633
		1634	/* Record the position of the start of the request so that
		1635	* should we detect the updated seqno part-way through the
		1636	* GPU processing the request, we never over-estimate the
		1637	* position of the head.
		1638	*/
		1639	request_ring_position = intel_ring_get_tail(ring);
		1640
2352	Serge	1641	ret = ring->add_request(ring, &seqno);
3031	serge	1642	if (ret) {
		1643	kfree(request);
2352	Serge	1644	return ret;
3031	serge	1645	}
2332	Serge	1646
2352	Serge	1647	trace_i915_gem_request_add(ring, seqno);
2332	Serge	1648
2352	Serge	1649	request->seqno = seqno;
		1650	request->ring = ring;
3031	serge	1651	request->tail = request_ring_position;
		1652	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1653	was_empty = list_empty(&ring->request_list);
		1654	list_add_tail(&request->list, &ring->request_list);
3031	serge	1655	request->file_priv = NULL;
2332	Serge	1656
		1657
3031	serge	1658	ring->outstanding_lazy_request = 0;
2332	Serge	1659
2360	Serge	1660	if (!dev_priv->mm.suspended) {
		1661	if (i915_enable_hangcheck) {
2352	Serge	1662	// mod_timer(&dev_priv->hangcheck_timer,
		1663	// jiffies +
		1664	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1665	}
3031	serge	1666	if (was_empty) {
2360	Serge	1667	queue_delayed_work(dev_priv->wq,
		1668	&dev_priv->mm.retire_work, HZ);
3031	serge	1669	intel_mark_busy(dev_priv->dev);
		1670	}
2360	Serge	1671	}
3031	serge	1672
		1673	if (out_seqno)
		1674	*out_seqno = seqno;
2352	Serge	1675	return 0;
		1676	}
2332	Serge	1677
		1678
		1679
		1680
3031	serge	1681	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1682	struct intel_ring_buffer *ring)
		1683	{
		1684	while (!list_empty(&ring->request_list)) {
		1685	struct drm_i915_gem_request *request;
2332	Serge	1686
3031	serge	1687	request = list_first_entry(&ring->request_list,
		1688	struct drm_i915_gem_request,
		1689	list);
2332	Serge	1690
3031	serge	1691	list_del(&request->list);
		1692	// i915_gem_request_remove_from_client(request);
		1693	kfree(request);
		1694	}
2332	Serge	1695
3031	serge	1696	while (!list_empty(&ring->active_list)) {
		1697	struct drm_i915_gem_object *obj;
2332	Serge	1698
3031	serge	1699	obj = list_first_entry(&ring->active_list,
		1700	struct drm_i915_gem_object,
		1701	ring_list);
2332	Serge	1702
3031	serge	1703	i915_gem_object_move_to_inactive(obj);
		1704	}
		1705	}
2332	Serge	1706
3031	serge	1707	static void i915_gem_reset_fences(struct drm_device *dev)
		1708	{
		1709	struct drm_i915_private *dev_priv = dev->dev_private;
		1710	int i;
2332	Serge	1711
3031	serge	1712	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1713	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1714
3031	serge	1715	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1716
3031	serge	1717	if (reg->obj)
		1718	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1719
3031	serge	1720	reg->pin_count = 0;
		1721	reg->obj = NULL;
		1722	INIT_LIST_HEAD(®->lru_list);
		1723	}
2360	Serge	1724
3031	serge	1725	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1726	}
2360	Serge	1727
3031	serge	1728	void i915_gem_reset(struct drm_device *dev)
		1729	{
		1730	struct drm_i915_private *dev_priv = dev->dev_private;
		1731	struct drm_i915_gem_object *obj;
		1732	struct intel_ring_buffer *ring;
		1733	int i;
2360	Serge	1734
3031	serge	1735	for_each_ring(ring, dev_priv, i)
		1736	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1737
3031	serge	1738	/* Move everything out of the GPU domains to ensure we do any
		1739	* necessary invalidation upon reuse.
		1740	*/
		1741	list_for_each_entry(obj,
		1742	&dev_priv->mm.inactive_list,
		1743	mm_list)
		1744	{
		1745	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1746	}
2360	Serge	1747
3031	serge	1748	/* The fence registers are invalidated so clear them out */
		1749	i915_gem_reset_fences(dev);
		1750	}
2360	Serge	1751
2352	Serge	1752	/**
		1753	* This function clears the request list as sequence numbers are passed.
		1754	*/
3031	serge	1755	void
2352	Serge	1756	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1757	{
		1758	uint32_t seqno;
		1759	int i;
2332	Serge	1760
2352	Serge	1761	if (list_empty(&ring->request_list))
		1762	return;
2332	Serge	1763
2352	Serge	1764	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1765
3031	serge	1766	seqno = ring->get_seqno(ring, true);
2332	Serge	1767
2352	Serge	1768	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
		1769	if (seqno >= ring->sync_seqno[i])
		1770	ring->sync_seqno[i] = 0;
2332	Serge	1771
2352	Serge	1772	while (!list_empty(&ring->request_list)) {
		1773	struct drm_i915_gem_request *request;
2332	Serge	1774
2352	Serge	1775	request = list_first_entry(&ring->request_list,
		1776	struct drm_i915_gem_request,
		1777	list);
2332	Serge	1778
2352	Serge	1779	if (!i915_seqno_passed(seqno, request->seqno))
		1780	break;
2332	Serge	1781
2352	Serge	1782	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1783	/* We know the GPU must have read the request to have
		1784	* sent us the seqno + interrupt, so use the position
		1785	* of tail of the request to update the last known position
		1786	* of the GPU head.
		1787	*/
		1788	ring->last_retired_head = request->tail;
2332	Serge	1789
2352	Serge	1790	list_del(&request->list);
		1791	kfree(request);
		1792	}
2332	Serge	1793
2352	Serge	1794	/* Move any buffers on the active list that are no longer referenced
		1795	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1796	*/
		1797	while (!list_empty(&ring->active_list)) {
		1798	struct drm_i915_gem_object *obj;
2332	Serge	1799
2352	Serge	1800	obj = list_first_entry(&ring->active_list,
		1801	struct drm_i915_gem_object,
		1802	ring_list);
2332	Serge	1803
3031	serge	1804	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1805	break;
2332	Serge	1806
2352	Serge	1807	i915_gem_object_move_to_inactive(obj);
		1808	}
2332	Serge	1809
2352	Serge	1810	if (unlikely(ring->trace_irq_seqno &&
		1811	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1812	ring->irq_put(ring);
		1813	ring->trace_irq_seqno = 0;
		1814	}
2332	Serge	1815
2352	Serge	1816	WARN_ON(i915_verify_lists(ring->dev));
		1817	}
2332	Serge	1818
2352	Serge	1819	void
		1820	i915_gem_retire_requests(struct drm_device *dev)
		1821	{
		1822	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1823	struct intel_ring_buffer *ring;
2352	Serge	1824	int i;
2332	Serge	1825
3031	serge	1826	for_each_ring(ring, dev_priv, i)
		1827	i915_gem_retire_requests_ring(ring);
2352	Serge	1828	}
		1829
2360	Serge	1830	static void
		1831	i915_gem_retire_work_handler(struct work_struct *work)
		1832	{
		1833	drm_i915_private_t *dev_priv;
		1834	struct drm_device *dev;
3031	serge	1835	struct intel_ring_buffer *ring;
2360	Serge	1836	bool idle;
		1837	int i;
2352	Serge	1838
2360	Serge	1839	dev_priv = container_of(work, drm_i915_private_t,
		1840	mm.retire_work.work);
		1841	dev = dev_priv->dev;
2352	Serge	1842
2360	Serge	1843	/* Come back later if the device is busy... */
		1844	if (!mutex_trylock(&dev->struct_mutex)) {
		1845	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
		1846	return;
		1847	}
2352	Serge	1848
2360	Serge	1849	i915_gem_retire_requests(dev);
2352	Serge	1850
2360	Serge	1851	/* Send a periodic flush down the ring so we don't hold onto GEM
		1852	* objects indefinitely.
		1853	*/
		1854	idle = true;
3031	serge	1855	for_each_ring(ring, dev_priv, i) {
		1856	if (ring->gpu_caches_dirty)
		1857	i915_add_request(ring, NULL, NULL);
2352	Serge	1858
2360	Serge	1859	idle &= list_empty(&ring->request_list);
		1860	}
2352	Serge	1861
2360	Serge	1862	if (!dev_priv->mm.suspended && !idle)
		1863	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	1864	if (idle)
		1865	intel_mark_idle(dev);
2360	Serge	1866
		1867	mutex_unlock(&dev->struct_mutex);
		1868	}
		1869
2344	Serge	1870	/**
3031	serge	1871	* Ensures that an object will eventually get non-busy by flushing any required
		1872	* write domains, emitting any outstanding lazy request and retiring and
		1873	* completed requests.
2352	Serge	1874	*/
3031	serge	1875	static int
		1876	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	1877	{
3031	serge	1878	int ret;
2352	Serge	1879
3031	serge	1880	if (obj->active) {
		1881	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		1882	if (ret)
		1883	return ret;
2352	Serge	1884
3031	serge	1885	i915_gem_retire_requests_ring(obj->ring);
		1886	}
2352	Serge	1887
3031	serge	1888	return 0;
		1889	}
2352	Serge	1890
		1891
		1892
		1893
		1894
		1895
		1896
		1897
		1898
		1899
		1900	/**
3031	serge	1901	* i915_gem_object_sync - sync an object to a ring.
		1902	*
		1903	* @obj: object which may be in use on another ring.
		1904	* @to: ring we wish to use the object on. May be NULL.
		1905	*
		1906	* This code is meant to abstract object synchronization with the GPU.
		1907	* Calling with NULL implies synchronizing the object with the CPU
		1908	* rather than a particular GPU ring.
		1909	*
		1910	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	1911	*/
		1912	int
3031	serge	1913	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		1914	struct intel_ring_buffer *to)
2344	Serge	1915	{
3031	serge	1916	struct intel_ring_buffer *from = obj->ring;
		1917	u32 seqno;
		1918	int ret, idx;
2332	Serge	1919
3031	serge	1920	if (from == NULL \|\| to == from)
		1921	return 0;
2332	Serge	1922
3031	serge	1923	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		1924	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	1925
3031	serge	1926	idx = intel_ring_sync_index(from, to);
		1927
		1928	seqno = obj->last_read_seqno;
		1929	if (seqno <= from->sync_seqno[idx])
		1930	return 0;
		1931
		1932	ret = i915_gem_check_olr(obj->ring, seqno);
		1933	if (ret)
		1934	return ret;
		1935
		1936	ret = to->sync_to(to, from, seqno);
		1937	if (!ret)
		1938	from->sync_seqno[idx] = seqno;
		1939
		1940	return ret;
2344	Serge	1941	}
2332	Serge	1942
2344	Serge	1943	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		1944	{
		1945	u32 old_write_domain, old_read_domains;
2332	Serge	1946
2344	Serge	1947	/* Act a barrier for all accesses through the GTT */
		1948	mb();
2332	Serge	1949
2344	Serge	1950	/* Force a pagefault for domain tracking on next user access */
		1951	// i915_gem_release_mmap(obj);
2332	Serge	1952
2344	Serge	1953	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		1954	return;
2332	Serge	1955
2344	Serge	1956	old_read_domains = obj->base.read_domains;
		1957	old_write_domain = obj->base.write_domain;
2351	Serge	1958
2344	Serge	1959	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		1960	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	1961
2351	Serge	1962	trace_i915_gem_object_change_domain(obj,
		1963	old_read_domains,
		1964	old_write_domain);
2344	Serge	1965	}
2332	Serge	1966
2344	Serge	1967	/**
		1968	* Unbinds an object from the GTT aperture.
		1969	*/
		1970	int
		1971	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		1972	{
3031	serge	1973	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	1974	int ret = 0;
2332	Serge	1975
2344	Serge	1976	if (obj->gtt_space == NULL)
		1977	return 0;
2332	Serge	1978
3031	serge	1979	if (obj->pin_count)
		1980	return -EBUSY;
2332	Serge	1981
3031	serge	1982	BUG_ON(obj->pages.page == NULL);
		1983
2344	Serge	1984	ret = i915_gem_object_finish_gpu(obj);
3031	serge	1985	if (ret)
2344	Serge	1986	return ret;
		1987	/* Continue on if we fail due to EIO, the GPU is hung so we
		1988	* should be safe and we need to cleanup or else we might
		1989	* cause memory corruption through use-after-free.
		1990	*/
2332	Serge	1991
2344	Serge	1992	i915_gem_object_finish_gtt(obj);
2332	Serge	1993
2344	Serge	1994	/* release the fence reg _after_ flushing */
		1995	ret = i915_gem_object_put_fence(obj);
3031	serge	1996	if (ret)
2344	Serge	1997	return ret;
2332	Serge	1998
2351	Serge	1999	trace_i915_gem_object_unbind(obj);
2332	Serge	2000
3031	serge	2001	if (obj->has_global_gtt_mapping)
2344	Serge	2002	i915_gem_gtt_unbind_object(obj);
3031	serge	2003	if (obj->has_aliasing_ppgtt_mapping) {
		2004	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2005	obj->has_aliasing_ppgtt_mapping = 0;
		2006	}
		2007	i915_gem_gtt_finish_object(obj);
2332	Serge	2008
3031	serge	2009	list_del(&obj->mm_list);
		2010	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2011	/* Avoid an unnecessary call to unbind on rebind. */
		2012	obj->map_and_fenceable = true;
2332	Serge	2013
2344	Serge	2014	drm_mm_put_block(obj->gtt_space);
		2015	obj->gtt_space = NULL;
		2016	obj->gtt_offset = 0;
2332	Serge	2017
2344	Serge	2018	return 0;
		2019	}
2332	Serge	2020
2344	Serge	2021	static int i915_ring_idle(struct intel_ring_buffer *ring)
		2022	{
3031	serge	2023	if (list_empty(&ring->active_list))
2344	Serge	2024	return 0;
2332	Serge	2025
3031	serge	2026	return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
2344	Serge	2027	}
2332	Serge	2028
3031	serge	2029	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2030	{
		2031	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2032	struct intel_ring_buffer *ring;
2344	Serge	2033	int ret, i;
2332	Serge	2034
2344	Serge	2035	/* Flush everything onto the inactive list. */
3031	serge	2036	for_each_ring(ring, dev_priv, i) {
		2037	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2038	if (ret)
		2039	return ret;
3031	serge	2040
		2041	ret = i915_ring_idle(ring);
		2042	if (ret)
		2043	return ret;
2344	Serge	2044	}
2332	Serge	2045
2344	Serge	2046	return 0;
		2047	}
2332	Serge	2048
3031	serge	2049	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2050	struct drm_i915_gem_object *obj)
		2051	{
		2052	drm_i915_private_t *dev_priv = dev->dev_private;
		2053	uint64_t val;
2332	Serge	2054
3031	serge	2055	if (obj) {
		2056	u32 size = obj->gtt_space->size;
2332	Serge	2057
3031	serge	2058	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2059	0xfffff000) << 32;
		2060	val \|= obj->gtt_offset & 0xfffff000;
		2061	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2062	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2063
3031	serge	2064	if (obj->tiling_mode == I915_TILING_Y)
		2065	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2066	val \|= I965_FENCE_REG_VALID;
		2067	} else
		2068	val = 0;
2332	Serge	2069
3031	serge	2070	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2071	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2072	}
2332	Serge	2073
3031	serge	2074	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2075	struct drm_i915_gem_object *obj)
		2076	{
		2077	drm_i915_private_t *dev_priv = dev->dev_private;
		2078	uint64_t val;
2332	Serge	2079
3031	serge	2080	if (obj) {
		2081	u32 size = obj->gtt_space->size;
2332	Serge	2082
3031	serge	2083	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2084	0xfffff000) << 32;
		2085	val \|= obj->gtt_offset & 0xfffff000;
		2086	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2087	if (obj->tiling_mode == I915_TILING_Y)
		2088	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2089	val \|= I965_FENCE_REG_VALID;
		2090	} else
		2091	val = 0;
2332	Serge	2092
3031	serge	2093	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2094	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2095	}
2332	Serge	2096
3031	serge	2097	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2098	struct drm_i915_gem_object *obj)
		2099	{
		2100	drm_i915_private_t *dev_priv = dev->dev_private;
		2101	u32 val;
2332	Serge	2102
3031	serge	2103	if (obj) {
		2104	u32 size = obj->gtt_space->size;
		2105	int pitch_val;
		2106	int tile_width;
2332	Serge	2107
3031	serge	2108	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2109	(size & -size) != size \|\|
		2110	(obj->gtt_offset & (size - 1)),
		2111	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2112	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2113
3031	serge	2114	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2115	tile_width = 128;
		2116	else
		2117	tile_width = 512;
2332	Serge	2118
3031	serge	2119	/* Note: pitch better be a power of two tile widths */
		2120	pitch_val = obj->stride / tile_width;
		2121	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2122
3031	serge	2123	val = obj->gtt_offset;
		2124	if (obj->tiling_mode == I915_TILING_Y)
		2125	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2126	val \|= I915_FENCE_SIZE_BITS(size);
		2127	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2128	val \|= I830_FENCE_REG_VALID;
		2129	} else
		2130	val = 0;
2332	Serge	2131
3031	serge	2132	if (reg < 8)
		2133	reg = FENCE_REG_830_0 + reg * 4;
		2134	else
		2135	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2136
3031	serge	2137	I915_WRITE(reg, val);
		2138	POSTING_READ(reg);
		2139	}
2332	Serge	2140
3031	serge	2141	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2142	struct drm_i915_gem_object *obj)
		2143	{
		2144	drm_i915_private_t *dev_priv = dev->dev_private;
		2145	uint32_t val;
2344	Serge	2146
3031	serge	2147	if (obj) {
		2148	u32 size = obj->gtt_space->size;
		2149	uint32_t pitch_val;
2344	Serge	2150
3031	serge	2151	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2152	(size & -size) != size \|\|
		2153	(obj->gtt_offset & (size - 1)),
		2154	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2155	obj->gtt_offset, size);
2344	Serge	2156
3031	serge	2157	pitch_val = obj->stride / 128;
		2158	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2159
3031	serge	2160	val = obj->gtt_offset;
		2161	if (obj->tiling_mode == I915_TILING_Y)
		2162	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2163	val \|= I830_FENCE_SIZE_BITS(size);
		2164	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2165	val \|= I830_FENCE_REG_VALID;
		2166	} else
		2167	val = 0;
		2168
		2169	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2170	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2171	}
		2172
		2173	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2174	struct drm_i915_gem_object *obj)
2332	Serge	2175	{
3031	serge	2176	switch (INTEL_INFO(dev)->gen) {
		2177	case 7:
		2178	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2179	case 5:
		2180	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2181	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2182	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2183	default: break;
		2184	}
2344	Serge	2185	}
		2186
3031	serge	2187	static inline int fence_number(struct drm_i915_private *dev_priv,
		2188	struct drm_i915_fence_reg *fence)
2344	Serge	2189	{
3031	serge	2190	return fence - dev_priv->fence_regs;
		2191	}
2332	Serge	2192
3031	serge	2193	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2194	struct drm_i915_fence_reg *fence,
		2195	bool enable)
		2196	{
		2197	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2198	int reg = fence_number(dev_priv, fence);
2332	Serge	2199
3031	serge	2200	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2201
		2202	if (enable) {
		2203	obj->fence_reg = reg;
		2204	fence->obj = obj;
		2205	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2206	} else {
		2207	obj->fence_reg = I915_FENCE_REG_NONE;
		2208	fence->obj = NULL;
		2209	list_del_init(&fence->lru_list);
2344	Serge	2210	}
3031	serge	2211	}
2344	Serge	2212
3031	serge	2213	static int
		2214	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2215	{
		2216	if (obj->last_fenced_seqno) {
		2217	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2218	if (ret)
		2219	return ret;
2344	Serge	2220
		2221	obj->last_fenced_seqno = 0;
		2222	}
		2223
		2224	/* Ensure that all CPU reads are completed before installing a fence
		2225	* and all writes before removing the fence.
2332	Serge	2226	*/
2344	Serge	2227	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2228	mb();
2332	Serge	2229
3031	serge	2230	obj->fenced_gpu_access = false;
2332	Serge	2231	return 0;
		2232	}
		2233
		2234	int
2344	Serge	2235	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2236	{
3031	serge	2237	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2238	int ret;
		2239
3031	serge	2240	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2241	if (ret)
		2242	return ret;
		2243
3031	serge	2244	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2245	return 0;
2332	Serge	2246
3031	serge	2247	i915_gem_object_update_fence(obj,
		2248	&dev_priv->fence_regs[obj->fence_reg],
		2249	false);
		2250	i915_gem_object_fence_lost(obj);
2344	Serge	2251
2332	Serge	2252	return 0;
		2253	}
		2254
3031	serge	2255	static struct drm_i915_fence_reg *
		2256	i915_find_fence_reg(struct drm_device *dev)
		2257	{
		2258	struct drm_i915_private *dev_priv = dev->dev_private;
		2259	struct drm_i915_fence_reg reg, avail;
		2260	int i;
2332	Serge	2261
3031	serge	2262	/* First try to find a free reg */
		2263	avail = NULL;
		2264	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2265	reg = &dev_priv->fence_regs[i];
		2266	if (!reg->obj)
		2267	return reg;
2332	Serge	2268
3031	serge	2269	if (!reg->pin_count)
		2270	avail = reg;
		2271	}
2332	Serge	2272
3031	serge	2273	if (avail == NULL)
		2274	return NULL;
2332	Serge	2275
3031	serge	2276	/* None available, try to steal one or wait for a user to finish */
		2277	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2278	if (reg->pin_count)
		2279	continue;
2332	Serge	2280
3031	serge	2281	return reg;
		2282	}
2332	Serge	2283
3031	serge	2284	return NULL;
		2285	}
2332	Serge	2286
3031	serge	2287	/**
		2288	* i915_gem_object_get_fence - set up fencing for an object
		2289	* @obj: object to map through a fence reg
		2290	*
		2291	* When mapping objects through the GTT, userspace wants to be able to write
		2292	* to them without having to worry about swizzling if the object is tiled.
		2293	* This function walks the fence regs looking for a free one for @obj,
		2294	* stealing one if it can't find any.
		2295	*
		2296	* It then sets up the reg based on the object's properties: address, pitch
		2297	* and tiling format.
		2298	*
		2299	* For an untiled surface, this removes any existing fence.
		2300	*/
		2301	int
		2302	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2303	{
		2304	struct drm_device *dev = obj->base.dev;
		2305	struct drm_i915_private *dev_priv = dev->dev_private;
		2306	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2307	struct drm_i915_fence_reg *reg;
		2308	int ret;
2332	Serge	2309
3031	serge	2310	/* Have we updated the tiling parameters upon the object and so
		2311	* will need to serialise the write to the associated fence register?
		2312	*/
		2313	if (obj->fence_dirty) {
		2314	ret = i915_gem_object_flush_fence(obj);
		2315	if (ret)
		2316	return ret;
		2317	}
2332	Serge	2318
3031	serge	2319	/* Just update our place in the LRU if our fence is getting reused. */
		2320	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2321	reg = &dev_priv->fence_regs[obj->fence_reg];
		2322	if (!obj->fence_dirty) {
		2323	list_move_tail(®->lru_list,
		2324	&dev_priv->mm.fence_list);
		2325	return 0;
		2326	}
		2327	} else if (enable) {
		2328	reg = i915_find_fence_reg(dev);
		2329	if (reg == NULL)
		2330	return -EDEADLK;
2332	Serge	2331
3031	serge	2332	if (reg->obj) {
		2333	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2334
3031	serge	2335	ret = i915_gem_object_flush_fence(old);
		2336	if (ret)
		2337	return ret;
2332	Serge	2338
3031	serge	2339	i915_gem_object_fence_lost(old);
		2340	}
		2341	} else
		2342	return 0;
2332	Serge	2343
3031	serge	2344	i915_gem_object_update_fence(obj, reg, enable);
		2345	obj->fence_dirty = false;
2332	Serge	2346
3031	serge	2347	return 0;
		2348	}
2332	Serge	2349
3031	serge	2350	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2351	struct drm_mm_node *gtt_space,
		2352	unsigned long cache_level)
		2353	{
		2354	struct drm_mm_node *other;
2332	Serge	2355
3031	serge	2356	/* On non-LLC machines we have to be careful when putting differing
		2357	* types of snoopable memory together to avoid the prefetcher
		2358	* crossing memory domains and dieing.
		2359	*/
		2360	if (HAS_LLC(dev))
		2361	return true;
2332	Serge	2362
3031	serge	2363	if (gtt_space == NULL)
		2364	return true;
2332	Serge	2365
3031	serge	2366	if (list_empty(>t_space->node_list))
		2367	return true;
2332	Serge	2368
3031	serge	2369	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2370	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2371	return false;
2344	Serge	2372
3031	serge	2373	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2374	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2375	return false;
2344	Serge	2376
3031	serge	2377	return true;
		2378	}
2344	Serge	2379
3031	serge	2380	static void i915_gem_verify_gtt(struct drm_device *dev)
		2381	{
		2382	#if WATCH_GTT
		2383	struct drm_i915_private *dev_priv = dev->dev_private;
		2384	struct drm_i915_gem_object *obj;
		2385	int err = 0;
2344	Serge	2386
3031	serge	2387	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2388	if (obj->gtt_space == NULL) {
		2389	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2390	err++;
		2391	continue;
		2392	}
2344	Serge	2393
3031	serge	2394	if (obj->cache_level != obj->gtt_space->color) {
		2395	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2396	obj->gtt_space->start,
		2397	obj->gtt_space->start + obj->gtt_space->size,
		2398	obj->cache_level,
		2399	obj->gtt_space->color);
		2400	err++;
		2401	continue;
		2402	}
2344	Serge	2403
3031	serge	2404	if (!i915_gem_valid_gtt_space(dev,
		2405	obj->gtt_space,
		2406	obj->cache_level)) {
		2407	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2408	obj->gtt_space->start,
		2409	obj->gtt_space->start + obj->gtt_space->size,
		2410	obj->cache_level);
		2411	err++;
		2412	continue;
		2413	}
		2414	}
2344	Serge	2415
3031	serge	2416	WARN_ON(err);
		2417	#endif
2326	Serge	2418	}
		2419
2332	Serge	2420	/**
		2421	* Finds free space in the GTT aperture and binds the object there.
		2422	*/
		2423	static int
		2424	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2425	unsigned alignment,
3031	serge	2426	bool map_and_fenceable,
		2427	bool nonblocking)
2332	Serge	2428	{
		2429	struct drm_device *dev = obj->base.dev;
		2430	drm_i915_private_t *dev_priv = dev->dev_private;
		2431	struct drm_mm_node *free_space;
		2432	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2433	bool mappable, fenceable;
		2434	int ret;
2326	Serge	2435
2332	Serge	2436	if (obj->madv != I915_MADV_WILLNEED) {
		2437	DRM_ERROR("Attempting to bind a purgeable object\n");
		2438	return -EINVAL;
		2439	}
		2440
		2441	fence_size = i915_gem_get_gtt_size(dev,
		2442	obj->base.size,
		2443	obj->tiling_mode);
		2444	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2445	obj->base.size,
		2446	obj->tiling_mode);
		2447	unfenced_alignment =
		2448	i915_gem_get_unfenced_gtt_alignment(dev,
		2449	obj->base.size,
		2450	obj->tiling_mode);
		2451
		2452	if (alignment == 0)
		2453	alignment = map_and_fenceable ? fence_alignment :
		2454	unfenced_alignment;
		2455	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2456	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2457	return -EINVAL;
		2458	}
		2459
		2460	size = map_and_fenceable ? fence_size : obj->base.size;
		2461
		2462	/* If the object is bigger than the entire aperture, reject it early
		2463	* before evicting everything in a vain attempt to find space.
		2464	*/
		2465	if (obj->base.size >
		2466	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2467	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2468	return -E2BIG;
		2469	}
		2470
3031	serge	2471	ret = i915_gem_object_get_pages(obj);
		2472	if (ret)
		2473	return ret;
		2474
2332	Serge	2475	search_free:
		2476	if (map_and_fenceable)
		2477	free_space =
3031	serge	2478	drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
		2479	size, alignment, obj->cache_level,
		2480	0, dev_priv->mm.gtt_mappable_end,
		2481	false);
2332	Serge	2482	else
3031	serge	2483	free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
		2484	size, alignment, obj->cache_level,
		2485	false);
2332	Serge	2486
		2487	if (free_space != NULL) {
		2488	if (map_and_fenceable)
		2489	obj->gtt_space =
		2490	drm_mm_get_block_range_generic(free_space,
3031	serge	2491	size, alignment, obj->cache_level,
		2492	0, dev_priv->mm.gtt_mappable_end,
		2493	false);
2332	Serge	2494	else
		2495	obj->gtt_space =
3031	serge	2496	drm_mm_get_block_generic(free_space,
		2497	size, alignment, obj->cache_level,
		2498	false);
2332	Serge	2499	}
		2500	if (obj->gtt_space == NULL) {
		2501	ret = 1; //i915_gem_evict_something(dev, size, alignment,
		2502	// map_and_fenceable);
		2503	if (ret)
		2504	return ret;
		2505
		2506	goto search_free;
		2507	}
3031	serge	2508	if (WARN_ON(!i915_gem_valid_gtt_space(dev,
		2509	obj->gtt_space,
		2510	obj->cache_level))) {
2332	Serge	2511	drm_mm_put_block(obj->gtt_space);
		2512	obj->gtt_space = NULL;
3031	serge	2513	return -EINVAL;
		2514	}
2332	Serge	2515
		2516
3031	serge	2517	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2518	if (ret) {
		2519	drm_mm_put_block(obj->gtt_space);
		2520	obj->gtt_space = NULL;
		2521	return ret;
		2522	}
		2523
3031	serge	2524	if (!dev_priv->mm.aliasing_ppgtt)
		2525	i915_gem_gtt_bind_object(obj, obj->cache_level);
		2526
		2527	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2528	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2529
		2530	obj->gtt_offset = obj->gtt_space->start;
		2531
		2532	fenceable =
		2533	obj->gtt_space->size == fence_size &&
2342	Serge	2534	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2332	Serge	2535
		2536	mappable =
		2537	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2538
		2539	obj->map_and_fenceable = mappable && fenceable;
		2540
2351	Serge	2541	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2542	i915_gem_verify_gtt(dev);
2332	Serge	2543	return 0;
		2544	}
		2545
		2546	void
		2547	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2548	{
		2549	/* If we don't have a page list set up, then we're not pinned
		2550	* to GPU, and we can ignore the cache flush because it'll happen
		2551	* again at bind time.
		2552	*/
3031	serge	2553	if (obj->pages.page == NULL)
2332	Serge	2554	return;
		2555
		2556	/* If the GPU is snooping the contents of the CPU cache,
		2557	* we do not need to manually clear the CPU cache lines. However,
		2558	* the caches are only snooped when the render cache is
		2559	* flushed/invalidated. As we always have to emit invalidations
		2560	* and flushes when moving into and out of the RENDER domain, correct
		2561	* snooping behaviour occurs naturally as the result of our domain
		2562	* tracking.
		2563	*/
		2564	if (obj->cache_level != I915_CACHE_NONE)
		2565	return;
		2566
2344	Serge	2567	if(obj->mapped != NULL)
		2568	{
		2569	uint8_t *page_virtual;
		2570	unsigned int i;
2332	Serge	2571
2344	Serge	2572	page_virtual = obj->mapped;
		2573	asm volatile("mfence");
		2574	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2575	clflush(page_virtual + i);
		2576	asm volatile("mfence");
		2577	}
		2578	else
		2579	{
		2580	uint8_t *page_virtual;
		2581	unsigned int i;
		2582	page_virtual = AllocKernelSpace(obj->base.size);
		2583	if(page_virtual != NULL)
		2584	{
3031	serge	2585	dma_addr_t src, dst;
2344	Serge	2586	u32 count;
		2587
		2588	#define page_tabs 0xFDC00000 /* really dirty hack */
		2589
3031	serge	2590	src = obj->pages.page;
		2591	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2592	count = obj->base.size/4096;
		2593
		2594	while(count--)
		2595	{
		2596	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2597	};
		2598
		2599	asm volatile("mfence");
		2600	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2601	clflush(page_virtual + i);
		2602	asm volatile("mfence");
		2603	FreeKernelSpace(page_virtual);
		2604	}
		2605	else
		2606	{
		2607	asm volatile (
		2608	"mfence \n"
		2609	"wbinvd \n" /* this is really ugly */
		2610	"mfence");
		2611	}
		2612	}
2332	Serge	2613	}
		2614
2344	Serge	2615	/** Flushes the GTT write domain for the object if it's dirty. */
		2616	static void
		2617	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2618	{
		2619	uint32_t old_write_domain;
2332	Serge	2620
2344	Serge	2621	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2622	return;
2332	Serge	2623
2344	Serge	2624	/* No actual flushing is required for the GTT write domain. Writes
		2625	* to it immediately go to main memory as far as we know, so there's
		2626	* no chipset flush. It also doesn't land in render cache.
		2627	*
		2628	* However, we do have to enforce the order so that all writes through
		2629	* the GTT land before any writes to the device, such as updates to
		2630	* the GATT itself.
		2631	*/
		2632	wmb();
2332	Serge	2633
2344	Serge	2634	old_write_domain = obj->base.write_domain;
		2635	obj->base.write_domain = 0;
2332	Serge	2636
2351	Serge	2637	trace_i915_gem_object_change_domain(obj,
		2638	obj->base.read_domains,
		2639	old_write_domain);
2344	Serge	2640	}
2332	Serge	2641
		2642	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2643	static void
2332	Serge	2644	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2645	{
		2646	uint32_t old_write_domain;
		2647
		2648	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2649	return;
		2650
		2651	i915_gem_clflush_object(obj);
		2652	intel_gtt_chipset_flush();
		2653	old_write_domain = obj->base.write_domain;
		2654	obj->base.write_domain = 0;
		2655
2351	Serge	2656	trace_i915_gem_object_change_domain(obj,
		2657	obj->base.read_domains,
		2658	old_write_domain);
2332	Serge	2659	}
		2660
		2661	/**
		2662	* Moves a single object to the GTT read, and possibly write domain.
		2663	*
		2664	* This function returns when the move is complete, including waiting on
		2665	* flushes to occur.
		2666	*/
		2667	int
		2668	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2669	{
3031	serge	2670	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2671	uint32_t old_write_domain, old_read_domains;
		2672	int ret;
		2673
		2674	/* Not valid to be called on unbound objects. */
		2675	if (obj->gtt_space == NULL)
		2676	return -EINVAL;
		2677
		2678	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2679	return 0;
		2680
3031	serge	2681	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2682	if (ret)
		2683	return ret;
		2684
		2685	i915_gem_object_flush_cpu_write_domain(obj);
		2686
		2687	old_write_domain = obj->base.write_domain;
		2688	old_read_domains = obj->base.read_domains;
		2689
		2690	/* It should now be out of any other write domains, and we can update
		2691	* the domain values for our changes.
		2692	*/
		2693	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2694	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2695	if (write) {
		2696	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2697	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2698	obj->dirty = 1;
		2699	}
		2700
2351	Serge	2701	trace_i915_gem_object_change_domain(obj,
		2702	old_read_domains,
		2703	old_write_domain);
		2704
3031	serge	2705	/* And bump the LRU for this access */
		2706	if (i915_gem_object_is_inactive(obj))
		2707	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2708
2332	Serge	2709	return 0;
		2710	}
		2711
2335	Serge	2712	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2713	enum i915_cache_level cache_level)
		2714	{
3031	serge	2715	struct drm_device *dev = obj->base.dev;
		2716	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2717	int ret;
2332	Serge	2718
2335	Serge	2719	if (obj->cache_level == cache_level)
		2720	return 0;
2332	Serge	2721
2335	Serge	2722	if (obj->pin_count) {
		2723	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2724	return -EBUSY;
		2725	}
2332	Serge	2726
3031	serge	2727	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2728	ret = i915_gem_object_unbind(obj);
		2729	if (ret)
		2730	return ret;
		2731	}
		2732
2335	Serge	2733	if (obj->gtt_space) {
		2734	ret = i915_gem_object_finish_gpu(obj);
		2735	if (ret)
		2736	return ret;
2332	Serge	2737
2335	Serge	2738	i915_gem_object_finish_gtt(obj);
2332	Serge	2739
2335	Serge	2740	/* Before SandyBridge, you could not use tiling or fence
		2741	* registers with snooped memory, so relinquish any fences
		2742	* currently pointing to our region in the aperture.
		2743	*/
3031	serge	2744	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2745	ret = i915_gem_object_put_fence(obj);
		2746	if (ret)
		2747	return ret;
		2748	}
2332	Serge	2749
3031	serge	2750	if (obj->has_global_gtt_mapping)
		2751	i915_gem_gtt_bind_object(obj, cache_level);
		2752	if (obj->has_aliasing_ppgtt_mapping)
		2753	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2754	obj, cache_level);
		2755
		2756	obj->gtt_space->color = cache_level;
2335	Serge	2757	}
2332	Serge	2758
2335	Serge	2759	if (cache_level == I915_CACHE_NONE) {
		2760	u32 old_read_domains, old_write_domain;
2332	Serge	2761
2335	Serge	2762	/* If we're coming from LLC cached, then we haven't
		2763	* actually been tracking whether the data is in the
		2764	* CPU cache or not, since we only allow one bit set
		2765	* in obj->write_domain and have been skipping the clflushes.
		2766	* Just set it to the CPU cache for now.
		2767	*/
		2768	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2769	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2770
2335	Serge	2771	old_read_domains = obj->base.read_domains;
		2772	old_write_domain = obj->base.write_domain;
2332	Serge	2773
2335	Serge	2774	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2775	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2776
2351	Serge	2777	trace_i915_gem_object_change_domain(obj,
		2778	old_read_domains,
		2779	old_write_domain);
2344	Serge	2780	}
2332	Serge	2781
2335	Serge	2782	obj->cache_level = cache_level;
3031	serge	2783	i915_gem_verify_gtt(dev);
2335	Serge	2784	return 0;
		2785	}
2332	Serge	2786
2335	Serge	2787	/*
		2788	* Prepare buffer for display plane (scanout, cursors, etc).
		2789	* Can be called from an uninterruptible phase (modesetting) and allows
		2790	* any flushes to be pipelined (for pageflips).
		2791	*/
		2792	int
		2793	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		2794	u32 alignment,
		2795	struct intel_ring_buffer *pipelined)
		2796	{
		2797	u32 old_read_domains, old_write_domain;
		2798	int ret;
2332	Serge	2799
3031	serge	2800	if (pipelined != obj->ring) {
		2801	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	2802	if (ret)
		2803	return ret;
		2804	}
2332	Serge	2805
2335	Serge	2806	/* The display engine is not coherent with the LLC cache on gen6. As
		2807	* a result, we make sure that the pinning that is about to occur is
		2808	* done with uncached PTEs. This is lowest common denominator for all
		2809	* chipsets.
		2810	*
		2811	* However for gen6+, we could do better by using the GFDT bit instead
		2812	* of uncaching, which would allow us to flush all the LLC-cached data
		2813	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		2814	*/
2360	Serge	2815	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		2816	if (ret)
		2817	return ret;
2332	Serge	2818
2335	Serge	2819	/* As the user may map the buffer once pinned in the display plane
		2820	* (e.g. libkms for the bootup splash), we have to ensure that we
		2821	* always use map_and_fenceable for all scanout buffers.
		2822	*/
3031	serge	2823	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	2824	if (ret)
		2825	return ret;
2332	Serge	2826
2335	Serge	2827	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	2828
2335	Serge	2829	old_write_domain = obj->base.write_domain;
		2830	old_read_domains = obj->base.read_domains;
2332	Serge	2831
2335	Serge	2832	/* It should now be out of any other write domains, and we can update
		2833	* the domain values for our changes.
		2834	*/
3031	serge	2835	obj->base.write_domain = 0;
2335	Serge	2836	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	2837
2351	Serge	2838	trace_i915_gem_object_change_domain(obj,
		2839	old_read_domains,
		2840	old_write_domain);
2332	Serge	2841
2335	Serge	2842	return 0;
		2843	}
2332	Serge	2844
2344	Serge	2845	int
		2846	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		2847	{
		2848	int ret;
2332	Serge	2849
2344	Serge	2850	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		2851	return 0;
2332	Serge	2852
3031	serge	2853	ret = i915_gem_object_wait_rendering(obj, false);
2344	Serge	2854	if (ret)
		2855	return ret;
2332	Serge	2856
2344	Serge	2857	/* Ensure that we invalidate the GPU's caches and TLBs. */
		2858	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	2859	return 0;
2344	Serge	2860	}
2332	Serge	2861
2344	Serge	2862	/**
		2863	* Moves a single object to the CPU read, and possibly write domain.
		2864	*
		2865	* This function returns when the move is complete, including waiting on
		2866	* flushes to occur.
		2867	*/
3031	serge	2868	int
2344	Serge	2869	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		2870	{
		2871	uint32_t old_write_domain, old_read_domains;
		2872	int ret;
2332	Serge	2873
2344	Serge	2874	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		2875	return 0;
2332	Serge	2876
3031	serge	2877	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	2878	if (ret)
		2879	return ret;
2332	Serge	2880
2344	Serge	2881	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	2882
2344	Serge	2883	old_write_domain = obj->base.write_domain;
		2884	old_read_domains = obj->base.read_domains;
2332	Serge	2885
2344	Serge	2886	/* Flush the CPU cache if it's still invalid. */
		2887	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		2888	i915_gem_clflush_object(obj);
2332	Serge	2889
2344	Serge	2890	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		2891	}
2332	Serge	2892
2344	Serge	2893	/* It should now be out of any other write domains, and we can update
		2894	* the domain values for our changes.
		2895	*/
		2896	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	2897
2344	Serge	2898	/* If we're writing through the CPU, then the GPU read domains will
		2899	* need to be invalidated at next use.
		2900	*/
		2901	if (write) {
		2902	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2903	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2904	}
2332	Serge	2905
2351	Serge	2906	trace_i915_gem_object_change_domain(obj,
		2907	old_read_domains,
		2908	old_write_domain);
2332	Serge	2909
2344	Serge	2910	return 0;
		2911	}
2332	Serge	2912
3031	serge	2913	#if 0
		2914	/* Throttle our rendering by waiting until the ring has completed our requests
		2915	* emitted over 20 msec ago.
2344	Serge	2916	*
3031	serge	2917	* Note that if we were to use the current jiffies each time around the loop,
		2918	* we wouldn't escape the function with any frames outstanding if the time to
		2919	* render a frame was over 20ms.
		2920	*
		2921	* This should get us reasonable parallelism between CPU and GPU but also
		2922	* relatively low latency when blocking on a particular request to finish.
2344	Serge	2923	*/
3031	serge	2924	static int
		2925	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	2926	{
3031	serge	2927	struct drm_i915_private *dev_priv = dev->dev_private;
		2928	struct drm_i915_file_private *file_priv = file->driver_priv;
		2929	unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20);
		2930	struct drm_i915_gem_request *request;
		2931	struct intel_ring_buffer *ring = NULL;
		2932	u32 seqno = 0;
		2933	int ret;
2332	Serge	2934
3031	serge	2935	if (atomic_read(&dev_priv->mm.wedged))
		2936	return -EIO;
2332	Serge	2937
3031	serge	2938	spin_lock(&file_priv->mm.lock);
		2939	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		2940	if (time_after_eq(request->emitted_jiffies, recent_enough))
		2941	break;
2332	Serge	2942
3031	serge	2943	ring = request->ring;
		2944	seqno = request->seqno;
		2945	}
		2946	spin_unlock(&file_priv->mm.lock);
2332	Serge	2947
3031	serge	2948	if (seqno == 0)
		2949	return 0;
2332	Serge	2950
3031	serge	2951	ret = __wait_seqno(ring, seqno, true, NULL);
		2952	if (ret == 0)
		2953	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	2954
3031	serge	2955	return ret;
2352	Serge	2956	}
3031	serge	2957	#endif
2332	Serge	2958
		2959	int
		2960	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		2961	uint32_t alignment,
3031	serge	2962	bool map_and_fenceable,
		2963	bool nonblocking)
2332	Serge	2964	{
		2965	int ret;
		2966
3031	serge	2967	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		2968	return -EBUSY;
2332	Serge	2969
		2970	#if 0
		2971	if (obj->gtt_space != NULL) {
		2972	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		2973	(map_and_fenceable && !obj->map_and_fenceable)) {
		2974	WARN(obj->pin_count,
		2975	"bo is already pinned with incorrect alignment:"
		2976	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		2977	" obj->map_and_fenceable=%d\n",
		2978	obj->gtt_offset, alignment,
		2979	map_and_fenceable,
		2980	obj->map_and_fenceable);
		2981	ret = i915_gem_object_unbind(obj);
		2982	if (ret)
		2983	return ret;
		2984	}
		2985	}
		2986	#endif
		2987
		2988	if (obj->gtt_space == NULL) {
		2989	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	2990	map_and_fenceable,
		2991	nonblocking);
2332	Serge	2992	if (ret)
		2993	return ret;
		2994	}
		2995
3031	serge	2996	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		2997	i915_gem_gtt_bind_object(obj, obj->cache_level);
		2998
		2999	obj->pin_count++;
2332	Serge	3000	obj->pin_mappable \|= map_and_fenceable;
		3001
		3002	return 0;
		3003	}
		3004
2344	Serge	3005	void
		3006	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3007	{
		3008	BUG_ON(obj->pin_count == 0);
		3009	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3010
3031	serge	3011	if (--obj->pin_count == 0)
2344	Serge	3012	obj->pin_mappable = false;
		3013	}
2332	Serge	3014
3031	serge	3015	#if 0
		3016	int
		3017	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3018	struct drm_file *file)
		3019	{
		3020	struct drm_i915_gem_pin *args = data;
		3021	struct drm_i915_gem_object *obj;
		3022	int ret;
2332	Serge	3023
3031	serge	3024	ret = i915_mutex_lock_interruptible(dev);
		3025	if (ret)
		3026	return ret;
2332	Serge	3027
3031	serge	3028	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3029	if (&obj->base == NULL) {
		3030	ret = -ENOENT;
		3031	goto unlock;
		3032	}
2332	Serge	3033
3031	serge	3034	if (obj->madv != I915_MADV_WILLNEED) {
		3035	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3036	ret = -EINVAL;
		3037	goto out;
		3038	}
2332	Serge	3039
3031	serge	3040	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3041	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3042	args->handle);
		3043	ret = -EINVAL;
		3044	goto out;
		3045	}
2332	Serge	3046
3031	serge	3047	obj->user_pin_count++;
		3048	obj->pin_filp = file;
		3049	if (obj->user_pin_count == 1) {
		3050	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3051	if (ret)
		3052	goto out;
		3053	}
2332	Serge	3054
3031	serge	3055	/* XXX - flush the CPU caches for pinned objects
		3056	* as the X server doesn't manage domains yet
		3057	*/
		3058	i915_gem_object_flush_cpu_write_domain(obj);
		3059	args->offset = obj->gtt_offset;
		3060	out:
		3061	drm_gem_object_unreference(&obj->base);
		3062	unlock:
		3063	mutex_unlock(&dev->struct_mutex);
		3064	return ret;
		3065	}
2332	Serge	3066
3031	serge	3067	int
		3068	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3069	struct drm_file *file)
		3070	{
		3071	struct drm_i915_gem_pin *args = data;
		3072	struct drm_i915_gem_object *obj;
		3073	int ret;
2332	Serge	3074
3031	serge	3075	ret = i915_mutex_lock_interruptible(dev);
		3076	if (ret)
		3077	return ret;
2332	Serge	3078
3031	serge	3079	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3080	if (&obj->base == NULL) {
		3081	ret = -ENOENT;
		3082	goto unlock;
		3083	}
2332	Serge	3084
3031	serge	3085	if (obj->pin_filp != file) {
		3086	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3087	args->handle);
		3088	ret = -EINVAL;
		3089	goto out;
		3090	}
		3091	obj->user_pin_count--;
		3092	if (obj->user_pin_count == 0) {
		3093	obj->pin_filp = NULL;
		3094	i915_gem_object_unpin(obj);
		3095	}
2332	Serge	3096
3031	serge	3097	out:
		3098	drm_gem_object_unreference(&obj->base);
		3099	unlock:
		3100	mutex_unlock(&dev->struct_mutex);
		3101	return ret;
		3102	}
2332	Serge	3103
3031	serge	3104	int
		3105	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3106	struct drm_file *file)
		3107	{
		3108	struct drm_i915_gem_busy *args = data;
		3109	struct drm_i915_gem_object *obj;
		3110	int ret;
2332	Serge	3111
3031	serge	3112	ret = i915_mutex_lock_interruptible(dev);
		3113	if (ret)
		3114	return ret;
2332	Serge	3115
3031	serge	3116	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3117	if (&obj->base == NULL) {
		3118	ret = -ENOENT;
		3119	goto unlock;
		3120	}
2332	Serge	3121
3031	serge	3122	/* Count all active objects as busy, even if they are currently not used
		3123	* by the gpu. Users of this interface expect objects to eventually
		3124	* become non-busy without any further actions, therefore emit any
		3125	* necessary flushes here.
		3126	*/
		3127	ret = i915_gem_object_flush_active(obj);
2332	Serge	3128
3031	serge	3129	args->busy = obj->active;
		3130	if (obj->ring) {
		3131	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3132	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3133	}
2332	Serge	3134
3031	serge	3135	drm_gem_object_unreference(&obj->base);
		3136	unlock:
		3137	mutex_unlock(&dev->struct_mutex);
		3138	return ret;
		3139	}
2332	Serge	3140
3031	serge	3141	int
		3142	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3143	struct drm_file *file_priv)
		3144	{
		3145	return i915_gem_ring_throttle(dev, file_priv);
		3146	}
2332	Serge	3147
3031	serge	3148	int
		3149	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3150	struct drm_file *file_priv)
		3151	{
		3152	struct drm_i915_gem_madvise *args = data;
		3153	struct drm_i915_gem_object *obj;
		3154	int ret;
2332	Serge	3155
3031	serge	3156	switch (args->madv) {
		3157	case I915_MADV_DONTNEED:
		3158	case I915_MADV_WILLNEED:
		3159	break;
		3160	default:
		3161	return -EINVAL;
		3162	}
2332	Serge	3163
3031	serge	3164	ret = i915_mutex_lock_interruptible(dev);
		3165	if (ret)
		3166	return ret;
2332	Serge	3167
3031	serge	3168	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3169	if (&obj->base == NULL) {
		3170	ret = -ENOENT;
		3171	goto unlock;
		3172	}
2332	Serge	3173
3031	serge	3174	if (obj->pin_count) {
		3175	ret = -EINVAL;
		3176	goto out;
		3177	}
2332	Serge	3178
3031	serge	3179	if (obj->madv != __I915_MADV_PURGED)
		3180	obj->madv = args->madv;
2332	Serge	3181
3031	serge	3182	/* if the object is no longer attached, discard its backing storage */
		3183	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3184	i915_gem_object_truncate(obj);
2332	Serge	3185
3031	serge	3186	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3187
3031	serge	3188	out:
		3189	drm_gem_object_unreference(&obj->base);
		3190	unlock:
		3191	mutex_unlock(&dev->struct_mutex);
		3192	return ret;
		3193	}
		3194	#endif
2332	Serge	3195
3031	serge	3196	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3197	const struct drm_i915_gem_object_ops *ops)
		3198	{
		3199	INIT_LIST_HEAD(&obj->mm_list);
		3200	INIT_LIST_HEAD(&obj->gtt_list);
		3201	INIT_LIST_HEAD(&obj->ring_list);
		3202	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3203
3031	serge	3204	obj->ops = ops;
		3205
		3206	obj->fence_reg = I915_FENCE_REG_NONE;
		3207	obj->madv = I915_MADV_WILLNEED;
		3208	/* Avoid an unnecessary call to unbind on the first bind. */
		3209	obj->map_and_fenceable = true;
		3210
		3211	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3212	}
		3213
		3214	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3215	.get_pages = i915_gem_object_get_pages_gtt,
		3216	.put_pages = i915_gem_object_put_pages_gtt,
		3217	};
		3218
2332	Serge	3219	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3220	size_t size)
		3221	{
		3222	struct drm_i915_gem_object *obj;
3031	serge	3223	struct address_space *mapping;
		3224	u32 mask;
2340	Serge	3225
2332	Serge	3226	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3227	if (obj == NULL)
		3228	return NULL;
		3229
		3230	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3231	kfree(obj);
		3232	return NULL;
		3233	}
		3234
		3235
3031	serge	3236	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3237
		3238	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3239	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3240
3031	serge	3241	if (HAS_LLC(dev)) {
		3242	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3243	* cache) for about a 10% performance improvement
		3244	* compared to uncached. Graphics requests other than
		3245	* display scanout are coherent with the CPU in
		3246	* accessing this cache. This means in this mode we
		3247	* don't need to clflush on the CPU side, and on the
		3248	* GPU side we only need to flush internal caches to
		3249	* get data visible to the CPU.
		3250	*
		3251	* However, we maintain the display planes as UC, and so
		3252	* need to rebind when first used as such.
		3253	*/
		3254	obj->cache_level = I915_CACHE_LLC;
		3255	} else
		3256	obj->cache_level = I915_CACHE_NONE;
		3257
		3258	return obj;
		3259	}
		3260
2344	Serge	3261	int i915_gem_init_object(struct drm_gem_object *obj)
		3262	{
		3263	BUG();
2332	Serge	3264
2344	Serge	3265	return 0;
		3266	}
2332	Serge	3267
3031	serge	3268	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3269	{
3031	serge	3270	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3271	struct drm_device *dev = obj->base.dev;
		3272	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3273
3031	serge	3274	trace_i915_gem_object_destroy(obj);
		3275
		3276	// if (obj->phys_obj)
		3277	// i915_gem_detach_phys_object(dev, obj);
		3278
		3279	obj->pin_count = 0;
		3280	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3281	bool was_interruptible;
		3282
		3283	was_interruptible = dev_priv->mm.interruptible;
		3284	dev_priv->mm.interruptible = false;
		3285
		3286	WARN_ON(i915_gem_object_unbind(obj));
		3287
		3288	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3289	}
2332	Serge	3290
3031	serge	3291	obj->pages_pin_count = 0;
		3292	i915_gem_object_put_pages(obj);
		3293	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3294
3031	serge	3295	BUG_ON(obj->pages.page);
2332	Serge	3296
3031	serge	3297	// if (obj->base.import_attach)
		3298	// drm_prime_gem_destroy(&obj->base, NULL);
		3299
2344	Serge	3300	drm_gem_object_release(&obj->base);
		3301	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3302
2344	Serge	3303	kfree(obj->bit_17);
		3304	kfree(obj);
		3305	}
2332	Serge	3306
3031	serge	3307	#if 0
		3308	int
		3309	i915_gem_idle(struct drm_device *dev)
2344	Serge	3310	{
3031	serge	3311	drm_i915_private_t *dev_priv = dev->dev_private;
		3312	int ret;
2332	Serge	3313
3031	serge	3314	mutex_lock(&dev->struct_mutex);
2332	Serge	3315
3031	serge	3316	if (dev_priv->mm.suspended) {
		3317	mutex_unlock(&dev->struct_mutex);
		3318	return 0;
		3319	}
2332	Serge	3320
3031	serge	3321	ret = i915_gpu_idle(dev);
		3322	if (ret) {
		3323	mutex_unlock(&dev->struct_mutex);
		3324	return ret;
		3325	}
		3326	i915_gem_retire_requests(dev);
		3327
		3328	i915_gem_reset_fences(dev);
		3329
		3330	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3331	* We need to replace this with a semaphore, or something.
		3332	* And not confound mm.suspended!
		3333	*/
		3334	dev_priv->mm.suspended = 1;
		3335	del_timer_sync(&dev_priv->hangcheck_timer);
		3336
		3337	i915_kernel_lost_context(dev);
		3338	i915_gem_cleanup_ringbuffer(dev);
		3339
		3340	mutex_unlock(&dev->struct_mutex);
		3341
		3342	/* Cancel the retire work handler, which should be idle now. */
		3343	// cancel_delayed_work_sync(&dev_priv->mm.retire_work);
		3344
		3345	return 0;
2344	Serge	3346	}
3031	serge	3347	#endif
2332	Serge	3348
3031	serge	3349	void i915_gem_l3_remap(struct drm_device *dev)
		3350	{
		3351	drm_i915_private_t *dev_priv = dev->dev_private;
		3352	u32 misccpctl;
		3353	int i;
2332	Serge	3354
3031	serge	3355	if (!IS_IVYBRIDGE(dev))
		3356	return;
2332	Serge	3357
3031	serge	3358	if (!dev_priv->mm.l3_remap_info)
		3359	return;
2332	Serge	3360
3031	serge	3361	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3362	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3363	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3364
3031	serge	3365	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3366	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
		3367	if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
		3368	DRM_DEBUG("0x%x was already programmed to %x\n",
		3369	GEN7_L3LOG_BASE + i, remap);
		3370	if (remap && !dev_priv->mm.l3_remap_info[i/4])
		3371	DRM_DEBUG_DRIVER("Clearing remapped register\n");
		3372	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
		3373	}
2332	Serge	3374
3031	serge	3375	/* Make sure all the writes land before disabling dop clock gating */
		3376	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3377
3031	serge	3378	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3379	}
2332	Serge	3380
3031	serge	3381	void i915_gem_init_swizzling(struct drm_device *dev)
		3382	{
		3383	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3384
3031	serge	3385	if (INTEL_INFO(dev)->gen < 5 \|\|
		3386	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3387	return;
2332	Serge	3388
3031	serge	3389	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3390	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3391
3031	serge	3392	if (IS_GEN5(dev))
		3393	return;
2344	Serge	3394
3031	serge	3395	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3396	if (IS_GEN6(dev))
		3397	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3398	else
		3399	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3400	}
		3401
		3402	void i915_gem_init_ppgtt(struct drm_device *dev)
		3403	{
		3404	drm_i915_private_t *dev_priv = dev->dev_private;
		3405	uint32_t pd_offset;
		3406	struct intel_ring_buffer *ring;
		3407	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
		3408	uint32_t __iomem *pd_addr;
		3409	uint32_t pd_entry;
		3410	int i;
		3411
		3412	if (!dev_priv->mm.aliasing_ppgtt)
		3413	return;
		3414
		3415
		3416	pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
		3417	for (i = 0; i < ppgtt->num_pd_entries; i++) {
		3418	dma_addr_t pt_addr;
		3419
		3420	if (dev_priv->mm.gtt->needs_dmar)
		3421	pt_addr = ppgtt->pt_dma_addr[i];
		3422	else
		3423	pt_addr = ppgtt->pt_pages[i];
		3424
		3425	pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
		3426	pd_entry \|= GEN6_PDE_VALID;
		3427
		3428	writel(pd_entry, pd_addr + i);
		3429	}
		3430	readl(pd_addr);
		3431
		3432	pd_offset = ppgtt->pd_offset;
		3433	pd_offset /= 64; /* in cachelines, */
		3434	pd_offset <<= 16;
		3435
		3436	if (INTEL_INFO(dev)->gen == 6) {
		3437	uint32_t ecochk, gab_ctl, ecobits;
		3438
		3439	ecobits = I915_READ(GAC_ECO_BITS);
		3440	I915_WRITE(GAC_ECO_BITS, ecobits \| ECOBITS_PPGTT_CACHE64B);
		3441
		3442	gab_ctl = I915_READ(GAB_CTL);
		3443	I915_WRITE(GAB_CTL, gab_ctl \| GAB_CTL_CONT_AFTER_PAGEFAULT);
		3444
		3445	ecochk = I915_READ(GAM_ECOCHK);
		3446	I915_WRITE(GAM_ECOCHK, ecochk \| ECOCHK_SNB_BIT \|
		3447	ECOCHK_PPGTT_CACHE64B);
		3448	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
		3449	} else if (INTEL_INFO(dev)->gen >= 7) {
		3450	I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
		3451	/* GFX_MODE is per-ring on gen7+ */
		3452	}
		3453
		3454	for_each_ring(ring, dev_priv, i) {
		3455	if (INTEL_INFO(dev)->gen >= 7)
		3456	I915_WRITE(RING_MODE_GEN7(ring),
		3457	_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
		3458
		3459	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
		3460	I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
		3461	}
		3462	}
		3463
		3464	static bool
		3465	intel_enable_blt(struct drm_device *dev)
		3466	{
		3467	if (!HAS_BLT(dev))
		3468	return false;
		3469
		3470	/* The blitter was dysfunctional on early prototypes */
		3471	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3472	DRM_INFO("BLT not supported on this pre-production hardware;"
		3473	" graphics performance will be degraded.\n");
		3474	return false;
		3475	}
		3476
		3477	return true;
		3478	}
		3479
2332	Serge	3480	int
3031	serge	3481	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3482	{
		3483	drm_i915_private_t *dev_priv = dev->dev_private;
		3484	int ret;
2351	Serge	3485
3031	serge	3486	if (!intel_enable_gtt())
		3487	return -EIO;
		3488
		3489	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3490	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3491
		3492	i915_gem_l3_remap(dev);
		3493
		3494	i915_gem_init_swizzling(dev);
		3495
2332	Serge	3496	ret = intel_init_render_ring_buffer(dev);
		3497	if (ret)
		3498	return ret;
		3499
		3500	if (HAS_BSD(dev)) {
		3501	ret = intel_init_bsd_ring_buffer(dev);
		3502	if (ret)
		3503	goto cleanup_render_ring;
		3504	}
		3505
3031	serge	3506	if (intel_enable_blt(dev)) {
2332	Serge	3507	ret = intel_init_blt_ring_buffer(dev);
		3508	if (ret)
		3509	goto cleanup_bsd_ring;
		3510	}
		3511
		3512	dev_priv->next_seqno = 1;
2351	Serge	3513
3031	serge	3514	/*
		3515	* XXX: There was some w/a described somewhere suggesting loading
		3516	* contexts before PPGTT.
		3517	*/
		3518	i915_gem_context_init(dev);
		3519	i915_gem_init_ppgtt(dev);
		3520
2332	Serge	3521	return 0;
		3522
		3523	cleanup_bsd_ring:
		3524	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3525	cleanup_render_ring:
		3526	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3527	return ret;
		3528	}
		3529
3031	serge	3530	static bool
		3531	intel_enable_ppgtt(struct drm_device *dev)
		3532	{
		3533	if (i915_enable_ppgtt >= 0)
		3534	return i915_enable_ppgtt;
		3535
		3536	#ifdef CONFIG_INTEL_IOMMU
		3537	/* Disable ppgtt on SNB if VT-d is on. */
		3538	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3539	return false;
		3540	#endif
		3541
		3542	return true;
		3543	}
		3544
		3545	#define LFB_SIZE 0xC00000
		3546
		3547	int i915_gem_init(struct drm_device *dev)
		3548	{
		3549	struct drm_i915_private *dev_priv = dev->dev_private;
		3550	unsigned long gtt_size, mappable_size;
		3551	int ret;
		3552
		3553	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3554	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3555
		3556	mutex_lock(&dev->struct_mutex);
		3557	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3558	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3559	* aperture accordingly when using aliasing ppgtt. */
		3560	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3561
		3562	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3563
		3564	ret = i915_gem_init_aliasing_ppgtt(dev);
		3565	if (ret) {
		3566	mutex_unlock(&dev->struct_mutex);
		3567	return ret;
		3568	}
		3569	} else {
		3570	/* Let GEM Manage all of the aperture.
		3571	*
		3572	* However, leave one page at the end still bound to the scratch
		3573	* page. There are a number of places where the hardware
		3574	* apparently prefetches past the end of the object, and we've
		3575	* seen multiple hangs with the GPU head pointer stuck in a
		3576	* batchbuffer bound at the last page of the aperture. One page
		3577	* should be enough to keep any prefetching inside of the
		3578	* aperture.
		3579	*/
		3580	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3581	}
		3582
		3583	ret = i915_gem_init_hw(dev);
		3584	mutex_unlock(&dev->struct_mutex);
		3585	if (ret) {
		3586	i915_gem_cleanup_aliasing_ppgtt(dev);
		3587	return ret;
		3588	}
		3589
		3590	return 0;
		3591	}
		3592
2332	Serge	3593	void
		3594	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3595	{
		3596	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3597	struct intel_ring_buffer *ring;
2332	Serge	3598	int i;
		3599
3031	serge	3600	for_each_ring(ring, dev_priv, i)
		3601	intel_cleanup_ring_buffer(ring);
2332	Serge	3602	}
		3603
3031	serge	3604	#if 0
		3605
2332	Serge	3606	int
		3607	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3608	struct drm_file *file_priv)
		3609	{
		3610	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3611	int ret;
2332	Serge	3612
		3613	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3614	return 0;
		3615
		3616	if (atomic_read(&dev_priv->mm.wedged)) {
		3617	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3618	atomic_set(&dev_priv->mm.wedged, 0);
		3619	}
		3620
		3621	mutex_lock(&dev->struct_mutex);
		3622	dev_priv->mm.suspended = 0;
		3623
3031	serge	3624	ret = i915_gem_init_hw(dev);
2332	Serge	3625	if (ret != 0) {
		3626	mutex_unlock(&dev->struct_mutex);
		3627	return ret;
		3628	}
		3629
		3630	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3631	mutex_unlock(&dev->struct_mutex);
		3632
		3633	ret = drm_irq_install(dev);
		3634	if (ret)
		3635	goto cleanup_ringbuffer;
		3636
		3637	return 0;
		3638
		3639	cleanup_ringbuffer:
		3640	mutex_lock(&dev->struct_mutex);
		3641	i915_gem_cleanup_ringbuffer(dev);
		3642	dev_priv->mm.suspended = 1;
		3643	mutex_unlock(&dev->struct_mutex);
		3644
		3645	return ret;
		3646	}
		3647
		3648	int
		3649	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3650	struct drm_file *file_priv)
		3651	{
		3652	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3653	return 0;
		3654
		3655	drm_irq_uninstall(dev);
		3656	return i915_gem_idle(dev);
		3657	}
		3658
		3659	void
		3660	i915_gem_lastclose(struct drm_device *dev)
		3661	{
		3662	int ret;
		3663
		3664	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3665	return;
		3666
		3667	ret = i915_gem_idle(dev);
		3668	if (ret)
		3669	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3670	}
		3671	#endif
		3672
		3673	static void
2326	Serge	3674	init_ring_lists(struct intel_ring_buffer *ring)
		3675	{
		3676	INIT_LIST_HEAD(&ring->active_list);
		3677	INIT_LIST_HEAD(&ring->request_list);
		3678	}
		3679
		3680	void
		3681	i915_gem_load(struct drm_device *dev)
		3682	{
		3683	int i;
		3684	drm_i915_private_t *dev_priv = dev->dev_private;
		3685
		3686	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3687	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3688	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3689	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3690	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3691	for (i = 0; i < I915_NUM_RINGS; i++)
		3692	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3693	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3694	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3695	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3696	i915_gem_retire_work_handler);
2326	Serge	3697
		3698	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3699	if (IS_GEN3(dev)) {
3031	serge	3700	I915_WRITE(MI_ARB_STATE,
		3701	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3702	}
		3703
		3704	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3705
		3706	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3707	dev_priv->num_fence_regs = 16;
		3708	else
		3709	dev_priv->num_fence_regs = 8;
		3710
		3711	/* Initialize fence registers to zero */
3031	serge	3712	i915_gem_reset_fences(dev);
2326	Serge	3713
		3714	i915_gem_detect_bit_6_swizzle(dev);
		3715
		3716	dev_priv->mm.interruptible = true;
		3717
		3718	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3719	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3720	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3721	}
		3722
		3723

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3037