WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
2344	Serge	40	#undef mb
		41	#undef rmb
		42	#undef wmb
		43	#define mb() asm volatile("mfence")
		44	#define rmb() asm volatile ("lfence")
		45	#define wmb() asm volatile ("sfence")
		46
		47	static inline void clflush(volatile void *__p)
		48	{
		49	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		50	}
		51
2332	Serge	52	#define MAX_ERRNO 4095
		53
		54	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		55
		56
2326	Serge	57	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		58	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		59	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		60	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		61
2332	Serge	62	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		63	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		64	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		65	unsigned alignment,
3031	serge	66	bool map_and_fenceable,
		67	bool nonblocking);
2332	Serge	68	static int i915_gem_phys_pwrite(struct drm_device *dev,
		69	struct drm_i915_gem_object *obj,
		70	struct drm_i915_gem_pwrite *args,
		71	struct drm_file *file);
2326	Serge	72
3031	serge	73	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		74	struct drm_i915_gem_object *obj);
		75	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		76	struct drm_i915_fence_reg *fence,
		77	bool enable);
2332	Serge	78
3031	serge	79	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		80	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		81	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		82
		83	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		84	{
		85	if (obj->tiling_mode)
		86	i915_gem_release_mmap(obj);
		87
		88	/* As we do not have an associated fence register, we will force
		89	* a tiling change if we ever need to acquire one.
		90	*/
		91	obj->fence_dirty = false;
		92	obj->fence_reg = I915_FENCE_REG_NONE;
		93	}
		94
2332	Serge	95	/* some bookkeeping */
		96	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		97	size_t size)
		98	{
		99	dev_priv->mm.object_count++;
		100	dev_priv->mm.object_memory += size;
		101	}
		102
		103	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		104	size_t size)
		105	{
		106	dev_priv->mm.object_count--;
		107	dev_priv->mm.object_memory -= size;
		108	}
		109
		110	static int
		111	i915_gem_wait_for_error(struct drm_device *dev)
		112	{
		113	struct drm_i915_private *dev_priv = dev->dev_private;
		114	struct completion *x = &dev_priv->error_completion;
		115	unsigned long flags;
		116	int ret;
		117
		118	if (!atomic_read(&dev_priv->mm.wedged))
		119	return 0;
3255	Serge	120	#if 0
3031	serge	121	/*
		122	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		123	* userspace. If it takes that long something really bad is going on and
		124	* we should simply try to bail out and fail as gracefully as possible.
		125	*/
		126	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		127	if (ret == 0) {
		128	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		129	return -EIO;
		130	} else if (ret < 0) {
2332	Serge	131	return ret;
3031	serge	132	}
2332	Serge	133
		134	if (atomic_read(&dev_priv->mm.wedged)) {
		135	/* GPU is hung, bump the completion count to account for
		136	* the token we just consumed so that we never hit zero and
		137	* end up waiting upon a subsequent completion event that
		138	* will never happen.
		139	*/
		140	spin_lock_irqsave(&x->wait.lock, flags);
		141	x->done++;
		142	spin_unlock_irqrestore(&x->wait.lock, flags);
		143	}
3255	Serge	144	#endif
		145
2332	Serge	146	return 0;
		147	}
		148
		149	int i915_mutex_lock_interruptible(struct drm_device *dev)
		150	{
		151	int ret;
		152
		153	ret = i915_gem_wait_for_error(dev);
		154	if (ret)
		155	return ret;
		156
3255	Serge	157	mutex_lock(&dev->struct_mutex);
2332	Serge	158
		159	WARN_ON(i915_verify_lists(dev));
		160	return 0;
		161	}
		162
		163	static inline bool
		164	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		165	{
3031	serge	166	return obj->gtt_space && !obj->active;
2332	Serge	167	}
		168
		169
		170	#if 0
		171
		172	int
		173	i915_gem_init_ioctl(struct drm_device dev, void data,
		174	struct drm_file *file)
		175	{
		176	struct drm_i915_gem_init *args = data;
		177
3031	serge	178	if (drm_core_check_feature(dev, DRIVER_MODESET))
		179	return -ENODEV;
		180
2332	Serge	181	if (args->gtt_start >= args->gtt_end \|\|
		182	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		183	return -EINVAL;
		184
3031	serge	185	/* GEM with user mode setting was never supported on ilk and later. */
		186	if (INTEL_INFO(dev)->gen >= 5)
		187	return -ENODEV;
		188
2332	Serge	189	mutex_lock(&dev->struct_mutex);
3031	serge	190	i915_gem_init_global_gtt(dev, args->gtt_start,
		191	args->gtt_end, args->gtt_end);
2332	Serge	192	mutex_unlock(&dev->struct_mutex);
		193
		194	return 0;
		195	}
2351	Serge	196	#endif
2332	Serge	197
		198	int
		199	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		200	struct drm_file *file)
		201	{
		202	struct drm_i915_private *dev_priv = dev->dev_private;
		203	struct drm_i915_gem_get_aperture *args = data;
		204	struct drm_i915_gem_object *obj;
		205	size_t pinned;
		206
		207	pinned = 0;
		208	mutex_lock(&dev->struct_mutex);
3031	serge	209	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		210	if (obj->pin_count)
2332	Serge	211	pinned += obj->gtt_space->size;
		212	mutex_unlock(&dev->struct_mutex);
		213
		214	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	215	args->aper_available_size = args->aper_size - pinned;
2332	Serge	216
		217	return 0;
		218	}
		219
3031	serge	220	static int
		221	i915_gem_create(struct drm_file *file,
2332	Serge	222	struct drm_device *dev,
		223	uint64_t size,
		224	uint32_t *handle_p)
		225	{
		226	struct drm_i915_gem_object *obj;
		227	int ret;
		228	u32 handle;
		229
		230	size = roundup(size, PAGE_SIZE);
2342	Serge	231	if (size == 0)
		232	return -EINVAL;
2332	Serge	233
		234	/* Allocate the new object */
		235	obj = i915_gem_alloc_object(dev, size);
		236	if (obj == NULL)
		237	return -ENOMEM;
		238
		239	ret = drm_gem_handle_create(file, &obj->base, &handle);
		240	if (ret) {
		241	drm_gem_object_release(&obj->base);
		242	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		243	kfree(obj);
		244	return ret;
		245	}
		246
		247	/* drop reference from allocate - handle holds it now */
		248	drm_gem_object_unreference(&obj->base);
2351	Serge	249	trace_i915_gem_object_create(obj);
2332	Serge	250
		251	*handle_p = handle;
		252	return 0;
		253	}
		254
		255	int
		256	i915_gem_dumb_create(struct drm_file *file,
		257	struct drm_device *dev,
		258	struct drm_mode_create_dumb *args)
		259	{
		260	/* have to work out size/pitch and return them */
		261	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		262	args->size = args->pitch * args->height;
		263	return i915_gem_create(file, dev,
		264	args->size, &args->handle);
		265	}
		266
		267	int i915_gem_dumb_destroy(struct drm_file *file,
		268	struct drm_device *dev,
		269	uint32_t handle)
		270	{
		271	return drm_gem_handle_delete(file, handle);
		272	}
		273
2326	Serge	274	/**
2332	Serge	275	* Creates a new mm object and returns a handle to it.
		276	*/
		277	int
		278	i915_gem_create_ioctl(struct drm_device dev, void data,
		279	struct drm_file *file)
		280	{
		281	struct drm_i915_gem_create *args = data;
3031	serge	282
2332	Serge	283	return i915_gem_create(file, dev,
		284	args->size, &args->handle);
		285	}
		286
		287	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		288	{
		289	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		290
		291	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		292	obj->tiling_mode != I915_TILING_NONE;
		293	}
3260	Serge	294	#if 0
2332	Serge	295
3031	serge	296	static inline int
		297	__copy_to_user_swizzled(char __user *cpu_vaddr,
		298	const char *gpu_vaddr, int gpu_offset,
2332	Serge	299	int length)
		300	{
3031	serge	301	int ret, cpu_offset = 0;
2332	Serge	302
3031	serge	303	while (length > 0) {
		304	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		305	int this_length = min(cacheline_end - gpu_offset, length);
		306	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	307
3031	serge	308	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		309	gpu_vaddr + swizzled_gpu_offset,
		310	this_length);
		311	if (ret)
		312	return ret + length;
2332	Serge	313
3031	serge	314	cpu_offset += this_length;
		315	gpu_offset += this_length;
		316	length -= this_length;
		317	}
		318
		319	return 0;
2332	Serge	320	}
		321
3031	serge	322	static inline int
		323	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		324	const char __user *cpu_vaddr,
		325	int length)
2332	Serge	326	{
3031	serge	327	int ret, cpu_offset = 0;
2332	Serge	328
		329	while (length > 0) {
		330	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		331	int this_length = min(cacheline_end - gpu_offset, length);
		332	int swizzled_gpu_offset = gpu_offset ^ 64;
		333
3031	serge	334	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	335	cpu_vaddr + cpu_offset,
		336	this_length);
3031	serge	337	if (ret)
		338	return ret + length;
		339
2332	Serge	340	cpu_offset += this_length;
		341	gpu_offset += this_length;
		342	length -= this_length;
		343	}
		344
3031	serge	345	return 0;
2332	Serge	346	}
		347
3031	serge	348	/* Per-page copy function for the shmem pread fastpath.
		349	* Flushes invalid cachelines before reading the target if
		350	* needs_clflush is set. */
2332	Serge	351	static int
3031	serge	352	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		353	char __user *user_data,
		354	bool page_do_bit17_swizzling, bool needs_clflush)
		355	{
		356	char *vaddr;
		357	int ret;
		358
		359	if (unlikely(page_do_bit17_swizzling))
		360	return -EINVAL;
		361
		362	vaddr = kmap_atomic(page);
		363	if (needs_clflush)
		364	drm_clflush_virt_range(vaddr + shmem_page_offset,
		365	page_length);
		366	ret = __copy_to_user_inatomic(user_data,
		367	vaddr + shmem_page_offset,
		368	page_length);
		369	kunmap_atomic(vaddr);
		370
		371	return ret ? -EFAULT : 0;
		372	}
		373
		374	static void
		375	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		376	bool swizzled)
		377	{
		378	if (unlikely(swizzled)) {
		379	unsigned long start = (unsigned long) addr;
		380	unsigned long end = (unsigned long) addr + length;
		381
		382	/* For swizzling simply ensure that we always flush both
		383	* channels. Lame, but simple and it works. Swizzled
		384	* pwrite/pread is far from a hotpath - current userspace
		385	* doesn't use it at all. */
		386	start = round_down(start, 128);
		387	end = round_up(end, 128);
		388
		389	drm_clflush_virt_range((void *)start, end - start);
		390	} else {
		391	drm_clflush_virt_range(addr, length);
		392	}
		393
		394	}
		395
		396	/* Only difference to the fast-path function is that this can handle bit17
		397	* and uses non-atomic copy and kmap functions. */
		398	static int
		399	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		400	char __user *user_data,
		401	bool page_do_bit17_swizzling, bool needs_clflush)
		402	{
		403	char *vaddr;
		404	int ret;
		405
		406	vaddr = kmap(page);
		407	if (needs_clflush)
		408	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		409	page_length,
		410	page_do_bit17_swizzling);
		411
		412	if (page_do_bit17_swizzling)
		413	ret = __copy_to_user_swizzled(user_data,
		414	vaddr, shmem_page_offset,
		415	page_length);
		416	else
		417	ret = __copy_to_user(user_data,
		418	vaddr + shmem_page_offset,
		419	page_length);
		420	kunmap(page);
		421
		422	return ret ? - EFAULT : 0;
		423	}
		424
		425	static int
		426	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	427	struct drm_i915_gem_object *obj,
		428	struct drm_i915_gem_pread *args,
		429	struct drm_file *file)
		430	{
3031	serge	431	char __user *user_data;
2332	Serge	432	ssize_t remain;
		433	loff_t offset;
3031	serge	434	int shmem_page_offset, page_length, ret = 0;
		435	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		436	int hit_slowpath = 0;
		437	int prefaulted = 0;
		438	int needs_clflush = 0;
		439	struct scatterlist *sg;
		440	int i;
2332	Serge	441
		442	user_data = (char __user *) (uintptr_t) args->data_ptr;
		443	remain = args->size;
		444
3031	serge	445	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		446
		447	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		448	/* If we're not in the cpu read domain, set ourself into the gtt
		449	* read domain and manually flush cachelines (if required). This
		450	* optimizes for the case when the gpu will dirty the data
		451	* anyway again before the next pread happens. */
		452	if (obj->cache_level == I915_CACHE_NONE)
		453	needs_clflush = 1;
		454	if (obj->gtt_space) {
		455	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		456	if (ret)
		457	return ret;
		458	}
		459	}
		460
		461	ret = i915_gem_object_get_pages(obj);
		462	if (ret)
		463	return ret;
		464
		465	i915_gem_object_pin_pages(obj);
		466
2332	Serge	467	offset = args->offset;
		468
3031	serge	469	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	470	struct page *page;
		471
3031	serge	472	if (i < offset >> PAGE_SHIFT)
		473	continue;
		474
		475	if (remain <= 0)
		476	break;
		477
2332	Serge	478	/* Operation in this page
		479	*
3031	serge	480	* shmem_page_offset = offset within page in shmem file
2332	Serge	481	* page_length = bytes to copy for this page
		482	*/
3031	serge	483	shmem_page_offset = offset_in_page(offset);
2332	Serge	484	page_length = remain;
3031	serge	485	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		486	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	487
3031	serge	488	page = sg_page(sg);
		489	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		490	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	491
3031	serge	492	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		493	user_data, page_do_bit17_swizzling,
		494	needs_clflush);
		495	if (ret == 0)
		496	goto next_page;
2332	Serge	497
3031	serge	498	hit_slowpath = 1;
		499	mutex_unlock(&dev->struct_mutex);
		500
		501	if (!prefaulted) {
		502	ret = fault_in_multipages_writeable(user_data, remain);
		503	/* Userspace is tricking us, but we've already clobbered
		504	* its pages with the prefault and promised to write the
		505	* data up to the first fault. Hence ignore any errors
		506	* and just continue. */
		507	(void)ret;
		508	prefaulted = 1;
		509	}
		510
		511	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		512	user_data, page_do_bit17_swizzling,
		513	needs_clflush);
		514
		515	mutex_lock(&dev->struct_mutex);
		516
		517	next_page:
2332	Serge	518	mark_page_accessed(page);
3031	serge	519
2332	Serge	520	if (ret)
3031	serge	521	goto out;
2332	Serge	522
		523	remain -= page_length;
		524	user_data += page_length;
		525	offset += page_length;
		526	}
		527
3031	serge	528	out:
		529	i915_gem_object_unpin_pages(obj);
		530
		531	if (hit_slowpath) {
		532	/* Fixup: Kill any reinstated backing storage pages */
		533	if (obj->madv == __I915_MADV_PURGED)
		534	i915_gem_object_truncate(obj);
		535	}
		536
		537	return ret;
2332	Serge	538	}
		539
		540	/**
3031	serge	541	* Reads data from the object referenced by handle.
		542	*
		543	* On error, the contents of *data are undefined.
2332	Serge	544	*/
3031	serge	545	int
		546	i915_gem_pread_ioctl(struct drm_device dev, void data,
		547	struct drm_file *file)
		548	{
		549	struct drm_i915_gem_pread *args = data;
		550	struct drm_i915_gem_object *obj;
		551	int ret = 0;
		552
		553	if (args->size == 0)
		554	return 0;
		555
		556	if (!access_ok(VERIFY_WRITE,
		557	(char __user *)(uintptr_t)args->data_ptr,
		558	args->size))
		559	return -EFAULT;
		560
		561	ret = i915_mutex_lock_interruptible(dev);
		562	if (ret)
		563	return ret;
		564
		565	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		566	if (&obj->base == NULL) {
		567	ret = -ENOENT;
		568	goto unlock;
		569	}
		570
		571	/* Bounds check source. */
		572	if (args->offset > obj->base.size \|\|
		573	args->size > obj->base.size - args->offset) {
		574	ret = -EINVAL;
		575	goto out;
		576	}
		577
		578	/* prime objects have no backing filp to GEM pread/pwrite
		579	* pages from.
		580	*/
		581	if (!obj->base.filp) {
		582	ret = -EINVAL;
		583	goto out;
		584	}
		585
		586	trace_i915_gem_object_pread(obj, args->offset, args->size);
		587
		588	ret = i915_gem_shmem_pread(dev, obj, args, file);
		589
		590	out:
		591	drm_gem_object_unreference(&obj->base);
		592	unlock:
		593	mutex_unlock(&dev->struct_mutex);
		594	return ret;
		595	}
		596
		597	/* This is the fast write path which cannot handle
		598	* page faults in the source data
		599	*/
		600
		601	static inline int
		602	fast_user_write(struct io_mapping *mapping,
		603	loff_t page_base, int page_offset,
		604	char __user *user_data,
		605	int length)
		606	{
		607	void __iomem *vaddr_atomic;
		608	void *vaddr;
		609	unsigned long unwritten;
		610
		611	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		612	/* We can use the cpu mem copy function because this is X86. */
		613	vaddr = (void __force*)vaddr_atomic + page_offset;
		614	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		615	user_data, length);
		616	io_mapping_unmap_atomic(vaddr_atomic);
		617	return unwritten;
		618	}
3260	Serge	619	#endif
3031	serge	620
3260	Serge	621	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	622	/**
		623	* This is the fast pwrite path, where we copy the data directly from the
		624	* user into the GTT, uncached.
		625	*/
2332	Serge	626	static int
3031	serge	627	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		628	struct drm_i915_gem_object *obj,
		629	struct drm_i915_gem_pwrite *args,
		630	struct drm_file *file)
2332	Serge	631	{
3031	serge	632	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	633	ssize_t remain;
3031	serge	634	loff_t offset, page_base;
		635	char __user *user_data;
		636	int page_offset, page_length, ret;
3260	Serge	637	char *vaddr;
2332	Serge	638
3031	serge	639	ret = i915_gem_object_pin(obj, 0, true, true);
		640	if (ret)
		641	goto out;
		642
		643	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		644	if (ret)
		645	goto out_unpin;
		646
		647	ret = i915_gem_object_put_fence(obj);
		648	if (ret)
		649	goto out_unpin;
		650
3260	Serge	651	vaddr = AllocKernelSpace(4096);
		652	if(vaddr == NULL)
		653	{
		654	ret = -ENOSPC;
		655	goto out_unpin;
		656	};
		657
3031	serge	658	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	659	remain = args->size;
		660
3031	serge	661	offset = obj->gtt_offset + args->offset;
2332	Serge	662
3031	serge	663	while (remain > 0) {
		664	/* Operation in this page
		665	*
		666	* page_base = page offset within aperture
		667	* page_offset = offset within page
		668	* page_length = bytes to copy for this page
		669	*/
		670	page_base = offset & PAGE_MASK;
		671	page_offset = offset_in_page(offset);
		672	page_length = remain;
		673	if ((page_offset + remain) > PAGE_SIZE)
		674	page_length = PAGE_SIZE - page_offset;
2332	Serge	675
3260	Serge	676	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	677
3260	Serge	678	memcpy(vaddr+page_offset, user_data, page_length);
		679
3031	serge	680	remain -= page_length;
		681	user_data += page_length;
		682	offset += page_length;
2332	Serge	683	}
		684
3260	Serge	685	FreeKernelSpace(vaddr);
		686
3031	serge	687	out_unpin:
		688	i915_gem_object_unpin(obj);
		689	out:
3260	Serge	690	printf("% s ret = %d\n", __FUNCTION__, ret);
		691
3031	serge	692	return ret;
		693	}
		694
		695	/* Per-page copy function for the shmem pwrite fastpath.
		696	* Flushes invalid cachelines before writing to the target if
		697	* needs_clflush_before is set and flushes out any written cachelines after
		698	* writing if needs_clflush is set. */
		699	static int
		700	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		701	char __user *user_data,
		702	bool page_do_bit17_swizzling,
		703	bool needs_clflush_before,
		704	bool needs_clflush_after)
		705	{
		706	char *vaddr;
3260	Serge	707	int ret = 0;
3031	serge	708
		709	if (unlikely(page_do_bit17_swizzling))
		710	return -EINVAL;
		711
3260	Serge	712	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	713	if (needs_clflush_before)
		714	drm_clflush_virt_range(vaddr + shmem_page_offset,
		715	page_length);
3260	Serge	716	memcpy(vaddr + shmem_page_offset,
3031	serge	717	user_data,
		718	page_length);
		719	if (needs_clflush_after)
		720	drm_clflush_virt_range(vaddr + shmem_page_offset,
		721	page_length);
3260	Serge	722	FreeKernelSpace(vaddr);
3031	serge	723
		724	return ret ? -EFAULT : 0;
		725	}
3260	Serge	726	#if 0
3031	serge	727
		728	/* Only difference to the fast-path function is that this can handle bit17
		729	* and uses non-atomic copy and kmap functions. */
		730	static int
		731	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		732	char __user *user_data,
		733	bool page_do_bit17_swizzling,
		734	bool needs_clflush_before,
		735	bool needs_clflush_after)
		736	{
		737	char *vaddr;
		738	int ret;
		739
		740	vaddr = kmap(page);
		741	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		742	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		743	page_length,
		744	page_do_bit17_swizzling);
		745	if (page_do_bit17_swizzling)
		746	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		747	user_data,
		748	page_length);
		749	else
		750	ret = __copy_from_user(vaddr + shmem_page_offset,
		751	user_data,
		752	page_length);
		753	if (needs_clflush_after)
		754	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		755	page_length,
		756	page_do_bit17_swizzling);
		757	kunmap(page);
		758
		759	return ret ? -EFAULT : 0;
		760	}
3260	Serge	761	#endif
3031	serge	762
3260	Serge	763
3031	serge	764	static int
		765	i915_gem_shmem_pwrite(struct drm_device *dev,
		766	struct drm_i915_gem_object *obj,
		767	struct drm_i915_gem_pwrite *args,
		768	struct drm_file *file)
		769	{
		770	ssize_t remain;
		771	loff_t offset;
		772	char __user *user_data;
		773	int shmem_page_offset, page_length, ret = 0;
		774	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		775	int hit_slowpath = 0;
		776	int needs_clflush_after = 0;
		777	int needs_clflush_before = 0;
		778	int i;
		779	struct scatterlist *sg;
		780
		781	user_data = (char __user *) (uintptr_t) args->data_ptr;
		782	remain = args->size;
		783
		784	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		785
		786	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		787	/* If we're not in the cpu write domain, set ourself into the gtt
		788	* write domain and manually flush cachelines (if required). This
		789	* optimizes for the case when the gpu will use the data
		790	* right away and we therefore have to clflush anyway. */
		791	if (obj->cache_level == I915_CACHE_NONE)
		792	needs_clflush_after = 1;
		793	if (obj->gtt_space) {
		794	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		795	if (ret)
		796	return ret;
		797	}
		798	}
		799	/* Same trick applies for invalidate partially written cachelines before
		800	* writing. */
		801	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		802	&& obj->cache_level == I915_CACHE_NONE)
		803	needs_clflush_before = 1;
		804
		805	ret = i915_gem_object_get_pages(obj);
2332	Serge	806	if (ret)
3031	serge	807	return ret;
2332	Serge	808
3031	serge	809	i915_gem_object_pin_pages(obj);
2332	Serge	810
		811	offset = args->offset;
3031	serge	812	obj->dirty = 1;
2332	Serge	813
3031	serge	814	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	815	struct page *page;
3031	serge	816	int partial_cacheline_write;
2332	Serge	817
3031	serge	818	if (i < offset >> PAGE_SHIFT)
		819	continue;
		820
		821	if (remain <= 0)
		822	break;
		823
2332	Serge	824	/* Operation in this page
		825	*
		826	* shmem_page_offset = offset within page in shmem file
		827	* page_length = bytes to copy for this page
		828	*/
		829	shmem_page_offset = offset_in_page(offset);
		830
		831	page_length = remain;
		832	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		833	page_length = PAGE_SIZE - shmem_page_offset;
		834
3031	serge	835	/* If we don't overwrite a cacheline completely we need to be
		836	* careful to have up-to-date data by first clflushing. Don't
		837	* overcomplicate things and flush the entire patch. */
		838	partial_cacheline_write = needs_clflush_before &&
		839	((shmem_page_offset \| page_length)
3260	Serge	840	& (x86_clflush_size - 1));
2332	Serge	841
3031	serge	842	page = sg_page(sg);
		843	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		844	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	845
3031	serge	846	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		847	user_data, page_do_bit17_swizzling,
		848	partial_cacheline_write,
		849	needs_clflush_after);
		850	if (ret == 0)
		851	goto next_page;
		852
		853	hit_slowpath = 1;
		854	mutex_unlock(&dev->struct_mutex);
3260	Serge	855	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	856
3260	Serge	857	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		858	// user_data, page_do_bit17_swizzling,
		859	// partial_cacheline_write,
		860	// needs_clflush_after);
		861
3031	serge	862	mutex_lock(&dev->struct_mutex);
		863
		864	next_page:
2332	Serge	865
3031	serge	866	if (ret)
		867	goto out;
		868
2332	Serge	869	remain -= page_length;
3031	serge	870	user_data += page_length;
2332	Serge	871	offset += page_length;
		872	}
		873
		874	out:
3031	serge	875	i915_gem_object_unpin_pages(obj);
		876
		877	if (hit_slowpath) {
		878	/* Fixup: Kill any reinstated backing storage pages */
		879	if (obj->madv == __I915_MADV_PURGED)
		880	i915_gem_object_truncate(obj);
		881	/* and flush dirty cachelines in case the object isn't in the cpu write
		882	* domain anymore. */
		883	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		884	i915_gem_clflush_object(obj);
3243	Serge	885	i915_gem_chipset_flush(dev);
3031	serge	886	}
2332	Serge	887	}
		888
3031	serge	889	if (needs_clflush_after)
3243	Serge	890	i915_gem_chipset_flush(dev);
3031	serge	891
2332	Serge	892	return ret;
		893	}
3031	serge	894
		895	/**
		896	* Writes data to the object referenced by handle.
		897	*
		898	* On error, the contents of the buffer that were to be modified are undefined.
		899	*/
		900	int
		901	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		902	struct drm_file *file)
		903	{
		904	struct drm_i915_gem_pwrite *args = data;
		905	struct drm_i915_gem_object *obj;
		906	int ret;
		907
		908	if (args->size == 0)
		909	return 0;
		910
		911	ret = i915_mutex_lock_interruptible(dev);
		912	if (ret)
		913	return ret;
		914
		915	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		916	if (&obj->base == NULL) {
		917	ret = -ENOENT;
		918	goto unlock;
		919	}
		920
		921	/* Bounds check destination. */
		922	if (args->offset > obj->base.size \|\|
		923	args->size > obj->base.size - args->offset) {
		924	ret = -EINVAL;
		925	goto out;
		926	}
		927
		928	/* prime objects have no backing filp to GEM pread/pwrite
		929	* pages from.
		930	*/
		931	if (!obj->base.filp) {
		932	ret = -EINVAL;
		933	goto out;
		934	}
		935
		936	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		937
		938	ret = -EFAULT;
		939	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		940	* it would end up going through the fenced access, and we'll get
		941	* different detiling behavior between reading and writing.
		942	* pread/pwrite currently are reading and writing from the CPU
		943	* perspective, requiring manual detiling by the client.
		944	*/
3260	Serge	945	// if (obj->phys_obj) {
		946	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		947	// goto out;
		948	// }
3031	serge	949
		950	if (obj->cache_level == I915_CACHE_NONE &&
		951	obj->tiling_mode == I915_TILING_NONE &&
		952	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		953	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		954	/* Note that the gtt paths might fail with non-page-backed user
		955	* pointers (e.g. gtt mappings when moving data between
		956	* textures). Fallback to the shmem path in that case. */
		957	}
		958
		959	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	960	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	961
		962	out:
		963	drm_gem_object_unreference(&obj->base);
		964	unlock:
		965	mutex_unlock(&dev->struct_mutex);
		966	return ret;
		967	}
		968
		969	int
		970	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		971	bool interruptible)
		972	{
		973	if (atomic_read(&dev_priv->mm.wedged)) {
		974	struct completion *x = &dev_priv->error_completion;
		975	bool recovery_complete;
		976	unsigned long flags;
2332	Serge	977
3031	serge	978	/* Give the error handler a chance to run. */
		979	spin_lock_irqsave(&x->wait.lock, flags);
		980	recovery_complete = x->done > 0;
		981	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	982
3031	serge	983	/* Non-interruptible callers can't handle -EAGAIN, hence return
		984	* -EIO unconditionally for these. */
		985	if (!interruptible)
		986	return -EIO;
2332	Serge	987
3031	serge	988	/* Recovery complete, but still wedged means reset failure. */
		989	if (recovery_complete)
		990	return -EIO;
2332	Serge	991
3031	serge	992	return -EAGAIN;
		993	}
2332	Serge	994
3031	serge	995	return 0;
		996	}
2332	Serge	997
3031	serge	998	/*
		999	* Compare seqno against outstanding lazy request. Emit a request if they are
		1000	* equal.
		1001	*/
		1002	static int
		1003	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1004	{
		1005	int ret;
2332	Serge	1006
3031	serge	1007	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1008
3031	serge	1009	ret = 0;
		1010	if (seqno == ring->outstanding_lazy_request)
		1011	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1012
3031	serge	1013	return ret;
		1014	}
2332	Serge	1015
3031	serge	1016	/**
		1017	* __wait_seqno - wait until execution of seqno has finished
		1018	* @ring: the ring expected to report seqno
		1019	* @seqno: duh!
		1020	* @interruptible: do an interruptible wait (normally yes)
		1021	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1022	*
		1023	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1024	* errno with remaining time filled in timeout argument.
		1025	*/
		1026	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1027	bool interruptible, struct timespec *timeout)
		1028	{
		1029	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1030	struct timespec before, now, wait_time={1,0};
		1031	unsigned long timeout_jiffies;
		1032	long end;
		1033	bool wait_forever = true;
		1034	int ret;
2332	Serge	1035
3031	serge	1036	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1037	return 0;
2332	Serge	1038
3031	serge	1039	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1040
3031	serge	1041	if (timeout != NULL) {
		1042	wait_time = *timeout;
		1043	wait_forever = false;
		1044	}
2332	Serge	1045
3031	serge	1046	// timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1047
3031	serge	1048	if (WARN_ON(!ring->irq_get(ring)))
		1049	return -ENODEV;
		1050	#if 0
2332	Serge	1051
3031	serge	1052	/* Record current time in case interrupted by signal, or wedged * */
		1053	getrawmonotonic(&before);
2332	Serge	1054
3031	serge	1055	#define EXIT_COND \
		1056	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1057	atomic_read(&dev_priv->mm.wedged))
		1058	do {
		1059	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1060	timeout_jiffies);
2332	Serge	1061
3031	serge	1062	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1063	if (ret)
		1064	end = ret;
		1065	} while (end == 0 && wait_forever);
2332	Serge	1066
3031	serge	1067	getrawmonotonic(&now);
2332	Serge	1068
3031	serge	1069	ring->irq_put(ring);
		1070	trace_i915_gem_request_wait_end(ring, seqno);
		1071	#undef EXIT_COND
2332	Serge	1072
3031	serge	1073	if (timeout) {
		1074	// struct timespec sleep_time = timespec_sub(now, before);
		1075	// timeout = timespec_sub(timeout, sleep_time);
		1076	}
2332	Serge	1077
3031	serge	1078	switch (end) {
		1079	case -EIO:
		1080	case -EAGAIN: /* Wedged */
		1081	case -ERESTARTSYS: /* Signal */
		1082	return (int)end;
		1083	case 0: /* Timeout */
		1084	// if (timeout)
		1085	// set_normalized_timespec(timeout, 0, 0);
		1086	return -ETIME;
		1087	default: /* Completed */
		1088	WARN_ON(end < 0); /* We're not aware of other errors */
		1089	return 0;
		1090	}
3260	Serge	1091
3031	serge	1092	#endif
2332	Serge	1093
3031	serge	1094	#define EXIT_COND \
		1095	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1096	atomic_read(&dev_priv->mm.wedged))
		1097	wait_event(ring->irq_queue, EXIT_COND);
		1098	#undef EXIT_COND
		1099	ring->irq_put(ring);
2332	Serge	1100
3031	serge	1101	return 0;
		1102	}
2332	Serge	1103
3031	serge	1104	/**
		1105	* Waits for a sequence number to be signaled, and cleans up the
		1106	* request and object lists appropriately for that event.
		1107	*/
		1108	int
		1109	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1110	{
		1111	struct drm_device *dev = ring->dev;
		1112	struct drm_i915_private *dev_priv = dev->dev_private;
		1113	bool interruptible = dev_priv->mm.interruptible;
		1114	int ret;
2332	Serge	1115
3031	serge	1116	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1117	BUG_ON(seqno == 0);
2332	Serge	1118
3031	serge	1119	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1120	if (ret)
		1121	return ret;
2332	Serge	1122
3031	serge	1123	ret = i915_gem_check_olr(ring, seqno);
		1124	if (ret)
		1125	return ret;
2332	Serge	1126
3031	serge	1127	return __wait_seqno(ring, seqno, interruptible, NULL);
		1128	}
2332	Serge	1129
3031	serge	1130	/**
		1131	* Ensures that all rendering to the object has completed and the object is
		1132	* safe to unbind from the GTT or access from the CPU.
		1133	*/
		1134	static __must_check int
		1135	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1136	bool readonly)
		1137	{
		1138	struct intel_ring_buffer *ring = obj->ring;
		1139	u32 seqno;
		1140	int ret;
2332	Serge	1141
3031	serge	1142	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1143	if (seqno == 0)
		1144	return 0;
2332	Serge	1145
3031	serge	1146	ret = i915_wait_seqno(ring, seqno);
		1147	if (ret)
		1148	return ret;
2332	Serge	1149
3031	serge	1150	i915_gem_retire_requests_ring(ring);
2332	Serge	1151
3031	serge	1152	/* Manually manage the write flush as we may have not yet
		1153	* retired the buffer.
		1154	*/
		1155	if (obj->last_write_seqno &&
		1156	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1157	obj->last_write_seqno = 0;
		1158	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1159	}
2332	Serge	1160
3031	serge	1161	return 0;
		1162	}
2332	Serge	1163
3260	Serge	1164	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1165	* as the object state may change during this call.
		1166	*/
		1167	static __must_check int
		1168	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1169	bool readonly)
		1170	{
		1171	struct drm_device *dev = obj->base.dev;
		1172	struct drm_i915_private *dev_priv = dev->dev_private;
		1173	struct intel_ring_buffer *ring = obj->ring;
		1174	u32 seqno;
		1175	int ret;
2332	Serge	1176
3260	Serge	1177	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1178	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1179
3260	Serge	1180	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1181	if (seqno == 0)
		1182	return 0;
2332	Serge	1183
3260	Serge	1184	ret = i915_gem_check_wedge(dev_priv, true);
		1185	if (ret)
		1186	return ret;
2332	Serge	1187
3260	Serge	1188	ret = i915_gem_check_olr(ring, seqno);
		1189	if (ret)
		1190	return ret;
2332	Serge	1191
3260	Serge	1192	mutex_unlock(&dev->struct_mutex);
		1193	ret = __wait_seqno(ring, seqno, true, NULL);
		1194	mutex_lock(&dev->struct_mutex);
2332	Serge	1195
3260	Serge	1196	i915_gem_retire_requests_ring(ring);
2332	Serge	1197
3260	Serge	1198	/* Manually manage the write flush as we may have not yet
		1199	* retired the buffer.
		1200	*/
		1201	if (obj->last_write_seqno &&
		1202	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1203	obj->last_write_seqno = 0;
		1204	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1205	}
2332	Serge	1206
3260	Serge	1207	return ret;
		1208	}
2332	Serge	1209
3260	Serge	1210	/**
		1211	* Called when user space prepares to use an object with the CPU, either
		1212	* through the mmap ioctl's mapping or a GTT mapping.
		1213	*/
		1214	int
		1215	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1216	struct drm_file *file)
		1217	{
		1218	struct drm_i915_gem_set_domain *args = data;
		1219	struct drm_i915_gem_object *obj;
		1220	uint32_t read_domains = args->read_domains;
		1221	uint32_t write_domain = args->write_domain;
		1222	int ret;
2332	Serge	1223
3260	Serge	1224	/* Only handle setting domains to types used by the CPU. */
		1225	if (write_domain & I915_GEM_GPU_DOMAINS)
		1226	return -EINVAL;
2332	Serge	1227
3260	Serge	1228	if (read_domains & I915_GEM_GPU_DOMAINS)
		1229	return -EINVAL;
2332	Serge	1230
3260	Serge	1231	/* Having something in the write domain implies it's in the read
		1232	* domain, and only that read domain. Enforce that in the request.
		1233	*/
		1234	if (write_domain != 0 && read_domains != write_domain)
		1235	return -EINVAL;
2332	Serge	1236
3260	Serge	1237	ret = i915_mutex_lock_interruptible(dev);
		1238	if (ret)
		1239	return ret;
2332	Serge	1240
3260	Serge	1241	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1242	if (&obj->base == NULL) {
		1243	ret = -ENOENT;
		1244	goto unlock;
		1245	}
2332	Serge	1246
3260	Serge	1247	/* Try to flush the object off the GPU without holding the lock.
		1248	* We will repeat the flush holding the lock in the normal manner
		1249	* to catch cases where we are gazumped.
		1250	*/
		1251	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1252	if (ret)
		1253	goto unref;
2332	Serge	1254
3260	Serge	1255	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1256	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1257
3260	Serge	1258	/* Silently promote "you're not bound, there was nothing to do"
		1259	* to success, since the client was just asking us to
		1260	* make sure everything was done.
		1261	*/
		1262	if (ret == -EINVAL)
		1263	ret = 0;
		1264	} else {
		1265	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1266	}
2332	Serge	1267
3260	Serge	1268	unref:
		1269	drm_gem_object_unreference(&obj->base);
		1270	unlock:
		1271	mutex_unlock(&dev->struct_mutex);
		1272	return ret;
		1273	}
2332	Serge	1274
		1275
		1276
		1277
		1278
		1279
3260	Serge	1280	/**
		1281	* Maps the contents of an object, returning the address it is mapped
		1282	* into.
		1283	*
		1284	* While the mapping holds a reference on the contents of the object, it doesn't
		1285	* imply a ref on the object itself.
		1286	*/
		1287	int
		1288	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1289	struct drm_file *file)
		1290	{
		1291	struct drm_i915_gem_mmap *args = data;
		1292	struct drm_gem_object *obj;
		1293	unsigned long addr = 0;
2332	Serge	1294
3260	Serge	1295	obj = drm_gem_object_lookup(dev, file, args->handle);
		1296	if (obj == NULL)
		1297	return -ENOENT;
		1298
		1299	dbgprintf("%s offset %lld size %lld not supported\n",
		1300	args->offset, args->size);
		1301	/* prime objects have no backing filp to GEM mmap
		1302	* pages from.
		1303	*/
		1304	if (!obj->filp) {
		1305	drm_gem_object_unreference_unlocked(obj);
		1306	return -EINVAL;
		1307	}
2332	Serge	1308
3260	Serge	1309	// addr = vm_mmap(obj->filp, 0, args->size,
		1310	// PROT_READ \| PROT_WRITE, MAP_SHARED,
		1311	// args->offset);
		1312	drm_gem_object_unreference_unlocked(obj);
		1313	// if (IS_ERR((void *)addr))
		1314	// return addr;
2332	Serge	1315
3260	Serge	1316	args->addr_ptr = (uint64_t) addr;
		1317	return -EINVAL;
2332	Serge	1318
3260	Serge	1319	// return 0;
		1320	}
2332	Serge	1321
		1322
		1323
		1324
		1325
		1326
		1327
		1328
3031	serge	1329
		1330
		1331
		1332
		1333
		1334	/**
		1335	* i915_gem_release_mmap - remove physical page mappings
		1336	* @obj: obj in question
		1337	*
		1338	* Preserve the reservation of the mmapping with the DRM core code, but
		1339	* relinquish ownership of the pages back to the system.
		1340	*
		1341	* It is vital that we remove the page mapping if we have mapped a tiled
		1342	* object through the GTT and then lose the fence register due to
		1343	* resource pressure. Similarly if the object has been moved out of the
		1344	* aperture, than pages mapped into userspace must be revoked. Removing the
		1345	* mapping will then trigger a page fault on the next user access, allowing
		1346	* fixup by i915_gem_fault().
		1347	*/
		1348	void
		1349	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1350	{
		1351	if (!obj->fault_mappable)
		1352	return;
		1353
		1354	if (obj->base.dev->dev_mapping)
		1355	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1356	// (loff_t)obj->base.map_list.hash.key<
		1357	// obj->base.size, 1);
		1358
		1359	obj->fault_mappable = false;
		1360	}
		1361
2332	Serge	1362	static uint32_t
		1363	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1364	{
		1365	uint32_t gtt_size;
		1366
		1367	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1368	tiling_mode == I915_TILING_NONE)
		1369	return size;
		1370
		1371	/* Previous chips need a power-of-two fence region when tiling */
		1372	if (INTEL_INFO(dev)->gen == 3)
		1373	gtt_size = 1024*1024;
		1374	else
		1375	gtt_size = 512*1024;
		1376
		1377	while (gtt_size < size)
		1378	gtt_size <<= 1;
		1379
		1380	return gtt_size;
		1381	}
		1382
		1383	/**
		1384	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1385	* @obj: object to check
		1386	*
		1387	* Return the required GTT alignment for an object, taking into account
		1388	* potential fence register mapping.
		1389	*/
		1390	static uint32_t
		1391	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1392	uint32_t size,
		1393	int tiling_mode)
		1394	{
		1395	/*
		1396	* Minimum alignment is 4k (GTT page size), but might be greater
		1397	* if a fence register is needed for the object.
		1398	*/
		1399	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1400	tiling_mode == I915_TILING_NONE)
		1401	return 4096;
		1402
		1403	/*
		1404	* Previous chips need to be aligned to the size of the smallest
		1405	* fence register that can contain the object.
		1406	*/
		1407	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1408	}
		1409
		1410	/**
		1411	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1412	* unfenced object
		1413	* @dev: the device
		1414	* @size: size of the object
		1415	* @tiling_mode: tiling mode of the object
		1416	*
		1417	* Return the required GTT alignment for an object, only taking into account
		1418	* unfenced tiled surface requirements.
		1419	*/
		1420	uint32_t
		1421	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1422	uint32_t size,
		1423	int tiling_mode)
		1424	{
		1425	/*
		1426	* Minimum alignment is 4k (GTT page size) for sane hw.
		1427	*/
		1428	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1429	tiling_mode == I915_TILING_NONE)
		1430	return 4096;
		1431
		1432	/* Previous hardware however needs to be aligned to a power-of-two
		1433	* tile height. The simplest method for determining this is to reuse
		1434	* the power-of-tile object size.
		1435	*/
		1436	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1437	}
		1438
3031	serge	1439	/* Immediately discard the backing storage */
		1440	static void
		1441	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1442	{
		1443	// struct inode *inode;
2332	Serge	1444
3031	serge	1445	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1446
3031	serge	1447	// if (obj->base.filp == NULL)
		1448	// return;
2332	Serge	1449
3031	serge	1450	/* Our goal here is to return as much of the memory as
		1451	* is possible back to the system as we are called from OOM.
		1452	* To do this we must instruct the shmfs to drop all of its
		1453	* backing pages, now.
		1454	*/
		1455	// inode = obj->base.filp->f_path.dentry->d_inode;
		1456	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1457
3031	serge	1458	obj->madv = __I915_MADV_PURGED;
		1459	}
2332	Serge	1460
3031	serge	1461	static inline int
		1462	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1463	{
		1464	return obj->madv == I915_MADV_DONTNEED;
		1465	}
2332	Serge	1466
3031	serge	1467	static void
		1468	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1469	{
3243	Serge	1470	int page_count = obj->base.size / PAGE_SIZE;
		1471	struct scatterlist *sg;
3031	serge	1472	int ret, i;
2332	Serge	1473
3031	serge	1474	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1475
3031	serge	1476	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1477	if (ret) {
		1478	/* In the event of a disaster, abandon all caches and
		1479	* hope for the best.
		1480	*/
		1481	WARN_ON(ret != -EIO);
		1482	i915_gem_clflush_object(obj);
		1483	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1484	}
2332	Serge	1485
3031	serge	1486	if (obj->madv == I915_MADV_DONTNEED)
		1487	obj->dirty = 0;
2332	Serge	1488
3243	Serge	1489	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1490	struct page *page = sg_page(sg);
2332	Serge	1491
3243	Serge	1492
		1493
		1494	page_cache_release(page);
		1495	}
		1496	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3031	serge	1497	obj->dirty = 0;
3243	Serge	1498
		1499	sg_free_table(obj->pages);
		1500	kfree(obj->pages);
3031	serge	1501	}
2332	Serge	1502
3031	serge	1503	static int
		1504	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1505	{
		1506	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1507
3243	Serge	1508	if (obj->pages == NULL)
3031	serge	1509	return 0;
2332	Serge	1510
3031	serge	1511	BUG_ON(obj->gtt_space);
		1512
		1513	if (obj->pages_pin_count)
		1514	return -EBUSY;
		1515
3243	Serge	1516	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1517	* array, hence protect them from being reaped by removing them from gtt
		1518	* lists early. */
		1519	list_del(&obj->gtt_list);
		1520
3031	serge	1521	ops->put_pages(obj);
3243	Serge	1522	obj->pages = NULL;
3031	serge	1523
		1524	if (i915_gem_object_is_purgeable(obj))
		1525	i915_gem_object_truncate(obj);
		1526
		1527	return 0;
		1528	}
		1529
		1530
		1531
		1532
		1533
		1534
		1535
		1536
2332	Serge	1537	static int
3031	serge	1538	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1539	{
3260	Serge	1540	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1541	int page_count, i;
3260	Serge	1542	struct address_space *mapping;
3243	Serge	1543	struct sg_table *st;
		1544	struct scatterlist *sg;
		1545	struct page *page;
		1546	gfp_t gfp;
2332	Serge	1547
3243	Serge	1548	/* Assert that the object is not currently in any GPU domain. As it
		1549	* wasn't in the GTT, there shouldn't be any way it could have been in
		1550	* a GPU cache
2332	Serge	1551	*/
3243	Serge	1552	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1553	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1554
		1555	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1556	if (st == NULL)
		1557	return -ENOMEM;
		1558
2332	Serge	1559	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1560	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1561	sg_free_table(st);
		1562	kfree(st);
2332	Serge	1563	return -ENOMEM;
3243	Serge	1564	}
2332	Serge	1565
3243	Serge	1566	/* Get the list of pages out of our struct file. They'll be pinned
		1567	* at this point until we release them.
		1568	*
		1569	* Fail silently without starting the shrinker
		1570	*/
		1571	for_each_sg(st->sgl, sg, page_count, i) {
3260	Serge	1572	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1573	if (IS_ERR(page)) {
		1574	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1575	goto err_pages;
		1576
3260	Serge	1577	}
3243	Serge	1578	sg_set_page(sg, page, PAGE_SIZE, 0);
		1579	}
3031	serge	1580
3243	Serge	1581	obj->pages = st;
3031	serge	1582
3260	Serge	1583	DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1584
		1585	return 0;
		1586
		1587	err_pages:
3243	Serge	1588	for_each_sg(st->sgl, sg, i, page_count)
		1589	page_cache_release(sg_page(sg));
		1590	sg_free_table(st);
		1591	kfree(st);
		1592	return PTR_ERR(page);
2332	Serge	1593	}
		1594
3031	serge	1595	/* Ensure that the associated pages are gathered from the backing storage
		1596	* and pinned into our object. i915_gem_object_get_pages() may be called
		1597	* multiple times before they are released by a single call to
		1598	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1599	* either as a result of memory pressure (reaping pages under the shrinker)
		1600	* or as the object is itself released.
		1601	*/
		1602	int
		1603	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1604	{
3031	serge	1605	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1606	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1607	int ret;
2332	Serge	1608
3243	Serge	1609	if (obj->pages)
3031	serge	1610	return 0;
2332	Serge	1611
3031	serge	1612	BUG_ON(obj->pages_pin_count);
2332	Serge	1613
3031	serge	1614	ret = ops->get_pages(obj);
		1615	if (ret)
		1616	return ret;
2344	Serge	1617
3031	serge	1618	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1619	return 0;
2332	Serge	1620	}
		1621
		1622	void
		1623	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1624	struct intel_ring_buffer *ring)
2332	Serge	1625	{
		1626	struct drm_device *dev = obj->base.dev;
		1627	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1628	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1629
		1630	BUG_ON(ring == NULL);
		1631	obj->ring = ring;
		1632
		1633	/* Add a reference if we're newly entering the active list. */
		1634	if (!obj->active) {
2344	Serge	1635	drm_gem_object_reference(&obj->base);
2332	Serge	1636	obj->active = 1;
		1637	}
		1638
		1639	/* Move from whatever list we were on to the tail of execution. */
		1640	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1641	list_move_tail(&obj->ring_list, &ring->active_list);
		1642
3031	serge	1643	obj->last_read_seqno = seqno;
		1644
2332	Serge	1645	if (obj->fenced_gpu_access) {
3031	serge	1646	obj->last_fenced_seqno = seqno;
		1647
		1648	/* Bump MRU to take account of the delayed flush */
		1649	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1650	struct drm_i915_fence_reg *reg;
		1651
		1652	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1653	list_move_tail(®->lru_list,
		1654	&dev_priv->mm.fence_list);
		1655	}
2332	Serge	1656	}
		1657	}
		1658
2344	Serge	1659	static void
3031	serge	1660	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1661	{
		1662	struct drm_device *dev = obj->base.dev;
3031	serge	1663	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1664
3031	serge	1665	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1666	BUG_ON(!obj->active);
2332	Serge	1667
3031	serge	1668	if (obj->pin_count) /* are we a framebuffer? */
		1669	intel_mark_fb_idle(obj);
2344	Serge	1670
2352	Serge	1671	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1672
3031	serge	1673	list_del_init(&obj->ring_list);
2352	Serge	1674	obj->ring = NULL;
2344	Serge	1675
3031	serge	1676	obj->last_read_seqno = 0;
		1677	obj->last_write_seqno = 0;
		1678	obj->base.write_domain = 0;
		1679
		1680	obj->last_fenced_seqno = 0;
2352	Serge	1681	obj->fenced_gpu_access = false;
2344	Serge	1682
2352	Serge	1683	obj->active = 0;
		1684	drm_gem_object_unreference(&obj->base);
		1685
		1686	WARN_ON(i915_verify_lists(dev));
		1687	}
		1688
3243	Serge	1689	static int
		1690	i915_gem_handle_seqno_wrap(struct drm_device *dev)
2344	Serge	1691	{
3243	Serge	1692	struct drm_i915_private *dev_priv = dev->dev_private;
		1693	struct intel_ring_buffer *ring;
		1694	int ret, i, j;
2344	Serge	1695
3243	Serge	1696	/* The hardware uses various monotonic 32-bit counters, if we
		1697	* detect that they will wraparound we need to idle the GPU
		1698	* and reset those counters.
		1699	*/
		1700	ret = 0;
		1701	for_each_ring(ring, dev_priv, i) {
		1702	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1703	ret \|= ring->sync_seqno[j] != 0;
		1704	}
		1705	if (ret == 0)
		1706	return ret;
2344	Serge	1707
3243	Serge	1708	ret = i915_gpu_idle(dev);
		1709	if (ret)
		1710	return ret;
		1711
		1712	i915_gem_retire_requests(dev);
		1713	for_each_ring(ring, dev_priv, i) {
		1714	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1715	ring->sync_seqno[j] = 0;
		1716	}
		1717
		1718	return 0;
2344	Serge	1719	}
		1720
3243	Serge	1721	int
		1722	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1723	{
3243	Serge	1724	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1725
3243	Serge	1726	/* reserve 0 for non-seqno */
		1727	if (dev_priv->next_seqno == 0) {
		1728	int ret = i915_gem_handle_seqno_wrap(dev);
		1729	if (ret)
		1730	return ret;
		1731
		1732	dev_priv->next_seqno = 1;
		1733	}
		1734
		1735	*seqno = dev_priv->next_seqno++;
		1736	return 0;
2332	Serge	1737	}
		1738
2352	Serge	1739	int
		1740	i915_add_request(struct intel_ring_buffer *ring,
		1741	struct drm_file *file,
3031	serge	1742	u32 *out_seqno)
2352	Serge	1743	{
		1744	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1745	struct drm_i915_gem_request *request;
		1746	u32 request_ring_position;
2352	Serge	1747	int was_empty;
		1748	int ret;
2332	Serge	1749
3031	serge	1750	/*
		1751	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1752	* after having emitted the batchbuffer command. Hence we need to fix
		1753	* things up similar to emitting the lazy request. The difference here
		1754	* is that the flush _must_ happen before the next request, no matter
		1755	* what.
		1756	*/
		1757	ret = intel_ring_flush_all_caches(ring);
		1758	if (ret)
		1759	return ret;
2332	Serge	1760
3031	serge	1761	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1762	if (request == NULL)
		1763	return -ENOMEM;
		1764
		1765
		1766	/* Record the position of the start of the request so that
		1767	* should we detect the updated seqno part-way through the
		1768	* GPU processing the request, we never over-estimate the
		1769	* position of the head.
		1770	*/
		1771	request_ring_position = intel_ring_get_tail(ring);
		1772
3243	Serge	1773	ret = ring->add_request(ring);
3031	serge	1774	if (ret) {
		1775	kfree(request);
2352	Serge	1776	return ret;
3031	serge	1777	}
2332	Serge	1778
3243	Serge	1779	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1780	request->ring = ring;
3031	serge	1781	request->tail = request_ring_position;
		1782	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1783	was_empty = list_empty(&ring->request_list);
		1784	list_add_tail(&request->list, &ring->request_list);
3031	serge	1785	request->file_priv = NULL;
2332	Serge	1786
		1787
3031	serge	1788	ring->outstanding_lazy_request = 0;
2332	Serge	1789
2360	Serge	1790	if (!dev_priv->mm.suspended) {
		1791	if (i915_enable_hangcheck) {
2352	Serge	1792	// mod_timer(&dev_priv->hangcheck_timer,
		1793	// jiffies +
		1794	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1795	}
3031	serge	1796	if (was_empty) {
2360	Serge	1797	queue_delayed_work(dev_priv->wq,
		1798	&dev_priv->mm.retire_work, HZ);
3031	serge	1799	intel_mark_busy(dev_priv->dev);
		1800	}
2360	Serge	1801	}
3031	serge	1802
		1803	if (out_seqno)
3243	Serge	1804	*out_seqno = request->seqno;
2352	Serge	1805	return 0;
		1806	}
2332	Serge	1807
		1808
		1809
		1810
3031	serge	1811	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1812	struct intel_ring_buffer *ring)
		1813	{
		1814	while (!list_empty(&ring->request_list)) {
		1815	struct drm_i915_gem_request *request;
2332	Serge	1816
3031	serge	1817	request = list_first_entry(&ring->request_list,
		1818	struct drm_i915_gem_request,
		1819	list);
2332	Serge	1820
3031	serge	1821	list_del(&request->list);
		1822	// i915_gem_request_remove_from_client(request);
		1823	kfree(request);
		1824	}
2332	Serge	1825
3031	serge	1826	while (!list_empty(&ring->active_list)) {
		1827	struct drm_i915_gem_object *obj;
2332	Serge	1828
3031	serge	1829	obj = list_first_entry(&ring->active_list,
		1830	struct drm_i915_gem_object,
		1831	ring_list);
2332	Serge	1832
3031	serge	1833	i915_gem_object_move_to_inactive(obj);
		1834	}
		1835	}
2332	Serge	1836
3031	serge	1837	static void i915_gem_reset_fences(struct drm_device *dev)
		1838	{
		1839	struct drm_i915_private *dev_priv = dev->dev_private;
		1840	int i;
2332	Serge	1841
3031	serge	1842	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1843	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1844
3031	serge	1845	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1846
3031	serge	1847	if (reg->obj)
		1848	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1849
3031	serge	1850	reg->pin_count = 0;
		1851	reg->obj = NULL;
		1852	INIT_LIST_HEAD(®->lru_list);
		1853	}
2360	Serge	1854
3031	serge	1855	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1856	}
2360	Serge	1857
3031	serge	1858	void i915_gem_reset(struct drm_device *dev)
		1859	{
		1860	struct drm_i915_private *dev_priv = dev->dev_private;
		1861	struct drm_i915_gem_object *obj;
		1862	struct intel_ring_buffer *ring;
		1863	int i;
2360	Serge	1864
3031	serge	1865	for_each_ring(ring, dev_priv, i)
		1866	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1867
3031	serge	1868	/* Move everything out of the GPU domains to ensure we do any
		1869	* necessary invalidation upon reuse.
		1870	*/
		1871	list_for_each_entry(obj,
		1872	&dev_priv->mm.inactive_list,
		1873	mm_list)
		1874	{
		1875	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1876	}
2360	Serge	1877
3031	serge	1878	/* The fence registers are invalidated so clear them out */
		1879	i915_gem_reset_fences(dev);
		1880	}
2360	Serge	1881
2352	Serge	1882	/**
		1883	* This function clears the request list as sequence numbers are passed.
		1884	*/
3031	serge	1885	void
2352	Serge	1886	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1887	{
		1888	uint32_t seqno;
2332	Serge	1889
2352	Serge	1890	if (list_empty(&ring->request_list))
		1891	return;
2332	Serge	1892
2352	Serge	1893	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1894
3031	serge	1895	seqno = ring->get_seqno(ring, true);
2332	Serge	1896
2352	Serge	1897	while (!list_empty(&ring->request_list)) {
		1898	struct drm_i915_gem_request *request;
2332	Serge	1899
2352	Serge	1900	request = list_first_entry(&ring->request_list,
		1901	struct drm_i915_gem_request,
		1902	list);
2332	Serge	1903
2352	Serge	1904	if (!i915_seqno_passed(seqno, request->seqno))
		1905	break;
2332	Serge	1906
2352	Serge	1907	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1908	/* We know the GPU must have read the request to have
		1909	* sent us the seqno + interrupt, so use the position
		1910	* of tail of the request to update the last known position
		1911	* of the GPU head.
		1912	*/
		1913	ring->last_retired_head = request->tail;
2332	Serge	1914
2352	Serge	1915	list_del(&request->list);
		1916	kfree(request);
		1917	}
2332	Serge	1918
2352	Serge	1919	/* Move any buffers on the active list that are no longer referenced
		1920	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1921	*/
		1922	while (!list_empty(&ring->active_list)) {
		1923	struct drm_i915_gem_object *obj;
2332	Serge	1924
2352	Serge	1925	obj = list_first_entry(&ring->active_list,
		1926	struct drm_i915_gem_object,
		1927	ring_list);
2332	Serge	1928
3031	serge	1929	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1930	break;
2332	Serge	1931
2352	Serge	1932	i915_gem_object_move_to_inactive(obj);
		1933	}
2332	Serge	1934
2352	Serge	1935	if (unlikely(ring->trace_irq_seqno &&
		1936	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1937	ring->irq_put(ring);
		1938	ring->trace_irq_seqno = 0;
		1939	}
2332	Serge	1940
2352	Serge	1941	WARN_ON(i915_verify_lists(ring->dev));
		1942	}
2332	Serge	1943
2352	Serge	1944	void
		1945	i915_gem_retire_requests(struct drm_device *dev)
		1946	{
		1947	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1948	struct intel_ring_buffer *ring;
2352	Serge	1949	int i;
2332	Serge	1950
3031	serge	1951	for_each_ring(ring, dev_priv, i)
		1952	i915_gem_retire_requests_ring(ring);
2352	Serge	1953	}
		1954
2360	Serge	1955	static void
		1956	i915_gem_retire_work_handler(struct work_struct *work)
		1957	{
		1958	drm_i915_private_t *dev_priv;
		1959	struct drm_device *dev;
3031	serge	1960	struct intel_ring_buffer *ring;
2360	Serge	1961	bool idle;
		1962	int i;
2352	Serge	1963
2360	Serge	1964	dev_priv = container_of(work, drm_i915_private_t,
		1965	mm.retire_work.work);
		1966	dev = dev_priv->dev;
2352	Serge	1967
2360	Serge	1968	/* Come back later if the device is busy... */
		1969	if (!mutex_trylock(&dev->struct_mutex)) {
		1970	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	1971	return;
2360	Serge	1972	}
2352	Serge	1973
2360	Serge	1974	i915_gem_retire_requests(dev);
2352	Serge	1975
2360	Serge	1976	/* Send a periodic flush down the ring so we don't hold onto GEM
		1977	* objects indefinitely.
		1978	*/
		1979	idle = true;
3031	serge	1980	for_each_ring(ring, dev_priv, i) {
		1981	if (ring->gpu_caches_dirty)
		1982	i915_add_request(ring, NULL, NULL);
2352	Serge	1983
2360	Serge	1984	idle &= list_empty(&ring->request_list);
		1985	}
2352	Serge	1986
2360	Serge	1987	if (!dev_priv->mm.suspended && !idle)
		1988	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	1989	if (idle)
		1990	intel_mark_idle(dev);
2360	Serge	1991
		1992	mutex_unlock(&dev->struct_mutex);
		1993	}
		1994
2344	Serge	1995	/**
3031	serge	1996	* Ensures that an object will eventually get non-busy by flushing any required
		1997	* write domains, emitting any outstanding lazy request and retiring and
		1998	* completed requests.
2352	Serge	1999	*/
3031	serge	2000	static int
		2001	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2002	{
3031	serge	2003	int ret;
2352	Serge	2004
3031	serge	2005	if (obj->active) {
		2006	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2007	if (ret)
		2008	return ret;
2352	Serge	2009
3031	serge	2010	i915_gem_retire_requests_ring(obj->ring);
		2011	}
2352	Serge	2012
3031	serge	2013	return 0;
		2014	}
2352	Serge	2015
3243	Serge	2016	/**
		2017	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2018	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2019	*
		2020	* Returns 0 if successful, else an error is returned with the remaining time in
		2021	* the timeout parameter.
		2022	* -ETIME: object is still busy after timeout
		2023	* -ERESTARTSYS: signal interrupted the wait
		2024	* -ENONENT: object doesn't exist
		2025	* Also possible, but rare:
		2026	* -EAGAIN: GPU wedged
		2027	* -ENOMEM: damn
		2028	* -ENODEV: Internal IRQ fail
		2029	* -E?: The add request failed
		2030	*
		2031	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2032	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2033	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2034	* without holding struct_mutex the object may become re-busied before this
		2035	* function completes. A similar but shorter * race condition exists in the busy
		2036	* ioctl
		2037	*/
2352	Serge	2038
		2039
		2040
		2041
		2042
		2043
		2044
		2045
		2046
3243	Serge	2047
		2048
		2049
2352	Serge	2050	/**
3031	serge	2051	* i915_gem_object_sync - sync an object to a ring.
		2052	*
		2053	* @obj: object which may be in use on another ring.
		2054	* @to: ring we wish to use the object on. May be NULL.
		2055	*
		2056	* This code is meant to abstract object synchronization with the GPU.
		2057	* Calling with NULL implies synchronizing the object with the CPU
		2058	* rather than a particular GPU ring.
		2059	*
		2060	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2061	*/
		2062	int
3031	serge	2063	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2064	struct intel_ring_buffer *to)
2344	Serge	2065	{
3031	serge	2066	struct intel_ring_buffer *from = obj->ring;
		2067	u32 seqno;
		2068	int ret, idx;
2332	Serge	2069
3031	serge	2070	if (from == NULL \|\| to == from)
		2071	return 0;
2332	Serge	2072
3031	serge	2073	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2074	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2075
3031	serge	2076	idx = intel_ring_sync_index(from, to);
		2077
		2078	seqno = obj->last_read_seqno;
		2079	if (seqno <= from->sync_seqno[idx])
		2080	return 0;
		2081
		2082	ret = i915_gem_check_olr(obj->ring, seqno);
		2083	if (ret)
		2084	return ret;
		2085
		2086	ret = to->sync_to(to, from, seqno);
		2087	if (!ret)
3243	Serge	2088	/* We use last_read_seqno because sync_to()
		2089	* might have just caused seqno wrap under
		2090	* the radar.
		2091	*/
		2092	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2093
		2094	return ret;
2344	Serge	2095	}
2332	Serge	2096
2344	Serge	2097	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2098	{
		2099	u32 old_write_domain, old_read_domains;
2332	Serge	2100
2344	Serge	2101	/* Act a barrier for all accesses through the GTT */
		2102	mb();
2332	Serge	2103
2344	Serge	2104	/* Force a pagefault for domain tracking on next user access */
		2105	// i915_gem_release_mmap(obj);
2332	Serge	2106
2344	Serge	2107	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2108	return;
2332	Serge	2109
2344	Serge	2110	old_read_domains = obj->base.read_domains;
		2111	old_write_domain = obj->base.write_domain;
2351	Serge	2112
2344	Serge	2113	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2114	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2115
2351	Serge	2116	trace_i915_gem_object_change_domain(obj,
		2117	old_read_domains,
		2118	old_write_domain);
2344	Serge	2119	}
2332	Serge	2120
2344	Serge	2121	/**
		2122	* Unbinds an object from the GTT aperture.
		2123	*/
		2124	int
		2125	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2126	{
3031	serge	2127	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	2128	int ret = 0;
2332	Serge	2129
2344	Serge	2130	if (obj->gtt_space == NULL)
		2131	return 0;
2332	Serge	2132
3031	serge	2133	if (obj->pin_count)
		2134	return -EBUSY;
2332	Serge	2135
3243	Serge	2136	BUG_ON(obj->pages == NULL);
3031	serge	2137
2344	Serge	2138	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2139	if (ret)
2344	Serge	2140	return ret;
		2141	/* Continue on if we fail due to EIO, the GPU is hung so we
		2142	* should be safe and we need to cleanup or else we might
		2143	* cause memory corruption through use-after-free.
		2144	*/
2332	Serge	2145
2344	Serge	2146	i915_gem_object_finish_gtt(obj);
2332	Serge	2147
2344	Serge	2148	/* release the fence reg _after_ flushing */
		2149	ret = i915_gem_object_put_fence(obj);
3031	serge	2150	if (ret)
2344	Serge	2151	return ret;
2332	Serge	2152
2351	Serge	2153	trace_i915_gem_object_unbind(obj);
2332	Serge	2154
3031	serge	2155	if (obj->has_global_gtt_mapping)
3243	Serge	2156	i915_gem_gtt_unbind_object(obj);
3031	serge	2157	if (obj->has_aliasing_ppgtt_mapping) {
		2158	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2159	obj->has_aliasing_ppgtt_mapping = 0;
		2160	}
		2161	i915_gem_gtt_finish_object(obj);
2332	Serge	2162
3031	serge	2163	list_del(&obj->mm_list);
		2164	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2165	/* Avoid an unnecessary call to unbind on rebind. */
		2166	obj->map_and_fenceable = true;
2332	Serge	2167
2344	Serge	2168	drm_mm_put_block(obj->gtt_space);
		2169	obj->gtt_space = NULL;
		2170	obj->gtt_offset = 0;
2332	Serge	2171
2344	Serge	2172	return 0;
		2173	}
2332	Serge	2174
3031	serge	2175	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2176	{
		2177	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2178	struct intel_ring_buffer *ring;
2344	Serge	2179	int ret, i;
2332	Serge	2180
2344	Serge	2181	/* Flush everything onto the inactive list. */
3031	serge	2182	for_each_ring(ring, dev_priv, i) {
		2183	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2184	if (ret)
		2185	return ret;
3031	serge	2186
3243	Serge	2187	ret = intel_ring_idle(ring);
3031	serge	2188	if (ret)
		2189	return ret;
2344	Serge	2190	}
2332	Serge	2191
2344	Serge	2192	return 0;
		2193	}
2332	Serge	2194
3031	serge	2195	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2196	struct drm_i915_gem_object *obj)
		2197	{
		2198	drm_i915_private_t *dev_priv = dev->dev_private;
		2199	uint64_t val;
2332	Serge	2200
3031	serge	2201	if (obj) {
		2202	u32 size = obj->gtt_space->size;
2332	Serge	2203
3031	serge	2204	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2205	0xfffff000) << 32;
		2206	val \|= obj->gtt_offset & 0xfffff000;
		2207	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2208	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2209
3031	serge	2210	if (obj->tiling_mode == I915_TILING_Y)
		2211	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2212	val \|= I965_FENCE_REG_VALID;
		2213	} else
		2214	val = 0;
2332	Serge	2215
3031	serge	2216	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2217	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2218	}
2332	Serge	2219
3031	serge	2220	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2221	struct drm_i915_gem_object *obj)
		2222	{
		2223	drm_i915_private_t *dev_priv = dev->dev_private;
		2224	uint64_t val;
2332	Serge	2225
3031	serge	2226	if (obj) {
		2227	u32 size = obj->gtt_space->size;
2332	Serge	2228
3031	serge	2229	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2230	0xfffff000) << 32;
		2231	val \|= obj->gtt_offset & 0xfffff000;
		2232	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2233	if (obj->tiling_mode == I915_TILING_Y)
		2234	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2235	val \|= I965_FENCE_REG_VALID;
		2236	} else
		2237	val = 0;
2332	Serge	2238
3031	serge	2239	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2240	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2241	}
2332	Serge	2242
3031	serge	2243	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2244	struct drm_i915_gem_object *obj)
		2245	{
		2246	drm_i915_private_t *dev_priv = dev->dev_private;
		2247	u32 val;
2332	Serge	2248
3031	serge	2249	if (obj) {
		2250	u32 size = obj->gtt_space->size;
		2251	int pitch_val;
		2252	int tile_width;
2332	Serge	2253
3031	serge	2254	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2255	(size & -size) != size \|\|
		2256	(obj->gtt_offset & (size - 1)),
		2257	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2258	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2259
3031	serge	2260	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2261	tile_width = 128;
		2262	else
		2263	tile_width = 512;
2332	Serge	2264
3031	serge	2265	/* Note: pitch better be a power of two tile widths */
		2266	pitch_val = obj->stride / tile_width;
		2267	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2268
3031	serge	2269	val = obj->gtt_offset;
		2270	if (obj->tiling_mode == I915_TILING_Y)
		2271	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2272	val \|= I915_FENCE_SIZE_BITS(size);
		2273	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2274	val \|= I830_FENCE_REG_VALID;
		2275	} else
		2276	val = 0;
2332	Serge	2277
3031	serge	2278	if (reg < 8)
		2279	reg = FENCE_REG_830_0 + reg * 4;
		2280	else
		2281	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2282
3031	serge	2283	I915_WRITE(reg, val);
		2284	POSTING_READ(reg);
		2285	}
2332	Serge	2286
3031	serge	2287	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2288	struct drm_i915_gem_object *obj)
		2289	{
		2290	drm_i915_private_t *dev_priv = dev->dev_private;
		2291	uint32_t val;
2344	Serge	2292
3031	serge	2293	if (obj) {
		2294	u32 size = obj->gtt_space->size;
		2295	uint32_t pitch_val;
2344	Serge	2296
3031	serge	2297	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2298	(size & -size) != size \|\|
		2299	(obj->gtt_offset & (size - 1)),
		2300	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2301	obj->gtt_offset, size);
2344	Serge	2302
3031	serge	2303	pitch_val = obj->stride / 128;
		2304	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2305
3031	serge	2306	val = obj->gtt_offset;
		2307	if (obj->tiling_mode == I915_TILING_Y)
		2308	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2309	val \|= I830_FENCE_SIZE_BITS(size);
		2310	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2311	val \|= I830_FENCE_REG_VALID;
		2312	} else
		2313	val = 0;
		2314
		2315	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2316	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2317	}
		2318
		2319	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2320	struct drm_i915_gem_object *obj)
2332	Serge	2321	{
3031	serge	2322	switch (INTEL_INFO(dev)->gen) {
		2323	case 7:
		2324	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2325	case 5:
		2326	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2327	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2328	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2329	default: break;
		2330	}
2344	Serge	2331	}
		2332
3031	serge	2333	static inline int fence_number(struct drm_i915_private *dev_priv,
		2334	struct drm_i915_fence_reg *fence)
2344	Serge	2335	{
3031	serge	2336	return fence - dev_priv->fence_regs;
		2337	}
2332	Serge	2338
3031	serge	2339	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2340	struct drm_i915_fence_reg *fence,
		2341	bool enable)
		2342	{
		2343	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2344	int reg = fence_number(dev_priv, fence);
2332	Serge	2345
3031	serge	2346	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2347
		2348	if (enable) {
		2349	obj->fence_reg = reg;
		2350	fence->obj = obj;
		2351	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2352	} else {
		2353	obj->fence_reg = I915_FENCE_REG_NONE;
		2354	fence->obj = NULL;
		2355	list_del_init(&fence->lru_list);
2344	Serge	2356	}
3031	serge	2357	}
2344	Serge	2358
3031	serge	2359	static int
		2360	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2361	{
		2362	if (obj->last_fenced_seqno) {
		2363	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2364	if (ret)
		2365	return ret;
2344	Serge	2366
		2367	obj->last_fenced_seqno = 0;
		2368	}
		2369
		2370	/* Ensure that all CPU reads are completed before installing a fence
		2371	* and all writes before removing the fence.
2332	Serge	2372	*/
2344	Serge	2373	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2374	mb();
2332	Serge	2375
3031	serge	2376	obj->fenced_gpu_access = false;
2332	Serge	2377	return 0;
		2378	}
		2379
		2380	int
2344	Serge	2381	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2382	{
3031	serge	2383	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2384	int ret;
		2385
3031	serge	2386	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2387	if (ret)
		2388	return ret;
		2389
3031	serge	2390	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2391	return 0;
2332	Serge	2392
3031	serge	2393	i915_gem_object_update_fence(obj,
		2394	&dev_priv->fence_regs[obj->fence_reg],
		2395	false);
		2396	i915_gem_object_fence_lost(obj);
2344	Serge	2397
2332	Serge	2398	return 0;
		2399	}
		2400
3031	serge	2401	static struct drm_i915_fence_reg *
		2402	i915_find_fence_reg(struct drm_device *dev)
		2403	{
		2404	struct drm_i915_private *dev_priv = dev->dev_private;
		2405	struct drm_i915_fence_reg reg, avail;
		2406	int i;
2332	Serge	2407
3031	serge	2408	/* First try to find a free reg */
		2409	avail = NULL;
		2410	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2411	reg = &dev_priv->fence_regs[i];
		2412	if (!reg->obj)
		2413	return reg;
2332	Serge	2414
3031	serge	2415	if (!reg->pin_count)
		2416	avail = reg;
		2417	}
2332	Serge	2418
3031	serge	2419	if (avail == NULL)
		2420	return NULL;
2332	Serge	2421
3031	serge	2422	/* None available, try to steal one or wait for a user to finish */
		2423	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2424	if (reg->pin_count)
		2425	continue;
2332	Serge	2426
3031	serge	2427	return reg;
		2428	}
2332	Serge	2429
3031	serge	2430	return NULL;
		2431	}
2332	Serge	2432
3031	serge	2433	/**
		2434	* i915_gem_object_get_fence - set up fencing for an object
		2435	* @obj: object to map through a fence reg
		2436	*
		2437	* When mapping objects through the GTT, userspace wants to be able to write
		2438	* to them without having to worry about swizzling if the object is tiled.
		2439	* This function walks the fence regs looking for a free one for @obj,
		2440	* stealing one if it can't find any.
		2441	*
		2442	* It then sets up the reg based on the object's properties: address, pitch
		2443	* and tiling format.
		2444	*
		2445	* For an untiled surface, this removes any existing fence.
		2446	*/
		2447	int
		2448	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2449	{
		2450	struct drm_device *dev = obj->base.dev;
		2451	struct drm_i915_private *dev_priv = dev->dev_private;
		2452	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2453	struct drm_i915_fence_reg *reg;
		2454	int ret;
2332	Serge	2455
3031	serge	2456	/* Have we updated the tiling parameters upon the object and so
		2457	* will need to serialise the write to the associated fence register?
		2458	*/
		2459	if (obj->fence_dirty) {
		2460	ret = i915_gem_object_flush_fence(obj);
		2461	if (ret)
		2462	return ret;
		2463	}
2332	Serge	2464
3031	serge	2465	/* Just update our place in the LRU if our fence is getting reused. */
		2466	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2467	reg = &dev_priv->fence_regs[obj->fence_reg];
		2468	if (!obj->fence_dirty) {
		2469	list_move_tail(®->lru_list,
		2470	&dev_priv->mm.fence_list);
		2471	return 0;
		2472	}
		2473	} else if (enable) {
		2474	reg = i915_find_fence_reg(dev);
		2475	if (reg == NULL)
		2476	return -EDEADLK;
2332	Serge	2477
3031	serge	2478	if (reg->obj) {
		2479	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2480
3031	serge	2481	ret = i915_gem_object_flush_fence(old);
		2482	if (ret)
		2483	return ret;
2332	Serge	2484
3031	serge	2485	i915_gem_object_fence_lost(old);
		2486	}
		2487	} else
		2488	return 0;
2332	Serge	2489
3031	serge	2490	i915_gem_object_update_fence(obj, reg, enable);
		2491	obj->fence_dirty = false;
2332	Serge	2492
3031	serge	2493	return 0;
		2494	}
2332	Serge	2495
3031	serge	2496	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2497	struct drm_mm_node *gtt_space,
		2498	unsigned long cache_level)
		2499	{
		2500	struct drm_mm_node *other;
2332	Serge	2501
3031	serge	2502	/* On non-LLC machines we have to be careful when putting differing
		2503	* types of snoopable memory together to avoid the prefetcher
		2504	* crossing memory domains and dieing.
		2505	*/
		2506	if (HAS_LLC(dev))
		2507	return true;
2332	Serge	2508
3031	serge	2509	if (gtt_space == NULL)
		2510	return true;
2332	Serge	2511
3031	serge	2512	if (list_empty(>t_space->node_list))
		2513	return true;
2332	Serge	2514
3031	serge	2515	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2516	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2517	return false;
2344	Serge	2518
3031	serge	2519	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2520	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2521	return false;
2344	Serge	2522
3031	serge	2523	return true;
		2524	}
2344	Serge	2525
3031	serge	2526	static void i915_gem_verify_gtt(struct drm_device *dev)
		2527	{
		2528	#if WATCH_GTT
		2529	struct drm_i915_private *dev_priv = dev->dev_private;
		2530	struct drm_i915_gem_object *obj;
		2531	int err = 0;
2344	Serge	2532
3031	serge	2533	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2534	if (obj->gtt_space == NULL) {
		2535	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2536	err++;
		2537	continue;
		2538	}
2344	Serge	2539
3031	serge	2540	if (obj->cache_level != obj->gtt_space->color) {
		2541	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2542	obj->gtt_space->start,
		2543	obj->gtt_space->start + obj->gtt_space->size,
		2544	obj->cache_level,
		2545	obj->gtt_space->color);
		2546	err++;
		2547	continue;
		2548	}
2344	Serge	2549
3031	serge	2550	if (!i915_gem_valid_gtt_space(dev,
		2551	obj->gtt_space,
		2552	obj->cache_level)) {
		2553	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2554	obj->gtt_space->start,
		2555	obj->gtt_space->start + obj->gtt_space->size,
		2556	obj->cache_level);
		2557	err++;
		2558	continue;
		2559	}
		2560	}
2344	Serge	2561
3031	serge	2562	WARN_ON(err);
		2563	#endif
2326	Serge	2564	}
		2565
2332	Serge	2566	/**
		2567	* Finds free space in the GTT aperture and binds the object there.
		2568	*/
		2569	static int
		2570	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2571	unsigned alignment,
3031	serge	2572	bool map_and_fenceable,
		2573	bool nonblocking)
2332	Serge	2574	{
		2575	struct drm_device *dev = obj->base.dev;
		2576	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2577	struct drm_mm_node *node;
2332	Serge	2578	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2579	bool mappable, fenceable;
		2580	int ret;
2326	Serge	2581
2332	Serge	2582	if (obj->madv != I915_MADV_WILLNEED) {
3243	Serge	2583	DRM_ERROR("Attempting to bind a purgeable object\n");
2332	Serge	2584	return -EINVAL;
		2585	}
		2586
		2587	fence_size = i915_gem_get_gtt_size(dev,
		2588	obj->base.size,
		2589	obj->tiling_mode);
		2590	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2591	obj->base.size,
		2592	obj->tiling_mode);
		2593	unfenced_alignment =
		2594	i915_gem_get_unfenced_gtt_alignment(dev,
		2595	obj->base.size,
		2596	obj->tiling_mode);
		2597
		2598	if (alignment == 0)
		2599	alignment = map_and_fenceable ? fence_alignment :
		2600	unfenced_alignment;
		2601	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2602	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2603	return -EINVAL;
		2604	}
		2605
		2606	size = map_and_fenceable ? fence_size : obj->base.size;
		2607
		2608	/* If the object is bigger than the entire aperture, reject it early
		2609	* before evicting everything in a vain attempt to find space.
		2610	*/
		2611	if (obj->base.size >
		2612	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2613	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2614	return -E2BIG;
		2615	}
		2616
3031	serge	2617	ret = i915_gem_object_get_pages(obj);
		2618	if (ret)
		2619	return ret;
		2620
3243	Serge	2621	i915_gem_object_pin_pages(obj);
		2622
		2623	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2624	if (node == NULL) {
		2625	i915_gem_object_unpin_pages(obj);
		2626	return -ENOMEM;
		2627	}
		2628
2332	Serge	2629	search_free:
		2630	if (map_and_fenceable)
3243	Serge	2631	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2632	size, alignment, obj->cache_level,
3243	Serge	2633	0, dev_priv->mm.gtt_mappable_end);
2332	Serge	2634	else
3243	Serge	2635	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2636	size, alignment, obj->cache_level);
		2637	if (ret) {
2332	Serge	2638
3243	Serge	2639	i915_gem_object_unpin_pages(obj);
		2640	kfree(node);
2332	Serge	2641	return ret;
		2642	}
3243	Serge	2643	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2644	i915_gem_object_unpin_pages(obj);
		2645	drm_mm_put_block(node);
3031	serge	2646	return -EINVAL;
		2647	}
2332	Serge	2648
3031	serge	2649	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2650	if (ret) {
3243	Serge	2651	i915_gem_object_unpin_pages(obj);
		2652	drm_mm_put_block(node);
2332	Serge	2653	return ret;
		2654	}
		2655
3031	serge	2656	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2657	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2658
3243	Serge	2659	obj->gtt_space = node;
		2660	obj->gtt_offset = node->start;
2332	Serge	2661
		2662	fenceable =
3243	Serge	2663	node->size == fence_size &&
		2664	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2665
		2666	mappable =
		2667	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2668
		2669	obj->map_and_fenceable = mappable && fenceable;
		2670
3243	Serge	2671	i915_gem_object_unpin_pages(obj);
2351	Serge	2672	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2673	i915_gem_verify_gtt(dev);
2332	Serge	2674	return 0;
		2675	}
		2676
		2677	void
		2678	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2679	{
		2680	/* If we don't have a page list set up, then we're not pinned
		2681	* to GPU, and we can ignore the cache flush because it'll happen
		2682	* again at bind time.
		2683	*/
3243	Serge	2684	if (obj->pages == NULL)
2332	Serge	2685	return;
		2686
		2687	/* If the GPU is snooping the contents of the CPU cache,
		2688	* we do not need to manually clear the CPU cache lines. However,
		2689	* the caches are only snooped when the render cache is
		2690	* flushed/invalidated. As we always have to emit invalidations
		2691	* and flushes when moving into and out of the RENDER domain, correct
		2692	* snooping behaviour occurs naturally as the result of our domain
		2693	* tracking.
		2694	*/
		2695	if (obj->cache_level != I915_CACHE_NONE)
		2696	return;
3243	Serge	2697	#if 0
2344	Serge	2698	if(obj->mapped != NULL)
		2699	{
		2700	uint8_t *page_virtual;
		2701	unsigned int i;
2332	Serge	2702
2344	Serge	2703	page_virtual = obj->mapped;
		2704	asm volatile("mfence");
		2705	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2706	clflush(page_virtual + i);
		2707	asm volatile("mfence");
		2708	}
		2709	else
		2710	{
		2711	uint8_t *page_virtual;
		2712	unsigned int i;
		2713	page_virtual = AllocKernelSpace(obj->base.size);
		2714	if(page_virtual != NULL)
		2715	{
3031	serge	2716	dma_addr_t src, dst;
2344	Serge	2717	u32 count;
		2718
		2719	#define page_tabs 0xFDC00000 /* really dirty hack */
		2720
3031	serge	2721	src = obj->pages.page;
		2722	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2723	count = obj->base.size/4096;
		2724
		2725	while(count--)
		2726	{
		2727	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2728	};
		2729
		2730	asm volatile("mfence");
		2731	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2732	clflush(page_virtual + i);
		2733	asm volatile("mfence");
		2734	FreeKernelSpace(page_virtual);
		2735	}
		2736	else
		2737	{
		2738	asm volatile (
		2739	"mfence \n"
		2740	"wbinvd \n" /* this is really ugly */
		2741	"mfence");
		2742	}
		2743	}
3243	Serge	2744	#endif
		2745
2332	Serge	2746	}
		2747
2344	Serge	2748	/** Flushes the GTT write domain for the object if it's dirty. */
		2749	static void
		2750	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2751	{
		2752	uint32_t old_write_domain;
2332	Serge	2753
2344	Serge	2754	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2755	return;
2332	Serge	2756
2344	Serge	2757	/* No actual flushing is required for the GTT write domain. Writes
		2758	* to it immediately go to main memory as far as we know, so there's
		2759	* no chipset flush. It also doesn't land in render cache.
		2760	*
		2761	* However, we do have to enforce the order so that all writes through
		2762	* the GTT land before any writes to the device, such as updates to
		2763	* the GATT itself.
		2764	*/
		2765	wmb();
2332	Serge	2766
2344	Serge	2767	old_write_domain = obj->base.write_domain;
		2768	obj->base.write_domain = 0;
2332	Serge	2769
2351	Serge	2770	trace_i915_gem_object_change_domain(obj,
		2771	obj->base.read_domains,
		2772	old_write_domain);
2344	Serge	2773	}
2332	Serge	2774
		2775	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2776	static void
2332	Serge	2777	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2778	{
		2779	uint32_t old_write_domain;
		2780
		2781	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2782	return;
		2783
		2784	i915_gem_clflush_object(obj);
3243	Serge	2785	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2786	old_write_domain = obj->base.write_domain;
		2787	obj->base.write_domain = 0;
		2788
2351	Serge	2789	trace_i915_gem_object_change_domain(obj,
		2790	obj->base.read_domains,
		2791	old_write_domain);
2332	Serge	2792	}
		2793
		2794	/**
		2795	* Moves a single object to the GTT read, and possibly write domain.
		2796	*
		2797	* This function returns when the move is complete, including waiting on
		2798	* flushes to occur.
		2799	*/
		2800	int
		2801	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2802	{
3031	serge	2803	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2804	uint32_t old_write_domain, old_read_domains;
		2805	int ret;
		2806
		2807	/* Not valid to be called on unbound objects. */
		2808	if (obj->gtt_space == NULL)
		2809	return -EINVAL;
		2810
		2811	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2812	return 0;
		2813
3031	serge	2814	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2815	if (ret)
		2816	return ret;
		2817
		2818	i915_gem_object_flush_cpu_write_domain(obj);
		2819
		2820	old_write_domain = obj->base.write_domain;
		2821	old_read_domains = obj->base.read_domains;
		2822
		2823	/* It should now be out of any other write domains, and we can update
		2824	* the domain values for our changes.
		2825	*/
		2826	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2827	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2828	if (write) {
		2829	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2830	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2831	obj->dirty = 1;
		2832	}
		2833
2351	Serge	2834	trace_i915_gem_object_change_domain(obj,
		2835	old_read_domains,
		2836	old_write_domain);
		2837
3031	serge	2838	/* And bump the LRU for this access */
		2839	if (i915_gem_object_is_inactive(obj))
		2840	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2841
2332	Serge	2842	return 0;
		2843	}
		2844
2335	Serge	2845	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2846	enum i915_cache_level cache_level)
		2847	{
3031	serge	2848	struct drm_device *dev = obj->base.dev;
		2849	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2850	int ret;
2332	Serge	2851
2335	Serge	2852	if (obj->cache_level == cache_level)
		2853	return 0;
2332	Serge	2854
2335	Serge	2855	if (obj->pin_count) {
		2856	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2857	return -EBUSY;
		2858	}
2332	Serge	2859
3031	serge	2860	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2861	ret = i915_gem_object_unbind(obj);
		2862	if (ret)
		2863	return ret;
		2864	}
		2865
2335	Serge	2866	if (obj->gtt_space) {
		2867	ret = i915_gem_object_finish_gpu(obj);
		2868	if (ret)
		2869	return ret;
2332	Serge	2870
2335	Serge	2871	i915_gem_object_finish_gtt(obj);
2332	Serge	2872
2335	Serge	2873	/* Before SandyBridge, you could not use tiling or fence
		2874	* registers with snooped memory, so relinquish any fences
		2875	* currently pointing to our region in the aperture.
		2876	*/
3031	serge	2877	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2878	ret = i915_gem_object_put_fence(obj);
		2879	if (ret)
		2880	return ret;
		2881	}
2332	Serge	2882
3031	serge	2883	if (obj->has_global_gtt_mapping)
		2884	i915_gem_gtt_bind_object(obj, cache_level);
		2885	if (obj->has_aliasing_ppgtt_mapping)
		2886	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2887	obj, cache_level);
		2888
		2889	obj->gtt_space->color = cache_level;
2335	Serge	2890	}
2332	Serge	2891
2335	Serge	2892	if (cache_level == I915_CACHE_NONE) {
		2893	u32 old_read_domains, old_write_domain;
2332	Serge	2894
2335	Serge	2895	/* If we're coming from LLC cached, then we haven't
		2896	* actually been tracking whether the data is in the
		2897	* CPU cache or not, since we only allow one bit set
		2898	* in obj->write_domain and have been skipping the clflushes.
		2899	* Just set it to the CPU cache for now.
		2900	*/
		2901	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2902	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2903
2335	Serge	2904	old_read_domains = obj->base.read_domains;
		2905	old_write_domain = obj->base.write_domain;
2332	Serge	2906
2335	Serge	2907	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2908	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2909
2351	Serge	2910	trace_i915_gem_object_change_domain(obj,
		2911	old_read_domains,
		2912	old_write_domain);
2344	Serge	2913	}
2332	Serge	2914
2335	Serge	2915	obj->cache_level = cache_level;
3031	serge	2916	i915_gem_verify_gtt(dev);
2335	Serge	2917	return 0;
		2918	}
2332	Serge	2919
3260	Serge	2920	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		2921	struct drm_file *file)
		2922	{
		2923	struct drm_i915_gem_caching *args = data;
		2924	struct drm_i915_gem_object *obj;
		2925	int ret;
		2926
		2927	ret = i915_mutex_lock_interruptible(dev);
		2928	if (ret)
		2929	return ret;
		2930
		2931	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2932	if (&obj->base == NULL) {
		2933	ret = -ENOENT;
		2934	goto unlock;
		2935	}
		2936
		2937	args->caching = obj->cache_level != I915_CACHE_NONE;
		2938
		2939	drm_gem_object_unreference(&obj->base);
		2940	unlock:
		2941	mutex_unlock(&dev->struct_mutex);
		2942	return ret;
		2943	}
		2944
		2945	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		2946	struct drm_file *file)
		2947	{
		2948	struct drm_i915_gem_caching *args = data;
		2949	struct drm_i915_gem_object *obj;
		2950	enum i915_cache_level level;
		2951	int ret;
		2952
		2953	switch (args->caching) {
		2954	case I915_CACHING_NONE:
		2955	level = I915_CACHE_NONE;
		2956	break;
		2957	case I915_CACHING_CACHED:
		2958	level = I915_CACHE_LLC;
		2959	break;
		2960	default:
		2961	return -EINVAL;
		2962	}
		2963
		2964	ret = i915_mutex_lock_interruptible(dev);
		2965	if (ret)
		2966	return ret;
		2967
		2968	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2969	if (&obj->base == NULL) {
		2970	ret = -ENOENT;
		2971	goto unlock;
		2972	}
		2973
		2974	ret = i915_gem_object_set_cache_level(obj, level);
		2975
		2976	drm_gem_object_unreference(&obj->base);
		2977	unlock:
		2978	mutex_unlock(&dev->struct_mutex);
		2979	return ret;
		2980	}
		2981
2335	Serge	2982	/*
		2983	* Prepare buffer for display plane (scanout, cursors, etc).
		2984	* Can be called from an uninterruptible phase (modesetting) and allows
		2985	* any flushes to be pipelined (for pageflips).
		2986	*/
		2987	int
		2988	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		2989	u32 alignment,
		2990	struct intel_ring_buffer *pipelined)
		2991	{
		2992	u32 old_read_domains, old_write_domain;
		2993	int ret;
2332	Serge	2994
3031	serge	2995	if (pipelined != obj->ring) {
		2996	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	2997	if (ret)
		2998	return ret;
		2999	}
2332	Serge	3000
2335	Serge	3001	/* The display engine is not coherent with the LLC cache on gen6. As
		3002	* a result, we make sure that the pinning that is about to occur is
		3003	* done with uncached PTEs. This is lowest common denominator for all
		3004	* chipsets.
		3005	*
		3006	* However for gen6+, we could do better by using the GFDT bit instead
		3007	* of uncaching, which would allow us to flush all the LLC-cached data
		3008	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3009	*/
2360	Serge	3010	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3011	if (ret)
		3012	return ret;
2332	Serge	3013
2335	Serge	3014	/* As the user may map the buffer once pinned in the display plane
		3015	* (e.g. libkms for the bootup splash), we have to ensure that we
		3016	* always use map_and_fenceable for all scanout buffers.
		3017	*/
3031	serge	3018	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3019	if (ret)
		3020	return ret;
2332	Serge	3021
2335	Serge	3022	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3023
2335	Serge	3024	old_write_domain = obj->base.write_domain;
		3025	old_read_domains = obj->base.read_domains;
2332	Serge	3026
2335	Serge	3027	/* It should now be out of any other write domains, and we can update
		3028	* the domain values for our changes.
		3029	*/
3031	serge	3030	obj->base.write_domain = 0;
2335	Serge	3031	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3032
2351	Serge	3033	trace_i915_gem_object_change_domain(obj,
		3034	old_read_domains,
		3035	old_write_domain);
2332	Serge	3036
2335	Serge	3037	return 0;
		3038	}
2332	Serge	3039
2344	Serge	3040	int
		3041	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3042	{
		3043	int ret;
2332	Serge	3044
2344	Serge	3045	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3046	return 0;
2332	Serge	3047
3031	serge	3048	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3049	if (ret)
		3050	return ret;
2332	Serge	3051
2344	Serge	3052	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3053	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3054	return 0;
2344	Serge	3055	}
2332	Serge	3056
2344	Serge	3057	/**
		3058	* Moves a single object to the CPU read, and possibly write domain.
		3059	*
		3060	* This function returns when the move is complete, including waiting on
		3061	* flushes to occur.
		3062	*/
3031	serge	3063	int
2344	Serge	3064	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3065	{
		3066	uint32_t old_write_domain, old_read_domains;
		3067	int ret;
2332	Serge	3068
2344	Serge	3069	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3070	return 0;
2332	Serge	3071
3031	serge	3072	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3073	if (ret)
		3074	return ret;
2332	Serge	3075
2344	Serge	3076	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3077
2344	Serge	3078	old_write_domain = obj->base.write_domain;
		3079	old_read_domains = obj->base.read_domains;
2332	Serge	3080
2344	Serge	3081	/* Flush the CPU cache if it's still invalid. */
		3082	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3083	i915_gem_clflush_object(obj);
2332	Serge	3084
2344	Serge	3085	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3086	}
2332	Serge	3087
2344	Serge	3088	/* It should now be out of any other write domains, and we can update
		3089	* the domain values for our changes.
		3090	*/
		3091	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3092
2344	Serge	3093	/* If we're writing through the CPU, then the GPU read domains will
		3094	* need to be invalidated at next use.
		3095	*/
		3096	if (write) {
		3097	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3098	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3099	}
2332	Serge	3100
2351	Serge	3101	trace_i915_gem_object_change_domain(obj,
		3102	old_read_domains,
		3103	old_write_domain);
2332	Serge	3104
2344	Serge	3105	return 0;
		3106	}
2332	Serge	3107
3031	serge	3108	#if 0
		3109	/* Throttle our rendering by waiting until the ring has completed our requests
		3110	* emitted over 20 msec ago.
2344	Serge	3111	*
3031	serge	3112	* Note that if we were to use the current jiffies each time around the loop,
		3113	* we wouldn't escape the function with any frames outstanding if the time to
		3114	* render a frame was over 20ms.
		3115	*
		3116	* This should get us reasonable parallelism between CPU and GPU but also
		3117	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3118	*/
3031	serge	3119	static int
		3120	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3121	{
3031	serge	3122	struct drm_i915_private *dev_priv = dev->dev_private;
		3123	struct drm_i915_file_private *file_priv = file->driver_priv;
		3124	unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20);
		3125	struct drm_i915_gem_request *request;
		3126	struct intel_ring_buffer *ring = NULL;
		3127	u32 seqno = 0;
		3128	int ret;
2332	Serge	3129
3031	serge	3130	if (atomic_read(&dev_priv->mm.wedged))
		3131	return -EIO;
2332	Serge	3132
3031	serge	3133	spin_lock(&file_priv->mm.lock);
		3134	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3135	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3136	break;
2332	Serge	3137
3031	serge	3138	ring = request->ring;
		3139	seqno = request->seqno;
		3140	}
		3141	spin_unlock(&file_priv->mm.lock);
2332	Serge	3142
3031	serge	3143	if (seqno == 0)
		3144	return 0;
2332	Serge	3145
3031	serge	3146	ret = __wait_seqno(ring, seqno, true, NULL);
		3147	if (ret == 0)
		3148	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3149
3031	serge	3150	return ret;
2352	Serge	3151	}
3031	serge	3152	#endif
2332	Serge	3153
		3154	int
		3155	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3156	uint32_t alignment,
3031	serge	3157	bool map_and_fenceable,
		3158	bool nonblocking)
2332	Serge	3159	{
		3160	int ret;
		3161
3031	serge	3162	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3163	return -EBUSY;
2332	Serge	3164
		3165	#if 0
		3166	if (obj->gtt_space != NULL) {
		3167	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3168	(map_and_fenceable && !obj->map_and_fenceable)) {
		3169	WARN(obj->pin_count,
		3170	"bo is already pinned with incorrect alignment:"
		3171	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3172	" obj->map_and_fenceable=%d\n",
		3173	obj->gtt_offset, alignment,
		3174	map_and_fenceable,
		3175	obj->map_and_fenceable);
		3176	ret = i915_gem_object_unbind(obj);
		3177	if (ret)
		3178	return ret;
		3179	}
		3180	}
		3181	#endif
		3182
		3183	if (obj->gtt_space == NULL) {
3243	Serge	3184	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3185
2332	Serge	3186	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3187	map_and_fenceable,
		3188	nonblocking);
2332	Serge	3189	if (ret)
		3190	return ret;
3243	Serge	3191
		3192	if (!dev_priv->mm.aliasing_ppgtt)
		3193	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3194	}
		3195
3031	serge	3196	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3197	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3198
		3199	obj->pin_count++;
2332	Serge	3200	obj->pin_mappable \|= map_and_fenceable;
		3201
		3202	return 0;
		3203	}
		3204
2344	Serge	3205	void
		3206	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3207	{
		3208	BUG_ON(obj->pin_count == 0);
		3209	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3210
3031	serge	3211	if (--obj->pin_count == 0)
2344	Serge	3212	obj->pin_mappable = false;
		3213	}
2332	Serge	3214
3031	serge	3215	int
		3216	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3217	struct drm_file *file)
		3218	{
		3219	struct drm_i915_gem_pin *args = data;
		3220	struct drm_i915_gem_object *obj;
		3221	int ret;
2332	Serge	3222
3031	serge	3223	ret = i915_mutex_lock_interruptible(dev);
		3224	if (ret)
		3225	return ret;
2332	Serge	3226
3031	serge	3227	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3228	if (&obj->base == NULL) {
		3229	ret = -ENOENT;
		3230	goto unlock;
		3231	}
2332	Serge	3232
3031	serge	3233	if (obj->madv != I915_MADV_WILLNEED) {
		3234	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3235	ret = -EINVAL;
		3236	goto out;
		3237	}
2332	Serge	3238
3031	serge	3239	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3240	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3241	args->handle);
		3242	ret = -EINVAL;
		3243	goto out;
		3244	}
2332	Serge	3245
3243	Serge	3246	if (obj->user_pin_count == 0) {
3031	serge	3247	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3248	if (ret)
		3249	goto out;
		3250	}
2332	Serge	3251
3243	Serge	3252	obj->user_pin_count++;
		3253	obj->pin_filp = file;
		3254
3031	serge	3255	/* XXX - flush the CPU caches for pinned objects
		3256	* as the X server doesn't manage domains yet
		3257	*/
		3258	i915_gem_object_flush_cpu_write_domain(obj);
		3259	args->offset = obj->gtt_offset;
		3260	out:
		3261	drm_gem_object_unreference(&obj->base);
		3262	unlock:
		3263	mutex_unlock(&dev->struct_mutex);
		3264	return ret;
		3265	}
2332	Serge	3266
3255	Serge	3267	#if 0
		3268
3031	serge	3269	int
		3270	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3271	struct drm_file *file)
		3272	{
		3273	struct drm_i915_gem_pin *args = data;
		3274	struct drm_i915_gem_object *obj;
		3275	int ret;
2332	Serge	3276
3031	serge	3277	ret = i915_mutex_lock_interruptible(dev);
		3278	if (ret)
		3279	return ret;
2332	Serge	3280
3031	serge	3281	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3282	if (&obj->base == NULL) {
		3283	ret = -ENOENT;
		3284	goto unlock;
		3285	}
2332	Serge	3286
3031	serge	3287	if (obj->pin_filp != file) {
		3288	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3289	args->handle);
		3290	ret = -EINVAL;
		3291	goto out;
		3292	}
		3293	obj->user_pin_count--;
		3294	if (obj->user_pin_count == 0) {
		3295	obj->pin_filp = NULL;
		3296	i915_gem_object_unpin(obj);
		3297	}
2332	Serge	3298
3031	serge	3299	out:
		3300	drm_gem_object_unreference(&obj->base);
		3301	unlock:
		3302	mutex_unlock(&dev->struct_mutex);
		3303	return ret;
		3304	}
2332	Serge	3305
3260	Serge	3306	#endif
		3307
3031	serge	3308	int
		3309	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3310	struct drm_file *file)
		3311	{
		3312	struct drm_i915_gem_busy *args = data;
		3313	struct drm_i915_gem_object *obj;
		3314	int ret;
2332	Serge	3315
3031	serge	3316	ret = i915_mutex_lock_interruptible(dev);
		3317	if (ret)
		3318	return ret;
2332	Serge	3319
3031	serge	3320	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3321	if (&obj->base == NULL) {
		3322	ret = -ENOENT;
		3323	goto unlock;
		3324	}
2332	Serge	3325
3031	serge	3326	/* Count all active objects as busy, even if they are currently not used
		3327	* by the gpu. Users of this interface expect objects to eventually
		3328	* become non-busy without any further actions, therefore emit any
		3329	* necessary flushes here.
		3330	*/
		3331	ret = i915_gem_object_flush_active(obj);
2332	Serge	3332
3031	serge	3333	args->busy = obj->active;
		3334	if (obj->ring) {
		3335	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3336	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3337	}
2332	Serge	3338
3031	serge	3339	drm_gem_object_unreference(&obj->base);
		3340	unlock:
		3341	mutex_unlock(&dev->struct_mutex);
		3342	return ret;
		3343	}
2332	Serge	3344
3260	Serge	3345	#if 0
3031	serge	3346	int
		3347	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3348	struct drm_file *file_priv)
		3349	{
		3350	return i915_gem_ring_throttle(dev, file_priv);
		3351	}
2332	Serge	3352
3031	serge	3353	int
		3354	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3355	struct drm_file *file_priv)
		3356	{
		3357	struct drm_i915_gem_madvise *args = data;
		3358	struct drm_i915_gem_object *obj;
		3359	int ret;
2332	Serge	3360
3031	serge	3361	switch (args->madv) {
		3362	case I915_MADV_DONTNEED:
		3363	case I915_MADV_WILLNEED:
		3364	break;
		3365	default:
		3366	return -EINVAL;
		3367	}
2332	Serge	3368
3031	serge	3369	ret = i915_mutex_lock_interruptible(dev);
		3370	if (ret)
		3371	return ret;
2332	Serge	3372
3031	serge	3373	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3374	if (&obj->base == NULL) {
		3375	ret = -ENOENT;
		3376	goto unlock;
		3377	}
2332	Serge	3378
3031	serge	3379	if (obj->pin_count) {
		3380	ret = -EINVAL;
		3381	goto out;
		3382	}
2332	Serge	3383
3031	serge	3384	if (obj->madv != __I915_MADV_PURGED)
		3385	obj->madv = args->madv;
2332	Serge	3386
3031	serge	3387	/* if the object is no longer attached, discard its backing storage */
		3388	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3389	i915_gem_object_truncate(obj);
2332	Serge	3390
3031	serge	3391	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3392
3031	serge	3393	out:
		3394	drm_gem_object_unreference(&obj->base);
		3395	unlock:
		3396	mutex_unlock(&dev->struct_mutex);
		3397	return ret;
		3398	}
		3399	#endif
2332	Serge	3400
3031	serge	3401	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3402	const struct drm_i915_gem_object_ops *ops)
		3403	{
		3404	INIT_LIST_HEAD(&obj->mm_list);
		3405	INIT_LIST_HEAD(&obj->gtt_list);
		3406	INIT_LIST_HEAD(&obj->ring_list);
		3407	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3408
3031	serge	3409	obj->ops = ops;
		3410
		3411	obj->fence_reg = I915_FENCE_REG_NONE;
		3412	obj->madv = I915_MADV_WILLNEED;
		3413	/* Avoid an unnecessary call to unbind on the first bind. */
		3414	obj->map_and_fenceable = true;
		3415
		3416	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3417	}
		3418
		3419	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3420	.get_pages = i915_gem_object_get_pages_gtt,
		3421	.put_pages = i915_gem_object_put_pages_gtt,
		3422	};
		3423
2332	Serge	3424	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3425	size_t size)
		3426	{
		3427	struct drm_i915_gem_object *obj;
3031	serge	3428	struct address_space *mapping;
		3429	u32 mask;
2340	Serge	3430
2332	Serge	3431	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3432	if (obj == NULL)
		3433	return NULL;
		3434
		3435	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3436	kfree(obj);
		3437	return NULL;
		3438	}
		3439
		3440
3031	serge	3441	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3442
		3443	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3444	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3445
3031	serge	3446	if (HAS_LLC(dev)) {
		3447	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3448	* cache) for about a 10% performance improvement
		3449	* compared to uncached. Graphics requests other than
		3450	* display scanout are coherent with the CPU in
		3451	* accessing this cache. This means in this mode we
		3452	* don't need to clflush on the CPU side, and on the
		3453	* GPU side we only need to flush internal caches to
		3454	* get data visible to the CPU.
		3455	*
		3456	* However, we maintain the display planes as UC, and so
		3457	* need to rebind when first used as such.
		3458	*/
		3459	obj->cache_level = I915_CACHE_LLC;
		3460	} else
		3461	obj->cache_level = I915_CACHE_NONE;
		3462
		3463	return obj;
		3464	}
		3465
2344	Serge	3466	int i915_gem_init_object(struct drm_gem_object *obj)
		3467	{
		3468	BUG();
2332	Serge	3469
2344	Serge	3470	return 0;
		3471	}
2332	Serge	3472
3031	serge	3473	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3474	{
3031	serge	3475	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3476	struct drm_device *dev = obj->base.dev;
		3477	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3478
3031	serge	3479	trace_i915_gem_object_destroy(obj);
		3480
		3481	// if (obj->phys_obj)
		3482	// i915_gem_detach_phys_object(dev, obj);
		3483
		3484	obj->pin_count = 0;
		3485	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3486	bool was_interruptible;
		3487
		3488	was_interruptible = dev_priv->mm.interruptible;
		3489	dev_priv->mm.interruptible = false;
		3490
		3491	WARN_ON(i915_gem_object_unbind(obj));
		3492
		3493	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3494	}
2332	Serge	3495
3031	serge	3496	obj->pages_pin_count = 0;
		3497	i915_gem_object_put_pages(obj);
		3498	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3499
3243	Serge	3500	BUG_ON(obj->pages);
2332	Serge	3501
3031	serge	3502	// if (obj->base.import_attach)
		3503	// drm_prime_gem_destroy(&obj->base, NULL);
		3504
2344	Serge	3505	drm_gem_object_release(&obj->base);
		3506	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3507
2344	Serge	3508	kfree(obj->bit_17);
		3509	kfree(obj);
		3510	}
2332	Serge	3511
3031	serge	3512	#if 0
		3513	int
		3514	i915_gem_idle(struct drm_device *dev)
2344	Serge	3515	{
3031	serge	3516	drm_i915_private_t *dev_priv = dev->dev_private;
		3517	int ret;
2332	Serge	3518
3031	serge	3519	mutex_lock(&dev->struct_mutex);
2332	Serge	3520
3031	serge	3521	if (dev_priv->mm.suspended) {
		3522	mutex_unlock(&dev->struct_mutex);
		3523	return 0;
		3524	}
2332	Serge	3525
3031	serge	3526	ret = i915_gpu_idle(dev);
		3527	if (ret) {
		3528	mutex_unlock(&dev->struct_mutex);
		3529	return ret;
		3530	}
		3531	i915_gem_retire_requests(dev);
		3532
		3533	i915_gem_reset_fences(dev);
		3534
		3535	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3536	* We need to replace this with a semaphore, or something.
		3537	* And not confound mm.suspended!
		3538	*/
		3539	dev_priv->mm.suspended = 1;
		3540	del_timer_sync(&dev_priv->hangcheck_timer);
		3541
		3542	i915_kernel_lost_context(dev);
		3543	i915_gem_cleanup_ringbuffer(dev);
		3544
		3545	mutex_unlock(&dev->struct_mutex);
		3546
		3547	/* Cancel the retire work handler, which should be idle now. */
		3548	// cancel_delayed_work_sync(&dev_priv->mm.retire_work);
		3549
		3550	return 0;
2344	Serge	3551	}
3031	serge	3552	#endif
2332	Serge	3553
3031	serge	3554	void i915_gem_l3_remap(struct drm_device *dev)
		3555	{
		3556	drm_i915_private_t *dev_priv = dev->dev_private;
		3557	u32 misccpctl;
		3558	int i;
2332	Serge	3559
3031	serge	3560	if (!IS_IVYBRIDGE(dev))
		3561	return;
2332	Serge	3562
3243	Serge	3563	if (!dev_priv->l3_parity.remap_info)
3031	serge	3564	return;
2332	Serge	3565
3031	serge	3566	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3567	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3568	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3569
3031	serge	3570	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3571	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3572	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3573	DRM_DEBUG("0x%x was already programmed to %x\n",
		3574	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3575	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3576	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3577	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3578	}
2332	Serge	3579
3031	serge	3580	/* Make sure all the writes land before disabling dop clock gating */
		3581	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3582
3031	serge	3583	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3584	}
2332	Serge	3585
3031	serge	3586	void i915_gem_init_swizzling(struct drm_device *dev)
		3587	{
		3588	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3589
3031	serge	3590	if (INTEL_INFO(dev)->gen < 5 \|\|
		3591	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3592	return;
2332	Serge	3593
3031	serge	3594	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3595	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3596
3031	serge	3597	if (IS_GEN5(dev))
		3598	return;
2344	Serge	3599
3031	serge	3600	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3601	if (IS_GEN6(dev))
		3602	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3603	else
		3604	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3605	}
		3606
		3607	static bool
		3608	intel_enable_blt(struct drm_device *dev)
		3609	{
		3610	if (!HAS_BLT(dev))
		3611	return false;
		3612
		3613	/* The blitter was dysfunctional on early prototypes */
		3614	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3615	DRM_INFO("BLT not supported on this pre-production hardware;"
		3616	" graphics performance will be degraded.\n");
		3617	return false;
		3618	}
		3619
		3620	return true;
		3621	}
		3622
2332	Serge	3623	int
3031	serge	3624	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3625	{
		3626	drm_i915_private_t *dev_priv = dev->dev_private;
		3627	int ret;
2351	Serge	3628
3243	Serge	3629	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
3031	serge	3630	return -EIO;
		3631
		3632	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3633	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3634
		3635	i915_gem_l3_remap(dev);
		3636
		3637	i915_gem_init_swizzling(dev);
		3638
2332	Serge	3639	ret = intel_init_render_ring_buffer(dev);
		3640	if (ret)
		3641	return ret;
		3642
		3643	if (HAS_BSD(dev)) {
		3644	ret = intel_init_bsd_ring_buffer(dev);
		3645	if (ret)
		3646	goto cleanup_render_ring;
		3647	}
		3648
3031	serge	3649	if (intel_enable_blt(dev)) {
2332	Serge	3650	ret = intel_init_blt_ring_buffer(dev);
		3651	if (ret)
		3652	goto cleanup_bsd_ring;
		3653	}
		3654
		3655	dev_priv->next_seqno = 1;
2351	Serge	3656
3031	serge	3657	/*
		3658	* XXX: There was some w/a described somewhere suggesting loading
		3659	* contexts before PPGTT.
		3660	*/
		3661	i915_gem_context_init(dev);
		3662	i915_gem_init_ppgtt(dev);
		3663
2332	Serge	3664	return 0;
		3665
		3666	cleanup_bsd_ring:
		3667	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3668	cleanup_render_ring:
		3669	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3670	return ret;
		3671	}
		3672
3031	serge	3673	static bool
		3674	intel_enable_ppgtt(struct drm_device *dev)
		3675	{
		3676	if (i915_enable_ppgtt >= 0)
		3677	return i915_enable_ppgtt;
		3678
		3679	#ifdef CONFIG_INTEL_IOMMU
		3680	/* Disable ppgtt on SNB if VT-d is on. */
		3681	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3682	return false;
		3683	#endif
		3684
		3685	return true;
		3686	}
		3687
		3688	#define LFB_SIZE 0xC00000
		3689
		3690	int i915_gem_init(struct drm_device *dev)
		3691	{
		3692	struct drm_i915_private *dev_priv = dev->dev_private;
		3693	unsigned long gtt_size, mappable_size;
		3694	int ret;
		3695
		3696	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3697	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3698
		3699	mutex_lock(&dev->struct_mutex);
		3700	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3701	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3702	* aperture accordingly when using aliasing ppgtt. */
		3703	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3704
		3705	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3706
		3707	ret = i915_gem_init_aliasing_ppgtt(dev);
		3708	if (ret) {
		3709	mutex_unlock(&dev->struct_mutex);
		3710	return ret;
		3711	}
		3712	} else {
		3713	/* Let GEM Manage all of the aperture.
		3714	*
		3715	* However, leave one page at the end still bound to the scratch
		3716	* page. There are a number of places where the hardware
		3717	* apparently prefetches past the end of the object, and we've
		3718	* seen multiple hangs with the GPU head pointer stuck in a
		3719	* batchbuffer bound at the last page of the aperture. One page
		3720	* should be enough to keep any prefetching inside of the
		3721	* aperture.
		3722	*/
		3723	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3724	}
		3725
		3726	ret = i915_gem_init_hw(dev);
		3727	mutex_unlock(&dev->struct_mutex);
		3728	if (ret) {
		3729	i915_gem_cleanup_aliasing_ppgtt(dev);
		3730	return ret;
		3731	}
		3732
		3733	return 0;
		3734	}
		3735
2332	Serge	3736	void
		3737	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3738	{
		3739	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3740	struct intel_ring_buffer *ring;
2332	Serge	3741	int i;
		3742
3031	serge	3743	for_each_ring(ring, dev_priv, i)
		3744	intel_cleanup_ring_buffer(ring);
2332	Serge	3745	}
		3746
3031	serge	3747	#if 0
		3748
2332	Serge	3749	int
		3750	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3751	struct drm_file *file_priv)
		3752	{
		3753	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3754	int ret;
2332	Serge	3755
		3756	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3757	return 0;
		3758
		3759	if (atomic_read(&dev_priv->mm.wedged)) {
		3760	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3761	atomic_set(&dev_priv->mm.wedged, 0);
		3762	}
		3763
		3764	mutex_lock(&dev->struct_mutex);
		3765	dev_priv->mm.suspended = 0;
		3766
3031	serge	3767	ret = i915_gem_init_hw(dev);
2332	Serge	3768	if (ret != 0) {
		3769	mutex_unlock(&dev->struct_mutex);
		3770	return ret;
		3771	}
		3772
		3773	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3774	mutex_unlock(&dev->struct_mutex);
		3775
		3776	ret = drm_irq_install(dev);
		3777	if (ret)
		3778	goto cleanup_ringbuffer;
		3779
		3780	return 0;
		3781
		3782	cleanup_ringbuffer:
		3783	mutex_lock(&dev->struct_mutex);
		3784	i915_gem_cleanup_ringbuffer(dev);
		3785	dev_priv->mm.suspended = 1;
		3786	mutex_unlock(&dev->struct_mutex);
		3787
		3788	return ret;
		3789	}
		3790
		3791	int
		3792	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3793	struct drm_file *file_priv)
		3794	{
		3795	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3796	return 0;
		3797
		3798	drm_irq_uninstall(dev);
		3799	return i915_gem_idle(dev);
		3800	}
		3801
		3802	void
		3803	i915_gem_lastclose(struct drm_device *dev)
		3804	{
		3805	int ret;
		3806
		3807	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3808	return;
		3809
		3810	ret = i915_gem_idle(dev);
		3811	if (ret)
		3812	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3813	}
		3814	#endif
		3815
		3816	static void
2326	Serge	3817	init_ring_lists(struct intel_ring_buffer *ring)
		3818	{
		3819	INIT_LIST_HEAD(&ring->active_list);
		3820	INIT_LIST_HEAD(&ring->request_list);
		3821	}
		3822
		3823	void
		3824	i915_gem_load(struct drm_device *dev)
		3825	{
		3826	int i;
		3827	drm_i915_private_t *dev_priv = dev->dev_private;
		3828
		3829	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3830	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3831	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3832	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3833	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3834	for (i = 0; i < I915_NUM_RINGS; i++)
		3835	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3836	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3837	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3838	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3839	i915_gem_retire_work_handler);
2326	Serge	3840
		3841	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3842	if (IS_GEN3(dev)) {
3031	serge	3843	I915_WRITE(MI_ARB_STATE,
		3844	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3845	}
		3846
		3847	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3848
		3849	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3850	dev_priv->num_fence_regs = 16;
		3851	else
		3852	dev_priv->num_fence_regs = 8;
		3853
		3854	/* Initialize fence registers to zero */
3031	serge	3855	i915_gem_reset_fences(dev);
2326	Serge	3856
		3857	i915_gem_detect_bit_6_swizzle(dev);
		3858
		3859	dev_priv->mm.interruptible = true;
		3860
		3861	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3862	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3863	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3864	}
		3865
		3866

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 3482 – Rev 3260