WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
3263	Serge	40	#define PROT_READ 0x1 /* page can be read */
		41	#define PROT_WRITE 0x2 /* page can be written */
		42	#define MAP_SHARED 0x01 /* Share changes */
		43
2344	Serge	44	#undef mb
		45	#undef rmb
		46	#undef wmb
		47	#define mb() asm volatile("mfence")
		48	#define rmb() asm volatile ("lfence")
		49	#define wmb() asm volatile ("sfence")
		50
3266	Serge	51	struct drm_i915_gem_object *get_fb_obj();
		52
3263	Serge	53	unsigned long vm_mmap(struct file *file, unsigned long addr,
		54	unsigned long len, unsigned long prot,
		55	unsigned long flag, unsigned long offset);
		56
2344	Serge	57	static inline void clflush(volatile void *__p)
		58	{
		59	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		60	}
		61
2332	Serge	62	#define MAX_ERRNO 4095
		63
		64	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		65
		66
2326	Serge	67	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		68	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		69	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		70	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		71
2332	Serge	72	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		73	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		74	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		75	unsigned alignment,
3031	serge	76	bool map_and_fenceable,
		77	bool nonblocking);
2332	Serge	78	static int i915_gem_phys_pwrite(struct drm_device *dev,
		79	struct drm_i915_gem_object *obj,
		80	struct drm_i915_gem_pwrite *args,
		81	struct drm_file *file);
2326	Serge	82
3031	serge	83	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		84	struct drm_i915_gem_object *obj);
		85	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		86	struct drm_i915_fence_reg *fence,
		87	bool enable);
2332	Serge	88
3031	serge	89	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		90	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		91	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		92
		93	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		94	{
		95	if (obj->tiling_mode)
		96	i915_gem_release_mmap(obj);
		97
		98	/* As we do not have an associated fence register, we will force
		99	* a tiling change if we ever need to acquire one.
		100	*/
		101	obj->fence_dirty = false;
		102	obj->fence_reg = I915_FENCE_REG_NONE;
		103	}
		104
2332	Serge	105	/* some bookkeeping */
		106	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		107	size_t size)
		108	{
		109	dev_priv->mm.object_count++;
		110	dev_priv->mm.object_memory += size;
		111	}
		112
		113	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		114	size_t size)
		115	{
		116	dev_priv->mm.object_count--;
		117	dev_priv->mm.object_memory -= size;
		118	}
		119
		120	static int
		121	i915_gem_wait_for_error(struct drm_device *dev)
		122	{
		123	struct drm_i915_private *dev_priv = dev->dev_private;
		124	struct completion *x = &dev_priv->error_completion;
		125	unsigned long flags;
		126	int ret;
		127
		128	if (!atomic_read(&dev_priv->mm.wedged))
		129	return 0;
3255	Serge	130	#if 0
3031	serge	131	/*
		132	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		133	* userspace. If it takes that long something really bad is going on and
		134	* we should simply try to bail out and fail as gracefully as possible.
		135	*/
		136	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		137	if (ret == 0) {
		138	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		139	return -EIO;
		140	} else if (ret < 0) {
2332	Serge	141	return ret;
3031	serge	142	}
2332	Serge	143
		144	if (atomic_read(&dev_priv->mm.wedged)) {
		145	/* GPU is hung, bump the completion count to account for
		146	* the token we just consumed so that we never hit zero and
		147	* end up waiting upon a subsequent completion event that
		148	* will never happen.
		149	*/
		150	spin_lock_irqsave(&x->wait.lock, flags);
		151	x->done++;
		152	spin_unlock_irqrestore(&x->wait.lock, flags);
		153	}
3255	Serge	154	#endif
		155
2332	Serge	156	return 0;
		157	}
		158
		159	int i915_mutex_lock_interruptible(struct drm_device *dev)
		160	{
		161	int ret;
		162
		163	ret = i915_gem_wait_for_error(dev);
		164	if (ret)
		165	return ret;
		166
3255	Serge	167	mutex_lock(&dev->struct_mutex);
2332	Serge	168
		169	WARN_ON(i915_verify_lists(dev));
		170	return 0;
		171	}
		172
		173	static inline bool
		174	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		175	{
3031	serge	176	return obj->gtt_space && !obj->active;
2332	Serge	177	}
		178
		179
		180	#if 0
		181
		182	int
		183	i915_gem_init_ioctl(struct drm_device dev, void data,
		184	struct drm_file *file)
		185	{
		186	struct drm_i915_gem_init *args = data;
		187
3031	serge	188	if (drm_core_check_feature(dev, DRIVER_MODESET))
		189	return -ENODEV;
		190
2332	Serge	191	if (args->gtt_start >= args->gtt_end \|\|
		192	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		193	return -EINVAL;
		194
3031	serge	195	/* GEM with user mode setting was never supported on ilk and later. */
		196	if (INTEL_INFO(dev)->gen >= 5)
		197	return -ENODEV;
		198
2332	Serge	199	mutex_lock(&dev->struct_mutex);
3031	serge	200	i915_gem_init_global_gtt(dev, args->gtt_start,
		201	args->gtt_end, args->gtt_end);
2332	Serge	202	mutex_unlock(&dev->struct_mutex);
		203
		204	return 0;
		205	}
2351	Serge	206	#endif
2332	Serge	207
		208	int
		209	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		210	struct drm_file *file)
		211	{
		212	struct drm_i915_private *dev_priv = dev->dev_private;
		213	struct drm_i915_gem_get_aperture *args = data;
		214	struct drm_i915_gem_object *obj;
		215	size_t pinned;
		216
		217	pinned = 0;
		218	mutex_lock(&dev->struct_mutex);
3031	serge	219	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		220	if (obj->pin_count)
2332	Serge	221	pinned += obj->gtt_space->size;
		222	mutex_unlock(&dev->struct_mutex);
		223
		224	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	225	args->aper_available_size = args->aper_size - pinned;
2332	Serge	226
		227	return 0;
		228	}
		229
3031	serge	230	static int
		231	i915_gem_create(struct drm_file *file,
2332	Serge	232	struct drm_device *dev,
		233	uint64_t size,
		234	uint32_t *handle_p)
		235	{
		236	struct drm_i915_gem_object *obj;
		237	int ret;
		238	u32 handle;
		239
3290	Serge	240	ENTER();
		241
2332	Serge	242	size = roundup(size, PAGE_SIZE);
2342	Serge	243	if (size == 0)
		244	return -EINVAL;
2332	Serge	245
		246	/* Allocate the new object */
		247	obj = i915_gem_alloc_object(dev, size);
		248	if (obj == NULL)
		249	return -ENOMEM;
		250
		251	ret = drm_gem_handle_create(file, &obj->base, &handle);
		252	if (ret) {
		253	drm_gem_object_release(&obj->base);
		254	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		255	kfree(obj);
		256	return ret;
		257	}
		258
		259	/* drop reference from allocate - handle holds it now */
		260	drm_gem_object_unreference(&obj->base);
2351	Serge	261	trace_i915_gem_object_create(obj);
2332	Serge	262
		263	*handle_p = handle;
3290	Serge	264
		265	printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle);
		266
2332	Serge	267	return 0;
		268	}
		269
		270	int
		271	i915_gem_dumb_create(struct drm_file *file,
		272	struct drm_device *dev,
		273	struct drm_mode_create_dumb *args)
		274	{
		275	/* have to work out size/pitch and return them */
		276	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		277	args->size = args->pitch * args->height;
		278	return i915_gem_create(file, dev,
		279	args->size, &args->handle);
		280	}
		281
		282	int i915_gem_dumb_destroy(struct drm_file *file,
		283	struct drm_device *dev,
		284	uint32_t handle)
		285	{
		286	return drm_gem_handle_delete(file, handle);
		287	}
		288
2326	Serge	289	/**
2332	Serge	290	* Creates a new mm object and returns a handle to it.
		291	*/
		292	int
		293	i915_gem_create_ioctl(struct drm_device dev, void data,
		294	struct drm_file *file)
		295	{
		296	struct drm_i915_gem_create *args = data;
3031	serge	297
2332	Serge	298	return i915_gem_create(file, dev,
		299	args->size, &args->handle);
		300	}
		301
		302	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		303	{
		304	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		305
		306	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		307	obj->tiling_mode != I915_TILING_NONE;
		308	}
3260	Serge	309	#if 0
2332	Serge	310
3031	serge	311	static inline int
		312	__copy_to_user_swizzled(char __user *cpu_vaddr,
		313	const char *gpu_vaddr, int gpu_offset,
2332	Serge	314	int length)
		315	{
3031	serge	316	int ret, cpu_offset = 0;
2332	Serge	317
3031	serge	318	while (length > 0) {
		319	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		320	int this_length = min(cacheline_end - gpu_offset, length);
		321	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	322
3031	serge	323	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		324	gpu_vaddr + swizzled_gpu_offset,
		325	this_length);
		326	if (ret)
		327	return ret + length;
2332	Serge	328
3031	serge	329	cpu_offset += this_length;
		330	gpu_offset += this_length;
		331	length -= this_length;
		332	}
		333
		334	return 0;
2332	Serge	335	}
		336
3031	serge	337	static inline int
		338	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		339	const char __user *cpu_vaddr,
		340	int length)
2332	Serge	341	{
3031	serge	342	int ret, cpu_offset = 0;
2332	Serge	343
		344	while (length > 0) {
		345	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		346	int this_length = min(cacheline_end - gpu_offset, length);
		347	int swizzled_gpu_offset = gpu_offset ^ 64;
		348
3031	serge	349	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	350	cpu_vaddr + cpu_offset,
		351	this_length);
3031	serge	352	if (ret)
		353	return ret + length;
		354
2332	Serge	355	cpu_offset += this_length;
		356	gpu_offset += this_length;
		357	length -= this_length;
		358	}
		359
3031	serge	360	return 0;
2332	Serge	361	}
		362
3031	serge	363	/* Per-page copy function for the shmem pread fastpath.
		364	* Flushes invalid cachelines before reading the target if
		365	* needs_clflush is set. */
2332	Serge	366	static int
3031	serge	367	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		368	char __user *user_data,
		369	bool page_do_bit17_swizzling, bool needs_clflush)
		370	{
		371	char *vaddr;
		372	int ret;
		373
		374	if (unlikely(page_do_bit17_swizzling))
		375	return -EINVAL;
		376
		377	vaddr = kmap_atomic(page);
		378	if (needs_clflush)
		379	drm_clflush_virt_range(vaddr + shmem_page_offset,
		380	page_length);
		381	ret = __copy_to_user_inatomic(user_data,
		382	vaddr + shmem_page_offset,
		383	page_length);
		384	kunmap_atomic(vaddr);
		385
		386	return ret ? -EFAULT : 0;
		387	}
		388
		389	static void
		390	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		391	bool swizzled)
		392	{
		393	if (unlikely(swizzled)) {
		394	unsigned long start = (unsigned long) addr;
		395	unsigned long end = (unsigned long) addr + length;
		396
		397	/* For swizzling simply ensure that we always flush both
		398	* channels. Lame, but simple and it works. Swizzled
		399	* pwrite/pread is far from a hotpath - current userspace
		400	* doesn't use it at all. */
		401	start = round_down(start, 128);
		402	end = round_up(end, 128);
		403
		404	drm_clflush_virt_range((void *)start, end - start);
		405	} else {
		406	drm_clflush_virt_range(addr, length);
		407	}
		408
		409	}
		410
		411	/* Only difference to the fast-path function is that this can handle bit17
		412	* and uses non-atomic copy and kmap functions. */
		413	static int
		414	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		415	char __user *user_data,
		416	bool page_do_bit17_swizzling, bool needs_clflush)
		417	{
		418	char *vaddr;
		419	int ret;
		420
		421	vaddr = kmap(page);
		422	if (needs_clflush)
		423	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		424	page_length,
		425	page_do_bit17_swizzling);
		426
		427	if (page_do_bit17_swizzling)
		428	ret = __copy_to_user_swizzled(user_data,
		429	vaddr, shmem_page_offset,
		430	page_length);
		431	else
		432	ret = __copy_to_user(user_data,
		433	vaddr + shmem_page_offset,
		434	page_length);
		435	kunmap(page);
		436
		437	return ret ? - EFAULT : 0;
		438	}
		439
		440	static int
		441	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	442	struct drm_i915_gem_object *obj,
		443	struct drm_i915_gem_pread *args,
		444	struct drm_file *file)
		445	{
3031	serge	446	char __user *user_data;
2332	Serge	447	ssize_t remain;
		448	loff_t offset;
3031	serge	449	int shmem_page_offset, page_length, ret = 0;
		450	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		451	int hit_slowpath = 0;
		452	int prefaulted = 0;
		453	int needs_clflush = 0;
		454	struct scatterlist *sg;
		455	int i;
2332	Serge	456
		457	user_data = (char __user *) (uintptr_t) args->data_ptr;
		458	remain = args->size;
		459
3031	serge	460	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		461
		462	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		463	/* If we're not in the cpu read domain, set ourself into the gtt
		464	* read domain and manually flush cachelines (if required). This
		465	* optimizes for the case when the gpu will dirty the data
		466	* anyway again before the next pread happens. */
		467	if (obj->cache_level == I915_CACHE_NONE)
		468	needs_clflush = 1;
		469	if (obj->gtt_space) {
		470	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		471	if (ret)
		472	return ret;
		473	}
		474	}
		475
		476	ret = i915_gem_object_get_pages(obj);
		477	if (ret)
		478	return ret;
		479
		480	i915_gem_object_pin_pages(obj);
		481
2332	Serge	482	offset = args->offset;
		483
3031	serge	484	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	485	struct page *page;
		486
3031	serge	487	if (i < offset >> PAGE_SHIFT)
		488	continue;
		489
		490	if (remain <= 0)
		491	break;
		492
2332	Serge	493	/* Operation in this page
		494	*
3031	serge	495	* shmem_page_offset = offset within page in shmem file
2332	Serge	496	* page_length = bytes to copy for this page
		497	*/
3031	serge	498	shmem_page_offset = offset_in_page(offset);
2332	Serge	499	page_length = remain;
3031	serge	500	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		501	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	502
3031	serge	503	page = sg_page(sg);
		504	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		505	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	506
3031	serge	507	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		508	user_data, page_do_bit17_swizzling,
		509	needs_clflush);
		510	if (ret == 0)
		511	goto next_page;
2332	Serge	512
3031	serge	513	hit_slowpath = 1;
		514	mutex_unlock(&dev->struct_mutex);
		515
		516	if (!prefaulted) {
		517	ret = fault_in_multipages_writeable(user_data, remain);
		518	/* Userspace is tricking us, but we've already clobbered
		519	* its pages with the prefault and promised to write the
		520	* data up to the first fault. Hence ignore any errors
		521	* and just continue. */
		522	(void)ret;
		523	prefaulted = 1;
		524	}
		525
		526	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		527	user_data, page_do_bit17_swizzling,
		528	needs_clflush);
		529
		530	mutex_lock(&dev->struct_mutex);
		531
		532	next_page:
2332	Serge	533	mark_page_accessed(page);
3031	serge	534
2332	Serge	535	if (ret)
3031	serge	536	goto out;
2332	Serge	537
		538	remain -= page_length;
		539	user_data += page_length;
		540	offset += page_length;
		541	}
		542
3031	serge	543	out:
		544	i915_gem_object_unpin_pages(obj);
		545
		546	if (hit_slowpath) {
		547	/* Fixup: Kill any reinstated backing storage pages */
		548	if (obj->madv == __I915_MADV_PURGED)
		549	i915_gem_object_truncate(obj);
		550	}
		551
		552	return ret;
2332	Serge	553	}
		554
		555	/**
3031	serge	556	* Reads data from the object referenced by handle.
		557	*
		558	* On error, the contents of *data are undefined.
2332	Serge	559	*/
3031	serge	560	int
		561	i915_gem_pread_ioctl(struct drm_device dev, void data,
		562	struct drm_file *file)
		563	{
		564	struct drm_i915_gem_pread *args = data;
		565	struct drm_i915_gem_object *obj;
		566	int ret = 0;
		567
		568	if (args->size == 0)
		569	return 0;
		570
		571	if (!access_ok(VERIFY_WRITE,
		572	(char __user *)(uintptr_t)args->data_ptr,
		573	args->size))
		574	return -EFAULT;
		575
		576	ret = i915_mutex_lock_interruptible(dev);
		577	if (ret)
		578	return ret;
		579
		580	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		581	if (&obj->base == NULL) {
		582	ret = -ENOENT;
		583	goto unlock;
		584	}
		585
		586	/* Bounds check source. */
		587	if (args->offset > obj->base.size \|\|
		588	args->size > obj->base.size - args->offset) {
		589	ret = -EINVAL;
		590	goto out;
		591	}
		592
		593	/* prime objects have no backing filp to GEM pread/pwrite
		594	* pages from.
		595	*/
		596	if (!obj->base.filp) {
		597	ret = -EINVAL;
		598	goto out;
		599	}
		600
		601	trace_i915_gem_object_pread(obj, args->offset, args->size);
		602
		603	ret = i915_gem_shmem_pread(dev, obj, args, file);
		604
		605	out:
		606	drm_gem_object_unreference(&obj->base);
		607	unlock:
		608	mutex_unlock(&dev->struct_mutex);
		609	return ret;
		610	}
		611
		612	/* This is the fast write path which cannot handle
		613	* page faults in the source data
		614	*/
		615
		616	static inline int
		617	fast_user_write(struct io_mapping *mapping,
		618	loff_t page_base, int page_offset,
		619	char __user *user_data,
		620	int length)
		621	{
		622	void __iomem *vaddr_atomic;
		623	void *vaddr;
		624	unsigned long unwritten;
		625
		626	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		627	/* We can use the cpu mem copy function because this is X86. */
		628	vaddr = (void __force*)vaddr_atomic + page_offset;
		629	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		630	user_data, length);
		631	io_mapping_unmap_atomic(vaddr_atomic);
		632	return unwritten;
		633	}
3260	Serge	634	#endif
3031	serge	635
3260	Serge	636	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	637	/**
		638	* This is the fast pwrite path, where we copy the data directly from the
		639	* user into the GTT, uncached.
		640	*/
2332	Serge	641	static int
3031	serge	642	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		643	struct drm_i915_gem_object *obj,
		644	struct drm_i915_gem_pwrite *args,
		645	struct drm_file *file)
2332	Serge	646	{
3031	serge	647	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	648	ssize_t remain;
3031	serge	649	loff_t offset, page_base;
		650	char __user *user_data;
		651	int page_offset, page_length, ret;
3260	Serge	652	char *vaddr;
2332	Serge	653
3031	serge	654	ret = i915_gem_object_pin(obj, 0, true, true);
		655	if (ret)
		656	goto out;
		657
		658	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		659	if (ret)
		660	goto out_unpin;
		661
		662	ret = i915_gem_object_put_fence(obj);
		663	if (ret)
		664	goto out_unpin;
		665
3260	Serge	666	vaddr = AllocKernelSpace(4096);
		667	if(vaddr == NULL)
		668	{
		669	ret = -ENOSPC;
		670	goto out_unpin;
		671	};
		672
3031	serge	673	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	674	remain = args->size;
		675
3031	serge	676	offset = obj->gtt_offset + args->offset;
2332	Serge	677
3031	serge	678	while (remain > 0) {
		679	/* Operation in this page
		680	*
		681	* page_base = page offset within aperture
		682	* page_offset = offset within page
		683	* page_length = bytes to copy for this page
		684	*/
		685	page_base = offset & PAGE_MASK;
		686	page_offset = offset_in_page(offset);
		687	page_length = remain;
		688	if ((page_offset + remain) > PAGE_SIZE)
		689	page_length = PAGE_SIZE - page_offset;
2332	Serge	690
3260	Serge	691	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	692
3260	Serge	693	memcpy(vaddr+page_offset, user_data, page_length);
		694
3031	serge	695	remain -= page_length;
		696	user_data += page_length;
		697	offset += page_length;
2332	Serge	698	}
		699
3260	Serge	700	FreeKernelSpace(vaddr);
		701
3031	serge	702	out_unpin:
		703	i915_gem_object_unpin(obj);
		704	out:
3260	Serge	705	printf("% s ret = %d\n", __FUNCTION__, ret);
		706
3031	serge	707	return ret;
		708	}
		709
		710	/* Per-page copy function for the shmem pwrite fastpath.
		711	* Flushes invalid cachelines before writing to the target if
		712	* needs_clflush_before is set and flushes out any written cachelines after
		713	* writing if needs_clflush is set. */
		714	static int
		715	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		716	char __user *user_data,
		717	bool page_do_bit17_swizzling,
		718	bool needs_clflush_before,
		719	bool needs_clflush_after)
		720	{
		721	char *vaddr;
3260	Serge	722	int ret = 0;
3031	serge	723
		724	if (unlikely(page_do_bit17_swizzling))
		725	return -EINVAL;
		726
3260	Serge	727	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	728	if (needs_clflush_before)
		729	drm_clflush_virt_range(vaddr + shmem_page_offset,
		730	page_length);
3260	Serge	731	memcpy(vaddr + shmem_page_offset,
3031	serge	732	user_data,
		733	page_length);
		734	if (needs_clflush_after)
		735	drm_clflush_virt_range(vaddr + shmem_page_offset,
		736	page_length);
3260	Serge	737	FreeKernelSpace(vaddr);
3031	serge	738
		739	return ret ? -EFAULT : 0;
		740	}
3260	Serge	741	#if 0
3031	serge	742
		743	/* Only difference to the fast-path function is that this can handle bit17
		744	* and uses non-atomic copy and kmap functions. */
		745	static int
		746	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		747	char __user *user_data,
		748	bool page_do_bit17_swizzling,
		749	bool needs_clflush_before,
		750	bool needs_clflush_after)
		751	{
		752	char *vaddr;
		753	int ret;
		754
		755	vaddr = kmap(page);
		756	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		757	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		758	page_length,
		759	page_do_bit17_swizzling);
		760	if (page_do_bit17_swizzling)
		761	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		762	user_data,
		763	page_length);
		764	else
		765	ret = __copy_from_user(vaddr + shmem_page_offset,
		766	user_data,
		767	page_length);
		768	if (needs_clflush_after)
		769	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		770	page_length,
		771	page_do_bit17_swizzling);
		772	kunmap(page);
		773
		774	return ret ? -EFAULT : 0;
		775	}
3260	Serge	776	#endif
3031	serge	777
3260	Serge	778
3031	serge	779	static int
		780	i915_gem_shmem_pwrite(struct drm_device *dev,
		781	struct drm_i915_gem_object *obj,
		782	struct drm_i915_gem_pwrite *args,
		783	struct drm_file *file)
		784	{
		785	ssize_t remain;
		786	loff_t offset;
		787	char __user *user_data;
		788	int shmem_page_offset, page_length, ret = 0;
		789	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		790	int hit_slowpath = 0;
		791	int needs_clflush_after = 0;
		792	int needs_clflush_before = 0;
		793	int i;
		794	struct scatterlist *sg;
		795
		796	user_data = (char __user *) (uintptr_t) args->data_ptr;
		797	remain = args->size;
		798
		799	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		800
		801	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		802	/* If we're not in the cpu write domain, set ourself into the gtt
		803	* write domain and manually flush cachelines (if required). This
		804	* optimizes for the case when the gpu will use the data
		805	* right away and we therefore have to clflush anyway. */
		806	if (obj->cache_level == I915_CACHE_NONE)
		807	needs_clflush_after = 1;
		808	if (obj->gtt_space) {
		809	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		810	if (ret)
		811	return ret;
		812	}
		813	}
		814	/* Same trick applies for invalidate partially written cachelines before
		815	* writing. */
		816	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		817	&& obj->cache_level == I915_CACHE_NONE)
		818	needs_clflush_before = 1;
		819
		820	ret = i915_gem_object_get_pages(obj);
2332	Serge	821	if (ret)
3031	serge	822	return ret;
2332	Serge	823
3031	serge	824	i915_gem_object_pin_pages(obj);
2332	Serge	825
		826	offset = args->offset;
3031	serge	827	obj->dirty = 1;
2332	Serge	828
3031	serge	829	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	830	struct page *page;
3031	serge	831	int partial_cacheline_write;
2332	Serge	832
3031	serge	833	if (i < offset >> PAGE_SHIFT)
		834	continue;
		835
		836	if (remain <= 0)
		837	break;
		838
2332	Serge	839	/* Operation in this page
		840	*
		841	* shmem_page_offset = offset within page in shmem file
		842	* page_length = bytes to copy for this page
		843	*/
		844	shmem_page_offset = offset_in_page(offset);
		845
		846	page_length = remain;
		847	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		848	page_length = PAGE_SIZE - shmem_page_offset;
		849
3031	serge	850	/* If we don't overwrite a cacheline completely we need to be
		851	* careful to have up-to-date data by first clflushing. Don't
		852	* overcomplicate things and flush the entire patch. */
		853	partial_cacheline_write = needs_clflush_before &&
		854	((shmem_page_offset \| page_length)
3260	Serge	855	& (x86_clflush_size - 1));
2332	Serge	856
3031	serge	857	page = sg_page(sg);
		858	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		859	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	860
3031	serge	861	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		862	user_data, page_do_bit17_swizzling,
		863	partial_cacheline_write,
		864	needs_clflush_after);
		865	if (ret == 0)
		866	goto next_page;
		867
		868	hit_slowpath = 1;
		869	mutex_unlock(&dev->struct_mutex);
3260	Serge	870	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	871
3260	Serge	872	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		873	// user_data, page_do_bit17_swizzling,
		874	// partial_cacheline_write,
		875	// needs_clflush_after);
		876
3031	serge	877	mutex_lock(&dev->struct_mutex);
		878
		879	next_page:
2332	Serge	880
3031	serge	881	if (ret)
		882	goto out;
		883
2332	Serge	884	remain -= page_length;
3031	serge	885	user_data += page_length;
2332	Serge	886	offset += page_length;
		887	}
		888
		889	out:
3031	serge	890	i915_gem_object_unpin_pages(obj);
		891
		892	if (hit_slowpath) {
		893	/* Fixup: Kill any reinstated backing storage pages */
		894	if (obj->madv == __I915_MADV_PURGED)
		895	i915_gem_object_truncate(obj);
		896	/* and flush dirty cachelines in case the object isn't in the cpu write
		897	* domain anymore. */
		898	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		899	i915_gem_clflush_object(obj);
3243	Serge	900	i915_gem_chipset_flush(dev);
3031	serge	901	}
2332	Serge	902	}
		903
3031	serge	904	if (needs_clflush_after)
3243	Serge	905	i915_gem_chipset_flush(dev);
3031	serge	906
2332	Serge	907	return ret;
		908	}
3031	serge	909
		910	/**
		911	* Writes data to the object referenced by handle.
		912	*
		913	* On error, the contents of the buffer that were to be modified are undefined.
		914	*/
		915	int
		916	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		917	struct drm_file *file)
		918	{
		919	struct drm_i915_gem_pwrite *args = data;
		920	struct drm_i915_gem_object *obj;
		921	int ret;
		922
		923	if (args->size == 0)
		924	return 0;
		925
		926	ret = i915_mutex_lock_interruptible(dev);
		927	if (ret)
		928	return ret;
		929
		930	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		931	if (&obj->base == NULL) {
		932	ret = -ENOENT;
		933	goto unlock;
		934	}
		935
		936	/* Bounds check destination. */
		937	if (args->offset > obj->base.size \|\|
		938	args->size > obj->base.size - args->offset) {
		939	ret = -EINVAL;
		940	goto out;
		941	}
		942
		943	/* prime objects have no backing filp to GEM pread/pwrite
		944	* pages from.
		945	*/
		946	if (!obj->base.filp) {
		947	ret = -EINVAL;
		948	goto out;
		949	}
		950
		951	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		952
		953	ret = -EFAULT;
		954	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		955	* it would end up going through the fenced access, and we'll get
		956	* different detiling behavior between reading and writing.
		957	* pread/pwrite currently are reading and writing from the CPU
		958	* perspective, requiring manual detiling by the client.
		959	*/
3260	Serge	960	// if (obj->phys_obj) {
		961	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		962	// goto out;
		963	// }
3031	serge	964
		965	if (obj->cache_level == I915_CACHE_NONE &&
		966	obj->tiling_mode == I915_TILING_NONE &&
		967	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		968	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		969	/* Note that the gtt paths might fail with non-page-backed user
		970	* pointers (e.g. gtt mappings when moving data between
		971	* textures). Fallback to the shmem path in that case. */
		972	}
		973
		974	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	975	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	976
		977	out:
		978	drm_gem_object_unreference(&obj->base);
		979	unlock:
		980	mutex_unlock(&dev->struct_mutex);
		981	return ret;
		982	}
		983
		984	int
		985	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		986	bool interruptible)
		987	{
		988	if (atomic_read(&dev_priv->mm.wedged)) {
		989	struct completion *x = &dev_priv->error_completion;
		990	bool recovery_complete;
		991	unsigned long flags;
2332	Serge	992
3031	serge	993	/* Give the error handler a chance to run. */
		994	spin_lock_irqsave(&x->wait.lock, flags);
		995	recovery_complete = x->done > 0;
		996	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	997
3031	serge	998	/* Non-interruptible callers can't handle -EAGAIN, hence return
		999	* -EIO unconditionally for these. */
		1000	if (!interruptible)
		1001	return -EIO;
2332	Serge	1002
3031	serge	1003	/* Recovery complete, but still wedged means reset failure. */
		1004	if (recovery_complete)
		1005	return -EIO;
2332	Serge	1006
3031	serge	1007	return -EAGAIN;
		1008	}
2332	Serge	1009
3031	serge	1010	return 0;
		1011	}
2332	Serge	1012
3031	serge	1013	/*
		1014	* Compare seqno against outstanding lazy request. Emit a request if they are
		1015	* equal.
		1016	*/
		1017	static int
		1018	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1019	{
		1020	int ret;
2332	Serge	1021
3031	serge	1022	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1023
3031	serge	1024	ret = 0;
		1025	if (seqno == ring->outstanding_lazy_request)
		1026	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1027
3031	serge	1028	return ret;
		1029	}
2332	Serge	1030
3031	serge	1031	/**
		1032	* __wait_seqno - wait until execution of seqno has finished
		1033	* @ring: the ring expected to report seqno
		1034	* @seqno: duh!
		1035	* @interruptible: do an interruptible wait (normally yes)
		1036	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1037	*
		1038	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1039	* errno with remaining time filled in timeout argument.
		1040	*/
		1041	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1042	bool interruptible, struct timespec *timeout)
		1043	{
		1044	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1045	struct timespec before, now, wait_time={1,0};
		1046	unsigned long timeout_jiffies;
		1047	long end;
		1048	bool wait_forever = true;
		1049	int ret;
2332	Serge	1050
3031	serge	1051	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1052	return 0;
2332	Serge	1053
3031	serge	1054	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1055
3031	serge	1056	if (timeout != NULL) {
		1057	wait_time = *timeout;
		1058	wait_forever = false;
		1059	}
2332	Serge	1060
3266	Serge	1061	timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1062
3031	serge	1063	if (WARN_ON(!ring->irq_get(ring)))
		1064	return -ENODEV;
2332	Serge	1065
3031	serge	1066	/* Record current time in case interrupted by signal, or wedged * */
		1067	getrawmonotonic(&before);
2332	Serge	1068
3031	serge	1069	#define EXIT_COND \
		1070	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1071	atomic_read(&dev_priv->mm.wedged))
		1072	do {
3266	Serge	1073	if (interruptible)
		1074	end = wait_event_interruptible_timeout(ring->irq_queue,
		1075	EXIT_COND,
		1076	timeout_jiffies);
		1077	else
3031	serge	1078	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1079	timeout_jiffies);
2332	Serge	1080
3031	serge	1081	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1082	if (ret)
		1083	end = ret;
		1084	} while (end == 0 && wait_forever);
2332	Serge	1085
3031	serge	1086	getrawmonotonic(&now);
2332	Serge	1087
3031	serge	1088	ring->irq_put(ring);
		1089	trace_i915_gem_request_wait_end(ring, seqno);
		1090	#undef EXIT_COND
2332	Serge	1091
3031	serge	1092	if (timeout) {
		1093	// struct timespec sleep_time = timespec_sub(now, before);
		1094	// timeout = timespec_sub(timeout, sleep_time);
		1095	}
2332	Serge	1096
3031	serge	1097	switch (end) {
		1098	case -EIO:
		1099	case -EAGAIN: /* Wedged */
		1100	case -ERESTARTSYS: /* Signal */
		1101	return (int)end;
		1102	case 0: /* Timeout */
3266	Serge	1103	if (timeout)
		1104	set_normalized_timespec(timeout, 0, 0);
3031	serge	1105	return -ETIME;
		1106	default: /* Completed */
		1107	WARN_ON(end < 0); /* We're not aware of other errors */
		1108	return 0;
		1109	}
		1110	}
2332	Serge	1111
3031	serge	1112	/**
		1113	* Waits for a sequence number to be signaled, and cleans up the
		1114	* request and object lists appropriately for that event.
		1115	*/
		1116	int
		1117	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1118	{
		1119	struct drm_device *dev = ring->dev;
		1120	struct drm_i915_private *dev_priv = dev->dev_private;
		1121	bool interruptible = dev_priv->mm.interruptible;
		1122	int ret;
2332	Serge	1123
3031	serge	1124	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1125	BUG_ON(seqno == 0);
2332	Serge	1126
3031	serge	1127	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1128	if (ret)
		1129	return ret;
2332	Serge	1130
3031	serge	1131	ret = i915_gem_check_olr(ring, seqno);
		1132	if (ret)
		1133	return ret;
2332	Serge	1134
3031	serge	1135	return __wait_seqno(ring, seqno, interruptible, NULL);
		1136	}
2332	Serge	1137
3031	serge	1138	/**
		1139	* Ensures that all rendering to the object has completed and the object is
		1140	* safe to unbind from the GTT or access from the CPU.
		1141	*/
		1142	static __must_check int
		1143	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1144	bool readonly)
		1145	{
		1146	struct intel_ring_buffer *ring = obj->ring;
		1147	u32 seqno;
		1148	int ret;
2332	Serge	1149
3031	serge	1150	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1151	if (seqno == 0)
		1152	return 0;
2332	Serge	1153
3031	serge	1154	ret = i915_wait_seqno(ring, seqno);
		1155	if (ret)
		1156	return ret;
2332	Serge	1157
3031	serge	1158	i915_gem_retire_requests_ring(ring);
2332	Serge	1159
3031	serge	1160	/* Manually manage the write flush as we may have not yet
		1161	* retired the buffer.
		1162	*/
		1163	if (obj->last_write_seqno &&
		1164	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1165	obj->last_write_seqno = 0;
		1166	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1167	}
2332	Serge	1168
3031	serge	1169	return 0;
		1170	}
2332	Serge	1171
3260	Serge	1172	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1173	* as the object state may change during this call.
		1174	*/
		1175	static __must_check int
		1176	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1177	bool readonly)
		1178	{
		1179	struct drm_device *dev = obj->base.dev;
		1180	struct drm_i915_private *dev_priv = dev->dev_private;
		1181	struct intel_ring_buffer *ring = obj->ring;
		1182	u32 seqno;
		1183	int ret;
2332	Serge	1184
3260	Serge	1185	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1186	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1187
3260	Serge	1188	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1189	if (seqno == 0)
		1190	return 0;
2332	Serge	1191
3260	Serge	1192	ret = i915_gem_check_wedge(dev_priv, true);
		1193	if (ret)
		1194	return ret;
2332	Serge	1195
3260	Serge	1196	ret = i915_gem_check_olr(ring, seqno);
		1197	if (ret)
		1198	return ret;
2332	Serge	1199
3260	Serge	1200	mutex_unlock(&dev->struct_mutex);
		1201	ret = __wait_seqno(ring, seqno, true, NULL);
		1202	mutex_lock(&dev->struct_mutex);
2332	Serge	1203
3260	Serge	1204	i915_gem_retire_requests_ring(ring);
2332	Serge	1205
3260	Serge	1206	/* Manually manage the write flush as we may have not yet
		1207	* retired the buffer.
		1208	*/
		1209	if (obj->last_write_seqno &&
		1210	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1211	obj->last_write_seqno = 0;
		1212	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1213	}
2332	Serge	1214
3260	Serge	1215	return ret;
		1216	}
2332	Serge	1217
3260	Serge	1218	/**
		1219	* Called when user space prepares to use an object with the CPU, either
		1220	* through the mmap ioctl's mapping or a GTT mapping.
		1221	*/
		1222	int
		1223	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1224	struct drm_file *file)
		1225	{
		1226	struct drm_i915_gem_set_domain *args = data;
		1227	struct drm_i915_gem_object *obj;
		1228	uint32_t read_domains = args->read_domains;
		1229	uint32_t write_domain = args->write_domain;
		1230	int ret;
2332	Serge	1231
3260	Serge	1232	/* Only handle setting domains to types used by the CPU. */
		1233	if (write_domain & I915_GEM_GPU_DOMAINS)
		1234	return -EINVAL;
2332	Serge	1235
3260	Serge	1236	if (read_domains & I915_GEM_GPU_DOMAINS)
		1237	return -EINVAL;
2332	Serge	1238
3260	Serge	1239	/* Having something in the write domain implies it's in the read
		1240	* domain, and only that read domain. Enforce that in the request.
		1241	*/
		1242	if (write_domain != 0 && read_domains != write_domain)
		1243	return -EINVAL;
2332	Serge	1244
3260	Serge	1245	ret = i915_mutex_lock_interruptible(dev);
		1246	if (ret)
		1247	return ret;
2332	Serge	1248
3260	Serge	1249	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1250	if (&obj->base == NULL) {
		1251	ret = -ENOENT;
		1252	goto unlock;
		1253	}
2332	Serge	1254
3260	Serge	1255	/* Try to flush the object off the GPU without holding the lock.
		1256	* We will repeat the flush holding the lock in the normal manner
		1257	* to catch cases where we are gazumped.
		1258	*/
		1259	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1260	if (ret)
		1261	goto unref;
2332	Serge	1262
3260	Serge	1263	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1264	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1265
3260	Serge	1266	/* Silently promote "you're not bound, there was nothing to do"
		1267	* to success, since the client was just asking us to
		1268	* make sure everything was done.
		1269	*/
		1270	if (ret == -EINVAL)
		1271	ret = 0;
		1272	} else {
		1273	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1274	}
2332	Serge	1275
3260	Serge	1276	unref:
		1277	drm_gem_object_unreference(&obj->base);
		1278	unlock:
		1279	mutex_unlock(&dev->struct_mutex);
		1280	return ret;
		1281	}
2332	Serge	1282
		1283
		1284
		1285
		1286
		1287
3260	Serge	1288	/**
		1289	* Maps the contents of an object, returning the address it is mapped
		1290	* into.
		1291	*
		1292	* While the mapping holds a reference on the contents of the object, it doesn't
		1293	* imply a ref on the object itself.
		1294	*/
		1295	int
		1296	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1297	struct drm_file *file)
		1298	{
		1299	struct drm_i915_gem_mmap *args = data;
		1300	struct drm_gem_object *obj;
		1301	unsigned long addr = 0;
2332	Serge	1302
3260	Serge	1303	obj = drm_gem_object_lookup(dev, file, args->handle);
		1304	if (obj == NULL)
		1305	return -ENOENT;
		1306
3263	Serge	1307	dbgprintf("%s offset %lld size %lld\n",
		1308	__FUNCTION__, args->offset, args->size);
3260	Serge	1309	/* prime objects have no backing filp to GEM mmap
		1310	* pages from.
		1311	*/
		1312	if (!obj->filp) {
		1313	drm_gem_object_unreference_unlocked(obj);
		1314	return -EINVAL;
		1315	}
2332	Serge	1316
3263	Serge	1317	addr = vm_mmap(obj->filp, 0, args->size,
		1318	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1319	args->offset);
3260	Serge	1320	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1321	if (IS_ERR((void *)addr))
		1322	return addr;
2332	Serge	1323
3260	Serge	1324	args->addr_ptr = (uint64_t) addr;
2332	Serge	1325
3263	Serge	1326	return 0;
3260	Serge	1327	}
2332	Serge	1328
		1329
		1330
		1331
		1332
		1333
		1334
		1335
3031	serge	1336
		1337
		1338
		1339
		1340
		1341	/**
		1342	* i915_gem_release_mmap - remove physical page mappings
		1343	* @obj: obj in question
		1344	*
		1345	* Preserve the reservation of the mmapping with the DRM core code, but
		1346	* relinquish ownership of the pages back to the system.
		1347	*
		1348	* It is vital that we remove the page mapping if we have mapped a tiled
		1349	* object through the GTT and then lose the fence register due to
		1350	* resource pressure. Similarly if the object has been moved out of the
		1351	* aperture, than pages mapped into userspace must be revoked. Removing the
		1352	* mapping will then trigger a page fault on the next user access, allowing
		1353	* fixup by i915_gem_fault().
		1354	*/
		1355	void
		1356	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1357	{
		1358	if (!obj->fault_mappable)
		1359	return;
		1360
		1361	if (obj->base.dev->dev_mapping)
		1362	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1363	// (loff_t)obj->base.map_list.hash.key<
		1364	// obj->base.size, 1);
		1365
		1366	obj->fault_mappable = false;
		1367	}
		1368
2332	Serge	1369	static uint32_t
		1370	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1371	{
		1372	uint32_t gtt_size;
		1373
		1374	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1375	tiling_mode == I915_TILING_NONE)
		1376	return size;
		1377
		1378	/* Previous chips need a power-of-two fence region when tiling */
		1379	if (INTEL_INFO(dev)->gen == 3)
		1380	gtt_size = 1024*1024;
		1381	else
		1382	gtt_size = 512*1024;
		1383
		1384	while (gtt_size < size)
		1385	gtt_size <<= 1;
		1386
		1387	return gtt_size;
		1388	}
		1389
		1390	/**
		1391	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1392	* @obj: object to check
		1393	*
		1394	* Return the required GTT alignment for an object, taking into account
		1395	* potential fence register mapping.
		1396	*/
		1397	static uint32_t
		1398	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1399	uint32_t size,
		1400	int tiling_mode)
		1401	{
		1402	/*
		1403	* Minimum alignment is 4k (GTT page size), but might be greater
		1404	* if a fence register is needed for the object.
		1405	*/
		1406	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1407	tiling_mode == I915_TILING_NONE)
		1408	return 4096;
		1409
		1410	/*
		1411	* Previous chips need to be aligned to the size of the smallest
		1412	* fence register that can contain the object.
		1413	*/
		1414	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1415	}
		1416
		1417	/**
		1418	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1419	* unfenced object
		1420	* @dev: the device
		1421	* @size: size of the object
		1422	* @tiling_mode: tiling mode of the object
		1423	*
		1424	* Return the required GTT alignment for an object, only taking into account
		1425	* unfenced tiled surface requirements.
		1426	*/
		1427	uint32_t
		1428	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1429	uint32_t size,
		1430	int tiling_mode)
		1431	{
		1432	/*
		1433	* Minimum alignment is 4k (GTT page size) for sane hw.
		1434	*/
		1435	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1436	tiling_mode == I915_TILING_NONE)
		1437	return 4096;
		1438
		1439	/* Previous hardware however needs to be aligned to a power-of-two
		1440	* tile height. The simplest method for determining this is to reuse
		1441	* the power-of-tile object size.
		1442	*/
		1443	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1444	}
		1445
3031	serge	1446	/* Immediately discard the backing storage */
		1447	static void
		1448	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1449	{
		1450	// struct inode *inode;
2332	Serge	1451
3031	serge	1452	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1453
3263	Serge	1454	if (obj->base.filp == NULL)
		1455	return;
2332	Serge	1456
3031	serge	1457	/* Our goal here is to return as much of the memory as
		1458	* is possible back to the system as we are called from OOM.
		1459	* To do this we must instruct the shmfs to drop all of its
		1460	* backing pages, now.
		1461	*/
		1462	// inode = obj->base.filp->f_path.dentry->d_inode;
		1463	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1464
3031	serge	1465	obj->madv = __I915_MADV_PURGED;
		1466	}
2332	Serge	1467
3031	serge	1468	static inline int
		1469	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1470	{
		1471	return obj->madv == I915_MADV_DONTNEED;
		1472	}
2332	Serge	1473
3031	serge	1474	static void
		1475	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1476	{
3243	Serge	1477	int page_count = obj->base.size / PAGE_SIZE;
		1478	struct scatterlist *sg;
3031	serge	1479	int ret, i;
2332	Serge	1480
3031	serge	1481	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1482
3031	serge	1483	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1484	if (ret) {
		1485	/* In the event of a disaster, abandon all caches and
		1486	* hope for the best.
		1487	*/
		1488	WARN_ON(ret != -EIO);
		1489	i915_gem_clflush_object(obj);
		1490	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1491	}
2332	Serge	1492
3031	serge	1493	if (obj->madv == I915_MADV_DONTNEED)
		1494	obj->dirty = 0;
2332	Serge	1495
3243	Serge	1496	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1497	struct page *page = sg_page(sg);
2332	Serge	1498
3290	Serge	1499	page_cache_release(page);
3243	Serge	1500	}
		1501	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1502	printf("%s release %d pages\n", __FUNCTION__, page_count);
		1503
3031	serge	1504	obj->dirty = 0;
3243	Serge	1505
		1506	sg_free_table(obj->pages);
		1507	kfree(obj->pages);
3031	serge	1508	}
2332	Serge	1509
3031	serge	1510	static int
		1511	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1512	{
		1513	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1514
3243	Serge	1515	if (obj->pages == NULL)
3031	serge	1516	return 0;
2332	Serge	1517
3031	serge	1518	BUG_ON(obj->gtt_space);
		1519
		1520	if (obj->pages_pin_count)
		1521	return -EBUSY;
		1522
3243	Serge	1523	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1524	* array, hence protect them from being reaped by removing them from gtt
		1525	* lists early. */
		1526	list_del(&obj->gtt_list);
		1527
3031	serge	1528	ops->put_pages(obj);
3243	Serge	1529	obj->pages = NULL;
3031	serge	1530
		1531	if (i915_gem_object_is_purgeable(obj))
		1532	i915_gem_object_truncate(obj);
		1533
		1534	return 0;
		1535	}
		1536
		1537
		1538
		1539
		1540
		1541
		1542
		1543
2332	Serge	1544	static int
3031	serge	1545	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1546	{
3260	Serge	1547	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1548	int page_count, i;
3260	Serge	1549	struct address_space *mapping;
3243	Serge	1550	struct sg_table *st;
		1551	struct scatterlist *sg;
		1552	struct page *page;
		1553	gfp_t gfp;
2332	Serge	1554
3243	Serge	1555	/* Assert that the object is not currently in any GPU domain. As it
		1556	* wasn't in the GTT, there shouldn't be any way it could have been in
		1557	* a GPU cache
2332	Serge	1558	*/
3243	Serge	1559	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1560	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1561
		1562	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1563	if (st == NULL)
		1564	return -ENOMEM;
		1565
2332	Serge	1566	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1567	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1568	sg_free_table(st);
		1569	kfree(st);
2332	Serge	1570	return -ENOMEM;
3243	Serge	1571	}
2332	Serge	1572
3243	Serge	1573	/* Get the list of pages out of our struct file. They'll be pinned
		1574	* at this point until we release them.
		1575	*
		1576	* Fail silently without starting the shrinker
		1577	*/
		1578	for_each_sg(st->sgl, sg, page_count, i) {
3260	Serge	1579	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1580	if (IS_ERR(page)) {
		1581	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1582	goto err_pages;
		1583
3260	Serge	1584	}
3243	Serge	1585	sg_set_page(sg, page, PAGE_SIZE, 0);
		1586	}
3031	serge	1587
3243	Serge	1588	obj->pages = st;
3031	serge	1589
3260	Serge	1590	DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1591
		1592	return 0;
		1593
		1594	err_pages:
3243	Serge	1595	for_each_sg(st->sgl, sg, i, page_count)
		1596	page_cache_release(sg_page(sg));
		1597	sg_free_table(st);
		1598	kfree(st);
		1599	return PTR_ERR(page);
2332	Serge	1600	}
		1601
3031	serge	1602	/* Ensure that the associated pages are gathered from the backing storage
		1603	* and pinned into our object. i915_gem_object_get_pages() may be called
		1604	* multiple times before they are released by a single call to
		1605	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1606	* either as a result of memory pressure (reaping pages under the shrinker)
		1607	* or as the object is itself released.
		1608	*/
		1609	int
		1610	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1611	{
3031	serge	1612	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1613	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1614	int ret;
2332	Serge	1615
3243	Serge	1616	if (obj->pages)
3031	serge	1617	return 0;
2332	Serge	1618
3031	serge	1619	BUG_ON(obj->pages_pin_count);
2332	Serge	1620
3031	serge	1621	ret = ops->get_pages(obj);
		1622	if (ret)
		1623	return ret;
2344	Serge	1624
3031	serge	1625	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1626	return 0;
2332	Serge	1627	}
		1628
		1629	void
		1630	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1631	struct intel_ring_buffer *ring)
2332	Serge	1632	{
		1633	struct drm_device *dev = obj->base.dev;
		1634	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1635	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1636
		1637	BUG_ON(ring == NULL);
		1638	obj->ring = ring;
		1639
		1640	/* Add a reference if we're newly entering the active list. */
		1641	if (!obj->active) {
2344	Serge	1642	drm_gem_object_reference(&obj->base);
2332	Serge	1643	obj->active = 1;
		1644	}
		1645
		1646	/* Move from whatever list we were on to the tail of execution. */
		1647	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1648	list_move_tail(&obj->ring_list, &ring->active_list);
		1649
3031	serge	1650	obj->last_read_seqno = seqno;
		1651
2332	Serge	1652	if (obj->fenced_gpu_access) {
3031	serge	1653	obj->last_fenced_seqno = seqno;
		1654
		1655	/* Bump MRU to take account of the delayed flush */
		1656	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1657	struct drm_i915_fence_reg *reg;
		1658
		1659	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1660	list_move_tail(®->lru_list,
		1661	&dev_priv->mm.fence_list);
		1662	}
2332	Serge	1663	}
		1664	}
		1665
2344	Serge	1666	static void
3031	serge	1667	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1668	{
		1669	struct drm_device *dev = obj->base.dev;
3031	serge	1670	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1671
3031	serge	1672	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1673	BUG_ON(!obj->active);
2332	Serge	1674
3031	serge	1675	if (obj->pin_count) /* are we a framebuffer? */
		1676	intel_mark_fb_idle(obj);
2344	Serge	1677
2352	Serge	1678	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1679
3031	serge	1680	list_del_init(&obj->ring_list);
2352	Serge	1681	obj->ring = NULL;
2344	Serge	1682
3031	serge	1683	obj->last_read_seqno = 0;
		1684	obj->last_write_seqno = 0;
		1685	obj->base.write_domain = 0;
		1686
		1687	obj->last_fenced_seqno = 0;
2352	Serge	1688	obj->fenced_gpu_access = false;
2344	Serge	1689
2352	Serge	1690	obj->active = 0;
		1691	drm_gem_object_unreference(&obj->base);
		1692
		1693	WARN_ON(i915_verify_lists(dev));
		1694	}
		1695
3243	Serge	1696	static int
		1697	i915_gem_handle_seqno_wrap(struct drm_device *dev)
2344	Serge	1698	{
3243	Serge	1699	struct drm_i915_private *dev_priv = dev->dev_private;
		1700	struct intel_ring_buffer *ring;
		1701	int ret, i, j;
2344	Serge	1702
3243	Serge	1703	/* The hardware uses various monotonic 32-bit counters, if we
		1704	* detect that they will wraparound we need to idle the GPU
		1705	* and reset those counters.
		1706	*/
		1707	ret = 0;
		1708	for_each_ring(ring, dev_priv, i) {
		1709	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1710	ret \|= ring->sync_seqno[j] != 0;
		1711	}
		1712	if (ret == 0)
		1713	return ret;
2344	Serge	1714
3243	Serge	1715	ret = i915_gpu_idle(dev);
		1716	if (ret)
		1717	return ret;
		1718
		1719	i915_gem_retire_requests(dev);
		1720	for_each_ring(ring, dev_priv, i) {
		1721	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1722	ring->sync_seqno[j] = 0;
		1723	}
		1724
		1725	return 0;
2344	Serge	1726	}
		1727
3243	Serge	1728	int
		1729	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1730	{
3243	Serge	1731	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1732
3243	Serge	1733	/* reserve 0 for non-seqno */
		1734	if (dev_priv->next_seqno == 0) {
		1735	int ret = i915_gem_handle_seqno_wrap(dev);
		1736	if (ret)
		1737	return ret;
		1738
		1739	dev_priv->next_seqno = 1;
		1740	}
		1741
		1742	*seqno = dev_priv->next_seqno++;
		1743	return 0;
2332	Serge	1744	}
		1745
2352	Serge	1746	int
		1747	i915_add_request(struct intel_ring_buffer *ring,
		1748	struct drm_file *file,
3031	serge	1749	u32 *out_seqno)
2352	Serge	1750	{
		1751	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1752	struct drm_i915_gem_request *request;
		1753	u32 request_ring_position;
2352	Serge	1754	int was_empty;
		1755	int ret;
2332	Serge	1756
3031	serge	1757	/*
		1758	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1759	* after having emitted the batchbuffer command. Hence we need to fix
		1760	* things up similar to emitting the lazy request. The difference here
		1761	* is that the flush _must_ happen before the next request, no matter
		1762	* what.
		1763	*/
		1764	ret = intel_ring_flush_all_caches(ring);
		1765	if (ret)
		1766	return ret;
2332	Serge	1767
3031	serge	1768	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1769	if (request == NULL)
		1770	return -ENOMEM;
		1771
		1772
		1773	/* Record the position of the start of the request so that
		1774	* should we detect the updated seqno part-way through the
		1775	* GPU processing the request, we never over-estimate the
		1776	* position of the head.
		1777	*/
		1778	request_ring_position = intel_ring_get_tail(ring);
		1779
3243	Serge	1780	ret = ring->add_request(ring);
3031	serge	1781	if (ret) {
		1782	kfree(request);
2352	Serge	1783	return ret;
3031	serge	1784	}
2332	Serge	1785
3243	Serge	1786	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1787	request->ring = ring;
3031	serge	1788	request->tail = request_ring_position;
		1789	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1790	was_empty = list_empty(&ring->request_list);
		1791	list_add_tail(&request->list, &ring->request_list);
3031	serge	1792	request->file_priv = NULL;
2332	Serge	1793
3263	Serge	1794	if (file) {
		1795	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1796
3263	Serge	1797	spin_lock(&file_priv->mm.lock);
		1798	request->file_priv = file_priv;
		1799	list_add_tail(&request->client_list,
		1800	&file_priv->mm.request_list);
		1801	spin_unlock(&file_priv->mm.lock);
		1802	}
		1803
		1804	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1805	ring->outstanding_lazy_request = 0;
2332	Serge	1806
2360	Serge	1807	if (!dev_priv->mm.suspended) {
		1808	if (i915_enable_hangcheck) {
2352	Serge	1809	// mod_timer(&dev_priv->hangcheck_timer,
		1810	// jiffies +
		1811	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1812	}
3031	serge	1813	if (was_empty) {
2360	Serge	1814	queue_delayed_work(dev_priv->wq,
		1815	&dev_priv->mm.retire_work, HZ);
3031	serge	1816	intel_mark_busy(dev_priv->dev);
		1817	}
2360	Serge	1818	}
3031	serge	1819
		1820	if (out_seqno)
3243	Serge	1821	*out_seqno = request->seqno;
2352	Serge	1822	return 0;
		1823	}
2332	Serge	1824
3263	Serge	1825	static inline void
		1826	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1827	{
		1828	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1829
3263	Serge	1830	if (!file_priv)
		1831	return;
2332	Serge	1832
3263	Serge	1833	spin_lock(&file_priv->mm.lock);
		1834	if (request->file_priv) {
		1835	list_del(&request->client_list);
		1836	request->file_priv = NULL;
		1837	}
		1838	spin_unlock(&file_priv->mm.lock);
		1839	}
2332	Serge	1840
3031	serge	1841	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1842	struct intel_ring_buffer *ring)
		1843	{
		1844	while (!list_empty(&ring->request_list)) {
		1845	struct drm_i915_gem_request *request;
2332	Serge	1846
3031	serge	1847	request = list_first_entry(&ring->request_list,
		1848	struct drm_i915_gem_request,
		1849	list);
2332	Serge	1850
3031	serge	1851	list_del(&request->list);
3263	Serge	1852	i915_gem_request_remove_from_client(request);
3031	serge	1853	kfree(request);
		1854	}
2332	Serge	1855
3031	serge	1856	while (!list_empty(&ring->active_list)) {
		1857	struct drm_i915_gem_object *obj;
2332	Serge	1858
3031	serge	1859	obj = list_first_entry(&ring->active_list,
		1860	struct drm_i915_gem_object,
		1861	ring_list);
2332	Serge	1862
3031	serge	1863	i915_gem_object_move_to_inactive(obj);
		1864	}
		1865	}
2332	Serge	1866
3031	serge	1867	static void i915_gem_reset_fences(struct drm_device *dev)
		1868	{
		1869	struct drm_i915_private *dev_priv = dev->dev_private;
		1870	int i;
2332	Serge	1871
3031	serge	1872	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1873	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1874
3031	serge	1875	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1876
3031	serge	1877	if (reg->obj)
		1878	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1879
3031	serge	1880	reg->pin_count = 0;
		1881	reg->obj = NULL;
		1882	INIT_LIST_HEAD(®->lru_list);
		1883	}
2360	Serge	1884
3031	serge	1885	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1886	}
2360	Serge	1887
3031	serge	1888	void i915_gem_reset(struct drm_device *dev)
		1889	{
		1890	struct drm_i915_private *dev_priv = dev->dev_private;
		1891	struct drm_i915_gem_object *obj;
		1892	struct intel_ring_buffer *ring;
		1893	int i;
2360	Serge	1894
3031	serge	1895	for_each_ring(ring, dev_priv, i)
		1896	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1897
3031	serge	1898	/* Move everything out of the GPU domains to ensure we do any
		1899	* necessary invalidation upon reuse.
		1900	*/
		1901	list_for_each_entry(obj,
		1902	&dev_priv->mm.inactive_list,
		1903	mm_list)
		1904	{
		1905	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1906	}
2360	Serge	1907
3031	serge	1908	/* The fence registers are invalidated so clear them out */
		1909	i915_gem_reset_fences(dev);
		1910	}
2360	Serge	1911
2352	Serge	1912	/**
		1913	* This function clears the request list as sequence numbers are passed.
		1914	*/
3031	serge	1915	void
2352	Serge	1916	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1917	{
		1918	uint32_t seqno;
2332	Serge	1919
2352	Serge	1920	if (list_empty(&ring->request_list))
		1921	return;
2332	Serge	1922
2352	Serge	1923	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1924
3031	serge	1925	seqno = ring->get_seqno(ring, true);
2332	Serge	1926
2352	Serge	1927	while (!list_empty(&ring->request_list)) {
		1928	struct drm_i915_gem_request *request;
2332	Serge	1929
2352	Serge	1930	request = list_first_entry(&ring->request_list,
		1931	struct drm_i915_gem_request,
		1932	list);
2332	Serge	1933
2352	Serge	1934	if (!i915_seqno_passed(seqno, request->seqno))
		1935	break;
2332	Serge	1936
2352	Serge	1937	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1938	/* We know the GPU must have read the request to have
		1939	* sent us the seqno + interrupt, so use the position
		1940	* of tail of the request to update the last known position
		1941	* of the GPU head.
		1942	*/
		1943	ring->last_retired_head = request->tail;
2332	Serge	1944
2352	Serge	1945	list_del(&request->list);
3263	Serge	1946	i915_gem_request_remove_from_client(request);
2352	Serge	1947	kfree(request);
		1948	}
2332	Serge	1949
2352	Serge	1950	/* Move any buffers on the active list that are no longer referenced
		1951	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1952	*/
		1953	while (!list_empty(&ring->active_list)) {
		1954	struct drm_i915_gem_object *obj;
2332	Serge	1955
2352	Serge	1956	obj = list_first_entry(&ring->active_list,
		1957	struct drm_i915_gem_object,
		1958	ring_list);
2332	Serge	1959
3031	serge	1960	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1961	break;
2332	Serge	1962
2352	Serge	1963	i915_gem_object_move_to_inactive(obj);
		1964	}
2332	Serge	1965
2352	Serge	1966	if (unlikely(ring->trace_irq_seqno &&
		1967	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1968	ring->irq_put(ring);
		1969	ring->trace_irq_seqno = 0;
		1970	}
2332	Serge	1971
2352	Serge	1972	WARN_ON(i915_verify_lists(ring->dev));
		1973	}
2332	Serge	1974
2352	Serge	1975	void
		1976	i915_gem_retire_requests(struct drm_device *dev)
		1977	{
		1978	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1979	struct intel_ring_buffer *ring;
2352	Serge	1980	int i;
2332	Serge	1981
3031	serge	1982	for_each_ring(ring, dev_priv, i)
		1983	i915_gem_retire_requests_ring(ring);
2352	Serge	1984	}
		1985
2360	Serge	1986	static void
		1987	i915_gem_retire_work_handler(struct work_struct *work)
		1988	{
		1989	drm_i915_private_t *dev_priv;
		1990	struct drm_device *dev;
3031	serge	1991	struct intel_ring_buffer *ring;
2360	Serge	1992	bool idle;
		1993	int i;
2352	Serge	1994
2360	Serge	1995	dev_priv = container_of(work, drm_i915_private_t,
		1996	mm.retire_work.work);
		1997	dev = dev_priv->dev;
2352	Serge	1998
2360	Serge	1999	/* Come back later if the device is busy... */
		2000	if (!mutex_trylock(&dev->struct_mutex)) {
		2001	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	2002	return;
2360	Serge	2003	}
2352	Serge	2004
2360	Serge	2005	i915_gem_retire_requests(dev);
2352	Serge	2006
2360	Serge	2007	/* Send a periodic flush down the ring so we don't hold onto GEM
		2008	* objects indefinitely.
		2009	*/
		2010	idle = true;
3031	serge	2011	for_each_ring(ring, dev_priv, i) {
		2012	if (ring->gpu_caches_dirty)
		2013	i915_add_request(ring, NULL, NULL);
2352	Serge	2014
2360	Serge	2015	idle &= list_empty(&ring->request_list);
		2016	}
2352	Serge	2017
2360	Serge	2018	if (!dev_priv->mm.suspended && !idle)
		2019	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	2020	if (idle)
		2021	intel_mark_idle(dev);
2360	Serge	2022
		2023	mutex_unlock(&dev->struct_mutex);
		2024	}
		2025
2344	Serge	2026	/**
3031	serge	2027	* Ensures that an object will eventually get non-busy by flushing any required
		2028	* write domains, emitting any outstanding lazy request and retiring and
		2029	* completed requests.
2352	Serge	2030	*/
3031	serge	2031	static int
		2032	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2033	{
3031	serge	2034	int ret;
2352	Serge	2035
3031	serge	2036	if (obj->active) {
		2037	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2038	if (ret)
		2039	return ret;
2352	Serge	2040
3031	serge	2041	i915_gem_retire_requests_ring(obj->ring);
		2042	}
2352	Serge	2043
3031	serge	2044	return 0;
		2045	}
2352	Serge	2046
3243	Serge	2047	/**
		2048	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2049	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2050	*
		2051	* Returns 0 if successful, else an error is returned with the remaining time in
		2052	* the timeout parameter.
		2053	* -ETIME: object is still busy after timeout
		2054	* -ERESTARTSYS: signal interrupted the wait
		2055	* -ENONENT: object doesn't exist
		2056	* Also possible, but rare:
		2057	* -EAGAIN: GPU wedged
		2058	* -ENOMEM: damn
		2059	* -ENODEV: Internal IRQ fail
		2060	* -E?: The add request failed
		2061	*
		2062	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2063	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2064	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2065	* without holding struct_mutex the object may become re-busied before this
		2066	* function completes. A similar but shorter * race condition exists in the busy
		2067	* ioctl
		2068	*/
2352	Serge	2069
		2070
		2071
		2072
		2073
		2074
		2075
		2076
		2077
3243	Serge	2078
		2079
		2080
2352	Serge	2081	/**
3031	serge	2082	* i915_gem_object_sync - sync an object to a ring.
		2083	*
		2084	* @obj: object which may be in use on another ring.
		2085	* @to: ring we wish to use the object on. May be NULL.
		2086	*
		2087	* This code is meant to abstract object synchronization with the GPU.
		2088	* Calling with NULL implies synchronizing the object with the CPU
		2089	* rather than a particular GPU ring.
		2090	*
		2091	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2092	*/
		2093	int
3031	serge	2094	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2095	struct intel_ring_buffer *to)
2344	Serge	2096	{
3031	serge	2097	struct intel_ring_buffer *from = obj->ring;
		2098	u32 seqno;
		2099	int ret, idx;
2332	Serge	2100
3031	serge	2101	if (from == NULL \|\| to == from)
		2102	return 0;
2332	Serge	2103
3031	serge	2104	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2105	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2106
3031	serge	2107	idx = intel_ring_sync_index(from, to);
		2108
		2109	seqno = obj->last_read_seqno;
		2110	if (seqno <= from->sync_seqno[idx])
		2111	return 0;
		2112
		2113	ret = i915_gem_check_olr(obj->ring, seqno);
		2114	if (ret)
		2115	return ret;
		2116
		2117	ret = to->sync_to(to, from, seqno);
		2118	if (!ret)
3243	Serge	2119	/* We use last_read_seqno because sync_to()
		2120	* might have just caused seqno wrap under
		2121	* the radar.
		2122	*/
		2123	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2124
		2125	return ret;
2344	Serge	2126	}
2332	Serge	2127
2344	Serge	2128	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2129	{
		2130	u32 old_write_domain, old_read_domains;
2332	Serge	2131
2344	Serge	2132	/* Act a barrier for all accesses through the GTT */
		2133	mb();
2332	Serge	2134
2344	Serge	2135	/* Force a pagefault for domain tracking on next user access */
		2136	// i915_gem_release_mmap(obj);
2332	Serge	2137
2344	Serge	2138	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2139	return;
2332	Serge	2140
2344	Serge	2141	old_read_domains = obj->base.read_domains;
		2142	old_write_domain = obj->base.write_domain;
2351	Serge	2143
2344	Serge	2144	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2145	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2146
2351	Serge	2147	trace_i915_gem_object_change_domain(obj,
		2148	old_read_domains,
		2149	old_write_domain);
2344	Serge	2150	}
2332	Serge	2151
2344	Serge	2152	/**
		2153	* Unbinds an object from the GTT aperture.
		2154	*/
		2155	int
		2156	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2157	{
3031	serge	2158	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	2159	int ret = 0;
2332	Serge	2160
3263	Serge	2161	if(obj == get_fb_obj())
		2162	return 0;
		2163
2344	Serge	2164	if (obj->gtt_space == NULL)
		2165	return 0;
2332	Serge	2166
3031	serge	2167	if (obj->pin_count)
		2168	return -EBUSY;
2332	Serge	2169
3243	Serge	2170	BUG_ON(obj->pages == NULL);
3031	serge	2171
2344	Serge	2172	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2173	if (ret)
2344	Serge	2174	return ret;
		2175	/* Continue on if we fail due to EIO, the GPU is hung so we
		2176	* should be safe and we need to cleanup or else we might
		2177	* cause memory corruption through use-after-free.
		2178	*/
2332	Serge	2179
2344	Serge	2180	i915_gem_object_finish_gtt(obj);
2332	Serge	2181
2344	Serge	2182	/* release the fence reg _after_ flushing */
		2183	ret = i915_gem_object_put_fence(obj);
3031	serge	2184	if (ret)
2344	Serge	2185	return ret;
2332	Serge	2186
2351	Serge	2187	trace_i915_gem_object_unbind(obj);
2332	Serge	2188
3031	serge	2189	if (obj->has_global_gtt_mapping)
3243	Serge	2190	i915_gem_gtt_unbind_object(obj);
3031	serge	2191	if (obj->has_aliasing_ppgtt_mapping) {
		2192	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2193	obj->has_aliasing_ppgtt_mapping = 0;
		2194	}
		2195	i915_gem_gtt_finish_object(obj);
2332	Serge	2196
3031	serge	2197	list_del(&obj->mm_list);
		2198	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2199	/* Avoid an unnecessary call to unbind on rebind. */
		2200	obj->map_and_fenceable = true;
2332	Serge	2201
2344	Serge	2202	drm_mm_put_block(obj->gtt_space);
		2203	obj->gtt_space = NULL;
		2204	obj->gtt_offset = 0;
2332	Serge	2205
2344	Serge	2206	return 0;
		2207	}
2332	Serge	2208
3031	serge	2209	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2210	{
		2211	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2212	struct intel_ring_buffer *ring;
2344	Serge	2213	int ret, i;
2332	Serge	2214
2344	Serge	2215	/* Flush everything onto the inactive list. */
3031	serge	2216	for_each_ring(ring, dev_priv, i) {
		2217	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2218	if (ret)
		2219	return ret;
3031	serge	2220
3243	Serge	2221	ret = intel_ring_idle(ring);
3031	serge	2222	if (ret)
		2223	return ret;
2344	Serge	2224	}
2332	Serge	2225
2344	Serge	2226	return 0;
		2227	}
2332	Serge	2228
3031	serge	2229	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2230	struct drm_i915_gem_object *obj)
		2231	{
		2232	drm_i915_private_t *dev_priv = dev->dev_private;
		2233	uint64_t val;
2332	Serge	2234
3031	serge	2235	if (obj) {
		2236	u32 size = obj->gtt_space->size;
2332	Serge	2237
3031	serge	2238	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2239	0xfffff000) << 32;
		2240	val \|= obj->gtt_offset & 0xfffff000;
		2241	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2242	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2243
3031	serge	2244	if (obj->tiling_mode == I915_TILING_Y)
		2245	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2246	val \|= I965_FENCE_REG_VALID;
		2247	} else
		2248	val = 0;
2332	Serge	2249
3031	serge	2250	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2251	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2252	}
2332	Serge	2253
3031	serge	2254	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2255	struct drm_i915_gem_object *obj)
		2256	{
		2257	drm_i915_private_t *dev_priv = dev->dev_private;
		2258	uint64_t val;
2332	Serge	2259
3031	serge	2260	if (obj) {
		2261	u32 size = obj->gtt_space->size;
2332	Serge	2262
3031	serge	2263	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2264	0xfffff000) << 32;
		2265	val \|= obj->gtt_offset & 0xfffff000;
		2266	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2267	if (obj->tiling_mode == I915_TILING_Y)
		2268	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2269	val \|= I965_FENCE_REG_VALID;
		2270	} else
		2271	val = 0;
2332	Serge	2272
3031	serge	2273	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2274	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2275	}
2332	Serge	2276
3031	serge	2277	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2278	struct drm_i915_gem_object *obj)
		2279	{
		2280	drm_i915_private_t *dev_priv = dev->dev_private;
		2281	u32 val;
2332	Serge	2282
3031	serge	2283	if (obj) {
		2284	u32 size = obj->gtt_space->size;
		2285	int pitch_val;
		2286	int tile_width;
2332	Serge	2287
3031	serge	2288	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2289	(size & -size) != size \|\|
		2290	(obj->gtt_offset & (size - 1)),
		2291	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2292	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2293
3031	serge	2294	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2295	tile_width = 128;
		2296	else
		2297	tile_width = 512;
2332	Serge	2298
3031	serge	2299	/* Note: pitch better be a power of two tile widths */
		2300	pitch_val = obj->stride / tile_width;
		2301	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2302
3031	serge	2303	val = obj->gtt_offset;
		2304	if (obj->tiling_mode == I915_TILING_Y)
		2305	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2306	val \|= I915_FENCE_SIZE_BITS(size);
		2307	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2308	val \|= I830_FENCE_REG_VALID;
		2309	} else
		2310	val = 0;
2332	Serge	2311
3031	serge	2312	if (reg < 8)
		2313	reg = FENCE_REG_830_0 + reg * 4;
		2314	else
		2315	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2316
3031	serge	2317	I915_WRITE(reg, val);
		2318	POSTING_READ(reg);
		2319	}
2332	Serge	2320
3031	serge	2321	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2322	struct drm_i915_gem_object *obj)
		2323	{
		2324	drm_i915_private_t *dev_priv = dev->dev_private;
		2325	uint32_t val;
2344	Serge	2326
3031	serge	2327	if (obj) {
		2328	u32 size = obj->gtt_space->size;
		2329	uint32_t pitch_val;
2344	Serge	2330
3031	serge	2331	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2332	(size & -size) != size \|\|
		2333	(obj->gtt_offset & (size - 1)),
		2334	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2335	obj->gtt_offset, size);
2344	Serge	2336
3031	serge	2337	pitch_val = obj->stride / 128;
		2338	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2339
3031	serge	2340	val = obj->gtt_offset;
		2341	if (obj->tiling_mode == I915_TILING_Y)
		2342	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2343	val \|= I830_FENCE_SIZE_BITS(size);
		2344	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2345	val \|= I830_FENCE_REG_VALID;
		2346	} else
		2347	val = 0;
		2348
		2349	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2350	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2351	}
		2352
		2353	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2354	struct drm_i915_gem_object *obj)
2332	Serge	2355	{
3031	serge	2356	switch (INTEL_INFO(dev)->gen) {
		2357	case 7:
		2358	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2359	case 5:
		2360	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2361	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2362	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2363	default: break;
		2364	}
2344	Serge	2365	}
		2366
3031	serge	2367	static inline int fence_number(struct drm_i915_private *dev_priv,
		2368	struct drm_i915_fence_reg *fence)
2344	Serge	2369	{
3031	serge	2370	return fence - dev_priv->fence_regs;
		2371	}
2332	Serge	2372
3031	serge	2373	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2374	struct drm_i915_fence_reg *fence,
		2375	bool enable)
		2376	{
		2377	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2378	int reg = fence_number(dev_priv, fence);
2332	Serge	2379
3031	serge	2380	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2381
		2382	if (enable) {
		2383	obj->fence_reg = reg;
		2384	fence->obj = obj;
		2385	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2386	} else {
		2387	obj->fence_reg = I915_FENCE_REG_NONE;
		2388	fence->obj = NULL;
		2389	list_del_init(&fence->lru_list);
2344	Serge	2390	}
3031	serge	2391	}
2344	Serge	2392
3031	serge	2393	static int
		2394	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2395	{
		2396	if (obj->last_fenced_seqno) {
		2397	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2398	if (ret)
		2399	return ret;
2344	Serge	2400
		2401	obj->last_fenced_seqno = 0;
		2402	}
		2403
		2404	/* Ensure that all CPU reads are completed before installing a fence
		2405	* and all writes before removing the fence.
2332	Serge	2406	*/
2344	Serge	2407	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2408	mb();
2332	Serge	2409
3031	serge	2410	obj->fenced_gpu_access = false;
2332	Serge	2411	return 0;
		2412	}
		2413
		2414	int
2344	Serge	2415	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2416	{
3031	serge	2417	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2418	int ret;
		2419
3031	serge	2420	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2421	if (ret)
		2422	return ret;
		2423
3031	serge	2424	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2425	return 0;
2332	Serge	2426
3031	serge	2427	i915_gem_object_update_fence(obj,
		2428	&dev_priv->fence_regs[obj->fence_reg],
		2429	false);
		2430	i915_gem_object_fence_lost(obj);
2344	Serge	2431
2332	Serge	2432	return 0;
		2433	}
		2434
3031	serge	2435	static struct drm_i915_fence_reg *
		2436	i915_find_fence_reg(struct drm_device *dev)
		2437	{
		2438	struct drm_i915_private *dev_priv = dev->dev_private;
		2439	struct drm_i915_fence_reg reg, avail;
		2440	int i;
2332	Serge	2441
3031	serge	2442	/* First try to find a free reg */
		2443	avail = NULL;
		2444	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2445	reg = &dev_priv->fence_regs[i];
		2446	if (!reg->obj)
		2447	return reg;
2332	Serge	2448
3031	serge	2449	if (!reg->pin_count)
		2450	avail = reg;
		2451	}
2332	Serge	2452
3031	serge	2453	if (avail == NULL)
		2454	return NULL;
2332	Serge	2455
3031	serge	2456	/* None available, try to steal one or wait for a user to finish */
		2457	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2458	if (reg->pin_count)
		2459	continue;
2332	Serge	2460
3031	serge	2461	return reg;
		2462	}
2332	Serge	2463
3031	serge	2464	return NULL;
		2465	}
2332	Serge	2466
3031	serge	2467	/**
		2468	* i915_gem_object_get_fence - set up fencing for an object
		2469	* @obj: object to map through a fence reg
		2470	*
		2471	* When mapping objects through the GTT, userspace wants to be able to write
		2472	* to them without having to worry about swizzling if the object is tiled.
		2473	* This function walks the fence regs looking for a free one for @obj,
		2474	* stealing one if it can't find any.
		2475	*
		2476	* It then sets up the reg based on the object's properties: address, pitch
		2477	* and tiling format.
		2478	*
		2479	* For an untiled surface, this removes any existing fence.
		2480	*/
		2481	int
		2482	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2483	{
		2484	struct drm_device *dev = obj->base.dev;
		2485	struct drm_i915_private *dev_priv = dev->dev_private;
		2486	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2487	struct drm_i915_fence_reg *reg;
		2488	int ret;
2332	Serge	2489
3031	serge	2490	/* Have we updated the tiling parameters upon the object and so
		2491	* will need to serialise the write to the associated fence register?
		2492	*/
		2493	if (obj->fence_dirty) {
		2494	ret = i915_gem_object_flush_fence(obj);
		2495	if (ret)
		2496	return ret;
		2497	}
2332	Serge	2498
3031	serge	2499	/* Just update our place in the LRU if our fence is getting reused. */
		2500	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2501	reg = &dev_priv->fence_regs[obj->fence_reg];
		2502	if (!obj->fence_dirty) {
		2503	list_move_tail(®->lru_list,
		2504	&dev_priv->mm.fence_list);
		2505	return 0;
		2506	}
		2507	} else if (enable) {
		2508	reg = i915_find_fence_reg(dev);
		2509	if (reg == NULL)
		2510	return -EDEADLK;
2332	Serge	2511
3031	serge	2512	if (reg->obj) {
		2513	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2514
3031	serge	2515	ret = i915_gem_object_flush_fence(old);
		2516	if (ret)
		2517	return ret;
2332	Serge	2518
3031	serge	2519	i915_gem_object_fence_lost(old);
		2520	}
		2521	} else
		2522	return 0;
2332	Serge	2523
3031	serge	2524	i915_gem_object_update_fence(obj, reg, enable);
		2525	obj->fence_dirty = false;
2332	Serge	2526
3031	serge	2527	return 0;
		2528	}
2332	Serge	2529
3031	serge	2530	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2531	struct drm_mm_node *gtt_space,
		2532	unsigned long cache_level)
		2533	{
		2534	struct drm_mm_node *other;
2332	Serge	2535
3031	serge	2536	/* On non-LLC machines we have to be careful when putting differing
		2537	* types of snoopable memory together to avoid the prefetcher
		2538	* crossing memory domains and dieing.
		2539	*/
		2540	if (HAS_LLC(dev))
		2541	return true;
2332	Serge	2542
3031	serge	2543	if (gtt_space == NULL)
		2544	return true;
2332	Serge	2545
3031	serge	2546	if (list_empty(>t_space->node_list))
		2547	return true;
2332	Serge	2548
3031	serge	2549	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2550	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2551	return false;
2344	Serge	2552
3031	serge	2553	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2554	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2555	return false;
2344	Serge	2556
3031	serge	2557	return true;
		2558	}
2344	Serge	2559
3031	serge	2560	static void i915_gem_verify_gtt(struct drm_device *dev)
		2561	{
		2562	#if WATCH_GTT
		2563	struct drm_i915_private *dev_priv = dev->dev_private;
		2564	struct drm_i915_gem_object *obj;
		2565	int err = 0;
2344	Serge	2566
3031	serge	2567	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2568	if (obj->gtt_space == NULL) {
		2569	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2570	err++;
		2571	continue;
		2572	}
2344	Serge	2573
3031	serge	2574	if (obj->cache_level != obj->gtt_space->color) {
		2575	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2576	obj->gtt_space->start,
		2577	obj->gtt_space->start + obj->gtt_space->size,
		2578	obj->cache_level,
		2579	obj->gtt_space->color);
		2580	err++;
		2581	continue;
		2582	}
2344	Serge	2583
3031	serge	2584	if (!i915_gem_valid_gtt_space(dev,
		2585	obj->gtt_space,
		2586	obj->cache_level)) {
		2587	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2588	obj->gtt_space->start,
		2589	obj->gtt_space->start + obj->gtt_space->size,
		2590	obj->cache_level);
		2591	err++;
		2592	continue;
		2593	}
		2594	}
2344	Serge	2595
3031	serge	2596	WARN_ON(err);
		2597	#endif
2326	Serge	2598	}
		2599
2332	Serge	2600	/**
		2601	* Finds free space in the GTT aperture and binds the object there.
		2602	*/
		2603	static int
		2604	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2605	unsigned alignment,
3031	serge	2606	bool map_and_fenceable,
		2607	bool nonblocking)
2332	Serge	2608	{
		2609	struct drm_device *dev = obj->base.dev;
		2610	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2611	struct drm_mm_node *node;
2332	Serge	2612	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2613	bool mappable, fenceable;
		2614	int ret;
2326	Serge	2615
2332	Serge	2616	if (obj->madv != I915_MADV_WILLNEED) {
3243	Serge	2617	DRM_ERROR("Attempting to bind a purgeable object\n");
2332	Serge	2618	return -EINVAL;
		2619	}
		2620
		2621	fence_size = i915_gem_get_gtt_size(dev,
		2622	obj->base.size,
		2623	obj->tiling_mode);
		2624	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2625	obj->base.size,
		2626	obj->tiling_mode);
		2627	unfenced_alignment =
		2628	i915_gem_get_unfenced_gtt_alignment(dev,
		2629	obj->base.size,
		2630	obj->tiling_mode);
		2631
		2632	if (alignment == 0)
		2633	alignment = map_and_fenceable ? fence_alignment :
		2634	unfenced_alignment;
		2635	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2636	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2637	return -EINVAL;
		2638	}
		2639
		2640	size = map_and_fenceable ? fence_size : obj->base.size;
		2641
		2642	/* If the object is bigger than the entire aperture, reject it early
		2643	* before evicting everything in a vain attempt to find space.
		2644	*/
		2645	if (obj->base.size >
		2646	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2647	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2648	return -E2BIG;
		2649	}
		2650
3031	serge	2651	ret = i915_gem_object_get_pages(obj);
		2652	if (ret)
		2653	return ret;
		2654
3243	Serge	2655	i915_gem_object_pin_pages(obj);
		2656
		2657	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2658	if (node == NULL) {
		2659	i915_gem_object_unpin_pages(obj);
		2660	return -ENOMEM;
		2661	}
		2662
2332	Serge	2663	search_free:
		2664	if (map_and_fenceable)
3243	Serge	2665	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2666	size, alignment, obj->cache_level,
3243	Serge	2667	0, dev_priv->mm.gtt_mappable_end);
2332	Serge	2668	else
3243	Serge	2669	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2670	size, alignment, obj->cache_level);
		2671	if (ret) {
2332	Serge	2672
3243	Serge	2673	i915_gem_object_unpin_pages(obj);
		2674	kfree(node);
2332	Serge	2675	return ret;
		2676	}
3243	Serge	2677	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2678	i915_gem_object_unpin_pages(obj);
		2679	drm_mm_put_block(node);
3031	serge	2680	return -EINVAL;
		2681	}
2332	Serge	2682
3031	serge	2683	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2684	if (ret) {
3243	Serge	2685	i915_gem_object_unpin_pages(obj);
		2686	drm_mm_put_block(node);
2332	Serge	2687	return ret;
		2688	}
		2689
3031	serge	2690	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2691	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2692
3243	Serge	2693	obj->gtt_space = node;
		2694	obj->gtt_offset = node->start;
2332	Serge	2695
		2696	fenceable =
3243	Serge	2697	node->size == fence_size &&
		2698	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2699
		2700	mappable =
		2701	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2702
		2703	obj->map_and_fenceable = mappable && fenceable;
		2704
3243	Serge	2705	i915_gem_object_unpin_pages(obj);
2351	Serge	2706	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2707	i915_gem_verify_gtt(dev);
2332	Serge	2708	return 0;
		2709	}
		2710
		2711	void
		2712	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2713	{
		2714	/* If we don't have a page list set up, then we're not pinned
		2715	* to GPU, and we can ignore the cache flush because it'll happen
		2716	* again at bind time.
		2717	*/
3243	Serge	2718	if (obj->pages == NULL)
2332	Serge	2719	return;
		2720
		2721	/* If the GPU is snooping the contents of the CPU cache,
		2722	* we do not need to manually clear the CPU cache lines. However,
		2723	* the caches are only snooped when the render cache is
		2724	* flushed/invalidated. As we always have to emit invalidations
		2725	* and flushes when moving into and out of the RENDER domain, correct
		2726	* snooping behaviour occurs naturally as the result of our domain
		2727	* tracking.
		2728	*/
		2729	if (obj->cache_level != I915_CACHE_NONE)
		2730	return;
3243	Serge	2731	#if 0
2344	Serge	2732	if(obj->mapped != NULL)
		2733	{
		2734	uint8_t *page_virtual;
		2735	unsigned int i;
2332	Serge	2736
2344	Serge	2737	page_virtual = obj->mapped;
		2738	asm volatile("mfence");
		2739	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2740	clflush(page_virtual + i);
		2741	asm volatile("mfence");
		2742	}
		2743	else
		2744	{
		2745	uint8_t *page_virtual;
		2746	unsigned int i;
		2747	page_virtual = AllocKernelSpace(obj->base.size);
		2748	if(page_virtual != NULL)
		2749	{
3031	serge	2750	dma_addr_t src, dst;
2344	Serge	2751	u32 count;
		2752
		2753	#define page_tabs 0xFDC00000 /* really dirty hack */
		2754
3031	serge	2755	src = obj->pages.page;
		2756	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2757	count = obj->base.size/4096;
		2758
		2759	while(count--)
		2760	{
		2761	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2762	};
		2763
		2764	asm volatile("mfence");
		2765	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2766	clflush(page_virtual + i);
		2767	asm volatile("mfence");
		2768	FreeKernelSpace(page_virtual);
		2769	}
		2770	else
		2771	{
		2772	asm volatile (
		2773	"mfence \n"
		2774	"wbinvd \n" /* this is really ugly */
		2775	"mfence");
		2776	}
		2777	}
3243	Serge	2778	#endif
		2779
2332	Serge	2780	}
		2781
2344	Serge	2782	/** Flushes the GTT write domain for the object if it's dirty. */
		2783	static void
		2784	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2785	{
		2786	uint32_t old_write_domain;
2332	Serge	2787
2344	Serge	2788	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2789	return;
2332	Serge	2790
2344	Serge	2791	/* No actual flushing is required for the GTT write domain. Writes
		2792	* to it immediately go to main memory as far as we know, so there's
		2793	* no chipset flush. It also doesn't land in render cache.
		2794	*
		2795	* However, we do have to enforce the order so that all writes through
		2796	* the GTT land before any writes to the device, such as updates to
		2797	* the GATT itself.
		2798	*/
		2799	wmb();
2332	Serge	2800
2344	Serge	2801	old_write_domain = obj->base.write_domain;
		2802	obj->base.write_domain = 0;
2332	Serge	2803
2351	Serge	2804	trace_i915_gem_object_change_domain(obj,
		2805	obj->base.read_domains,
		2806	old_write_domain);
2344	Serge	2807	}
2332	Serge	2808
		2809	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2810	static void
2332	Serge	2811	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2812	{
		2813	uint32_t old_write_domain;
		2814
		2815	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2816	return;
		2817
		2818	i915_gem_clflush_object(obj);
3243	Serge	2819	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2820	old_write_domain = obj->base.write_domain;
		2821	obj->base.write_domain = 0;
		2822
2351	Serge	2823	trace_i915_gem_object_change_domain(obj,
		2824	obj->base.read_domains,
		2825	old_write_domain);
2332	Serge	2826	}
		2827
		2828	/**
		2829	* Moves a single object to the GTT read, and possibly write domain.
		2830	*
		2831	* This function returns when the move is complete, including waiting on
		2832	* flushes to occur.
		2833	*/
		2834	int
		2835	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2836	{
3031	serge	2837	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2838	uint32_t old_write_domain, old_read_domains;
		2839	int ret;
		2840
		2841	/* Not valid to be called on unbound objects. */
		2842	if (obj->gtt_space == NULL)
		2843	return -EINVAL;
		2844
		2845	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2846	return 0;
		2847
3031	serge	2848	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2849	if (ret)
		2850	return ret;
		2851
		2852	i915_gem_object_flush_cpu_write_domain(obj);
		2853
		2854	old_write_domain = obj->base.write_domain;
		2855	old_read_domains = obj->base.read_domains;
		2856
		2857	/* It should now be out of any other write domains, and we can update
		2858	* the domain values for our changes.
		2859	*/
		2860	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2861	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2862	if (write) {
		2863	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2864	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2865	obj->dirty = 1;
		2866	}
		2867
2351	Serge	2868	trace_i915_gem_object_change_domain(obj,
		2869	old_read_domains,
		2870	old_write_domain);
		2871
3031	serge	2872	/* And bump the LRU for this access */
		2873	if (i915_gem_object_is_inactive(obj))
		2874	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2875
2332	Serge	2876	return 0;
		2877	}
		2878
2335	Serge	2879	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2880	enum i915_cache_level cache_level)
		2881	{
3031	serge	2882	struct drm_device *dev = obj->base.dev;
		2883	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2884	int ret;
2332	Serge	2885
2335	Serge	2886	if (obj->cache_level == cache_level)
		2887	return 0;
2332	Serge	2888
2335	Serge	2889	if (obj->pin_count) {
		2890	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2891	return -EBUSY;
		2892	}
2332	Serge	2893
3031	serge	2894	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2895	ret = i915_gem_object_unbind(obj);
		2896	if (ret)
		2897	return ret;
		2898	}
		2899
2335	Serge	2900	if (obj->gtt_space) {
		2901	ret = i915_gem_object_finish_gpu(obj);
		2902	if (ret)
		2903	return ret;
2332	Serge	2904
2335	Serge	2905	i915_gem_object_finish_gtt(obj);
2332	Serge	2906
2335	Serge	2907	/* Before SandyBridge, you could not use tiling or fence
		2908	* registers with snooped memory, so relinquish any fences
		2909	* currently pointing to our region in the aperture.
		2910	*/
3031	serge	2911	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2912	ret = i915_gem_object_put_fence(obj);
		2913	if (ret)
		2914	return ret;
		2915	}
2332	Serge	2916
3031	serge	2917	if (obj->has_global_gtt_mapping)
		2918	i915_gem_gtt_bind_object(obj, cache_level);
		2919	if (obj->has_aliasing_ppgtt_mapping)
		2920	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2921	obj, cache_level);
		2922
		2923	obj->gtt_space->color = cache_level;
2335	Serge	2924	}
2332	Serge	2925
2335	Serge	2926	if (cache_level == I915_CACHE_NONE) {
		2927	u32 old_read_domains, old_write_domain;
2332	Serge	2928
2335	Serge	2929	/* If we're coming from LLC cached, then we haven't
		2930	* actually been tracking whether the data is in the
		2931	* CPU cache or not, since we only allow one bit set
		2932	* in obj->write_domain and have been skipping the clflushes.
		2933	* Just set it to the CPU cache for now.
		2934	*/
		2935	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2936	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2937
2335	Serge	2938	old_read_domains = obj->base.read_domains;
		2939	old_write_domain = obj->base.write_domain;
2332	Serge	2940
2335	Serge	2941	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2942	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2943
2351	Serge	2944	trace_i915_gem_object_change_domain(obj,
		2945	old_read_domains,
		2946	old_write_domain);
2344	Serge	2947	}
2332	Serge	2948
2335	Serge	2949	obj->cache_level = cache_level;
3031	serge	2950	i915_gem_verify_gtt(dev);
2335	Serge	2951	return 0;
		2952	}
2332	Serge	2953
3260	Serge	2954	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		2955	struct drm_file *file)
		2956	{
		2957	struct drm_i915_gem_caching *args = data;
		2958	struct drm_i915_gem_object *obj;
		2959	int ret;
		2960
		2961	ret = i915_mutex_lock_interruptible(dev);
		2962	if (ret)
		2963	return ret;
		2964
		2965	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2966	if (&obj->base == NULL) {
		2967	ret = -ENOENT;
		2968	goto unlock;
		2969	}
		2970
		2971	args->caching = obj->cache_level != I915_CACHE_NONE;
		2972
		2973	drm_gem_object_unreference(&obj->base);
		2974	unlock:
		2975	mutex_unlock(&dev->struct_mutex);
		2976	return ret;
		2977	}
		2978
		2979	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		2980	struct drm_file *file)
		2981	{
		2982	struct drm_i915_gem_caching *args = data;
		2983	struct drm_i915_gem_object *obj;
		2984	enum i915_cache_level level;
		2985	int ret;
		2986
		2987	switch (args->caching) {
		2988	case I915_CACHING_NONE:
		2989	level = I915_CACHE_NONE;
		2990	break;
		2991	case I915_CACHING_CACHED:
		2992	level = I915_CACHE_LLC;
		2993	break;
		2994	default:
		2995	return -EINVAL;
		2996	}
		2997
		2998	ret = i915_mutex_lock_interruptible(dev);
		2999	if (ret)
		3000	return ret;
		3001
		3002	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3003	if (&obj->base == NULL) {
		3004	ret = -ENOENT;
		3005	goto unlock;
		3006	}
		3007
		3008	ret = i915_gem_object_set_cache_level(obj, level);
		3009
		3010	drm_gem_object_unreference(&obj->base);
		3011	unlock:
		3012	mutex_unlock(&dev->struct_mutex);
		3013	return ret;
		3014	}
		3015
2335	Serge	3016	/*
		3017	* Prepare buffer for display plane (scanout, cursors, etc).
		3018	* Can be called from an uninterruptible phase (modesetting) and allows
		3019	* any flushes to be pipelined (for pageflips).
		3020	*/
		3021	int
		3022	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3023	u32 alignment,
		3024	struct intel_ring_buffer *pipelined)
		3025	{
		3026	u32 old_read_domains, old_write_domain;
		3027	int ret;
2332	Serge	3028
3031	serge	3029	if (pipelined != obj->ring) {
		3030	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3031	if (ret)
		3032	return ret;
		3033	}
2332	Serge	3034
2335	Serge	3035	/* The display engine is not coherent with the LLC cache on gen6. As
		3036	* a result, we make sure that the pinning that is about to occur is
		3037	* done with uncached PTEs. This is lowest common denominator for all
		3038	* chipsets.
		3039	*
		3040	* However for gen6+, we could do better by using the GFDT bit instead
		3041	* of uncaching, which would allow us to flush all the LLC-cached data
		3042	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3043	*/
2360	Serge	3044	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3045	if (ret)
		3046	return ret;
2332	Serge	3047
2335	Serge	3048	/* As the user may map the buffer once pinned in the display plane
		3049	* (e.g. libkms for the bootup splash), we have to ensure that we
		3050	* always use map_and_fenceable for all scanout buffers.
		3051	*/
3031	serge	3052	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3053	if (ret)
		3054	return ret;
2332	Serge	3055
2335	Serge	3056	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3057
2335	Serge	3058	old_write_domain = obj->base.write_domain;
		3059	old_read_domains = obj->base.read_domains;
2332	Serge	3060
2335	Serge	3061	/* It should now be out of any other write domains, and we can update
		3062	* the domain values for our changes.
		3063	*/
3031	serge	3064	obj->base.write_domain = 0;
2335	Serge	3065	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3066
2351	Serge	3067	trace_i915_gem_object_change_domain(obj,
		3068	old_read_domains,
		3069	old_write_domain);
2332	Serge	3070
2335	Serge	3071	return 0;
		3072	}
2332	Serge	3073
2344	Serge	3074	int
		3075	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3076	{
		3077	int ret;
2332	Serge	3078
2344	Serge	3079	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3080	return 0;
2332	Serge	3081
3031	serge	3082	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3083	if (ret)
		3084	return ret;
2332	Serge	3085
2344	Serge	3086	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3087	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3088	return 0;
2344	Serge	3089	}
2332	Serge	3090
2344	Serge	3091	/**
		3092	* Moves a single object to the CPU read, and possibly write domain.
		3093	*
		3094	* This function returns when the move is complete, including waiting on
		3095	* flushes to occur.
		3096	*/
3031	serge	3097	int
2344	Serge	3098	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3099	{
		3100	uint32_t old_write_domain, old_read_domains;
		3101	int ret;
2332	Serge	3102
2344	Serge	3103	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3104	return 0;
2332	Serge	3105
3031	serge	3106	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3107	if (ret)
		3108	return ret;
2332	Serge	3109
2344	Serge	3110	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3111
2344	Serge	3112	old_write_domain = obj->base.write_domain;
		3113	old_read_domains = obj->base.read_domains;
2332	Serge	3114
2344	Serge	3115	/* Flush the CPU cache if it's still invalid. */
		3116	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3117	i915_gem_clflush_object(obj);
2332	Serge	3118
2344	Serge	3119	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3120	}
2332	Serge	3121
2344	Serge	3122	/* It should now be out of any other write domains, and we can update
		3123	* the domain values for our changes.
		3124	*/
		3125	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3126
2344	Serge	3127	/* If we're writing through the CPU, then the GPU read domains will
		3128	* need to be invalidated at next use.
		3129	*/
		3130	if (write) {
		3131	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3132	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3133	}
2332	Serge	3134
2351	Serge	3135	trace_i915_gem_object_change_domain(obj,
		3136	old_read_domains,
		3137	old_write_domain);
2332	Serge	3138
2344	Serge	3139	return 0;
		3140	}
2332	Serge	3141
3031	serge	3142	/* Throttle our rendering by waiting until the ring has completed our requests
		3143	* emitted over 20 msec ago.
2344	Serge	3144	*
3031	serge	3145	* Note that if we were to use the current jiffies each time around the loop,
		3146	* we wouldn't escape the function with any frames outstanding if the time to
		3147	* render a frame was over 20ms.
		3148	*
		3149	* This should get us reasonable parallelism between CPU and GPU but also
		3150	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3151	*/
3031	serge	3152	static int
		3153	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3154	{
3031	serge	3155	struct drm_i915_private *dev_priv = dev->dev_private;
		3156	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3157	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3158	struct drm_i915_gem_request *request;
		3159	struct intel_ring_buffer *ring = NULL;
		3160	u32 seqno = 0;
		3161	int ret;
2332	Serge	3162
3031	serge	3163	if (atomic_read(&dev_priv->mm.wedged))
		3164	return -EIO;
2332	Serge	3165
3031	serge	3166	spin_lock(&file_priv->mm.lock);
		3167	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3168	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3169	break;
2332	Serge	3170
3031	serge	3171	ring = request->ring;
		3172	seqno = request->seqno;
		3173	}
		3174	spin_unlock(&file_priv->mm.lock);
2332	Serge	3175
3031	serge	3176	if (seqno == 0)
		3177	return 0;
2332	Serge	3178
3031	serge	3179	ret = __wait_seqno(ring, seqno, true, NULL);
		3180	if (ret == 0)
		3181	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3182
3031	serge	3183	return ret;
2352	Serge	3184	}
2332	Serge	3185
		3186	int
		3187	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3188	uint32_t alignment,
3031	serge	3189	bool map_and_fenceable,
		3190	bool nonblocking)
2332	Serge	3191	{
		3192	int ret;
		3193
3031	serge	3194	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3195	return -EBUSY;
2332	Serge	3196
		3197	if (obj->gtt_space != NULL) {
		3198	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3199	(map_and_fenceable && !obj->map_and_fenceable)) {
		3200	WARN(obj->pin_count,
		3201	"bo is already pinned with incorrect alignment:"
		3202	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3203	" obj->map_and_fenceable=%d\n",
		3204	obj->gtt_offset, alignment,
		3205	map_and_fenceable,
		3206	obj->map_and_fenceable);
		3207	ret = i915_gem_object_unbind(obj);
		3208	if (ret)
		3209	return ret;
		3210	}
		3211	}
		3212
		3213	if (obj->gtt_space == NULL) {
3243	Serge	3214	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3215
2332	Serge	3216	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3217	map_and_fenceable,
		3218	nonblocking);
2332	Serge	3219	if (ret)
		3220	return ret;
3243	Serge	3221
		3222	if (!dev_priv->mm.aliasing_ppgtt)
		3223	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3224	}
		3225
3031	serge	3226	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3227	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3228
		3229	obj->pin_count++;
2332	Serge	3230	obj->pin_mappable \|= map_and_fenceable;
		3231
		3232	return 0;
		3233	}
		3234
2344	Serge	3235	void
		3236	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3237	{
		3238	BUG_ON(obj->pin_count == 0);
		3239	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3240
3031	serge	3241	if (--obj->pin_count == 0)
2344	Serge	3242	obj->pin_mappable = false;
		3243	}
2332	Serge	3244
3031	serge	3245	int
		3246	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3247	struct drm_file *file)
		3248	{
		3249	struct drm_i915_gem_pin *args = data;
		3250	struct drm_i915_gem_object *obj;
		3251	int ret;
2332	Serge	3252
3031	serge	3253	ret = i915_mutex_lock_interruptible(dev);
		3254	if (ret)
		3255	return ret;
2332	Serge	3256
3031	serge	3257	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3258	if (&obj->base == NULL) {
		3259	ret = -ENOENT;
		3260	goto unlock;
		3261	}
2332	Serge	3262
3031	serge	3263	if (obj->madv != I915_MADV_WILLNEED) {
		3264	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3265	ret = -EINVAL;
		3266	goto out;
		3267	}
2332	Serge	3268
3031	serge	3269	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3270	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3271	args->handle);
		3272	ret = -EINVAL;
		3273	goto out;
		3274	}
2332	Serge	3275
3243	Serge	3276	if (obj->user_pin_count == 0) {
3031	serge	3277	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3278	if (ret)
		3279	goto out;
		3280	}
2332	Serge	3281
3243	Serge	3282	obj->user_pin_count++;
		3283	obj->pin_filp = file;
		3284
3031	serge	3285	/* XXX - flush the CPU caches for pinned objects
		3286	* as the X server doesn't manage domains yet
		3287	*/
		3288	i915_gem_object_flush_cpu_write_domain(obj);
		3289	args->offset = obj->gtt_offset;
		3290	out:
		3291	drm_gem_object_unreference(&obj->base);
		3292	unlock:
		3293	mutex_unlock(&dev->struct_mutex);
		3294	return ret;
		3295	}
2332	Serge	3296
3255	Serge	3297	#if 0
		3298
3031	serge	3299	int
		3300	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3301	struct drm_file *file)
		3302	{
		3303	struct drm_i915_gem_pin *args = data;
		3304	struct drm_i915_gem_object *obj;
		3305	int ret;
2332	Serge	3306
3031	serge	3307	ret = i915_mutex_lock_interruptible(dev);
		3308	if (ret)
		3309	return ret;
2332	Serge	3310
3031	serge	3311	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3312	if (&obj->base == NULL) {
		3313	ret = -ENOENT;
		3314	goto unlock;
		3315	}
2332	Serge	3316
3031	serge	3317	if (obj->pin_filp != file) {
		3318	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3319	args->handle);
		3320	ret = -EINVAL;
		3321	goto out;
		3322	}
		3323	obj->user_pin_count--;
		3324	if (obj->user_pin_count == 0) {
		3325	obj->pin_filp = NULL;
		3326	i915_gem_object_unpin(obj);
		3327	}
2332	Serge	3328
3031	serge	3329	out:
		3330	drm_gem_object_unreference(&obj->base);
		3331	unlock:
		3332	mutex_unlock(&dev->struct_mutex);
		3333	return ret;
		3334	}
2332	Serge	3335
3260	Serge	3336	#endif
		3337
3031	serge	3338	int
		3339	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3340	struct drm_file *file)
		3341	{
		3342	struct drm_i915_gem_busy *args = data;
		3343	struct drm_i915_gem_object *obj;
		3344	int ret;
2332	Serge	3345
3031	serge	3346	ret = i915_mutex_lock_interruptible(dev);
		3347	if (ret)
		3348	return ret;
2332	Serge	3349
3031	serge	3350	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3351	if (&obj->base == NULL) {
		3352	ret = -ENOENT;
		3353	goto unlock;
		3354	}
2332	Serge	3355
3031	serge	3356	/* Count all active objects as busy, even if they are currently not used
		3357	* by the gpu. Users of this interface expect objects to eventually
		3358	* become non-busy without any further actions, therefore emit any
		3359	* necessary flushes here.
		3360	*/
		3361	ret = i915_gem_object_flush_active(obj);
2332	Serge	3362
3031	serge	3363	args->busy = obj->active;
		3364	if (obj->ring) {
		3365	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3366	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3367	}
2332	Serge	3368
3031	serge	3369	drm_gem_object_unreference(&obj->base);
		3370	unlock:
		3371	mutex_unlock(&dev->struct_mutex);
		3372	return ret;
		3373	}
2332	Serge	3374
3031	serge	3375	int
		3376	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3377	struct drm_file *file_priv)
		3378	{
		3379	return i915_gem_ring_throttle(dev, file_priv);
		3380	}
2332	Serge	3381
3263	Serge	3382	#if 0
		3383
3031	serge	3384	int
		3385	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3386	struct drm_file *file_priv)
		3387	{
		3388	struct drm_i915_gem_madvise *args = data;
		3389	struct drm_i915_gem_object *obj;
		3390	int ret;
2332	Serge	3391
3031	serge	3392	switch (args->madv) {
		3393	case I915_MADV_DONTNEED:
		3394	case I915_MADV_WILLNEED:
		3395	break;
		3396	default:
		3397	return -EINVAL;
		3398	}
2332	Serge	3399
3031	serge	3400	ret = i915_mutex_lock_interruptible(dev);
		3401	if (ret)
		3402	return ret;
2332	Serge	3403
3031	serge	3404	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3405	if (&obj->base == NULL) {
		3406	ret = -ENOENT;
		3407	goto unlock;
		3408	}
2332	Serge	3409
3031	serge	3410	if (obj->pin_count) {
		3411	ret = -EINVAL;
		3412	goto out;
		3413	}
2332	Serge	3414
3031	serge	3415	if (obj->madv != __I915_MADV_PURGED)
		3416	obj->madv = args->madv;
2332	Serge	3417
3031	serge	3418	/* if the object is no longer attached, discard its backing storage */
		3419	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3420	i915_gem_object_truncate(obj);
2332	Serge	3421
3031	serge	3422	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3423
3031	serge	3424	out:
		3425	drm_gem_object_unreference(&obj->base);
		3426	unlock:
		3427	mutex_unlock(&dev->struct_mutex);
		3428	return ret;
		3429	}
		3430	#endif
2332	Serge	3431
3031	serge	3432	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3433	const struct drm_i915_gem_object_ops *ops)
		3434	{
		3435	INIT_LIST_HEAD(&obj->mm_list);
		3436	INIT_LIST_HEAD(&obj->gtt_list);
		3437	INIT_LIST_HEAD(&obj->ring_list);
		3438	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3439
3031	serge	3440	obj->ops = ops;
		3441
		3442	obj->fence_reg = I915_FENCE_REG_NONE;
		3443	obj->madv = I915_MADV_WILLNEED;
		3444	/* Avoid an unnecessary call to unbind on the first bind. */
		3445	obj->map_and_fenceable = true;
		3446
		3447	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3448	}
		3449
		3450	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3451	.get_pages = i915_gem_object_get_pages_gtt,
		3452	.put_pages = i915_gem_object_put_pages_gtt,
		3453	};
		3454
2332	Serge	3455	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3456	size_t size)
		3457	{
		3458	struct drm_i915_gem_object *obj;
3031	serge	3459	struct address_space *mapping;
		3460	u32 mask;
2340	Serge	3461
2332	Serge	3462	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3463	if (obj == NULL)
		3464	return NULL;
		3465
		3466	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3467	kfree(obj);
		3468	return NULL;
		3469	}
		3470
		3471
3031	serge	3472	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3473
		3474	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3475	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3476
3031	serge	3477	if (HAS_LLC(dev)) {
		3478	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3479	* cache) for about a 10% performance improvement
		3480	* compared to uncached. Graphics requests other than
		3481	* display scanout are coherent with the CPU in
		3482	* accessing this cache. This means in this mode we
		3483	* don't need to clflush on the CPU side, and on the
		3484	* GPU side we only need to flush internal caches to
		3485	* get data visible to the CPU.
		3486	*
		3487	* However, we maintain the display planes as UC, and so
		3488	* need to rebind when first used as such.
		3489	*/
		3490	obj->cache_level = I915_CACHE_LLC;
		3491	} else
		3492	obj->cache_level = I915_CACHE_NONE;
		3493
		3494	return obj;
		3495	}
		3496
2344	Serge	3497	int i915_gem_init_object(struct drm_gem_object *obj)
		3498	{
		3499	BUG();
2332	Serge	3500
2344	Serge	3501	return 0;
		3502	}
2332	Serge	3503
3031	serge	3504	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3505	{
3031	serge	3506	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3507	struct drm_device *dev = obj->base.dev;
		3508	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3509
3031	serge	3510	trace_i915_gem_object_destroy(obj);
		3511
		3512	// if (obj->phys_obj)
		3513	// i915_gem_detach_phys_object(dev, obj);
3290	Serge	3514	printf("%s obj %p\n", __FUNCTION__, obj);
3031	serge	3515
		3516	obj->pin_count = 0;
		3517	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3518	bool was_interruptible;
		3519
		3520	was_interruptible = dev_priv->mm.interruptible;
		3521	dev_priv->mm.interruptible = false;
		3522
		3523	WARN_ON(i915_gem_object_unbind(obj));
		3524
		3525	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3526	}
2332	Serge	3527
3031	serge	3528	obj->pages_pin_count = 0;
		3529	i915_gem_object_put_pages(obj);
		3530	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3531
3243	Serge	3532	BUG_ON(obj->pages);
2332	Serge	3533
3031	serge	3534	// if (obj->base.import_attach)
		3535	// drm_prime_gem_destroy(&obj->base, NULL);
		3536
3290	Serge	3537	if(obj->base.filp != NULL)
		3538	{
		3539	printf("filp %p\n", obj->base.filp);
		3540	shmem_file_delete(obj->base.filp);
		3541	}
		3542
2344	Serge	3543	drm_gem_object_release(&obj->base);
		3544	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3545
2344	Serge	3546	kfree(obj->bit_17);
		3547	kfree(obj);
		3548	}
2332	Serge	3549
3031	serge	3550	#if 0
		3551	int
		3552	i915_gem_idle(struct drm_device *dev)
2344	Serge	3553	{
3031	serge	3554	drm_i915_private_t *dev_priv = dev->dev_private;
		3555	int ret;
2332	Serge	3556
3031	serge	3557	mutex_lock(&dev->struct_mutex);
2332	Serge	3558
3031	serge	3559	if (dev_priv->mm.suspended) {
		3560	mutex_unlock(&dev->struct_mutex);
		3561	return 0;
		3562	}
2332	Serge	3563
3031	serge	3564	ret = i915_gpu_idle(dev);
		3565	if (ret) {
		3566	mutex_unlock(&dev->struct_mutex);
		3567	return ret;
		3568	}
		3569	i915_gem_retire_requests(dev);
		3570
		3571	i915_gem_reset_fences(dev);
		3572
		3573	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3574	* We need to replace this with a semaphore, or something.
		3575	* And not confound mm.suspended!
		3576	*/
		3577	dev_priv->mm.suspended = 1;
		3578	del_timer_sync(&dev_priv->hangcheck_timer);
		3579
		3580	i915_kernel_lost_context(dev);
		3581	i915_gem_cleanup_ringbuffer(dev);
		3582
		3583	mutex_unlock(&dev->struct_mutex);
		3584
		3585	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	3586	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	3587
		3588	return 0;
2344	Serge	3589	}
3031	serge	3590	#endif
2332	Serge	3591
3031	serge	3592	void i915_gem_l3_remap(struct drm_device *dev)
		3593	{
		3594	drm_i915_private_t *dev_priv = dev->dev_private;
		3595	u32 misccpctl;
		3596	int i;
2332	Serge	3597
3031	serge	3598	if (!IS_IVYBRIDGE(dev))
		3599	return;
2332	Serge	3600
3243	Serge	3601	if (!dev_priv->l3_parity.remap_info)
3031	serge	3602	return;
2332	Serge	3603
3031	serge	3604	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3605	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3606	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3607
3031	serge	3608	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3609	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3610	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3611	DRM_DEBUG("0x%x was already programmed to %x\n",
		3612	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3613	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3614	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3615	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3616	}
2332	Serge	3617
3031	serge	3618	/* Make sure all the writes land before disabling dop clock gating */
		3619	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3620
3031	serge	3621	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3622	}
2332	Serge	3623
3031	serge	3624	void i915_gem_init_swizzling(struct drm_device *dev)
		3625	{
		3626	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3627
3031	serge	3628	if (INTEL_INFO(dev)->gen < 5 \|\|
		3629	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3630	return;
2332	Serge	3631
3031	serge	3632	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3633	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3634
3031	serge	3635	if (IS_GEN5(dev))
		3636	return;
2344	Serge	3637
3031	serge	3638	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3639	if (IS_GEN6(dev))
		3640	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3641	else
		3642	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3643	}
		3644
		3645	static bool
		3646	intel_enable_blt(struct drm_device *dev)
		3647	{
		3648	if (!HAS_BLT(dev))
		3649	return false;
		3650
		3651	/* The blitter was dysfunctional on early prototypes */
		3652	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3653	DRM_INFO("BLT not supported on this pre-production hardware;"
		3654	" graphics performance will be degraded.\n");
		3655	return false;
		3656	}
		3657
		3658	return true;
		3659	}
		3660
2332	Serge	3661	int
3031	serge	3662	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3663	{
		3664	drm_i915_private_t *dev_priv = dev->dev_private;
		3665	int ret;
2351	Serge	3666
3243	Serge	3667	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
3031	serge	3668	return -EIO;
		3669
		3670	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3671	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3672
		3673	i915_gem_l3_remap(dev);
		3674
		3675	i915_gem_init_swizzling(dev);
		3676
2332	Serge	3677	ret = intel_init_render_ring_buffer(dev);
		3678	if (ret)
		3679	return ret;
		3680
		3681	if (HAS_BSD(dev)) {
		3682	ret = intel_init_bsd_ring_buffer(dev);
		3683	if (ret)
		3684	goto cleanup_render_ring;
		3685	}
		3686
3031	serge	3687	if (intel_enable_blt(dev)) {
2332	Serge	3688	ret = intel_init_blt_ring_buffer(dev);
		3689	if (ret)
		3690	goto cleanup_bsd_ring;
		3691	}
		3692
		3693	dev_priv->next_seqno = 1;
2351	Serge	3694
3031	serge	3695	/*
		3696	* XXX: There was some w/a described somewhere suggesting loading
		3697	* contexts before PPGTT.
		3698	*/
		3699	i915_gem_context_init(dev);
		3700	i915_gem_init_ppgtt(dev);
		3701
2332	Serge	3702	return 0;
		3703
		3704	cleanup_bsd_ring:
		3705	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3706	cleanup_render_ring:
		3707	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3708	return ret;
		3709	}
		3710
3031	serge	3711	static bool
		3712	intel_enable_ppgtt(struct drm_device *dev)
		3713	{
		3714	if (i915_enable_ppgtt >= 0)
		3715	return i915_enable_ppgtt;
		3716
		3717	#ifdef CONFIG_INTEL_IOMMU
		3718	/* Disable ppgtt on SNB if VT-d is on. */
		3719	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3720	return false;
		3721	#endif
		3722
		3723	return true;
		3724	}
		3725
		3726	#define LFB_SIZE 0xC00000
		3727
		3728	int i915_gem_init(struct drm_device *dev)
		3729	{
		3730	struct drm_i915_private *dev_priv = dev->dev_private;
		3731	unsigned long gtt_size, mappable_size;
		3732	int ret;
		3733
		3734	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3735	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3736
		3737	mutex_lock(&dev->struct_mutex);
		3738	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3739	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3740	* aperture accordingly when using aliasing ppgtt. */
		3741	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3742
		3743	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3744
		3745	ret = i915_gem_init_aliasing_ppgtt(dev);
		3746	if (ret) {
		3747	mutex_unlock(&dev->struct_mutex);
		3748	return ret;
		3749	}
		3750	} else {
		3751	/* Let GEM Manage all of the aperture.
		3752	*
		3753	* However, leave one page at the end still bound to the scratch
		3754	* page. There are a number of places where the hardware
		3755	* apparently prefetches past the end of the object, and we've
		3756	* seen multiple hangs with the GPU head pointer stuck in a
		3757	* batchbuffer bound at the last page of the aperture. One page
		3758	* should be enough to keep any prefetching inside of the
		3759	* aperture.
		3760	*/
		3761	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3762	}
		3763
		3764	ret = i915_gem_init_hw(dev);
		3765	mutex_unlock(&dev->struct_mutex);
		3766	if (ret) {
		3767	i915_gem_cleanup_aliasing_ppgtt(dev);
		3768	return ret;
		3769	}
		3770
		3771	return 0;
		3772	}
		3773
2332	Serge	3774	void
		3775	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3776	{
		3777	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3778	struct intel_ring_buffer *ring;
2332	Serge	3779	int i;
		3780
3031	serge	3781	for_each_ring(ring, dev_priv, i)
		3782	intel_cleanup_ring_buffer(ring);
2332	Serge	3783	}
		3784
3031	serge	3785	#if 0
		3786
2332	Serge	3787	int
		3788	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3789	struct drm_file *file_priv)
		3790	{
		3791	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3792	int ret;
2332	Serge	3793
		3794	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3795	return 0;
		3796
		3797	if (atomic_read(&dev_priv->mm.wedged)) {
		3798	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3799	atomic_set(&dev_priv->mm.wedged, 0);
		3800	}
		3801
		3802	mutex_lock(&dev->struct_mutex);
		3803	dev_priv->mm.suspended = 0;
		3804
3031	serge	3805	ret = i915_gem_init_hw(dev);
2332	Serge	3806	if (ret != 0) {
		3807	mutex_unlock(&dev->struct_mutex);
		3808	return ret;
		3809	}
		3810
		3811	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3812	mutex_unlock(&dev->struct_mutex);
		3813
		3814	ret = drm_irq_install(dev);
		3815	if (ret)
		3816	goto cleanup_ringbuffer;
		3817
		3818	return 0;
		3819
		3820	cleanup_ringbuffer:
		3821	mutex_lock(&dev->struct_mutex);
		3822	i915_gem_cleanup_ringbuffer(dev);
		3823	dev_priv->mm.suspended = 1;
		3824	mutex_unlock(&dev->struct_mutex);
		3825
		3826	return ret;
		3827	}
		3828
		3829	int
		3830	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3831	struct drm_file *file_priv)
		3832	{
		3833	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3834	return 0;
		3835
		3836	drm_irq_uninstall(dev);
		3837	return i915_gem_idle(dev);
		3838	}
		3839
		3840	void
		3841	i915_gem_lastclose(struct drm_device *dev)
		3842	{
		3843	int ret;
		3844
		3845	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3846	return;
		3847
		3848	ret = i915_gem_idle(dev);
		3849	if (ret)
		3850	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3851	}
		3852	#endif
		3853
		3854	static void
2326	Serge	3855	init_ring_lists(struct intel_ring_buffer *ring)
		3856	{
		3857	INIT_LIST_HEAD(&ring->active_list);
		3858	INIT_LIST_HEAD(&ring->request_list);
		3859	}
		3860
		3861	void
		3862	i915_gem_load(struct drm_device *dev)
		3863	{
		3864	int i;
		3865	drm_i915_private_t *dev_priv = dev->dev_private;
		3866
		3867	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3868	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3869	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3870	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3871	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3872	for (i = 0; i < I915_NUM_RINGS; i++)
		3873	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3874	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3875	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3876	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3877	i915_gem_retire_work_handler);
2326	Serge	3878
		3879	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3880	if (IS_GEN3(dev)) {
3031	serge	3881	I915_WRITE(MI_ARB_STATE,
		3882	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3883	}
		3884
		3885	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3886
		3887	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3888	dev_priv->num_fence_regs = 16;
		3889	else
		3890	dev_priv->num_fence_regs = 8;
		3891
		3892	/* Initialize fence registers to zero */
3031	serge	3893	i915_gem_reset_fences(dev);
2326	Serge	3894
		3895	i915_gem_detect_bit_6_swizzle(dev);
		3896
		3897	dev_priv->mm.interruptible = true;
		3898
		3899	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3900	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3901	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3902	}
		3903
		3904

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 5367 – Rev 3290