WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
3746	Serge	36	#include
2326	Serge	37	#include
		38
2344	Serge	39	extern int x86_clflush_size;
2332	Serge	40
3263	Serge	41	#define PROT_READ 0x1 /* page can be read */
		42	#define PROT_WRITE 0x2 /* page can be written */
		43	#define MAP_SHARED 0x01 /* Share changes */
		44
2344	Serge	45	#undef mb
		46	#undef rmb
		47	#undef wmb
		48	#define mb() asm volatile("mfence")
		49	#define rmb() asm volatile ("lfence")
		50	#define wmb() asm volatile ("sfence")
		51
3266	Serge	52	struct drm_i915_gem_object *get_fb_obj();
		53
3263	Serge	54	unsigned long vm_mmap(struct file *file, unsigned long addr,
		55	unsigned long len, unsigned long prot,
		56	unsigned long flag, unsigned long offset);
		57
2344	Serge	58	static inline void clflush(volatile void *__p)
		59	{
		60	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		61	}
		62
2332	Serge	63	#define MAX_ERRNO 4095
		64
		65	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		66
		67
2326	Serge	68	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		69	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		70	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		71	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		72
2332	Serge	73	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		74	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		75	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		76	unsigned alignment,
3031	serge	77	bool map_and_fenceable,
		78	bool nonblocking);
2332	Serge	79	static int i915_gem_phys_pwrite(struct drm_device *dev,
		80	struct drm_i915_gem_object *obj,
		81	struct drm_i915_gem_pwrite *args,
		82	struct drm_file *file);
2326	Serge	83
3031	serge	84	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		85	struct drm_i915_gem_object *obj);
		86	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		87	struct drm_i915_fence_reg *fence,
		88	bool enable);
2332	Serge	89
3031	serge	90	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		91	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		92	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		93
		94	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		95	{
		96	if (obj->tiling_mode)
		97	i915_gem_release_mmap(obj);
		98
		99	/* As we do not have an associated fence register, we will force
		100	* a tiling change if we ever need to acquire one.
		101	*/
		102	obj->fence_dirty = false;
		103	obj->fence_reg = I915_FENCE_REG_NONE;
		104	}
		105
2332	Serge	106	/* some bookkeeping */
		107	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		108	size_t size)
		109	{
		110	dev_priv->mm.object_count++;
		111	dev_priv->mm.object_memory += size;
		112	}
		113
		114	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		115	size_t size)
		116	{
		117	dev_priv->mm.object_count--;
		118	dev_priv->mm.object_memory -= size;
		119	}
		120
		121	static int
3480	Serge	122	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	123	{
		124	int ret;
		125
3480	Serge	126	#define EXIT_COND (!i915_reset_in_progress(error))
		127	if (EXIT_COND)
2332	Serge	128	return 0;
3255	Serge	129	#if 0
3031	serge	130	/*
		131	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		132	* userspace. If it takes that long something really bad is going on and
		133	* we should simply try to bail out and fail as gracefully as possible.
		134	*/
3480	Serge	135	ret = wait_event_interruptible_timeout(error->reset_queue,
		136	EXIT_COND,
		137	10*HZ);
3031	serge	138	if (ret == 0) {
		139	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		140	return -EIO;
		141	} else if (ret < 0) {
2332	Serge	142	return ret;
3031	serge	143	}
2332	Serge	144
3255	Serge	145	#endif
3480	Serge	146	#undef EXIT_COND
3255	Serge	147
2332	Serge	148	return 0;
		149	}
		150
		151	int i915_mutex_lock_interruptible(struct drm_device *dev)
		152	{
3480	Serge	153	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	154	int ret;
		155
3480	Serge	156	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	157	if (ret)
		158	return ret;
		159
3480	Serge	160	ret = mutex_lock_interruptible(&dev->struct_mutex);
		161	if (ret)
		162	return ret;
2332	Serge	163
		164	WARN_ON(i915_verify_lists(dev));
		165	return 0;
		166	}
		167
		168	static inline bool
		169	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		170	{
3031	serge	171	return obj->gtt_space && !obj->active;
2332	Serge	172	}
		173
		174
		175	#if 0
		176
		177	int
		178	i915_gem_init_ioctl(struct drm_device dev, void data,
		179	struct drm_file *file)
		180	{
3480	Serge	181	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	182	struct drm_i915_gem_init *args = data;
		183
3031	serge	184	if (drm_core_check_feature(dev, DRIVER_MODESET))
		185	return -ENODEV;
		186
2332	Serge	187	if (args->gtt_start >= args->gtt_end \|\|
		188	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		189	return -EINVAL;
		190
3031	serge	191	/* GEM with user mode setting was never supported on ilk and later. */
		192	if (INTEL_INFO(dev)->gen >= 5)
		193	return -ENODEV;
		194
2332	Serge	195	mutex_lock(&dev->struct_mutex);
3480	Serge	196	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		197	args->gtt_end);
		198	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	199	mutex_unlock(&dev->struct_mutex);
		200
		201	return 0;
		202	}
2351	Serge	203	#endif
2332	Serge	204
		205	int
		206	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		207	struct drm_file *file)
		208	{
		209	struct drm_i915_private *dev_priv = dev->dev_private;
		210	struct drm_i915_gem_get_aperture *args = data;
		211	struct drm_i915_gem_object *obj;
		212	size_t pinned;
		213
		214	pinned = 0;
		215	mutex_lock(&dev->struct_mutex);
3031	serge	216	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		217	if (obj->pin_count)
2332	Serge	218	pinned += obj->gtt_space->size;
		219	mutex_unlock(&dev->struct_mutex);
		220
3480	Serge	221	args->aper_size = dev_priv->gtt.total;
2342	Serge	222	args->aper_available_size = args->aper_size - pinned;
2332	Serge	223
		224	return 0;
		225	}
		226
3480	Serge	227	void i915_gem_object_alloc(struct drm_device dev)
		228	{
		229	struct drm_i915_private *dev_priv = dev->dev_private;
		230	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		231	}
		232
		233	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		234	{
		235	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		236	kfree(obj);
		237	}
		238
3031	serge	239	static int
		240	i915_gem_create(struct drm_file *file,
2332	Serge	241	struct drm_device *dev,
		242	uint64_t size,
		243	uint32_t *handle_p)
		244	{
		245	struct drm_i915_gem_object *obj;
		246	int ret;
		247	u32 handle;
		248
		249	size = roundup(size, PAGE_SIZE);
2342	Serge	250	if (size == 0)
		251	return -EINVAL;
2332	Serge	252
		253	/* Allocate the new object */
		254	obj = i915_gem_alloc_object(dev, size);
		255	if (obj == NULL)
		256	return -ENOMEM;
		257
		258	ret = drm_gem_handle_create(file, &obj->base, &handle);
		259	if (ret) {
		260	drm_gem_object_release(&obj->base);
		261	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		262	kfree(obj);
		263	return ret;
		264	}
		265
		266	/* drop reference from allocate - handle holds it now */
		267	drm_gem_object_unreference(&obj->base);
2351	Serge	268	trace_i915_gem_object_create(obj);
2332	Serge	269
		270	*handle_p = handle;
		271	return 0;
		272	}
		273
		274	int
		275	i915_gem_dumb_create(struct drm_file *file,
		276	struct drm_device *dev,
		277	struct drm_mode_create_dumb *args)
		278	{
		279	/* have to work out size/pitch and return them */
		280	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		281	args->size = args->pitch * args->height;
		282	return i915_gem_create(file, dev,
		283	args->size, &args->handle);
		284	}
		285
		286	int i915_gem_dumb_destroy(struct drm_file *file,
		287	struct drm_device *dev,
		288	uint32_t handle)
		289	{
		290	return drm_gem_handle_delete(file, handle);
		291	}
		292
2326	Serge	293	/**
2332	Serge	294	* Creates a new mm object and returns a handle to it.
		295	*/
		296	int
		297	i915_gem_create_ioctl(struct drm_device dev, void data,
		298	struct drm_file *file)
		299	{
		300	struct drm_i915_gem_create *args = data;
3031	serge	301
2332	Serge	302	return i915_gem_create(file, dev,
		303	args->size, &args->handle);
		304	}
		305
		306
3260	Serge	307	#if 0
2332	Serge	308
3031	serge	309	static inline int
		310	__copy_to_user_swizzled(char __user *cpu_vaddr,
		311	const char *gpu_vaddr, int gpu_offset,
2332	Serge	312	int length)
		313	{
3031	serge	314	int ret, cpu_offset = 0;
2332	Serge	315
3031	serge	316	while (length > 0) {
		317	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		318	int this_length = min(cacheline_end - gpu_offset, length);
		319	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	320
3031	serge	321	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		322	gpu_vaddr + swizzled_gpu_offset,
		323	this_length);
		324	if (ret)
		325	return ret + length;
2332	Serge	326
3031	serge	327	cpu_offset += this_length;
		328	gpu_offset += this_length;
		329	length -= this_length;
		330	}
		331
		332	return 0;
2332	Serge	333	}
		334
3031	serge	335	static inline int
		336	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		337	const char __user *cpu_vaddr,
		338	int length)
2332	Serge	339	{
3031	serge	340	int ret, cpu_offset = 0;
2332	Serge	341
		342	while (length > 0) {
		343	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		344	int this_length = min(cacheline_end - gpu_offset, length);
		345	int swizzled_gpu_offset = gpu_offset ^ 64;
		346
3031	serge	347	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	348	cpu_vaddr + cpu_offset,
		349	this_length);
3031	serge	350	if (ret)
		351	return ret + length;
		352
2332	Serge	353	cpu_offset += this_length;
		354	gpu_offset += this_length;
		355	length -= this_length;
		356	}
		357
3031	serge	358	return 0;
2332	Serge	359	}
		360
3031	serge	361	/* Per-page copy function for the shmem pread fastpath.
		362	* Flushes invalid cachelines before reading the target if
		363	* needs_clflush is set. */
2332	Serge	364	static int
3031	serge	365	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		366	char __user *user_data,
		367	bool page_do_bit17_swizzling, bool needs_clflush)
		368	{
		369	char *vaddr;
		370	int ret;
		371
		372	if (unlikely(page_do_bit17_swizzling))
		373	return -EINVAL;
		374
		375	vaddr = kmap_atomic(page);
		376	if (needs_clflush)
		377	drm_clflush_virt_range(vaddr + shmem_page_offset,
		378	page_length);
		379	ret = __copy_to_user_inatomic(user_data,
		380	vaddr + shmem_page_offset,
		381	page_length);
		382	kunmap_atomic(vaddr);
		383
		384	return ret ? -EFAULT : 0;
		385	}
		386
		387	static void
		388	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		389	bool swizzled)
		390	{
		391	if (unlikely(swizzled)) {
		392	unsigned long start = (unsigned long) addr;
		393	unsigned long end = (unsigned long) addr + length;
		394
		395	/* For swizzling simply ensure that we always flush both
		396	* channels. Lame, but simple and it works. Swizzled
		397	* pwrite/pread is far from a hotpath - current userspace
		398	* doesn't use it at all. */
		399	start = round_down(start, 128);
		400	end = round_up(end, 128);
		401
		402	drm_clflush_virt_range((void *)start, end - start);
		403	} else {
		404	drm_clflush_virt_range(addr, length);
		405	}
		406
		407	}
		408
		409	/* Only difference to the fast-path function is that this can handle bit17
		410	* and uses non-atomic copy and kmap functions. */
		411	static int
		412	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		413	char __user *user_data,
		414	bool page_do_bit17_swizzling, bool needs_clflush)
		415	{
		416	char *vaddr;
		417	int ret;
		418
		419	vaddr = kmap(page);
		420	if (needs_clflush)
		421	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		422	page_length,
		423	page_do_bit17_swizzling);
		424
		425	if (page_do_bit17_swizzling)
		426	ret = __copy_to_user_swizzled(user_data,
		427	vaddr, shmem_page_offset,
		428	page_length);
		429	else
		430	ret = __copy_to_user(user_data,
		431	vaddr + shmem_page_offset,
		432	page_length);
		433	kunmap(page);
		434
		435	return ret ? - EFAULT : 0;
		436	}
		437
		438	static int
		439	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	440	struct drm_i915_gem_object *obj,
		441	struct drm_i915_gem_pread *args,
		442	struct drm_file *file)
		443	{
3031	serge	444	char __user *user_data;
2332	Serge	445	ssize_t remain;
		446	loff_t offset;
3031	serge	447	int shmem_page_offset, page_length, ret = 0;
		448	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		449	int prefaulted = 0;
		450	int needs_clflush = 0;
3746	Serge	451	struct sg_page_iter sg_iter;
2332	Serge	452
3746	Serge	453	user_data = to_user_ptr(args->data_ptr);
2332	Serge	454	remain = args->size;
		455
3031	serge	456	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		457
		458	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		459	/* If we're not in the cpu read domain, set ourself into the gtt
		460	* read domain and manually flush cachelines (if required). This
		461	* optimizes for the case when the gpu will dirty the data
		462	* anyway again before the next pread happens. */
		463	if (obj->cache_level == I915_CACHE_NONE)
		464	needs_clflush = 1;
		465	if (obj->gtt_space) {
		466	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		467	if (ret)
		468	return ret;
		469	}
		470	}
		471
		472	ret = i915_gem_object_get_pages(obj);
		473	if (ret)
		474	return ret;
		475
		476	i915_gem_object_pin_pages(obj);
		477
2332	Serge	478	offset = args->offset;
		479
3746	Serge	480	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		481	offset >> PAGE_SHIFT) {
		482	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	483
3031	serge	484	if (remain <= 0)
		485	break;
		486
2332	Serge	487	/* Operation in this page
		488	*
3031	serge	489	* shmem_page_offset = offset within page in shmem file
2332	Serge	490	* page_length = bytes to copy for this page
		491	*/
3031	serge	492	shmem_page_offset = offset_in_page(offset);
2332	Serge	493	page_length = remain;
3031	serge	494	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		495	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	496
3031	serge	497	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		498	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	499
3031	serge	500	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		501	user_data, page_do_bit17_swizzling,
		502	needs_clflush);
		503	if (ret == 0)
		504	goto next_page;
2332	Serge	505
3031	serge	506	mutex_unlock(&dev->struct_mutex);
		507
		508	if (!prefaulted) {
		509	ret = fault_in_multipages_writeable(user_data, remain);
		510	/* Userspace is tricking us, but we've already clobbered
		511	* its pages with the prefault and promised to write the
		512	* data up to the first fault. Hence ignore any errors
		513	* and just continue. */
		514	(void)ret;
		515	prefaulted = 1;
		516	}
		517
		518	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		519	user_data, page_do_bit17_swizzling,
		520	needs_clflush);
		521
		522	mutex_lock(&dev->struct_mutex);
		523
		524	next_page:
2332	Serge	525	mark_page_accessed(page);
3031	serge	526
2332	Serge	527	if (ret)
3031	serge	528	goto out;
2332	Serge	529
		530	remain -= page_length;
		531	user_data += page_length;
		532	offset += page_length;
		533	}
		534
3031	serge	535	out:
		536	i915_gem_object_unpin_pages(obj);
		537
		538	return ret;
2332	Serge	539	}
		540
		541	/**
3031	serge	542	* Reads data from the object referenced by handle.
		543	*
		544	* On error, the contents of *data are undefined.
2332	Serge	545	*/
3031	serge	546	int
		547	i915_gem_pread_ioctl(struct drm_device dev, void data,
		548	struct drm_file *file)
		549	{
		550	struct drm_i915_gem_pread *args = data;
		551	struct drm_i915_gem_object *obj;
		552	int ret = 0;
		553
		554	if (args->size == 0)
		555	return 0;
		556
		557	if (!access_ok(VERIFY_WRITE,
3746	Serge	558	to_user_ptr(args->data_ptr),
3031	serge	559	args->size))
		560	return -EFAULT;
		561
		562	ret = i915_mutex_lock_interruptible(dev);
		563	if (ret)
		564	return ret;
		565
		566	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		567	if (&obj->base == NULL) {
		568	ret = -ENOENT;
		569	goto unlock;
		570	}
		571
		572	/* Bounds check source. */
		573	if (args->offset > obj->base.size \|\|
		574	args->size > obj->base.size - args->offset) {
		575	ret = -EINVAL;
		576	goto out;
		577	}
		578
		579	/* prime objects have no backing filp to GEM pread/pwrite
		580	* pages from.
		581	*/
		582	if (!obj->base.filp) {
		583	ret = -EINVAL;
		584	goto out;
		585	}
		586
		587	trace_i915_gem_object_pread(obj, args->offset, args->size);
		588
		589	ret = i915_gem_shmem_pread(dev, obj, args, file);
		590
		591	out:
		592	drm_gem_object_unreference(&obj->base);
		593	unlock:
		594	mutex_unlock(&dev->struct_mutex);
		595	return ret;
		596	}
		597
		598	/* This is the fast write path which cannot handle
		599	* page faults in the source data
		600	*/
		601
		602	static inline int
		603	fast_user_write(struct io_mapping *mapping,
		604	loff_t page_base, int page_offset,
		605	char __user *user_data,
		606	int length)
		607	{
		608	void __iomem *vaddr_atomic;
		609	void *vaddr;
		610	unsigned long unwritten;
		611
		612	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		613	/* We can use the cpu mem copy function because this is X86. */
		614	vaddr = (void __force*)vaddr_atomic + page_offset;
		615	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		616	user_data, length);
		617	io_mapping_unmap_atomic(vaddr_atomic);
		618	return unwritten;
		619	}
3260	Serge	620	#endif
3031	serge	621
3260	Serge	622	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	623	/**
		624	* This is the fast pwrite path, where we copy the data directly from the
		625	* user into the GTT, uncached.
		626	*/
2332	Serge	627	static int
3031	serge	628	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		629	struct drm_i915_gem_object *obj,
		630	struct drm_i915_gem_pwrite *args,
		631	struct drm_file *file)
2332	Serge	632	{
3031	serge	633	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	634	ssize_t remain;
3031	serge	635	loff_t offset, page_base;
		636	char __user *user_data;
		637	int page_offset, page_length, ret;
3260	Serge	638	char *vaddr;
2332	Serge	639
3031	serge	640	ret = i915_gem_object_pin(obj, 0, true, true);
		641	if (ret)
		642	goto out;
		643
		644	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		645	if (ret)
		646	goto out_unpin;
		647
		648	ret = i915_gem_object_put_fence(obj);
		649	if (ret)
		650	goto out_unpin;
		651
3260	Serge	652	vaddr = AllocKernelSpace(4096);
		653	if(vaddr == NULL)
		654	{
		655	ret = -ENOSPC;
		656	goto out_unpin;
		657	};
		658
3031	serge	659	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	660	remain = args->size;
		661
3031	serge	662	offset = obj->gtt_offset + args->offset;
2332	Serge	663
3031	serge	664	while (remain > 0) {
		665	/* Operation in this page
		666	*
		667	* page_base = page offset within aperture
		668	* page_offset = offset within page
		669	* page_length = bytes to copy for this page
		670	*/
		671	page_base = offset & PAGE_MASK;
		672	page_offset = offset_in_page(offset);
		673	page_length = remain;
		674	if ((page_offset + remain) > PAGE_SIZE)
		675	page_length = PAGE_SIZE - page_offset;
2332	Serge	676
3260	Serge	677	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	678
3260	Serge	679	memcpy(vaddr+page_offset, user_data, page_length);
		680
3031	serge	681	remain -= page_length;
		682	user_data += page_length;
		683	offset += page_length;
2332	Serge	684	}
		685
3260	Serge	686	FreeKernelSpace(vaddr);
		687
3031	serge	688	out_unpin:
		689	i915_gem_object_unpin(obj);
		690	out:
		691	return ret;
		692	}
		693
		694	/* Per-page copy function for the shmem pwrite fastpath.
		695	* Flushes invalid cachelines before writing to the target if
		696	* needs_clflush_before is set and flushes out any written cachelines after
		697	* writing if needs_clflush is set. */
		698	static int
		699	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		700	char __user *user_data,
		701	bool page_do_bit17_swizzling,
		702	bool needs_clflush_before,
		703	bool needs_clflush_after)
		704	{
		705	char *vaddr;
3260	Serge	706	int ret = 0;
3031	serge	707
		708	if (unlikely(page_do_bit17_swizzling))
		709	return -EINVAL;
		710
3260	Serge	711	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	712	if (needs_clflush_before)
		713	drm_clflush_virt_range(vaddr + shmem_page_offset,
		714	page_length);
3260	Serge	715	memcpy(vaddr + shmem_page_offset,
3031	serge	716	user_data,
		717	page_length);
		718	if (needs_clflush_after)
		719	drm_clflush_virt_range(vaddr + shmem_page_offset,
		720	page_length);
3260	Serge	721	FreeKernelSpace(vaddr);
3031	serge	722
		723	return ret ? -EFAULT : 0;
		724	}
3260	Serge	725	#if 0
3031	serge	726
		727	/* Only difference to the fast-path function is that this can handle bit17
		728	* and uses non-atomic copy and kmap functions. */
		729	static int
		730	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		731	char __user *user_data,
		732	bool page_do_bit17_swizzling,
		733	bool needs_clflush_before,
		734	bool needs_clflush_after)
		735	{
		736	char *vaddr;
		737	int ret;
		738
		739	vaddr = kmap(page);
		740	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		741	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		742	page_length,
		743	page_do_bit17_swizzling);
		744	if (page_do_bit17_swizzling)
		745	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		746	user_data,
		747	page_length);
		748	else
		749	ret = __copy_from_user(vaddr + shmem_page_offset,
		750	user_data,
		751	page_length);
		752	if (needs_clflush_after)
		753	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		754	page_length,
		755	page_do_bit17_swizzling);
		756	kunmap(page);
		757
		758	return ret ? -EFAULT : 0;
		759	}
3260	Serge	760	#endif
3031	serge	761
3260	Serge	762
3031	serge	763	static int
		764	i915_gem_shmem_pwrite(struct drm_device *dev,
		765	struct drm_i915_gem_object *obj,
		766	struct drm_i915_gem_pwrite *args,
		767	struct drm_file *file)
		768	{
		769	ssize_t remain;
		770	loff_t offset;
		771	char __user *user_data;
		772	int shmem_page_offset, page_length, ret = 0;
		773	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		774	int hit_slowpath = 0;
		775	int needs_clflush_after = 0;
		776	int needs_clflush_before = 0;
3746	Serge	777	struct sg_page_iter sg_iter;
3031	serge	778
3746	Serge	779	user_data = to_user_ptr(args->data_ptr);
3031	serge	780	remain = args->size;
		781
		782	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		783
		784	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		785	/* If we're not in the cpu write domain, set ourself into the gtt
		786	* write domain and manually flush cachelines (if required). This
		787	* optimizes for the case when the gpu will use the data
		788	* right away and we therefore have to clflush anyway. */
		789	if (obj->cache_level == I915_CACHE_NONE)
		790	needs_clflush_after = 1;
		791	if (obj->gtt_space) {
		792	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		793	if (ret)
		794	return ret;
		795	}
		796	}
		797	/* Same trick applies for invalidate partially written cachelines before
		798	* writing. */
		799	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		800	&& obj->cache_level == I915_CACHE_NONE)
		801	needs_clflush_before = 1;
		802
		803	ret = i915_gem_object_get_pages(obj);
2332	Serge	804	if (ret)
3031	serge	805	return ret;
2332	Serge	806
3031	serge	807	i915_gem_object_pin_pages(obj);
2332	Serge	808
		809	offset = args->offset;
3031	serge	810	obj->dirty = 1;
2332	Serge	811
3746	Serge	812	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		813	offset >> PAGE_SHIFT) {
		814	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	815	int partial_cacheline_write;
2332	Serge	816
3031	serge	817	if (remain <= 0)
		818	break;
		819
2332	Serge	820	/* Operation in this page
		821	*
		822	* shmem_page_offset = offset within page in shmem file
		823	* page_length = bytes to copy for this page
		824	*/
		825	shmem_page_offset = offset_in_page(offset);
		826
		827	page_length = remain;
		828	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		829	page_length = PAGE_SIZE - shmem_page_offset;
		830
3031	serge	831	/* If we don't overwrite a cacheline completely we need to be
		832	* careful to have up-to-date data by first clflushing. Don't
		833	* overcomplicate things and flush the entire patch. */
		834	partial_cacheline_write = needs_clflush_before &&
		835	((shmem_page_offset \| page_length)
3260	Serge	836	& (x86_clflush_size - 1));
2332	Serge	837
3031	serge	838	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		839	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	840
3031	serge	841	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		842	user_data, page_do_bit17_swizzling,
		843	partial_cacheline_write,
		844	needs_clflush_after);
		845	if (ret == 0)
		846	goto next_page;
		847
		848	hit_slowpath = 1;
		849	mutex_unlock(&dev->struct_mutex);
3260	Serge	850	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	851
3260	Serge	852	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		853	// user_data, page_do_bit17_swizzling,
		854	// partial_cacheline_write,
		855	// needs_clflush_after);
		856
3031	serge	857	mutex_lock(&dev->struct_mutex);
		858
		859	next_page:
2332	Serge	860
3031	serge	861	if (ret)
		862	goto out;
		863
2332	Serge	864	remain -= page_length;
3031	serge	865	user_data += page_length;
2332	Serge	866	offset += page_length;
		867	}
		868
		869	out:
3031	serge	870	i915_gem_object_unpin_pages(obj);
		871
		872	if (hit_slowpath) {
3480	Serge	873	/*
		874	* Fixup: Flush cpu caches in case we didn't flush the dirty
		875	* cachelines in-line while writing and the object moved
		876	* out of the cpu write domain while we've dropped the lock.
		877	*/
		878	if (!needs_clflush_after &&
		879	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3031	serge	880	i915_gem_clflush_object(obj);
3243	Serge	881	i915_gem_chipset_flush(dev);
3031	serge	882	}
2332	Serge	883	}
		884
3031	serge	885	if (needs_clflush_after)
3243	Serge	886	i915_gem_chipset_flush(dev);
3031	serge	887
2332	Serge	888	return ret;
		889	}
3031	serge	890
		891	/**
		892	* Writes data to the object referenced by handle.
		893	*
		894	* On error, the contents of the buffer that were to be modified are undefined.
		895	*/
		896	int
		897	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		898	struct drm_file *file)
		899	{
		900	struct drm_i915_gem_pwrite *args = data;
		901	struct drm_i915_gem_object *obj;
		902	int ret;
		903
3480	Serge	904	if(args->handle == -2)
		905	{
		906	printf("%s handle %d\n", __FUNCTION__, args->handle);
		907	return 0;
		908	}
		909
3031	serge	910	if (args->size == 0)
		911	return 0;
		912
		913	ret = i915_mutex_lock_interruptible(dev);
		914	if (ret)
		915	return ret;
		916
		917	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		918	if (&obj->base == NULL) {
		919	ret = -ENOENT;
		920	goto unlock;
		921	}
		922
		923	/* Bounds check destination. */
		924	if (args->offset > obj->base.size \|\|
		925	args->size > obj->base.size - args->offset) {
		926	ret = -EINVAL;
		927	goto out;
		928	}
		929
		930	/* prime objects have no backing filp to GEM pread/pwrite
		931	* pages from.
		932	*/
		933	if (!obj->base.filp) {
		934	ret = -EINVAL;
		935	goto out;
		936	}
		937
		938	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		939
		940	ret = -EFAULT;
		941	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		942	* it would end up going through the fenced access, and we'll get
		943	* different detiling behavior between reading and writing.
		944	* pread/pwrite currently are reading and writing from the CPU
		945	* perspective, requiring manual detiling by the client.
		946	*/
3260	Serge	947	// if (obj->phys_obj) {
		948	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		949	// goto out;
		950	// }
3031	serge	951
		952	if (obj->cache_level == I915_CACHE_NONE &&
		953	obj->tiling_mode == I915_TILING_NONE &&
		954	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		955	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		956	/* Note that the gtt paths might fail with non-page-backed user
		957	* pointers (e.g. gtt mappings when moving data between
		958	* textures). Fallback to the shmem path in that case. */
		959	}
		960
		961	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	962	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	963
		964	out:
		965	drm_gem_object_unreference(&obj->base);
		966	unlock:
		967	mutex_unlock(&dev->struct_mutex);
		968	return ret;
		969	}
		970
		971	int
3480	Serge	972	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	973	bool interruptible)
		974	{
3480	Serge	975	if (i915_reset_in_progress(error)) {
3031	serge	976	/* Non-interruptible callers can't handle -EAGAIN, hence return
		977	* -EIO unconditionally for these. */
		978	if (!interruptible)
		979	return -EIO;
2332	Serge	980
3480	Serge	981	/* Recovery complete, but the reset failed ... */
		982	if (i915_terminally_wedged(error))
3031	serge	983	return -EIO;
2332	Serge	984
3031	serge	985	return -EAGAIN;
		986	}
2332	Serge	987
3031	serge	988	return 0;
		989	}
2332	Serge	990
3031	serge	991	/*
		992	* Compare seqno against outstanding lazy request. Emit a request if they are
		993	* equal.
		994	*/
		995	static int
		996	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		997	{
		998	int ret;
2332	Serge	999
3031	serge	1000	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1001
3031	serge	1002	ret = 0;
		1003	if (seqno == ring->outstanding_lazy_request)
		1004	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1005
3031	serge	1006	return ret;
		1007	}
2332	Serge	1008
3031	serge	1009	/**
		1010	* __wait_seqno - wait until execution of seqno has finished
		1011	* @ring: the ring expected to report seqno
		1012	* @seqno: duh!
3480	Serge	1013	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1014	* @interruptible: do an interruptible wait (normally yes)
		1015	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1016	*
3480	Serge	1017	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1018	* values have been read by the caller in an smp safe manner. Where read-side
		1019	* locks are involved, it is sufficient to read the reset_counter before
		1020	* unlocking the lock that protects the seqno. For lockless tricks, the
		1021	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1022	* inserted.
		1023	*
3031	serge	1024	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1025	* errno with remaining time filled in timeout argument.
		1026	*/
		1027	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1028	unsigned reset_counter,
3031	serge	1029	bool interruptible, struct timespec *timeout)
		1030	{
		1031	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1032	struct timespec before, now, wait_time={1,0};
		1033	unsigned long timeout_jiffies;
		1034	long end;
		1035	bool wait_forever = true;
		1036	int ret;
2332	Serge	1037
3031	serge	1038	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1039	return 0;
2332	Serge	1040
3031	serge	1041	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1042
3031	serge	1043	if (timeout != NULL) {
		1044	wait_time = *timeout;
		1045	wait_forever = false;
		1046	}
2332	Serge	1047
3266	Serge	1048	timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1049
3031	serge	1050	if (WARN_ON(!ring->irq_get(ring)))
		1051	return -ENODEV;
2332	Serge	1052
3031	serge	1053	/* Record current time in case interrupted by signal, or wedged * */
		1054	getrawmonotonic(&before);
2332	Serge	1055
3031	serge	1056	#define EXIT_COND \
		1057	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1058	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1059	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1060	do {
3266	Serge	1061	if (interruptible)
		1062	end = wait_event_interruptible_timeout(ring->irq_queue,
		1063	EXIT_COND,
		1064	timeout_jiffies);
		1065	else
3031	serge	1066	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1067	timeout_jiffies);
2332	Serge	1068
3480	Serge	1069	/* We need to check whether any gpu reset happened in between
		1070	* the caller grabbing the seqno and now ... */
		1071	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1072	end = -EAGAIN;
		1073
		1074	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1075	* gone. */
		1076	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1077	if (ret)
		1078	end = ret;
		1079	} while (end == 0 && wait_forever);
2332	Serge	1080
3031	serge	1081	getrawmonotonic(&now);
2332	Serge	1082
3031	serge	1083	ring->irq_put(ring);
		1084	trace_i915_gem_request_wait_end(ring, seqno);
		1085	#undef EXIT_COND
2332	Serge	1086
3031	serge	1087	if (timeout) {
		1088	// struct timespec sleep_time = timespec_sub(now, before);
		1089	// timeout = timespec_sub(timeout, sleep_time);
		1090	}
2332	Serge	1091
3031	serge	1092	switch (end) {
		1093	case -EIO:
		1094	case -EAGAIN: /* Wedged */
		1095	case -ERESTARTSYS: /* Signal */
		1096	return (int)end;
		1097	case 0: /* Timeout */
		1098	return -ETIME;
		1099	default: /* Completed */
		1100	WARN_ON(end < 0); /* We're not aware of other errors */
		1101	return 0;
		1102	}
		1103	}
2332	Serge	1104
3031	serge	1105	/**
		1106	* Waits for a sequence number to be signaled, and cleans up the
		1107	* request and object lists appropriately for that event.
		1108	*/
		1109	int
		1110	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1111	{
		1112	struct drm_device *dev = ring->dev;
		1113	struct drm_i915_private *dev_priv = dev->dev_private;
		1114	bool interruptible = dev_priv->mm.interruptible;
		1115	int ret;
2332	Serge	1116
3031	serge	1117	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1118	BUG_ON(seqno == 0);
2332	Serge	1119
3480	Serge	1120	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1121	if (ret)
		1122	return ret;
2332	Serge	1123
3031	serge	1124	ret = i915_gem_check_olr(ring, seqno);
		1125	if (ret)
		1126	return ret;
2332	Serge	1127
3480	Serge	1128	return __wait_seqno(ring, seqno,
		1129	atomic_read(&dev_priv->gpu_error.reset_counter),
		1130	interruptible, NULL);
3031	serge	1131	}
2332	Serge	1132
3031	serge	1133	/**
		1134	* Ensures that all rendering to the object has completed and the object is
		1135	* safe to unbind from the GTT or access from the CPU.
		1136	*/
		1137	static __must_check int
		1138	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1139	bool readonly)
		1140	{
		1141	struct intel_ring_buffer *ring = obj->ring;
		1142	u32 seqno;
		1143	int ret;
2332	Serge	1144
3031	serge	1145	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1146	if (seqno == 0)
		1147	return 0;
2332	Serge	1148
3031	serge	1149	ret = i915_wait_seqno(ring, seqno);
		1150	if (ret)
		1151	return ret;
2332	Serge	1152
3031	serge	1153	i915_gem_retire_requests_ring(ring);
2332	Serge	1154
3031	serge	1155	/* Manually manage the write flush as we may have not yet
		1156	* retired the buffer.
		1157	*/
		1158	if (obj->last_write_seqno &&
		1159	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1160	obj->last_write_seqno = 0;
		1161	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1162	}
2332	Serge	1163
3031	serge	1164	return 0;
		1165	}
2332	Serge	1166
3260	Serge	1167	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1168	* as the object state may change during this call.
		1169	*/
		1170	static __must_check int
		1171	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1172	bool readonly)
		1173	{
		1174	struct drm_device *dev = obj->base.dev;
		1175	struct drm_i915_private *dev_priv = dev->dev_private;
		1176	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1177	unsigned reset_counter;
3260	Serge	1178	u32 seqno;
		1179	int ret;
2332	Serge	1180
3260	Serge	1181	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1182	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1183
3260	Serge	1184	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1185	if (seqno == 0)
		1186	return 0;
2332	Serge	1187
3480	Serge	1188	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1189	if (ret)
		1190	return ret;
2332	Serge	1191
3260	Serge	1192	ret = i915_gem_check_olr(ring, seqno);
		1193	if (ret)
		1194	return ret;
2332	Serge	1195
3480	Serge	1196	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1197	mutex_unlock(&dev->struct_mutex);
3480	Serge	1198	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1199	mutex_lock(&dev->struct_mutex);
2332	Serge	1200
3260	Serge	1201	i915_gem_retire_requests_ring(ring);
2332	Serge	1202
3260	Serge	1203	/* Manually manage the write flush as we may have not yet
		1204	* retired the buffer.
		1205	*/
		1206	if (obj->last_write_seqno &&
		1207	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1208	obj->last_write_seqno = 0;
		1209	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1210	}
2332	Serge	1211
3260	Serge	1212	return ret;
		1213	}
2332	Serge	1214
3260	Serge	1215	/**
		1216	* Called when user space prepares to use an object with the CPU, either
		1217	* through the mmap ioctl's mapping or a GTT mapping.
		1218	*/
		1219	int
		1220	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1221	struct drm_file *file)
		1222	{
		1223	struct drm_i915_gem_set_domain *args = data;
		1224	struct drm_i915_gem_object *obj;
		1225	uint32_t read_domains = args->read_domains;
		1226	uint32_t write_domain = args->write_domain;
		1227	int ret;
2332	Serge	1228
3480	Serge	1229
		1230	if(args->handle == -2)
		1231	{
		1232	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1233	return 0;
		1234	}
		1235
3260	Serge	1236	/* Only handle setting domains to types used by the CPU. */
		1237	if (write_domain & I915_GEM_GPU_DOMAINS)
		1238	return -EINVAL;
2332	Serge	1239
3260	Serge	1240	if (read_domains & I915_GEM_GPU_DOMAINS)
		1241	return -EINVAL;
2332	Serge	1242
3260	Serge	1243	/* Having something in the write domain implies it's in the read
		1244	* domain, and only that read domain. Enforce that in the request.
		1245	*/
		1246	if (write_domain != 0 && read_domains != write_domain)
		1247	return -EINVAL;
2332	Serge	1248
3260	Serge	1249	ret = i915_mutex_lock_interruptible(dev);
		1250	if (ret)
		1251	return ret;
2332	Serge	1252
3260	Serge	1253	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1254	if (&obj->base == NULL) {
		1255	ret = -ENOENT;
		1256	goto unlock;
		1257	}
2332	Serge	1258
3260	Serge	1259	/* Try to flush the object off the GPU without holding the lock.
		1260	* We will repeat the flush holding the lock in the normal manner
		1261	* to catch cases where we are gazumped.
		1262	*/
		1263	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1264	if (ret)
		1265	goto unref;
2332	Serge	1266
3260	Serge	1267	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1268	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1269
3260	Serge	1270	/* Silently promote "you're not bound, there was nothing to do"
		1271	* to success, since the client was just asking us to
		1272	* make sure everything was done.
		1273	*/
		1274	if (ret == -EINVAL)
		1275	ret = 0;
		1276	} else {
		1277	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1278	}
2332	Serge	1279
3260	Serge	1280	unref:
		1281	drm_gem_object_unreference(&obj->base);
		1282	unlock:
		1283	mutex_unlock(&dev->struct_mutex);
		1284	return ret;
		1285	}
2332	Serge	1286
		1287
		1288
		1289
		1290
		1291
3260	Serge	1292	/**
		1293	* Maps the contents of an object, returning the address it is mapped
		1294	* into.
		1295	*
		1296	* While the mapping holds a reference on the contents of the object, it doesn't
		1297	* imply a ref on the object itself.
		1298	*/
		1299	int
		1300	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1301	struct drm_file *file)
		1302	{
		1303	struct drm_i915_gem_mmap *args = data;
		1304	struct drm_gem_object *obj;
		1305	unsigned long addr = 0;
2332	Serge	1306
3480	Serge	1307	if(args->handle == -2)
		1308	{
		1309	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1310	return 0;
		1311	}
		1312
3260	Serge	1313	obj = drm_gem_object_lookup(dev, file, args->handle);
		1314	if (obj == NULL)
		1315	return -ENOENT;
		1316
3298	Serge	1317	//dbgprintf("%s offset %lld size %lld\n",
		1318	// __FUNCTION__, args->offset, args->size);
3260	Serge	1319	/* prime objects have no backing filp to GEM mmap
		1320	* pages from.
		1321	*/
		1322	if (!obj->filp) {
		1323	drm_gem_object_unreference_unlocked(obj);
		1324	return -EINVAL;
		1325	}
2332	Serge	1326
3263	Serge	1327	addr = vm_mmap(obj->filp, 0, args->size,
		1328	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1329	args->offset);
3260	Serge	1330	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1331	if (IS_ERR((void *)addr))
		1332	return addr;
2332	Serge	1333
3260	Serge	1334	args->addr_ptr = (uint64_t) addr;
2332	Serge	1335
3263	Serge	1336	return 0;
3260	Serge	1337	}
2332	Serge	1338
		1339
		1340
		1341
		1342
		1343
		1344
		1345
3031	serge	1346
		1347
		1348
		1349
		1350
		1351	/**
		1352	* i915_gem_release_mmap - remove physical page mappings
		1353	* @obj: obj in question
		1354	*
		1355	* Preserve the reservation of the mmapping with the DRM core code, but
		1356	* relinquish ownership of the pages back to the system.
		1357	*
		1358	* It is vital that we remove the page mapping if we have mapped a tiled
		1359	* object through the GTT and then lose the fence register due to
		1360	* resource pressure. Similarly if the object has been moved out of the
		1361	* aperture, than pages mapped into userspace must be revoked. Removing the
		1362	* mapping will then trigger a page fault on the next user access, allowing
		1363	* fixup by i915_gem_fault().
		1364	*/
		1365	void
		1366	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1367	{
		1368	if (!obj->fault_mappable)
		1369	return;
		1370
		1371	if (obj->base.dev->dev_mapping)
		1372	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1373	// (loff_t)obj->base.map_list.hash.key<
		1374	// obj->base.size, 1);
		1375
		1376	obj->fault_mappable = false;
		1377	}
		1378
3480	Serge	1379	uint32_t
2332	Serge	1380	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1381	{
		1382	uint32_t gtt_size;
		1383
		1384	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1385	tiling_mode == I915_TILING_NONE)
		1386	return size;
		1387
		1388	/* Previous chips need a power-of-two fence region when tiling */
		1389	if (INTEL_INFO(dev)->gen == 3)
		1390	gtt_size = 1024*1024;
		1391	else
		1392	gtt_size = 512*1024;
		1393
		1394	while (gtt_size < size)
		1395	gtt_size <<= 1;
		1396
		1397	return gtt_size;
		1398	}
		1399
		1400	/**
		1401	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1402	* @obj: object to check
		1403	*
		1404	* Return the required GTT alignment for an object, taking into account
		1405	* potential fence register mapping.
		1406	*/
3480	Serge	1407	uint32_t
		1408	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1409	int tiling_mode, bool fenced)
2332	Serge	1410	{
		1411	/*
		1412	* Minimum alignment is 4k (GTT page size), but might be greater
		1413	* if a fence register is needed for the object.
		1414	*/
3480	Serge	1415	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1416	tiling_mode == I915_TILING_NONE)
		1417	return 4096;
		1418
		1419	/*
		1420	* Previous chips need to be aligned to the size of the smallest
		1421	* fence register that can contain the object.
		1422	*/
		1423	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1424	}
		1425
		1426	/**
		1427	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1428	* unfenced object
		1429	* @dev: the device
		1430	* @size: size of the object
		1431	* @tiling_mode: tiling mode of the object
		1432	*
		1433	* Return the required GTT alignment for an object, only taking into account
		1434	* unfenced tiled surface requirements.
		1435	*/
		1436	uint32_t
		1437	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1438	uint32_t size,
		1439	int tiling_mode)
		1440	{
		1441	/*
		1442	* Minimum alignment is 4k (GTT page size) for sane hw.
		1443	*/
		1444	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1445	tiling_mode == I915_TILING_NONE)
		1446	return 4096;
		1447
		1448	/* Previous hardware however needs to be aligned to a power-of-two
		1449	* tile height. The simplest method for determining this is to reuse
		1450	* the power-of-tile object size.
		1451	*/
		1452	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1453	}
		1454
3480	Serge	1455	int
		1456	i915_gem_mmap_gtt(struct drm_file *file,
		1457	struct drm_device *dev,
		1458	uint32_t handle,
		1459	uint64_t *offset)
		1460	{
		1461	struct drm_i915_private *dev_priv = dev->dev_private;
		1462	struct drm_i915_gem_object *obj;
		1463	unsigned long pfn;
		1464	char mem, ptr;
		1465	int ret;
		1466
		1467	ret = i915_mutex_lock_interruptible(dev);
		1468	if (ret)
		1469	return ret;
		1470
		1471	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1472	if (&obj->base == NULL) {
		1473	ret = -ENOENT;
		1474	goto unlock;
		1475	}
		1476
		1477	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1478	ret = -E2BIG;
		1479	goto out;
		1480	}
		1481
		1482	if (obj->madv != I915_MADV_WILLNEED) {
		1483	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1484	ret = -EINVAL;
		1485	goto out;
		1486	}
		1487	/* Now bind it into the GTT if needed */
		1488	ret = i915_gem_object_pin(obj, 0, true, false);
		1489	if (ret)
		1490	goto out;
		1491
		1492	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1493	if (ret)
		1494	goto unpin;
		1495
		1496	ret = i915_gem_object_get_fence(obj);
		1497	if (ret)
		1498	goto unpin;
		1499
		1500	obj->fault_mappable = true;
		1501
		1502	pfn = dev_priv->gtt.mappable_base + obj->gtt_offset;
		1503
		1504	/* Finally, remap it using the new GTT offset */
		1505
		1506	mem = UserAlloc(obj->base.size);
		1507	if(unlikely(mem == NULL))
		1508	{
		1509	ret = -ENOMEM;
		1510	goto unpin;
		1511	}
		1512
		1513	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1514	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1515
		1516	unpin:
		1517	i915_gem_object_unpin(obj);
		1518
		1519
		1520	*offset = (u64)mem;
		1521
		1522	out:
		1523	drm_gem_object_unreference(&obj->base);
		1524	unlock:
		1525	mutex_unlock(&dev->struct_mutex);
		1526	return ret;
		1527	}
		1528
		1529	/**
		1530	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1531	* @dev: DRM device
		1532	* @data: GTT mapping ioctl data
		1533	* @file: GEM object info
		1534	*
		1535	* Simply returns the fake offset to userspace so it can mmap it.
		1536	* The mmap call will end up in drm_gem_mmap(), which will set things
		1537	* up so we can get faults in the handler above.
		1538	*
		1539	* The fault handler will take care of binding the object into the GTT
		1540	* (since it may have been evicted to make room for something), allocating
		1541	* a fence register, and mapping the appropriate aperture address into
		1542	* userspace.
		1543	*/
		1544	int
		1545	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1546	struct drm_file *file)
		1547	{
		1548	struct drm_i915_gem_mmap_gtt *args = data;
		1549
		1550	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1551	}
		1552
3031	serge	1553	/* Immediately discard the backing storage */
		1554	static void
		1555	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1556	{
		1557	// struct inode *inode;
2332	Serge	1558
3031	serge	1559	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1560
3263	Serge	1561	if (obj->base.filp == NULL)
		1562	return;
2332	Serge	1563
3031	serge	1564	/* Our goal here is to return as much of the memory as
		1565	* is possible back to the system as we are called from OOM.
		1566	* To do this we must instruct the shmfs to drop all of its
		1567	* backing pages, now.
		1568	*/
		1569	// inode = obj->base.filp->f_path.dentry->d_inode;
		1570	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1571
3031	serge	1572	obj->madv = __I915_MADV_PURGED;
		1573	}
2332	Serge	1574
3031	serge	1575	static inline int
		1576	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1577	{
		1578	return obj->madv == I915_MADV_DONTNEED;
		1579	}
2332	Serge	1580
3031	serge	1581	static void
		1582	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1583	{
3746	Serge	1584	struct sg_page_iter sg_iter;
		1585	int ret;
2332	Serge	1586
3031	serge	1587	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1588
3031	serge	1589	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1590	if (ret) {
		1591	/* In the event of a disaster, abandon all caches and
		1592	* hope for the best.
		1593	*/
		1594	WARN_ON(ret != -EIO);
		1595	i915_gem_clflush_object(obj);
		1596	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1597	}
2332	Serge	1598
3031	serge	1599	if (obj->madv == I915_MADV_DONTNEED)
		1600	obj->dirty = 0;
2332	Serge	1601
3746	Serge	1602	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1603	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1604
3290	Serge	1605	page_cache_release(page);
3243	Serge	1606	}
		1607	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1608
3031	serge	1609	obj->dirty = 0;
3243	Serge	1610
		1611	sg_free_table(obj->pages);
		1612	kfree(obj->pages);
3031	serge	1613	}
2332	Serge	1614
3480	Serge	1615	int
3031	serge	1616	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1617	{
		1618	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1619
3243	Serge	1620	if (obj->pages == NULL)
3031	serge	1621	return 0;
2332	Serge	1622
3031	serge	1623	BUG_ON(obj->gtt_space);
		1624
		1625	if (obj->pages_pin_count)
		1626	return -EBUSY;
		1627
3243	Serge	1628	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1629	* array, hence protect them from being reaped by removing them from gtt
		1630	* lists early. */
		1631	list_del(&obj->gtt_list);
		1632
3031	serge	1633	ops->put_pages(obj);
3243	Serge	1634	obj->pages = NULL;
3031	serge	1635
		1636	if (i915_gem_object_is_purgeable(obj))
		1637	i915_gem_object_truncate(obj);
		1638
		1639	return 0;
		1640	}
		1641
		1642
		1643
		1644
		1645
		1646
		1647
		1648
2332	Serge	1649	static int
3031	serge	1650	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1651	{
3260	Serge	1652	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1653	int page_count, i;
		1654	struct sg_table *st;
		1655	struct scatterlist *sg;
3746	Serge	1656	struct sg_page_iter sg_iter;
3243	Serge	1657	struct page *page;
3746	Serge	1658	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1659	gfp_t gfp;
2332	Serge	1660
3243	Serge	1661	/* Assert that the object is not currently in any GPU domain. As it
		1662	* wasn't in the GTT, there shouldn't be any way it could have been in
		1663	* a GPU cache
2332	Serge	1664	*/
3243	Serge	1665	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1666	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1667
		1668	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1669	if (st == NULL)
		1670	return -ENOMEM;
		1671
2332	Serge	1672	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1673	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1674	sg_free_table(st);
		1675	kfree(st);
3746	Serge	1676	FAIL();
2332	Serge	1677	return -ENOMEM;
3243	Serge	1678	}
2332	Serge	1679
3243	Serge	1680	/* Get the list of pages out of our struct file. They'll be pinned
		1681	* at this point until we release them.
		1682	*
		1683	* Fail silently without starting the shrinker
		1684	*/
3746	Serge	1685	sg = st->sgl;
		1686	st->nents = 0;
		1687	for (i = 0; i < page_count; i++) {
3260	Serge	1688	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1689	if (IS_ERR(page)) {
		1690	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1691	goto err_pages;
		1692
3260	Serge	1693	}
3746	Serge	1694
		1695	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1696	if (i)
		1697	sg = sg_next(sg);
		1698	st->nents++;
3243	Serge	1699	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1700	} else {
		1701	sg->length += PAGE_SIZE;
		1702	}
		1703	last_pfn = page_to_pfn(page);
3243	Serge	1704	}
3031	serge	1705
3746	Serge	1706	sg_mark_end(sg);
3243	Serge	1707	obj->pages = st;
3031	serge	1708
2332	Serge	1709	return 0;
		1710
		1711	err_pages:
3746	Serge	1712	sg_mark_end(sg);
		1713	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1714	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1715	sg_free_table(st);
		1716	kfree(st);
3746	Serge	1717	FAIL();
3243	Serge	1718	return PTR_ERR(page);
2332	Serge	1719	}
		1720
3031	serge	1721	/* Ensure that the associated pages are gathered from the backing storage
		1722	* and pinned into our object. i915_gem_object_get_pages() may be called
		1723	* multiple times before they are released by a single call to
		1724	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1725	* either as a result of memory pressure (reaping pages under the shrinker)
		1726	* or as the object is itself released.
		1727	*/
		1728	int
		1729	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1730	{
3031	serge	1731	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1732	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1733	int ret;
2332	Serge	1734
3243	Serge	1735	if (obj->pages)
3031	serge	1736	return 0;
2332	Serge	1737
3031	serge	1738	BUG_ON(obj->pages_pin_count);
2332	Serge	1739
3031	serge	1740	ret = ops->get_pages(obj);
		1741	if (ret)
		1742	return ret;
2344	Serge	1743
3031	serge	1744	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1745	return 0;
2332	Serge	1746	}
		1747
		1748	void
		1749	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1750	struct intel_ring_buffer *ring)
2332	Serge	1751	{
		1752	struct drm_device *dev = obj->base.dev;
		1753	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1754	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1755
		1756	BUG_ON(ring == NULL);
		1757	obj->ring = ring;
		1758
		1759	/* Add a reference if we're newly entering the active list. */
		1760	if (!obj->active) {
2344	Serge	1761	drm_gem_object_reference(&obj->base);
2332	Serge	1762	obj->active = 1;
		1763	}
		1764
		1765	/* Move from whatever list we were on to the tail of execution. */
		1766	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1767	list_move_tail(&obj->ring_list, &ring->active_list);
		1768
3031	serge	1769	obj->last_read_seqno = seqno;
		1770
2332	Serge	1771	if (obj->fenced_gpu_access) {
3031	serge	1772	obj->last_fenced_seqno = seqno;
		1773
		1774	/* Bump MRU to take account of the delayed flush */
		1775	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1776	struct drm_i915_fence_reg *reg;
		1777
		1778	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1779	list_move_tail(®->lru_list,
		1780	&dev_priv->mm.fence_list);
		1781	}
2332	Serge	1782	}
		1783	}
		1784
2344	Serge	1785	static void
3031	serge	1786	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1787	{
		1788	struct drm_device *dev = obj->base.dev;
3031	serge	1789	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1790
3031	serge	1791	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1792	BUG_ON(!obj->active);
2332	Serge	1793
2352	Serge	1794	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1795
3031	serge	1796	list_del_init(&obj->ring_list);
2352	Serge	1797	obj->ring = NULL;
2344	Serge	1798
3031	serge	1799	obj->last_read_seqno = 0;
		1800	obj->last_write_seqno = 0;
		1801	obj->base.write_domain = 0;
		1802
		1803	obj->last_fenced_seqno = 0;
2352	Serge	1804	obj->fenced_gpu_access = false;
2344	Serge	1805
2352	Serge	1806	obj->active = 0;
		1807	drm_gem_object_unreference(&obj->base);
		1808
		1809	WARN_ON(i915_verify_lists(dev));
		1810	}
		1811
3243	Serge	1812	static int
3480	Serge	1813	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1814	{
3243	Serge	1815	struct drm_i915_private *dev_priv = dev->dev_private;
		1816	struct intel_ring_buffer *ring;
		1817	int ret, i, j;
2344	Serge	1818
3480	Serge	1819	/* Carefully retire all requests without writing to the rings */
3243	Serge	1820	for_each_ring(ring, dev_priv, i) {
3480	Serge	1821	ret = intel_ring_idle(ring);
3243	Serge	1822	if (ret)
		1823	return ret;
3480	Serge	1824	}
		1825	i915_gem_retire_requests(dev);
3243	Serge	1826
3480	Serge	1827	/* Finally reset hw state */
3243	Serge	1828	for_each_ring(ring, dev_priv, i) {
3480	Serge	1829	intel_ring_init_seqno(ring, seqno);
		1830
3243	Serge	1831	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1832	ring->sync_seqno[j] = 0;
		1833	}
		1834
		1835	return 0;
2344	Serge	1836	}
		1837
3480	Serge	1838	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1839	{
		1840	struct drm_i915_private *dev_priv = dev->dev_private;
		1841	int ret;
		1842
		1843	if (seqno == 0)
		1844	return -EINVAL;
		1845
		1846	/* HWS page needs to be set less than what we
		1847	* will inject to ring
		1848	*/
		1849	ret = i915_gem_init_seqno(dev, seqno - 1);
		1850	if (ret)
		1851	return ret;
		1852
		1853	/* Carefully set the last_seqno value so that wrap
		1854	* detection still works
		1855	*/
		1856	dev_priv->next_seqno = seqno;
		1857	dev_priv->last_seqno = seqno - 1;
		1858	if (dev_priv->last_seqno == 0)
		1859	dev_priv->last_seqno--;
		1860
		1861	return 0;
		1862	}
		1863
3243	Serge	1864	int
		1865	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1866	{
3243	Serge	1867	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1868
3243	Serge	1869	/* reserve 0 for non-seqno */
		1870	if (dev_priv->next_seqno == 0) {
3480	Serge	1871	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1872	if (ret)
		1873	return ret;
		1874
		1875	dev_priv->next_seqno = 1;
		1876	}
		1877
3480	Serge	1878	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1879	return 0;
2332	Serge	1880	}
		1881
2352	Serge	1882	int
		1883	i915_add_request(struct intel_ring_buffer *ring,
		1884	struct drm_file *file,
3031	serge	1885	u32 *out_seqno)
2352	Serge	1886	{
		1887	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1888	struct drm_i915_gem_request *request;
		1889	u32 request_ring_position;
2352	Serge	1890	int was_empty;
		1891	int ret;
2332	Serge	1892
3031	serge	1893	/*
		1894	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1895	* after having emitted the batchbuffer command. Hence we need to fix
		1896	* things up similar to emitting the lazy request. The difference here
		1897	* is that the flush _must_ happen before the next request, no matter
		1898	* what.
		1899	*/
		1900	ret = intel_ring_flush_all_caches(ring);
		1901	if (ret)
		1902	return ret;
2332	Serge	1903
3031	serge	1904	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1905	if (request == NULL)
		1906	return -ENOMEM;
		1907
		1908
		1909	/* Record the position of the start of the request so that
		1910	* should we detect the updated seqno part-way through the
		1911	* GPU processing the request, we never over-estimate the
		1912	* position of the head.
		1913	*/
		1914	request_ring_position = intel_ring_get_tail(ring);
		1915
3243	Serge	1916	ret = ring->add_request(ring);
3031	serge	1917	if (ret) {
		1918	kfree(request);
2352	Serge	1919	return ret;
3031	serge	1920	}
2332	Serge	1921
3243	Serge	1922	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1923	request->ring = ring;
3031	serge	1924	request->tail = request_ring_position;
		1925	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1926	was_empty = list_empty(&ring->request_list);
		1927	list_add_tail(&request->list, &ring->request_list);
3031	serge	1928	request->file_priv = NULL;
2332	Serge	1929
3263	Serge	1930	if (file) {
		1931	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1932
3263	Serge	1933	spin_lock(&file_priv->mm.lock);
		1934	request->file_priv = file_priv;
		1935	list_add_tail(&request->client_list,
		1936	&file_priv->mm.request_list);
		1937	spin_unlock(&file_priv->mm.lock);
		1938	}
		1939
		1940	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1941	ring->outstanding_lazy_request = 0;
2332	Serge	1942
2360	Serge	1943	if (!dev_priv->mm.suspended) {
		1944	if (i915_enable_hangcheck) {
2352	Serge	1945	// mod_timer(&dev_priv->hangcheck_timer,
		1946	// jiffies +
		1947	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1948	}
3031	serge	1949	if (was_empty) {
2360	Serge	1950	queue_delayed_work(dev_priv->wq,
3482	Serge	1951	&dev_priv->mm.retire_work,
		1952	round_jiffies_up_relative(HZ));
3031	serge	1953	intel_mark_busy(dev_priv->dev);
		1954	}
2360	Serge	1955	}
3031	serge	1956
		1957	if (out_seqno)
3243	Serge	1958	*out_seqno = request->seqno;
2352	Serge	1959	return 0;
		1960	}
2332	Serge	1961
3263	Serge	1962	static inline void
		1963	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1964	{
		1965	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1966
3263	Serge	1967	if (!file_priv)
		1968	return;
2332	Serge	1969
3263	Serge	1970	spin_lock(&file_priv->mm.lock);
		1971	if (request->file_priv) {
		1972	list_del(&request->client_list);
		1973	request->file_priv = NULL;
		1974	}
		1975	spin_unlock(&file_priv->mm.lock);
		1976	}
2332	Serge	1977
3031	serge	1978	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1979	struct intel_ring_buffer *ring)
		1980	{
		1981	while (!list_empty(&ring->request_list)) {
		1982	struct drm_i915_gem_request *request;
2332	Serge	1983
3031	serge	1984	request = list_first_entry(&ring->request_list,
		1985	struct drm_i915_gem_request,
		1986	list);
2332	Serge	1987
3031	serge	1988	list_del(&request->list);
3263	Serge	1989	i915_gem_request_remove_from_client(request);
3031	serge	1990	kfree(request);
		1991	}
2332	Serge	1992
3031	serge	1993	while (!list_empty(&ring->active_list)) {
		1994	struct drm_i915_gem_object *obj;
2332	Serge	1995
3031	serge	1996	obj = list_first_entry(&ring->active_list,
		1997	struct drm_i915_gem_object,
		1998	ring_list);
2332	Serge	1999
3031	serge	2000	i915_gem_object_move_to_inactive(obj);
		2001	}
		2002	}
2332	Serge	2003
3746	Serge	2004	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2005	{
		2006	struct drm_i915_private *dev_priv = dev->dev_private;
		2007	int i;
2332	Serge	2008
3031	serge	2009	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2010	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
3746	Serge	2011	i915_gem_write_fence(dev, i, reg->obj);
3031	serge	2012	}
		2013	}
2360	Serge	2014
3031	serge	2015	void i915_gem_reset(struct drm_device *dev)
		2016	{
		2017	struct drm_i915_private *dev_priv = dev->dev_private;
		2018	struct drm_i915_gem_object *obj;
		2019	struct intel_ring_buffer *ring;
		2020	int i;
2360	Serge	2021
3031	serge	2022	for_each_ring(ring, dev_priv, i)
		2023	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	2024
3031	serge	2025	/* Move everything out of the GPU domains to ensure we do any
		2026	* necessary invalidation upon reuse.
		2027	*/
		2028	list_for_each_entry(obj,
		2029	&dev_priv->mm.inactive_list,
		2030	mm_list)
		2031	{
		2032	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		2033	}
2360	Serge	2034
3746	Serge	2035	i915_gem_restore_fences(dev);
3031	serge	2036	}
2360	Serge	2037
2352	Serge	2038	/**
		2039	* This function clears the request list as sequence numbers are passed.
		2040	*/
3031	serge	2041	void
2352	Serge	2042	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2043	{
		2044	uint32_t seqno;
2332	Serge	2045
2352	Serge	2046	if (list_empty(&ring->request_list))
		2047	return;
2332	Serge	2048
2352	Serge	2049	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2050
3031	serge	2051	seqno = ring->get_seqno(ring, true);
2332	Serge	2052
2352	Serge	2053	while (!list_empty(&ring->request_list)) {
		2054	struct drm_i915_gem_request *request;
2332	Serge	2055
2352	Serge	2056	request = list_first_entry(&ring->request_list,
		2057	struct drm_i915_gem_request,
		2058	list);
2332	Serge	2059
2352	Serge	2060	if (!i915_seqno_passed(seqno, request->seqno))
		2061	break;
2332	Serge	2062
2352	Serge	2063	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2064	/* We know the GPU must have read the request to have
		2065	* sent us the seqno + interrupt, so use the position
		2066	* of tail of the request to update the last known position
		2067	* of the GPU head.
		2068	*/
		2069	ring->last_retired_head = request->tail;
2332	Serge	2070
2352	Serge	2071	list_del(&request->list);
3263	Serge	2072	i915_gem_request_remove_from_client(request);
2352	Serge	2073	kfree(request);
		2074	}
2332	Serge	2075
2352	Serge	2076	/* Move any buffers on the active list that are no longer referenced
		2077	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2078	*/
		2079	while (!list_empty(&ring->active_list)) {
		2080	struct drm_i915_gem_object *obj;
2332	Serge	2081
2352	Serge	2082	obj = list_first_entry(&ring->active_list,
		2083	struct drm_i915_gem_object,
		2084	ring_list);
2332	Serge	2085
3031	serge	2086	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2087	break;
2332	Serge	2088
2352	Serge	2089	i915_gem_object_move_to_inactive(obj);
		2090	}
2332	Serge	2091
2352	Serge	2092	if (unlikely(ring->trace_irq_seqno &&
		2093	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2094	ring->irq_put(ring);
		2095	ring->trace_irq_seqno = 0;
		2096	}
2332	Serge	2097
2352	Serge	2098	WARN_ON(i915_verify_lists(ring->dev));
		2099	}
2332	Serge	2100
2352	Serge	2101	void
		2102	i915_gem_retire_requests(struct drm_device *dev)
		2103	{
		2104	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2105	struct intel_ring_buffer *ring;
2352	Serge	2106	int i;
2332	Serge	2107
3031	serge	2108	for_each_ring(ring, dev_priv, i)
		2109	i915_gem_retire_requests_ring(ring);
2352	Serge	2110	}
		2111
2360	Serge	2112	static void
		2113	i915_gem_retire_work_handler(struct work_struct *work)
		2114	{
		2115	drm_i915_private_t *dev_priv;
		2116	struct drm_device *dev;
3031	serge	2117	struct intel_ring_buffer *ring;
2360	Serge	2118	bool idle;
		2119	int i;
2352	Serge	2120
2360	Serge	2121	dev_priv = container_of(work, drm_i915_private_t,
		2122	mm.retire_work.work);
		2123	dev = dev_priv->dev;
2352	Serge	2124
2360	Serge	2125	/* Come back later if the device is busy... */
		2126	if (!mutex_trylock(&dev->struct_mutex)) {
3482	Serge	2127	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2128	round_jiffies_up_relative(HZ));
3243	Serge	2129	return;
2360	Serge	2130	}
2352	Serge	2131
2360	Serge	2132	i915_gem_retire_requests(dev);
2352	Serge	2133
2360	Serge	2134	/* Send a periodic flush down the ring so we don't hold onto GEM
		2135	* objects indefinitely.
		2136	*/
		2137	idle = true;
3031	serge	2138	for_each_ring(ring, dev_priv, i) {
		2139	if (ring->gpu_caches_dirty)
		2140	i915_add_request(ring, NULL, NULL);
2352	Serge	2141
2360	Serge	2142	idle &= list_empty(&ring->request_list);
		2143	}
2352	Serge	2144
2360	Serge	2145	if (!dev_priv->mm.suspended && !idle)
3482	Serge	2146	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2147	round_jiffies_up_relative(HZ));
3031	serge	2148	if (idle)
		2149	intel_mark_idle(dev);
2360	Serge	2150
		2151	mutex_unlock(&dev->struct_mutex);
		2152	}
		2153
2344	Serge	2154	/**
3031	serge	2155	* Ensures that an object will eventually get non-busy by flushing any required
		2156	* write domains, emitting any outstanding lazy request and retiring and
		2157	* completed requests.
2352	Serge	2158	*/
3031	serge	2159	static int
		2160	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2161	{
3031	serge	2162	int ret;
2352	Serge	2163
3031	serge	2164	if (obj->active) {
		2165	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2166	if (ret)
		2167	return ret;
2352	Serge	2168
3031	serge	2169	i915_gem_retire_requests_ring(obj->ring);
		2170	}
2352	Serge	2171
3031	serge	2172	return 0;
		2173	}
2352	Serge	2174
3243	Serge	2175	/**
		2176	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2177	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2178	*
		2179	* Returns 0 if successful, else an error is returned with the remaining time in
		2180	* the timeout parameter.
		2181	* -ETIME: object is still busy after timeout
		2182	* -ERESTARTSYS: signal interrupted the wait
		2183	* -ENONENT: object doesn't exist
		2184	* Also possible, but rare:
		2185	* -EAGAIN: GPU wedged
		2186	* -ENOMEM: damn
		2187	* -ENODEV: Internal IRQ fail
		2188	* -E?: The add request failed
		2189	*
		2190	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2191	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2192	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2193	* without holding struct_mutex the object may become re-busied before this
		2194	* function completes. A similar but shorter * race condition exists in the busy
		2195	* ioctl
		2196	*/
2352	Serge	2197
		2198
		2199
		2200
		2201
		2202
		2203
		2204
		2205
3243	Serge	2206
		2207
		2208
2352	Serge	2209	/**
3031	serge	2210	* i915_gem_object_sync - sync an object to a ring.
		2211	*
		2212	* @obj: object which may be in use on another ring.
		2213	* @to: ring we wish to use the object on. May be NULL.
		2214	*
		2215	* This code is meant to abstract object synchronization with the GPU.
		2216	* Calling with NULL implies synchronizing the object with the CPU
		2217	* rather than a particular GPU ring.
		2218	*
		2219	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2220	*/
		2221	int
3031	serge	2222	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2223	struct intel_ring_buffer *to)
2344	Serge	2224	{
3031	serge	2225	struct intel_ring_buffer *from = obj->ring;
		2226	u32 seqno;
		2227	int ret, idx;
2332	Serge	2228
3031	serge	2229	if (from == NULL \|\| to == from)
		2230	return 0;
2332	Serge	2231
3031	serge	2232	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2233	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2234
3031	serge	2235	idx = intel_ring_sync_index(from, to);
		2236
		2237	seqno = obj->last_read_seqno;
		2238	if (seqno <= from->sync_seqno[idx])
		2239	return 0;
		2240
		2241	ret = i915_gem_check_olr(obj->ring, seqno);
		2242	if (ret)
		2243	return ret;
		2244
		2245	ret = to->sync_to(to, from, seqno);
		2246	if (!ret)
3243	Serge	2247	/* We use last_read_seqno because sync_to()
		2248	* might have just caused seqno wrap under
		2249	* the radar.
		2250	*/
		2251	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2252
		2253	return ret;
2344	Serge	2254	}
2332	Serge	2255
2344	Serge	2256	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2257	{
		2258	u32 old_write_domain, old_read_domains;
2332	Serge	2259
2344	Serge	2260	/* Force a pagefault for domain tracking on next user access */
		2261	// i915_gem_release_mmap(obj);
2332	Serge	2262
2344	Serge	2263	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2264	return;
2332	Serge	2265
3480	Serge	2266	/* Wait for any direct GTT access to complete */
		2267	mb();
		2268
2344	Serge	2269	old_read_domains = obj->base.read_domains;
		2270	old_write_domain = obj->base.write_domain;
2351	Serge	2271
2344	Serge	2272	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2273	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2274
2351	Serge	2275	trace_i915_gem_object_change_domain(obj,
		2276	old_read_domains,
		2277	old_write_domain);
2344	Serge	2278	}
2332	Serge	2279
2344	Serge	2280	/**
		2281	* Unbinds an object from the GTT aperture.
		2282	*/
		2283	int
		2284	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2285	{
3031	serge	2286	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2287	int ret;
2332	Serge	2288
3263	Serge	2289	if(obj == get_fb_obj())
		2290	return 0;
		2291
2344	Serge	2292	if (obj->gtt_space == NULL)
		2293	return 0;
2332	Serge	2294
3031	serge	2295	if (obj->pin_count)
		2296	return -EBUSY;
2332	Serge	2297
3243	Serge	2298	BUG_ON(obj->pages == NULL);
3031	serge	2299
2344	Serge	2300	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2301	if (ret)
2344	Serge	2302	return ret;
		2303	/* Continue on if we fail due to EIO, the GPU is hung so we
		2304	* should be safe and we need to cleanup or else we might
		2305	* cause memory corruption through use-after-free.
		2306	*/
2332	Serge	2307
2344	Serge	2308	i915_gem_object_finish_gtt(obj);
2332	Serge	2309
2344	Serge	2310	/* release the fence reg _after_ flushing */
		2311	ret = i915_gem_object_put_fence(obj);
3031	serge	2312	if (ret)
2344	Serge	2313	return ret;
2332	Serge	2314
2351	Serge	2315	trace_i915_gem_object_unbind(obj);
2332	Serge	2316
3031	serge	2317	if (obj->has_global_gtt_mapping)
3243	Serge	2318	i915_gem_gtt_unbind_object(obj);
3031	serge	2319	if (obj->has_aliasing_ppgtt_mapping) {
		2320	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2321	obj->has_aliasing_ppgtt_mapping = 0;
		2322	}
		2323	i915_gem_gtt_finish_object(obj);
2332	Serge	2324
3031	serge	2325	list_del(&obj->mm_list);
		2326	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2327	/* Avoid an unnecessary call to unbind on rebind. */
		2328	obj->map_and_fenceable = true;
2332	Serge	2329
2344	Serge	2330	drm_mm_put_block(obj->gtt_space);
		2331	obj->gtt_space = NULL;
		2332	obj->gtt_offset = 0;
2332	Serge	2333
2344	Serge	2334	return 0;
		2335	}
2332	Serge	2336
3031	serge	2337	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2338	{
		2339	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2340	struct intel_ring_buffer *ring;
2344	Serge	2341	int ret, i;
2332	Serge	2342
2344	Serge	2343	/* Flush everything onto the inactive list. */
3031	serge	2344	for_each_ring(ring, dev_priv, i) {
		2345	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2346	if (ret)
		2347	return ret;
3031	serge	2348
3243	Serge	2349	ret = intel_ring_idle(ring);
3031	serge	2350	if (ret)
		2351	return ret;
2344	Serge	2352	}
2332	Serge	2353
2344	Serge	2354	return 0;
		2355	}
2332	Serge	2356
3480	Serge	2357	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2358	struct drm_i915_gem_object *obj)
		2359	{
		2360	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2361	int fence_reg;
		2362	int fence_pitch_shift;
3031	serge	2363	uint64_t val;
2332	Serge	2364
3480	Serge	2365	if (INTEL_INFO(dev)->gen >= 6) {
		2366	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2367	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2368	} else {
		2369	fence_reg = FENCE_REG_965_0;
		2370	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2371	}
2332	Serge	2372
3031	serge	2373	if (obj) {
		2374	u32 size = obj->gtt_space->size;
2332	Serge	2375
3031	serge	2376	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2377	0xfffff000) << 32;
		2378	val \|= obj->gtt_offset & 0xfffff000;
3480	Serge	2379	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2380	if (obj->tiling_mode == I915_TILING_Y)
		2381	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2382	val \|= I965_FENCE_REG_VALID;
		2383	} else
		2384	val = 0;
2332	Serge	2385
3480	Serge	2386	fence_reg += reg * 8;
		2387	I915_WRITE64(fence_reg, val);
		2388	POSTING_READ(fence_reg);
3031	serge	2389	}
2332	Serge	2390
3031	serge	2391	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2392	struct drm_i915_gem_object *obj)
		2393	{
		2394	drm_i915_private_t *dev_priv = dev->dev_private;
		2395	u32 val;
2332	Serge	2396
3031	serge	2397	if (obj) {
		2398	u32 size = obj->gtt_space->size;
		2399	int pitch_val;
		2400	int tile_width;
2332	Serge	2401
3031	serge	2402	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2403	(size & -size) != size \|\|
		2404	(obj->gtt_offset & (size - 1)),
		2405	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2406	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2407
3031	serge	2408	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2409	tile_width = 128;
		2410	else
		2411	tile_width = 512;
2332	Serge	2412
3031	serge	2413	/* Note: pitch better be a power of two tile widths */
		2414	pitch_val = obj->stride / tile_width;
		2415	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2416
3031	serge	2417	val = obj->gtt_offset;
		2418	if (obj->tiling_mode == I915_TILING_Y)
		2419	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2420	val \|= I915_FENCE_SIZE_BITS(size);
		2421	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2422	val \|= I830_FENCE_REG_VALID;
		2423	} else
		2424	val = 0;
2332	Serge	2425
3031	serge	2426	if (reg < 8)
		2427	reg = FENCE_REG_830_0 + reg * 4;
		2428	else
		2429	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2430
3031	serge	2431	I915_WRITE(reg, val);
		2432	POSTING_READ(reg);
		2433	}
2332	Serge	2434
3031	serge	2435	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2436	struct drm_i915_gem_object *obj)
		2437	{
		2438	drm_i915_private_t *dev_priv = dev->dev_private;
		2439	uint32_t val;
2344	Serge	2440
3031	serge	2441	if (obj) {
		2442	u32 size = obj->gtt_space->size;
		2443	uint32_t pitch_val;
2344	Serge	2444
3031	serge	2445	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2446	(size & -size) != size \|\|
		2447	(obj->gtt_offset & (size - 1)),
		2448	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2449	obj->gtt_offset, size);
2344	Serge	2450
3031	serge	2451	pitch_val = obj->stride / 128;
		2452	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2453
3031	serge	2454	val = obj->gtt_offset;
		2455	if (obj->tiling_mode == I915_TILING_Y)
		2456	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2457	val \|= I830_FENCE_SIZE_BITS(size);
		2458	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2459	val \|= I830_FENCE_REG_VALID;
		2460	} else
		2461	val = 0;
		2462
		2463	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2464	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2465	}
		2466
3480	Serge	2467	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2468	{
		2469	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2470	}
		2471
3031	serge	2472	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2473	struct drm_i915_gem_object *obj)
2332	Serge	2474	{
3480	Serge	2475	struct drm_i915_private *dev_priv = dev->dev_private;
		2476
		2477	/* Ensure that all CPU reads are completed before installing a fence
		2478	* and all writes before removing the fence.
		2479	*/
		2480	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2481	mb();
		2482
3031	serge	2483	switch (INTEL_INFO(dev)->gen) {
		2484	case 7:
3480	Serge	2485	case 6:
3031	serge	2486	case 5:
		2487	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2488	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2489	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2490	default: BUG();
3031	serge	2491	}
3480	Serge	2492
		2493	/* And similarly be paranoid that no direct access to this region
		2494	* is reordered to before the fence is installed.
		2495	*/
		2496	if (i915_gem_object_needs_mb(obj))
		2497	mb();
2344	Serge	2498	}
		2499
3031	serge	2500	static inline int fence_number(struct drm_i915_private *dev_priv,
		2501	struct drm_i915_fence_reg *fence)
2344	Serge	2502	{
3031	serge	2503	return fence - dev_priv->fence_regs;
		2504	}
2332	Serge	2505
3746	Serge	2506	static void i915_gem_write_fence__ipi(void *data)
		2507	{
		2508	asm volatile("wbinvd");
		2509
		2510	}
		2511
3031	serge	2512	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2513	struct drm_i915_fence_reg *fence,
		2514	bool enable)
		2515	{
3746	Serge	2516	struct drm_device *dev = obj->base.dev;
		2517	struct drm_i915_private *dev_priv = dev->dev_private;
		2518	int fence_reg = fence_number(dev_priv, fence);
2332	Serge	2519
3746	Serge	2520	/* In order to fully serialize access to the fenced region and
		2521	* the update to the fence register we need to take extreme
		2522	* measures on SNB+. In theory, the write to the fence register
		2523	* flushes all memory transactions before, and coupled with the
		2524	* mb() placed around the register write we serialise all memory
		2525	* operations with respect to the changes in the tiler. Yet, on
		2526	* SNB+ we need to take a step further and emit an explicit wbinvd()
		2527	* on each processor in order to manually flush all memory
		2528	* transactions before updating the fence register.
		2529	*/
		2530	if (HAS_LLC(obj->base.dev))
		2531	on_each_cpu(i915_gem_write_fence__ipi, NULL, 1);
		2532	i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL);
3031	serge	2533
		2534	if (enable) {
3746	Serge	2535	obj->fence_reg = fence_reg;
3031	serge	2536	fence->obj = obj;
		2537	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2538	} else {
		2539	obj->fence_reg = I915_FENCE_REG_NONE;
		2540	fence->obj = NULL;
		2541	list_del_init(&fence->lru_list);
2344	Serge	2542	}
3031	serge	2543	}
2344	Serge	2544
3031	serge	2545	static int
3480	Serge	2546	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2547	{
		2548	if (obj->last_fenced_seqno) {
		2549	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2550	if (ret)
		2551	return ret;
2344	Serge	2552
		2553	obj->last_fenced_seqno = 0;
		2554	}
		2555
3031	serge	2556	obj->fenced_gpu_access = false;
2332	Serge	2557	return 0;
		2558	}
		2559
		2560	int
2344	Serge	2561	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2562	{
3031	serge	2563	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2564	struct drm_i915_fence_reg *fence;
2332	Serge	2565	int ret;
		2566
3480	Serge	2567	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2568	if (ret)
		2569	return ret;
		2570
3031	serge	2571	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2572	return 0;
2332	Serge	2573
3746	Serge	2574	fence = &dev_priv->fence_regs[obj->fence_reg];
		2575
3031	serge	2576	i915_gem_object_fence_lost(obj);
3746	Serge	2577	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2578
2332	Serge	2579	return 0;
		2580	}
		2581
3031	serge	2582	static struct drm_i915_fence_reg *
		2583	i915_find_fence_reg(struct drm_device *dev)
		2584	{
		2585	struct drm_i915_private *dev_priv = dev->dev_private;
		2586	struct drm_i915_fence_reg reg, avail;
		2587	int i;
2332	Serge	2588
3031	serge	2589	/* First try to find a free reg */
		2590	avail = NULL;
		2591	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2592	reg = &dev_priv->fence_regs[i];
		2593	if (!reg->obj)
		2594	return reg;
2332	Serge	2595
3031	serge	2596	if (!reg->pin_count)
		2597	avail = reg;
		2598	}
2332	Serge	2599
3031	serge	2600	if (avail == NULL)
		2601	return NULL;
2332	Serge	2602
3031	serge	2603	/* None available, try to steal one or wait for a user to finish */
		2604	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2605	if (reg->pin_count)
		2606	continue;
2332	Serge	2607
3031	serge	2608	return reg;
		2609	}
2332	Serge	2610
3031	serge	2611	return NULL;
		2612	}
2332	Serge	2613
3031	serge	2614	/**
		2615	* i915_gem_object_get_fence - set up fencing for an object
		2616	* @obj: object to map through a fence reg
		2617	*
		2618	* When mapping objects through the GTT, userspace wants to be able to write
		2619	* to them without having to worry about swizzling if the object is tiled.
		2620	* This function walks the fence regs looking for a free one for @obj,
		2621	* stealing one if it can't find any.
		2622	*
		2623	* It then sets up the reg based on the object's properties: address, pitch
		2624	* and tiling format.
		2625	*
		2626	* For an untiled surface, this removes any existing fence.
		2627	*/
		2628	int
		2629	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2630	{
		2631	struct drm_device *dev = obj->base.dev;
		2632	struct drm_i915_private *dev_priv = dev->dev_private;
		2633	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2634	struct drm_i915_fence_reg *reg;
		2635	int ret;
2332	Serge	2636
3031	serge	2637	/* Have we updated the tiling parameters upon the object and so
		2638	* will need to serialise the write to the associated fence register?
		2639	*/
		2640	if (obj->fence_dirty) {
3480	Serge	2641	ret = i915_gem_object_wait_fence(obj);
3031	serge	2642	if (ret)
		2643	return ret;
		2644	}
2332	Serge	2645
3031	serge	2646	/* Just update our place in the LRU if our fence is getting reused. */
		2647	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2648	reg = &dev_priv->fence_regs[obj->fence_reg];
		2649	if (!obj->fence_dirty) {
		2650	list_move_tail(®->lru_list,
		2651	&dev_priv->mm.fence_list);
		2652	return 0;
		2653	}
		2654	} else if (enable) {
		2655	reg = i915_find_fence_reg(dev);
		2656	if (reg == NULL)
		2657	return -EDEADLK;
2332	Serge	2658
3031	serge	2659	if (reg->obj) {
		2660	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2661
3480	Serge	2662	ret = i915_gem_object_wait_fence(old);
3031	serge	2663	if (ret)
		2664	return ret;
2332	Serge	2665
3031	serge	2666	i915_gem_object_fence_lost(old);
		2667	}
		2668	} else
		2669	return 0;
2332	Serge	2670
3031	serge	2671	i915_gem_object_update_fence(obj, reg, enable);
		2672	obj->fence_dirty = false;
2332	Serge	2673
3031	serge	2674	return 0;
		2675	}
2332	Serge	2676
3031	serge	2677	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2678	struct drm_mm_node *gtt_space,
		2679	unsigned long cache_level)
		2680	{
		2681	struct drm_mm_node *other;
2332	Serge	2682
3031	serge	2683	/* On non-LLC machines we have to be careful when putting differing
		2684	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2685	* crossing memory domains and dying.
3031	serge	2686	*/
		2687	if (HAS_LLC(dev))
		2688	return true;
2332	Serge	2689
3031	serge	2690	if (gtt_space == NULL)
		2691	return true;
2332	Serge	2692
3031	serge	2693	if (list_empty(>t_space->node_list))
		2694	return true;
2332	Serge	2695
3031	serge	2696	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2697	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2698	return false;
2344	Serge	2699
3031	serge	2700	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2701	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2702	return false;
2344	Serge	2703
3031	serge	2704	return true;
		2705	}
2344	Serge	2706
3031	serge	2707	static void i915_gem_verify_gtt(struct drm_device *dev)
		2708	{
		2709	#if WATCH_GTT
		2710	struct drm_i915_private *dev_priv = dev->dev_private;
		2711	struct drm_i915_gem_object *obj;
		2712	int err = 0;
2344	Serge	2713
3031	serge	2714	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2715	if (obj->gtt_space == NULL) {
		2716	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2717	err++;
		2718	continue;
		2719	}
2344	Serge	2720
3031	serge	2721	if (obj->cache_level != obj->gtt_space->color) {
		2722	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2723	obj->gtt_space->start,
		2724	obj->gtt_space->start + obj->gtt_space->size,
		2725	obj->cache_level,
		2726	obj->gtt_space->color);
		2727	err++;
		2728	continue;
		2729	}
2344	Serge	2730
3031	serge	2731	if (!i915_gem_valid_gtt_space(dev,
		2732	obj->gtt_space,
		2733	obj->cache_level)) {
		2734	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2735	obj->gtt_space->start,
		2736	obj->gtt_space->start + obj->gtt_space->size,
		2737	obj->cache_level);
		2738	err++;
		2739	continue;
		2740	}
		2741	}
2344	Serge	2742
3031	serge	2743	WARN_ON(err);
		2744	#endif
2326	Serge	2745	}
		2746
2332	Serge	2747	/**
		2748	* Finds free space in the GTT aperture and binds the object there.
		2749	*/
		2750	static int
		2751	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2752	unsigned alignment,
3031	serge	2753	bool map_and_fenceable,
		2754	bool nonblocking)
2332	Serge	2755	{
		2756	struct drm_device *dev = obj->base.dev;
		2757	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2758	struct drm_mm_node *node;
2332	Serge	2759	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2760	bool mappable, fenceable;
		2761	int ret;
2326	Serge	2762
2332	Serge	2763	fence_size = i915_gem_get_gtt_size(dev,
		2764	obj->base.size,
		2765	obj->tiling_mode);
		2766	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2767	obj->base.size,
3480	Serge	2768	obj->tiling_mode, true);
2332	Serge	2769	unfenced_alignment =
3480	Serge	2770	i915_gem_get_gtt_alignment(dev,
2332	Serge	2771	obj->base.size,
3480	Serge	2772	obj->tiling_mode, false);
2332	Serge	2773
		2774	if (alignment == 0)
		2775	alignment = map_and_fenceable ? fence_alignment :
		2776	unfenced_alignment;
		2777	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2778	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2779	return -EINVAL;
		2780	}
		2781
		2782	size = map_and_fenceable ? fence_size : obj->base.size;
		2783
		2784	/* If the object is bigger than the entire aperture, reject it early
		2785	* before evicting everything in a vain attempt to find space.
		2786	*/
		2787	if (obj->base.size >
3480	Serge	2788	(map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) {
2332	Serge	2789	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
3746	Serge	2790	FAIL();
2332	Serge	2791	return -E2BIG;
		2792	}
		2793
3031	serge	2794	ret = i915_gem_object_get_pages(obj);
		2795	if (ret)
		2796	return ret;
		2797
3243	Serge	2798	i915_gem_object_pin_pages(obj);
		2799
		2800	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2801	if (node == NULL) {
		2802	i915_gem_object_unpin_pages(obj);
		2803	return -ENOMEM;
		2804	}
		2805
2332	Serge	2806	search_free:
		2807	if (map_and_fenceable)
3243	Serge	2808	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2809	size, alignment, obj->cache_level,
3480	Serge	2810	0, dev_priv->gtt.mappable_end);
2332	Serge	2811	else
3243	Serge	2812	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2813	size, alignment, obj->cache_level);
		2814	if (ret) {
2332	Serge	2815
3243	Serge	2816	i915_gem_object_unpin_pages(obj);
		2817	kfree(node);
2332	Serge	2818	return ret;
		2819	}
3243	Serge	2820	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2821	i915_gem_object_unpin_pages(obj);
		2822	drm_mm_put_block(node);
3031	serge	2823	return -EINVAL;
		2824	}
2332	Serge	2825
3031	serge	2826	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2827	if (ret) {
3243	Serge	2828	i915_gem_object_unpin_pages(obj);
		2829	drm_mm_put_block(node);
2332	Serge	2830	return ret;
		2831	}
		2832
3031	serge	2833	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2834	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2835
3243	Serge	2836	obj->gtt_space = node;
		2837	obj->gtt_offset = node->start;
2332	Serge	2838
		2839	fenceable =
3243	Serge	2840	node->size == fence_size &&
		2841	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2842
		2843	mappable =
3480	Serge	2844	obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end;
2332	Serge	2845
		2846	obj->map_and_fenceable = mappable && fenceable;
		2847
3243	Serge	2848	i915_gem_object_unpin_pages(obj);
2351	Serge	2849	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2850	i915_gem_verify_gtt(dev);
2332	Serge	2851	return 0;
		2852	}
		2853
		2854	void
		2855	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2856	{
		2857	/* If we don't have a page list set up, then we're not pinned
		2858	* to GPU, and we can ignore the cache flush because it'll happen
		2859	* again at bind time.
		2860	*/
3243	Serge	2861	if (obj->pages == NULL)
2332	Serge	2862	return;
		2863
3480	Serge	2864	/*
		2865	* Stolen memory is always coherent with the GPU as it is explicitly
		2866	* marked as wc by the system, or the system is cache-coherent.
		2867	*/
		2868	if (obj->stolen)
		2869	return;
		2870
2332	Serge	2871	/* If the GPU is snooping the contents of the CPU cache,
		2872	* we do not need to manually clear the CPU cache lines. However,
		2873	* the caches are only snooped when the render cache is
		2874	* flushed/invalidated. As we always have to emit invalidations
		2875	* and flushes when moving into and out of the RENDER domain, correct
		2876	* snooping behaviour occurs naturally as the result of our domain
		2877	* tracking.
		2878	*/
		2879	if (obj->cache_level != I915_CACHE_NONE)
		2880	return;
3243	Serge	2881	#if 0
2344	Serge	2882	if(obj->mapped != NULL)
		2883	{
		2884	uint8_t *page_virtual;
		2885	unsigned int i;
2332	Serge	2886
2344	Serge	2887	page_virtual = obj->mapped;
		2888	asm volatile("mfence");
		2889	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2890	clflush(page_virtual + i);
		2891	asm volatile("mfence");
		2892	}
		2893	else
		2894	{
		2895	uint8_t *page_virtual;
		2896	unsigned int i;
		2897	page_virtual = AllocKernelSpace(obj->base.size);
		2898	if(page_virtual != NULL)
		2899	{
3031	serge	2900	dma_addr_t src, dst;
2344	Serge	2901	u32 count;
		2902
		2903	#define page_tabs 0xFDC00000 /* really dirty hack */
		2904
3031	serge	2905	src = obj->pages.page;
		2906	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2907	count = obj->base.size/4096;
		2908
		2909	while(count--)
		2910	{
		2911	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2912	};
		2913
		2914	asm volatile("mfence");
		2915	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2916	clflush(page_virtual + i);
		2917	asm volatile("mfence");
		2918	FreeKernelSpace(page_virtual);
		2919	}
		2920	else
		2921	{
		2922	asm volatile (
		2923	"mfence \n"
		2924	"wbinvd \n" /* this is really ugly */
		2925	"mfence");
		2926	}
		2927	}
3243	Serge	2928	#endif
		2929
2332	Serge	2930	}
		2931
2344	Serge	2932	/** Flushes the GTT write domain for the object if it's dirty. */
		2933	static void
		2934	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2935	{
		2936	uint32_t old_write_domain;
2332	Serge	2937
2344	Serge	2938	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2939	return;
2332	Serge	2940
2344	Serge	2941	/* No actual flushing is required for the GTT write domain. Writes
		2942	* to it immediately go to main memory as far as we know, so there's
		2943	* no chipset flush. It also doesn't land in render cache.
		2944	*
		2945	* However, we do have to enforce the order so that all writes through
		2946	* the GTT land before any writes to the device, such as updates to
		2947	* the GATT itself.
		2948	*/
		2949	wmb();
2332	Serge	2950
2344	Serge	2951	old_write_domain = obj->base.write_domain;
		2952	obj->base.write_domain = 0;
2332	Serge	2953
2351	Serge	2954	trace_i915_gem_object_change_domain(obj,
		2955	obj->base.read_domains,
		2956	old_write_domain);
2344	Serge	2957	}
2332	Serge	2958
		2959	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2960	static void
2332	Serge	2961	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2962	{
		2963	uint32_t old_write_domain;
		2964
		2965	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2966	return;
		2967
		2968	i915_gem_clflush_object(obj);
3243	Serge	2969	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2970	old_write_domain = obj->base.write_domain;
		2971	obj->base.write_domain = 0;
		2972
2351	Serge	2973	trace_i915_gem_object_change_domain(obj,
		2974	obj->base.read_domains,
		2975	old_write_domain);
2332	Serge	2976	}
		2977
		2978	/**
		2979	* Moves a single object to the GTT read, and possibly write domain.
		2980	*
		2981	* This function returns when the move is complete, including waiting on
		2982	* flushes to occur.
		2983	*/
		2984	int
		2985	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2986	{
3031	serge	2987	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2988	uint32_t old_write_domain, old_read_domains;
		2989	int ret;
		2990
		2991	/* Not valid to be called on unbound objects. */
		2992	if (obj->gtt_space == NULL)
		2993	return -EINVAL;
		2994
		2995	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2996	return 0;
		2997
3031	serge	2998	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2999	if (ret)
		3000	return ret;
		3001
		3002	i915_gem_object_flush_cpu_write_domain(obj);
		3003
3480	Serge	3004	/* Serialise direct access to this object with the barriers for
		3005	* coherent writes from the GPU, by effectively invalidating the
		3006	* GTT domain upon first access.
		3007	*/
		3008	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3009	mb();
		3010
2332	Serge	3011	old_write_domain = obj->base.write_domain;
		3012	old_read_domains = obj->base.read_domains;
		3013
		3014	/* It should now be out of any other write domains, and we can update
		3015	* the domain values for our changes.
		3016	*/
		3017	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3018	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3019	if (write) {
		3020	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3021	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3022	obj->dirty = 1;
		3023	}
		3024
2351	Serge	3025	trace_i915_gem_object_change_domain(obj,
		3026	old_read_domains,
		3027	old_write_domain);
		3028
3031	serge	3029	/* And bump the LRU for this access */
		3030	if (i915_gem_object_is_inactive(obj))
		3031	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		3032
2332	Serge	3033	return 0;
		3034	}
		3035
2335	Serge	3036	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3037	enum i915_cache_level cache_level)
		3038	{
3031	serge	3039	struct drm_device *dev = obj->base.dev;
		3040	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	3041	int ret;
2332	Serge	3042
2335	Serge	3043	if (obj->cache_level == cache_level)
		3044	return 0;
2332	Serge	3045
2335	Serge	3046	if (obj->pin_count) {
		3047	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3048	return -EBUSY;
		3049	}
2332	Serge	3050
3031	serge	3051	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		3052	ret = i915_gem_object_unbind(obj);
		3053	if (ret)
		3054	return ret;
		3055	}
		3056
2335	Serge	3057	if (obj->gtt_space) {
		3058	ret = i915_gem_object_finish_gpu(obj);
		3059	if (ret)
		3060	return ret;
2332	Serge	3061
2335	Serge	3062	i915_gem_object_finish_gtt(obj);
2332	Serge	3063
2335	Serge	3064	/* Before SandyBridge, you could not use tiling or fence
		3065	* registers with snooped memory, so relinquish any fences
		3066	* currently pointing to our region in the aperture.
		3067	*/
3031	serge	3068	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3069	ret = i915_gem_object_put_fence(obj);
		3070	if (ret)
		3071	return ret;
		3072	}
2332	Serge	3073
3031	serge	3074	if (obj->has_global_gtt_mapping)
		3075	i915_gem_gtt_bind_object(obj, cache_level);
		3076	if (obj->has_aliasing_ppgtt_mapping)
		3077	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3078	obj, cache_level);
		3079
		3080	obj->gtt_space->color = cache_level;
2335	Serge	3081	}
2332	Serge	3082
2335	Serge	3083	if (cache_level == I915_CACHE_NONE) {
		3084	u32 old_read_domains, old_write_domain;
2332	Serge	3085
2335	Serge	3086	/* If we're coming from LLC cached, then we haven't
		3087	* actually been tracking whether the data is in the
		3088	* CPU cache or not, since we only allow one bit set
		3089	* in obj->write_domain and have been skipping the clflushes.
		3090	* Just set it to the CPU cache for now.
		3091	*/
		3092	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		3093	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3094
2335	Serge	3095	old_read_domains = obj->base.read_domains;
		3096	old_write_domain = obj->base.write_domain;
2332	Serge	3097
2335	Serge	3098	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3099	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3100
2351	Serge	3101	trace_i915_gem_object_change_domain(obj,
		3102	old_read_domains,
		3103	old_write_domain);
2344	Serge	3104	}
2332	Serge	3105
2335	Serge	3106	obj->cache_level = cache_level;
3031	serge	3107	i915_gem_verify_gtt(dev);
2335	Serge	3108	return 0;
		3109	}
2332	Serge	3110
3260	Serge	3111	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3112	struct drm_file *file)
		3113	{
		3114	struct drm_i915_gem_caching *args = data;
		3115	struct drm_i915_gem_object *obj;
		3116	int ret;
		3117
3480	Serge	3118	if(args->handle == -2)
		3119	{
		3120	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3121	return 0;
		3122	}
		3123
3260	Serge	3124	ret = i915_mutex_lock_interruptible(dev);
		3125	if (ret)
		3126	return ret;
		3127
		3128	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3129	if (&obj->base == NULL) {
		3130	ret = -ENOENT;
		3131	goto unlock;
		3132	}
		3133
		3134	args->caching = obj->cache_level != I915_CACHE_NONE;
		3135
		3136	drm_gem_object_unreference(&obj->base);
		3137	unlock:
		3138	mutex_unlock(&dev->struct_mutex);
		3139	return ret;
		3140	}
		3141
		3142	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3143	struct drm_file *file)
		3144	{
		3145	struct drm_i915_gem_caching *args = data;
		3146	struct drm_i915_gem_object *obj;
		3147	enum i915_cache_level level;
		3148	int ret;
		3149
3480	Serge	3150	if(args->handle == -2)
		3151	{
		3152	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3153	return 0;
		3154	}
		3155
3260	Serge	3156	switch (args->caching) {
		3157	case I915_CACHING_NONE:
		3158	level = I915_CACHE_NONE;
		3159	break;
		3160	case I915_CACHING_CACHED:
		3161	level = I915_CACHE_LLC;
		3162	break;
		3163	default:
		3164	return -EINVAL;
		3165	}
		3166
		3167	ret = i915_mutex_lock_interruptible(dev);
		3168	if (ret)
		3169	return ret;
		3170
		3171	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3172	if (&obj->base == NULL) {
		3173	ret = -ENOENT;
		3174	goto unlock;
		3175	}
		3176
		3177	ret = i915_gem_object_set_cache_level(obj, level);
		3178
		3179	drm_gem_object_unreference(&obj->base);
		3180	unlock:
		3181	mutex_unlock(&dev->struct_mutex);
		3182	return ret;
		3183	}
		3184
2335	Serge	3185	/*
		3186	* Prepare buffer for display plane (scanout, cursors, etc).
		3187	* Can be called from an uninterruptible phase (modesetting) and allows
		3188	* any flushes to be pipelined (for pageflips).
		3189	*/
		3190	int
		3191	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3192	u32 alignment,
		3193	struct intel_ring_buffer *pipelined)
		3194	{
		3195	u32 old_read_domains, old_write_domain;
		3196	int ret;
2332	Serge	3197
3031	serge	3198	if (pipelined != obj->ring) {
		3199	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3200	if (ret)
		3201	return ret;
		3202	}
2332	Serge	3203
2335	Serge	3204	/* The display engine is not coherent with the LLC cache on gen6. As
		3205	* a result, we make sure that the pinning that is about to occur is
		3206	* done with uncached PTEs. This is lowest common denominator for all
		3207	* chipsets.
		3208	*
		3209	* However for gen6+, we could do better by using the GFDT bit instead
		3210	* of uncaching, which would allow us to flush all the LLC-cached data
		3211	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3212	*/
2360	Serge	3213	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3214	if (ret)
		3215	return ret;
2332	Serge	3216
2335	Serge	3217	/* As the user may map the buffer once pinned in the display plane
		3218	* (e.g. libkms for the bootup splash), we have to ensure that we
		3219	* always use map_and_fenceable for all scanout buffers.
		3220	*/
3031	serge	3221	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3222	if (ret)
		3223	return ret;
2332	Serge	3224
2335	Serge	3225	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3226
2335	Serge	3227	old_write_domain = obj->base.write_domain;
		3228	old_read_domains = obj->base.read_domains;
2332	Serge	3229
2335	Serge	3230	/* It should now be out of any other write domains, and we can update
		3231	* the domain values for our changes.
		3232	*/
3031	serge	3233	obj->base.write_domain = 0;
2335	Serge	3234	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3235
2351	Serge	3236	trace_i915_gem_object_change_domain(obj,
		3237	old_read_domains,
		3238	old_write_domain);
2332	Serge	3239
2335	Serge	3240	return 0;
		3241	}
2332	Serge	3242
2344	Serge	3243	int
		3244	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3245	{
		3246	int ret;
2332	Serge	3247
2344	Serge	3248	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3249	return 0;
2332	Serge	3250
3031	serge	3251	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3252	if (ret)
		3253	return ret;
2332	Serge	3254
2344	Serge	3255	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3256	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3257	return 0;
2344	Serge	3258	}
2332	Serge	3259
2344	Serge	3260	/**
		3261	* Moves a single object to the CPU read, and possibly write domain.
		3262	*
		3263	* This function returns when the move is complete, including waiting on
		3264	* flushes to occur.
		3265	*/
3031	serge	3266	int
2344	Serge	3267	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3268	{
		3269	uint32_t old_write_domain, old_read_domains;
		3270	int ret;
2332	Serge	3271
2344	Serge	3272	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3273	return 0;
2332	Serge	3274
3031	serge	3275	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3276	if (ret)
		3277	return ret;
2332	Serge	3278
2344	Serge	3279	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3280
2344	Serge	3281	old_write_domain = obj->base.write_domain;
		3282	old_read_domains = obj->base.read_domains;
2332	Serge	3283
2344	Serge	3284	/* Flush the CPU cache if it's still invalid. */
		3285	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3286	i915_gem_clflush_object(obj);
2332	Serge	3287
2344	Serge	3288	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3289	}
2332	Serge	3290
2344	Serge	3291	/* It should now be out of any other write domains, and we can update
		3292	* the domain values for our changes.
		3293	*/
		3294	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3295
2344	Serge	3296	/* If we're writing through the CPU, then the GPU read domains will
		3297	* need to be invalidated at next use.
		3298	*/
		3299	if (write) {
		3300	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3301	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3302	}
2332	Serge	3303
2351	Serge	3304	trace_i915_gem_object_change_domain(obj,
		3305	old_read_domains,
		3306	old_write_domain);
2332	Serge	3307
2344	Serge	3308	return 0;
		3309	}
2332	Serge	3310
3031	serge	3311	/* Throttle our rendering by waiting until the ring has completed our requests
		3312	* emitted over 20 msec ago.
2344	Serge	3313	*
3031	serge	3314	* Note that if we were to use the current jiffies each time around the loop,
		3315	* we wouldn't escape the function with any frames outstanding if the time to
		3316	* render a frame was over 20ms.
		3317	*
		3318	* This should get us reasonable parallelism between CPU and GPU but also
		3319	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3320	*/
3031	serge	3321	static int
		3322	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3323	{
3031	serge	3324	struct drm_i915_private *dev_priv = dev->dev_private;
		3325	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3326	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3327	struct drm_i915_gem_request *request;
		3328	struct intel_ring_buffer *ring = NULL;
3480	Serge	3329	unsigned reset_counter;
3031	serge	3330	u32 seqno = 0;
		3331	int ret;
2332	Serge	3332
3480	Serge	3333	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3334	if (ret)
		3335	return ret;
2332	Serge	3336
3480	Serge	3337	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3338	if (ret)
		3339	return ret;
		3340
3031	serge	3341	spin_lock(&file_priv->mm.lock);
		3342	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3343	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3344	break;
2332	Serge	3345
3031	serge	3346	ring = request->ring;
		3347	seqno = request->seqno;
		3348	}
3480	Serge	3349	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3350	spin_unlock(&file_priv->mm.lock);
2332	Serge	3351
3031	serge	3352	if (seqno == 0)
		3353	return 0;
2332	Serge	3354
3480	Serge	3355	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3356	if (ret == 0)
		3357	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3358
3031	serge	3359	return ret;
2352	Serge	3360	}
2332	Serge	3361
		3362	int
		3363	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3364	uint32_t alignment,
3031	serge	3365	bool map_and_fenceable,
		3366	bool nonblocking)
2332	Serge	3367	{
		3368	int ret;
		3369
3031	serge	3370	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3371	return -EBUSY;
2332	Serge	3372
		3373	if (obj->gtt_space != NULL) {
		3374	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3375	(map_and_fenceable && !obj->map_and_fenceable)) {
		3376	WARN(obj->pin_count,
		3377	"bo is already pinned with incorrect alignment:"
		3378	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3379	" obj->map_and_fenceable=%d\n",
		3380	obj->gtt_offset, alignment,
		3381	map_and_fenceable,
		3382	obj->map_and_fenceable);
		3383	ret = i915_gem_object_unbind(obj);
		3384	if (ret)
		3385	return ret;
		3386	}
		3387	}
		3388
		3389	if (obj->gtt_space == NULL) {
3243	Serge	3390	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3391
2332	Serge	3392	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3393	map_and_fenceable,
		3394	nonblocking);
2332	Serge	3395	if (ret)
		3396	return ret;
3243	Serge	3397
		3398	if (!dev_priv->mm.aliasing_ppgtt)
		3399	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3400	}
		3401
3031	serge	3402	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3403	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3404
		3405	obj->pin_count++;
2332	Serge	3406	obj->pin_mappable \|= map_and_fenceable;
		3407
		3408	return 0;
		3409	}
		3410
2344	Serge	3411	void
		3412	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3413	{
		3414	BUG_ON(obj->pin_count == 0);
		3415	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3416
3031	serge	3417	if (--obj->pin_count == 0)
2344	Serge	3418	obj->pin_mappable = false;
		3419	}
2332	Serge	3420
3031	serge	3421	int
		3422	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3423	struct drm_file *file)
		3424	{
		3425	struct drm_i915_gem_pin *args = data;
		3426	struct drm_i915_gem_object *obj;
		3427	int ret;
2332	Serge	3428
3480	Serge	3429	if(args->handle == -2)
		3430	{
		3431	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3432	return 0;
		3433	}
		3434
3031	serge	3435	ret = i915_mutex_lock_interruptible(dev);
		3436	if (ret)
		3437	return ret;
2332	Serge	3438
3031	serge	3439	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3440	if (&obj->base == NULL) {
		3441	ret = -ENOENT;
		3442	goto unlock;
		3443	}
2332	Serge	3444
3031	serge	3445	if (obj->madv != I915_MADV_WILLNEED) {
		3446	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3447	ret = -EINVAL;
		3448	goto out;
		3449	}
2332	Serge	3450
3031	serge	3451	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3452	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3453	args->handle);
		3454	ret = -EINVAL;
		3455	goto out;
		3456	}
2332	Serge	3457
3243	Serge	3458	if (obj->user_pin_count == 0) {
3031	serge	3459	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3460	if (ret)
		3461	goto out;
		3462	}
2332	Serge	3463
3243	Serge	3464	obj->user_pin_count++;
		3465	obj->pin_filp = file;
		3466
3031	serge	3467	/* XXX - flush the CPU caches for pinned objects
		3468	* as the X server doesn't manage domains yet
		3469	*/
		3470	i915_gem_object_flush_cpu_write_domain(obj);
		3471	args->offset = obj->gtt_offset;
		3472	out:
		3473	drm_gem_object_unreference(&obj->base);
		3474	unlock:
		3475	mutex_unlock(&dev->struct_mutex);
		3476	return ret;
		3477	}
2332	Serge	3478
3255	Serge	3479	#if 0
		3480
3031	serge	3481	int
		3482	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3483	struct drm_file *file)
		3484	{
		3485	struct drm_i915_gem_pin *args = data;
		3486	struct drm_i915_gem_object *obj;
		3487	int ret;
2332	Serge	3488
3031	serge	3489	ret = i915_mutex_lock_interruptible(dev);
		3490	if (ret)
		3491	return ret;
2332	Serge	3492
3031	serge	3493	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3494	if (&obj->base == NULL) {
		3495	ret = -ENOENT;
		3496	goto unlock;
		3497	}
2332	Serge	3498
3031	serge	3499	if (obj->pin_filp != file) {
		3500	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3501	args->handle);
		3502	ret = -EINVAL;
		3503	goto out;
		3504	}
		3505	obj->user_pin_count--;
		3506	if (obj->user_pin_count == 0) {
		3507	obj->pin_filp = NULL;
		3508	i915_gem_object_unpin(obj);
		3509	}
2332	Serge	3510
3031	serge	3511	out:
		3512	drm_gem_object_unreference(&obj->base);
		3513	unlock:
		3514	mutex_unlock(&dev->struct_mutex);
		3515	return ret;
		3516	}
2332	Serge	3517
3260	Serge	3518	#endif
		3519
3031	serge	3520	int
		3521	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3522	struct drm_file *file)
		3523	{
		3524	struct drm_i915_gem_busy *args = data;
		3525	struct drm_i915_gem_object *obj;
		3526	int ret;
2332	Serge	3527
3031	serge	3528	ret = i915_mutex_lock_interruptible(dev);
		3529	if (ret)
		3530	return ret;
2332	Serge	3531
3480	Serge	3532	if(args->handle == -2)
		3533	{
		3534	obj = get_fb_obj();
		3535	drm_gem_object_reference(&obj->base);
		3536	}
		3537	else
3031	serge	3538	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3539	if (&obj->base == NULL) {
		3540	ret = -ENOENT;
		3541	goto unlock;
		3542	}
2332	Serge	3543
3031	serge	3544	/* Count all active objects as busy, even if they are currently not used
		3545	* by the gpu. Users of this interface expect objects to eventually
		3546	* become non-busy without any further actions, therefore emit any
		3547	* necessary flushes here.
		3548	*/
		3549	ret = i915_gem_object_flush_active(obj);
2332	Serge	3550
3031	serge	3551	args->busy = obj->active;
		3552	if (obj->ring) {
		3553	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3554	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3555	}
2332	Serge	3556
3031	serge	3557	drm_gem_object_unreference(&obj->base);
		3558	unlock:
		3559	mutex_unlock(&dev->struct_mutex);
		3560	return ret;
		3561	}
2332	Serge	3562
3031	serge	3563	int
		3564	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3565	struct drm_file *file_priv)
		3566	{
		3567	return i915_gem_ring_throttle(dev, file_priv);
		3568	}
2332	Serge	3569
3263	Serge	3570	#if 0
		3571
3031	serge	3572	int
		3573	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3574	struct drm_file *file_priv)
		3575	{
		3576	struct drm_i915_gem_madvise *args = data;
		3577	struct drm_i915_gem_object *obj;
		3578	int ret;
2332	Serge	3579
3031	serge	3580	switch (args->madv) {
		3581	case I915_MADV_DONTNEED:
		3582	case I915_MADV_WILLNEED:
		3583	break;
		3584	default:
		3585	return -EINVAL;
		3586	}
2332	Serge	3587
3031	serge	3588	ret = i915_mutex_lock_interruptible(dev);
		3589	if (ret)
		3590	return ret;
2332	Serge	3591
3031	serge	3592	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3593	if (&obj->base == NULL) {
		3594	ret = -ENOENT;
		3595	goto unlock;
		3596	}
2332	Serge	3597
3031	serge	3598	if (obj->pin_count) {
		3599	ret = -EINVAL;
		3600	goto out;
		3601	}
2332	Serge	3602
3031	serge	3603	if (obj->madv != __I915_MADV_PURGED)
		3604	obj->madv = args->madv;
2332	Serge	3605
3031	serge	3606	/* if the object is no longer attached, discard its backing storage */
		3607	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3608	i915_gem_object_truncate(obj);
2332	Serge	3609
3031	serge	3610	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3611
3031	serge	3612	out:
		3613	drm_gem_object_unreference(&obj->base);
		3614	unlock:
		3615	mutex_unlock(&dev->struct_mutex);
		3616	return ret;
		3617	}
		3618	#endif
2332	Serge	3619
3031	serge	3620	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3621	const struct drm_i915_gem_object_ops *ops)
		3622	{
		3623	INIT_LIST_HEAD(&obj->mm_list);
		3624	INIT_LIST_HEAD(&obj->gtt_list);
		3625	INIT_LIST_HEAD(&obj->ring_list);
		3626	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3627
3031	serge	3628	obj->ops = ops;
		3629
		3630	obj->fence_reg = I915_FENCE_REG_NONE;
		3631	obj->madv = I915_MADV_WILLNEED;
		3632	/* Avoid an unnecessary call to unbind on the first bind. */
		3633	obj->map_and_fenceable = true;
		3634
		3635	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3636	}
		3637
		3638	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3639	.get_pages = i915_gem_object_get_pages_gtt,
		3640	.put_pages = i915_gem_object_put_pages_gtt,
		3641	};
		3642
2332	Serge	3643	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3644	size_t size)
		3645	{
		3646	struct drm_i915_gem_object *obj;
3031	serge	3647	struct address_space *mapping;
3480	Serge	3648	gfp_t mask;
2340	Serge	3649
3746	Serge	3650	obj = i915_gem_object_alloc(dev);
		3651
2332	Serge	3652	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3653	if (obj == NULL)
3746	Serge	3654	{
		3655	FAIL();
2332	Serge	3656	return NULL;
3746	Serge	3657	};
2332	Serge	3658
		3659	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3660	kfree(obj);
3746	Serge	3661	FAIL();
2332	Serge	3662	return NULL;
		3663	}
		3664
		3665
3031	serge	3666	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3667
		3668	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3669	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3670
3031	serge	3671	if (HAS_LLC(dev)) {
		3672	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3673	* cache) for about a 10% performance improvement
		3674	* compared to uncached. Graphics requests other than
		3675	* display scanout are coherent with the CPU in
		3676	* accessing this cache. This means in this mode we
		3677	* don't need to clflush on the CPU side, and on the
		3678	* GPU side we only need to flush internal caches to
		3679	* get data visible to the CPU.
		3680	*
		3681	* However, we maintain the display planes as UC, and so
		3682	* need to rebind when first used as such.
		3683	*/
		3684	obj->cache_level = I915_CACHE_LLC;
		3685	} else
		3686	obj->cache_level = I915_CACHE_NONE;
		3687
		3688	return obj;
		3689	}
		3690
2344	Serge	3691	int i915_gem_init_object(struct drm_gem_object *obj)
		3692	{
		3693	BUG();
2332	Serge	3694
2344	Serge	3695	return 0;
		3696	}
2332	Serge	3697
3031	serge	3698	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3699	{
3031	serge	3700	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3701	struct drm_device *dev = obj->base.dev;
		3702	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3703
3031	serge	3704	trace_i915_gem_object_destroy(obj);
		3705
		3706	// if (obj->phys_obj)
		3707	// i915_gem_detach_phys_object(dev, obj);
3298	Serge	3708	// printf("%s obj %p\n", __FUNCTION__, obj);
3031	serge	3709
		3710	obj->pin_count = 0;
		3711	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3712	bool was_interruptible;
		3713
		3714	was_interruptible = dev_priv->mm.interruptible;
		3715	dev_priv->mm.interruptible = false;
		3716
		3717	WARN_ON(i915_gem_object_unbind(obj));
		3718
		3719	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3720	}
2332	Serge	3721
3031	serge	3722	obj->pages_pin_count = 0;
		3723	i915_gem_object_put_pages(obj);
		3724	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3725
3243	Serge	3726	BUG_ON(obj->pages);
2332	Serge	3727
3031	serge	3728	// if (obj->base.import_attach)
		3729	// drm_prime_gem_destroy(&obj->base, NULL);
		3730
3290	Serge	3731	if(obj->base.filp != NULL)
		3732	{
3298	Serge	3733	// printf("filp %p\n", obj->base.filp);
3290	Serge	3734	shmem_file_delete(obj->base.filp);
		3735	}
		3736
2344	Serge	3737	drm_gem_object_release(&obj->base);
		3738	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3739
2344	Serge	3740	kfree(obj->bit_17);
		3741	kfree(obj);
		3742	}
2332	Serge	3743
3031	serge	3744	#if 0
		3745	int
		3746	i915_gem_idle(struct drm_device *dev)
2344	Serge	3747	{
3031	serge	3748	drm_i915_private_t *dev_priv = dev->dev_private;
		3749	int ret;
2332	Serge	3750
3031	serge	3751	mutex_lock(&dev->struct_mutex);
2332	Serge	3752
3031	serge	3753	if (dev_priv->mm.suspended) {
		3754	mutex_unlock(&dev->struct_mutex);
		3755	return 0;
		3756	}
2332	Serge	3757
3031	serge	3758	ret = i915_gpu_idle(dev);
		3759	if (ret) {
		3760	mutex_unlock(&dev->struct_mutex);
		3761	return ret;
		3762	}
		3763	i915_gem_retire_requests(dev);
		3764
3480	Serge	3765	/* Under UMS, be paranoid and evict. */
		3766	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		3767	i915_gem_evict_everything(dev);
		3768
3031	serge	3769	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3770	* We need to replace this with a semaphore, or something.
		3771	* And not confound mm.suspended!
		3772	*/
		3773	dev_priv->mm.suspended = 1;
3480	Serge	3774	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	3775
		3776	i915_kernel_lost_context(dev);
		3777	i915_gem_cleanup_ringbuffer(dev);
		3778
		3779	mutex_unlock(&dev->struct_mutex);
		3780
		3781	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	3782	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	3783
		3784	return 0;
2344	Serge	3785	}
3031	serge	3786	#endif
2332	Serge	3787
3031	serge	3788	void i915_gem_l3_remap(struct drm_device *dev)
		3789	{
		3790	drm_i915_private_t *dev_priv = dev->dev_private;
		3791	u32 misccpctl;
		3792	int i;
2332	Serge	3793
3480	Serge	3794	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	3795	return;
2332	Serge	3796
3243	Serge	3797	if (!dev_priv->l3_parity.remap_info)
3031	serge	3798	return;
2332	Serge	3799
3031	serge	3800	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3801	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3802	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3803
3031	serge	3804	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3805	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3806	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3807	DRM_DEBUG("0x%x was already programmed to %x\n",
		3808	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3809	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3810	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3811	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3812	}
2332	Serge	3813
3031	serge	3814	/* Make sure all the writes land before disabling dop clock gating */
		3815	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3816
3031	serge	3817	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3818	}
2332	Serge	3819
3031	serge	3820	void i915_gem_init_swizzling(struct drm_device *dev)
		3821	{
		3822	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3823
3031	serge	3824	if (INTEL_INFO(dev)->gen < 5 \|\|
		3825	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3826	return;
2332	Serge	3827
3031	serge	3828	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3829	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3830
3031	serge	3831	if (IS_GEN5(dev))
		3832	return;
2344	Serge	3833
3031	serge	3834	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3835	if (IS_GEN6(dev))
		3836	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	3837	else if (IS_GEN7(dev))
		3838	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	3839	else
3480	Serge	3840	BUG();
3031	serge	3841	}
		3842
		3843	static bool
		3844	intel_enable_blt(struct drm_device *dev)
		3845	{
		3846	if (!HAS_BLT(dev))
		3847	return false;
		3848
		3849	/* The blitter was dysfunctional on early prototypes */
		3850	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3851	DRM_INFO("BLT not supported on this pre-production hardware;"
		3852	" graphics performance will be degraded.\n");
		3853	return false;
		3854	}
		3855
		3856	return true;
		3857	}
		3858
3480	Serge	3859	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	3860	{
3480	Serge	3861	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	3862	int ret;
2351	Serge	3863
2332	Serge	3864	ret = intel_init_render_ring_buffer(dev);
		3865	if (ret)
		3866	return ret;
		3867
		3868	if (HAS_BSD(dev)) {
		3869	ret = intel_init_bsd_ring_buffer(dev);
		3870	if (ret)
		3871	goto cleanup_render_ring;
		3872	}
		3873
3031	serge	3874	if (intel_enable_blt(dev)) {
2332	Serge	3875	ret = intel_init_blt_ring_buffer(dev);
		3876	if (ret)
		3877	goto cleanup_bsd_ring;
		3878	}
		3879
3480	Serge	3880	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		3881	if (ret)
		3882	goto cleanup_blt_ring;
2351	Serge	3883
2332	Serge	3884	return 0;
		3885
3480	Serge	3886	cleanup_blt_ring:
		3887	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	3888	cleanup_bsd_ring:
		3889	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3890	cleanup_render_ring:
		3891	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	3892
2332	Serge	3893	return ret;
		3894	}
		3895
3480	Serge	3896	int
		3897	i915_gem_init_hw(struct drm_device *dev)
3031	serge	3898	{
3480	Serge	3899	drm_i915_private_t *dev_priv = dev->dev_private;
		3900	int ret;
3031	serge	3901
3480	Serge	3902	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		3903	return -EIO;
3031	serge	3904
3480	Serge	3905	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3906	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3907
3746	Serge	3908	if (HAS_PCH_NOP(dev)) {
		3909	u32 temp = I915_READ(GEN7_MSG_CTL);
		3910	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		3911	I915_WRITE(GEN7_MSG_CTL, temp);
		3912	}
		3913
3480	Serge	3914	i915_gem_l3_remap(dev);
		3915
		3916	i915_gem_init_swizzling(dev);
		3917
		3918	ret = i915_gem_init_rings(dev);
		3919	if (ret)
		3920	return ret;
		3921
		3922	/*
		3923	* XXX: There was some w/a described somewhere suggesting loading
		3924	* contexts before PPGTT.
		3925	*/
		3926	i915_gem_context_init(dev);
3746	Serge	3927	if (dev_priv->mm.aliasing_ppgtt) {
		3928	ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		3929	if (ret) {
		3930	i915_gem_cleanup_aliasing_ppgtt(dev);
		3931	DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		3932	}
		3933	}
3480	Serge	3934
		3935	return 0;
3031	serge	3936	}
		3937
		3938	#define LFB_SIZE 0xC00000
		3939
		3940	int i915_gem_init(struct drm_device *dev)
		3941	{
		3942	struct drm_i915_private *dev_priv = dev->dev_private;
		3943	int ret;
		3944
		3945	mutex_lock(&dev->struct_mutex);
3746	Serge	3946
		3947	if (IS_VALLEYVIEW(dev)) {
		3948	/* VLVA0 (potential hack), BIOS isn't actually waking us */
		3949	I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		3950	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
		3951	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		3952	}
		3953
3480	Serge	3954	i915_gem_init_global_gtt(dev);
3746	Serge	3955
3031	serge	3956	ret = i915_gem_init_hw(dev);
		3957	mutex_unlock(&dev->struct_mutex);
		3958	if (ret) {
		3959	i915_gem_cleanup_aliasing_ppgtt(dev);
		3960	return ret;
		3961	}
		3962
3746	Serge	3963
3031	serge	3964	return 0;
		3965	}
		3966
2332	Serge	3967	void
		3968	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3969	{
		3970	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3971	struct intel_ring_buffer *ring;
2332	Serge	3972	int i;
		3973
3031	serge	3974	for_each_ring(ring, dev_priv, i)
		3975	intel_cleanup_ring_buffer(ring);
2332	Serge	3976	}
		3977
3031	serge	3978	#if 0
		3979
2332	Serge	3980	int
		3981	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3982	struct drm_file *file_priv)
		3983	{
		3984	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3985	int ret;
2332	Serge	3986
		3987	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3988	return 0;
		3989
3480	Serge	3990	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	3991	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	3992	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	3993	}
		3994
		3995	mutex_lock(&dev->struct_mutex);
		3996	dev_priv->mm.suspended = 0;
		3997
3031	serge	3998	ret = i915_gem_init_hw(dev);
2332	Serge	3999	if (ret != 0) {
		4000	mutex_unlock(&dev->struct_mutex);
		4001	return ret;
		4002	}
		4003
		4004	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		4005	mutex_unlock(&dev->struct_mutex);
		4006
		4007	ret = drm_irq_install(dev);
		4008	if (ret)
		4009	goto cleanup_ringbuffer;
		4010
		4011	return 0;
		4012
		4013	cleanup_ringbuffer:
		4014	mutex_lock(&dev->struct_mutex);
		4015	i915_gem_cleanup_ringbuffer(dev);
		4016	dev_priv->mm.suspended = 1;
		4017	mutex_unlock(&dev->struct_mutex);
		4018
		4019	return ret;
		4020	}
		4021
		4022	int
		4023	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4024	struct drm_file *file_priv)
		4025	{
		4026	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4027	return 0;
		4028
		4029	drm_irq_uninstall(dev);
		4030	return i915_gem_idle(dev);
		4031	}
		4032
		4033	void
		4034	i915_gem_lastclose(struct drm_device *dev)
		4035	{
		4036	int ret;
		4037
		4038	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4039	return;
		4040
		4041	ret = i915_gem_idle(dev);
		4042	if (ret)
		4043	DRM_ERROR("failed to idle hardware: %d\n", ret);
		4044	}
		4045	#endif
		4046
		4047	static void
2326	Serge	4048	init_ring_lists(struct intel_ring_buffer *ring)
		4049	{
		4050	INIT_LIST_HEAD(&ring->active_list);
		4051	INIT_LIST_HEAD(&ring->request_list);
		4052	}
		4053
		4054	void
		4055	i915_gem_load(struct drm_device *dev)
		4056	{
3480	Serge	4057	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4058	int i;
		4059
		4060	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		4061	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	4062	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4063	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4064	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4065	for (i = 0; i < I915_NUM_RINGS; i++)
		4066	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4067	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4068	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4069	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4070	i915_gem_retire_work_handler);
3480	Serge	4071	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4072
		4073	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4074	if (IS_GEN3(dev)) {
3031	serge	4075	I915_WRITE(MI_ARB_STATE,
		4076	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4077	}
		4078
		4079	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4080
3746	Serge	4081	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4082	dev_priv->num_fence_regs = 32;
		4083	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4084	dev_priv->num_fence_regs = 16;
		4085	else
		4086	dev_priv->num_fence_regs = 8;
		4087
		4088	/* Initialize fence registers to zero */
3746	Serge	4089	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4090	i915_gem_restore_fences(dev);
2326	Serge	4091
		4092	i915_gem_detect_bit_6_swizzle(dev);
		4093
		4094	dev_priv->mm.interruptible = true;
		4095
		4096	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		4097	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		4098	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		4099	}
		4100
		4101

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3746