WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
3746	Serge	36	#include
2326	Serge	37	#include
		38
2344	Serge	39	extern int x86_clflush_size;
2332	Serge	40
3263	Serge	41	#define PROT_READ 0x1 /* page can be read */
		42	#define PROT_WRITE 0x2 /* page can be written */
		43	#define MAP_SHARED 0x01 /* Share changes */
		44
2344	Serge	45	#undef mb
		46	#undef rmb
		47	#undef wmb
		48	#define mb() asm volatile("mfence")
		49	#define rmb() asm volatile ("lfence")
		50	#define wmb() asm volatile ("sfence")
		51
3266	Serge	52	struct drm_i915_gem_object *get_fb_obj();
		53
3263	Serge	54	unsigned long vm_mmap(struct file *file, unsigned long addr,
		55	unsigned long len, unsigned long prot,
		56	unsigned long flag, unsigned long offset);
		57
2344	Serge	58	static inline void clflush(volatile void *__p)
		59	{
		60	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		61	}
		62
2332	Serge	63	#define MAX_ERRNO 4095
		64
		65	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		66
		67
2326	Serge	68	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		69	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		70	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		71	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		72
2332	Serge	73	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	74	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		75	bool force);
		76	static __must_check int
		77	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		78	struct i915_address_space *vm,
2332	Serge	79	unsigned alignment,
3031	serge	80	bool map_and_fenceable,
		81	bool nonblocking);
2332	Serge	82	static int i915_gem_phys_pwrite(struct drm_device *dev,
		83	struct drm_i915_gem_object *obj,
		84	struct drm_i915_gem_pwrite *args,
		85	struct drm_file *file);
2326	Serge	86
3031	serge	87	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		88	struct drm_i915_gem_object *obj);
		89	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		90	struct drm_i915_fence_reg *fence,
		91	bool enable);
2332	Serge	92
3031	serge	93	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
4104	Serge	94	static long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	95	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		96
4104	Serge	97	static bool cpu_cache_is_coherent(struct drm_device *dev,
		98	enum i915_cache_level level)
		99	{
		100	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		101	}
		102
		103	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		104	{
		105	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		106	return true;
		107
		108	return obj->pin_display;
		109	}
		110
3031	serge	111	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		112	{
		113	if (obj->tiling_mode)
		114	i915_gem_release_mmap(obj);
		115
		116	/* As we do not have an associated fence register, we will force
		117	* a tiling change if we ever need to acquire one.
		118	*/
		119	obj->fence_dirty = false;
		120	obj->fence_reg = I915_FENCE_REG_NONE;
		121	}
		122
2332	Serge	123	/* some bookkeeping */
		124	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		125	size_t size)
		126	{
4104	Serge	127	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	128	dev_priv->mm.object_count++;
		129	dev_priv->mm.object_memory += size;
4104	Serge	130	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	131	}
		132
		133	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		134	size_t size)
		135	{
4104	Serge	136	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	137	dev_priv->mm.object_count--;
		138	dev_priv->mm.object_memory -= size;
4104	Serge	139	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	140	}
		141
		142	static int
3480	Serge	143	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	144	{
		145	int ret;
		146
3480	Serge	147	#define EXIT_COND (!i915_reset_in_progress(error))
		148	if (EXIT_COND)
2332	Serge	149	return 0;
3255	Serge	150	#if 0
3031	serge	151	/*
		152	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		153	* userspace. If it takes that long something really bad is going on and
		154	* we should simply try to bail out and fail as gracefully as possible.
		155	*/
3480	Serge	156	ret = wait_event_interruptible_timeout(error->reset_queue,
		157	EXIT_COND,
		158	10*HZ);
3031	serge	159	if (ret == 0) {
		160	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		161	return -EIO;
		162	} else if (ret < 0) {
2332	Serge	163	return ret;
3031	serge	164	}
2332	Serge	165
3255	Serge	166	#endif
3480	Serge	167	#undef EXIT_COND
3255	Serge	168
2332	Serge	169	return 0;
		170	}
		171
		172	int i915_mutex_lock_interruptible(struct drm_device *dev)
		173	{
3480	Serge	174	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	175	int ret;
		176
3480	Serge	177	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	178	if (ret)
		179	return ret;
		180
3480	Serge	181	ret = mutex_lock_interruptible(&dev->struct_mutex);
		182	if (ret)
		183	return ret;
2332	Serge	184
		185	WARN_ON(i915_verify_lists(dev));
		186	return 0;
		187	}
		188
		189	static inline bool
		190	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		191	{
4104	Serge	192	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	193	}
		194
		195
		196	#if 0
		197
		198	int
		199	i915_gem_init_ioctl(struct drm_device dev, void data,
		200	struct drm_file *file)
		201	{
3480	Serge	202	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	203	struct drm_i915_gem_init *args = data;
		204
3031	serge	205	if (drm_core_check_feature(dev, DRIVER_MODESET))
		206	return -ENODEV;
		207
2332	Serge	208	if (args->gtt_start >= args->gtt_end \|\|
		209	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		210	return -EINVAL;
		211
3031	serge	212	/* GEM with user mode setting was never supported on ilk and later. */
		213	if (INTEL_INFO(dev)->gen >= 5)
		214	return -ENODEV;
		215
2332	Serge	216	mutex_lock(&dev->struct_mutex);
3480	Serge	217	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		218	args->gtt_end);
		219	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	220	mutex_unlock(&dev->struct_mutex);
		221
		222	return 0;
		223	}
2351	Serge	224	#endif
2332	Serge	225
		226	int
		227	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		228	struct drm_file *file)
		229	{
		230	struct drm_i915_private *dev_priv = dev->dev_private;
		231	struct drm_i915_gem_get_aperture *args = data;
		232	struct drm_i915_gem_object *obj;
		233	size_t pinned;
		234
		235	pinned = 0;
		236	mutex_lock(&dev->struct_mutex);
4104	Serge	237	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
3031	serge	238	if (obj->pin_count)
4104	Serge	239	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	240	mutex_unlock(&dev->struct_mutex);
		241
4104	Serge	242	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	243	args->aper_available_size = args->aper_size - pinned;
2332	Serge	244
		245	return 0;
		246	}
		247
3480	Serge	248	void i915_gem_object_alloc(struct drm_device dev)
		249	{
		250	struct drm_i915_private *dev_priv = dev->dev_private;
		251	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		252	}
		253
		254	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		255	{
		256	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		257	kfree(obj);
		258	}
		259
3031	serge	260	static int
		261	i915_gem_create(struct drm_file *file,
2332	Serge	262	struct drm_device *dev,
		263	uint64_t size,
		264	uint32_t *handle_p)
		265	{
		266	struct drm_i915_gem_object *obj;
		267	int ret;
		268	u32 handle;
		269
		270	size = roundup(size, PAGE_SIZE);
2342	Serge	271	if (size == 0)
		272	return -EINVAL;
2332	Serge	273
		274	/* Allocate the new object */
		275	obj = i915_gem_alloc_object(dev, size);
		276	if (obj == NULL)
		277	return -ENOMEM;
		278
		279	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	280	/* drop reference from allocate - handle holds it now */
		281	drm_gem_object_unreference_unlocked(&obj->base);
		282	if (ret)
2332	Serge	283	return ret;
		284
		285	*handle_p = handle;
		286	return 0;
		287	}
		288
		289	int
		290	i915_gem_dumb_create(struct drm_file *file,
		291	struct drm_device *dev,
		292	struct drm_mode_create_dumb *args)
		293	{
		294	/* have to work out size/pitch and return them */
		295	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		296	args->size = args->pitch * args->height;
		297	return i915_gem_create(file, dev,
		298	args->size, &args->handle);
		299	}
		300
2326	Serge	301	/**
2332	Serge	302	* Creates a new mm object and returns a handle to it.
		303	*/
		304	int
		305	i915_gem_create_ioctl(struct drm_device dev, void data,
		306	struct drm_file *file)
		307	{
		308	struct drm_i915_gem_create *args = data;
3031	serge	309
2332	Serge	310	return i915_gem_create(file, dev,
		311	args->size, &args->handle);
		312	}
		313
		314
3260	Serge	315	#if 0
2332	Serge	316
3031	serge	317	static inline int
		318	__copy_to_user_swizzled(char __user *cpu_vaddr,
		319	const char *gpu_vaddr, int gpu_offset,
2332	Serge	320	int length)
		321	{
3031	serge	322	int ret, cpu_offset = 0;
2332	Serge	323
3031	serge	324	while (length > 0) {
		325	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		326	int this_length = min(cacheline_end - gpu_offset, length);
		327	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	328
3031	serge	329	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		330	gpu_vaddr + swizzled_gpu_offset,
		331	this_length);
		332	if (ret)
		333	return ret + length;
2332	Serge	334
3031	serge	335	cpu_offset += this_length;
		336	gpu_offset += this_length;
		337	length -= this_length;
		338	}
		339
		340	return 0;
2332	Serge	341	}
		342
3031	serge	343	static inline int
		344	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		345	const char __user *cpu_vaddr,
		346	int length)
2332	Serge	347	{
3031	serge	348	int ret, cpu_offset = 0;
2332	Serge	349
		350	while (length > 0) {
		351	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		352	int this_length = min(cacheline_end - gpu_offset, length);
		353	int swizzled_gpu_offset = gpu_offset ^ 64;
		354
3031	serge	355	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	356	cpu_vaddr + cpu_offset,
		357	this_length);
3031	serge	358	if (ret)
		359	return ret + length;
		360
2332	Serge	361	cpu_offset += this_length;
		362	gpu_offset += this_length;
		363	length -= this_length;
		364	}
		365
3031	serge	366	return 0;
2332	Serge	367	}
		368
3031	serge	369	/* Per-page copy function for the shmem pread fastpath.
		370	* Flushes invalid cachelines before reading the target if
		371	* needs_clflush is set. */
2332	Serge	372	static int
3031	serge	373	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		374	char __user *user_data,
		375	bool page_do_bit17_swizzling, bool needs_clflush)
		376	{
		377	char *vaddr;
		378	int ret;
		379
		380	if (unlikely(page_do_bit17_swizzling))
		381	return -EINVAL;
		382
		383	vaddr = kmap_atomic(page);
		384	if (needs_clflush)
		385	drm_clflush_virt_range(vaddr + shmem_page_offset,
		386	page_length);
		387	ret = __copy_to_user_inatomic(user_data,
		388	vaddr + shmem_page_offset,
		389	page_length);
		390	kunmap_atomic(vaddr);
		391
		392	return ret ? -EFAULT : 0;
		393	}
		394
		395	static void
		396	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		397	bool swizzled)
		398	{
		399	if (unlikely(swizzled)) {
		400	unsigned long start = (unsigned long) addr;
		401	unsigned long end = (unsigned long) addr + length;
		402
		403	/* For swizzling simply ensure that we always flush both
		404	* channels. Lame, but simple and it works. Swizzled
		405	* pwrite/pread is far from a hotpath - current userspace
		406	* doesn't use it at all. */
		407	start = round_down(start, 128);
		408	end = round_up(end, 128);
		409
		410	drm_clflush_virt_range((void *)start, end - start);
		411	} else {
		412	drm_clflush_virt_range(addr, length);
		413	}
		414
		415	}
		416
		417	/* Only difference to the fast-path function is that this can handle bit17
		418	* and uses non-atomic copy and kmap functions. */
		419	static int
		420	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		421	char __user *user_data,
		422	bool page_do_bit17_swizzling, bool needs_clflush)
		423	{
		424	char *vaddr;
		425	int ret;
		426
		427	vaddr = kmap(page);
		428	if (needs_clflush)
		429	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		430	page_length,
		431	page_do_bit17_swizzling);
		432
		433	if (page_do_bit17_swizzling)
		434	ret = __copy_to_user_swizzled(user_data,
		435	vaddr, shmem_page_offset,
		436	page_length);
		437	else
		438	ret = __copy_to_user(user_data,
		439	vaddr + shmem_page_offset,
		440	page_length);
		441	kunmap(page);
		442
		443	return ret ? - EFAULT : 0;
		444	}
		445
		446	static int
		447	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	448	struct drm_i915_gem_object *obj,
		449	struct drm_i915_gem_pread *args,
		450	struct drm_file *file)
		451	{
3031	serge	452	char __user *user_data;
2332	Serge	453	ssize_t remain;
		454	loff_t offset;
3031	serge	455	int shmem_page_offset, page_length, ret = 0;
		456	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		457	int prefaulted = 0;
		458	int needs_clflush = 0;
3746	Serge	459	struct sg_page_iter sg_iter;
2332	Serge	460
3746	Serge	461	user_data = to_user_ptr(args->data_ptr);
2332	Serge	462	remain = args->size;
		463
3031	serge	464	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		465
		466	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		467	/* If we're not in the cpu read domain, set ourself into the gtt
		468	* read domain and manually flush cachelines (if required). This
		469	* optimizes for the case when the gpu will dirty the data
		470	* anyway again before the next pread happens. */
4104	Serge	471	needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
		472	if (i915_gem_obj_bound_any(obj)) {
3031	serge	473	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		474	if (ret)
		475	return ret;
		476	}
		477	}
		478
		479	ret = i915_gem_object_get_pages(obj);
		480	if (ret)
		481	return ret;
		482
		483	i915_gem_object_pin_pages(obj);
		484
2332	Serge	485	offset = args->offset;
		486
3746	Serge	487	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		488	offset >> PAGE_SHIFT) {
		489	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	490
3031	serge	491	if (remain <= 0)
		492	break;
		493
2332	Serge	494	/* Operation in this page
		495	*
3031	serge	496	* shmem_page_offset = offset within page in shmem file
2332	Serge	497	* page_length = bytes to copy for this page
		498	*/
3031	serge	499	shmem_page_offset = offset_in_page(offset);
2332	Serge	500	page_length = remain;
3031	serge	501	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		502	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	503
3031	serge	504	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		505	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	506
3031	serge	507	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		508	user_data, page_do_bit17_swizzling,
		509	needs_clflush);
		510	if (ret == 0)
		511	goto next_page;
2332	Serge	512
3031	serge	513	mutex_unlock(&dev->struct_mutex);
		514
4104	Serge	515	if (likely(!i915_prefault_disable) && !prefaulted) {
3031	serge	516	ret = fault_in_multipages_writeable(user_data, remain);
		517	/* Userspace is tricking us, but we've already clobbered
		518	* its pages with the prefault and promised to write the
		519	* data up to the first fault. Hence ignore any errors
		520	* and just continue. */
		521	(void)ret;
		522	prefaulted = 1;
		523	}
		524
		525	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		526	user_data, page_do_bit17_swizzling,
		527	needs_clflush);
		528
		529	mutex_lock(&dev->struct_mutex);
		530
		531	next_page:
2332	Serge	532	mark_page_accessed(page);
3031	serge	533
2332	Serge	534	if (ret)
3031	serge	535	goto out;
2332	Serge	536
		537	remain -= page_length;
		538	user_data += page_length;
		539	offset += page_length;
		540	}
		541
3031	serge	542	out:
		543	i915_gem_object_unpin_pages(obj);
		544
		545	return ret;
2332	Serge	546	}
		547
		548	/**
3031	serge	549	* Reads data from the object referenced by handle.
		550	*
		551	* On error, the contents of *data are undefined.
2332	Serge	552	*/
3031	serge	553	int
		554	i915_gem_pread_ioctl(struct drm_device dev, void data,
		555	struct drm_file *file)
		556	{
		557	struct drm_i915_gem_pread *args = data;
		558	struct drm_i915_gem_object *obj;
		559	int ret = 0;
		560
		561	if (args->size == 0)
		562	return 0;
		563
		564	if (!access_ok(VERIFY_WRITE,
3746	Serge	565	to_user_ptr(args->data_ptr),
3031	serge	566	args->size))
		567	return -EFAULT;
		568
		569	ret = i915_mutex_lock_interruptible(dev);
		570	if (ret)
		571	return ret;
		572
		573	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		574	if (&obj->base == NULL) {
		575	ret = -ENOENT;
		576	goto unlock;
		577	}
		578
		579	/* Bounds check source. */
		580	if (args->offset > obj->base.size \|\|
		581	args->size > obj->base.size - args->offset) {
		582	ret = -EINVAL;
		583	goto out;
		584	}
		585
		586	/* prime objects have no backing filp to GEM pread/pwrite
		587	* pages from.
		588	*/
		589	if (!obj->base.filp) {
		590	ret = -EINVAL;
		591	goto out;
		592	}
		593
		594	trace_i915_gem_object_pread(obj, args->offset, args->size);
		595
		596	ret = i915_gem_shmem_pread(dev, obj, args, file);
		597
		598	out:
		599	drm_gem_object_unreference(&obj->base);
		600	unlock:
		601	mutex_unlock(&dev->struct_mutex);
		602	return ret;
		603	}
		604
		605	/* This is the fast write path which cannot handle
		606	* page faults in the source data
		607	*/
		608
		609	static inline int
		610	fast_user_write(struct io_mapping *mapping,
		611	loff_t page_base, int page_offset,
		612	char __user *user_data,
		613	int length)
		614	{
		615	void __iomem *vaddr_atomic;
		616	void *vaddr;
		617	unsigned long unwritten;
		618
		619	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		620	/* We can use the cpu mem copy function because this is X86. */
		621	vaddr = (void __force*)vaddr_atomic + page_offset;
		622	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		623	user_data, length);
		624	io_mapping_unmap_atomic(vaddr_atomic);
		625	return unwritten;
		626	}
3260	Serge	627	#endif
3031	serge	628
3260	Serge	629	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	630	/**
		631	* This is the fast pwrite path, where we copy the data directly from the
		632	* user into the GTT, uncached.
		633	*/
2332	Serge	634	static int
3031	serge	635	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		636	struct drm_i915_gem_object *obj,
		637	struct drm_i915_gem_pwrite *args,
		638	struct drm_file *file)
2332	Serge	639	{
3031	serge	640	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	641	ssize_t remain;
3031	serge	642	loff_t offset, page_base;
		643	char __user *user_data;
		644	int page_offset, page_length, ret;
3260	Serge	645	char *vaddr;
2332	Serge	646
4104	Serge	647	ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
3031	serge	648	if (ret)
		649	goto out;
		650
		651	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		652	if (ret)
		653	goto out_unpin;
		654
		655	ret = i915_gem_object_put_fence(obj);
		656	if (ret)
		657	goto out_unpin;
		658
3260	Serge	659	vaddr = AllocKernelSpace(4096);
		660	if(vaddr == NULL)
		661	{
		662	ret = -ENOSPC;
		663	goto out_unpin;
		664	};
		665
3031	serge	666	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	667	remain = args->size;
		668
4104	Serge	669	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	670
3031	serge	671	while (remain > 0) {
		672	/* Operation in this page
		673	*
		674	* page_base = page offset within aperture
		675	* page_offset = offset within page
		676	* page_length = bytes to copy for this page
		677	*/
		678	page_base = offset & PAGE_MASK;
		679	page_offset = offset_in_page(offset);
		680	page_length = remain;
		681	if ((page_offset + remain) > PAGE_SIZE)
		682	page_length = PAGE_SIZE - page_offset;
2332	Serge	683
3260	Serge	684	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	685
3260	Serge	686	memcpy(vaddr+page_offset, user_data, page_length);
		687
3031	serge	688	remain -= page_length;
		689	user_data += page_length;
		690	offset += page_length;
2332	Serge	691	}
		692
3260	Serge	693	FreeKernelSpace(vaddr);
		694
3031	serge	695	out_unpin:
		696	i915_gem_object_unpin(obj);
		697	out:
		698	return ret;
		699	}
		700
		701	/* Per-page copy function for the shmem pwrite fastpath.
		702	* Flushes invalid cachelines before writing to the target if
		703	* needs_clflush_before is set and flushes out any written cachelines after
		704	* writing if needs_clflush is set. */
		705	static int
		706	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		707	char __user *user_data,
		708	bool page_do_bit17_swizzling,
		709	bool needs_clflush_before,
		710	bool needs_clflush_after)
		711	{
		712	char *vaddr;
3260	Serge	713	int ret = 0;
3031	serge	714
		715	if (unlikely(page_do_bit17_swizzling))
		716	return -EINVAL;
		717
3260	Serge	718	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	719	if (needs_clflush_before)
		720	drm_clflush_virt_range(vaddr + shmem_page_offset,
		721	page_length);
3260	Serge	722	memcpy(vaddr + shmem_page_offset,
3031	serge	723	user_data,
		724	page_length);
		725	if (needs_clflush_after)
		726	drm_clflush_virt_range(vaddr + shmem_page_offset,
		727	page_length);
3260	Serge	728	FreeKernelSpace(vaddr);
3031	serge	729
		730	return ret ? -EFAULT : 0;
		731	}
3260	Serge	732	#if 0
3031	serge	733
		734	/* Only difference to the fast-path function is that this can handle bit17
		735	* and uses non-atomic copy and kmap functions. */
		736	static int
		737	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		738	char __user *user_data,
		739	bool page_do_bit17_swizzling,
		740	bool needs_clflush_before,
		741	bool needs_clflush_after)
		742	{
		743	char *vaddr;
		744	int ret;
		745
		746	vaddr = kmap(page);
		747	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		748	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		749	page_length,
		750	page_do_bit17_swizzling);
		751	if (page_do_bit17_swizzling)
		752	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		753	user_data,
		754	page_length);
		755	else
		756	ret = __copy_from_user(vaddr + shmem_page_offset,
		757	user_data,
		758	page_length);
		759	if (needs_clflush_after)
		760	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		761	page_length,
		762	page_do_bit17_swizzling);
		763	kunmap(page);
		764
		765	return ret ? -EFAULT : 0;
		766	}
3260	Serge	767	#endif
3031	serge	768
3260	Serge	769
3031	serge	770	static int
		771	i915_gem_shmem_pwrite(struct drm_device *dev,
		772	struct drm_i915_gem_object *obj,
		773	struct drm_i915_gem_pwrite *args,
		774	struct drm_file *file)
		775	{
		776	ssize_t remain;
		777	loff_t offset;
		778	char __user *user_data;
		779	int shmem_page_offset, page_length, ret = 0;
		780	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		781	int hit_slowpath = 0;
		782	int needs_clflush_after = 0;
		783	int needs_clflush_before = 0;
3746	Serge	784	struct sg_page_iter sg_iter;
3031	serge	785
3746	Serge	786	user_data = to_user_ptr(args->data_ptr);
3031	serge	787	remain = args->size;
		788
		789	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		790
		791	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		792	/* If we're not in the cpu write domain, set ourself into the gtt
		793	* write domain and manually flush cachelines (if required). This
		794	* optimizes for the case when the gpu will use the data
		795	* right away and we therefore have to clflush anyway. */
4104	Serge	796	needs_clflush_after = cpu_write_needs_clflush(obj);
		797	if (i915_gem_obj_bound_any(obj)) {
3031	serge	798	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		799	if (ret)
		800	return ret;
		801	}
		802	}
4104	Serge	803	/* Same trick applies to invalidate partially written cachelines read
		804	* before writing. */
		805	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		806	needs_clflush_before =
		807	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	808
		809	ret = i915_gem_object_get_pages(obj);
2332	Serge	810	if (ret)
3031	serge	811	return ret;
2332	Serge	812
3031	serge	813	i915_gem_object_pin_pages(obj);
2332	Serge	814
		815	offset = args->offset;
3031	serge	816	obj->dirty = 1;
2332	Serge	817
3746	Serge	818	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		819	offset >> PAGE_SHIFT) {
		820	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	821	int partial_cacheline_write;
2332	Serge	822
3031	serge	823	if (remain <= 0)
		824	break;
		825
2332	Serge	826	/* Operation in this page
		827	*
		828	* shmem_page_offset = offset within page in shmem file
		829	* page_length = bytes to copy for this page
		830	*/
		831	shmem_page_offset = offset_in_page(offset);
		832
		833	page_length = remain;
		834	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		835	page_length = PAGE_SIZE - shmem_page_offset;
		836
3031	serge	837	/* If we don't overwrite a cacheline completely we need to be
		838	* careful to have up-to-date data by first clflushing. Don't
		839	* overcomplicate things and flush the entire patch. */
		840	partial_cacheline_write = needs_clflush_before &&
		841	((shmem_page_offset \| page_length)
3260	Serge	842	& (x86_clflush_size - 1));
2332	Serge	843
3031	serge	844	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		845	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	846
3031	serge	847	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		848	user_data, page_do_bit17_swizzling,
		849	partial_cacheline_write,
		850	needs_clflush_after);
		851	if (ret == 0)
		852	goto next_page;
		853
		854	hit_slowpath = 1;
		855	mutex_unlock(&dev->struct_mutex);
3260	Serge	856	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	857
3260	Serge	858	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		859	// user_data, page_do_bit17_swizzling,
		860	// partial_cacheline_write,
		861	// needs_clflush_after);
		862
3031	serge	863	mutex_lock(&dev->struct_mutex);
		864
		865	next_page:
2332	Serge	866
3031	serge	867	if (ret)
		868	goto out;
		869
2332	Serge	870	remain -= page_length;
3031	serge	871	user_data += page_length;
2332	Serge	872	offset += page_length;
		873	}
		874
		875	out:
3031	serge	876	i915_gem_object_unpin_pages(obj);
		877
		878	if (hit_slowpath) {
3480	Serge	879	/*
		880	* Fixup: Flush cpu caches in case we didn't flush the dirty
		881	* cachelines in-line while writing and the object moved
		882	* out of the cpu write domain while we've dropped the lock.
		883	*/
		884	if (!needs_clflush_after &&
		885	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	886	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	887	i915_gem_chipset_flush(dev);
3031	serge	888	}
2332	Serge	889	}
		890
3031	serge	891	if (needs_clflush_after)
3243	Serge	892	i915_gem_chipset_flush(dev);
3031	serge	893
2332	Serge	894	return ret;
		895	}
3031	serge	896
		897	/**
		898	* Writes data to the object referenced by handle.
		899	*
		900	* On error, the contents of the buffer that were to be modified are undefined.
		901	*/
		902	int
		903	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		904	struct drm_file *file)
		905	{
		906	struct drm_i915_gem_pwrite *args = data;
		907	struct drm_i915_gem_object *obj;
		908	int ret;
		909
4104	Serge	910	if (args->size == 0)
		911	return 0;
		912
3480	Serge	913	if(args->handle == -2)
		914	{
		915	printf("%s handle %d\n", __FUNCTION__, args->handle);
		916	return 0;
		917	}
		918
3031	serge	919	ret = i915_mutex_lock_interruptible(dev);
		920	if (ret)
		921	return ret;
		922
		923	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		924	if (&obj->base == NULL) {
		925	ret = -ENOENT;
		926	goto unlock;
		927	}
		928
		929	/* Bounds check destination. */
		930	if (args->offset > obj->base.size \|\|
		931	args->size > obj->base.size - args->offset) {
		932	ret = -EINVAL;
		933	goto out;
		934	}
		935
		936	/* prime objects have no backing filp to GEM pread/pwrite
		937	* pages from.
		938	*/
		939	if (!obj->base.filp) {
		940	ret = -EINVAL;
		941	goto out;
		942	}
		943
		944	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		945
		946	ret = -EFAULT;
		947	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		948	* it would end up going through the fenced access, and we'll get
		949	* different detiling behavior between reading and writing.
		950	* pread/pwrite currently are reading and writing from the CPU
		951	* perspective, requiring manual detiling by the client.
		952	*/
3260	Serge	953	// if (obj->phys_obj) {
		954	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		955	// goto out;
		956	// }
3031	serge	957
4104	Serge	958	if (obj->tiling_mode == I915_TILING_NONE &&
		959	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		960	cpu_write_needs_clflush(obj)) {
3031	serge	961	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		962	/* Note that the gtt paths might fail with non-page-backed user
		963	* pointers (e.g. gtt mappings when moving data between
		964	* textures). Fallback to the shmem path in that case. */
		965	}
		966
		967	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	968	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	969
		970	out:
		971	drm_gem_object_unreference(&obj->base);
		972	unlock:
		973	mutex_unlock(&dev->struct_mutex);
		974	return ret;
		975	}
		976
		977	int
3480	Serge	978	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	979	bool interruptible)
		980	{
3480	Serge	981	if (i915_reset_in_progress(error)) {
3031	serge	982	/* Non-interruptible callers can't handle -EAGAIN, hence return
		983	* -EIO unconditionally for these. */
		984	if (!interruptible)
		985	return -EIO;
2332	Serge	986
3480	Serge	987	/* Recovery complete, but the reset failed ... */
		988	if (i915_terminally_wedged(error))
3031	serge	989	return -EIO;
2332	Serge	990
3031	serge	991	return -EAGAIN;
		992	}
2332	Serge	993
3031	serge	994	return 0;
		995	}
2332	Serge	996
3031	serge	997	/*
		998	* Compare seqno against outstanding lazy request. Emit a request if they are
		999	* equal.
		1000	*/
		1001	static int
		1002	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1003	{
		1004	int ret;
2332	Serge	1005
3031	serge	1006	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1007
3031	serge	1008	ret = 0;
		1009	if (seqno == ring->outstanding_lazy_request)
4104	Serge	1010	ret = i915_add_request(ring, NULL);
2332	Serge	1011
3031	serge	1012	return ret;
		1013	}
2332	Serge	1014
3031	serge	1015	/**
		1016	* __wait_seqno - wait until execution of seqno has finished
		1017	* @ring: the ring expected to report seqno
		1018	* @seqno: duh!
3480	Serge	1019	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1020	* @interruptible: do an interruptible wait (normally yes)
		1021	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1022	*
3480	Serge	1023	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1024	* values have been read by the caller in an smp safe manner. Where read-side
		1025	* locks are involved, it is sufficient to read the reset_counter before
		1026	* unlocking the lock that protects the seqno. For lockless tricks, the
		1027	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1028	* inserted.
		1029	*
3031	serge	1030	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1031	* errno with remaining time filled in timeout argument.
		1032	*/
		1033	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1034	unsigned reset_counter,
3031	serge	1035	bool interruptible, struct timespec *timeout)
		1036	{
		1037	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1038	struct timespec before, now, wait_time={1,0};
		1039	unsigned long timeout_jiffies;
		1040	long end;
		1041	bool wait_forever = true;
		1042	int ret;
2332	Serge	1043
4104	Serge	1044	WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
		1045
3031	serge	1046	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1047	return 0;
2332	Serge	1048
3031	serge	1049	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1050
3031	serge	1051	if (timeout != NULL) {
		1052	wait_time = *timeout;
		1053	wait_forever = false;
		1054	}
2332	Serge	1055
4104	Serge	1056	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
2332	Serge	1057
3031	serge	1058	if (WARN_ON(!ring->irq_get(ring)))
		1059	return -ENODEV;
2332	Serge	1060
3031	serge	1061	/* Record current time in case interrupted by signal, or wedged * */
		1062	getrawmonotonic(&before);
2332	Serge	1063
3031	serge	1064	#define EXIT_COND \
		1065	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1066	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1067	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1068	do {
3266	Serge	1069	if (interruptible)
		1070	end = wait_event_interruptible_timeout(ring->irq_queue,
		1071	EXIT_COND,
		1072	timeout_jiffies);
		1073	else
3031	serge	1074	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1075	timeout_jiffies);
2332	Serge	1076
3480	Serge	1077	/* We need to check whether any gpu reset happened in between
		1078	* the caller grabbing the seqno and now ... */
		1079	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1080	end = -EAGAIN;
		1081
		1082	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1083	* gone. */
		1084	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1085	if (ret)
		1086	end = ret;
		1087	} while (end == 0 && wait_forever);
2332	Serge	1088
3031	serge	1089	getrawmonotonic(&now);
2332	Serge	1090
3031	serge	1091	ring->irq_put(ring);
		1092	trace_i915_gem_request_wait_end(ring, seqno);
		1093	#undef EXIT_COND
2332	Serge	1094
3031	serge	1095	if (timeout) {
4104	Serge	1096	// struct timespec sleep_time = timespec_sub(now, before);
		1097	// timeout = timespec_sub(timeout, sleep_time);
3031	serge	1098	}
2332	Serge	1099
3031	serge	1100	switch (end) {
		1101	case -EIO:
		1102	case -EAGAIN: /* Wedged */
		1103	case -ERESTARTSYS: /* Signal */
		1104	return (int)end;
		1105	case 0: /* Timeout */
		1106	return -ETIME;
		1107	default: /* Completed */
		1108	WARN_ON(end < 0); /* We're not aware of other errors */
		1109	return 0;
		1110	}
		1111	}
2332	Serge	1112
3031	serge	1113	/**
		1114	* Waits for a sequence number to be signaled, and cleans up the
		1115	* request and object lists appropriately for that event.
		1116	*/
		1117	int
		1118	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1119	{
		1120	struct drm_device *dev = ring->dev;
		1121	struct drm_i915_private *dev_priv = dev->dev_private;
		1122	bool interruptible = dev_priv->mm.interruptible;
		1123	int ret;
2332	Serge	1124
3031	serge	1125	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1126	BUG_ON(seqno == 0);
2332	Serge	1127
3480	Serge	1128	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1129	if (ret)
		1130	return ret;
2332	Serge	1131
3031	serge	1132	ret = i915_gem_check_olr(ring, seqno);
		1133	if (ret)
		1134	return ret;
2332	Serge	1135
3480	Serge	1136	return __wait_seqno(ring, seqno,
		1137	atomic_read(&dev_priv->gpu_error.reset_counter),
		1138	interruptible, NULL);
3031	serge	1139	}
2332	Serge	1140
4104	Serge	1141	static int
		1142	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
		1143	struct intel_ring_buffer *ring)
		1144	{
		1145	i915_gem_retire_requests_ring(ring);
		1146
		1147	/* Manually manage the write flush as we may have not yet
		1148	* retired the buffer.
		1149	*
		1150	* Note that the last_write_seqno is always the earlier of
		1151	* the two (read/write) seqno, so if we haved successfully waited,
		1152	* we know we have passed the last write.
		1153	*/
		1154	obj->last_write_seqno = 0;
		1155	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1156
		1157	return 0;
		1158	}
		1159
3031	serge	1160	/**
		1161	* Ensures that all rendering to the object has completed and the object is
		1162	* safe to unbind from the GTT or access from the CPU.
		1163	*/
		1164	static __must_check int
		1165	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1166	bool readonly)
		1167	{
		1168	struct intel_ring_buffer *ring = obj->ring;
		1169	u32 seqno;
		1170	int ret;
2332	Serge	1171
3031	serge	1172	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1173	if (seqno == 0)
		1174	return 0;
2332	Serge	1175
3031	serge	1176	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1177	if (ret)
		1178	return ret;
2332	Serge	1179
4104	Serge	1180	return i915_gem_object_wait_rendering__tail(obj, ring);
3031	serge	1181	}
2332	Serge	1182
3260	Serge	1183	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1184	* as the object state may change during this call.
		1185	*/
		1186	static __must_check int
		1187	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1188	bool readonly)
		1189	{
		1190	struct drm_device *dev = obj->base.dev;
		1191	struct drm_i915_private *dev_priv = dev->dev_private;
		1192	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1193	unsigned reset_counter;
3260	Serge	1194	u32 seqno;
		1195	int ret;
2332	Serge	1196
3260	Serge	1197	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1198	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1199
3260	Serge	1200	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1201	if (seqno == 0)
		1202	return 0;
2332	Serge	1203
3480	Serge	1204	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1205	if (ret)
		1206	return ret;
2332	Serge	1207
3260	Serge	1208	ret = i915_gem_check_olr(ring, seqno);
		1209	if (ret)
		1210	return ret;
2332	Serge	1211
3480	Serge	1212	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1213	mutex_unlock(&dev->struct_mutex);
3480	Serge	1214	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1215	mutex_lock(&dev->struct_mutex);
4104	Serge	1216	if (ret)
		1217	return ret;
2332	Serge	1218
4104	Serge	1219	return i915_gem_object_wait_rendering__tail(obj, ring);
3260	Serge	1220	}
2332	Serge	1221
3260	Serge	1222	/**
		1223	* Called when user space prepares to use an object with the CPU, either
		1224	* through the mmap ioctl's mapping or a GTT mapping.
		1225	*/
		1226	int
		1227	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1228	struct drm_file *file)
		1229	{
		1230	struct drm_i915_gem_set_domain *args = data;
		1231	struct drm_i915_gem_object *obj;
		1232	uint32_t read_domains = args->read_domains;
		1233	uint32_t write_domain = args->write_domain;
		1234	int ret;
2332	Serge	1235
3480	Serge	1236
		1237	if(args->handle == -2)
		1238	{
		1239	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1240	return 0;
		1241	}
		1242
3260	Serge	1243	/* Only handle setting domains to types used by the CPU. */
		1244	if (write_domain & I915_GEM_GPU_DOMAINS)
		1245	return -EINVAL;
2332	Serge	1246
3260	Serge	1247	if (read_domains & I915_GEM_GPU_DOMAINS)
		1248	return -EINVAL;
2332	Serge	1249
3260	Serge	1250	/* Having something in the write domain implies it's in the read
		1251	* domain, and only that read domain. Enforce that in the request.
		1252	*/
		1253	if (write_domain != 0 && read_domains != write_domain)
		1254	return -EINVAL;
2332	Serge	1255
3260	Serge	1256	ret = i915_mutex_lock_interruptible(dev);
		1257	if (ret)
		1258	return ret;
2332	Serge	1259
3260	Serge	1260	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1261	if (&obj->base == NULL) {
		1262	ret = -ENOENT;
		1263	goto unlock;
		1264	}
2332	Serge	1265
3260	Serge	1266	/* Try to flush the object off the GPU without holding the lock.
		1267	* We will repeat the flush holding the lock in the normal manner
		1268	* to catch cases where we are gazumped.
		1269	*/
		1270	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1271	if (ret)
		1272	goto unref;
2332	Serge	1273
3260	Serge	1274	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1275	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1276
3260	Serge	1277	/* Silently promote "you're not bound, there was nothing to do"
		1278	* to success, since the client was just asking us to
		1279	* make sure everything was done.
		1280	*/
		1281	if (ret == -EINVAL)
		1282	ret = 0;
		1283	} else {
		1284	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1285	}
2332	Serge	1286
3260	Serge	1287	unref:
		1288	drm_gem_object_unreference(&obj->base);
		1289	unlock:
		1290	mutex_unlock(&dev->struct_mutex);
		1291	return ret;
		1292	}
2332	Serge	1293
		1294
		1295
		1296
		1297
		1298
3260	Serge	1299	/**
		1300	* Maps the contents of an object, returning the address it is mapped
		1301	* into.
		1302	*
		1303	* While the mapping holds a reference on the contents of the object, it doesn't
		1304	* imply a ref on the object itself.
		1305	*/
		1306	int
		1307	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1308	struct drm_file *file)
		1309	{
		1310	struct drm_i915_gem_mmap *args = data;
		1311	struct drm_gem_object *obj;
		1312	unsigned long addr = 0;
2332	Serge	1313
3480	Serge	1314	if(args->handle == -2)
		1315	{
		1316	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1317	return 0;
		1318	}
		1319
3260	Serge	1320	obj = drm_gem_object_lookup(dev, file, args->handle);
		1321	if (obj == NULL)
		1322	return -ENOENT;
4104	Serge	1323
3298	Serge	1324	//dbgprintf("%s offset %lld size %lld\n",
		1325	// __FUNCTION__, args->offset, args->size);
3260	Serge	1326	/* prime objects have no backing filp to GEM mmap
		1327	* pages from.
		1328	*/
		1329	if (!obj->filp) {
		1330	drm_gem_object_unreference_unlocked(obj);
		1331	return -EINVAL;
		1332	}
2332	Serge	1333
3263	Serge	1334	addr = vm_mmap(obj->filp, 0, args->size,
		1335	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1336	args->offset);
3260	Serge	1337	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1338	if (IS_ERR((void *)addr))
		1339	return addr;
2332	Serge	1340
3260	Serge	1341	args->addr_ptr = (uint64_t) addr;
2332	Serge	1342
3263	Serge	1343	return 0;
3260	Serge	1344	}
2332	Serge	1345
		1346
		1347
		1348
		1349
		1350
		1351
		1352
3031	serge	1353
		1354
		1355
		1356
		1357
		1358	/**
		1359	* i915_gem_release_mmap - remove physical page mappings
		1360	* @obj: obj in question
		1361	*
		1362	* Preserve the reservation of the mmapping with the DRM core code, but
		1363	* relinquish ownership of the pages back to the system.
		1364	*
		1365	* It is vital that we remove the page mapping if we have mapped a tiled
		1366	* object through the GTT and then lose the fence register due to
		1367	* resource pressure. Similarly if the object has been moved out of the
		1368	* aperture, than pages mapped into userspace must be revoked. Removing the
		1369	* mapping will then trigger a page fault on the next user access, allowing
		1370	* fixup by i915_gem_fault().
		1371	*/
		1372	void
		1373	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1374	{
		1375	if (!obj->fault_mappable)
		1376	return;
		1377
4104	Serge	1378	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1379	obj->fault_mappable = false;
		1380	}
		1381
3480	Serge	1382	uint32_t
2332	Serge	1383	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1384	{
		1385	uint32_t gtt_size;
		1386
		1387	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1388	tiling_mode == I915_TILING_NONE)
		1389	return size;
		1390
		1391	/* Previous chips need a power-of-two fence region when tiling */
		1392	if (INTEL_INFO(dev)->gen == 3)
		1393	gtt_size = 1024*1024;
		1394	else
		1395	gtt_size = 512*1024;
		1396
		1397	while (gtt_size < size)
		1398	gtt_size <<= 1;
		1399
		1400	return gtt_size;
		1401	}
		1402
		1403	/**
		1404	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1405	* @obj: object to check
		1406	*
		1407	* Return the required GTT alignment for an object, taking into account
		1408	* potential fence register mapping.
		1409	*/
3480	Serge	1410	uint32_t
		1411	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1412	int tiling_mode, bool fenced)
2332	Serge	1413	{
		1414	/*
		1415	* Minimum alignment is 4k (GTT page size), but might be greater
		1416	* if a fence register is needed for the object.
		1417	*/
3480	Serge	1418	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1419	tiling_mode == I915_TILING_NONE)
		1420	return 4096;
		1421
		1422	/*
		1423	* Previous chips need to be aligned to the size of the smallest
		1424	* fence register that can contain the object.
		1425	*/
		1426	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1427	}
		1428
		1429	/**
		1430	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1431	* unfenced object
		1432	* @dev: the device
		1433	* @size: size of the object
		1434	* @tiling_mode: tiling mode of the object
		1435	*
		1436	* Return the required GTT alignment for an object, only taking into account
		1437	* unfenced tiled surface requirements.
		1438	*/
		1439	uint32_t
		1440	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1441	uint32_t size,
		1442	int tiling_mode)
		1443	{
		1444	/*
		1445	* Minimum alignment is 4k (GTT page size) for sane hw.
		1446	*/
		1447	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1448	tiling_mode == I915_TILING_NONE)
		1449	return 4096;
		1450
		1451	/* Previous hardware however needs to be aligned to a power-of-two
		1452	* tile height. The simplest method for determining this is to reuse
		1453	* the power-of-tile object size.
		1454	*/
		1455	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1456	}
		1457
3480	Serge	1458	int
		1459	i915_gem_mmap_gtt(struct drm_file *file,
		1460	struct drm_device *dev,
		1461	uint32_t handle,
		1462	uint64_t *offset)
		1463	{
		1464	struct drm_i915_private *dev_priv = dev->dev_private;
		1465	struct drm_i915_gem_object *obj;
		1466	unsigned long pfn;
		1467	char mem, ptr;
		1468	int ret;
		1469
		1470	ret = i915_mutex_lock_interruptible(dev);
		1471	if (ret)
		1472	return ret;
		1473
		1474	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1475	if (&obj->base == NULL) {
		1476	ret = -ENOENT;
		1477	goto unlock;
		1478	}
		1479
		1480	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1481	ret = -E2BIG;
		1482	goto out;
		1483	}
		1484
		1485	if (obj->madv != I915_MADV_WILLNEED) {
		1486	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1487	ret = -EINVAL;
		1488	goto out;
		1489	}
		1490	/* Now bind it into the GTT if needed */
4104	Serge	1491	ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
3480	Serge	1492	if (ret)
		1493	goto out;
		1494
		1495	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1496	if (ret)
		1497	goto unpin;
		1498
		1499	ret = i915_gem_object_get_fence(obj);
		1500	if (ret)
		1501	goto unpin;
		1502
		1503	obj->fault_mappable = true;
		1504
4104	Serge	1505	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1506
		1507	/* Finally, remap it using the new GTT offset */
		1508
		1509	mem = UserAlloc(obj->base.size);
		1510	if(unlikely(mem == NULL))
		1511	{
		1512	ret = -ENOMEM;
		1513	goto unpin;
		1514	}
		1515
		1516	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1517	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1518
		1519	unpin:
		1520	i915_gem_object_unpin(obj);
		1521
		1522
4104	Serge	1523	*offset = mem;
3480	Serge	1524
		1525	out:
		1526	drm_gem_object_unreference(&obj->base);
		1527	unlock:
		1528	mutex_unlock(&dev->struct_mutex);
		1529	return ret;
		1530	}
		1531
		1532	/**
		1533	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1534	* @dev: DRM device
		1535	* @data: GTT mapping ioctl data
		1536	* @file: GEM object info
		1537	*
		1538	* Simply returns the fake offset to userspace so it can mmap it.
		1539	* The mmap call will end up in drm_gem_mmap(), which will set things
		1540	* up so we can get faults in the handler above.
		1541	*
		1542	* The fault handler will take care of binding the object into the GTT
		1543	* (since it may have been evicted to make room for something), allocating
		1544	* a fence register, and mapping the appropriate aperture address into
		1545	* userspace.
		1546	*/
		1547	int
		1548	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1549	struct drm_file *file)
		1550	{
		1551	struct drm_i915_gem_mmap_gtt *args = data;
		1552
		1553	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1554	}
		1555
3031	serge	1556	/* Immediately discard the backing storage */
		1557	static void
		1558	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1559	{
		1560	// struct inode *inode;
2332	Serge	1561
3031	serge	1562	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1563
3263	Serge	1564	if (obj->base.filp == NULL)
		1565	return;
2332	Serge	1566
3031	serge	1567	/* Our goal here is to return as much of the memory as
		1568	* is possible back to the system as we are called from OOM.
		1569	* To do this we must instruct the shmfs to drop all of its
		1570	* backing pages, now.
		1571	*/
		1572	// inode = obj->base.filp->f_path.dentry->d_inode;
		1573	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1574
3031	serge	1575	obj->madv = __I915_MADV_PURGED;
		1576	}
2332	Serge	1577
3031	serge	1578	static inline int
		1579	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1580	{
		1581	return obj->madv == I915_MADV_DONTNEED;
		1582	}
2332	Serge	1583
3031	serge	1584	static void
		1585	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1586	{
3746	Serge	1587	struct sg_page_iter sg_iter;
		1588	int ret;
2332	Serge	1589
3031	serge	1590	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1591
3031	serge	1592	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1593	if (ret) {
		1594	/* In the event of a disaster, abandon all caches and
		1595	* hope for the best.
		1596	*/
		1597	WARN_ON(ret != -EIO);
4104	Serge	1598	i915_gem_clflush_object(obj, true);
3031	serge	1599	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1600	}
2332	Serge	1601
3031	serge	1602	if (obj->madv == I915_MADV_DONTNEED)
		1603	obj->dirty = 0;
2332	Serge	1604
3746	Serge	1605	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1606	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1607
3290	Serge	1608	page_cache_release(page);
3243	Serge	1609	}
		1610	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1611
4104	Serge	1612	obj->dirty = 0;
3243	Serge	1613
		1614	sg_free_table(obj->pages);
		1615	kfree(obj->pages);
3031	serge	1616	}
2332	Serge	1617
3480	Serge	1618	int
3031	serge	1619	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1620	{
		1621	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1622
3243	Serge	1623	if (obj->pages == NULL)
3031	serge	1624	return 0;
2332	Serge	1625
3031	serge	1626	if (obj->pages_pin_count)
		1627	return -EBUSY;
		1628
4104	Serge	1629	BUG_ON(i915_gem_obj_bound_any(obj));
		1630
3243	Serge	1631	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1632	* array, hence protect them from being reaped by removing them from gtt
		1633	* lists early. */
4104	Serge	1634	list_del(&obj->global_list);
3243	Serge	1635
3031	serge	1636	ops->put_pages(obj);
3243	Serge	1637	obj->pages = NULL;
3031	serge	1638
		1639	if (i915_gem_object_is_purgeable(obj))
		1640	i915_gem_object_truncate(obj);
		1641
		1642	return 0;
		1643	}
		1644
		1645
		1646
		1647
		1648
		1649
		1650
		1651
2332	Serge	1652	static int
3031	serge	1653	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1654	{
3260	Serge	1655	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1656	int page_count, i;
4104	Serge	1657	struct sg_table *st;
3243	Serge	1658	struct scatterlist *sg;
3746	Serge	1659	struct sg_page_iter sg_iter;
3243	Serge	1660	struct page *page;
3746	Serge	1661	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1662	gfp_t gfp;
2332	Serge	1663
3243	Serge	1664	/* Assert that the object is not currently in any GPU domain. As it
		1665	* wasn't in the GTT, there shouldn't be any way it could have been in
		1666	* a GPU cache
2332	Serge	1667	*/
3243	Serge	1668	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1669	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1670
		1671	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1672	if (st == NULL)
		1673	return -ENOMEM;
		1674
2332	Serge	1675	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1676	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1677	kfree(st);
3746	Serge	1678	FAIL();
2332	Serge	1679	return -ENOMEM;
3243	Serge	1680	}
2332	Serge	1681
3243	Serge	1682	/* Get the list of pages out of our struct file. They'll be pinned
		1683	* at this point until we release them.
		1684	*
		1685	* Fail silently without starting the shrinker
		1686	*/
3746	Serge	1687	sg = st->sgl;
		1688	st->nents = 0;
		1689	for (i = 0; i < page_count; i++) {
4104	Serge	1690	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1691	if (IS_ERR(page)) {
		1692	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1693	goto err_pages;
		1694
3260	Serge	1695	}
3746	Serge	1696
		1697	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1698	if (i)
		1699	sg = sg_next(sg);
		1700	st->nents++;
3243	Serge	1701	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1702	} else {
		1703	sg->length += PAGE_SIZE;
		1704	}
		1705	last_pfn = page_to_pfn(page);
3243	Serge	1706	}
3031	serge	1707
3746	Serge	1708	sg_mark_end(sg);
3243	Serge	1709	obj->pages = st;
3031	serge	1710
2332	Serge	1711	return 0;
		1712
		1713	err_pages:
3746	Serge	1714	sg_mark_end(sg);
		1715	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1716	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1717	sg_free_table(st);
		1718	kfree(st);
3746	Serge	1719	FAIL();
3243	Serge	1720	return PTR_ERR(page);
2332	Serge	1721	}
		1722
3031	serge	1723	/* Ensure that the associated pages are gathered from the backing storage
		1724	* and pinned into our object. i915_gem_object_get_pages() may be called
		1725	* multiple times before they are released by a single call to
		1726	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1727	* either as a result of memory pressure (reaping pages under the shrinker)
		1728	* or as the object is itself released.
		1729	*/
		1730	int
		1731	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1732	{
3031	serge	1733	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1734	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1735	int ret;
2332	Serge	1736
3243	Serge	1737	if (obj->pages)
3031	serge	1738	return 0;
2332	Serge	1739
3031	serge	1740	BUG_ON(obj->pages_pin_count);
2332	Serge	1741
3031	serge	1742	ret = ops->get_pages(obj);
		1743	if (ret)
		1744	return ret;
2344	Serge	1745
4104	Serge	1746	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1747	return 0;
2332	Serge	1748	}
		1749
		1750	void
		1751	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1752	struct intel_ring_buffer *ring)
2332	Serge	1753	{
		1754	struct drm_device *dev = obj->base.dev;
		1755	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1756	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1757
		1758	BUG_ON(ring == NULL);
4104	Serge	1759	if (obj->ring != ring && obj->last_write_seqno) {
		1760	/* Keep the seqno relative to the current ring */
		1761	obj->last_write_seqno = seqno;
		1762	}
2332	Serge	1763	obj->ring = ring;
		1764
		1765	/* Add a reference if we're newly entering the active list. */
		1766	if (!obj->active) {
2344	Serge	1767	drm_gem_object_reference(&obj->base);
2332	Serge	1768	obj->active = 1;
		1769	}
		1770
		1771	list_move_tail(&obj->ring_list, &ring->active_list);
		1772
3031	serge	1773	obj->last_read_seqno = seqno;
		1774
2332	Serge	1775	if (obj->fenced_gpu_access) {
3031	serge	1776	obj->last_fenced_seqno = seqno;
		1777
		1778	/* Bump MRU to take account of the delayed flush */
		1779	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1780	struct drm_i915_fence_reg *reg;
		1781
		1782	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1783	list_move_tail(®->lru_list,
		1784	&dev_priv->mm.fence_list);
		1785	}
2332	Serge	1786	}
		1787	}
		1788
2344	Serge	1789	static void
3031	serge	1790	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1791	{
4104	Serge	1792	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1793	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
		1794	struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2332	Serge	1795
3031	serge	1796	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1797	BUG_ON(!obj->active);
2332	Serge	1798
4104	Serge	1799	list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2344	Serge	1800
3031	serge	1801	list_del_init(&obj->ring_list);
2352	Serge	1802	obj->ring = NULL;
2344	Serge	1803
3031	serge	1804	obj->last_read_seqno = 0;
		1805	obj->last_write_seqno = 0;
		1806	obj->base.write_domain = 0;
		1807
		1808	obj->last_fenced_seqno = 0;
2352	Serge	1809	obj->fenced_gpu_access = false;
2344	Serge	1810
2352	Serge	1811	obj->active = 0;
		1812	drm_gem_object_unreference(&obj->base);
		1813
		1814	WARN_ON(i915_verify_lists(dev));
		1815	}
		1816
3243	Serge	1817	static int
3480	Serge	1818	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1819	{
3243	Serge	1820	struct drm_i915_private *dev_priv = dev->dev_private;
		1821	struct intel_ring_buffer *ring;
		1822	int ret, i, j;
2344	Serge	1823
3480	Serge	1824	/* Carefully retire all requests without writing to the rings */
3243	Serge	1825	for_each_ring(ring, dev_priv, i) {
3480	Serge	1826	ret = intel_ring_idle(ring);
3243	Serge	1827	if (ret)
		1828	return ret;
3480	Serge	1829	}
		1830	i915_gem_retire_requests(dev);
3243	Serge	1831
3480	Serge	1832	/* Finally reset hw state */
3243	Serge	1833	for_each_ring(ring, dev_priv, i) {
3480	Serge	1834	intel_ring_init_seqno(ring, seqno);
		1835
3243	Serge	1836	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1837	ring->sync_seqno[j] = 0;
		1838	}
		1839
		1840	return 0;
2344	Serge	1841	}
		1842
3480	Serge	1843	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1844	{
		1845	struct drm_i915_private *dev_priv = dev->dev_private;
		1846	int ret;
		1847
		1848	if (seqno == 0)
		1849	return -EINVAL;
		1850
		1851	/* HWS page needs to be set less than what we
		1852	* will inject to ring
		1853	*/
		1854	ret = i915_gem_init_seqno(dev, seqno - 1);
		1855	if (ret)
		1856	return ret;
		1857
		1858	/* Carefully set the last_seqno value so that wrap
		1859	* detection still works
		1860	*/
		1861	dev_priv->next_seqno = seqno;
		1862	dev_priv->last_seqno = seqno - 1;
		1863	if (dev_priv->last_seqno == 0)
		1864	dev_priv->last_seqno--;
		1865
		1866	return 0;
		1867	}
		1868
3243	Serge	1869	int
		1870	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1871	{
3243	Serge	1872	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1873
3243	Serge	1874	/* reserve 0 for non-seqno */
		1875	if (dev_priv->next_seqno == 0) {
3480	Serge	1876	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1877	if (ret)
		1878	return ret;
		1879
		1880	dev_priv->next_seqno = 1;
		1881	}
		1882
3480	Serge	1883	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1884	return 0;
2332	Serge	1885	}
		1886
4104	Serge	1887	int __i915_add_request(struct intel_ring_buffer *ring,
2352	Serge	1888	struct drm_file *file,
4104	Serge	1889	struct drm_i915_gem_object *obj,
3031	serge	1890	u32 *out_seqno)
2352	Serge	1891	{
		1892	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1893	struct drm_i915_gem_request *request;
4104	Serge	1894	u32 request_ring_position, request_start;
2352	Serge	1895	int was_empty;
		1896	int ret;
2332	Serge	1897
4104	Serge	1898	request_start = intel_ring_get_tail(ring);
3031	serge	1899	/*
		1900	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1901	* after having emitted the batchbuffer command. Hence we need to fix
		1902	* things up similar to emitting the lazy request. The difference here
		1903	* is that the flush _must_ happen before the next request, no matter
		1904	* what.
		1905	*/
4104	Serge	1906	ret = intel_ring_flush_all_caches(ring);
		1907	if (ret)
		1908	return ret;
2332	Serge	1909
3031	serge	1910	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1911	if (request == NULL)
		1912	return -ENOMEM;
		1913
		1914
		1915	/* Record the position of the start of the request so that
		1916	* should we detect the updated seqno part-way through the
4104	Serge	1917	* GPU processing the request, we never over-estimate the
3031	serge	1918	* position of the head.
		1919	*/
4104	Serge	1920	request_ring_position = intel_ring_get_tail(ring);
3031	serge	1921
3243	Serge	1922	ret = ring->add_request(ring);
3031	serge	1923	if (ret) {
		1924	kfree(request);
4104	Serge	1925	return ret;
3031	serge	1926	}
2332	Serge	1927
3243	Serge	1928	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1929	request->ring = ring;
4104	Serge	1930	request->head = request_start;
3031	serge	1931	request->tail = request_ring_position;
4104	Serge	1932	request->ctx = ring->last_context;
		1933	request->batch_obj = obj;
		1934
		1935	/* Whilst this request exists, batch_obj will be on the
		1936	* active_list, and so will hold the active reference. Only when this
		1937	* request is retired will the the batch_obj be moved onto the
		1938	* inactive_list and lose its active reference. Hence we do not need
		1939	* to explicitly hold another reference here.
		1940	*/
		1941
		1942	if (request->ctx)
		1943	i915_gem_context_reference(request->ctx);
		1944
3031	serge	1945	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1946	was_empty = list_empty(&ring->request_list);
		1947	list_add_tail(&request->list, &ring->request_list);
3031	serge	1948	request->file_priv = NULL;
2332	Serge	1949
3263	Serge	1950	if (file) {
		1951	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1952
3263	Serge	1953	spin_lock(&file_priv->mm.lock);
		1954	request->file_priv = file_priv;
		1955	list_add_tail(&request->client_list,
		1956	&file_priv->mm.request_list);
		1957	spin_unlock(&file_priv->mm.lock);
		1958	}
		1959
		1960	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1961	ring->outstanding_lazy_request = 0;
2332	Serge	1962
4104	Serge	1963	if (!dev_priv->ums.mm_suspended) {
		1964	// i915_queue_hangcheck(ring->dev);
		1965
		1966	if (was_empty) {
2360	Serge	1967	queue_delayed_work(dev_priv->wq,
3482	Serge	1968	&dev_priv->mm.retire_work,
		1969	round_jiffies_up_relative(HZ));
4104	Serge	1970	intel_mark_busy(dev_priv->dev);
		1971	}
		1972	}
3031	serge	1973
		1974	if (out_seqno)
3243	Serge	1975	*out_seqno = request->seqno;
2352	Serge	1976	return 0;
		1977	}
2332	Serge	1978
3263	Serge	1979	static inline void
		1980	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1981	{
		1982	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1983
3263	Serge	1984	if (!file_priv)
		1985	return;
2332	Serge	1986
3263	Serge	1987	spin_lock(&file_priv->mm.lock);
		1988	if (request->file_priv) {
		1989	list_del(&request->client_list);
		1990	request->file_priv = NULL;
		1991	}
		1992	spin_unlock(&file_priv->mm.lock);
		1993	}
2332	Serge	1994
4104	Serge	1995	static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
		1996	struct i915_address_space *vm)
		1997	{
		1998	if (acthd >= i915_gem_obj_offset(obj, vm) &&
		1999	acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
		2000	return true;
		2001
		2002	return false;
		2003	}
		2004
		2005	static bool i915_head_inside_request(const u32 acthd_unmasked,
		2006	const u32 request_start,
		2007	const u32 request_end)
		2008	{
		2009	const u32 acthd = acthd_unmasked & HEAD_ADDR;
		2010
		2011	if (request_start < request_end) {
		2012	if (acthd >= request_start && acthd < request_end)
		2013	return true;
		2014	} else if (request_start > request_end) {
		2015	if (acthd >= request_start \|\| acthd < request_end)
		2016	return true;
		2017	}
		2018
		2019	return false;
		2020	}
		2021
		2022	static struct i915_address_space *
		2023	request_to_vm(struct drm_i915_gem_request *request)
		2024	{
		2025	struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
		2026	struct i915_address_space *vm;
		2027
		2028	vm = &dev_priv->gtt.base;
		2029
		2030	return vm;
		2031	}
		2032
		2033	static bool i915_request_guilty(struct drm_i915_gem_request *request,
		2034	const u32 acthd, bool *inside)
		2035	{
		2036	/* There is a possibility that unmasked head address
		2037	* pointing inside the ring, matches the batch_obj address range.
		2038	* However this is extremely unlikely.
		2039	*/
		2040	if (request->batch_obj) {
		2041	if (i915_head_inside_object(acthd, request->batch_obj,
		2042	request_to_vm(request))) {
		2043	*inside = true;
		2044	return true;
		2045	}
		2046	}
		2047
		2048	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		2049	*inside = false;
		2050	return true;
		2051	}
		2052
		2053	return false;
		2054	}
		2055
		2056	static void i915_set_reset_status(struct intel_ring_buffer *ring,
		2057	struct drm_i915_gem_request *request,
		2058	u32 acthd)
		2059	{
		2060	struct i915_ctx_hang_stats *hs = NULL;
		2061	bool inside, guilty;
		2062	unsigned long offset = 0;
		2063
		2064	/* Innocent until proven guilty */
		2065	guilty = false;
		2066
		2067	if (request->batch_obj)
		2068	offset = i915_gem_obj_offset(request->batch_obj,
		2069	request_to_vm(request));
		2070
		2071	if (ring->hangcheck.action != HANGCHECK_WAIT &&
		2072	i915_request_guilty(request, acthd, &inside)) {
		2073	DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
		2074	ring->name,
		2075	inside ? "inside" : "flushing",
		2076	offset,
		2077	request->ctx ? request->ctx->id : 0,
		2078	acthd);
		2079
		2080	guilty = true;
		2081	}
		2082
		2083	/* If contexts are disabled or this is the default context, use
		2084	* file_priv->reset_state
		2085	*/
		2086	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		2087	hs = &request->ctx->hang_stats;
		2088	else if (request->file_priv)
		2089	hs = &request->file_priv->hang_stats;
		2090
		2091	if (hs) {
		2092	if (guilty)
		2093	hs->batch_active++;
		2094	else
		2095	hs->batch_pending++;
		2096	}
		2097	}
		2098
		2099	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2100	{
		2101	list_del(&request->list);
		2102	i915_gem_request_remove_from_client(request);
		2103
		2104	if (request->ctx)
		2105	i915_gem_context_unreference(request->ctx);
		2106
		2107	kfree(request);
		2108	}
		2109
3031	serge	2110	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		2111	struct intel_ring_buffer *ring)
		2112	{
4104	Serge	2113	u32 completed_seqno;
		2114	u32 acthd;
		2115
		2116	acthd = intel_ring_get_active_head(ring);
		2117	completed_seqno = ring->get_seqno(ring, false);
		2118
3031	serge	2119	while (!list_empty(&ring->request_list)) {
		2120	struct drm_i915_gem_request *request;
2332	Serge	2121
3031	serge	2122	request = list_first_entry(&ring->request_list,
		2123	struct drm_i915_gem_request,
		2124	list);
2332	Serge	2125
4104	Serge	2126	if (request->seqno > completed_seqno)
		2127	i915_set_reset_status(ring, request, acthd);
		2128
		2129	i915_gem_free_request(request);
3031	serge	2130	}
2332	Serge	2131
3031	serge	2132	while (!list_empty(&ring->active_list)) {
		2133	struct drm_i915_gem_object *obj;
2332	Serge	2134
3031	serge	2135	obj = list_first_entry(&ring->active_list,
		2136	struct drm_i915_gem_object,
		2137	ring_list);
2332	Serge	2138
3031	serge	2139	i915_gem_object_move_to_inactive(obj);
		2140	}
		2141	}
2332	Serge	2142
3746	Serge	2143	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2144	{
		2145	struct drm_i915_private *dev_priv = dev->dev_private;
		2146	int i;
2332	Serge	2147
3031	serge	2148	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2149	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2150
		2151	/*
		2152	* Commit delayed tiling changes if we have an object still
		2153	* attached to the fence, otherwise just clear the fence.
		2154	*/
		2155	if (reg->obj) {
		2156	i915_gem_object_update_fence(reg->obj, reg,
		2157	reg->obj->tiling_mode);
		2158	} else {
		2159	i915_gem_write_fence(dev, i, NULL);
		2160	}
3031	serge	2161	}
		2162	}
2360	Serge	2163
3031	serge	2164	void i915_gem_reset(struct drm_device *dev)
		2165	{
		2166	struct drm_i915_private *dev_priv = dev->dev_private;
		2167	struct intel_ring_buffer *ring;
		2168	int i;
2360	Serge	2169
3031	serge	2170	for_each_ring(ring, dev_priv, i)
		2171	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	2172
3746	Serge	2173	i915_gem_restore_fences(dev);
3031	serge	2174	}
2360	Serge	2175
2352	Serge	2176	/**
		2177	* This function clears the request list as sequence numbers are passed.
		2178	*/
3031	serge	2179	void
2352	Serge	2180	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2181	{
		2182	uint32_t seqno;
2332	Serge	2183
2352	Serge	2184	if (list_empty(&ring->request_list))
		2185	return;
2332	Serge	2186
2352	Serge	2187	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2188
3031	serge	2189	seqno = ring->get_seqno(ring, true);
2332	Serge	2190
2352	Serge	2191	while (!list_empty(&ring->request_list)) {
		2192	struct drm_i915_gem_request *request;
2332	Serge	2193
2352	Serge	2194	request = list_first_entry(&ring->request_list,
		2195	struct drm_i915_gem_request,
		2196	list);
2332	Serge	2197
2352	Serge	2198	if (!i915_seqno_passed(seqno, request->seqno))
		2199	break;
2332	Serge	2200
2352	Serge	2201	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2202	/* We know the GPU must have read the request to have
		2203	* sent us the seqno + interrupt, so use the position
		2204	* of tail of the request to update the last known position
		2205	* of the GPU head.
		2206	*/
		2207	ring->last_retired_head = request->tail;
2332	Serge	2208
4104	Serge	2209	i915_gem_free_request(request);
2352	Serge	2210	}
2332	Serge	2211
2352	Serge	2212	/* Move any buffers on the active list that are no longer referenced
		2213	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2214	*/
		2215	while (!list_empty(&ring->active_list)) {
		2216	struct drm_i915_gem_object *obj;
2332	Serge	2217
2352	Serge	2218	obj = list_first_entry(&ring->active_list,
		2219	struct drm_i915_gem_object,
		2220	ring_list);
2332	Serge	2221
3031	serge	2222	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2223	break;
2332	Serge	2224
2352	Serge	2225	i915_gem_object_move_to_inactive(obj);
		2226	}
2332	Serge	2227
2352	Serge	2228	if (unlikely(ring->trace_irq_seqno &&
		2229	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2230	ring->irq_put(ring);
		2231	ring->trace_irq_seqno = 0;
		2232	}
2332	Serge	2233
2352	Serge	2234	WARN_ON(i915_verify_lists(ring->dev));
		2235	}
2332	Serge	2236
2352	Serge	2237	void
		2238	i915_gem_retire_requests(struct drm_device *dev)
		2239	{
		2240	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2241	struct intel_ring_buffer *ring;
2352	Serge	2242	int i;
2332	Serge	2243
3031	serge	2244	for_each_ring(ring, dev_priv, i)
		2245	i915_gem_retire_requests_ring(ring);
2352	Serge	2246	}
		2247
2360	Serge	2248	static void
		2249	i915_gem_retire_work_handler(struct work_struct *work)
		2250	{
		2251	drm_i915_private_t *dev_priv;
		2252	struct drm_device *dev;
3031	serge	2253	struct intel_ring_buffer *ring;
2360	Serge	2254	bool idle;
		2255	int i;
2352	Serge	2256
2360	Serge	2257	dev_priv = container_of(work, drm_i915_private_t,
		2258	mm.retire_work.work);
		2259	dev = dev_priv->dev;
2352	Serge	2260
2360	Serge	2261	/* Come back later if the device is busy... */
		2262	if (!mutex_trylock(&dev->struct_mutex)) {
3482	Serge	2263	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2264	round_jiffies_up_relative(HZ));
3243	Serge	2265	return;
2360	Serge	2266	}
2352	Serge	2267
2360	Serge	2268	i915_gem_retire_requests(dev);
2352	Serge	2269
2360	Serge	2270	/* Send a periodic flush down the ring so we don't hold onto GEM
		2271	* objects indefinitely.
		2272	*/
		2273	idle = true;
3031	serge	2274	for_each_ring(ring, dev_priv, i) {
		2275	if (ring->gpu_caches_dirty)
4104	Serge	2276	i915_add_request(ring, NULL);
2352	Serge	2277
2360	Serge	2278	idle &= list_empty(&ring->request_list);
		2279	}
2352	Serge	2280
4104	Serge	2281	if (!dev_priv->ums.mm_suspended && !idle)
3482	Serge	2282	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2283	round_jiffies_up_relative(HZ));
3031	serge	2284	if (idle)
		2285	intel_mark_idle(dev);
2360	Serge	2286
		2287	mutex_unlock(&dev->struct_mutex);
		2288	}
		2289
2344	Serge	2290	/**
3031	serge	2291	* Ensures that an object will eventually get non-busy by flushing any required
		2292	* write domains, emitting any outstanding lazy request and retiring and
		2293	* completed requests.
2352	Serge	2294	*/
3031	serge	2295	static int
		2296	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2297	{
3031	serge	2298	int ret;
2352	Serge	2299
3031	serge	2300	if (obj->active) {
		2301	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2302	if (ret)
		2303	return ret;
2352	Serge	2304
3031	serge	2305	i915_gem_retire_requests_ring(obj->ring);
		2306	}
2352	Serge	2307
3031	serge	2308	return 0;
		2309	}
2352	Serge	2310
3243	Serge	2311	/**
		2312	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2313	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2314	*
		2315	* Returns 0 if successful, else an error is returned with the remaining time in
		2316	* the timeout parameter.
		2317	* -ETIME: object is still busy after timeout
		2318	* -ERESTARTSYS: signal interrupted the wait
		2319	* -ENONENT: object doesn't exist
		2320	* Also possible, but rare:
		2321	* -EAGAIN: GPU wedged
		2322	* -ENOMEM: damn
		2323	* -ENODEV: Internal IRQ fail
		2324	* -E?: The add request failed
		2325	*
		2326	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2327	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2328	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2329	* without holding struct_mutex the object may become re-busied before this
		2330	* function completes. A similar but shorter * race condition exists in the busy
		2331	* ioctl
		2332	*/
2352	Serge	2333
		2334
		2335
		2336
		2337
		2338
		2339
		2340
		2341
3243	Serge	2342
		2343
		2344
2352	Serge	2345	/**
3031	serge	2346	* i915_gem_object_sync - sync an object to a ring.
		2347	*
		2348	* @obj: object which may be in use on another ring.
		2349	* @to: ring we wish to use the object on. May be NULL.
		2350	*
		2351	* This code is meant to abstract object synchronization with the GPU.
		2352	* Calling with NULL implies synchronizing the object with the CPU
		2353	* rather than a particular GPU ring.
		2354	*
		2355	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2356	*/
		2357	int
3031	serge	2358	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2359	struct intel_ring_buffer *to)
2344	Serge	2360	{
3031	serge	2361	struct intel_ring_buffer *from = obj->ring;
		2362	u32 seqno;
		2363	int ret, idx;
2332	Serge	2364
3031	serge	2365	if (from == NULL \|\| to == from)
		2366	return 0;
2332	Serge	2367
3031	serge	2368	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2369	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2370
3031	serge	2371	idx = intel_ring_sync_index(from, to);
		2372
		2373	seqno = obj->last_read_seqno;
		2374	if (seqno <= from->sync_seqno[idx])
		2375	return 0;
		2376
		2377	ret = i915_gem_check_olr(obj->ring, seqno);
		2378	if (ret)
		2379	return ret;
		2380
		2381	ret = to->sync_to(to, from, seqno);
		2382	if (!ret)
3243	Serge	2383	/* We use last_read_seqno because sync_to()
		2384	* might have just caused seqno wrap under
		2385	* the radar.
		2386	*/
		2387	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2388
		2389	return ret;
2344	Serge	2390	}
2332	Serge	2391
2344	Serge	2392	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2393	{
		2394	u32 old_write_domain, old_read_domains;
2332	Serge	2395
2344	Serge	2396	/* Force a pagefault for domain tracking on next user access */
		2397	// i915_gem_release_mmap(obj);
2332	Serge	2398
2344	Serge	2399	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2400	return;
2332	Serge	2401
3480	Serge	2402	/* Wait for any direct GTT access to complete */
		2403	mb();
		2404
2344	Serge	2405	old_read_domains = obj->base.read_domains;
		2406	old_write_domain = obj->base.write_domain;
2351	Serge	2407
2344	Serge	2408	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2409	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2410
2351	Serge	2411	trace_i915_gem_object_change_domain(obj,
		2412	old_read_domains,
		2413	old_write_domain);
2344	Serge	2414	}
2332	Serge	2415
4104	Serge	2416	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2417	{
4104	Serge	2418	struct drm_i915_gem_object *obj = vma->obj;
3031	serge	2419	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2420	int ret;
2332	Serge	2421
3263	Serge	2422	if(obj == get_fb_obj())
		2423	return 0;
		2424
4104	Serge	2425	if (list_empty(&vma->vma_link))
2344	Serge	2426	return 0;
2332	Serge	2427
4104	Serge	2428	if (!drm_mm_node_allocated(&vma->node))
		2429	goto destroy;
		2430
3031	serge	2431	if (obj->pin_count)
		2432	return -EBUSY;
2332	Serge	2433
3243	Serge	2434	BUG_ON(obj->pages == NULL);
3031	serge	2435
2344	Serge	2436	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2437	if (ret)
2344	Serge	2438	return ret;
		2439	/* Continue on if we fail due to EIO, the GPU is hung so we
		2440	* should be safe and we need to cleanup or else we might
		2441	* cause memory corruption through use-after-free.
		2442	*/
2332	Serge	2443
2344	Serge	2444	i915_gem_object_finish_gtt(obj);
2332	Serge	2445
2344	Serge	2446	/* release the fence reg _after_ flushing */
		2447	ret = i915_gem_object_put_fence(obj);
3031	serge	2448	if (ret)
2344	Serge	2449	return ret;
2332	Serge	2450
4104	Serge	2451	trace_i915_vma_unbind(vma);
2332	Serge	2452
3031	serge	2453	if (obj->has_global_gtt_mapping)
3243	Serge	2454	i915_gem_gtt_unbind_object(obj);
3031	serge	2455	if (obj->has_aliasing_ppgtt_mapping) {
		2456	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2457	obj->has_aliasing_ppgtt_mapping = 0;
		2458	}
		2459	i915_gem_gtt_finish_object(obj);
4104	Serge	2460	i915_gem_object_unpin_pages(obj);
2332	Serge	2461
4104	Serge	2462	list_del(&vma->mm_list);
2344	Serge	2463	/* Avoid an unnecessary call to unbind on rebind. */
4104	Serge	2464	if (i915_is_ggtt(vma->vm))
2344	Serge	2465	obj->map_and_fenceable = true;
2332	Serge	2466
4104	Serge	2467	drm_mm_remove_node(&vma->node);
2332	Serge	2468
4104	Serge	2469	destroy:
		2470	i915_gem_vma_destroy(vma);
		2471
		2472	/* Since the unbound list is global, only move to that list if
		2473	* no more VMAs exist.
		2474	* NB: Until we have real VMAs there will only ever be one */
		2475	WARN_ON(!list_empty(&obj->vma_list));
		2476	if (list_empty(&obj->vma_list))
		2477	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2478
2344	Serge	2479	return 0;
		2480	}
2332	Serge	2481
4104	Serge	2482	/**
		2483	* Unbinds an object from the global GTT aperture.
		2484	*/
		2485	int
		2486	i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
		2487	{
		2488	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2489	struct i915_address_space *ggtt = &dev_priv->gtt.base;
		2490
		2491	if (!i915_gem_obj_ggtt_bound(obj))
		2492	return 0;
		2493
		2494	if (obj->pin_count)
		2495	return -EBUSY;
		2496
		2497	BUG_ON(obj->pages == NULL);
		2498
		2499	return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
		2500	}
		2501
3031	serge	2502	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2503	{
		2504	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2505	struct intel_ring_buffer *ring;
2344	Serge	2506	int ret, i;
2332	Serge	2507
2344	Serge	2508	/* Flush everything onto the inactive list. */
3031	serge	2509	for_each_ring(ring, dev_priv, i) {
		2510	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2511	if (ret)
		2512	return ret;
3031	serge	2513
3243	Serge	2514	ret = intel_ring_idle(ring);
3031	serge	2515	if (ret)
		2516	return ret;
2344	Serge	2517	}
2332	Serge	2518
2344	Serge	2519	return 0;
		2520	}
2332	Serge	2521
3480	Serge	2522	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2523	struct drm_i915_gem_object *obj)
		2524	{
		2525	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2526	int fence_reg;
		2527	int fence_pitch_shift;
2332	Serge	2528
3480	Serge	2529	if (INTEL_INFO(dev)->gen >= 6) {
		2530	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2531	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2532	} else {
		2533	fence_reg = FENCE_REG_965_0;
		2534	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2535	}
2332	Serge	2536
4104	Serge	2537	fence_reg += reg * 8;
		2538
		2539	/* To w/a incoherency with non-atomic 64-bit register updates,
		2540	* we split the 64-bit update into two 32-bit writes. In order
		2541	* for a partial fence not to be evaluated between writes, we
		2542	* precede the update with write to turn off the fence register,
		2543	* and only enable the fence as the last step.
		2544	*
		2545	* For extra levels of paranoia, we make sure each step lands
		2546	* before applying the next step.
		2547	*/
		2548	I915_WRITE(fence_reg, 0);
		2549	POSTING_READ(fence_reg);
		2550
3031	serge	2551	if (obj) {
4104	Serge	2552	u32 size = i915_gem_obj_ggtt_size(obj);
		2553	uint64_t val;
2332	Serge	2554
4104	Serge	2555	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2556	0xfffff000) << 32;
4104	Serge	2557	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2558	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2559	if (obj->tiling_mode == I915_TILING_Y)
		2560	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2561	val \|= I965_FENCE_REG_VALID;
2332	Serge	2562
4104	Serge	2563	I915_WRITE(fence_reg + 4, val >> 32);
		2564	POSTING_READ(fence_reg + 4);
		2565
		2566	I915_WRITE(fence_reg + 0, val);
3480	Serge	2567	POSTING_READ(fence_reg);
4104	Serge	2568	} else {
		2569	I915_WRITE(fence_reg + 4, 0);
		2570	POSTING_READ(fence_reg + 4);
		2571	}
3031	serge	2572	}
2332	Serge	2573
3031	serge	2574	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2575	struct drm_i915_gem_object *obj)
		2576	{
		2577	drm_i915_private_t *dev_priv = dev->dev_private;
		2578	u32 val;
2332	Serge	2579
3031	serge	2580	if (obj) {
4104	Serge	2581	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2582	int pitch_val;
		2583	int tile_width;
2332	Serge	2584
4104	Serge	2585	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2586	(size & -size) != size \|\|
4104	Serge	2587	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2588	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2589	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2590
3031	serge	2591	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2592	tile_width = 128;
		2593	else
		2594	tile_width = 512;
2332	Serge	2595
3031	serge	2596	/* Note: pitch better be a power of two tile widths */
		2597	pitch_val = obj->stride / tile_width;
		2598	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2599
4104	Serge	2600	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2601	if (obj->tiling_mode == I915_TILING_Y)
		2602	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2603	val \|= I915_FENCE_SIZE_BITS(size);
		2604	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2605	val \|= I830_FENCE_REG_VALID;
		2606	} else
		2607	val = 0;
2332	Serge	2608
3031	serge	2609	if (reg < 8)
		2610	reg = FENCE_REG_830_0 + reg * 4;
		2611	else
		2612	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2613
3031	serge	2614	I915_WRITE(reg, val);
		2615	POSTING_READ(reg);
		2616	}
2332	Serge	2617
3031	serge	2618	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2619	struct drm_i915_gem_object *obj)
		2620	{
		2621	drm_i915_private_t *dev_priv = dev->dev_private;
		2622	uint32_t val;
2344	Serge	2623
3031	serge	2624	if (obj) {
4104	Serge	2625	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2626	uint32_t pitch_val;
2344	Serge	2627
4104	Serge	2628	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2629	(size & -size) != size \|\|
4104	Serge	2630	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2631	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2632	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2633
3031	serge	2634	pitch_val = obj->stride / 128;
		2635	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2636
4104	Serge	2637	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2638	if (obj->tiling_mode == I915_TILING_Y)
		2639	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2640	val \|= I830_FENCE_SIZE_BITS(size);
		2641	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2642	val \|= I830_FENCE_REG_VALID;
		2643	} else
		2644	val = 0;
		2645
		2646	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2647	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2648	}
		2649
3480	Serge	2650	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2651	{
		2652	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2653	}
		2654
3031	serge	2655	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2656	struct drm_i915_gem_object *obj)
2332	Serge	2657	{
3480	Serge	2658	struct drm_i915_private *dev_priv = dev->dev_private;
		2659
		2660	/* Ensure that all CPU reads are completed before installing a fence
		2661	* and all writes before removing the fence.
		2662	*/
		2663	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2664	mb();
		2665
4104	Serge	2666	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2667	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2668	obj->stride, obj->tiling_mode);
		2669
3031	serge	2670	switch (INTEL_INFO(dev)->gen) {
		2671	case 7:
3480	Serge	2672	case 6:
3031	serge	2673	case 5:
		2674	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2675	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2676	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2677	default: BUG();
3031	serge	2678	}
3480	Serge	2679
		2680	/* And similarly be paranoid that no direct access to this region
		2681	* is reordered to before the fence is installed.
		2682	*/
		2683	if (i915_gem_object_needs_mb(obj))
		2684	mb();
2344	Serge	2685	}
		2686
3031	serge	2687	static inline int fence_number(struct drm_i915_private *dev_priv,
		2688	struct drm_i915_fence_reg *fence)
2344	Serge	2689	{
3031	serge	2690	return fence - dev_priv->fence_regs;
		2691	}
2332	Serge	2692
3031	serge	2693	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2694	struct drm_i915_fence_reg *fence,
		2695	bool enable)
		2696	{
4104	Serge	2697	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2698	int reg = fence_number(dev_priv, fence);
2332	Serge	2699
4104	Serge	2700	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2701
		2702	if (enable) {
4104	Serge	2703	obj->fence_reg = reg;
3031	serge	2704	fence->obj = obj;
		2705	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2706	} else {
		2707	obj->fence_reg = I915_FENCE_REG_NONE;
		2708	fence->obj = NULL;
		2709	list_del_init(&fence->lru_list);
2344	Serge	2710	}
4104	Serge	2711	obj->fence_dirty = false;
3031	serge	2712	}
2344	Serge	2713
3031	serge	2714	static int
3480	Serge	2715	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2716	{
		2717	if (obj->last_fenced_seqno) {
		2718	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2719	if (ret)
		2720	return ret;
2344	Serge	2721
		2722	obj->last_fenced_seqno = 0;
		2723	}
		2724
3031	serge	2725	obj->fenced_gpu_access = false;
2332	Serge	2726	return 0;
		2727	}
		2728
		2729	int
2344	Serge	2730	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2731	{
3031	serge	2732	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2733	struct drm_i915_fence_reg *fence;
2332	Serge	2734	int ret;
		2735
3480	Serge	2736	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2737	if (ret)
		2738	return ret;
		2739
3031	serge	2740	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2741	return 0;
2332	Serge	2742
3746	Serge	2743	fence = &dev_priv->fence_regs[obj->fence_reg];
		2744
3031	serge	2745	i915_gem_object_fence_lost(obj);
3746	Serge	2746	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2747
2332	Serge	2748	return 0;
		2749	}
		2750
3031	serge	2751	static struct drm_i915_fence_reg *
		2752	i915_find_fence_reg(struct drm_device *dev)
		2753	{
		2754	struct drm_i915_private *dev_priv = dev->dev_private;
		2755	struct drm_i915_fence_reg reg, avail;
		2756	int i;
2332	Serge	2757
3031	serge	2758	/* First try to find a free reg */
		2759	avail = NULL;
		2760	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2761	reg = &dev_priv->fence_regs[i];
		2762	if (!reg->obj)
		2763	return reg;
2332	Serge	2764
3031	serge	2765	if (!reg->pin_count)
		2766	avail = reg;
		2767	}
2332	Serge	2768
3031	serge	2769	if (avail == NULL)
		2770	return NULL;
2332	Serge	2771
3031	serge	2772	/* None available, try to steal one or wait for a user to finish */
		2773	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2774	if (reg->pin_count)
		2775	continue;
2332	Serge	2776
3031	serge	2777	return reg;
		2778	}
2332	Serge	2779
3031	serge	2780	return NULL;
		2781	}
2332	Serge	2782
3031	serge	2783	/**
		2784	* i915_gem_object_get_fence - set up fencing for an object
		2785	* @obj: object to map through a fence reg
		2786	*
		2787	* When mapping objects through the GTT, userspace wants to be able to write
		2788	* to them without having to worry about swizzling if the object is tiled.
		2789	* This function walks the fence regs looking for a free one for @obj,
		2790	* stealing one if it can't find any.
		2791	*
		2792	* It then sets up the reg based on the object's properties: address, pitch
		2793	* and tiling format.
		2794	*
		2795	* For an untiled surface, this removes any existing fence.
		2796	*/
		2797	int
		2798	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2799	{
		2800	struct drm_device *dev = obj->base.dev;
		2801	struct drm_i915_private *dev_priv = dev->dev_private;
		2802	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2803	struct drm_i915_fence_reg *reg;
		2804	int ret;
2332	Serge	2805
3031	serge	2806	/* Have we updated the tiling parameters upon the object and so
		2807	* will need to serialise the write to the associated fence register?
		2808	*/
		2809	if (obj->fence_dirty) {
3480	Serge	2810	ret = i915_gem_object_wait_fence(obj);
3031	serge	2811	if (ret)
		2812	return ret;
		2813	}
2332	Serge	2814
3031	serge	2815	/* Just update our place in the LRU if our fence is getting reused. */
		2816	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2817	reg = &dev_priv->fence_regs[obj->fence_reg];
		2818	if (!obj->fence_dirty) {
		2819	list_move_tail(®->lru_list,
		2820	&dev_priv->mm.fence_list);
		2821	return 0;
		2822	}
		2823	} else if (enable) {
		2824	reg = i915_find_fence_reg(dev);
		2825	if (reg == NULL)
		2826	return -EDEADLK;
2332	Serge	2827
3031	serge	2828	if (reg->obj) {
		2829	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2830
3480	Serge	2831	ret = i915_gem_object_wait_fence(old);
3031	serge	2832	if (ret)
		2833	return ret;
2332	Serge	2834
3031	serge	2835	i915_gem_object_fence_lost(old);
		2836	}
		2837	} else
		2838	return 0;
2332	Serge	2839
3031	serge	2840	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2841
3031	serge	2842	return 0;
		2843	}
2332	Serge	2844
3031	serge	2845	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2846	struct drm_mm_node *gtt_space,
		2847	unsigned long cache_level)
		2848	{
		2849	struct drm_mm_node *other;
2332	Serge	2850
3031	serge	2851	/* On non-LLC machines we have to be careful when putting differing
		2852	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2853	* crossing memory domains and dying.
3031	serge	2854	*/
		2855	if (HAS_LLC(dev))
		2856	return true;
2332	Serge	2857
4104	Serge	2858	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2859	return true;
2332	Serge	2860
3031	serge	2861	if (list_empty(>t_space->node_list))
		2862	return true;
2332	Serge	2863
3031	serge	2864	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2865	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2866	return false;
2344	Serge	2867
3031	serge	2868	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2869	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2870	return false;
2344	Serge	2871
3031	serge	2872	return true;
		2873	}
2344	Serge	2874
3031	serge	2875	static void i915_gem_verify_gtt(struct drm_device *dev)
		2876	{
		2877	#if WATCH_GTT
		2878	struct drm_i915_private *dev_priv = dev->dev_private;
		2879	struct drm_i915_gem_object *obj;
		2880	int err = 0;
2344	Serge	2881
4104	Serge	2882	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031	serge	2883	if (obj->gtt_space == NULL) {
		2884	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2885	err++;
		2886	continue;
		2887	}
2344	Serge	2888
3031	serge	2889	if (obj->cache_level != obj->gtt_space->color) {
		2890	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
4104	Serge	2891	i915_gem_obj_ggtt_offset(obj),
		2892	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2893	obj->cache_level,
		2894	obj->gtt_space->color);
		2895	err++;
		2896	continue;
		2897	}
2344	Serge	2898
3031	serge	2899	if (!i915_gem_valid_gtt_space(dev,
		2900	obj->gtt_space,
		2901	obj->cache_level)) {
		2902	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
4104	Serge	2903	i915_gem_obj_ggtt_offset(obj),
		2904	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2905	obj->cache_level);
		2906	err++;
		2907	continue;
		2908	}
		2909	}
2344	Serge	2910
3031	serge	2911	WARN_ON(err);
		2912	#endif
2326	Serge	2913	}
		2914
2332	Serge	2915	/**
		2916	* Finds free space in the GTT aperture and binds the object there.
		2917	*/
		2918	static int
4104	Serge	2919	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2920	struct i915_address_space *vm,
2332	Serge	2921	unsigned alignment,
3031	serge	2922	bool map_and_fenceable,
		2923	bool nonblocking)
2332	Serge	2924	{
		2925	struct drm_device *dev = obj->base.dev;
		2926	drm_i915_private_t *dev_priv = dev->dev_private;
		2927	u32 size, fence_size, fence_alignment, unfenced_alignment;
4104	Serge	2928	size_t gtt_max =
		2929	map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
		2930	struct i915_vma *vma;
2332	Serge	2931	int ret;
2326	Serge	2932
2332	Serge	2933	fence_size = i915_gem_get_gtt_size(dev,
		2934	obj->base.size,
		2935	obj->tiling_mode);
		2936	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2937	obj->base.size,
3480	Serge	2938	obj->tiling_mode, true);
2332	Serge	2939	unfenced_alignment =
3480	Serge	2940	i915_gem_get_gtt_alignment(dev,
2332	Serge	2941	obj->base.size,
3480	Serge	2942	obj->tiling_mode, false);
2332	Serge	2943
		2944	if (alignment == 0)
		2945	alignment = map_and_fenceable ? fence_alignment :
		2946	unfenced_alignment;
		2947	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2948	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2949	return -EINVAL;
		2950	}
		2951
		2952	size = map_and_fenceable ? fence_size : obj->base.size;
		2953
		2954	/* If the object is bigger than the entire aperture, reject it early
		2955	* before evicting everything in a vain attempt to find space.
		2956	*/
4104	Serge	2957	if (obj->base.size > gtt_max) {
		2958	DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
		2959	obj->base.size,
		2960	map_and_fenceable ? "mappable" : "total",
		2961	gtt_max);
2332	Serge	2962	return -E2BIG;
		2963	}
		2964
3031	serge	2965	ret = i915_gem_object_get_pages(obj);
		2966	if (ret)
		2967	return ret;
		2968
3243	Serge	2969	i915_gem_object_pin_pages(obj);
		2970
4104	Serge	2971	BUG_ON(!i915_is_ggtt(vm));
		2972
		2973	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
		2974	if (IS_ERR(vma)) {
		2975	ret = PTR_ERR(vma);
		2976	goto err_unpin;
3243	Serge	2977	}
		2978
4104	Serge	2979	/* For now we only ever use 1 vma per object */
		2980	WARN_ON(!list_is_singular(&obj->vma_list));
		2981
		2982	search_free:
		2983	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		2984	size, alignment,
		2985	obj->cache_level, 0, gtt_max,
		2986	DRM_MM_SEARCH_DEFAULT);
3243	Serge	2987	if (ret) {
2332	Serge	2988
4104	Serge	2989	goto err_free_vma;
2332	Serge	2990	}
4104	Serge	2991	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
		2992	obj->cache_level))) {
		2993	ret = -EINVAL;
		2994	goto err_remove_node;
3031	serge	2995	}
2332	Serge	2996
3031	serge	2997	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	2998	if (ret)
		2999	goto err_remove_node;
2332	Serge	3000
4104	Serge	3001	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3002	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3003
4104	Serge	3004	if (i915_is_ggtt(vm)) {
		3005	bool mappable, fenceable;
2332	Serge	3006
4104	Serge	3007	fenceable = (vma->node.size == fence_size &&
		3008	(vma->node.start & (fence_alignment - 1)) == 0);
2332	Serge	3009
4104	Serge	3010	mappable = (vma->node.start + obj->base.size <=
		3011	dev_priv->gtt.mappable_end);
2332	Serge	3012
		3013	obj->map_and_fenceable = mappable && fenceable;
4104	Serge	3014	}
2332	Serge	3015
4104	Serge	3016	WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
		3017
		3018	trace_i915_vma_bind(vma, map_and_fenceable);
3031	serge	3019	i915_gem_verify_gtt(dev);
2332	Serge	3020	return 0;
4104	Serge	3021
		3022	err_remove_node:
		3023	drm_mm_remove_node(&vma->node);
		3024	err_free_vma:
		3025	i915_gem_vma_destroy(vma);
		3026	err_unpin:
		3027	i915_gem_object_unpin_pages(obj);
		3028	return ret;
2332	Serge	3029	}
		3030
4104	Serge	3031	bool
		3032	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3033	bool force)
2332	Serge	3034	{
		3035	/* If we don't have a page list set up, then we're not pinned
		3036	* to GPU, and we can ignore the cache flush because it'll happen
		3037	* again at bind time.
		3038	*/
3243	Serge	3039	if (obj->pages == NULL)
4104	Serge	3040	return false;
2332	Serge	3041
3480	Serge	3042	/*
		3043	* Stolen memory is always coherent with the GPU as it is explicitly
		3044	* marked as wc by the system, or the system is cache-coherent.
		3045	*/
		3046	if (obj->stolen)
4104	Serge	3047	return false;
3480	Serge	3048
2332	Serge	3049	/* If the GPU is snooping the contents of the CPU cache,
		3050	* we do not need to manually clear the CPU cache lines. However,
		3051	* the caches are only snooped when the render cache is
		3052	* flushed/invalidated. As we always have to emit invalidations
		3053	* and flushes when moving into and out of the RENDER domain, correct
		3054	* snooping behaviour occurs naturally as the result of our domain
		3055	* tracking.
		3056	*/
4104	Serge	3057	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3058	return false;
3243	Serge	3059	#if 0
2344	Serge	3060	if(obj->mapped != NULL)
		3061	{
		3062	uint8_t *page_virtual;
		3063	unsigned int i;
2332	Serge	3064
2344	Serge	3065	page_virtual = obj->mapped;
		3066	asm volatile("mfence");
		3067	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		3068	clflush(page_virtual + i);
		3069	asm volatile("mfence");
		3070	}
		3071	else
		3072	{
		3073	uint8_t *page_virtual;
		3074	unsigned int i;
		3075	page_virtual = AllocKernelSpace(obj->base.size);
		3076	if(page_virtual != NULL)
		3077	{
3031	serge	3078	dma_addr_t src, dst;
2344	Serge	3079	u32 count;
		3080
		3081	#define page_tabs 0xFDC00000 /* really dirty hack */
		3082
3031	serge	3083	src = obj->pages.page;
		3084	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	3085	count = obj->base.size/4096;
		3086
		3087	while(count--)
		3088	{
		3089	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		3090	};
		3091
		3092	asm volatile("mfence");
		3093	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		3094	clflush(page_virtual + i);
		3095	asm volatile("mfence");
		3096	FreeKernelSpace(page_virtual);
		3097	}
		3098	else
		3099	{
		3100	asm volatile (
		3101	"mfence \n"
		3102	"wbinvd \n" /* this is really ugly */
		3103	"mfence");
		3104	}
		3105	}
3243	Serge	3106	#endif
		3107
4104	Serge	3108	return true;
2332	Serge	3109	}
		3110
2344	Serge	3111	/** Flushes the GTT write domain for the object if it's dirty. */
		3112	static void
		3113	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3114	{
		3115	uint32_t old_write_domain;
2332	Serge	3116
2344	Serge	3117	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3118	return;
2332	Serge	3119
2344	Serge	3120	/* No actual flushing is required for the GTT write domain. Writes
		3121	* to it immediately go to main memory as far as we know, so there's
		3122	* no chipset flush. It also doesn't land in render cache.
		3123	*
		3124	* However, we do have to enforce the order so that all writes through
		3125	* the GTT land before any writes to the device, such as updates to
		3126	* the GATT itself.
		3127	*/
		3128	wmb();
2332	Serge	3129
2344	Serge	3130	old_write_domain = obj->base.write_domain;
		3131	obj->base.write_domain = 0;
2332	Serge	3132
2351	Serge	3133	trace_i915_gem_object_change_domain(obj,
		3134	obj->base.read_domains,
		3135	old_write_domain);
2344	Serge	3136	}
2332	Serge	3137
		3138	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3139	static void
4104	Serge	3140	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3141	bool force)
2332	Serge	3142	{
		3143	uint32_t old_write_domain;
		3144
		3145	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3146	return;
		3147
4104	Serge	3148	if (i915_gem_clflush_object(obj, force))
3243	Serge	3149	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3150
2332	Serge	3151	old_write_domain = obj->base.write_domain;
		3152	obj->base.write_domain = 0;
		3153
2351	Serge	3154	trace_i915_gem_object_change_domain(obj,
		3155	obj->base.read_domains,
		3156	old_write_domain);
2332	Serge	3157	}
		3158
		3159	/**
		3160	* Moves a single object to the GTT read, and possibly write domain.
		3161	*
		3162	* This function returns when the move is complete, including waiting on
		3163	* flushes to occur.
		3164	*/
		3165	int
		3166	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3167	{
3031	serge	3168	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	3169	uint32_t old_write_domain, old_read_domains;
		3170	int ret;
		3171
		3172	/* Not valid to be called on unbound objects. */
4104	Serge	3173	if (!i915_gem_obj_bound_any(obj))
2332	Serge	3174	return -EINVAL;
		3175
		3176	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3177	return 0;
		3178
3031	serge	3179	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3180	if (ret)
		3181	return ret;
		3182
4104	Serge	3183	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3184
3480	Serge	3185	/* Serialise direct access to this object with the barriers for
		3186	* coherent writes from the GPU, by effectively invalidating the
		3187	* GTT domain upon first access.
		3188	*/
		3189	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3190	mb();
		3191
2332	Serge	3192	old_write_domain = obj->base.write_domain;
		3193	old_read_domains = obj->base.read_domains;
		3194
		3195	/* It should now be out of any other write domains, and we can update
		3196	* the domain values for our changes.
		3197	*/
		3198	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3199	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3200	if (write) {
		3201	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3202	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3203	obj->dirty = 1;
		3204	}
		3205
2351	Serge	3206	trace_i915_gem_object_change_domain(obj,
		3207	old_read_domains,
		3208	old_write_domain);
		3209
3031	serge	3210	/* And bump the LRU for this access */
4104	Serge	3211	if (i915_gem_object_is_inactive(obj)) {
		3212	struct i915_vma *vma = i915_gem_obj_to_vma(obj,
		3213	&dev_priv->gtt.base);
		3214	if (vma)
		3215	list_move_tail(&vma->mm_list,
		3216	&dev_priv->gtt.base.inactive_list);
3031	serge	3217
4104	Serge	3218	}
		3219
2332	Serge	3220	return 0;
		3221	}
		3222
2335	Serge	3223	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3224	enum i915_cache_level cache_level)
		3225	{
3031	serge	3226	struct drm_device *dev = obj->base.dev;
		3227	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3228	struct i915_vma *vma;
2335	Serge	3229	int ret;
2332	Serge	3230
2335	Serge	3231	if (obj->cache_level == cache_level)
		3232	return 0;
2332	Serge	3233
2335	Serge	3234	if (obj->pin_count) {
		3235	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3236	return -EBUSY;
		3237	}
2332	Serge	3238
4104	Serge	3239	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3240	if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
		3241	ret = i915_vma_unbind(vma);
3031	serge	3242	if (ret)
		3243	return ret;
4104	Serge	3244
		3245	break;
		3246	}
3031	serge	3247	}
		3248
4104	Serge	3249	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3250	ret = i915_gem_object_finish_gpu(obj);
		3251	if (ret)
		3252	return ret;
2332	Serge	3253
2335	Serge	3254	i915_gem_object_finish_gtt(obj);
2332	Serge	3255
2335	Serge	3256	/* Before SandyBridge, you could not use tiling or fence
		3257	* registers with snooped memory, so relinquish any fences
		3258	* currently pointing to our region in the aperture.
		3259	*/
3031	serge	3260	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3261	ret = i915_gem_object_put_fence(obj);
		3262	if (ret)
		3263	return ret;
		3264	}
2332	Serge	3265
3031	serge	3266	if (obj->has_global_gtt_mapping)
		3267	i915_gem_gtt_bind_object(obj, cache_level);
		3268	if (obj->has_aliasing_ppgtt_mapping)
		3269	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3270	obj, cache_level);
2335	Serge	3271	}
2332	Serge	3272
4104	Serge	3273	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3274	vma->node.color = cache_level;
		3275	obj->cache_level = cache_level;
		3276
		3277	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3278	u32 old_read_domains, old_write_domain;
2332	Serge	3279
2335	Serge	3280	/* If we're coming from LLC cached, then we haven't
		3281	* actually been tracking whether the data is in the
		3282	* CPU cache or not, since we only allow one bit set
		3283	* in obj->write_domain and have been skipping the clflushes.
		3284	* Just set it to the CPU cache for now.
		3285	*/
		3286	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3287
2335	Serge	3288	old_read_domains = obj->base.read_domains;
		3289	old_write_domain = obj->base.write_domain;
2332	Serge	3290
2335	Serge	3291	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3292	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3293
2351	Serge	3294	trace_i915_gem_object_change_domain(obj,
		3295	old_read_domains,
		3296	old_write_domain);
2344	Serge	3297	}
2332	Serge	3298
3031	serge	3299	i915_gem_verify_gtt(dev);
2335	Serge	3300	return 0;
		3301	}
2332	Serge	3302
3260	Serge	3303	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3304	struct drm_file *file)
		3305	{
		3306	struct drm_i915_gem_caching *args = data;
		3307	struct drm_i915_gem_object *obj;
		3308	int ret;
		3309
3480	Serge	3310	if(args->handle == -2)
		3311	{
		3312	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3313	return 0;
		3314	}
		3315
3260	Serge	3316	ret = i915_mutex_lock_interruptible(dev);
		3317	if (ret)
		3318	return ret;
		3319
		3320	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3321	if (&obj->base == NULL) {
		3322	ret = -ENOENT;
		3323	goto unlock;
		3324	}
		3325
4104	Serge	3326	switch (obj->cache_level) {
		3327	case I915_CACHE_LLC:
		3328	case I915_CACHE_L3_LLC:
		3329	args->caching = I915_CACHING_CACHED;
		3330	break;
3260	Serge	3331
4104	Serge	3332	case I915_CACHE_WT:
		3333	args->caching = I915_CACHING_DISPLAY;
		3334	break;
		3335
		3336	default:
		3337	args->caching = I915_CACHING_NONE;
		3338	break;
		3339	}
		3340
3260	Serge	3341	drm_gem_object_unreference(&obj->base);
		3342	unlock:
		3343	mutex_unlock(&dev->struct_mutex);
		3344	return ret;
		3345	}
		3346
		3347	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3348	struct drm_file *file)
		3349	{
		3350	struct drm_i915_gem_caching *args = data;
		3351	struct drm_i915_gem_object *obj;
		3352	enum i915_cache_level level;
		3353	int ret;
		3354
3480	Serge	3355	if(args->handle == -2)
		3356	{
		3357	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3358	return 0;
		3359	}
		3360
3260	Serge	3361	switch (args->caching) {
		3362	case I915_CACHING_NONE:
		3363	level = I915_CACHE_NONE;
		3364	break;
		3365	case I915_CACHING_CACHED:
		3366	level = I915_CACHE_LLC;
		3367	break;
4104	Serge	3368	case I915_CACHING_DISPLAY:
		3369	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3370	break;
3260	Serge	3371	default:
		3372	return -EINVAL;
		3373	}
		3374
		3375	ret = i915_mutex_lock_interruptible(dev);
		3376	if (ret)
		3377	return ret;
		3378
		3379	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3380	if (&obj->base == NULL) {
		3381	ret = -ENOENT;
		3382	goto unlock;
		3383	}
		3384
		3385	ret = i915_gem_object_set_cache_level(obj, level);
		3386
		3387	drm_gem_object_unreference(&obj->base);
		3388	unlock:
		3389	mutex_unlock(&dev->struct_mutex);
		3390	return ret;
		3391	}
		3392
4104	Serge	3393	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3394	{
		3395	/* There are 3 sources that pin objects:
		3396	* 1. The display engine (scanouts, sprites, cursors);
		3397	* 2. Reservations for execbuffer;
		3398	* 3. The user.
		3399	*
		3400	* We can ignore reservations as we hold the struct_mutex and
		3401	* are only called outside of the reservation path. The user
		3402	* can only increment pin_count once, and so if after
		3403	* subtracting the potential reference by the user, any pin_count
		3404	* remains, it must be due to another use by the display engine.
		3405	*/
		3406	return obj->pin_count - !!obj->user_pin_count;
		3407	}
		3408
2335	Serge	3409	/*
		3410	* Prepare buffer for display plane (scanout, cursors, etc).
		3411	* Can be called from an uninterruptible phase (modesetting) and allows
		3412	* any flushes to be pipelined (for pageflips).
		3413	*/
		3414	int
		3415	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3416	u32 alignment,
		3417	struct intel_ring_buffer *pipelined)
		3418	{
		3419	u32 old_read_domains, old_write_domain;
		3420	int ret;
2332	Serge	3421
3031	serge	3422	if (pipelined != obj->ring) {
		3423	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3424	if (ret)
		3425	return ret;
		3426	}
2332	Serge	3427
4104	Serge	3428	/* Mark the pin_display early so that we account for the
		3429	* display coherency whilst setting up the cache domains.
		3430	*/
		3431	obj->pin_display = true;
		3432
2335	Serge	3433	/* The display engine is not coherent with the LLC cache on gen6. As
		3434	* a result, we make sure that the pinning that is about to occur is
		3435	* done with uncached PTEs. This is lowest common denominator for all
		3436	* chipsets.
		3437	*
		3438	* However for gen6+, we could do better by using the GFDT bit instead
		3439	* of uncaching, which would allow us to flush all the LLC-cached data
		3440	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3441	*/
4104	Serge	3442	ret = i915_gem_object_set_cache_level(obj,
		3443	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3444	if (ret)
4104	Serge	3445	goto err_unpin_display;
2332	Serge	3446
2335	Serge	3447	/* As the user may map the buffer once pinned in the display plane
		3448	* (e.g. libkms for the bootup splash), we have to ensure that we
		3449	* always use map_and_fenceable for all scanout buffers.
		3450	*/
4104	Serge	3451	ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
2335	Serge	3452	if (ret)
4104	Serge	3453	goto err_unpin_display;
2332	Serge	3454
4104	Serge	3455	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3456
2335	Serge	3457	old_write_domain = obj->base.write_domain;
		3458	old_read_domains = obj->base.read_domains;
2332	Serge	3459
2335	Serge	3460	/* It should now be out of any other write domains, and we can update
		3461	* the domain values for our changes.
		3462	*/
3031	serge	3463	obj->base.write_domain = 0;
2335	Serge	3464	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3465
2351	Serge	3466	trace_i915_gem_object_change_domain(obj,
		3467	old_read_domains,
		3468	old_write_domain);
2332	Serge	3469
2335	Serge	3470	return 0;
4104	Serge	3471
		3472	err_unpin_display:
		3473	obj->pin_display = is_pin_display(obj);
		3474	return ret;
2335	Serge	3475	}
2332	Serge	3476
4104	Serge	3477	void
		3478	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3479	{
		3480	i915_gem_object_unpin(obj);
		3481	obj->pin_display = is_pin_display(obj);
		3482	}
		3483
2344	Serge	3484	int
		3485	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3486	{
		3487	int ret;
2332	Serge	3488
2344	Serge	3489	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3490	return 0;
2332	Serge	3491
3031	serge	3492	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3493	if (ret)
		3494	return ret;
2332	Serge	3495
2344	Serge	3496	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3497	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3498	return 0;
2344	Serge	3499	}
2332	Serge	3500
2344	Serge	3501	/**
		3502	* Moves a single object to the CPU read, and possibly write domain.
		3503	*
		3504	* This function returns when the move is complete, including waiting on
		3505	* flushes to occur.
		3506	*/
3031	serge	3507	int
2344	Serge	3508	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3509	{
		3510	uint32_t old_write_domain, old_read_domains;
		3511	int ret;
2332	Serge	3512
2344	Serge	3513	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3514	return 0;
2332	Serge	3515
3031	serge	3516	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3517	if (ret)
		3518	return ret;
2332	Serge	3519
2344	Serge	3520	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3521
2344	Serge	3522	old_write_domain = obj->base.write_domain;
		3523	old_read_domains = obj->base.read_domains;
2332	Serge	3524
2344	Serge	3525	/* Flush the CPU cache if it's still invalid. */
		3526	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3527	i915_gem_clflush_object(obj, false);
2332	Serge	3528
2344	Serge	3529	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3530	}
2332	Serge	3531
2344	Serge	3532	/* It should now be out of any other write domains, and we can update
		3533	* the domain values for our changes.
		3534	*/
		3535	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3536
2344	Serge	3537	/* If we're writing through the CPU, then the GPU read domains will
		3538	* need to be invalidated at next use.
		3539	*/
		3540	if (write) {
		3541	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3542	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3543	}
2332	Serge	3544
2351	Serge	3545	trace_i915_gem_object_change_domain(obj,
		3546	old_read_domains,
		3547	old_write_domain);
2332	Serge	3548
2344	Serge	3549	return 0;
		3550	}
2332	Serge	3551
3031	serge	3552	/* Throttle our rendering by waiting until the ring has completed our requests
		3553	* emitted over 20 msec ago.
2344	Serge	3554	*
3031	serge	3555	* Note that if we were to use the current jiffies each time around the loop,
		3556	* we wouldn't escape the function with any frames outstanding if the time to
		3557	* render a frame was over 20ms.
		3558	*
		3559	* This should get us reasonable parallelism between CPU and GPU but also
		3560	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3561	*/
3031	serge	3562	static int
		3563	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3564	{
3031	serge	3565	struct drm_i915_private *dev_priv = dev->dev_private;
		3566	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3567	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3568	struct drm_i915_gem_request *request;
		3569	struct intel_ring_buffer *ring = NULL;
3480	Serge	3570	unsigned reset_counter;
3031	serge	3571	u32 seqno = 0;
		3572	int ret;
2332	Serge	3573
3480	Serge	3574	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3575	if (ret)
		3576	return ret;
2332	Serge	3577
3480	Serge	3578	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3579	if (ret)
		3580	return ret;
		3581
3031	serge	3582	spin_lock(&file_priv->mm.lock);
		3583	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3584	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3585	break;
2332	Serge	3586
3031	serge	3587	ring = request->ring;
		3588	seqno = request->seqno;
		3589	}
3480	Serge	3590	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3591	spin_unlock(&file_priv->mm.lock);
2332	Serge	3592
3031	serge	3593	if (seqno == 0)
		3594	return 0;
2332	Serge	3595
3480	Serge	3596	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3597	if (ret == 0)
		3598	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3599
3031	serge	3600	return ret;
2352	Serge	3601	}
2332	Serge	3602
		3603	int
		3604	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3605	struct i915_address_space *vm,
2332	Serge	3606	uint32_t alignment,
3031	serge	3607	bool map_and_fenceable,
		3608	bool nonblocking)
2332	Serge	3609	{
4104	Serge	3610	struct i915_vma *vma;
2332	Serge	3611	int ret;
		3612
3031	serge	3613	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3614	return -EBUSY;
2332	Serge	3615
4104	Serge	3616	WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
		3617
		3618	vma = i915_gem_obj_to_vma(obj, vm);
		3619
		3620	if (vma) {
		3621	if ((alignment &&
		3622	vma->node.start & (alignment - 1)) \|\|
2332	Serge	3623	(map_and_fenceable && !obj->map_and_fenceable)) {
		3624	WARN(obj->pin_count,
		3625	"bo is already pinned with incorrect alignment:"
4104	Serge	3626	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3627	" obj->map_and_fenceable=%d\n",
4104	Serge	3628	i915_gem_obj_offset(obj, vm), alignment,
2332	Serge	3629	map_and_fenceable,
		3630	obj->map_and_fenceable);
4104	Serge	3631	ret = i915_vma_unbind(vma);
2332	Serge	3632	if (ret)
		3633	return ret;
		3634	}
		3635	}
		3636
4104	Serge	3637	if (!i915_gem_obj_bound(obj, vm)) {
3243	Serge	3638	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3639
4104	Serge	3640	ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3031	serge	3641	map_and_fenceable,
		3642	nonblocking);
2332	Serge	3643	if (ret)
		3644	return ret;
3243	Serge	3645
		3646	if (!dev_priv->mm.aliasing_ppgtt)
		3647	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3648	}
		3649
3031	serge	3650	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3651	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3652
		3653	obj->pin_count++;
2332	Serge	3654	obj->pin_mappable \|= map_and_fenceable;
		3655
		3656	return 0;
		3657	}
		3658
2344	Serge	3659	void
		3660	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3661	{
		3662	BUG_ON(obj->pin_count == 0);
4104	Serge	3663	BUG_ON(!i915_gem_obj_bound_any(obj));
2332	Serge	3664
3031	serge	3665	if (--obj->pin_count == 0)
2344	Serge	3666	obj->pin_mappable = false;
		3667	}
2332	Serge	3668
3031	serge	3669	int
		3670	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3671	struct drm_file *file)
		3672	{
		3673	struct drm_i915_gem_pin *args = data;
		3674	struct drm_i915_gem_object *obj;
		3675	int ret;
2332	Serge	3676
3480	Serge	3677	if(args->handle == -2)
		3678	{
		3679	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3680	return 0;
		3681	}
		3682
3031	serge	3683	ret = i915_mutex_lock_interruptible(dev);
		3684	if (ret)
		3685	return ret;
2332	Serge	3686
3031	serge	3687	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3688	if (&obj->base == NULL) {
		3689	ret = -ENOENT;
		3690	goto unlock;
		3691	}
2332	Serge	3692
3031	serge	3693	if (obj->madv != I915_MADV_WILLNEED) {
		3694	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3695	ret = -EINVAL;
		3696	goto out;
		3697	}
2332	Serge	3698
3031	serge	3699	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3700	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3701	args->handle);
		3702	ret = -EINVAL;
		3703	goto out;
		3704	}
2332	Serge	3705
3243	Serge	3706	if (obj->user_pin_count == 0) {
4104	Serge	3707	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3031	serge	3708	if (ret)
		3709	goto out;
		3710	}
2332	Serge	3711
3243	Serge	3712	obj->user_pin_count++;
		3713	obj->pin_filp = file;
		3714
4104	Serge	3715	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3716	out:
		3717	drm_gem_object_unreference(&obj->base);
		3718	unlock:
		3719	mutex_unlock(&dev->struct_mutex);
		3720	return ret;
		3721	}
2332	Serge	3722
3255	Serge	3723	#if 0
		3724
3031	serge	3725	int
		3726	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3727	struct drm_file *file)
		3728	{
		3729	struct drm_i915_gem_pin *args = data;
		3730	struct drm_i915_gem_object *obj;
		3731	int ret;
2332	Serge	3732
3031	serge	3733	ret = i915_mutex_lock_interruptible(dev);
		3734	if (ret)
		3735	return ret;
2332	Serge	3736
3031	serge	3737	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3738	if (&obj->base == NULL) {
		3739	ret = -ENOENT;
		3740	goto unlock;
		3741	}
2332	Serge	3742
3031	serge	3743	if (obj->pin_filp != file) {
		3744	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3745	args->handle);
		3746	ret = -EINVAL;
		3747	goto out;
		3748	}
		3749	obj->user_pin_count--;
		3750	if (obj->user_pin_count == 0) {
		3751	obj->pin_filp = NULL;
		3752	i915_gem_object_unpin(obj);
		3753	}
2332	Serge	3754
3031	serge	3755	out:
		3756	drm_gem_object_unreference(&obj->base);
		3757	unlock:
		3758	mutex_unlock(&dev->struct_mutex);
		3759	return ret;
		3760	}
2332	Serge	3761
3260	Serge	3762	#endif
		3763
3031	serge	3764	int
		3765	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3766	struct drm_file *file)
		3767	{
		3768	struct drm_i915_gem_busy *args = data;
		3769	struct drm_i915_gem_object *obj;
		3770	int ret;
2332	Serge	3771
3031	serge	3772	ret = i915_mutex_lock_interruptible(dev);
		3773	if (ret)
		3774	return ret;
2332	Serge	3775
3480	Serge	3776	if(args->handle == -2)
		3777	{
		3778	obj = get_fb_obj();
		3779	drm_gem_object_reference(&obj->base);
		3780	}
		3781	else
4104	Serge	3782	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3783	if (&obj->base == NULL) {
		3784	ret = -ENOENT;
		3785	goto unlock;
		3786	}
2332	Serge	3787
3031	serge	3788	/* Count all active objects as busy, even if they are currently not used
		3789	* by the gpu. Users of this interface expect objects to eventually
		3790	* become non-busy without any further actions, therefore emit any
		3791	* necessary flushes here.
		3792	*/
		3793	ret = i915_gem_object_flush_active(obj);
2332	Serge	3794
3031	serge	3795	args->busy = obj->active;
		3796	if (obj->ring) {
		3797	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3798	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3799	}
2332	Serge	3800
3031	serge	3801	drm_gem_object_unreference(&obj->base);
		3802	unlock:
		3803	mutex_unlock(&dev->struct_mutex);
		3804	return ret;
		3805	}
2332	Serge	3806
3031	serge	3807	int
		3808	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3809	struct drm_file *file_priv)
		3810	{
		3811	return i915_gem_ring_throttle(dev, file_priv);
		3812	}
2332	Serge	3813
3263	Serge	3814	#if 0
		3815
3031	serge	3816	int
		3817	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3818	struct drm_file *file_priv)
		3819	{
		3820	struct drm_i915_gem_madvise *args = data;
		3821	struct drm_i915_gem_object *obj;
		3822	int ret;
2332	Serge	3823
3031	serge	3824	switch (args->madv) {
		3825	case I915_MADV_DONTNEED:
		3826	case I915_MADV_WILLNEED:
		3827	break;
		3828	default:
		3829	return -EINVAL;
		3830	}
2332	Serge	3831
3031	serge	3832	ret = i915_mutex_lock_interruptible(dev);
		3833	if (ret)
		3834	return ret;
2332	Serge	3835
3031	serge	3836	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3837	if (&obj->base == NULL) {
		3838	ret = -ENOENT;
		3839	goto unlock;
		3840	}
2332	Serge	3841
3031	serge	3842	if (obj->pin_count) {
		3843	ret = -EINVAL;
		3844	goto out;
		3845	}
2332	Serge	3846
3031	serge	3847	if (obj->madv != __I915_MADV_PURGED)
		3848	obj->madv = args->madv;
2332	Serge	3849
3031	serge	3850	/* if the object is no longer attached, discard its backing storage */
		3851	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3852	i915_gem_object_truncate(obj);
2332	Serge	3853
3031	serge	3854	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3855
3031	serge	3856	out:
		3857	drm_gem_object_unreference(&obj->base);
		3858	unlock:
		3859	mutex_unlock(&dev->struct_mutex);
		3860	return ret;
		3861	}
		3862	#endif
2332	Serge	3863
3031	serge	3864	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3865	const struct drm_i915_gem_object_ops *ops)
		3866	{
4104	Serge	3867	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3868	INIT_LIST_HEAD(&obj->ring_list);
		3869	INIT_LIST_HEAD(&obj->exec_list);
4104	Serge	3870	INIT_LIST_HEAD(&obj->obj_exec_link);
		3871	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3872
3031	serge	3873	obj->ops = ops;
		3874
		3875	obj->fence_reg = I915_FENCE_REG_NONE;
		3876	obj->madv = I915_MADV_WILLNEED;
		3877	/* Avoid an unnecessary call to unbind on the first bind. */
		3878	obj->map_and_fenceable = true;
		3879
		3880	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3881	}
		3882
		3883	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3884	.get_pages = i915_gem_object_get_pages_gtt,
		3885	.put_pages = i915_gem_object_put_pages_gtt,
		3886	};
		3887
2332	Serge	3888	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3889	size_t size)
		3890	{
		3891	struct drm_i915_gem_object *obj;
3031	serge	3892	struct address_space *mapping;
3480	Serge	3893	gfp_t mask;
2340	Serge	3894
3746	Serge	3895	obj = i915_gem_object_alloc(dev);
2332	Serge	3896	if (obj == NULL)
3746	Serge	3897	{
		3898	FAIL();
2332	Serge	3899	return NULL;
3746	Serge	3900	};
2332	Serge	3901
		3902	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3903	i915_gem_object_free(obj);
2332	Serge	3904	return NULL;
		3905	}
		3906
		3907
3031	serge	3908	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3909
		3910	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3911	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3912
3031	serge	3913	if (HAS_LLC(dev)) {
		3914	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3915	* cache) for about a 10% performance improvement
		3916	* compared to uncached. Graphics requests other than
		3917	* display scanout are coherent with the CPU in
		3918	* accessing this cache. This means in this mode we
		3919	* don't need to clflush on the CPU side, and on the
		3920	* GPU side we only need to flush internal caches to
		3921	* get data visible to the CPU.
		3922	*
		3923	* However, we maintain the display planes as UC, and so
		3924	* need to rebind when first used as such.
		3925	*/
		3926	obj->cache_level = I915_CACHE_LLC;
		3927	} else
		3928	obj->cache_level = I915_CACHE_NONE;
		3929
		3930	return obj;
		3931	}
		3932
2344	Serge	3933	int i915_gem_init_object(struct drm_gem_object *obj)
		3934	{
		3935	BUG();
2332	Serge	3936
2344	Serge	3937	return 0;
		3938	}
2332	Serge	3939
3031	serge	3940	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3941	{
3031	serge	3942	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3943	struct drm_device *dev = obj->base.dev;
		3944	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3945	struct i915_vma vma, next;
2332	Serge	3946
3031	serge	3947	trace_i915_gem_object_destroy(obj);
		3948
		3949
		3950	obj->pin_count = 0;
4104	Serge	3951	/* NB: 0 or 1 elements */
		3952	WARN_ON(!list_empty(&obj->vma_list) &&
		3953	!list_is_singular(&obj->vma_list));
		3954	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3955	int ret = i915_vma_unbind(vma);
		3956	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	3957	bool was_interruptible;
		3958
		3959	was_interruptible = dev_priv->mm.interruptible;
		3960	dev_priv->mm.interruptible = false;
		3961
4104	Serge	3962	WARN_ON(i915_vma_unbind(vma));
3031	serge	3963
		3964	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3965	}
4104	Serge	3966	}
2332	Serge	3967
4104	Serge	3968	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		3969	* before progressing. */
		3970	if (obj->stolen)
		3971	i915_gem_object_unpin_pages(obj);
		3972
		3973	if (WARN_ON(obj->pages_pin_count))
3031	serge	3974	obj->pages_pin_count = 0;
		3975	i915_gem_object_put_pages(obj);
		3976	// i915_gem_object_free_mmap_offset(obj);
4104	Serge	3977	i915_gem_object_release_stolen(obj);
2332	Serge	3978
3243	Serge	3979	BUG_ON(obj->pages);
2332	Serge	3980
3031	serge	3981
3290	Serge	3982	if(obj->base.filp != NULL)
		3983	{
3298	Serge	3984	// printf("filp %p\n", obj->base.filp);
3290	Serge	3985	shmem_file_delete(obj->base.filp);
		3986	}
		3987
2344	Serge	3988	drm_gem_object_release(&obj->base);
		3989	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3990
2344	Serge	3991	kfree(obj->bit_17);
4104	Serge	3992	i915_gem_object_free(obj);
2344	Serge	3993	}
2332	Serge	3994
4104	Serge	3995	struct i915_vma i915_gem_vma_create(struct drm_i915_gem_object obj,
		3996	struct i915_address_space *vm)
		3997	{
		3998	struct i915_vma vma = kzalloc(sizeof(vma), GFP_KERNEL);
		3999	if (vma == NULL)
		4000	return ERR_PTR(-ENOMEM);
		4001
		4002	INIT_LIST_HEAD(&vma->vma_link);
		4003	INIT_LIST_HEAD(&vma->mm_list);
		4004	INIT_LIST_HEAD(&vma->exec_list);
		4005	vma->vm = vm;
		4006	vma->obj = obj;
		4007
		4008	/* Keep GGTT vmas first to make debug easier */
		4009	if (i915_is_ggtt(vm))
		4010	list_add(&vma->vma_link, &obj->vma_list);
		4011	else
		4012	list_add_tail(&vma->vma_link, &obj->vma_list);
		4013
		4014	return vma;
		4015	}
		4016
		4017	void i915_gem_vma_destroy(struct i915_vma *vma)
		4018	{
		4019	WARN_ON(vma->node.allocated);
		4020	list_del(&vma->vma_link);
		4021	kfree(vma);
		4022	}
		4023
3031	serge	4024	#if 0
		4025	int
		4026	i915_gem_idle(struct drm_device *dev)
2344	Serge	4027	{
3031	serge	4028	drm_i915_private_t *dev_priv = dev->dev_private;
		4029	int ret;
2332	Serge	4030
4104	Serge	4031	if (dev_priv->ums.mm_suspended) {
3031	serge	4032	mutex_unlock(&dev->struct_mutex);
		4033	return 0;
		4034	}
2332	Serge	4035
3031	serge	4036	ret = i915_gpu_idle(dev);
		4037	if (ret) {
		4038	mutex_unlock(&dev->struct_mutex);
		4039	return ret;
		4040	}
		4041	i915_gem_retire_requests(dev);
		4042
3480	Serge	4043	/* Under UMS, be paranoid and evict. */
		4044	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		4045	i915_gem_evict_everything(dev);
		4046
		4047	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	4048
		4049	i915_kernel_lost_context(dev);
		4050	i915_gem_cleanup_ringbuffer(dev);
		4051
		4052	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	4053	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	4054
		4055	return 0;
2344	Serge	4056	}
3031	serge	4057	#endif
2332	Serge	4058
3031	serge	4059	void i915_gem_l3_remap(struct drm_device *dev)
		4060	{
		4061	drm_i915_private_t *dev_priv = dev->dev_private;
		4062	u32 misccpctl;
		4063	int i;
2332	Serge	4064
3480	Serge	4065	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	4066	return;
2332	Serge	4067
3243	Serge	4068	if (!dev_priv->l3_parity.remap_info)
3031	serge	4069	return;
2332	Serge	4070
3031	serge	4071	misccpctl = I915_READ(GEN7_MISCCPCTL);
		4072	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		4073	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	4074
3031	serge	4075	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		4076	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	4077	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	4078	DRM_DEBUG("0x%x was already programmed to %x\n",
		4079	GEN7_L3LOG_BASE + i, remap);
3243	Serge	4080	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	4081	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	4082	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	4083	}
2332	Serge	4084
3031	serge	4085	/* Make sure all the writes land before disabling dop clock gating */
		4086	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	4087
3031	serge	4088	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		4089	}
2332	Serge	4090
3031	serge	4091	void i915_gem_init_swizzling(struct drm_device *dev)
		4092	{
		4093	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	4094
3031	serge	4095	if (INTEL_INFO(dev)->gen < 5 \|\|
		4096	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4097	return;
2332	Serge	4098
3031	serge	4099	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4100	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4101
3031	serge	4102	if (IS_GEN5(dev))
		4103	return;
2344	Serge	4104
3031	serge	4105	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4106	if (IS_GEN6(dev))
		4107	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4108	else if (IS_GEN7(dev))
		4109	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	4110	else
3480	Serge	4111	BUG();
3031	serge	4112	}
		4113
		4114	static bool
		4115	intel_enable_blt(struct drm_device *dev)
		4116	{
		4117	if (!HAS_BLT(dev))
		4118	return false;
		4119
		4120	/* The blitter was dysfunctional on early prototypes */
		4121	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4122	DRM_INFO("BLT not supported on this pre-production hardware;"
		4123	" graphics performance will be degraded.\n");
		4124	return false;
		4125	}
		4126
		4127	return true;
		4128	}
		4129
3480	Serge	4130	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	4131	{
3480	Serge	4132	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4133	int ret;
2351	Serge	4134
2332	Serge	4135	ret = intel_init_render_ring_buffer(dev);
		4136	if (ret)
		4137	return ret;
		4138
		4139	if (HAS_BSD(dev)) {
		4140	ret = intel_init_bsd_ring_buffer(dev);
		4141	if (ret)
		4142	goto cleanup_render_ring;
		4143	}
		4144
3031	serge	4145	if (intel_enable_blt(dev)) {
2332	Serge	4146	ret = intel_init_blt_ring_buffer(dev);
		4147	if (ret)
		4148	goto cleanup_bsd_ring;
		4149	}
		4150
4104	Serge	4151	if (HAS_VEBOX(dev)) {
		4152	ret = intel_init_vebox_ring_buffer(dev);
		4153	if (ret)
		4154	goto cleanup_blt_ring;
		4155	}
		4156
		4157
3480	Serge	4158	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4159	if (ret)
4104	Serge	4160	goto cleanup_vebox_ring;
2351	Serge	4161
2332	Serge	4162	return 0;
		4163
4104	Serge	4164	cleanup_vebox_ring:
		4165	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4166	cleanup_blt_ring:
		4167	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4168	cleanup_bsd_ring:
		4169	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4170	cleanup_render_ring:
		4171	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4172
2332	Serge	4173	return ret;
		4174	}
		4175
3480	Serge	4176	int
		4177	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4178	{
3480	Serge	4179	drm_i915_private_t *dev_priv = dev->dev_private;
		4180	int ret;
3031	serge	4181
3480	Serge	4182	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4183	return -EIO;
3031	serge	4184
4104	Serge	4185	if (dev_priv->ellc_size)
		4186	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4187
3746	Serge	4188	if (HAS_PCH_NOP(dev)) {
		4189	u32 temp = I915_READ(GEN7_MSG_CTL);
		4190	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4191	I915_WRITE(GEN7_MSG_CTL, temp);
		4192	}
		4193
3480	Serge	4194	i915_gem_l3_remap(dev);
		4195
		4196	i915_gem_init_swizzling(dev);
		4197
		4198	ret = i915_gem_init_rings(dev);
		4199	if (ret)
		4200	return ret;
		4201
		4202	/*
		4203	* XXX: There was some w/a described somewhere suggesting loading
		4204	* contexts before PPGTT.
		4205	*/
		4206	i915_gem_context_init(dev);
3746	Serge	4207	if (dev_priv->mm.aliasing_ppgtt) {
		4208	ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		4209	if (ret) {
		4210	i915_gem_cleanup_aliasing_ppgtt(dev);
		4211	DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		4212	}
		4213	}
3480	Serge	4214
		4215	return 0;
3031	serge	4216	}
		4217
		4218	#define LFB_SIZE 0xC00000
		4219
		4220	int i915_gem_init(struct drm_device *dev)
		4221	{
		4222	struct drm_i915_private *dev_priv = dev->dev_private;
		4223	int ret;
		4224
		4225	mutex_lock(&dev->struct_mutex);
3746	Serge	4226
		4227	if (IS_VALLEYVIEW(dev)) {
		4228	/* VLVA0 (potential hack), BIOS isn't actually waking us */
		4229	I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		4230	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
		4231	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4232	}
		4233
3480	Serge	4234	i915_gem_init_global_gtt(dev);
3746	Serge	4235
3031	serge	4236	ret = i915_gem_init_hw(dev);
		4237	mutex_unlock(&dev->struct_mutex);
		4238	if (ret) {
		4239	i915_gem_cleanup_aliasing_ppgtt(dev);
		4240	return ret;
		4241	}
		4242
3746	Serge	4243
3031	serge	4244	return 0;
		4245	}
		4246
2332	Serge	4247	void
		4248	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4249	{
		4250	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	4251	struct intel_ring_buffer *ring;
2332	Serge	4252	int i;
		4253
3031	serge	4254	for_each_ring(ring, dev_priv, i)
		4255	intel_cleanup_ring_buffer(ring);
2332	Serge	4256	}
		4257
3031	serge	4258	#if 0
		4259
2332	Serge	4260	int
		4261	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		4262	struct drm_file *file_priv)
		4263	{
4104	Serge	4264	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4265	int ret;
2332	Serge	4266
		4267	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4268	return 0;
		4269
3480	Serge	4270	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	4271	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	4272	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	4273	}
		4274
		4275	mutex_lock(&dev->struct_mutex);
4104	Serge	4276	dev_priv->ums.mm_suspended = 0;
2332	Serge	4277
3031	serge	4278	ret = i915_gem_init_hw(dev);
2332	Serge	4279	if (ret != 0) {
		4280	mutex_unlock(&dev->struct_mutex);
		4281	return ret;
		4282	}
		4283
4104	Serge	4284	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
2332	Serge	4285	mutex_unlock(&dev->struct_mutex);
		4286
		4287	ret = drm_irq_install(dev);
		4288	if (ret)
		4289	goto cleanup_ringbuffer;
		4290
		4291	return 0;
		4292
		4293	cleanup_ringbuffer:
		4294	mutex_lock(&dev->struct_mutex);
		4295	i915_gem_cleanup_ringbuffer(dev);
4104	Serge	4296	dev_priv->ums.mm_suspended = 1;
2332	Serge	4297	mutex_unlock(&dev->struct_mutex);
		4298
		4299	return ret;
		4300	}
		4301
		4302	int
		4303	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4304	struct drm_file *file_priv)
		4305	{
4104	Serge	4306	struct drm_i915_private *dev_priv = dev->dev_private;
		4307	int ret;
		4308
2332	Serge	4309	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4310	return 0;
		4311
		4312	drm_irq_uninstall(dev);
4104	Serge	4313
		4314	mutex_lock(&dev->struct_mutex);
		4315	ret = i915_gem_idle(dev);
		4316
		4317	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		4318	* We need to replace this with a semaphore, or something.
		4319	* And not confound ums.mm_suspended!
		4320	*/
		4321	if (ret != 0)
		4322	dev_priv->ums.mm_suspended = 1;
		4323	mutex_unlock(&dev->struct_mutex);
		4324
		4325	return ret;
2332	Serge	4326	}
		4327
		4328	void
		4329	i915_gem_lastclose(struct drm_device *dev)
		4330	{
		4331	int ret;
		4332
		4333	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4334	return;
		4335
4104	Serge	4336	mutex_lock(&dev->struct_mutex);
2332	Serge	4337	ret = i915_gem_idle(dev);
		4338	if (ret)
		4339	DRM_ERROR("failed to idle hardware: %d\n", ret);
4104	Serge	4340	mutex_unlock(&dev->struct_mutex);
2332	Serge	4341	}
		4342	#endif
		4343
		4344	static void
2326	Serge	4345	init_ring_lists(struct intel_ring_buffer *ring)
		4346	{
		4347	INIT_LIST_HEAD(&ring->active_list);
		4348	INIT_LIST_HEAD(&ring->request_list);
		4349	}
		4350
4104	Serge	4351	static void i915_init_vm(struct drm_i915_private *dev_priv,
		4352	struct i915_address_space *vm)
		4353	{
		4354	vm->dev = dev_priv->dev;
		4355	INIT_LIST_HEAD(&vm->active_list);
		4356	INIT_LIST_HEAD(&vm->inactive_list);
		4357	INIT_LIST_HEAD(&vm->global_link);
		4358	list_add(&vm->global_link, &dev_priv->vm_list);
		4359	}
		4360
2326	Serge	4361	void
		4362	i915_gem_load(struct drm_device *dev)
		4363	{
3480	Serge	4364	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4365	int i;
		4366
4104	Serge	4367	INIT_LIST_HEAD(&dev_priv->vm_list);
		4368	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4369
3031	serge	4370	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4371	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4372	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4373	for (i = 0; i < I915_NUM_RINGS; i++)
		4374	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4375	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4376	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4377	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4378	i915_gem_retire_work_handler);
3480	Serge	4379	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4380
		4381	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4382	if (IS_GEN3(dev)) {
3031	serge	4383	I915_WRITE(MI_ARB_STATE,
		4384	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4385	}
		4386
		4387	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4388
3746	Serge	4389	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4390	dev_priv->num_fence_regs = 32;
		4391	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4392	dev_priv->num_fence_regs = 16;
		4393	else
		4394	dev_priv->num_fence_regs = 8;
		4395
		4396	/* Initialize fence registers to zero */
3746	Serge	4397	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4398	i915_gem_restore_fences(dev);
2326	Serge	4399
		4400	i915_gem_detect_bit_6_swizzle(dev);
		4401
		4402	dev_priv->mm.interruptible = true;
		4403
		4404	}
		4405
4104	Serge	4406	#if 0
		4407	/*
		4408	* Create a physically contiguous memory object for this object
		4409	* e.g. for cursor + overlay regs
		4410	*/
		4411	static int i915_gem_init_phys_object(struct drm_device *dev,
		4412	int id, int size, int align)
		4413	{
		4414	drm_i915_private_t *dev_priv = dev->dev_private;
		4415	struct drm_i915_gem_phys_object *phys_obj;
		4416	int ret;
2326	Serge	4417
4104	Serge	4418	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)
		4419	return 0;
		4420
		4421	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
		4422	if (!phys_obj)
		4423	return -ENOMEM;
		4424
		4425	phys_obj->id = id;
		4426
		4427	phys_obj->handle = drm_pci_alloc(dev, size, align);
		4428	if (!phys_obj->handle) {
		4429	ret = -ENOMEM;
		4430	goto kfree_obj;
		4431	}
		4432	#ifdef CONFIG_X86
		4433	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4434	#endif
		4435
		4436	dev_priv->mm.phys_objs[id - 1] = phys_obj;
		4437
		4438	return 0;
		4439	kfree_obj:
		4440	kfree(phys_obj);
		4441	return ret;
		4442	}
		4443
		4444	static void i915_gem_free_phys_object(struct drm_device *dev, int id)
		4445	{
		4446	drm_i915_private_t *dev_priv = dev->dev_private;
		4447	struct drm_i915_gem_phys_object *phys_obj;
		4448
		4449	if (!dev_priv->mm.phys_objs[id - 1])
		4450	return;
		4451
		4452	phys_obj = dev_priv->mm.phys_objs[id - 1];
		4453	if (phys_obj->cur_obj) {
		4454	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
		4455	}
		4456
		4457	#ifdef CONFIG_X86
		4458	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4459	#endif
		4460	drm_pci_free(dev, phys_obj->handle);
		4461	kfree(phys_obj);
		4462	dev_priv->mm.phys_objs[id - 1] = NULL;
		4463	}
		4464
		4465	void i915_gem_free_all_phys_object(struct drm_device *dev)
		4466	{
		4467	int i;
		4468
		4469	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
		4470	i915_gem_free_phys_object(dev, i);
		4471	}
		4472
		4473	void i915_gem_detach_phys_object(struct drm_device *dev,
		4474	struct drm_i915_gem_object *obj)
		4475	{
		4476	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4477	char *vaddr;
		4478	int i;
		4479	int page_count;
		4480
		4481	if (!obj->phys_obj)
		4482	return;
		4483	vaddr = obj->phys_obj->handle->vaddr;
		4484
		4485	page_count = obj->base.size / PAGE_SIZE;
		4486	for (i = 0; i < page_count; i++) {
		4487	struct page *page = shmem_read_mapping_page(mapping, i);
		4488	if (!IS_ERR(page)) {
		4489	char *dst = kmap_atomic(page);
		4490	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
		4491	kunmap_atomic(dst);
		4492
		4493	drm_clflush_pages(&page, 1);
		4494
		4495	set_page_dirty(page);
		4496	mark_page_accessed(page);
		4497	page_cache_release(page);
		4498	}
		4499	}
		4500	i915_gem_chipset_flush(dev);
		4501
		4502	obj->phys_obj->cur_obj = NULL;
		4503	obj->phys_obj = NULL;
		4504	}
		4505
		4506	int
		4507	i915_gem_attach_phys_object(struct drm_device *dev,
		4508	struct drm_i915_gem_object *obj,
		4509	int id,
		4510	int align)
		4511	{
		4512	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4513	drm_i915_private_t *dev_priv = dev->dev_private;
		4514	int ret = 0;
		4515	int page_count;
		4516	int i;
		4517
		4518	if (id > I915_MAX_PHYS_OBJECT)
		4519	return -EINVAL;
		4520
		4521	if (obj->phys_obj) {
		4522	if (obj->phys_obj->id == id)
		4523	return 0;
		4524	i915_gem_detach_phys_object(dev, obj);
		4525	}
		4526
		4527	/* create a new object */
		4528	if (!dev_priv->mm.phys_objs[id - 1]) {
		4529	ret = i915_gem_init_phys_object(dev, id,
		4530	obj->base.size, align);
		4531	if (ret) {
		4532	DRM_ERROR("failed to init phys object %d size: %zu\n",
		4533	id, obj->base.size);
		4534	return ret;
		4535	}
		4536	}
		4537
		4538	/* bind to the object */
		4539	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
		4540	obj->phys_obj->cur_obj = obj;
		4541
		4542	page_count = obj->base.size / PAGE_SIZE;
		4543
		4544	for (i = 0; i < page_count; i++) {
		4545	struct page *page;
		4546	char dst, src;
		4547
		4548	page = shmem_read_mapping_page(mapping, i);
		4549	if (IS_ERR(page))
		4550	return PTR_ERR(page);
		4551
		4552	src = kmap_atomic(page);
		4553	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
		4554	memcpy(dst, src, PAGE_SIZE);
		4555	kunmap_atomic(src);
		4556
		4557	mark_page_accessed(page);
		4558	page_cache_release(page);
		4559	}
		4560
		4561	return 0;
		4562	}
		4563
		4564	static int
		4565	i915_gem_phys_pwrite(struct drm_device *dev,
		4566	struct drm_i915_gem_object *obj,
		4567	struct drm_i915_gem_pwrite *args,
		4568	struct drm_file *file_priv)
		4569	{
		4570	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
		4571	char __user *user_data = to_user_ptr(args->data_ptr);
		4572
		4573	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		4574	unsigned long unwritten;
		4575
		4576	/* The physical object once assigned is fixed for the lifetime
		4577	* of the obj, so we can safely drop the lock and continue
		4578	* to access vaddr.
		4579	*/
		4580	mutex_unlock(&dev->struct_mutex);
		4581	unwritten = copy_from_user(vaddr, user_data, args->size);
		4582	mutex_lock(&dev->struct_mutex);
		4583	if (unwritten)
		4584	return -EFAULT;
		4585	}
		4586
		4587	i915_gem_chipset_flush(dev);
		4588	return 0;
		4589	}
		4590
		4591	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4592	{
		4593	struct drm_i915_file_private *file_priv = file->driver_priv;
		4594
		4595	/* Clean up our request list when the client is going away, so that
		4596	* later retire_requests won't dereference our soon-to-be-gone
		4597	* file_priv.
		4598	*/
		4599	spin_lock(&file_priv->mm.lock);
		4600	while (!list_empty(&file_priv->mm.request_list)) {
		4601	struct drm_i915_gem_request *request;
		4602
		4603	request = list_first_entry(&file_priv->mm.request_list,
		4604	struct drm_i915_gem_request,
		4605	client_list);
		4606	list_del(&request->client_list);
		4607	request->file_priv = NULL;
		4608	}
		4609	spin_unlock(&file_priv->mm.lock);
		4610	}
		4611	#endif
		4612
		4613	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4614	{
		4615	if (!mutex_is_locked(mutex))
		4616	return false;
		4617
		4618	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4619	return mutex->owner == task;
		4620	#else
		4621	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4622	return false;
		4623	#endif
		4624	}
		4625
		4626	/* All the new VM stuff */
		4627	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4628	struct i915_address_space *vm)
		4629	{
		4630	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4631	struct i915_vma *vma;
		4632
		4633	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4634	vm = &dev_priv->gtt.base;
		4635
		4636	BUG_ON(list_empty(&o->vma_list));
		4637	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4638	if (vma->vm == vm)
		4639	return vma->node.start;
		4640
		4641	}
		4642	return 0; //-1;
		4643	}
		4644
		4645	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4646	struct i915_address_space *vm)
		4647	{
		4648	struct i915_vma *vma;
		4649
		4650	list_for_each_entry(vma, &o->vma_list, vma_link)
		4651	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4652	return true;
		4653
		4654	return false;
		4655	}
		4656
		4657	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4658	{
		4659	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4660	struct i915_address_space *vm;
		4661
		4662	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
		4663	if (i915_gem_obj_bound(o, vm))
		4664	return true;
		4665
		4666	return false;
		4667	}
		4668
		4669	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4670	struct i915_address_space *vm)
		4671	{
		4672	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4673	struct i915_vma *vma;
		4674
		4675	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4676	vm = &dev_priv->gtt.base;
		4677
		4678	BUG_ON(list_empty(&o->vma_list));
		4679
		4680	list_for_each_entry(vma, &o->vma_list, vma_link)
		4681	if (vma->vm == vm)
		4682	return vma->node.size;
		4683
		4684	return 0;
		4685	}
		4686	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
		4687	struct i915_address_space *vm)
		4688	{
		4689	struct i915_vma *vma;
		4690	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4691	if (vma->vm == vm)
		4692	return vma;
		4693
		4694	return NULL;
		4695	}
		4696
		4697	struct i915_vma *
		4698	i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
		4699	struct i915_address_space *vm)
		4700	{
		4701	struct i915_vma *vma;
		4702
		4703	vma = i915_gem_obj_to_vma(obj, vm);
		4704	if (!vma)
		4705	vma = i915_gem_vma_create(obj, vm);
		4706
		4707	return vma;
		4708	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 4104