WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
3260	Serge	34	#include
2330	Serge	35	#include
2326	Serge	36	//#include
3746	Serge	37	#include
2326	Serge	38	#include
		39
2344	Serge	40	extern int x86_clflush_size;
2332	Serge	41
3263	Serge	42	#define PROT_READ 0x1 /* page can be read */
		43	#define PROT_WRITE 0x2 /* page can be written */
		44	#define MAP_SHARED 0x01 /* Share changes */
		45
2344	Serge	46
3266	Serge	47	struct drm_i915_gem_object *get_fb_obj();
		48
3263	Serge	49	unsigned long vm_mmap(struct file *file, unsigned long addr,
		50	unsigned long len, unsigned long prot,
		51	unsigned long flag, unsigned long offset);
		52
2344	Serge	53	static inline void clflush(volatile void *__p)
		54	{
		55	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		56	}
		57
2332	Serge	58	#define MAX_ERRNO 4095
		59
		60	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		61
		62
2326	Serge	63
2332	Serge	64	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	65	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		66	bool force);
		67	static __must_check int
		68	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		69	struct i915_address_space *vm,
2332	Serge	70	unsigned alignment,
3031	serge	71	bool map_and_fenceable,
		72	bool nonblocking);
2332	Serge	73	static int i915_gem_phys_pwrite(struct drm_device *dev,
		74	struct drm_i915_gem_object *obj,
		75	struct drm_i915_gem_pwrite *args,
		76	struct drm_file *file);
2326	Serge	77
3031	serge	78	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		79	struct drm_i915_gem_object *obj);
		80	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		81	struct drm_i915_fence_reg *fence,
		82	bool enable);
2332	Serge	83
3031	serge	84	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
4104	Serge	85	static long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	86	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		87
4104	Serge	88	static bool cpu_cache_is_coherent(struct drm_device *dev,
		89	enum i915_cache_level level)
		90	{
		91	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		92	}
		93
		94	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		95	{
		96	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		97	return true;
		98
		99	return obj->pin_display;
		100	}
		101
3031	serge	102	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		103	{
		104	if (obj->tiling_mode)
		105	i915_gem_release_mmap(obj);
		106
		107	/* As we do not have an associated fence register, we will force
		108	* a tiling change if we ever need to acquire one.
		109	*/
		110	obj->fence_dirty = false;
		111	obj->fence_reg = I915_FENCE_REG_NONE;
		112	}
		113
2332	Serge	114	/* some bookkeeping */
		115	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		116	size_t size)
		117	{
4104	Serge	118	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	119	dev_priv->mm.object_count++;
		120	dev_priv->mm.object_memory += size;
4104	Serge	121	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	122	}
		123
		124	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		125	size_t size)
		126	{
4104	Serge	127	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	128	dev_priv->mm.object_count--;
		129	dev_priv->mm.object_memory -= size;
4104	Serge	130	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	131	}
		132
		133	static int
3480	Serge	134	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	135	{
		136	int ret;
		137
3480	Serge	138	#define EXIT_COND (!i915_reset_in_progress(error))
		139	if (EXIT_COND)
2332	Serge	140	return 0;
3255	Serge	141	#if 0
3031	serge	142	/*
		143	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		144	* userspace. If it takes that long something really bad is going on and
		145	* we should simply try to bail out and fail as gracefully as possible.
		146	*/
3480	Serge	147	ret = wait_event_interruptible_timeout(error->reset_queue,
		148	EXIT_COND,
		149	10*HZ);
3031	serge	150	if (ret == 0) {
		151	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		152	return -EIO;
		153	} else if (ret < 0) {
2332	Serge	154	return ret;
3031	serge	155	}
2332	Serge	156
3255	Serge	157	#endif
3480	Serge	158	#undef EXIT_COND
3255	Serge	159
2332	Serge	160	return 0;
		161	}
		162
		163	int i915_mutex_lock_interruptible(struct drm_device *dev)
		164	{
3480	Serge	165	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	166	int ret;
		167
3480	Serge	168	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	169	if (ret)
		170	return ret;
		171
3480	Serge	172	ret = mutex_lock_interruptible(&dev->struct_mutex);
		173	if (ret)
		174	return ret;
2332	Serge	175
		176	WARN_ON(i915_verify_lists(dev));
		177	return 0;
		178	}
		179
		180	static inline bool
		181	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		182	{
4104	Serge	183	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	184	}
		185
		186
		187	#if 0
		188
		189	int
		190	i915_gem_init_ioctl(struct drm_device dev, void data,
		191	struct drm_file *file)
		192	{
3480	Serge	193	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	194	struct drm_i915_gem_init *args = data;
		195
3031	serge	196	if (drm_core_check_feature(dev, DRIVER_MODESET))
		197	return -ENODEV;
		198
2332	Serge	199	if (args->gtt_start >= args->gtt_end \|\|
		200	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		201	return -EINVAL;
		202
3031	serge	203	/* GEM with user mode setting was never supported on ilk and later. */
		204	if (INTEL_INFO(dev)->gen >= 5)
		205	return -ENODEV;
		206
2332	Serge	207	mutex_lock(&dev->struct_mutex);
3480	Serge	208	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		209	args->gtt_end);
		210	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	211	mutex_unlock(&dev->struct_mutex);
		212
		213	return 0;
		214	}
2351	Serge	215	#endif
2332	Serge	216
		217	int
		218	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		219	struct drm_file *file)
		220	{
		221	struct drm_i915_private *dev_priv = dev->dev_private;
		222	struct drm_i915_gem_get_aperture *args = data;
		223	struct drm_i915_gem_object *obj;
		224	size_t pinned;
		225
		226	pinned = 0;
		227	mutex_lock(&dev->struct_mutex);
4104	Serge	228	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
3031	serge	229	if (obj->pin_count)
4104	Serge	230	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	231	mutex_unlock(&dev->struct_mutex);
		232
4104	Serge	233	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	234	args->aper_available_size = args->aper_size - pinned;
2332	Serge	235
		236	return 0;
		237	}
		238
3480	Serge	239	void i915_gem_object_alloc(struct drm_device dev)
		240	{
		241	struct drm_i915_private *dev_priv = dev->dev_private;
		242	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		243	}
		244
		245	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		246	{
		247	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		248	kfree(obj);
		249	}
		250
3031	serge	251	static int
		252	i915_gem_create(struct drm_file *file,
2332	Serge	253	struct drm_device *dev,
		254	uint64_t size,
		255	uint32_t *handle_p)
		256	{
		257	struct drm_i915_gem_object *obj;
		258	int ret;
		259	u32 handle;
		260
		261	size = roundup(size, PAGE_SIZE);
2342	Serge	262	if (size == 0)
		263	return -EINVAL;
2332	Serge	264
		265	/* Allocate the new object */
		266	obj = i915_gem_alloc_object(dev, size);
		267	if (obj == NULL)
		268	return -ENOMEM;
		269
		270	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	271	/* drop reference from allocate - handle holds it now */
		272	drm_gem_object_unreference_unlocked(&obj->base);
		273	if (ret)
2332	Serge	274	return ret;
		275
		276	*handle_p = handle;
		277	return 0;
		278	}
		279
		280	int
		281	i915_gem_dumb_create(struct drm_file *file,
		282	struct drm_device *dev,
		283	struct drm_mode_create_dumb *args)
		284	{
		285	/* have to work out size/pitch and return them */
		286	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		287	args->size = args->pitch * args->height;
		288	return i915_gem_create(file, dev,
		289	args->size, &args->handle);
		290	}
		291
2326	Serge	292	/**
2332	Serge	293	* Creates a new mm object and returns a handle to it.
		294	*/
		295	int
		296	i915_gem_create_ioctl(struct drm_device dev, void data,
		297	struct drm_file *file)
		298	{
		299	struct drm_i915_gem_create *args = data;
3031	serge	300
2332	Serge	301	return i915_gem_create(file, dev,
		302	args->size, &args->handle);
		303	}
		304
		305
3260	Serge	306	#if 0
2332	Serge	307
3031	serge	308	static inline int
		309	__copy_to_user_swizzled(char __user *cpu_vaddr,
		310	const char *gpu_vaddr, int gpu_offset,
2332	Serge	311	int length)
		312	{
3031	serge	313	int ret, cpu_offset = 0;
2332	Serge	314
3031	serge	315	while (length > 0) {
		316	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		317	int this_length = min(cacheline_end - gpu_offset, length);
		318	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	319
3031	serge	320	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		321	gpu_vaddr + swizzled_gpu_offset,
		322	this_length);
		323	if (ret)
		324	return ret + length;
2332	Serge	325
3031	serge	326	cpu_offset += this_length;
		327	gpu_offset += this_length;
		328	length -= this_length;
		329	}
		330
		331	return 0;
2332	Serge	332	}
		333
3031	serge	334	static inline int
		335	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		336	const char __user *cpu_vaddr,
		337	int length)
2332	Serge	338	{
3031	serge	339	int ret, cpu_offset = 0;
2332	Serge	340
		341	while (length > 0) {
		342	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		343	int this_length = min(cacheline_end - gpu_offset, length);
		344	int swizzled_gpu_offset = gpu_offset ^ 64;
		345
3031	serge	346	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	347	cpu_vaddr + cpu_offset,
		348	this_length);
3031	serge	349	if (ret)
		350	return ret + length;
		351
2332	Serge	352	cpu_offset += this_length;
		353	gpu_offset += this_length;
		354	length -= this_length;
		355	}
		356
3031	serge	357	return 0;
2332	Serge	358	}
		359
3031	serge	360	/* Per-page copy function for the shmem pread fastpath.
		361	* Flushes invalid cachelines before reading the target if
		362	* needs_clflush is set. */
2332	Serge	363	static int
3031	serge	364	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		365	char __user *user_data,
		366	bool page_do_bit17_swizzling, bool needs_clflush)
		367	{
		368	char *vaddr;
		369	int ret;
		370
		371	if (unlikely(page_do_bit17_swizzling))
		372	return -EINVAL;
		373
		374	vaddr = kmap_atomic(page);
		375	if (needs_clflush)
		376	drm_clflush_virt_range(vaddr + shmem_page_offset,
		377	page_length);
		378	ret = __copy_to_user_inatomic(user_data,
		379	vaddr + shmem_page_offset,
		380	page_length);
		381	kunmap_atomic(vaddr);
		382
		383	return ret ? -EFAULT : 0;
		384	}
		385
		386	static void
		387	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		388	bool swizzled)
		389	{
		390	if (unlikely(swizzled)) {
		391	unsigned long start = (unsigned long) addr;
		392	unsigned long end = (unsigned long) addr + length;
		393
		394	/* For swizzling simply ensure that we always flush both
		395	* channels. Lame, but simple and it works. Swizzled
		396	* pwrite/pread is far from a hotpath - current userspace
		397	* doesn't use it at all. */
		398	start = round_down(start, 128);
		399	end = round_up(end, 128);
		400
		401	drm_clflush_virt_range((void *)start, end - start);
		402	} else {
		403	drm_clflush_virt_range(addr, length);
		404	}
		405
		406	}
		407
		408	/* Only difference to the fast-path function is that this can handle bit17
		409	* and uses non-atomic copy and kmap functions. */
		410	static int
		411	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		412	char __user *user_data,
		413	bool page_do_bit17_swizzling, bool needs_clflush)
		414	{
		415	char *vaddr;
		416	int ret;
		417
		418	vaddr = kmap(page);
		419	if (needs_clflush)
		420	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		421	page_length,
		422	page_do_bit17_swizzling);
		423
		424	if (page_do_bit17_swizzling)
		425	ret = __copy_to_user_swizzled(user_data,
		426	vaddr, shmem_page_offset,
		427	page_length);
		428	else
		429	ret = __copy_to_user(user_data,
		430	vaddr + shmem_page_offset,
		431	page_length);
		432	kunmap(page);
		433
		434	return ret ? - EFAULT : 0;
		435	}
		436
		437	static int
		438	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	439	struct drm_i915_gem_object *obj,
		440	struct drm_i915_gem_pread *args,
		441	struct drm_file *file)
		442	{
3031	serge	443	char __user *user_data;
2332	Serge	444	ssize_t remain;
		445	loff_t offset;
3031	serge	446	int shmem_page_offset, page_length, ret = 0;
		447	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		448	int prefaulted = 0;
		449	int needs_clflush = 0;
3746	Serge	450	struct sg_page_iter sg_iter;
2332	Serge	451
3746	Serge	452	user_data = to_user_ptr(args->data_ptr);
2332	Serge	453	remain = args->size;
		454
3031	serge	455	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		456
		457	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		458	/* If we're not in the cpu read domain, set ourself into the gtt
		459	* read domain and manually flush cachelines (if required). This
		460	* optimizes for the case when the gpu will dirty the data
		461	* anyway again before the next pread happens. */
4104	Serge	462	needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
		463	if (i915_gem_obj_bound_any(obj)) {
3031	serge	464	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		465	if (ret)
		466	return ret;
		467	}
		468	}
		469
		470	ret = i915_gem_object_get_pages(obj);
		471	if (ret)
		472	return ret;
		473
		474	i915_gem_object_pin_pages(obj);
		475
2332	Serge	476	offset = args->offset;
		477
3746	Serge	478	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		479	offset >> PAGE_SHIFT) {
		480	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	481
3031	serge	482	if (remain <= 0)
		483	break;
		484
2332	Serge	485	/* Operation in this page
		486	*
3031	serge	487	* shmem_page_offset = offset within page in shmem file
2332	Serge	488	* page_length = bytes to copy for this page
		489	*/
3031	serge	490	shmem_page_offset = offset_in_page(offset);
2332	Serge	491	page_length = remain;
3031	serge	492	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		493	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	494
3031	serge	495	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		496	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	497
3031	serge	498	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		499	user_data, page_do_bit17_swizzling,
		500	needs_clflush);
		501	if (ret == 0)
		502	goto next_page;
2332	Serge	503
3031	serge	504	mutex_unlock(&dev->struct_mutex);
		505
4104	Serge	506	if (likely(!i915_prefault_disable) && !prefaulted) {
3031	serge	507	ret = fault_in_multipages_writeable(user_data, remain);
		508	/* Userspace is tricking us, but we've already clobbered
		509	* its pages with the prefault and promised to write the
		510	* data up to the first fault. Hence ignore any errors
		511	* and just continue. */
		512	(void)ret;
		513	prefaulted = 1;
		514	}
		515
		516	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		517	user_data, page_do_bit17_swizzling,
		518	needs_clflush);
		519
		520	mutex_lock(&dev->struct_mutex);
		521
		522	next_page:
2332	Serge	523	mark_page_accessed(page);
3031	serge	524
2332	Serge	525	if (ret)
3031	serge	526	goto out;
2332	Serge	527
		528	remain -= page_length;
		529	user_data += page_length;
		530	offset += page_length;
		531	}
		532
3031	serge	533	out:
		534	i915_gem_object_unpin_pages(obj);
		535
		536	return ret;
2332	Serge	537	}
		538
		539	/**
3031	serge	540	* Reads data from the object referenced by handle.
		541	*
		542	* On error, the contents of *data are undefined.
2332	Serge	543	*/
3031	serge	544	int
		545	i915_gem_pread_ioctl(struct drm_device dev, void data,
		546	struct drm_file *file)
		547	{
		548	struct drm_i915_gem_pread *args = data;
		549	struct drm_i915_gem_object *obj;
		550	int ret = 0;
		551
		552	if (args->size == 0)
		553	return 0;
		554
		555	if (!access_ok(VERIFY_WRITE,
3746	Serge	556	to_user_ptr(args->data_ptr),
3031	serge	557	args->size))
		558	return -EFAULT;
		559
		560	ret = i915_mutex_lock_interruptible(dev);
		561	if (ret)
		562	return ret;
		563
		564	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		565	if (&obj->base == NULL) {
		566	ret = -ENOENT;
		567	goto unlock;
		568	}
		569
		570	/* Bounds check source. */
		571	if (args->offset > obj->base.size \|\|
		572	args->size > obj->base.size - args->offset) {
		573	ret = -EINVAL;
		574	goto out;
		575	}
		576
		577	/* prime objects have no backing filp to GEM pread/pwrite
		578	* pages from.
		579	*/
		580	if (!obj->base.filp) {
		581	ret = -EINVAL;
		582	goto out;
		583	}
		584
		585	trace_i915_gem_object_pread(obj, args->offset, args->size);
		586
		587	ret = i915_gem_shmem_pread(dev, obj, args, file);
		588
		589	out:
		590	drm_gem_object_unreference(&obj->base);
		591	unlock:
		592	mutex_unlock(&dev->struct_mutex);
		593	return ret;
		594	}
		595
		596	/* This is the fast write path which cannot handle
		597	* page faults in the source data
		598	*/
		599
		600	static inline int
		601	fast_user_write(struct io_mapping *mapping,
		602	loff_t page_base, int page_offset,
		603	char __user *user_data,
		604	int length)
		605	{
		606	void __iomem *vaddr_atomic;
		607	void *vaddr;
		608	unsigned long unwritten;
		609
		610	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		611	/* We can use the cpu mem copy function because this is X86. */
		612	vaddr = (void __force*)vaddr_atomic + page_offset;
		613	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		614	user_data, length);
		615	io_mapping_unmap_atomic(vaddr_atomic);
		616	return unwritten;
		617	}
3260	Serge	618	#endif
3031	serge	619
3260	Serge	620	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	621	/**
		622	* This is the fast pwrite path, where we copy the data directly from the
		623	* user into the GTT, uncached.
		624	*/
2332	Serge	625	static int
3031	serge	626	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		627	struct drm_i915_gem_object *obj,
		628	struct drm_i915_gem_pwrite *args,
		629	struct drm_file *file)
2332	Serge	630	{
3031	serge	631	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	632	ssize_t remain;
3031	serge	633	loff_t offset, page_base;
		634	char __user *user_data;
		635	int page_offset, page_length, ret;
3260	Serge	636	char *vaddr;
2332	Serge	637
4104	Serge	638	ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
3031	serge	639	if (ret)
		640	goto out;
		641
		642	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		643	if (ret)
		644	goto out_unpin;
		645
		646	ret = i915_gem_object_put_fence(obj);
		647	if (ret)
		648	goto out_unpin;
		649
3260	Serge	650	vaddr = AllocKernelSpace(4096);
		651	if(vaddr == NULL)
		652	{
		653	ret = -ENOSPC;
		654	goto out_unpin;
		655	};
		656
3031	serge	657	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	658	remain = args->size;
		659
4104	Serge	660	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	661
3031	serge	662	while (remain > 0) {
		663	/* Operation in this page
		664	*
		665	* page_base = page offset within aperture
		666	* page_offset = offset within page
		667	* page_length = bytes to copy for this page
		668	*/
		669	page_base = offset & PAGE_MASK;
		670	page_offset = offset_in_page(offset);
		671	page_length = remain;
		672	if ((page_offset + remain) > PAGE_SIZE)
		673	page_length = PAGE_SIZE - page_offset;
2332	Serge	674
4371	Serge	675	MapPage(vaddr, dev_priv->gtt.mappable_base+page_base, PG_SW\|PG_NOCACHE);
3031	serge	676
3260	Serge	677	memcpy(vaddr+page_offset, user_data, page_length);
		678
3031	serge	679	remain -= page_length;
		680	user_data += page_length;
		681	offset += page_length;
2332	Serge	682	}
		683
3260	Serge	684	FreeKernelSpace(vaddr);
		685
3031	serge	686	out_unpin:
		687	i915_gem_object_unpin(obj);
		688	out:
		689	return ret;
		690	}
		691
		692	/* Per-page copy function for the shmem pwrite fastpath.
		693	* Flushes invalid cachelines before writing to the target if
		694	* needs_clflush_before is set and flushes out any written cachelines after
		695	* writing if needs_clflush is set. */
		696	static int
		697	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		698	char __user *user_data,
		699	bool page_do_bit17_swizzling,
		700	bool needs_clflush_before,
		701	bool needs_clflush_after)
		702	{
		703	char *vaddr;
3260	Serge	704	int ret = 0;
3031	serge	705
		706	if (unlikely(page_do_bit17_swizzling))
		707	return -EINVAL;
		708
4371	Serge	709	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW\|PG_NOCACHE);
3031	serge	710	if (needs_clflush_before)
		711	drm_clflush_virt_range(vaddr + shmem_page_offset,
		712	page_length);
3260	Serge	713	memcpy(vaddr + shmem_page_offset,
3031	serge	714	user_data,
		715	page_length);
		716	if (needs_clflush_after)
		717	drm_clflush_virt_range(vaddr + shmem_page_offset,
		718	page_length);
3260	Serge	719	FreeKernelSpace(vaddr);
3031	serge	720
		721	return ret ? -EFAULT : 0;
		722	}
3260	Serge	723	#if 0
3031	serge	724
		725	/* Only difference to the fast-path function is that this can handle bit17
		726	* and uses non-atomic copy and kmap functions. */
		727	static int
		728	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		729	char __user *user_data,
		730	bool page_do_bit17_swizzling,
		731	bool needs_clflush_before,
		732	bool needs_clflush_after)
		733	{
		734	char *vaddr;
		735	int ret;
		736
		737	vaddr = kmap(page);
		738	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		739	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		740	page_length,
		741	page_do_bit17_swizzling);
		742	if (page_do_bit17_swizzling)
		743	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		744	user_data,
		745	page_length);
		746	else
		747	ret = __copy_from_user(vaddr + shmem_page_offset,
		748	user_data,
		749	page_length);
		750	if (needs_clflush_after)
		751	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		752	page_length,
		753	page_do_bit17_swizzling);
		754	kunmap(page);
		755
		756	return ret ? -EFAULT : 0;
		757	}
3260	Serge	758	#endif
3031	serge	759
3260	Serge	760
3031	serge	761	static int
		762	i915_gem_shmem_pwrite(struct drm_device *dev,
		763	struct drm_i915_gem_object *obj,
		764	struct drm_i915_gem_pwrite *args,
		765	struct drm_file *file)
		766	{
		767	ssize_t remain;
		768	loff_t offset;
		769	char __user *user_data;
		770	int shmem_page_offset, page_length, ret = 0;
		771	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		772	int hit_slowpath = 0;
		773	int needs_clflush_after = 0;
		774	int needs_clflush_before = 0;
3746	Serge	775	struct sg_page_iter sg_iter;
3031	serge	776
3746	Serge	777	user_data = to_user_ptr(args->data_ptr);
3031	serge	778	remain = args->size;
		779
		780	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		781
		782	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		783	/* If we're not in the cpu write domain, set ourself into the gtt
		784	* write domain and manually flush cachelines (if required). This
		785	* optimizes for the case when the gpu will use the data
		786	* right away and we therefore have to clflush anyway. */
4104	Serge	787	needs_clflush_after = cpu_write_needs_clflush(obj);
		788	if (i915_gem_obj_bound_any(obj)) {
3031	serge	789	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		790	if (ret)
		791	return ret;
		792	}
		793	}
4104	Serge	794	/* Same trick applies to invalidate partially written cachelines read
		795	* before writing. */
		796	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		797	needs_clflush_before =
		798	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	799
		800	ret = i915_gem_object_get_pages(obj);
2332	Serge	801	if (ret)
3031	serge	802	return ret;
2332	Serge	803
3031	serge	804	i915_gem_object_pin_pages(obj);
2332	Serge	805
		806	offset = args->offset;
3031	serge	807	obj->dirty = 1;
2332	Serge	808
3746	Serge	809	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		810	offset >> PAGE_SHIFT) {
		811	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	812	int partial_cacheline_write;
2332	Serge	813
3031	serge	814	if (remain <= 0)
		815	break;
		816
2332	Serge	817	/* Operation in this page
		818	*
		819	* shmem_page_offset = offset within page in shmem file
		820	* page_length = bytes to copy for this page
		821	*/
		822	shmem_page_offset = offset_in_page(offset);
		823
		824	page_length = remain;
		825	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		826	page_length = PAGE_SIZE - shmem_page_offset;
		827
3031	serge	828	/* If we don't overwrite a cacheline completely we need to be
		829	* careful to have up-to-date data by first clflushing. Don't
		830	* overcomplicate things and flush the entire patch. */
		831	partial_cacheline_write = needs_clflush_before &&
		832	((shmem_page_offset \| page_length)
3260	Serge	833	& (x86_clflush_size - 1));
2332	Serge	834
3031	serge	835	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		836	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	837
3031	serge	838	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		839	user_data, page_do_bit17_swizzling,
		840	partial_cacheline_write,
		841	needs_clflush_after);
		842	if (ret == 0)
		843	goto next_page;
		844
		845	hit_slowpath = 1;
		846	mutex_unlock(&dev->struct_mutex);
3260	Serge	847	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	848
3260	Serge	849	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		850	// user_data, page_do_bit17_swizzling,
		851	// partial_cacheline_write,
		852	// needs_clflush_after);
		853
3031	serge	854	mutex_lock(&dev->struct_mutex);
		855
		856	next_page:
2332	Serge	857
3031	serge	858	if (ret)
		859	goto out;
		860
2332	Serge	861	remain -= page_length;
3031	serge	862	user_data += page_length;
2332	Serge	863	offset += page_length;
		864	}
		865
		866	out:
3031	serge	867	i915_gem_object_unpin_pages(obj);
		868
		869	if (hit_slowpath) {
3480	Serge	870	/*
		871	* Fixup: Flush cpu caches in case we didn't flush the dirty
		872	* cachelines in-line while writing and the object moved
		873	* out of the cpu write domain while we've dropped the lock.
		874	*/
		875	if (!needs_clflush_after &&
		876	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	877	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	878	i915_gem_chipset_flush(dev);
3031	serge	879	}
2332	Serge	880	}
		881
3031	serge	882	if (needs_clflush_after)
3243	Serge	883	i915_gem_chipset_flush(dev);
3031	serge	884
2332	Serge	885	return ret;
		886	}
3031	serge	887
		888	/**
		889	* Writes data to the object referenced by handle.
		890	*
		891	* On error, the contents of the buffer that were to be modified are undefined.
		892	*/
		893	int
		894	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		895	struct drm_file *file)
		896	{
		897	struct drm_i915_gem_pwrite *args = data;
		898	struct drm_i915_gem_object *obj;
		899	int ret;
		900
4104	Serge	901	if (args->size == 0)
		902	return 0;
		903
3480	Serge	904
3031	serge	905	ret = i915_mutex_lock_interruptible(dev);
		906	if (ret)
		907	return ret;
		908
		909	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		910	if (&obj->base == NULL) {
		911	ret = -ENOENT;
		912	goto unlock;
		913	}
		914
		915	/* Bounds check destination. */
		916	if (args->offset > obj->base.size \|\|
		917	args->size > obj->base.size - args->offset) {
		918	ret = -EINVAL;
		919	goto out;
		920	}
		921
		922	/* prime objects have no backing filp to GEM pread/pwrite
		923	* pages from.
		924	*/
		925	if (!obj->base.filp) {
		926	ret = -EINVAL;
		927	goto out;
		928	}
		929
		930	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		931
		932	ret = -EFAULT;
		933	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		934	* it would end up going through the fenced access, and we'll get
		935	* different detiling behavior between reading and writing.
		936	* pread/pwrite currently are reading and writing from the CPU
		937	* perspective, requiring manual detiling by the client.
		938	*/
3260	Serge	939	// if (obj->phys_obj) {
		940	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		941	// goto out;
		942	// }
3031	serge	943
4104	Serge	944	if (obj->tiling_mode == I915_TILING_NONE &&
		945	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		946	cpu_write_needs_clflush(obj)) {
3031	serge	947	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		948	/* Note that the gtt paths might fail with non-page-backed user
		949	* pointers (e.g. gtt mappings when moving data between
		950	* textures). Fallback to the shmem path in that case. */
		951	}
		952
		953	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	954	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	955
		956	out:
		957	drm_gem_object_unreference(&obj->base);
		958	unlock:
		959	mutex_unlock(&dev->struct_mutex);
		960	return ret;
		961	}
		962
		963	int
3480	Serge	964	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	965	bool interruptible)
		966	{
3480	Serge	967	if (i915_reset_in_progress(error)) {
3031	serge	968	/* Non-interruptible callers can't handle -EAGAIN, hence return
		969	* -EIO unconditionally for these. */
		970	if (!interruptible)
		971	return -EIO;
2332	Serge	972
3480	Serge	973	/* Recovery complete, but the reset failed ... */
		974	if (i915_terminally_wedged(error))
3031	serge	975	return -EIO;
2332	Serge	976
3031	serge	977	return -EAGAIN;
		978	}
2332	Serge	979
3031	serge	980	return 0;
		981	}
2332	Serge	982
3031	serge	983	/*
		984	* Compare seqno against outstanding lazy request. Emit a request if they are
		985	* equal.
		986	*/
		987	static int
		988	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		989	{
		990	int ret;
2332	Serge	991
3031	serge	992	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	993
3031	serge	994	ret = 0;
		995	if (seqno == ring->outstanding_lazy_request)
4104	Serge	996	ret = i915_add_request(ring, NULL);
2332	Serge	997
3031	serge	998	return ret;
		999	}
2332	Serge	1000
3031	serge	1001	/**
		1002	* __wait_seqno - wait until execution of seqno has finished
		1003	* @ring: the ring expected to report seqno
		1004	* @seqno: duh!
3480	Serge	1005	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1006	* @interruptible: do an interruptible wait (normally yes)
		1007	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1008	*
3480	Serge	1009	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1010	* values have been read by the caller in an smp safe manner. Where read-side
		1011	* locks are involved, it is sufficient to read the reset_counter before
		1012	* unlocking the lock that protects the seqno. For lockless tricks, the
		1013	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1014	* inserted.
		1015	*
3031	serge	1016	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1017	* errno with remaining time filled in timeout argument.
		1018	*/
		1019	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1020	unsigned reset_counter,
3031	serge	1021	bool interruptible, struct timespec *timeout)
		1022	{
		1023	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1024	struct timespec before, now, wait_time={1,0};
		1025	unsigned long timeout_jiffies;
		1026	long end;
		1027	bool wait_forever = true;
		1028	int ret;
2332	Serge	1029
4104	Serge	1030	WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
		1031
3031	serge	1032	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1033	return 0;
2332	Serge	1034
3031	serge	1035	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1036
3031	serge	1037	if (timeout != NULL) {
		1038	wait_time = *timeout;
		1039	wait_forever = false;
		1040	}
2332	Serge	1041
4104	Serge	1042	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
2332	Serge	1043
3031	serge	1044	if (WARN_ON(!ring->irq_get(ring)))
		1045	return -ENODEV;
2332	Serge	1046
3031	serge	1047	/* Record current time in case interrupted by signal, or wedged * */
		1048	getrawmonotonic(&before);
2332	Serge	1049
3031	serge	1050	#define EXIT_COND \
		1051	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1052	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1053	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1054	do {
3266	Serge	1055	if (interruptible)
		1056	end = wait_event_interruptible_timeout(ring->irq_queue,
		1057	EXIT_COND,
		1058	timeout_jiffies);
		1059	else
3031	serge	1060	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1061	timeout_jiffies);
2332	Serge	1062
3480	Serge	1063	/* We need to check whether any gpu reset happened in between
		1064	* the caller grabbing the seqno and now ... */
		1065	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1066	end = -EAGAIN;
		1067
		1068	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1069	* gone. */
		1070	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1071	if (ret)
		1072	end = ret;
		1073	} while (end == 0 && wait_forever);
2332	Serge	1074
3031	serge	1075	getrawmonotonic(&now);
2332	Serge	1076
3031	serge	1077	ring->irq_put(ring);
		1078	trace_i915_gem_request_wait_end(ring, seqno);
		1079	#undef EXIT_COND
2332	Serge	1080
3031	serge	1081	if (timeout) {
4104	Serge	1082	// struct timespec sleep_time = timespec_sub(now, before);
		1083	// timeout = timespec_sub(timeout, sleep_time);
3031	serge	1084	}
2332	Serge	1085
3031	serge	1086	switch (end) {
		1087	case -EIO:
		1088	case -EAGAIN: /* Wedged */
		1089	case -ERESTARTSYS: /* Signal */
		1090	return (int)end;
		1091	case 0: /* Timeout */
		1092	return -ETIME;
		1093	default: /* Completed */
		1094	WARN_ON(end < 0); /* We're not aware of other errors */
		1095	return 0;
		1096	}
		1097	}
2332	Serge	1098
3031	serge	1099	/**
		1100	* Waits for a sequence number to be signaled, and cleans up the
		1101	* request and object lists appropriately for that event.
		1102	*/
		1103	int
		1104	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1105	{
		1106	struct drm_device *dev = ring->dev;
		1107	struct drm_i915_private *dev_priv = dev->dev_private;
		1108	bool interruptible = dev_priv->mm.interruptible;
		1109	int ret;
2332	Serge	1110
3031	serge	1111	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1112	BUG_ON(seqno == 0);
2332	Serge	1113
3480	Serge	1114	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1115	if (ret)
		1116	return ret;
2332	Serge	1117
3031	serge	1118	ret = i915_gem_check_olr(ring, seqno);
		1119	if (ret)
		1120	return ret;
2332	Serge	1121
3480	Serge	1122	return __wait_seqno(ring, seqno,
		1123	atomic_read(&dev_priv->gpu_error.reset_counter),
		1124	interruptible, NULL);
3031	serge	1125	}
2332	Serge	1126
4104	Serge	1127	static int
		1128	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
		1129	struct intel_ring_buffer *ring)
		1130	{
		1131	i915_gem_retire_requests_ring(ring);
		1132
		1133	/* Manually manage the write flush as we may have not yet
		1134	* retired the buffer.
		1135	*
		1136	* Note that the last_write_seqno is always the earlier of
		1137	* the two (read/write) seqno, so if we haved successfully waited,
		1138	* we know we have passed the last write.
		1139	*/
		1140	obj->last_write_seqno = 0;
		1141	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1142
		1143	return 0;
		1144	}
		1145
3031	serge	1146	/**
		1147	* Ensures that all rendering to the object has completed and the object is
		1148	* safe to unbind from the GTT or access from the CPU.
		1149	*/
		1150	static __must_check int
		1151	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1152	bool readonly)
		1153	{
		1154	struct intel_ring_buffer *ring = obj->ring;
		1155	u32 seqno;
		1156	int ret;
2332	Serge	1157
3031	serge	1158	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1159	if (seqno == 0)
		1160	return 0;
2332	Serge	1161
3031	serge	1162	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1163	if (ret)
		1164	return ret;
2332	Serge	1165
4104	Serge	1166	return i915_gem_object_wait_rendering__tail(obj, ring);
3031	serge	1167	}
2332	Serge	1168
3260	Serge	1169	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1170	* as the object state may change during this call.
		1171	*/
		1172	static __must_check int
		1173	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1174	bool readonly)
		1175	{
		1176	struct drm_device *dev = obj->base.dev;
		1177	struct drm_i915_private *dev_priv = dev->dev_private;
		1178	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1179	unsigned reset_counter;
3260	Serge	1180	u32 seqno;
		1181	int ret;
2332	Serge	1182
3260	Serge	1183	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1184	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1185
3260	Serge	1186	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1187	if (seqno == 0)
		1188	return 0;
2332	Serge	1189
3480	Serge	1190	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1191	if (ret)
		1192	return ret;
2332	Serge	1193
3260	Serge	1194	ret = i915_gem_check_olr(ring, seqno);
		1195	if (ret)
		1196	return ret;
2332	Serge	1197
3480	Serge	1198	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1199	mutex_unlock(&dev->struct_mutex);
3480	Serge	1200	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1201	mutex_lock(&dev->struct_mutex);
4104	Serge	1202	if (ret)
		1203	return ret;
2332	Serge	1204
4104	Serge	1205	return i915_gem_object_wait_rendering__tail(obj, ring);
3260	Serge	1206	}
2332	Serge	1207
3260	Serge	1208	/**
		1209	* Called when user space prepares to use an object with the CPU, either
		1210	* through the mmap ioctl's mapping or a GTT mapping.
		1211	*/
		1212	int
		1213	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1214	struct drm_file *file)
		1215	{
		1216	struct drm_i915_gem_set_domain *args = data;
		1217	struct drm_i915_gem_object *obj;
		1218	uint32_t read_domains = args->read_domains;
		1219	uint32_t write_domain = args->write_domain;
		1220	int ret;
2332	Serge	1221
3260	Serge	1222	/* Only handle setting domains to types used by the CPU. */
		1223	if (write_domain & I915_GEM_GPU_DOMAINS)
		1224	return -EINVAL;
2332	Serge	1225
3260	Serge	1226	if (read_domains & I915_GEM_GPU_DOMAINS)
		1227	return -EINVAL;
2332	Serge	1228
3260	Serge	1229	/* Having something in the write domain implies it's in the read
		1230	* domain, and only that read domain. Enforce that in the request.
		1231	*/
		1232	if (write_domain != 0 && read_domains != write_domain)
		1233	return -EINVAL;
2332	Serge	1234
3260	Serge	1235	ret = i915_mutex_lock_interruptible(dev);
		1236	if (ret)
		1237	return ret;
2332	Serge	1238
3260	Serge	1239	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1240	if (&obj->base == NULL) {
		1241	ret = -ENOENT;
		1242	goto unlock;
		1243	}
2332	Serge	1244
3260	Serge	1245	/* Try to flush the object off the GPU without holding the lock.
		1246	* We will repeat the flush holding the lock in the normal manner
		1247	* to catch cases where we are gazumped.
		1248	*/
		1249	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1250	if (ret)
		1251	goto unref;
2332	Serge	1252
3260	Serge	1253	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1254	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1255
3260	Serge	1256	/* Silently promote "you're not bound, there was nothing to do"
		1257	* to success, since the client was just asking us to
		1258	* make sure everything was done.
		1259	*/
		1260	if (ret == -EINVAL)
		1261	ret = 0;
		1262	} else {
		1263	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1264	}
2332	Serge	1265
3260	Serge	1266	unref:
		1267	drm_gem_object_unreference(&obj->base);
		1268	unlock:
		1269	mutex_unlock(&dev->struct_mutex);
		1270	return ret;
		1271	}
2332	Serge	1272
4293	Serge	1273	/**
		1274	* Called when user space has done writes to this buffer
		1275	*/
		1276	int
		1277	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
		1278	struct drm_file *file)
		1279	{
		1280	struct drm_i915_gem_sw_finish *args = data;
		1281	struct drm_i915_gem_object *obj;
		1282	int ret = 0;
2332	Serge	1283
4293	Serge	1284	ret = i915_mutex_lock_interruptible(dev);
		1285	if (ret)
		1286	return ret;
2332	Serge	1287
4293	Serge	1288	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1289	if (&obj->base == NULL) {
		1290	ret = -ENOENT;
		1291	goto unlock;
		1292	}
2332	Serge	1293
4293	Serge	1294	/* Pinned buffers may be scanout, so flush the cache */
		1295	if (obj->pin_display)
		1296	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	1297
4293	Serge	1298	drm_gem_object_unreference(&obj->base);
		1299	unlock:
		1300	mutex_unlock(&dev->struct_mutex);
		1301	return ret;
		1302	}
		1303
3260	Serge	1304	/**
		1305	* Maps the contents of an object, returning the address it is mapped
		1306	* into.
		1307	*
		1308	* While the mapping holds a reference on the contents of the object, it doesn't
		1309	* imply a ref on the object itself.
		1310	*/
		1311	int
		1312	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1313	struct drm_file *file)
		1314	{
		1315	struct drm_i915_gem_mmap *args = data;
		1316	struct drm_gem_object *obj;
4392	Serge	1317	unsigned long addr;
2332	Serge	1318
3260	Serge	1319	obj = drm_gem_object_lookup(dev, file, args->handle);
		1320	if (obj == NULL)
		1321	return -ENOENT;
4104	Serge	1322
3260	Serge	1323	/* prime objects have no backing filp to GEM mmap
		1324	* pages from.
		1325	*/
		1326	if (!obj->filp) {
		1327	drm_gem_object_unreference_unlocked(obj);
		1328	return -EINVAL;
		1329	}
2332	Serge	1330
3263	Serge	1331	addr = vm_mmap(obj->filp, 0, args->size,
		1332	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1333	args->offset);
3260	Serge	1334	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1335	if (IS_ERR((void *)addr))
		1336	return addr;
2332	Serge	1337
3260	Serge	1338	args->addr_ptr = (uint64_t) addr;
2332	Serge	1339
3263	Serge	1340	return 0;
3260	Serge	1341	}
2332	Serge	1342
		1343
		1344
		1345
		1346
		1347
		1348
		1349
3031	serge	1350
		1351
		1352
		1353
		1354
		1355	/**
		1356	* i915_gem_release_mmap - remove physical page mappings
		1357	* @obj: obj in question
		1358	*
		1359	* Preserve the reservation of the mmapping with the DRM core code, but
		1360	* relinquish ownership of the pages back to the system.
		1361	*
		1362	* It is vital that we remove the page mapping if we have mapped a tiled
		1363	* object through the GTT and then lose the fence register due to
		1364	* resource pressure. Similarly if the object has been moved out of the
		1365	* aperture, than pages mapped into userspace must be revoked. Removing the
		1366	* mapping will then trigger a page fault on the next user access, allowing
		1367	* fixup by i915_gem_fault().
		1368	*/
		1369	void
		1370	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1371	{
		1372	if (!obj->fault_mappable)
		1373	return;
		1374
4104	Serge	1375	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1376	obj->fault_mappable = false;
		1377	}
		1378
3480	Serge	1379	uint32_t
2332	Serge	1380	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1381	{
		1382	uint32_t gtt_size;
		1383
		1384	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1385	tiling_mode == I915_TILING_NONE)
		1386	return size;
		1387
		1388	/* Previous chips need a power-of-two fence region when tiling */
		1389	if (INTEL_INFO(dev)->gen == 3)
		1390	gtt_size = 1024*1024;
		1391	else
		1392	gtt_size = 512*1024;
		1393
		1394	while (gtt_size < size)
		1395	gtt_size <<= 1;
		1396
		1397	return gtt_size;
		1398	}
		1399
		1400	/**
		1401	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1402	* @obj: object to check
		1403	*
		1404	* Return the required GTT alignment for an object, taking into account
		1405	* potential fence register mapping.
		1406	*/
3480	Serge	1407	uint32_t
		1408	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1409	int tiling_mode, bool fenced)
2332	Serge	1410	{
		1411	/*
		1412	* Minimum alignment is 4k (GTT page size), but might be greater
		1413	* if a fence register is needed for the object.
		1414	*/
3480	Serge	1415	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1416	tiling_mode == I915_TILING_NONE)
		1417	return 4096;
		1418
		1419	/*
		1420	* Previous chips need to be aligned to the size of the smallest
		1421	* fence register that can contain the object.
		1422	*/
		1423	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1424	}
		1425
		1426
		1427
3480	Serge	1428	int
		1429	i915_gem_mmap_gtt(struct drm_file *file,
		1430	struct drm_device *dev,
		1431	uint32_t handle,
		1432	uint64_t *offset)
		1433	{
		1434	struct drm_i915_private *dev_priv = dev->dev_private;
		1435	struct drm_i915_gem_object *obj;
		1436	unsigned long pfn;
		1437	char mem, ptr;
		1438	int ret;
		1439
		1440	ret = i915_mutex_lock_interruptible(dev);
		1441	if (ret)
		1442	return ret;
		1443
		1444	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1445	if (&obj->base == NULL) {
		1446	ret = -ENOENT;
		1447	goto unlock;
		1448	}
		1449
		1450	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1451	ret = -E2BIG;
		1452	goto out;
		1453	}
		1454
		1455	if (obj->madv != I915_MADV_WILLNEED) {
		1456	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1457	ret = -EINVAL;
		1458	goto out;
		1459	}
		1460	/* Now bind it into the GTT if needed */
4104	Serge	1461	ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
3480	Serge	1462	if (ret)
		1463	goto out;
		1464
		1465	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1466	if (ret)
		1467	goto unpin;
		1468
		1469	ret = i915_gem_object_get_fence(obj);
		1470	if (ret)
		1471	goto unpin;
		1472
		1473	obj->fault_mappable = true;
		1474
4104	Serge	1475	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1476
		1477	/* Finally, remap it using the new GTT offset */
		1478
		1479	mem = UserAlloc(obj->base.size);
		1480	if(unlikely(mem == NULL))
		1481	{
		1482	ret = -ENOMEM;
		1483	goto unpin;
		1484	}
		1485
		1486	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1487	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1488
		1489	unpin:
		1490	i915_gem_object_unpin(obj);
		1491
		1492
4104	Serge	1493	*offset = mem;
3480	Serge	1494
		1495	out:
		1496	drm_gem_object_unreference(&obj->base);
		1497	unlock:
		1498	mutex_unlock(&dev->struct_mutex);
		1499	return ret;
		1500	}
		1501
		1502	/**
		1503	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1504	* @dev: DRM device
		1505	* @data: GTT mapping ioctl data
		1506	* @file: GEM object info
		1507	*
		1508	* Simply returns the fake offset to userspace so it can mmap it.
		1509	* The mmap call will end up in drm_gem_mmap(), which will set things
		1510	* up so we can get faults in the handler above.
		1511	*
		1512	* The fault handler will take care of binding the object into the GTT
		1513	* (since it may have been evicted to make room for something), allocating
		1514	* a fence register, and mapping the appropriate aperture address into
		1515	* userspace.
		1516	*/
		1517	int
		1518	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1519	struct drm_file *file)
		1520	{
		1521	struct drm_i915_gem_mmap_gtt *args = data;
		1522
		1523	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1524	}
		1525
3031	serge	1526	/* Immediately discard the backing storage */
		1527	static void
		1528	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1529	{
		1530	// struct inode *inode;
2332	Serge	1531
3031	serge	1532	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1533
3263	Serge	1534	if (obj->base.filp == NULL)
		1535	return;
2332	Serge	1536
3031	serge	1537	/* Our goal here is to return as much of the memory as
		1538	* is possible back to the system as we are called from OOM.
		1539	* To do this we must instruct the shmfs to drop all of its
		1540	* backing pages, now.
		1541	*/
		1542	// inode = obj->base.filp->f_path.dentry->d_inode;
		1543	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1544
3031	serge	1545	obj->madv = __I915_MADV_PURGED;
		1546	}
2332	Serge	1547
3031	serge	1548	static inline int
		1549	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1550	{
		1551	return obj->madv == I915_MADV_DONTNEED;
		1552	}
2332	Serge	1553
3031	serge	1554	static void
		1555	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1556	{
3746	Serge	1557	struct sg_page_iter sg_iter;
		1558	int ret;
2332	Serge	1559
3031	serge	1560	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1561
3031	serge	1562	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1563	if (ret) {
		1564	/* In the event of a disaster, abandon all caches and
		1565	* hope for the best.
		1566	*/
		1567	WARN_ON(ret != -EIO);
4104	Serge	1568	i915_gem_clflush_object(obj, true);
3031	serge	1569	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1570	}
2332	Serge	1571
3031	serge	1572	if (obj->madv == I915_MADV_DONTNEED)
		1573	obj->dirty = 0;
2332	Serge	1574
3746	Serge	1575	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1576	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1577
3290	Serge	1578	page_cache_release(page);
3243	Serge	1579	}
		1580	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1581
4104	Serge	1582	obj->dirty = 0;
3243	Serge	1583
		1584	sg_free_table(obj->pages);
		1585	kfree(obj->pages);
3031	serge	1586	}
2332	Serge	1587
3480	Serge	1588	int
3031	serge	1589	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1590	{
		1591	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1592
3243	Serge	1593	if (obj->pages == NULL)
3031	serge	1594	return 0;
2332	Serge	1595
3031	serge	1596	if (obj->pages_pin_count)
		1597	return -EBUSY;
		1598
4104	Serge	1599	BUG_ON(i915_gem_obj_bound_any(obj));
		1600
3243	Serge	1601	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1602	* array, hence protect them from being reaped by removing them from gtt
		1603	* lists early. */
4104	Serge	1604	list_del(&obj->global_list);
3243	Serge	1605
3031	serge	1606	ops->put_pages(obj);
3243	Serge	1607	obj->pages = NULL;
3031	serge	1608
		1609	if (i915_gem_object_is_purgeable(obj))
		1610	i915_gem_object_truncate(obj);
		1611
		1612	return 0;
		1613	}
		1614
		1615
		1616
		1617
		1618
		1619
		1620
		1621
2332	Serge	1622	static int
3031	serge	1623	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1624	{
3260	Serge	1625	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1626	int page_count, i;
4104	Serge	1627	struct sg_table *st;
3243	Serge	1628	struct scatterlist *sg;
3746	Serge	1629	struct sg_page_iter sg_iter;
3243	Serge	1630	struct page *page;
3746	Serge	1631	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1632	gfp_t gfp;
2332	Serge	1633
3243	Serge	1634	/* Assert that the object is not currently in any GPU domain. As it
		1635	* wasn't in the GTT, there shouldn't be any way it could have been in
		1636	* a GPU cache
2332	Serge	1637	*/
3243	Serge	1638	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1639	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1640
		1641	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1642	if (st == NULL)
		1643	return -ENOMEM;
		1644
2332	Serge	1645	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1646	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1647	kfree(st);
3746	Serge	1648	FAIL();
2332	Serge	1649	return -ENOMEM;
3243	Serge	1650	}
2332	Serge	1651
3243	Serge	1652	/* Get the list of pages out of our struct file. They'll be pinned
		1653	* at this point until we release them.
		1654	*
		1655	* Fail silently without starting the shrinker
		1656	*/
3746	Serge	1657	sg = st->sgl;
		1658	st->nents = 0;
		1659	for (i = 0; i < page_count; i++) {
4104	Serge	1660	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1661	if (IS_ERR(page)) {
		1662	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1663	goto err_pages;
		1664
3260	Serge	1665	}
3746	Serge	1666
		1667	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1668	if (i)
		1669	sg = sg_next(sg);
		1670	st->nents++;
3243	Serge	1671	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1672	} else {
		1673	sg->length += PAGE_SIZE;
		1674	}
		1675	last_pfn = page_to_pfn(page);
3243	Serge	1676	}
3031	serge	1677
3746	Serge	1678	sg_mark_end(sg);
3243	Serge	1679	obj->pages = st;
3031	serge	1680
2332	Serge	1681	return 0;
		1682
		1683	err_pages:
3746	Serge	1684	sg_mark_end(sg);
		1685	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1686	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1687	sg_free_table(st);
		1688	kfree(st);
3746	Serge	1689	FAIL();
3243	Serge	1690	return PTR_ERR(page);
2332	Serge	1691	}
		1692
3031	serge	1693	/* Ensure that the associated pages are gathered from the backing storage
		1694	* and pinned into our object. i915_gem_object_get_pages() may be called
		1695	* multiple times before they are released by a single call to
		1696	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1697	* either as a result of memory pressure (reaping pages under the shrinker)
		1698	* or as the object is itself released.
		1699	*/
		1700	int
		1701	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1702	{
3031	serge	1703	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1704	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1705	int ret;
2332	Serge	1706
3243	Serge	1707	if (obj->pages)
3031	serge	1708	return 0;
2332	Serge	1709
4392	Serge	1710	if (obj->madv != I915_MADV_WILLNEED) {
		1711	DRM_ERROR("Attempting to obtain a purgeable object\n");
		1712	return -EINVAL;
		1713	}
		1714
3031	serge	1715	BUG_ON(obj->pages_pin_count);
2332	Serge	1716
3031	serge	1717	ret = ops->get_pages(obj);
		1718	if (ret)
		1719	return ret;
2344	Serge	1720
4104	Serge	1721	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1722	return 0;
2332	Serge	1723	}
		1724
		1725	void
		1726	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1727	struct intel_ring_buffer *ring)
2332	Serge	1728	{
		1729	struct drm_device *dev = obj->base.dev;
		1730	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1731	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1732
		1733	BUG_ON(ring == NULL);
4104	Serge	1734	if (obj->ring != ring && obj->last_write_seqno) {
		1735	/* Keep the seqno relative to the current ring */
		1736	obj->last_write_seqno = seqno;
		1737	}
2332	Serge	1738	obj->ring = ring;
		1739
		1740	/* Add a reference if we're newly entering the active list. */
		1741	if (!obj->active) {
2344	Serge	1742	drm_gem_object_reference(&obj->base);
2332	Serge	1743	obj->active = 1;
		1744	}
		1745
		1746	list_move_tail(&obj->ring_list, &ring->active_list);
		1747
3031	serge	1748	obj->last_read_seqno = seqno;
		1749
2332	Serge	1750	if (obj->fenced_gpu_access) {
3031	serge	1751	obj->last_fenced_seqno = seqno;
		1752
		1753	/* Bump MRU to take account of the delayed flush */
		1754	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1755	struct drm_i915_fence_reg *reg;
		1756
		1757	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1758	list_move_tail(®->lru_list,
		1759	&dev_priv->mm.fence_list);
		1760	}
2332	Serge	1761	}
		1762	}
		1763
2344	Serge	1764	static void
3031	serge	1765	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1766	{
4104	Serge	1767	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1768	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
		1769	struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2332	Serge	1770
3031	serge	1771	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1772	BUG_ON(!obj->active);
2332	Serge	1773
4104	Serge	1774	list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2344	Serge	1775
3031	serge	1776	list_del_init(&obj->ring_list);
2352	Serge	1777	obj->ring = NULL;
2344	Serge	1778
3031	serge	1779	obj->last_read_seqno = 0;
		1780	obj->last_write_seqno = 0;
		1781	obj->base.write_domain = 0;
		1782
		1783	obj->last_fenced_seqno = 0;
2352	Serge	1784	obj->fenced_gpu_access = false;
2344	Serge	1785
2352	Serge	1786	obj->active = 0;
		1787	drm_gem_object_unreference(&obj->base);
		1788
		1789	WARN_ON(i915_verify_lists(dev));
		1790	}
		1791
3243	Serge	1792	static int
3480	Serge	1793	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1794	{
3243	Serge	1795	struct drm_i915_private *dev_priv = dev->dev_private;
		1796	struct intel_ring_buffer *ring;
		1797	int ret, i, j;
2344	Serge	1798
3480	Serge	1799	/* Carefully retire all requests without writing to the rings */
3243	Serge	1800	for_each_ring(ring, dev_priv, i) {
3480	Serge	1801	ret = intel_ring_idle(ring);
3243	Serge	1802	if (ret)
		1803	return ret;
3480	Serge	1804	}
		1805	i915_gem_retire_requests(dev);
3243	Serge	1806
3480	Serge	1807	/* Finally reset hw state */
3243	Serge	1808	for_each_ring(ring, dev_priv, i) {
3480	Serge	1809	intel_ring_init_seqno(ring, seqno);
		1810
3243	Serge	1811	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1812	ring->sync_seqno[j] = 0;
		1813	}
		1814
		1815	return 0;
2344	Serge	1816	}
		1817
3480	Serge	1818	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1819	{
		1820	struct drm_i915_private *dev_priv = dev->dev_private;
		1821	int ret;
		1822
		1823	if (seqno == 0)
		1824	return -EINVAL;
		1825
		1826	/* HWS page needs to be set less than what we
		1827	* will inject to ring
		1828	*/
		1829	ret = i915_gem_init_seqno(dev, seqno - 1);
		1830	if (ret)
		1831	return ret;
		1832
		1833	/* Carefully set the last_seqno value so that wrap
		1834	* detection still works
		1835	*/
		1836	dev_priv->next_seqno = seqno;
		1837	dev_priv->last_seqno = seqno - 1;
		1838	if (dev_priv->last_seqno == 0)
		1839	dev_priv->last_seqno--;
		1840
		1841	return 0;
		1842	}
		1843
3243	Serge	1844	int
		1845	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1846	{
3243	Serge	1847	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1848
3243	Serge	1849	/* reserve 0 for non-seqno */
		1850	if (dev_priv->next_seqno == 0) {
3480	Serge	1851	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1852	if (ret)
		1853	return ret;
		1854
		1855	dev_priv->next_seqno = 1;
		1856	}
		1857
3480	Serge	1858	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1859	return 0;
2332	Serge	1860	}
		1861
4104	Serge	1862	int __i915_add_request(struct intel_ring_buffer *ring,
2352	Serge	1863	struct drm_file *file,
4104	Serge	1864	struct drm_i915_gem_object *obj,
3031	serge	1865	u32 *out_seqno)
2352	Serge	1866	{
		1867	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1868	struct drm_i915_gem_request *request;
4104	Serge	1869	u32 request_ring_position, request_start;
2352	Serge	1870	int was_empty;
		1871	int ret;
2332	Serge	1872
4104	Serge	1873	request_start = intel_ring_get_tail(ring);
3031	serge	1874	/*
		1875	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1876	* after having emitted the batchbuffer command. Hence we need to fix
		1877	* things up similar to emitting the lazy request. The difference here
		1878	* is that the flush _must_ happen before the next request, no matter
		1879	* what.
		1880	*/
4104	Serge	1881	ret = intel_ring_flush_all_caches(ring);
		1882	if (ret)
		1883	return ret;
2332	Serge	1884
3031	serge	1885	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1886	if (request == NULL)
		1887	return -ENOMEM;
		1888
		1889
		1890	/* Record the position of the start of the request so that
		1891	* should we detect the updated seqno part-way through the
4104	Serge	1892	* GPU processing the request, we never over-estimate the
3031	serge	1893	* position of the head.
		1894	*/
4104	Serge	1895	request_ring_position = intel_ring_get_tail(ring);
3031	serge	1896
3243	Serge	1897	ret = ring->add_request(ring);
3031	serge	1898	if (ret) {
		1899	kfree(request);
4104	Serge	1900	return ret;
3031	serge	1901	}
2332	Serge	1902
3243	Serge	1903	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1904	request->ring = ring;
4104	Serge	1905	request->head = request_start;
3031	serge	1906	request->tail = request_ring_position;
4104	Serge	1907	request->ctx = ring->last_context;
		1908	request->batch_obj = obj;
		1909
		1910	/* Whilst this request exists, batch_obj will be on the
		1911	* active_list, and so will hold the active reference. Only when this
		1912	* request is retired will the the batch_obj be moved onto the
		1913	* inactive_list and lose its active reference. Hence we do not need
		1914	* to explicitly hold another reference here.
		1915	*/
		1916
		1917	if (request->ctx)
		1918	i915_gem_context_reference(request->ctx);
		1919
3031	serge	1920	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1921	was_empty = list_empty(&ring->request_list);
		1922	list_add_tail(&request->list, &ring->request_list);
3031	serge	1923	request->file_priv = NULL;
2332	Serge	1924
3263	Serge	1925	if (file) {
		1926	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1927
3263	Serge	1928	spin_lock(&file_priv->mm.lock);
		1929	request->file_priv = file_priv;
		1930	list_add_tail(&request->client_list,
		1931	&file_priv->mm.request_list);
		1932	spin_unlock(&file_priv->mm.lock);
		1933	}
		1934
		1935	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1936	ring->outstanding_lazy_request = 0;
2332	Serge	1937
4104	Serge	1938	if (!dev_priv->ums.mm_suspended) {
		1939	// i915_queue_hangcheck(ring->dev);
		1940
		1941	if (was_empty) {
2360	Serge	1942	queue_delayed_work(dev_priv->wq,
3482	Serge	1943	&dev_priv->mm.retire_work,
		1944	round_jiffies_up_relative(HZ));
4104	Serge	1945	intel_mark_busy(dev_priv->dev);
		1946	}
		1947	}
3031	serge	1948
		1949	if (out_seqno)
3243	Serge	1950	*out_seqno = request->seqno;
2352	Serge	1951	return 0;
		1952	}
2332	Serge	1953
3263	Serge	1954	static inline void
		1955	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1956	{
		1957	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1958
3263	Serge	1959	if (!file_priv)
		1960	return;
2332	Serge	1961
3263	Serge	1962	spin_lock(&file_priv->mm.lock);
		1963	if (request->file_priv) {
		1964	list_del(&request->client_list);
		1965	request->file_priv = NULL;
		1966	}
		1967	spin_unlock(&file_priv->mm.lock);
		1968	}
2332	Serge	1969
4104	Serge	1970	static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
		1971	struct i915_address_space *vm)
		1972	{
		1973	if (acthd >= i915_gem_obj_offset(obj, vm) &&
		1974	acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
		1975	return true;
		1976
		1977	return false;
		1978	}
		1979
		1980	static bool i915_head_inside_request(const u32 acthd_unmasked,
		1981	const u32 request_start,
		1982	const u32 request_end)
		1983	{
		1984	const u32 acthd = acthd_unmasked & HEAD_ADDR;
		1985
		1986	if (request_start < request_end) {
		1987	if (acthd >= request_start && acthd < request_end)
		1988	return true;
		1989	} else if (request_start > request_end) {
		1990	if (acthd >= request_start \|\| acthd < request_end)
		1991	return true;
		1992	}
		1993
		1994	return false;
		1995	}
		1996
		1997	static struct i915_address_space *
		1998	request_to_vm(struct drm_i915_gem_request *request)
		1999	{
		2000	struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
		2001	struct i915_address_space *vm;
		2002
		2003	vm = &dev_priv->gtt.base;
		2004
		2005	return vm;
		2006	}
		2007
		2008	static bool i915_request_guilty(struct drm_i915_gem_request *request,
		2009	const u32 acthd, bool *inside)
		2010	{
		2011	/* There is a possibility that unmasked head address
		2012	* pointing inside the ring, matches the batch_obj address range.
		2013	* However this is extremely unlikely.
		2014	*/
		2015	if (request->batch_obj) {
		2016	if (i915_head_inside_object(acthd, request->batch_obj,
		2017	request_to_vm(request))) {
		2018	*inside = true;
		2019	return true;
		2020	}
		2021	}
		2022
		2023	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		2024	*inside = false;
		2025	return true;
		2026	}
		2027
		2028	return false;
		2029	}
		2030
		2031	static void i915_set_reset_status(struct intel_ring_buffer *ring,
		2032	struct drm_i915_gem_request *request,
		2033	u32 acthd)
		2034	{
		2035	struct i915_ctx_hang_stats *hs = NULL;
		2036	bool inside, guilty;
		2037	unsigned long offset = 0;
		2038
		2039	/* Innocent until proven guilty */
		2040	guilty = false;
		2041
		2042	if (request->batch_obj)
		2043	offset = i915_gem_obj_offset(request->batch_obj,
		2044	request_to_vm(request));
		2045
		2046	if (ring->hangcheck.action != HANGCHECK_WAIT &&
		2047	i915_request_guilty(request, acthd, &inside)) {
		2048	DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
		2049	ring->name,
		2050	inside ? "inside" : "flushing",
		2051	offset,
		2052	request->ctx ? request->ctx->id : 0,
		2053	acthd);
		2054
		2055	guilty = true;
		2056	}
		2057
		2058	/* If contexts are disabled or this is the default context, use
		2059	* file_priv->reset_state
		2060	*/
		2061	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		2062	hs = &request->ctx->hang_stats;
		2063	else if (request->file_priv)
		2064	hs = &request->file_priv->hang_stats;
		2065
		2066	if (hs) {
		2067	if (guilty)
		2068	hs->batch_active++;
		2069	else
		2070	hs->batch_pending++;
		2071	}
		2072	}
		2073
		2074	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2075	{
		2076	list_del(&request->list);
		2077	i915_gem_request_remove_from_client(request);
		2078
		2079	if (request->ctx)
		2080	i915_gem_context_unreference(request->ctx);
		2081
		2082	kfree(request);
		2083	}
		2084
3031	serge	2085	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		2086	struct intel_ring_buffer *ring)
		2087	{
4104	Serge	2088	u32 completed_seqno;
		2089	u32 acthd;
		2090
		2091	acthd = intel_ring_get_active_head(ring);
		2092	completed_seqno = ring->get_seqno(ring, false);
		2093
3031	serge	2094	while (!list_empty(&ring->request_list)) {
		2095	struct drm_i915_gem_request *request;
2332	Serge	2096
3031	serge	2097	request = list_first_entry(&ring->request_list,
		2098	struct drm_i915_gem_request,
		2099	list);
2332	Serge	2100
4104	Serge	2101	if (request->seqno > completed_seqno)
		2102	i915_set_reset_status(ring, request, acthd);
		2103
		2104	i915_gem_free_request(request);
3031	serge	2105	}
2332	Serge	2106
3031	serge	2107	while (!list_empty(&ring->active_list)) {
		2108	struct drm_i915_gem_object *obj;
2332	Serge	2109
3031	serge	2110	obj = list_first_entry(&ring->active_list,
		2111	struct drm_i915_gem_object,
		2112	ring_list);
2332	Serge	2113
3031	serge	2114	i915_gem_object_move_to_inactive(obj);
		2115	}
		2116	}
2332	Serge	2117
3746	Serge	2118	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2119	{
		2120	struct drm_i915_private *dev_priv = dev->dev_private;
		2121	int i;
2332	Serge	2122
3031	serge	2123	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2124	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2125
		2126	/*
		2127	* Commit delayed tiling changes if we have an object still
		2128	* attached to the fence, otherwise just clear the fence.
		2129	*/
		2130	if (reg->obj) {
		2131	i915_gem_object_update_fence(reg->obj, reg,
		2132	reg->obj->tiling_mode);
		2133	} else {
		2134	i915_gem_write_fence(dev, i, NULL);
		2135	}
3031	serge	2136	}
		2137	}
2360	Serge	2138
3031	serge	2139	void i915_gem_reset(struct drm_device *dev)
		2140	{
		2141	struct drm_i915_private *dev_priv = dev->dev_private;
		2142	struct intel_ring_buffer *ring;
		2143	int i;
2360	Serge	2144
3031	serge	2145	for_each_ring(ring, dev_priv, i)
		2146	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	2147
3746	Serge	2148	i915_gem_restore_fences(dev);
3031	serge	2149	}
2360	Serge	2150
2352	Serge	2151	/**
		2152	* This function clears the request list as sequence numbers are passed.
		2153	*/
3031	serge	2154	void
2352	Serge	2155	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2156	{
		2157	uint32_t seqno;
2332	Serge	2158
2352	Serge	2159	if (list_empty(&ring->request_list))
		2160	return;
2332	Serge	2161
2352	Serge	2162	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2163
3031	serge	2164	seqno = ring->get_seqno(ring, true);
2332	Serge	2165
2352	Serge	2166	while (!list_empty(&ring->request_list)) {
		2167	struct drm_i915_gem_request *request;
2332	Serge	2168
2352	Serge	2169	request = list_first_entry(&ring->request_list,
		2170	struct drm_i915_gem_request,
		2171	list);
2332	Serge	2172
2352	Serge	2173	if (!i915_seqno_passed(seqno, request->seqno))
		2174	break;
2332	Serge	2175
2352	Serge	2176	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2177	/* We know the GPU must have read the request to have
		2178	* sent us the seqno + interrupt, so use the position
		2179	* of tail of the request to update the last known position
		2180	* of the GPU head.
		2181	*/
		2182	ring->last_retired_head = request->tail;
2332	Serge	2183
4104	Serge	2184	i915_gem_free_request(request);
2352	Serge	2185	}
2332	Serge	2186
2352	Serge	2187	/* Move any buffers on the active list that are no longer referenced
		2188	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2189	*/
		2190	while (!list_empty(&ring->active_list)) {
		2191	struct drm_i915_gem_object *obj;
2332	Serge	2192
2352	Serge	2193	obj = list_first_entry(&ring->active_list,
		2194	struct drm_i915_gem_object,
		2195	ring_list);
2332	Serge	2196
3031	serge	2197	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2198	break;
2332	Serge	2199
2352	Serge	2200	i915_gem_object_move_to_inactive(obj);
		2201	}
2332	Serge	2202
2352	Serge	2203	if (unlikely(ring->trace_irq_seqno &&
		2204	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2205	ring->irq_put(ring);
		2206	ring->trace_irq_seqno = 0;
		2207	}
2332	Serge	2208
2352	Serge	2209	WARN_ON(i915_verify_lists(ring->dev));
		2210	}
2332	Serge	2211
2352	Serge	2212	void
		2213	i915_gem_retire_requests(struct drm_device *dev)
		2214	{
		2215	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2216	struct intel_ring_buffer *ring;
2352	Serge	2217	int i;
2332	Serge	2218
3031	serge	2219	for_each_ring(ring, dev_priv, i)
		2220	i915_gem_retire_requests_ring(ring);
2352	Serge	2221	}
		2222
2360	Serge	2223	static void
		2224	i915_gem_retire_work_handler(struct work_struct *work)
		2225	{
		2226	drm_i915_private_t *dev_priv;
		2227	struct drm_device *dev;
3031	serge	2228	struct intel_ring_buffer *ring;
2360	Serge	2229	bool idle;
		2230	int i;
2352	Serge	2231
2360	Serge	2232	dev_priv = container_of(work, drm_i915_private_t,
		2233	mm.retire_work.work);
		2234	dev = dev_priv->dev;
2352	Serge	2235
2360	Serge	2236	/* Come back later if the device is busy... */
		2237	if (!mutex_trylock(&dev->struct_mutex)) {
3482	Serge	2238	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2239	round_jiffies_up_relative(HZ));
3243	Serge	2240	return;
2360	Serge	2241	}
2352	Serge	2242
2360	Serge	2243	i915_gem_retire_requests(dev);
2352	Serge	2244
2360	Serge	2245	/* Send a periodic flush down the ring so we don't hold onto GEM
		2246	* objects indefinitely.
		2247	*/
		2248	idle = true;
3031	serge	2249	for_each_ring(ring, dev_priv, i) {
		2250	if (ring->gpu_caches_dirty)
4104	Serge	2251	i915_add_request(ring, NULL);
2352	Serge	2252
2360	Serge	2253	idle &= list_empty(&ring->request_list);
		2254	}
2352	Serge	2255
4104	Serge	2256	if (!dev_priv->ums.mm_suspended && !idle)
3482	Serge	2257	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2258	round_jiffies_up_relative(HZ));
3031	serge	2259	if (idle)
		2260	intel_mark_idle(dev);
2360	Serge	2261
		2262	mutex_unlock(&dev->struct_mutex);
		2263	}
		2264
2344	Serge	2265	/**
3031	serge	2266	* Ensures that an object will eventually get non-busy by flushing any required
		2267	* write domains, emitting any outstanding lazy request and retiring and
		2268	* completed requests.
2352	Serge	2269	*/
3031	serge	2270	static int
		2271	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2272	{
3031	serge	2273	int ret;
2352	Serge	2274
3031	serge	2275	if (obj->active) {
		2276	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2277	if (ret)
		2278	return ret;
2352	Serge	2279
3031	serge	2280	i915_gem_retire_requests_ring(obj->ring);
		2281	}
2352	Serge	2282
3031	serge	2283	return 0;
		2284	}
2352	Serge	2285
3243	Serge	2286	/**
		2287	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2288	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2289	*
		2290	* Returns 0 if successful, else an error is returned with the remaining time in
		2291	* the timeout parameter.
		2292	* -ETIME: object is still busy after timeout
		2293	* -ERESTARTSYS: signal interrupted the wait
		2294	* -ENONENT: object doesn't exist
		2295	* Also possible, but rare:
		2296	* -EAGAIN: GPU wedged
		2297	* -ENOMEM: damn
		2298	* -ENODEV: Internal IRQ fail
		2299	* -E?: The add request failed
		2300	*
		2301	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2302	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2303	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2304	* without holding struct_mutex the object may become re-busied before this
		2305	* function completes. A similar but shorter * race condition exists in the busy
		2306	* ioctl
		2307	*/
4246	Serge	2308	int
		2309	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2310	{
		2311	drm_i915_private_t *dev_priv = dev->dev_private;
		2312	struct drm_i915_gem_wait *args = data;
		2313	struct drm_i915_gem_object *obj;
		2314	struct intel_ring_buffer *ring = NULL;
		2315	struct timespec timeout_stack, *timeout = NULL;
		2316	unsigned reset_counter;
		2317	u32 seqno = 0;
		2318	int ret = 0;
2352	Serge	2319
4246	Serge	2320	if (args->timeout_ns >= 0) {
		2321	timeout_stack = ns_to_timespec(args->timeout_ns);
		2322	timeout = &timeout_stack;
		2323	}
2352	Serge	2324
4246	Serge	2325	ret = i915_mutex_lock_interruptible(dev);
		2326	if (ret)
		2327	return ret;
2352	Serge	2328
4246	Serge	2329	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2330	if (&obj->base == NULL) {
		2331	mutex_unlock(&dev->struct_mutex);
		2332	return -ENOENT;
		2333	}
2352	Serge	2334
4246	Serge	2335	/* Need to make sure the object gets inactive eventually. */
		2336	ret = i915_gem_object_flush_active(obj);
		2337	if (ret)
		2338	goto out;
2352	Serge	2339
4246	Serge	2340	if (obj->active) {
		2341	seqno = obj->last_read_seqno;
		2342	ring = obj->ring;
		2343	}
2352	Serge	2344
4246	Serge	2345	if (seqno == 0)
		2346	goto out;
2352	Serge	2347
4246	Serge	2348	/* Do this after OLR check to make sure we make forward progress polling
		2349	* on this IOCTL with a 0 timeout (like busy ioctl)
		2350	*/
		2351	if (!args->timeout_ns) {
		2352	ret = -ETIME;
		2353	goto out;
		2354	}
2352	Serge	2355
4246	Serge	2356	drm_gem_object_unreference(&obj->base);
		2357	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		2358	mutex_unlock(&dev->struct_mutex);
2352	Serge	2359
4246	Serge	2360	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
		2361	if (timeout)
		2362	args->timeout_ns = timespec_to_ns(timeout);
		2363	return ret;
3243	Serge	2364
4246	Serge	2365	out:
		2366	drm_gem_object_unreference(&obj->base);
		2367	mutex_unlock(&dev->struct_mutex);
		2368	return ret;
		2369	}
3243	Serge	2370
2352	Serge	2371	/**
3031	serge	2372	* i915_gem_object_sync - sync an object to a ring.
		2373	*
		2374	* @obj: object which may be in use on another ring.
		2375	* @to: ring we wish to use the object on. May be NULL.
		2376	*
		2377	* This code is meant to abstract object synchronization with the GPU.
		2378	* Calling with NULL implies synchronizing the object with the CPU
		2379	* rather than a particular GPU ring.
		2380	*
		2381	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2382	*/
		2383	int
3031	serge	2384	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2385	struct intel_ring_buffer *to)
2344	Serge	2386	{
3031	serge	2387	struct intel_ring_buffer *from = obj->ring;
		2388	u32 seqno;
		2389	int ret, idx;
2332	Serge	2390
3031	serge	2391	if (from == NULL \|\| to == from)
		2392	return 0;
2332	Serge	2393
3031	serge	2394	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2395	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2396
3031	serge	2397	idx = intel_ring_sync_index(from, to);
		2398
		2399	seqno = obj->last_read_seqno;
		2400	if (seqno <= from->sync_seqno[idx])
		2401	return 0;
		2402
		2403	ret = i915_gem_check_olr(obj->ring, seqno);
		2404	if (ret)
		2405	return ret;
		2406
		2407	ret = to->sync_to(to, from, seqno);
		2408	if (!ret)
3243	Serge	2409	/* We use last_read_seqno because sync_to()
		2410	* might have just caused seqno wrap under
		2411	* the radar.
		2412	*/
		2413	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2414
		2415	return ret;
2344	Serge	2416	}
2332	Serge	2417
2344	Serge	2418	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2419	{
		2420	u32 old_write_domain, old_read_domains;
2332	Serge	2421
2344	Serge	2422	/* Force a pagefault for domain tracking on next user access */
		2423	// i915_gem_release_mmap(obj);
2332	Serge	2424
2344	Serge	2425	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2426	return;
2332	Serge	2427
3480	Serge	2428	/* Wait for any direct GTT access to complete */
		2429	mb();
		2430
2344	Serge	2431	old_read_domains = obj->base.read_domains;
		2432	old_write_domain = obj->base.write_domain;
2351	Serge	2433
2344	Serge	2434	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2435	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2436
2351	Serge	2437	trace_i915_gem_object_change_domain(obj,
		2438	old_read_domains,
		2439	old_write_domain);
2344	Serge	2440	}
2332	Serge	2441
4104	Serge	2442	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2443	{
4104	Serge	2444	struct drm_i915_gem_object *obj = vma->obj;
3031	serge	2445	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2446	int ret;
2332	Serge	2447
3263	Serge	2448	if(obj == get_fb_obj())
		2449	return 0;
		2450
4104	Serge	2451	if (list_empty(&vma->vma_link))
2344	Serge	2452	return 0;
2332	Serge	2453
4104	Serge	2454	if (!drm_mm_node_allocated(&vma->node))
		2455	goto destroy;
		2456
3031	serge	2457	if (obj->pin_count)
		2458	return -EBUSY;
2332	Serge	2459
3243	Serge	2460	BUG_ON(obj->pages == NULL);
3031	serge	2461
2344	Serge	2462	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2463	if (ret)
2344	Serge	2464	return ret;
		2465	/* Continue on if we fail due to EIO, the GPU is hung so we
		2466	* should be safe and we need to cleanup or else we might
		2467	* cause memory corruption through use-after-free.
		2468	*/
2332	Serge	2469
2344	Serge	2470	i915_gem_object_finish_gtt(obj);
2332	Serge	2471
2344	Serge	2472	/* release the fence reg _after_ flushing */
		2473	ret = i915_gem_object_put_fence(obj);
3031	serge	2474	if (ret)
2344	Serge	2475	return ret;
2332	Serge	2476
4104	Serge	2477	trace_i915_vma_unbind(vma);
2332	Serge	2478
3031	serge	2479	if (obj->has_global_gtt_mapping)
3243	Serge	2480	i915_gem_gtt_unbind_object(obj);
3031	serge	2481	if (obj->has_aliasing_ppgtt_mapping) {
		2482	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2483	obj->has_aliasing_ppgtt_mapping = 0;
		2484	}
		2485	i915_gem_gtt_finish_object(obj);
4104	Serge	2486	i915_gem_object_unpin_pages(obj);
2332	Serge	2487
4104	Serge	2488	list_del(&vma->mm_list);
2344	Serge	2489	/* Avoid an unnecessary call to unbind on rebind. */
4104	Serge	2490	if (i915_is_ggtt(vma->vm))
2344	Serge	2491	obj->map_and_fenceable = true;
2332	Serge	2492
4104	Serge	2493	drm_mm_remove_node(&vma->node);
2332	Serge	2494
4104	Serge	2495	destroy:
		2496	i915_gem_vma_destroy(vma);
		2497
		2498	/* Since the unbound list is global, only move to that list if
		2499	* no more VMAs exist.
		2500	* NB: Until we have real VMAs there will only ever be one */
		2501	WARN_ON(!list_empty(&obj->vma_list));
		2502	if (list_empty(&obj->vma_list))
		2503	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2504
2344	Serge	2505	return 0;
		2506	}
2332	Serge	2507
4104	Serge	2508	/**
		2509	* Unbinds an object from the global GTT aperture.
		2510	*/
		2511	int
		2512	i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
		2513	{
		2514	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2515	struct i915_address_space *ggtt = &dev_priv->gtt.base;
		2516
		2517	if (!i915_gem_obj_ggtt_bound(obj))
		2518	return 0;
		2519
		2520	if (obj->pin_count)
		2521	return -EBUSY;
		2522
		2523	BUG_ON(obj->pages == NULL);
		2524
		2525	return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
		2526	}
		2527
3031	serge	2528	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2529	{
		2530	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2531	struct intel_ring_buffer *ring;
2344	Serge	2532	int ret, i;
2332	Serge	2533
2344	Serge	2534	/* Flush everything onto the inactive list. */
3031	serge	2535	for_each_ring(ring, dev_priv, i) {
		2536	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2537	if (ret)
		2538	return ret;
3031	serge	2539
3243	Serge	2540	ret = intel_ring_idle(ring);
3031	serge	2541	if (ret)
		2542	return ret;
2344	Serge	2543	}
2332	Serge	2544
2344	Serge	2545	return 0;
		2546	}
2332	Serge	2547
3480	Serge	2548	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2549	struct drm_i915_gem_object *obj)
		2550	{
		2551	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2552	int fence_reg;
		2553	int fence_pitch_shift;
2332	Serge	2554
3480	Serge	2555	if (INTEL_INFO(dev)->gen >= 6) {
		2556	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2557	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2558	} else {
		2559	fence_reg = FENCE_REG_965_0;
		2560	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2561	}
2332	Serge	2562
4104	Serge	2563	fence_reg += reg * 8;
		2564
		2565	/* To w/a incoherency with non-atomic 64-bit register updates,
		2566	* we split the 64-bit update into two 32-bit writes. In order
		2567	* for a partial fence not to be evaluated between writes, we
		2568	* precede the update with write to turn off the fence register,
		2569	* and only enable the fence as the last step.
		2570	*
		2571	* For extra levels of paranoia, we make sure each step lands
		2572	* before applying the next step.
		2573	*/
		2574	I915_WRITE(fence_reg, 0);
		2575	POSTING_READ(fence_reg);
		2576
3031	serge	2577	if (obj) {
4104	Serge	2578	u32 size = i915_gem_obj_ggtt_size(obj);
		2579	uint64_t val;
2332	Serge	2580
4104	Serge	2581	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2582	0xfffff000) << 32;
4104	Serge	2583	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2584	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2585	if (obj->tiling_mode == I915_TILING_Y)
		2586	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2587	val \|= I965_FENCE_REG_VALID;
2332	Serge	2588
4104	Serge	2589	I915_WRITE(fence_reg + 4, val >> 32);
		2590	POSTING_READ(fence_reg + 4);
		2591
		2592	I915_WRITE(fence_reg + 0, val);
3480	Serge	2593	POSTING_READ(fence_reg);
4104	Serge	2594	} else {
		2595	I915_WRITE(fence_reg + 4, 0);
		2596	POSTING_READ(fence_reg + 4);
		2597	}
3031	serge	2598	}
2332	Serge	2599
3031	serge	2600	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2601	struct drm_i915_gem_object *obj)
		2602	{
		2603	drm_i915_private_t *dev_priv = dev->dev_private;
		2604	u32 val;
2332	Serge	2605
3031	serge	2606	if (obj) {
4104	Serge	2607	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2608	int pitch_val;
		2609	int tile_width;
2332	Serge	2610
4104	Serge	2611	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2612	(size & -size) != size \|\|
4104	Serge	2613	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2614	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2615	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2616
3031	serge	2617	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2618	tile_width = 128;
		2619	else
		2620	tile_width = 512;
2332	Serge	2621
3031	serge	2622	/* Note: pitch better be a power of two tile widths */
		2623	pitch_val = obj->stride / tile_width;
		2624	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2625
4104	Serge	2626	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2627	if (obj->tiling_mode == I915_TILING_Y)
		2628	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2629	val \|= I915_FENCE_SIZE_BITS(size);
		2630	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2631	val \|= I830_FENCE_REG_VALID;
		2632	} else
		2633	val = 0;
2332	Serge	2634
3031	serge	2635	if (reg < 8)
		2636	reg = FENCE_REG_830_0 + reg * 4;
		2637	else
		2638	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2639
3031	serge	2640	I915_WRITE(reg, val);
		2641	POSTING_READ(reg);
		2642	}
2332	Serge	2643
3031	serge	2644	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2645	struct drm_i915_gem_object *obj)
		2646	{
		2647	drm_i915_private_t *dev_priv = dev->dev_private;
		2648	uint32_t val;
2344	Serge	2649
3031	serge	2650	if (obj) {
4104	Serge	2651	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2652	uint32_t pitch_val;
2344	Serge	2653
4104	Serge	2654	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2655	(size & -size) != size \|\|
4104	Serge	2656	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2657	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2658	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2659
3031	serge	2660	pitch_val = obj->stride / 128;
		2661	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2662
4104	Serge	2663	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2664	if (obj->tiling_mode == I915_TILING_Y)
		2665	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2666	val \|= I830_FENCE_SIZE_BITS(size);
		2667	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2668	val \|= I830_FENCE_REG_VALID;
		2669	} else
		2670	val = 0;
		2671
		2672	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2673	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2674	}
		2675
3480	Serge	2676	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2677	{
		2678	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2679	}
		2680
3031	serge	2681	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2682	struct drm_i915_gem_object *obj)
2332	Serge	2683	{
3480	Serge	2684	struct drm_i915_private *dev_priv = dev->dev_private;
		2685
		2686	/* Ensure that all CPU reads are completed before installing a fence
		2687	* and all writes before removing the fence.
		2688	*/
		2689	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2690	mb();
		2691
4104	Serge	2692	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2693	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2694	obj->stride, obj->tiling_mode);
		2695
3031	serge	2696	switch (INTEL_INFO(dev)->gen) {
		2697	case 7:
3480	Serge	2698	case 6:
3031	serge	2699	case 5:
		2700	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2701	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2702	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2703	default: BUG();
3031	serge	2704	}
3480	Serge	2705
		2706	/* And similarly be paranoid that no direct access to this region
		2707	* is reordered to before the fence is installed.
		2708	*/
		2709	if (i915_gem_object_needs_mb(obj))
		2710	mb();
2344	Serge	2711	}
		2712
3031	serge	2713	static inline int fence_number(struct drm_i915_private *dev_priv,
		2714	struct drm_i915_fence_reg *fence)
2344	Serge	2715	{
3031	serge	2716	return fence - dev_priv->fence_regs;
		2717	}
2332	Serge	2718
3031	serge	2719	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2720	struct drm_i915_fence_reg *fence,
		2721	bool enable)
		2722	{
4104	Serge	2723	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2724	int reg = fence_number(dev_priv, fence);
2332	Serge	2725
4104	Serge	2726	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2727
		2728	if (enable) {
4104	Serge	2729	obj->fence_reg = reg;
3031	serge	2730	fence->obj = obj;
		2731	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2732	} else {
		2733	obj->fence_reg = I915_FENCE_REG_NONE;
		2734	fence->obj = NULL;
		2735	list_del_init(&fence->lru_list);
2344	Serge	2736	}
4104	Serge	2737	obj->fence_dirty = false;
3031	serge	2738	}
2344	Serge	2739
3031	serge	2740	static int
3480	Serge	2741	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2742	{
		2743	if (obj->last_fenced_seqno) {
		2744	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2745	if (ret)
		2746	return ret;
2344	Serge	2747
		2748	obj->last_fenced_seqno = 0;
		2749	}
		2750
3031	serge	2751	obj->fenced_gpu_access = false;
2332	Serge	2752	return 0;
		2753	}
		2754
		2755	int
2344	Serge	2756	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2757	{
3031	serge	2758	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2759	struct drm_i915_fence_reg *fence;
2332	Serge	2760	int ret;
		2761
3480	Serge	2762	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2763	if (ret)
		2764	return ret;
		2765
3031	serge	2766	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2767	return 0;
2332	Serge	2768
3746	Serge	2769	fence = &dev_priv->fence_regs[obj->fence_reg];
		2770
3031	serge	2771	i915_gem_object_fence_lost(obj);
3746	Serge	2772	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2773
2332	Serge	2774	return 0;
		2775	}
		2776
3031	serge	2777	static struct drm_i915_fence_reg *
		2778	i915_find_fence_reg(struct drm_device *dev)
		2779	{
		2780	struct drm_i915_private *dev_priv = dev->dev_private;
		2781	struct drm_i915_fence_reg reg, avail;
		2782	int i;
2332	Serge	2783
3031	serge	2784	/* First try to find a free reg */
		2785	avail = NULL;
		2786	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2787	reg = &dev_priv->fence_regs[i];
		2788	if (!reg->obj)
		2789	return reg;
2332	Serge	2790
3031	serge	2791	if (!reg->pin_count)
		2792	avail = reg;
		2793	}
2332	Serge	2794
3031	serge	2795	if (avail == NULL)
		2796	return NULL;
2332	Serge	2797
3031	serge	2798	/* None available, try to steal one or wait for a user to finish */
		2799	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2800	if (reg->pin_count)
		2801	continue;
2332	Serge	2802
3031	serge	2803	return reg;
		2804	}
2332	Serge	2805
3031	serge	2806	return NULL;
		2807	}
2332	Serge	2808
3031	serge	2809	/**
		2810	* i915_gem_object_get_fence - set up fencing for an object
		2811	* @obj: object to map through a fence reg
		2812	*
		2813	* When mapping objects through the GTT, userspace wants to be able to write
		2814	* to them without having to worry about swizzling if the object is tiled.
		2815	* This function walks the fence regs looking for a free one for @obj,
		2816	* stealing one if it can't find any.
		2817	*
		2818	* It then sets up the reg based on the object's properties: address, pitch
		2819	* and tiling format.
		2820	*
		2821	* For an untiled surface, this removes any existing fence.
		2822	*/
		2823	int
		2824	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2825	{
		2826	struct drm_device *dev = obj->base.dev;
		2827	struct drm_i915_private *dev_priv = dev->dev_private;
		2828	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2829	struct drm_i915_fence_reg *reg;
		2830	int ret;
2332	Serge	2831
3031	serge	2832	/* Have we updated the tiling parameters upon the object and so
		2833	* will need to serialise the write to the associated fence register?
		2834	*/
		2835	if (obj->fence_dirty) {
3480	Serge	2836	ret = i915_gem_object_wait_fence(obj);
3031	serge	2837	if (ret)
		2838	return ret;
		2839	}
2332	Serge	2840
3031	serge	2841	/* Just update our place in the LRU if our fence is getting reused. */
		2842	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2843	reg = &dev_priv->fence_regs[obj->fence_reg];
		2844	if (!obj->fence_dirty) {
		2845	list_move_tail(®->lru_list,
		2846	&dev_priv->mm.fence_list);
		2847	return 0;
		2848	}
		2849	} else if (enable) {
		2850	reg = i915_find_fence_reg(dev);
		2851	if (reg == NULL)
		2852	return -EDEADLK;
2332	Serge	2853
3031	serge	2854	if (reg->obj) {
		2855	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2856
3480	Serge	2857	ret = i915_gem_object_wait_fence(old);
3031	serge	2858	if (ret)
		2859	return ret;
2332	Serge	2860
3031	serge	2861	i915_gem_object_fence_lost(old);
		2862	}
		2863	} else
		2864	return 0;
2332	Serge	2865
3031	serge	2866	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2867
3031	serge	2868	return 0;
		2869	}
2332	Serge	2870
3031	serge	2871	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2872	struct drm_mm_node *gtt_space,
		2873	unsigned long cache_level)
		2874	{
		2875	struct drm_mm_node *other;
2332	Serge	2876
3031	serge	2877	/* On non-LLC machines we have to be careful when putting differing
		2878	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2879	* crossing memory domains and dying.
3031	serge	2880	*/
		2881	if (HAS_LLC(dev))
		2882	return true;
2332	Serge	2883
4104	Serge	2884	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2885	return true;
2332	Serge	2886
3031	serge	2887	if (list_empty(>t_space->node_list))
		2888	return true;
2332	Serge	2889
3031	serge	2890	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2891	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2892	return false;
2344	Serge	2893
3031	serge	2894	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2895	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2896	return false;
2344	Serge	2897
3031	serge	2898	return true;
		2899	}
2344	Serge	2900
3031	serge	2901	static void i915_gem_verify_gtt(struct drm_device *dev)
		2902	{
		2903	#if WATCH_GTT
		2904	struct drm_i915_private *dev_priv = dev->dev_private;
		2905	struct drm_i915_gem_object *obj;
		2906	int err = 0;
2344	Serge	2907
4104	Serge	2908	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031	serge	2909	if (obj->gtt_space == NULL) {
		2910	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2911	err++;
		2912	continue;
		2913	}
2344	Serge	2914
3031	serge	2915	if (obj->cache_level != obj->gtt_space->color) {
		2916	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
4104	Serge	2917	i915_gem_obj_ggtt_offset(obj),
		2918	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2919	obj->cache_level,
		2920	obj->gtt_space->color);
		2921	err++;
		2922	continue;
		2923	}
2344	Serge	2924
3031	serge	2925	if (!i915_gem_valid_gtt_space(dev,
		2926	obj->gtt_space,
		2927	obj->cache_level)) {
		2928	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
4104	Serge	2929	i915_gem_obj_ggtt_offset(obj),
		2930	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2931	obj->cache_level);
		2932	err++;
		2933	continue;
		2934	}
		2935	}
2344	Serge	2936
3031	serge	2937	WARN_ON(err);
		2938	#endif
2326	Serge	2939	}
		2940
2332	Serge	2941	/**
		2942	* Finds free space in the GTT aperture and binds the object there.
		2943	*/
		2944	static int
4104	Serge	2945	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2946	struct i915_address_space *vm,
2332	Serge	2947	unsigned alignment,
3031	serge	2948	bool map_and_fenceable,
		2949	bool nonblocking)
2332	Serge	2950	{
		2951	struct drm_device *dev = obj->base.dev;
		2952	drm_i915_private_t *dev_priv = dev->dev_private;
		2953	u32 size, fence_size, fence_alignment, unfenced_alignment;
4104	Serge	2954	size_t gtt_max =
		2955	map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
		2956	struct i915_vma *vma;
2332	Serge	2957	int ret;
2326	Serge	2958
2332	Serge	2959	fence_size = i915_gem_get_gtt_size(dev,
		2960	obj->base.size,
		2961	obj->tiling_mode);
		2962	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2963	obj->base.size,
3480	Serge	2964	obj->tiling_mode, true);
2332	Serge	2965	unfenced_alignment =
3480	Serge	2966	i915_gem_get_gtt_alignment(dev,
2332	Serge	2967	obj->base.size,
3480	Serge	2968	obj->tiling_mode, false);
2332	Serge	2969
		2970	if (alignment == 0)
		2971	alignment = map_and_fenceable ? fence_alignment :
		2972	unfenced_alignment;
		2973	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2974	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2975	return -EINVAL;
		2976	}
		2977
		2978	size = map_and_fenceable ? fence_size : obj->base.size;
		2979
		2980	/* If the object is bigger than the entire aperture, reject it early
		2981	* before evicting everything in a vain attempt to find space.
		2982	*/
4104	Serge	2983	if (obj->base.size > gtt_max) {
		2984	DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
		2985	obj->base.size,
		2986	map_and_fenceable ? "mappable" : "total",
		2987	gtt_max);
2332	Serge	2988	return -E2BIG;
		2989	}
		2990
3031	serge	2991	ret = i915_gem_object_get_pages(obj);
		2992	if (ret)
		2993	return ret;
		2994
3243	Serge	2995	i915_gem_object_pin_pages(obj);
		2996
4104	Serge	2997	BUG_ON(!i915_is_ggtt(vm));
		2998
		2999	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
		3000	if (IS_ERR(vma)) {
		3001	ret = PTR_ERR(vma);
		3002	goto err_unpin;
3243	Serge	3003	}
		3004
4104	Serge	3005	/* For now we only ever use 1 vma per object */
		3006	WARN_ON(!list_is_singular(&obj->vma_list));
		3007
		3008	search_free:
		3009	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3010	size, alignment,
		3011	obj->cache_level, 0, gtt_max,
		3012	DRM_MM_SEARCH_DEFAULT);
3243	Serge	3013	if (ret) {
2332	Serge	3014
4104	Serge	3015	goto err_free_vma;
2332	Serge	3016	}
4104	Serge	3017	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
		3018	obj->cache_level))) {
		3019	ret = -EINVAL;
		3020	goto err_remove_node;
3031	serge	3021	}
2332	Serge	3022
3031	serge	3023	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	3024	if (ret)
		3025	goto err_remove_node;
2332	Serge	3026
4104	Serge	3027	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3028	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3029
4104	Serge	3030	if (i915_is_ggtt(vm)) {
		3031	bool mappable, fenceable;
2332	Serge	3032
4104	Serge	3033	fenceable = (vma->node.size == fence_size &&
		3034	(vma->node.start & (fence_alignment - 1)) == 0);
2332	Serge	3035
4104	Serge	3036	mappable = (vma->node.start + obj->base.size <=
		3037	dev_priv->gtt.mappable_end);
2332	Serge	3038
		3039	obj->map_and_fenceable = mappable && fenceable;
4104	Serge	3040	}
2332	Serge	3041
4104	Serge	3042	WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
		3043
		3044	trace_i915_vma_bind(vma, map_and_fenceable);
3031	serge	3045	i915_gem_verify_gtt(dev);
2332	Serge	3046	return 0;
4104	Serge	3047
		3048	err_remove_node:
		3049	drm_mm_remove_node(&vma->node);
		3050	err_free_vma:
		3051	i915_gem_vma_destroy(vma);
		3052	err_unpin:
		3053	i915_gem_object_unpin_pages(obj);
		3054	return ret;
2332	Serge	3055	}
		3056
4104	Serge	3057	bool
		3058	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3059	bool force)
2332	Serge	3060	{
		3061	/* If we don't have a page list set up, then we're not pinned
		3062	* to GPU, and we can ignore the cache flush because it'll happen
		3063	* again at bind time.
		3064	*/
3243	Serge	3065	if (obj->pages == NULL)
4104	Serge	3066	return false;
2332	Serge	3067
3480	Serge	3068	/*
		3069	* Stolen memory is always coherent with the GPU as it is explicitly
		3070	* marked as wc by the system, or the system is cache-coherent.
		3071	*/
		3072	if (obj->stolen)
4104	Serge	3073	return false;
3480	Serge	3074
2332	Serge	3075	/* If the GPU is snooping the contents of the CPU cache,
		3076	* we do not need to manually clear the CPU cache lines. However,
		3077	* the caches are only snooped when the render cache is
		3078	* flushed/invalidated. As we always have to emit invalidations
		3079	* and flushes when moving into and out of the RENDER domain, correct
		3080	* snooping behaviour occurs naturally as the result of our domain
		3081	* tracking.
		3082	*/
4104	Serge	3083	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3084	return false;
2332	Serge	3085
4293	Serge	3086	trace_i915_gem_object_clflush(obj);
		3087	drm_clflush_sg(obj->pages);
2344	Serge	3088
4104	Serge	3089	return true;
2332	Serge	3090	}
		3091
2344	Serge	3092	/** Flushes the GTT write domain for the object if it's dirty. */
		3093	static void
		3094	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3095	{
		3096	uint32_t old_write_domain;
2332	Serge	3097
2344	Serge	3098	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3099	return;
2332	Serge	3100
2344	Serge	3101	/* No actual flushing is required for the GTT write domain. Writes
		3102	* to it immediately go to main memory as far as we know, so there's
		3103	* no chipset flush. It also doesn't land in render cache.
		3104	*
		3105	* However, we do have to enforce the order so that all writes through
		3106	* the GTT land before any writes to the device, such as updates to
		3107	* the GATT itself.
		3108	*/
		3109	wmb();
2332	Serge	3110
2344	Serge	3111	old_write_domain = obj->base.write_domain;
		3112	obj->base.write_domain = 0;
2332	Serge	3113
2351	Serge	3114	trace_i915_gem_object_change_domain(obj,
		3115	obj->base.read_domains,
		3116	old_write_domain);
2344	Serge	3117	}
2332	Serge	3118
		3119	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3120	static void
4104	Serge	3121	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3122	bool force)
2332	Serge	3123	{
		3124	uint32_t old_write_domain;
		3125
		3126	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3127	return;
		3128
4104	Serge	3129	if (i915_gem_clflush_object(obj, force))
3243	Serge	3130	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3131
2332	Serge	3132	old_write_domain = obj->base.write_domain;
		3133	obj->base.write_domain = 0;
		3134
2351	Serge	3135	trace_i915_gem_object_change_domain(obj,
		3136	obj->base.read_domains,
		3137	old_write_domain);
2332	Serge	3138	}
		3139
		3140	/**
		3141	* Moves a single object to the GTT read, and possibly write domain.
		3142	*
		3143	* This function returns when the move is complete, including waiting on
		3144	* flushes to occur.
		3145	*/
		3146	int
		3147	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3148	{
3031	serge	3149	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	3150	uint32_t old_write_domain, old_read_domains;
		3151	int ret;
		3152
		3153	/* Not valid to be called on unbound objects. */
4104	Serge	3154	if (!i915_gem_obj_bound_any(obj))
2332	Serge	3155	return -EINVAL;
		3156
		3157	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3158	return 0;
		3159
3031	serge	3160	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3161	if (ret)
		3162	return ret;
		3163
4104	Serge	3164	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3165
3480	Serge	3166	/* Serialise direct access to this object with the barriers for
		3167	* coherent writes from the GPU, by effectively invalidating the
		3168	* GTT domain upon first access.
		3169	*/
		3170	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3171	mb();
		3172
2332	Serge	3173	old_write_domain = obj->base.write_domain;
		3174	old_read_domains = obj->base.read_domains;
		3175
		3176	/* It should now be out of any other write domains, and we can update
		3177	* the domain values for our changes.
		3178	*/
		3179	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3180	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3181	if (write) {
		3182	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3183	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3184	obj->dirty = 1;
		3185	}
		3186
2351	Serge	3187	trace_i915_gem_object_change_domain(obj,
		3188	old_read_domains,
		3189	old_write_domain);
		3190
3031	serge	3191	/* And bump the LRU for this access */
4104	Serge	3192	if (i915_gem_object_is_inactive(obj)) {
		3193	struct i915_vma *vma = i915_gem_obj_to_vma(obj,
		3194	&dev_priv->gtt.base);
		3195	if (vma)
		3196	list_move_tail(&vma->mm_list,
		3197	&dev_priv->gtt.base.inactive_list);
3031	serge	3198
4104	Serge	3199	}
		3200
2332	Serge	3201	return 0;
		3202	}
		3203
2335	Serge	3204	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3205	enum i915_cache_level cache_level)
		3206	{
3031	serge	3207	struct drm_device *dev = obj->base.dev;
		3208	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3209	struct i915_vma *vma;
2335	Serge	3210	int ret;
2332	Serge	3211
2335	Serge	3212	if (obj->cache_level == cache_level)
		3213	return 0;
2332	Serge	3214
2335	Serge	3215	if (obj->pin_count) {
		3216	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3217	return -EBUSY;
		3218	}
2332	Serge	3219
4104	Serge	3220	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3221	if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
		3222	ret = i915_vma_unbind(vma);
3031	serge	3223	if (ret)
		3224	return ret;
4104	Serge	3225
		3226	break;
		3227	}
3031	serge	3228	}
		3229
4104	Serge	3230	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3231	ret = i915_gem_object_finish_gpu(obj);
		3232	if (ret)
		3233	return ret;
2332	Serge	3234
2335	Serge	3235	i915_gem_object_finish_gtt(obj);
2332	Serge	3236
2335	Serge	3237	/* Before SandyBridge, you could not use tiling or fence
		3238	* registers with snooped memory, so relinquish any fences
		3239	* currently pointing to our region in the aperture.
		3240	*/
3031	serge	3241	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3242	ret = i915_gem_object_put_fence(obj);
		3243	if (ret)
		3244	return ret;
		3245	}
2332	Serge	3246
3031	serge	3247	if (obj->has_global_gtt_mapping)
		3248	i915_gem_gtt_bind_object(obj, cache_level);
		3249	if (obj->has_aliasing_ppgtt_mapping)
		3250	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3251	obj, cache_level);
2335	Serge	3252	}
2332	Serge	3253
4104	Serge	3254	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3255	vma->node.color = cache_level;
		3256	obj->cache_level = cache_level;
		3257
		3258	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3259	u32 old_read_domains, old_write_domain;
2332	Serge	3260
2335	Serge	3261	/* If we're coming from LLC cached, then we haven't
		3262	* actually been tracking whether the data is in the
		3263	* CPU cache or not, since we only allow one bit set
		3264	* in obj->write_domain and have been skipping the clflushes.
		3265	* Just set it to the CPU cache for now.
		3266	*/
		3267	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3268
2335	Serge	3269	old_read_domains = obj->base.read_domains;
		3270	old_write_domain = obj->base.write_domain;
2332	Serge	3271
2335	Serge	3272	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3273	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3274
2351	Serge	3275	trace_i915_gem_object_change_domain(obj,
		3276	old_read_domains,
		3277	old_write_domain);
2344	Serge	3278	}
2332	Serge	3279
3031	serge	3280	i915_gem_verify_gtt(dev);
2335	Serge	3281	return 0;
		3282	}
2332	Serge	3283
3260	Serge	3284	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3285	struct drm_file *file)
		3286	{
		3287	struct drm_i915_gem_caching *args = data;
		3288	struct drm_i915_gem_object *obj;
		3289	int ret;
		3290
		3291	ret = i915_mutex_lock_interruptible(dev);
		3292	if (ret)
		3293	return ret;
		3294
		3295	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3296	if (&obj->base == NULL) {
		3297	ret = -ENOENT;
		3298	goto unlock;
		3299	}
		3300
4104	Serge	3301	switch (obj->cache_level) {
		3302	case I915_CACHE_LLC:
		3303	case I915_CACHE_L3_LLC:
		3304	args->caching = I915_CACHING_CACHED;
		3305	break;
3260	Serge	3306
4104	Serge	3307	case I915_CACHE_WT:
		3308	args->caching = I915_CACHING_DISPLAY;
		3309	break;
		3310
		3311	default:
		3312	args->caching = I915_CACHING_NONE;
		3313	break;
		3314	}
		3315
3260	Serge	3316	drm_gem_object_unreference(&obj->base);
		3317	unlock:
		3318	mutex_unlock(&dev->struct_mutex);
		3319	return ret;
		3320	}
		3321
		3322	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3323	struct drm_file *file)
		3324	{
		3325	struct drm_i915_gem_caching *args = data;
		3326	struct drm_i915_gem_object *obj;
		3327	enum i915_cache_level level;
		3328	int ret;
		3329
		3330	switch (args->caching) {
		3331	case I915_CACHING_NONE:
		3332	level = I915_CACHE_NONE;
		3333	break;
		3334	case I915_CACHING_CACHED:
		3335	level = I915_CACHE_LLC;
		3336	break;
4104	Serge	3337	case I915_CACHING_DISPLAY:
		3338	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3339	break;
3260	Serge	3340	default:
		3341	return -EINVAL;
		3342	}
		3343
		3344	ret = i915_mutex_lock_interruptible(dev);
		3345	if (ret)
		3346	return ret;
		3347
		3348	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3349	if (&obj->base == NULL) {
		3350	ret = -ENOENT;
		3351	goto unlock;
		3352	}
		3353
		3354	ret = i915_gem_object_set_cache_level(obj, level);
		3355
		3356	drm_gem_object_unreference(&obj->base);
		3357	unlock:
		3358	mutex_unlock(&dev->struct_mutex);
		3359	return ret;
		3360	}
		3361
4104	Serge	3362	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3363	{
		3364	/* There are 3 sources that pin objects:
		3365	* 1. The display engine (scanouts, sprites, cursors);
		3366	* 2. Reservations for execbuffer;
		3367	* 3. The user.
		3368	*
		3369	* We can ignore reservations as we hold the struct_mutex and
		3370	* are only called outside of the reservation path. The user
		3371	* can only increment pin_count once, and so if after
		3372	* subtracting the potential reference by the user, any pin_count
		3373	* remains, it must be due to another use by the display engine.
		3374	*/
		3375	return obj->pin_count - !!obj->user_pin_count;
		3376	}
		3377
2335	Serge	3378	/*
		3379	* Prepare buffer for display plane (scanout, cursors, etc).
		3380	* Can be called from an uninterruptible phase (modesetting) and allows
		3381	* any flushes to be pipelined (for pageflips).
		3382	*/
		3383	int
		3384	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3385	u32 alignment,
		3386	struct intel_ring_buffer *pipelined)
		3387	{
		3388	u32 old_read_domains, old_write_domain;
		3389	int ret;
2332	Serge	3390
3031	serge	3391	if (pipelined != obj->ring) {
		3392	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3393	if (ret)
		3394	return ret;
		3395	}
2332	Serge	3396
4104	Serge	3397	/* Mark the pin_display early so that we account for the
		3398	* display coherency whilst setting up the cache domains.
		3399	*/
		3400	obj->pin_display = true;
		3401
2335	Serge	3402	/* The display engine is not coherent with the LLC cache on gen6. As
		3403	* a result, we make sure that the pinning that is about to occur is
		3404	* done with uncached PTEs. This is lowest common denominator for all
		3405	* chipsets.
		3406	*
		3407	* However for gen6+, we could do better by using the GFDT bit instead
		3408	* of uncaching, which would allow us to flush all the LLC-cached data
		3409	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3410	*/
4104	Serge	3411	ret = i915_gem_object_set_cache_level(obj,
		3412	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3413	if (ret)
4104	Serge	3414	goto err_unpin_display;
2332	Serge	3415
2335	Serge	3416	/* As the user may map the buffer once pinned in the display plane
		3417	* (e.g. libkms for the bootup splash), we have to ensure that we
		3418	* always use map_and_fenceable for all scanout buffers.
		3419	*/
4104	Serge	3420	ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
2335	Serge	3421	if (ret)
4104	Serge	3422	goto err_unpin_display;
2332	Serge	3423
4104	Serge	3424	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3425
2335	Serge	3426	old_write_domain = obj->base.write_domain;
		3427	old_read_domains = obj->base.read_domains;
2332	Serge	3428
2335	Serge	3429	/* It should now be out of any other write domains, and we can update
		3430	* the domain values for our changes.
		3431	*/
3031	serge	3432	obj->base.write_domain = 0;
2335	Serge	3433	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3434
2351	Serge	3435	trace_i915_gem_object_change_domain(obj,
		3436	old_read_domains,
		3437	old_write_domain);
2332	Serge	3438
2335	Serge	3439	return 0;
4104	Serge	3440
		3441	err_unpin_display:
		3442	obj->pin_display = is_pin_display(obj);
		3443	return ret;
2335	Serge	3444	}
2332	Serge	3445
4104	Serge	3446	void
		3447	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3448	{
		3449	i915_gem_object_unpin(obj);
		3450	obj->pin_display = is_pin_display(obj);
		3451	}
		3452
2344	Serge	3453	int
		3454	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3455	{
		3456	int ret;
2332	Serge	3457
2344	Serge	3458	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3459	return 0;
2332	Serge	3460
3031	serge	3461	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3462	if (ret)
		3463	return ret;
2332	Serge	3464
2344	Serge	3465	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3466	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3467	return 0;
2344	Serge	3468	}
2332	Serge	3469
2344	Serge	3470	/**
		3471	* Moves a single object to the CPU read, and possibly write domain.
		3472	*
		3473	* This function returns when the move is complete, including waiting on
		3474	* flushes to occur.
		3475	*/
3031	serge	3476	int
2344	Serge	3477	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3478	{
		3479	uint32_t old_write_domain, old_read_domains;
		3480	int ret;
2332	Serge	3481
2344	Serge	3482	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3483	return 0;
2332	Serge	3484
3031	serge	3485	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3486	if (ret)
		3487	return ret;
2332	Serge	3488
2344	Serge	3489	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3490
2344	Serge	3491	old_write_domain = obj->base.write_domain;
		3492	old_read_domains = obj->base.read_domains;
2332	Serge	3493
2344	Serge	3494	/* Flush the CPU cache if it's still invalid. */
		3495	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3496	i915_gem_clflush_object(obj, false);
2332	Serge	3497
2344	Serge	3498	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3499	}
2332	Serge	3500
2344	Serge	3501	/* It should now be out of any other write domains, and we can update
		3502	* the domain values for our changes.
		3503	*/
		3504	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3505
2344	Serge	3506	/* If we're writing through the CPU, then the GPU read domains will
		3507	* need to be invalidated at next use.
		3508	*/
		3509	if (write) {
		3510	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3511	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3512	}
2332	Serge	3513
2351	Serge	3514	trace_i915_gem_object_change_domain(obj,
		3515	old_read_domains,
		3516	old_write_domain);
2332	Serge	3517
2344	Serge	3518	return 0;
		3519	}
2332	Serge	3520
3031	serge	3521	/* Throttle our rendering by waiting until the ring has completed our requests
		3522	* emitted over 20 msec ago.
2344	Serge	3523	*
3031	serge	3524	* Note that if we were to use the current jiffies each time around the loop,
		3525	* we wouldn't escape the function with any frames outstanding if the time to
		3526	* render a frame was over 20ms.
		3527	*
		3528	* This should get us reasonable parallelism between CPU and GPU but also
		3529	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3530	*/
3031	serge	3531	static int
		3532	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3533	{
3031	serge	3534	struct drm_i915_private *dev_priv = dev->dev_private;
		3535	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3536	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3537	struct drm_i915_gem_request *request;
		3538	struct intel_ring_buffer *ring = NULL;
3480	Serge	3539	unsigned reset_counter;
3031	serge	3540	u32 seqno = 0;
		3541	int ret;
2332	Serge	3542
3480	Serge	3543	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3544	if (ret)
		3545	return ret;
2332	Serge	3546
3480	Serge	3547	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3548	if (ret)
		3549	return ret;
		3550
3031	serge	3551	spin_lock(&file_priv->mm.lock);
		3552	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3553	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3554	break;
2332	Serge	3555
3031	serge	3556	ring = request->ring;
		3557	seqno = request->seqno;
		3558	}
3480	Serge	3559	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3560	spin_unlock(&file_priv->mm.lock);
2332	Serge	3561
3031	serge	3562	if (seqno == 0)
		3563	return 0;
2332	Serge	3564
3480	Serge	3565	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3566	if (ret == 0)
		3567	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3568
3031	serge	3569	return ret;
2352	Serge	3570	}
2332	Serge	3571
		3572	int
		3573	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3574	struct i915_address_space *vm,
2332	Serge	3575	uint32_t alignment,
3031	serge	3576	bool map_and_fenceable,
		3577	bool nonblocking)
2332	Serge	3578	{
4104	Serge	3579	struct i915_vma *vma;
2332	Serge	3580	int ret;
		3581
3031	serge	3582	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3583	return -EBUSY;
2332	Serge	3584
4104	Serge	3585	WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
		3586
		3587	vma = i915_gem_obj_to_vma(obj, vm);
		3588
		3589	if (vma) {
		3590	if ((alignment &&
		3591	vma->node.start & (alignment - 1)) \|\|
2332	Serge	3592	(map_and_fenceable && !obj->map_and_fenceable)) {
		3593	WARN(obj->pin_count,
		3594	"bo is already pinned with incorrect alignment:"
4104	Serge	3595	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3596	" obj->map_and_fenceable=%d\n",
4104	Serge	3597	i915_gem_obj_offset(obj, vm), alignment,
2332	Serge	3598	map_and_fenceable,
		3599	obj->map_and_fenceable);
4104	Serge	3600	ret = i915_vma_unbind(vma);
2332	Serge	3601	if (ret)
		3602	return ret;
		3603	}
		3604	}
		3605
4104	Serge	3606	if (!i915_gem_obj_bound(obj, vm)) {
3243	Serge	3607	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3608
4104	Serge	3609	ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3031	serge	3610	map_and_fenceable,
		3611	nonblocking);
2332	Serge	3612	if (ret)
		3613	return ret;
3243	Serge	3614
		3615	if (!dev_priv->mm.aliasing_ppgtt)
		3616	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3617	}
		3618
3031	serge	3619	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3620	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3621
		3622	obj->pin_count++;
2332	Serge	3623	obj->pin_mappable \|= map_and_fenceable;
		3624
		3625	return 0;
		3626	}
		3627
2344	Serge	3628	void
		3629	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3630	{
		3631	BUG_ON(obj->pin_count == 0);
4104	Serge	3632	BUG_ON(!i915_gem_obj_bound_any(obj));
2332	Serge	3633
3031	serge	3634	if (--obj->pin_count == 0)
2344	Serge	3635	obj->pin_mappable = false;
		3636	}
2332	Serge	3637
3031	serge	3638	int
		3639	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3640	struct drm_file *file)
		3641	{
		3642	struct drm_i915_gem_pin *args = data;
		3643	struct drm_i915_gem_object *obj;
		3644	int ret;
2332	Serge	3645
3031	serge	3646	ret = i915_mutex_lock_interruptible(dev);
		3647	if (ret)
		3648	return ret;
2332	Serge	3649
3031	serge	3650	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3651	if (&obj->base == NULL) {
		3652	ret = -ENOENT;
		3653	goto unlock;
		3654	}
2332	Serge	3655
3031	serge	3656	if (obj->madv != I915_MADV_WILLNEED) {
		3657	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3658	ret = -EINVAL;
		3659	goto out;
		3660	}
2332	Serge	3661
3031	serge	3662	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3663	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3664	args->handle);
		3665	ret = -EINVAL;
		3666	goto out;
		3667	}
2332	Serge	3668
3243	Serge	3669	if (obj->user_pin_count == 0) {
4104	Serge	3670	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3031	serge	3671	if (ret)
		3672	goto out;
		3673	}
2332	Serge	3674
3243	Serge	3675	obj->user_pin_count++;
		3676	obj->pin_filp = file;
		3677
4104	Serge	3678	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3679	out:
		3680	drm_gem_object_unreference(&obj->base);
		3681	unlock:
		3682	mutex_unlock(&dev->struct_mutex);
		3683	return ret;
		3684	}
2332	Serge	3685
3031	serge	3686	int
		3687	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3688	struct drm_file *file)
		3689	{
		3690	struct drm_i915_gem_pin *args = data;
		3691	struct drm_i915_gem_object *obj;
		3692	int ret;
2332	Serge	3693
3031	serge	3694	ret = i915_mutex_lock_interruptible(dev);
		3695	if (ret)
		3696	return ret;
2332	Serge	3697
3031	serge	3698	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3699	if (&obj->base == NULL) {
		3700	ret = -ENOENT;
		3701	goto unlock;
		3702	}
2332	Serge	3703
3031	serge	3704	if (obj->pin_filp != file) {
		3705	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3706	args->handle);
		3707	ret = -EINVAL;
		3708	goto out;
		3709	}
		3710	obj->user_pin_count--;
		3711	if (obj->user_pin_count == 0) {
		3712	obj->pin_filp = NULL;
		3713	i915_gem_object_unpin(obj);
		3714	}
2332	Serge	3715
3031	serge	3716	out:
		3717	drm_gem_object_unreference(&obj->base);
		3718	unlock:
		3719	mutex_unlock(&dev->struct_mutex);
		3720	return ret;
		3721	}
2332	Serge	3722
3031	serge	3723	int
		3724	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3725	struct drm_file *file)
		3726	{
		3727	struct drm_i915_gem_busy *args = data;
		3728	struct drm_i915_gem_object *obj;
		3729	int ret;
2332	Serge	3730
3031	serge	3731	ret = i915_mutex_lock_interruptible(dev);
		3732	if (ret)
		3733	return ret;
2332	Serge	3734
4104	Serge	3735	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3736	if (&obj->base == NULL) {
		3737	ret = -ENOENT;
		3738	goto unlock;
		3739	}
2332	Serge	3740
3031	serge	3741	/* Count all active objects as busy, even if they are currently not used
		3742	* by the gpu. Users of this interface expect objects to eventually
		3743	* become non-busy without any further actions, therefore emit any
		3744	* necessary flushes here.
		3745	*/
		3746	ret = i915_gem_object_flush_active(obj);
2332	Serge	3747
3031	serge	3748	args->busy = obj->active;
		3749	if (obj->ring) {
		3750	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3751	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3752	}
2332	Serge	3753
3031	serge	3754	drm_gem_object_unreference(&obj->base);
		3755	unlock:
		3756	mutex_unlock(&dev->struct_mutex);
		3757	return ret;
		3758	}
2332	Serge	3759
3031	serge	3760	int
		3761	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3762	struct drm_file *file_priv)
		3763	{
		3764	return i915_gem_ring_throttle(dev, file_priv);
		3765	}
2332	Serge	3766
3263	Serge	3767	#if 0
		3768
3031	serge	3769	int
		3770	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3771	struct drm_file *file_priv)
		3772	{
		3773	struct drm_i915_gem_madvise *args = data;
		3774	struct drm_i915_gem_object *obj;
		3775	int ret;
2332	Serge	3776
3031	serge	3777	switch (args->madv) {
		3778	case I915_MADV_DONTNEED:
		3779	case I915_MADV_WILLNEED:
		3780	break;
		3781	default:
		3782	return -EINVAL;
		3783	}
2332	Serge	3784
3031	serge	3785	ret = i915_mutex_lock_interruptible(dev);
		3786	if (ret)
		3787	return ret;
2332	Serge	3788
3031	serge	3789	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3790	if (&obj->base == NULL) {
		3791	ret = -ENOENT;
		3792	goto unlock;
		3793	}
2332	Serge	3794
3031	serge	3795	if (obj->pin_count) {
		3796	ret = -EINVAL;
		3797	goto out;
		3798	}
2332	Serge	3799
3031	serge	3800	if (obj->madv != __I915_MADV_PURGED)
		3801	obj->madv = args->madv;
2332	Serge	3802
3031	serge	3803	/* if the object is no longer attached, discard its backing storage */
		3804	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3805	i915_gem_object_truncate(obj);
2332	Serge	3806
3031	serge	3807	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3808
3031	serge	3809	out:
		3810	drm_gem_object_unreference(&obj->base);
		3811	unlock:
		3812	mutex_unlock(&dev->struct_mutex);
		3813	return ret;
		3814	}
		3815	#endif
2332	Serge	3816
3031	serge	3817	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3818	const struct drm_i915_gem_object_ops *ops)
		3819	{
4104	Serge	3820	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3821	INIT_LIST_HEAD(&obj->ring_list);
		3822	INIT_LIST_HEAD(&obj->exec_list);
4104	Serge	3823	INIT_LIST_HEAD(&obj->obj_exec_link);
		3824	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3825
3031	serge	3826	obj->ops = ops;
		3827
		3828	obj->fence_reg = I915_FENCE_REG_NONE;
		3829	obj->madv = I915_MADV_WILLNEED;
		3830	/* Avoid an unnecessary call to unbind on the first bind. */
		3831	obj->map_and_fenceable = true;
		3832
		3833	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3834	}
		3835
		3836	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3837	.get_pages = i915_gem_object_get_pages_gtt,
		3838	.put_pages = i915_gem_object_put_pages_gtt,
		3839	};
		3840
2332	Serge	3841	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3842	size_t size)
		3843	{
		3844	struct drm_i915_gem_object *obj;
3031	serge	3845	struct address_space *mapping;
3480	Serge	3846	gfp_t mask;
2340	Serge	3847
3746	Serge	3848	obj = i915_gem_object_alloc(dev);
2332	Serge	3849	if (obj == NULL)
		3850	return NULL;
		3851
		3852	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3853	i915_gem_object_free(obj);
2332	Serge	3854	return NULL;
		3855	}
		3856
		3857
3031	serge	3858	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3859
		3860	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3861	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3862
3031	serge	3863	if (HAS_LLC(dev)) {
		3864	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3865	* cache) for about a 10% performance improvement
		3866	* compared to uncached. Graphics requests other than
		3867	* display scanout are coherent with the CPU in
		3868	* accessing this cache. This means in this mode we
		3869	* don't need to clflush on the CPU side, and on the
		3870	* GPU side we only need to flush internal caches to
		3871	* get data visible to the CPU.
		3872	*
		3873	* However, we maintain the display planes as UC, and so
		3874	* need to rebind when first used as such.
		3875	*/
		3876	obj->cache_level = I915_CACHE_LLC;
		3877	} else
		3878	obj->cache_level = I915_CACHE_NONE;
		3879
		3880	return obj;
		3881	}
		3882
2344	Serge	3883	int i915_gem_init_object(struct drm_gem_object *obj)
		3884	{
		3885	BUG();
2332	Serge	3886
2344	Serge	3887	return 0;
		3888	}
2332	Serge	3889
3031	serge	3890	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3891	{
3031	serge	3892	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3893	struct drm_device *dev = obj->base.dev;
		3894	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3895	struct i915_vma vma, next;
2332	Serge	3896
3031	serge	3897	trace_i915_gem_object_destroy(obj);
		3898
		3899
		3900	obj->pin_count = 0;
4104	Serge	3901	/* NB: 0 or 1 elements */
		3902	WARN_ON(!list_empty(&obj->vma_list) &&
		3903	!list_is_singular(&obj->vma_list));
		3904	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3905	int ret = i915_vma_unbind(vma);
		3906	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	3907	bool was_interruptible;
		3908
		3909	was_interruptible = dev_priv->mm.interruptible;
		3910	dev_priv->mm.interruptible = false;
		3911
4104	Serge	3912	WARN_ON(i915_vma_unbind(vma));
3031	serge	3913
		3914	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3915	}
4104	Serge	3916	}
2332	Serge	3917
4104	Serge	3918	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		3919	* before progressing. */
		3920	if (obj->stolen)
		3921	i915_gem_object_unpin_pages(obj);
		3922
		3923	if (WARN_ON(obj->pages_pin_count))
3031	serge	3924	obj->pages_pin_count = 0;
		3925	i915_gem_object_put_pages(obj);
		3926	// i915_gem_object_free_mmap_offset(obj);
4104	Serge	3927	i915_gem_object_release_stolen(obj);
2332	Serge	3928
3243	Serge	3929	BUG_ON(obj->pages);
2332	Serge	3930
3031	serge	3931
3290	Serge	3932	if(obj->base.filp != NULL)
		3933	{
3298	Serge	3934	// printf("filp %p\n", obj->base.filp);
3290	Serge	3935	shmem_file_delete(obj->base.filp);
		3936	}
		3937
2344	Serge	3938	drm_gem_object_release(&obj->base);
		3939	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3940
2344	Serge	3941	kfree(obj->bit_17);
4104	Serge	3942	i915_gem_object_free(obj);
2344	Serge	3943	}
2332	Serge	3944
4104	Serge	3945	struct i915_vma i915_gem_vma_create(struct drm_i915_gem_object obj,
		3946	struct i915_address_space *vm)
		3947	{
		3948	struct i915_vma vma = kzalloc(sizeof(vma), GFP_KERNEL);
		3949	if (vma == NULL)
		3950	return ERR_PTR(-ENOMEM);
		3951
		3952	INIT_LIST_HEAD(&vma->vma_link);
		3953	INIT_LIST_HEAD(&vma->mm_list);
		3954	INIT_LIST_HEAD(&vma->exec_list);
		3955	vma->vm = vm;
		3956	vma->obj = obj;
		3957
		3958	/* Keep GGTT vmas first to make debug easier */
		3959	if (i915_is_ggtt(vm))
		3960	list_add(&vma->vma_link, &obj->vma_list);
		3961	else
		3962	list_add_tail(&vma->vma_link, &obj->vma_list);
		3963
		3964	return vma;
		3965	}
		3966
		3967	void i915_gem_vma_destroy(struct i915_vma *vma)
		3968	{
		3969	WARN_ON(vma->node.allocated);
		3970	list_del(&vma->vma_link);
		3971	kfree(vma);
		3972	}
		3973
3031	serge	3974	#if 0
		3975	int
		3976	i915_gem_idle(struct drm_device *dev)
2344	Serge	3977	{
3031	serge	3978	drm_i915_private_t *dev_priv = dev->dev_private;
		3979	int ret;
2332	Serge	3980
4104	Serge	3981	if (dev_priv->ums.mm_suspended) {
3031	serge	3982	mutex_unlock(&dev->struct_mutex);
		3983	return 0;
		3984	}
2332	Serge	3985
3031	serge	3986	ret = i915_gpu_idle(dev);
		3987	if (ret) {
		3988	mutex_unlock(&dev->struct_mutex);
		3989	return ret;
		3990	}
		3991	i915_gem_retire_requests(dev);
		3992
3480	Serge	3993	/* Under UMS, be paranoid and evict. */
		3994	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		3995	i915_gem_evict_everything(dev);
		3996
		3997	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	3998
		3999	i915_kernel_lost_context(dev);
		4000	i915_gem_cleanup_ringbuffer(dev);
		4001
		4002	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	4003	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	4004
		4005	return 0;
2344	Serge	4006	}
3031	serge	4007	#endif
2332	Serge	4008
3031	serge	4009	void i915_gem_l3_remap(struct drm_device *dev)
		4010	{
		4011	drm_i915_private_t *dev_priv = dev->dev_private;
		4012	u32 misccpctl;
		4013	int i;
2332	Serge	4014
3480	Serge	4015	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	4016	return;
2332	Serge	4017
3243	Serge	4018	if (!dev_priv->l3_parity.remap_info)
3031	serge	4019	return;
2332	Serge	4020
3031	serge	4021	misccpctl = I915_READ(GEN7_MISCCPCTL);
		4022	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		4023	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	4024
3031	serge	4025	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		4026	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	4027	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	4028	DRM_DEBUG("0x%x was already programmed to %x\n",
		4029	GEN7_L3LOG_BASE + i, remap);
3243	Serge	4030	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	4031	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	4032	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	4033	}
2332	Serge	4034
3031	serge	4035	/* Make sure all the writes land before disabling dop clock gating */
		4036	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	4037
3031	serge	4038	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		4039	}
2332	Serge	4040
3031	serge	4041	void i915_gem_init_swizzling(struct drm_device *dev)
		4042	{
		4043	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	4044
3031	serge	4045	if (INTEL_INFO(dev)->gen < 5 \|\|
		4046	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4047	return;
2332	Serge	4048
3031	serge	4049	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4050	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4051
3031	serge	4052	if (IS_GEN5(dev))
		4053	return;
2344	Serge	4054
3031	serge	4055	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4056	if (IS_GEN6(dev))
		4057	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4058	else if (IS_GEN7(dev))
		4059	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	4060	else
3480	Serge	4061	BUG();
3031	serge	4062	}
		4063
		4064	static bool
		4065	intel_enable_blt(struct drm_device *dev)
		4066	{
		4067	if (!HAS_BLT(dev))
		4068	return false;
		4069
		4070	/* The blitter was dysfunctional on early prototypes */
		4071	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4072	DRM_INFO("BLT not supported on this pre-production hardware;"
		4073	" graphics performance will be degraded.\n");
		4074	return false;
		4075	}
		4076
		4077	return true;
		4078	}
		4079
3480	Serge	4080	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	4081	{
3480	Serge	4082	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4083	int ret;
2351	Serge	4084
2332	Serge	4085	ret = intel_init_render_ring_buffer(dev);
		4086	if (ret)
		4087	return ret;
		4088
		4089	if (HAS_BSD(dev)) {
		4090	ret = intel_init_bsd_ring_buffer(dev);
		4091	if (ret)
		4092	goto cleanup_render_ring;
		4093	}
		4094
3031	serge	4095	if (intel_enable_blt(dev)) {
2332	Serge	4096	ret = intel_init_blt_ring_buffer(dev);
		4097	if (ret)
		4098	goto cleanup_bsd_ring;
		4099	}
		4100
4104	Serge	4101	if (HAS_VEBOX(dev)) {
		4102	ret = intel_init_vebox_ring_buffer(dev);
		4103	if (ret)
		4104	goto cleanup_blt_ring;
		4105	}
		4106
		4107
3480	Serge	4108	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4109	if (ret)
4104	Serge	4110	goto cleanup_vebox_ring;
2351	Serge	4111
2332	Serge	4112	return 0;
		4113
4104	Serge	4114	cleanup_vebox_ring:
		4115	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4116	cleanup_blt_ring:
		4117	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4118	cleanup_bsd_ring:
		4119	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4120	cleanup_render_ring:
		4121	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4122
2332	Serge	4123	return ret;
		4124	}
		4125
3480	Serge	4126	int
		4127	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4128	{
3480	Serge	4129	drm_i915_private_t *dev_priv = dev->dev_private;
		4130	int ret;
3031	serge	4131
3480	Serge	4132	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4133	return -EIO;
3031	serge	4134
4104	Serge	4135	if (dev_priv->ellc_size)
		4136	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4137
3746	Serge	4138	if (HAS_PCH_NOP(dev)) {
		4139	u32 temp = I915_READ(GEN7_MSG_CTL);
		4140	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4141	I915_WRITE(GEN7_MSG_CTL, temp);
		4142	}
		4143
3480	Serge	4144	i915_gem_l3_remap(dev);
		4145
		4146	i915_gem_init_swizzling(dev);
		4147
		4148	ret = i915_gem_init_rings(dev);
		4149	if (ret)
		4150	return ret;
		4151
		4152	/*
		4153	* XXX: There was some w/a described somewhere suggesting loading
		4154	* contexts before PPGTT.
		4155	*/
		4156	i915_gem_context_init(dev);
3746	Serge	4157	if (dev_priv->mm.aliasing_ppgtt) {
		4158	ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		4159	if (ret) {
		4160	i915_gem_cleanup_aliasing_ppgtt(dev);
		4161	DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		4162	}
		4163	}
3480	Serge	4164
		4165	return 0;
3031	serge	4166	}
		4167
		4168	int i915_gem_init(struct drm_device *dev)
		4169	{
		4170	struct drm_i915_private *dev_priv = dev->dev_private;
		4171	int ret;
		4172
		4173	mutex_lock(&dev->struct_mutex);
3746	Serge	4174
		4175	if (IS_VALLEYVIEW(dev)) {
		4176	/* VLVA0 (potential hack), BIOS isn't actually waking us */
		4177	I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		4178	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
		4179	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4180	}
		4181
3480	Serge	4182	i915_gem_init_global_gtt(dev);
3746	Serge	4183
3031	serge	4184	ret = i915_gem_init_hw(dev);
		4185	mutex_unlock(&dev->struct_mutex);
		4186	if (ret) {
		4187	i915_gem_cleanup_aliasing_ppgtt(dev);
		4188	return ret;
		4189	}
		4190
3746	Serge	4191
3031	serge	4192	return 0;
		4193	}
		4194
2332	Serge	4195	void
		4196	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4197	{
		4198	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	4199	struct intel_ring_buffer *ring;
2332	Serge	4200	int i;
		4201
3031	serge	4202	for_each_ring(ring, dev_priv, i)
		4203	intel_cleanup_ring_buffer(ring);
2332	Serge	4204	}
		4205
3031	serge	4206	#if 0
		4207
2332	Serge	4208	int
		4209	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		4210	struct drm_file *file_priv)
		4211	{
4104	Serge	4212	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4213	int ret;
2332	Serge	4214
		4215	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4216	return 0;
		4217
3480	Serge	4218	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	4219	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	4220	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	4221	}
		4222
		4223	mutex_lock(&dev->struct_mutex);
4104	Serge	4224	dev_priv->ums.mm_suspended = 0;
2332	Serge	4225
3031	serge	4226	ret = i915_gem_init_hw(dev);
2332	Serge	4227	if (ret != 0) {
		4228	mutex_unlock(&dev->struct_mutex);
		4229	return ret;
		4230	}
		4231
4104	Serge	4232	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
2332	Serge	4233	mutex_unlock(&dev->struct_mutex);
		4234
		4235	ret = drm_irq_install(dev);
		4236	if (ret)
		4237	goto cleanup_ringbuffer;
		4238
		4239	return 0;
		4240
		4241	cleanup_ringbuffer:
		4242	mutex_lock(&dev->struct_mutex);
		4243	i915_gem_cleanup_ringbuffer(dev);
4104	Serge	4244	dev_priv->ums.mm_suspended = 1;
2332	Serge	4245	mutex_unlock(&dev->struct_mutex);
		4246
		4247	return ret;
		4248	}
		4249
		4250	int
		4251	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4252	struct drm_file *file_priv)
		4253	{
4104	Serge	4254	struct drm_i915_private *dev_priv = dev->dev_private;
		4255	int ret;
		4256
2332	Serge	4257	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4258	return 0;
		4259
		4260	drm_irq_uninstall(dev);
4104	Serge	4261
		4262	mutex_lock(&dev->struct_mutex);
		4263	ret = i915_gem_idle(dev);
		4264
		4265	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		4266	* We need to replace this with a semaphore, or something.
		4267	* And not confound ums.mm_suspended!
		4268	*/
		4269	if (ret != 0)
		4270	dev_priv->ums.mm_suspended = 1;
		4271	mutex_unlock(&dev->struct_mutex);
		4272
		4273	return ret;
2332	Serge	4274	}
		4275
		4276	void
		4277	i915_gem_lastclose(struct drm_device *dev)
		4278	{
		4279	int ret;
		4280
		4281	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4282	return;
		4283
4104	Serge	4284	mutex_lock(&dev->struct_mutex);
2332	Serge	4285	ret = i915_gem_idle(dev);
		4286	if (ret)
		4287	DRM_ERROR("failed to idle hardware: %d\n", ret);
4104	Serge	4288	mutex_unlock(&dev->struct_mutex);
2332	Serge	4289	}
		4290	#endif
		4291
		4292	static void
2326	Serge	4293	init_ring_lists(struct intel_ring_buffer *ring)
		4294	{
		4295	INIT_LIST_HEAD(&ring->active_list);
		4296	INIT_LIST_HEAD(&ring->request_list);
		4297	}
		4298
4104	Serge	4299	static void i915_init_vm(struct drm_i915_private *dev_priv,
		4300	struct i915_address_space *vm)
		4301	{
		4302	vm->dev = dev_priv->dev;
		4303	INIT_LIST_HEAD(&vm->active_list);
		4304	INIT_LIST_HEAD(&vm->inactive_list);
		4305	INIT_LIST_HEAD(&vm->global_link);
		4306	list_add(&vm->global_link, &dev_priv->vm_list);
		4307	}
		4308
2326	Serge	4309	void
		4310	i915_gem_load(struct drm_device *dev)
		4311	{
3480	Serge	4312	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4313	int i;
		4314
4104	Serge	4315	INIT_LIST_HEAD(&dev_priv->vm_list);
		4316	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4317
3031	serge	4318	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4319	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4320	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4321	for (i = 0; i < I915_NUM_RINGS; i++)
		4322	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4323	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4324	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4325	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4326	i915_gem_retire_work_handler);
3480	Serge	4327	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4328
		4329	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4330	if (IS_GEN3(dev)) {
3031	serge	4331	I915_WRITE(MI_ARB_STATE,
		4332	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4333	}
		4334
		4335	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4336
3746	Serge	4337	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4338	dev_priv->num_fence_regs = 32;
		4339	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4340	dev_priv->num_fence_regs = 16;
		4341	else
		4342	dev_priv->num_fence_regs = 8;
		4343
		4344	/* Initialize fence registers to zero */
3746	Serge	4345	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4346	i915_gem_restore_fences(dev);
2326	Serge	4347
		4348	i915_gem_detect_bit_6_swizzle(dev);
		4349
		4350	dev_priv->mm.interruptible = true;
		4351
		4352	}
		4353
4104	Serge	4354	#if 0
		4355	/*
		4356	* Create a physically contiguous memory object for this object
		4357	* e.g. for cursor + overlay regs
		4358	*/
		4359	static int i915_gem_init_phys_object(struct drm_device *dev,
		4360	int id, int size, int align)
		4361	{
		4362	drm_i915_private_t *dev_priv = dev->dev_private;
		4363	struct drm_i915_gem_phys_object *phys_obj;
		4364	int ret;
2326	Serge	4365
4104	Serge	4366	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)
		4367	return 0;
		4368
		4369	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
		4370	if (!phys_obj)
		4371	return -ENOMEM;
		4372
		4373	phys_obj->id = id;
		4374
		4375	phys_obj->handle = drm_pci_alloc(dev, size, align);
		4376	if (!phys_obj->handle) {
		4377	ret = -ENOMEM;
		4378	goto kfree_obj;
		4379	}
		4380	#ifdef CONFIG_X86
		4381	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4382	#endif
		4383
		4384	dev_priv->mm.phys_objs[id - 1] = phys_obj;
		4385
		4386	return 0;
		4387	kfree_obj:
		4388	kfree(phys_obj);
		4389	return ret;
		4390	}
		4391
		4392	static void i915_gem_free_phys_object(struct drm_device *dev, int id)
		4393	{
		4394	drm_i915_private_t *dev_priv = dev->dev_private;
		4395	struct drm_i915_gem_phys_object *phys_obj;
		4396
		4397	if (!dev_priv->mm.phys_objs[id - 1])
		4398	return;
		4399
		4400	phys_obj = dev_priv->mm.phys_objs[id - 1];
		4401	if (phys_obj->cur_obj) {
		4402	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
		4403	}
		4404
		4405	#ifdef CONFIG_X86
		4406	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4407	#endif
		4408	drm_pci_free(dev, phys_obj->handle);
		4409	kfree(phys_obj);
		4410	dev_priv->mm.phys_objs[id - 1] = NULL;
		4411	}
		4412
		4413	void i915_gem_free_all_phys_object(struct drm_device *dev)
		4414	{
		4415	int i;
		4416
		4417	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
		4418	i915_gem_free_phys_object(dev, i);
		4419	}
		4420
		4421	void i915_gem_detach_phys_object(struct drm_device *dev,
		4422	struct drm_i915_gem_object *obj)
		4423	{
		4424	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4425	char *vaddr;
		4426	int i;
		4427	int page_count;
		4428
		4429	if (!obj->phys_obj)
		4430	return;
		4431	vaddr = obj->phys_obj->handle->vaddr;
		4432
		4433	page_count = obj->base.size / PAGE_SIZE;
		4434	for (i = 0; i < page_count; i++) {
		4435	struct page *page = shmem_read_mapping_page(mapping, i);
		4436	if (!IS_ERR(page)) {
		4437	char *dst = kmap_atomic(page);
		4438	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
		4439	kunmap_atomic(dst);
		4440
		4441	drm_clflush_pages(&page, 1);
		4442
		4443	set_page_dirty(page);
		4444	mark_page_accessed(page);
		4445	page_cache_release(page);
		4446	}
		4447	}
		4448	i915_gem_chipset_flush(dev);
		4449
		4450	obj->phys_obj->cur_obj = NULL;
		4451	obj->phys_obj = NULL;
		4452	}
		4453
		4454	int
		4455	i915_gem_attach_phys_object(struct drm_device *dev,
		4456	struct drm_i915_gem_object *obj,
		4457	int id,
		4458	int align)
		4459	{
		4460	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4461	drm_i915_private_t *dev_priv = dev->dev_private;
		4462	int ret = 0;
		4463	int page_count;
		4464	int i;
		4465
		4466	if (id > I915_MAX_PHYS_OBJECT)
		4467	return -EINVAL;
		4468
		4469	if (obj->phys_obj) {
		4470	if (obj->phys_obj->id == id)
		4471	return 0;
		4472	i915_gem_detach_phys_object(dev, obj);
		4473	}
		4474
		4475	/* create a new object */
		4476	if (!dev_priv->mm.phys_objs[id - 1]) {
		4477	ret = i915_gem_init_phys_object(dev, id,
		4478	obj->base.size, align);
		4479	if (ret) {
		4480	DRM_ERROR("failed to init phys object %d size: %zu\n",
		4481	id, obj->base.size);
		4482	return ret;
		4483	}
		4484	}
		4485
		4486	/* bind to the object */
		4487	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
		4488	obj->phys_obj->cur_obj = obj;
		4489
		4490	page_count = obj->base.size / PAGE_SIZE;
		4491
		4492	for (i = 0; i < page_count; i++) {
		4493	struct page *page;
		4494	char dst, src;
		4495
		4496	page = shmem_read_mapping_page(mapping, i);
		4497	if (IS_ERR(page))
		4498	return PTR_ERR(page);
		4499
		4500	src = kmap_atomic(page);
		4501	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
		4502	memcpy(dst, src, PAGE_SIZE);
		4503	kunmap_atomic(src);
		4504
		4505	mark_page_accessed(page);
		4506	page_cache_release(page);
		4507	}
		4508
		4509	return 0;
		4510	}
		4511
		4512	static int
		4513	i915_gem_phys_pwrite(struct drm_device *dev,
		4514	struct drm_i915_gem_object *obj,
		4515	struct drm_i915_gem_pwrite *args,
		4516	struct drm_file *file_priv)
		4517	{
		4518	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
		4519	char __user *user_data = to_user_ptr(args->data_ptr);
		4520
		4521	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		4522	unsigned long unwritten;
		4523
		4524	/* The physical object once assigned is fixed for the lifetime
		4525	* of the obj, so we can safely drop the lock and continue
		4526	* to access vaddr.
		4527	*/
		4528	mutex_unlock(&dev->struct_mutex);
		4529	unwritten = copy_from_user(vaddr, user_data, args->size);
		4530	mutex_lock(&dev->struct_mutex);
		4531	if (unwritten)
		4532	return -EFAULT;
		4533	}
		4534
		4535	i915_gem_chipset_flush(dev);
		4536	return 0;
		4537	}
		4538
		4539	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4540	{
		4541	struct drm_i915_file_private *file_priv = file->driver_priv;
		4542
		4543	/* Clean up our request list when the client is going away, so that
		4544	* later retire_requests won't dereference our soon-to-be-gone
		4545	* file_priv.
		4546	*/
		4547	spin_lock(&file_priv->mm.lock);
		4548	while (!list_empty(&file_priv->mm.request_list)) {
		4549	struct drm_i915_gem_request *request;
		4550
		4551	request = list_first_entry(&file_priv->mm.request_list,
		4552	struct drm_i915_gem_request,
		4553	client_list);
		4554	list_del(&request->client_list);
		4555	request->file_priv = NULL;
		4556	}
		4557	spin_unlock(&file_priv->mm.lock);
		4558	}
		4559	#endif
		4560
		4561	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4562	{
		4563	if (!mutex_is_locked(mutex))
		4564	return false;
		4565
		4566	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4567	return mutex->owner == task;
		4568	#else
		4569	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4570	return false;
		4571	#endif
		4572	}
		4573
		4574	/* All the new VM stuff */
		4575	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4576	struct i915_address_space *vm)
		4577	{
		4578	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4579	struct i915_vma *vma;
		4580
		4581	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4582	vm = &dev_priv->gtt.base;
		4583
		4584	BUG_ON(list_empty(&o->vma_list));
		4585	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4586	if (vma->vm == vm)
		4587	return vma->node.start;
		4588
		4589	}
		4590	return 0; //-1;
		4591	}
		4592
		4593	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4594	struct i915_address_space *vm)
		4595	{
		4596	struct i915_vma *vma;
		4597
		4598	list_for_each_entry(vma, &o->vma_list, vma_link)
		4599	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4600	return true;
		4601
		4602	return false;
		4603	}
		4604
		4605	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4606	{
		4607	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4608	struct i915_address_space *vm;
		4609
		4610	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
		4611	if (i915_gem_obj_bound(o, vm))
		4612	return true;
		4613
		4614	return false;
		4615	}
		4616
		4617	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4618	struct i915_address_space *vm)
		4619	{
		4620	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4621	struct i915_vma *vma;
		4622
		4623	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4624	vm = &dev_priv->gtt.base;
		4625
		4626	BUG_ON(list_empty(&o->vma_list));
		4627
		4628	list_for_each_entry(vma, &o->vma_list, vma_link)
		4629	if (vma->vm == vm)
		4630	return vma->node.size;
		4631
		4632	return 0;
		4633	}
		4634	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
		4635	struct i915_address_space *vm)
		4636	{
		4637	struct i915_vma *vma;
		4638	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4639	if (vma->vm == vm)
		4640	return vma;
		4641
		4642	return NULL;
		4643	}
		4644
		4645	struct i915_vma *
		4646	i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
		4647	struct i915_address_space *vm)
		4648	{
		4649	struct i915_vma *vma;
		4650
		4651	vma = i915_gem_obj_to_vma(obj, vm);
		4652	if (!vma)
		4653	vma = i915_gem_vma_create(obj, vm);
		4654
		4655	return vma;
		4656	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 4392