WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
3260	Serge	34	#include
2330	Serge	35	#include
2326	Serge	36	//#include
3746	Serge	37	#include
2326	Serge	38	#include
		39
2344	Serge	40	extern int x86_clflush_size;
2332	Serge	41
3263	Serge	42	#define PROT_READ 0x1 /* page can be read */
		43	#define PROT_WRITE 0x2 /* page can be written */
		44	#define MAP_SHARED 0x01 /* Share changes */
		45
2344	Serge	46
3266	Serge	47	struct drm_i915_gem_object *get_fb_obj();
		48
3263	Serge	49	unsigned long vm_mmap(struct file *file, unsigned long addr,
		50	unsigned long len, unsigned long prot,
		51	unsigned long flag, unsigned long offset);
		52
2344	Serge	53	static inline void clflush(volatile void *__p)
		54	{
		55	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		56	}
		57
2332	Serge	58	#define MAX_ERRNO 4095
		59
		60	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		61
		62
2326	Serge	63
2332	Serge	64	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	65	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		66	bool force);
		67	static __must_check int
		68	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		69	struct i915_address_space *vm,
2332	Serge	70	unsigned alignment,
3031	serge	71	bool map_and_fenceable,
		72	bool nonblocking);
2332	Serge	73	static int i915_gem_phys_pwrite(struct drm_device *dev,
		74	struct drm_i915_gem_object *obj,
		75	struct drm_i915_gem_pwrite *args,
		76	struct drm_file *file);
2326	Serge	77
3031	serge	78	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		79	struct drm_i915_gem_object *obj);
		80	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		81	struct drm_i915_fence_reg *fence,
		82	bool enable);
2332	Serge	83
3031	serge	84	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
4104	Serge	85	static long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	86	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		87
4104	Serge	88	static bool cpu_cache_is_coherent(struct drm_device *dev,
		89	enum i915_cache_level level)
		90	{
		91	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		92	}
		93
		94	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		95	{
		96	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		97	return true;
		98
		99	return obj->pin_display;
		100	}
		101
3031	serge	102	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		103	{
		104	if (obj->tiling_mode)
		105	i915_gem_release_mmap(obj);
		106
		107	/* As we do not have an associated fence register, we will force
		108	* a tiling change if we ever need to acquire one.
		109	*/
		110	obj->fence_dirty = false;
		111	obj->fence_reg = I915_FENCE_REG_NONE;
		112	}
		113
2332	Serge	114	/* some bookkeeping */
		115	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		116	size_t size)
		117	{
4104	Serge	118	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	119	dev_priv->mm.object_count++;
		120	dev_priv->mm.object_memory += size;
4104	Serge	121	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	122	}
		123
		124	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		125	size_t size)
		126	{
4104	Serge	127	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	128	dev_priv->mm.object_count--;
		129	dev_priv->mm.object_memory -= size;
4104	Serge	130	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	131	}
		132
		133	static int
3480	Serge	134	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	135	{
		136	int ret;
		137
3480	Serge	138	#define EXIT_COND (!i915_reset_in_progress(error))
		139	if (EXIT_COND)
2332	Serge	140	return 0;
3255	Serge	141	#if 0
3031	serge	142	/*
		143	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		144	* userspace. If it takes that long something really bad is going on and
		145	* we should simply try to bail out and fail as gracefully as possible.
		146	*/
3480	Serge	147	ret = wait_event_interruptible_timeout(error->reset_queue,
		148	EXIT_COND,
		149	10*HZ);
3031	serge	150	if (ret == 0) {
		151	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		152	return -EIO;
		153	} else if (ret < 0) {
2332	Serge	154	return ret;
3031	serge	155	}
2332	Serge	156
3255	Serge	157	#endif
3480	Serge	158	#undef EXIT_COND
3255	Serge	159
2332	Serge	160	return 0;
		161	}
		162
		163	int i915_mutex_lock_interruptible(struct drm_device *dev)
		164	{
3480	Serge	165	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	166	int ret;
		167
3480	Serge	168	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	169	if (ret)
		170	return ret;
		171
3480	Serge	172	ret = mutex_lock_interruptible(&dev->struct_mutex);
		173	if (ret)
		174	return ret;
2332	Serge	175
		176	WARN_ON(i915_verify_lists(dev));
		177	return 0;
		178	}
		179
		180	static inline bool
		181	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		182	{
4104	Serge	183	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	184	}
		185
		186
		187	#if 0
		188
		189	int
		190	i915_gem_init_ioctl(struct drm_device dev, void data,
		191	struct drm_file *file)
		192	{
3480	Serge	193	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	194	struct drm_i915_gem_init *args = data;
		195
3031	serge	196	if (drm_core_check_feature(dev, DRIVER_MODESET))
		197	return -ENODEV;
		198
2332	Serge	199	if (args->gtt_start >= args->gtt_end \|\|
		200	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		201	return -EINVAL;
		202
3031	serge	203	/* GEM with user mode setting was never supported on ilk and later. */
		204	if (INTEL_INFO(dev)->gen >= 5)
		205	return -ENODEV;
		206
2332	Serge	207	mutex_lock(&dev->struct_mutex);
3480	Serge	208	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		209	args->gtt_end);
		210	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	211	mutex_unlock(&dev->struct_mutex);
		212
		213	return 0;
		214	}
2351	Serge	215	#endif
2332	Serge	216
		217	int
		218	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		219	struct drm_file *file)
		220	{
		221	struct drm_i915_private *dev_priv = dev->dev_private;
		222	struct drm_i915_gem_get_aperture *args = data;
		223	struct drm_i915_gem_object *obj;
		224	size_t pinned;
		225
		226	pinned = 0;
		227	mutex_lock(&dev->struct_mutex);
4104	Serge	228	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
3031	serge	229	if (obj->pin_count)
4104	Serge	230	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	231	mutex_unlock(&dev->struct_mutex);
		232
4104	Serge	233	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	234	args->aper_available_size = args->aper_size - pinned;
2332	Serge	235
		236	return 0;
		237	}
		238
3480	Serge	239	void i915_gem_object_alloc(struct drm_device dev)
		240	{
		241	struct drm_i915_private *dev_priv = dev->dev_private;
		242	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		243	}
		244
		245	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		246	{
		247	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		248	kfree(obj);
		249	}
		250
3031	serge	251	static int
		252	i915_gem_create(struct drm_file *file,
2332	Serge	253	struct drm_device *dev,
		254	uint64_t size,
		255	uint32_t *handle_p)
		256	{
		257	struct drm_i915_gem_object *obj;
		258	int ret;
		259	u32 handle;
		260
		261	size = roundup(size, PAGE_SIZE);
2342	Serge	262	if (size == 0)
		263	return -EINVAL;
2332	Serge	264
		265	/* Allocate the new object */
		266	obj = i915_gem_alloc_object(dev, size);
		267	if (obj == NULL)
		268	return -ENOMEM;
		269
		270	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	271	/* drop reference from allocate - handle holds it now */
		272	drm_gem_object_unreference_unlocked(&obj->base);
		273	if (ret)
2332	Serge	274	return ret;
		275
		276	*handle_p = handle;
		277	return 0;
		278	}
		279
		280	int
		281	i915_gem_dumb_create(struct drm_file *file,
		282	struct drm_device *dev,
		283	struct drm_mode_create_dumb *args)
		284	{
		285	/* have to work out size/pitch and return them */
		286	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		287	args->size = args->pitch * args->height;
		288	return i915_gem_create(file, dev,
		289	args->size, &args->handle);
		290	}
		291
2326	Serge	292	/**
2332	Serge	293	* Creates a new mm object and returns a handle to it.
		294	*/
		295	int
		296	i915_gem_create_ioctl(struct drm_device dev, void data,
		297	struct drm_file *file)
		298	{
		299	struct drm_i915_gem_create *args = data;
3031	serge	300
2332	Serge	301	return i915_gem_create(file, dev,
		302	args->size, &args->handle);
		303	}
		304
		305
3260	Serge	306	#if 0
2332	Serge	307
3031	serge	308	static inline int
		309	__copy_to_user_swizzled(char __user *cpu_vaddr,
		310	const char *gpu_vaddr, int gpu_offset,
2332	Serge	311	int length)
		312	{
3031	serge	313	int ret, cpu_offset = 0;
2332	Serge	314
3031	serge	315	while (length > 0) {
		316	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		317	int this_length = min(cacheline_end - gpu_offset, length);
		318	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	319
3031	serge	320	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		321	gpu_vaddr + swizzled_gpu_offset,
		322	this_length);
		323	if (ret)
		324	return ret + length;
2332	Serge	325
3031	serge	326	cpu_offset += this_length;
		327	gpu_offset += this_length;
		328	length -= this_length;
		329	}
		330
		331	return 0;
2332	Serge	332	}
		333
3031	serge	334	static inline int
		335	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		336	const char __user *cpu_vaddr,
		337	int length)
2332	Serge	338	{
3031	serge	339	int ret, cpu_offset = 0;
2332	Serge	340
		341	while (length > 0) {
		342	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		343	int this_length = min(cacheline_end - gpu_offset, length);
		344	int swizzled_gpu_offset = gpu_offset ^ 64;
		345
3031	serge	346	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	347	cpu_vaddr + cpu_offset,
		348	this_length);
3031	serge	349	if (ret)
		350	return ret + length;
		351
2332	Serge	352	cpu_offset += this_length;
		353	gpu_offset += this_length;
		354	length -= this_length;
		355	}
		356
3031	serge	357	return 0;
2332	Serge	358	}
		359
3031	serge	360	/* Per-page copy function for the shmem pread fastpath.
		361	* Flushes invalid cachelines before reading the target if
		362	* needs_clflush is set. */
2332	Serge	363	static int
3031	serge	364	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		365	char __user *user_data,
		366	bool page_do_bit17_swizzling, bool needs_clflush)
		367	{
		368	char *vaddr;
		369	int ret;
		370
		371	if (unlikely(page_do_bit17_swizzling))
		372	return -EINVAL;
		373
		374	vaddr = kmap_atomic(page);
		375	if (needs_clflush)
		376	drm_clflush_virt_range(vaddr + shmem_page_offset,
		377	page_length);
		378	ret = __copy_to_user_inatomic(user_data,
		379	vaddr + shmem_page_offset,
		380	page_length);
		381	kunmap_atomic(vaddr);
		382
		383	return ret ? -EFAULT : 0;
		384	}
		385
		386	static void
		387	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		388	bool swizzled)
		389	{
		390	if (unlikely(swizzled)) {
		391	unsigned long start = (unsigned long) addr;
		392	unsigned long end = (unsigned long) addr + length;
		393
		394	/* For swizzling simply ensure that we always flush both
		395	* channels. Lame, but simple and it works. Swizzled
		396	* pwrite/pread is far from a hotpath - current userspace
		397	* doesn't use it at all. */
		398	start = round_down(start, 128);
		399	end = round_up(end, 128);
		400
		401	drm_clflush_virt_range((void *)start, end - start);
		402	} else {
		403	drm_clflush_virt_range(addr, length);
		404	}
		405
		406	}
		407
		408	/* Only difference to the fast-path function is that this can handle bit17
		409	* and uses non-atomic copy and kmap functions. */
		410	static int
		411	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		412	char __user *user_data,
		413	bool page_do_bit17_swizzling, bool needs_clflush)
		414	{
		415	char *vaddr;
		416	int ret;
		417
		418	vaddr = kmap(page);
		419	if (needs_clflush)
		420	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		421	page_length,
		422	page_do_bit17_swizzling);
		423
		424	if (page_do_bit17_swizzling)
		425	ret = __copy_to_user_swizzled(user_data,
		426	vaddr, shmem_page_offset,
		427	page_length);
		428	else
		429	ret = __copy_to_user(user_data,
		430	vaddr + shmem_page_offset,
		431	page_length);
		432	kunmap(page);
		433
		434	return ret ? - EFAULT : 0;
		435	}
		436
		437	static int
		438	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	439	struct drm_i915_gem_object *obj,
		440	struct drm_i915_gem_pread *args,
		441	struct drm_file *file)
		442	{
3031	serge	443	char __user *user_data;
2332	Serge	444	ssize_t remain;
		445	loff_t offset;
3031	serge	446	int shmem_page_offset, page_length, ret = 0;
		447	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		448	int prefaulted = 0;
		449	int needs_clflush = 0;
3746	Serge	450	struct sg_page_iter sg_iter;
2332	Serge	451
3746	Serge	452	user_data = to_user_ptr(args->data_ptr);
2332	Serge	453	remain = args->size;
		454
3031	serge	455	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		456
		457	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		458	/* If we're not in the cpu read domain, set ourself into the gtt
		459	* read domain and manually flush cachelines (if required). This
		460	* optimizes for the case when the gpu will dirty the data
		461	* anyway again before the next pread happens. */
4104	Serge	462	needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
		463	if (i915_gem_obj_bound_any(obj)) {
3031	serge	464	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		465	if (ret)
		466	return ret;
		467	}
		468	}
		469
		470	ret = i915_gem_object_get_pages(obj);
		471	if (ret)
		472	return ret;
		473
		474	i915_gem_object_pin_pages(obj);
		475
2332	Serge	476	offset = args->offset;
		477
3746	Serge	478	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		479	offset >> PAGE_SHIFT) {
		480	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	481
3031	serge	482	if (remain <= 0)
		483	break;
		484
2332	Serge	485	/* Operation in this page
		486	*
3031	serge	487	* shmem_page_offset = offset within page in shmem file
2332	Serge	488	* page_length = bytes to copy for this page
		489	*/
3031	serge	490	shmem_page_offset = offset_in_page(offset);
2332	Serge	491	page_length = remain;
3031	serge	492	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		493	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	494
3031	serge	495	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		496	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	497
3031	serge	498	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		499	user_data, page_do_bit17_swizzling,
		500	needs_clflush);
		501	if (ret == 0)
		502	goto next_page;
2332	Serge	503
3031	serge	504	mutex_unlock(&dev->struct_mutex);
		505
4104	Serge	506	if (likely(!i915_prefault_disable) && !prefaulted) {
3031	serge	507	ret = fault_in_multipages_writeable(user_data, remain);
		508	/* Userspace is tricking us, but we've already clobbered
		509	* its pages with the prefault and promised to write the
		510	* data up to the first fault. Hence ignore any errors
		511	* and just continue. */
		512	(void)ret;
		513	prefaulted = 1;
		514	}
		515
		516	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		517	user_data, page_do_bit17_swizzling,
		518	needs_clflush);
		519
		520	mutex_lock(&dev->struct_mutex);
		521
		522	next_page:
2332	Serge	523	mark_page_accessed(page);
3031	serge	524
2332	Serge	525	if (ret)
3031	serge	526	goto out;
2332	Serge	527
		528	remain -= page_length;
		529	user_data += page_length;
		530	offset += page_length;
		531	}
		532
3031	serge	533	out:
		534	i915_gem_object_unpin_pages(obj);
		535
		536	return ret;
2332	Serge	537	}
		538
		539	/**
3031	serge	540	* Reads data from the object referenced by handle.
		541	*
		542	* On error, the contents of *data are undefined.
2332	Serge	543	*/
3031	serge	544	int
		545	i915_gem_pread_ioctl(struct drm_device dev, void data,
		546	struct drm_file *file)
		547	{
		548	struct drm_i915_gem_pread *args = data;
		549	struct drm_i915_gem_object *obj;
		550	int ret = 0;
		551
		552	if (args->size == 0)
		553	return 0;
		554
		555	if (!access_ok(VERIFY_WRITE,
3746	Serge	556	to_user_ptr(args->data_ptr),
3031	serge	557	args->size))
		558	return -EFAULT;
		559
		560	ret = i915_mutex_lock_interruptible(dev);
		561	if (ret)
		562	return ret;
		563
		564	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		565	if (&obj->base == NULL) {
		566	ret = -ENOENT;
		567	goto unlock;
		568	}
		569
		570	/* Bounds check source. */
		571	if (args->offset > obj->base.size \|\|
		572	args->size > obj->base.size - args->offset) {
		573	ret = -EINVAL;
		574	goto out;
		575	}
		576
		577	/* prime objects have no backing filp to GEM pread/pwrite
		578	* pages from.
		579	*/
		580	if (!obj->base.filp) {
		581	ret = -EINVAL;
		582	goto out;
		583	}
		584
		585	trace_i915_gem_object_pread(obj, args->offset, args->size);
		586
		587	ret = i915_gem_shmem_pread(dev, obj, args, file);
		588
		589	out:
		590	drm_gem_object_unreference(&obj->base);
		591	unlock:
		592	mutex_unlock(&dev->struct_mutex);
		593	return ret;
		594	}
		595
		596	/* This is the fast write path which cannot handle
		597	* page faults in the source data
		598	*/
		599
		600	static inline int
		601	fast_user_write(struct io_mapping *mapping,
		602	loff_t page_base, int page_offset,
		603	char __user *user_data,
		604	int length)
		605	{
		606	void __iomem *vaddr_atomic;
		607	void *vaddr;
		608	unsigned long unwritten;
		609
		610	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		611	/* We can use the cpu mem copy function because this is X86. */
		612	vaddr = (void __force*)vaddr_atomic + page_offset;
		613	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		614	user_data, length);
		615	io_mapping_unmap_atomic(vaddr_atomic);
		616	return unwritten;
		617	}
3260	Serge	618	#endif
3031	serge	619
3260	Serge	620	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	621	/**
		622	* This is the fast pwrite path, where we copy the data directly from the
		623	* user into the GTT, uncached.
		624	*/
2332	Serge	625	static int
3031	serge	626	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		627	struct drm_i915_gem_object *obj,
		628	struct drm_i915_gem_pwrite *args,
		629	struct drm_file *file)
2332	Serge	630	{
3031	serge	631	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	632	ssize_t remain;
3031	serge	633	loff_t offset, page_base;
		634	char __user *user_data;
		635	int page_offset, page_length, ret;
2332	Serge	636
4104	Serge	637	ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
3031	serge	638	if (ret)
		639	goto out;
		640
		641	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		642	if (ret)
		643	goto out_unpin;
		644
		645	ret = i915_gem_object_put_fence(obj);
		646	if (ret)
		647	goto out_unpin;
		648
4539	Serge	649	user_data = to_user_ptr(args->data_ptr);
2332	Serge	650	remain = args->size;
		651
4104	Serge	652	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	653
3031	serge	654	while (remain > 0) {
		655	/* Operation in this page
		656	*
		657	* page_base = page offset within aperture
		658	* page_offset = offset within page
		659	* page_length = bytes to copy for this page
		660	*/
		661	page_base = offset & PAGE_MASK;
		662	page_offset = offset_in_page(offset);
		663	page_length = remain;
		664	if ((page_offset + remain) > PAGE_SIZE)
		665	page_length = PAGE_SIZE - page_offset;
2332	Serge	666
4539	Serge	667	MapPage(dev_priv->gtt.mappable, dev_priv->gtt.mappable_base+page_base, PG_SW);
3031	serge	668
4539	Serge	669	memcpy(dev_priv->gtt.mappable+page_offset, user_data, page_length);
3260	Serge	670
3031	serge	671	remain -= page_length;
		672	user_data += page_length;
		673	offset += page_length;
2332	Serge	674	}
		675
3031	serge	676	out_unpin:
		677	i915_gem_object_unpin(obj);
		678	out:
		679	return ret;
		680	}
		681
		682	/* Per-page copy function for the shmem pwrite fastpath.
		683	* Flushes invalid cachelines before writing to the target if
		684	* needs_clflush_before is set and flushes out any written cachelines after
		685	* writing if needs_clflush is set. */
		686	static int
		687	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		688	char __user *user_data,
		689	bool page_do_bit17_swizzling,
		690	bool needs_clflush_before,
		691	bool needs_clflush_after)
		692	{
		693	char *vaddr;
3260	Serge	694	int ret = 0;
3031	serge	695
		696	if (unlikely(page_do_bit17_swizzling))
		697	return -EINVAL;
		698
4539	Serge	699	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	700	if (needs_clflush_before)
		701	drm_clflush_virt_range(vaddr + shmem_page_offset,
		702	page_length);
3260	Serge	703	memcpy(vaddr + shmem_page_offset,
3031	serge	704	user_data,
		705	page_length);
		706	if (needs_clflush_after)
		707	drm_clflush_virt_range(vaddr + shmem_page_offset,
		708	page_length);
3260	Serge	709	FreeKernelSpace(vaddr);
3031	serge	710
		711	return ret ? -EFAULT : 0;
		712	}
3260	Serge	713	#if 0
3031	serge	714
		715	/* Only difference to the fast-path function is that this can handle bit17
		716	* and uses non-atomic copy and kmap functions. */
		717	static int
		718	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		719	char __user *user_data,
		720	bool page_do_bit17_swizzling,
		721	bool needs_clflush_before,
		722	bool needs_clflush_after)
		723	{
		724	char *vaddr;
		725	int ret;
		726
		727	vaddr = kmap(page);
		728	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		729	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		730	page_length,
		731	page_do_bit17_swizzling);
		732	if (page_do_bit17_swizzling)
		733	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		734	user_data,
		735	page_length);
		736	else
		737	ret = __copy_from_user(vaddr + shmem_page_offset,
		738	user_data,
		739	page_length);
		740	if (needs_clflush_after)
		741	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		742	page_length,
		743	page_do_bit17_swizzling);
		744	kunmap(page);
		745
		746	return ret ? -EFAULT : 0;
		747	}
3260	Serge	748	#endif
3031	serge	749
3260	Serge	750
3031	serge	751	static int
		752	i915_gem_shmem_pwrite(struct drm_device *dev,
		753	struct drm_i915_gem_object *obj,
		754	struct drm_i915_gem_pwrite *args,
		755	struct drm_file *file)
		756	{
		757	ssize_t remain;
		758	loff_t offset;
		759	char __user *user_data;
		760	int shmem_page_offset, page_length, ret = 0;
		761	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		762	int hit_slowpath = 0;
		763	int needs_clflush_after = 0;
		764	int needs_clflush_before = 0;
3746	Serge	765	struct sg_page_iter sg_iter;
3031	serge	766
3746	Serge	767	user_data = to_user_ptr(args->data_ptr);
3031	serge	768	remain = args->size;
		769
		770	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		771
		772	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		773	/* If we're not in the cpu write domain, set ourself into the gtt
		774	* write domain and manually flush cachelines (if required). This
		775	* optimizes for the case when the gpu will use the data
		776	* right away and we therefore have to clflush anyway. */
4104	Serge	777	needs_clflush_after = cpu_write_needs_clflush(obj);
		778	if (i915_gem_obj_bound_any(obj)) {
3031	serge	779	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		780	if (ret)
		781	return ret;
		782	}
		783	}
4104	Serge	784	/* Same trick applies to invalidate partially written cachelines read
		785	* before writing. */
		786	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		787	needs_clflush_before =
		788	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	789
		790	ret = i915_gem_object_get_pages(obj);
2332	Serge	791	if (ret)
3031	serge	792	return ret;
2332	Serge	793
3031	serge	794	i915_gem_object_pin_pages(obj);
2332	Serge	795
		796	offset = args->offset;
3031	serge	797	obj->dirty = 1;
2332	Serge	798
3746	Serge	799	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		800	offset >> PAGE_SHIFT) {
		801	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	802	int partial_cacheline_write;
2332	Serge	803
3031	serge	804	if (remain <= 0)
		805	break;
		806
2332	Serge	807	/* Operation in this page
		808	*
		809	* shmem_page_offset = offset within page in shmem file
		810	* page_length = bytes to copy for this page
		811	*/
		812	shmem_page_offset = offset_in_page(offset);
		813
		814	page_length = remain;
		815	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		816	page_length = PAGE_SIZE - shmem_page_offset;
		817
3031	serge	818	/* If we don't overwrite a cacheline completely we need to be
		819	* careful to have up-to-date data by first clflushing. Don't
		820	* overcomplicate things and flush the entire patch. */
		821	partial_cacheline_write = needs_clflush_before &&
		822	((shmem_page_offset \| page_length)
3260	Serge	823	& (x86_clflush_size - 1));
2332	Serge	824
3031	serge	825	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		826	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	827
3031	serge	828	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		829	user_data, page_do_bit17_swizzling,
		830	partial_cacheline_write,
		831	needs_clflush_after);
		832	if (ret == 0)
		833	goto next_page;
		834
		835	hit_slowpath = 1;
		836	mutex_unlock(&dev->struct_mutex);
3260	Serge	837	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	838
3260	Serge	839	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		840	// user_data, page_do_bit17_swizzling,
		841	// partial_cacheline_write,
		842	// needs_clflush_after);
		843
3031	serge	844	mutex_lock(&dev->struct_mutex);
		845
		846	next_page:
2332	Serge	847
3031	serge	848	if (ret)
		849	goto out;
		850
2332	Serge	851	remain -= page_length;
3031	serge	852	user_data += page_length;
2332	Serge	853	offset += page_length;
		854	}
		855
		856	out:
3031	serge	857	i915_gem_object_unpin_pages(obj);
		858
		859	if (hit_slowpath) {
3480	Serge	860	/*
		861	* Fixup: Flush cpu caches in case we didn't flush the dirty
		862	* cachelines in-line while writing and the object moved
		863	* out of the cpu write domain while we've dropped the lock.
		864	*/
		865	if (!needs_clflush_after &&
		866	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	867	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	868	i915_gem_chipset_flush(dev);
3031	serge	869	}
2332	Serge	870	}
		871
3031	serge	872	if (needs_clflush_after)
3243	Serge	873	i915_gem_chipset_flush(dev);
3031	serge	874
2332	Serge	875	return ret;
		876	}
3031	serge	877
		878	/**
		879	* Writes data to the object referenced by handle.
		880	*
		881	* On error, the contents of the buffer that were to be modified are undefined.
		882	*/
		883	int
		884	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		885	struct drm_file *file)
		886	{
		887	struct drm_i915_gem_pwrite *args = data;
		888	struct drm_i915_gem_object *obj;
		889	int ret;
		890
4104	Serge	891	if (args->size == 0)
		892	return 0;
		893
3480	Serge	894
3031	serge	895	ret = i915_mutex_lock_interruptible(dev);
		896	if (ret)
		897	return ret;
		898
		899	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		900	if (&obj->base == NULL) {
		901	ret = -ENOENT;
		902	goto unlock;
		903	}
		904
		905	/* Bounds check destination. */
		906	if (args->offset > obj->base.size \|\|
		907	args->size > obj->base.size - args->offset) {
		908	ret = -EINVAL;
		909	goto out;
		910	}
		911
		912	/* prime objects have no backing filp to GEM pread/pwrite
		913	* pages from.
		914	*/
		915	if (!obj->base.filp) {
		916	ret = -EINVAL;
		917	goto out;
		918	}
		919
		920	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		921
		922	ret = -EFAULT;
		923	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		924	* it would end up going through the fenced access, and we'll get
		925	* different detiling behavior between reading and writing.
		926	* pread/pwrite currently are reading and writing from the CPU
		927	* perspective, requiring manual detiling by the client.
		928	*/
3260	Serge	929	// if (obj->phys_obj) {
		930	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		931	// goto out;
		932	// }
3031	serge	933
4104	Serge	934	if (obj->tiling_mode == I915_TILING_NONE &&
		935	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		936	cpu_write_needs_clflush(obj)) {
3031	serge	937	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		938	/* Note that the gtt paths might fail with non-page-backed user
		939	* pointers (e.g. gtt mappings when moving data between
		940	* textures). Fallback to the shmem path in that case. */
		941	}
		942
		943	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	944	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	945
		946	out:
		947	drm_gem_object_unreference(&obj->base);
		948	unlock:
		949	mutex_unlock(&dev->struct_mutex);
		950	return ret;
		951	}
		952
		953	int
3480	Serge	954	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	955	bool interruptible)
		956	{
3480	Serge	957	if (i915_reset_in_progress(error)) {
3031	serge	958	/* Non-interruptible callers can't handle -EAGAIN, hence return
		959	* -EIO unconditionally for these. */
		960	if (!interruptible)
		961	return -EIO;
2332	Serge	962
3480	Serge	963	/* Recovery complete, but the reset failed ... */
		964	if (i915_terminally_wedged(error))
3031	serge	965	return -EIO;
2332	Serge	966
3031	serge	967	return -EAGAIN;
		968	}
2332	Serge	969
3031	serge	970	return 0;
		971	}
2332	Serge	972
3031	serge	973	/*
		974	* Compare seqno against outstanding lazy request. Emit a request if they are
		975	* equal.
		976	*/
		977	static int
		978	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		979	{
		980	int ret;
2332	Serge	981
3031	serge	982	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	983
3031	serge	984	ret = 0;
		985	if (seqno == ring->outstanding_lazy_request)
4104	Serge	986	ret = i915_add_request(ring, NULL);
2332	Serge	987
3031	serge	988	return ret;
		989	}
2332	Serge	990
3031	serge	991	/**
		992	* __wait_seqno - wait until execution of seqno has finished
		993	* @ring: the ring expected to report seqno
		994	* @seqno: duh!
3480	Serge	995	* @reset_counter: reset sequence associated with the given seqno
3031	serge	996	* @interruptible: do an interruptible wait (normally yes)
		997	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		998	*
3480	Serge	999	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1000	* values have been read by the caller in an smp safe manner. Where read-side
		1001	* locks are involved, it is sufficient to read the reset_counter before
		1002	* unlocking the lock that protects the seqno. For lockless tricks, the
		1003	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1004	* inserted.
		1005	*
3031	serge	1006	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1007	* errno with remaining time filled in timeout argument.
		1008	*/
		1009	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1010	unsigned reset_counter,
3031	serge	1011	bool interruptible, struct timespec *timeout)
		1012	{
		1013	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1014	struct timespec before, now, wait_time={1,0};
		1015	unsigned long timeout_jiffies;
		1016	long end;
		1017	bool wait_forever = true;
		1018	int ret;
2332	Serge	1019
4104	Serge	1020	WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
		1021
3031	serge	1022	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1023	return 0;
2332	Serge	1024
3031	serge	1025	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1026
3031	serge	1027	if (timeout != NULL) {
		1028	wait_time = *timeout;
		1029	wait_forever = false;
		1030	}
2332	Serge	1031
4104	Serge	1032	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
2332	Serge	1033
3031	serge	1034	if (WARN_ON(!ring->irq_get(ring)))
		1035	return -ENODEV;
2332	Serge	1036
3031	serge	1037	/* Record current time in case interrupted by signal, or wedged * */
		1038	getrawmonotonic(&before);
2332	Serge	1039
3031	serge	1040	#define EXIT_COND \
		1041	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1042	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1043	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1044	do {
3266	Serge	1045	if (interruptible)
		1046	end = wait_event_interruptible_timeout(ring->irq_queue,
		1047	EXIT_COND,
		1048	timeout_jiffies);
		1049	else
3031	serge	1050	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1051	timeout_jiffies);
2332	Serge	1052
3480	Serge	1053	/* We need to check whether any gpu reset happened in between
		1054	* the caller grabbing the seqno and now ... */
		1055	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1056	end = -EAGAIN;
		1057
		1058	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1059	* gone. */
		1060	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1061	if (ret)
		1062	end = ret;
		1063	} while (end == 0 && wait_forever);
2332	Serge	1064
3031	serge	1065	getrawmonotonic(&now);
2332	Serge	1066
3031	serge	1067	ring->irq_put(ring);
		1068	trace_i915_gem_request_wait_end(ring, seqno);
		1069	#undef EXIT_COND
2332	Serge	1070
3031	serge	1071	if (timeout) {
4104	Serge	1072	// struct timespec sleep_time = timespec_sub(now, before);
		1073	// timeout = timespec_sub(timeout, sleep_time);
3031	serge	1074	}
2332	Serge	1075
3031	serge	1076	switch (end) {
		1077	case -EIO:
		1078	case -EAGAIN: /* Wedged */
		1079	case -ERESTARTSYS: /* Signal */
		1080	return (int)end;
		1081	case 0: /* Timeout */
		1082	return -ETIME;
		1083	default: /* Completed */
		1084	WARN_ON(end < 0); /* We're not aware of other errors */
		1085	return 0;
		1086	}
		1087	}
2332	Serge	1088
3031	serge	1089	/**
		1090	* Waits for a sequence number to be signaled, and cleans up the
		1091	* request and object lists appropriately for that event.
		1092	*/
		1093	int
		1094	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1095	{
		1096	struct drm_device *dev = ring->dev;
		1097	struct drm_i915_private *dev_priv = dev->dev_private;
		1098	bool interruptible = dev_priv->mm.interruptible;
		1099	int ret;
2332	Serge	1100
3031	serge	1101	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1102	BUG_ON(seqno == 0);
2332	Serge	1103
3480	Serge	1104	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1105	if (ret)
		1106	return ret;
2332	Serge	1107
3031	serge	1108	ret = i915_gem_check_olr(ring, seqno);
		1109	if (ret)
		1110	return ret;
2332	Serge	1111
3480	Serge	1112	return __wait_seqno(ring, seqno,
		1113	atomic_read(&dev_priv->gpu_error.reset_counter),
		1114	interruptible, NULL);
3031	serge	1115	}
2332	Serge	1116
4104	Serge	1117	static int
		1118	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
		1119	struct intel_ring_buffer *ring)
		1120	{
		1121	i915_gem_retire_requests_ring(ring);
		1122
		1123	/* Manually manage the write flush as we may have not yet
		1124	* retired the buffer.
		1125	*
		1126	* Note that the last_write_seqno is always the earlier of
		1127	* the two (read/write) seqno, so if we haved successfully waited,
		1128	* we know we have passed the last write.
		1129	*/
		1130	obj->last_write_seqno = 0;
		1131	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1132
		1133	return 0;
		1134	}
		1135
3031	serge	1136	/**
		1137	* Ensures that all rendering to the object has completed and the object is
		1138	* safe to unbind from the GTT or access from the CPU.
		1139	*/
		1140	static __must_check int
		1141	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1142	bool readonly)
		1143	{
		1144	struct intel_ring_buffer *ring = obj->ring;
		1145	u32 seqno;
		1146	int ret;
2332	Serge	1147
3031	serge	1148	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1149	if (seqno == 0)
		1150	return 0;
2332	Serge	1151
3031	serge	1152	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1153	if (ret)
		1154	return ret;
2332	Serge	1155
4104	Serge	1156	return i915_gem_object_wait_rendering__tail(obj, ring);
3031	serge	1157	}
2332	Serge	1158
3260	Serge	1159	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1160	* as the object state may change during this call.
		1161	*/
		1162	static __must_check int
		1163	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1164	bool readonly)
		1165	{
		1166	struct drm_device *dev = obj->base.dev;
		1167	struct drm_i915_private *dev_priv = dev->dev_private;
		1168	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1169	unsigned reset_counter;
3260	Serge	1170	u32 seqno;
		1171	int ret;
2332	Serge	1172
3260	Serge	1173	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1174	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1175
3260	Serge	1176	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1177	if (seqno == 0)
		1178	return 0;
2332	Serge	1179
3480	Serge	1180	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1181	if (ret)
		1182	return ret;
2332	Serge	1183
3260	Serge	1184	ret = i915_gem_check_olr(ring, seqno);
		1185	if (ret)
		1186	return ret;
2332	Serge	1187
3480	Serge	1188	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1189	mutex_unlock(&dev->struct_mutex);
3480	Serge	1190	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1191	mutex_lock(&dev->struct_mutex);
4104	Serge	1192	if (ret)
		1193	return ret;
2332	Serge	1194
4104	Serge	1195	return i915_gem_object_wait_rendering__tail(obj, ring);
3260	Serge	1196	}
2332	Serge	1197
3260	Serge	1198	/**
		1199	* Called when user space prepares to use an object with the CPU, either
		1200	* through the mmap ioctl's mapping or a GTT mapping.
		1201	*/
		1202	int
		1203	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1204	struct drm_file *file)
		1205	{
		1206	struct drm_i915_gem_set_domain *args = data;
		1207	struct drm_i915_gem_object *obj;
		1208	uint32_t read_domains = args->read_domains;
		1209	uint32_t write_domain = args->write_domain;
		1210	int ret;
2332	Serge	1211
3260	Serge	1212	/* Only handle setting domains to types used by the CPU. */
		1213	if (write_domain & I915_GEM_GPU_DOMAINS)
		1214	return -EINVAL;
2332	Serge	1215
3260	Serge	1216	if (read_domains & I915_GEM_GPU_DOMAINS)
		1217	return -EINVAL;
2332	Serge	1218
3260	Serge	1219	/* Having something in the write domain implies it's in the read
		1220	* domain, and only that read domain. Enforce that in the request.
		1221	*/
		1222	if (write_domain != 0 && read_domains != write_domain)
		1223	return -EINVAL;
2332	Serge	1224
3260	Serge	1225	ret = i915_mutex_lock_interruptible(dev);
		1226	if (ret)
		1227	return ret;
2332	Serge	1228
3260	Serge	1229	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1230	if (&obj->base == NULL) {
		1231	ret = -ENOENT;
		1232	goto unlock;
		1233	}
2332	Serge	1234
3260	Serge	1235	/* Try to flush the object off the GPU without holding the lock.
		1236	* We will repeat the flush holding the lock in the normal manner
		1237	* to catch cases where we are gazumped.
		1238	*/
		1239	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1240	if (ret)
		1241	goto unref;
2332	Serge	1242
3260	Serge	1243	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1244	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1245
3260	Serge	1246	/* Silently promote "you're not bound, there was nothing to do"
		1247	* to success, since the client was just asking us to
		1248	* make sure everything was done.
		1249	*/
		1250	if (ret == -EINVAL)
		1251	ret = 0;
		1252	} else {
		1253	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1254	}
2332	Serge	1255
3260	Serge	1256	unref:
		1257	drm_gem_object_unreference(&obj->base);
		1258	unlock:
		1259	mutex_unlock(&dev->struct_mutex);
		1260	return ret;
		1261	}
2332	Serge	1262
4293	Serge	1263	/**
		1264	* Called when user space has done writes to this buffer
		1265	*/
		1266	int
		1267	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
		1268	struct drm_file *file)
		1269	{
		1270	struct drm_i915_gem_sw_finish *args = data;
		1271	struct drm_i915_gem_object *obj;
		1272	int ret = 0;
2332	Serge	1273
4293	Serge	1274	ret = i915_mutex_lock_interruptible(dev);
		1275	if (ret)
		1276	return ret;
2332	Serge	1277
4293	Serge	1278	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1279	if (&obj->base == NULL) {
		1280	ret = -ENOENT;
		1281	goto unlock;
		1282	}
2332	Serge	1283
4293	Serge	1284	/* Pinned buffers may be scanout, so flush the cache */
		1285	if (obj->pin_display)
		1286	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	1287
4293	Serge	1288	drm_gem_object_unreference(&obj->base);
		1289	unlock:
		1290	mutex_unlock(&dev->struct_mutex);
		1291	return ret;
		1292	}
		1293
3260	Serge	1294	/**
		1295	* Maps the contents of an object, returning the address it is mapped
		1296	* into.
		1297	*
		1298	* While the mapping holds a reference on the contents of the object, it doesn't
		1299	* imply a ref on the object itself.
		1300	*/
		1301	int
		1302	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1303	struct drm_file *file)
		1304	{
		1305	struct drm_i915_gem_mmap *args = data;
		1306	struct drm_gem_object *obj;
4392	Serge	1307	unsigned long addr;
2332	Serge	1308
3260	Serge	1309	obj = drm_gem_object_lookup(dev, file, args->handle);
		1310	if (obj == NULL)
		1311	return -ENOENT;
4104	Serge	1312
3260	Serge	1313	/* prime objects have no backing filp to GEM mmap
		1314	* pages from.
		1315	*/
		1316	if (!obj->filp) {
		1317	drm_gem_object_unreference_unlocked(obj);
		1318	return -EINVAL;
		1319	}
2332	Serge	1320
3263	Serge	1321	addr = vm_mmap(obj->filp, 0, args->size,
		1322	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1323	args->offset);
3260	Serge	1324	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1325	if (IS_ERR((void *)addr))
		1326	return addr;
2332	Serge	1327
3260	Serge	1328	args->addr_ptr = (uint64_t) addr;
2332	Serge	1329
3263	Serge	1330	return 0;
3260	Serge	1331	}
2332	Serge	1332
		1333
		1334
		1335
		1336
		1337
		1338
		1339
3031	serge	1340
		1341
		1342
		1343
		1344
		1345	/**
		1346	* i915_gem_release_mmap - remove physical page mappings
		1347	* @obj: obj in question
		1348	*
		1349	* Preserve the reservation of the mmapping with the DRM core code, but
		1350	* relinquish ownership of the pages back to the system.
		1351	*
		1352	* It is vital that we remove the page mapping if we have mapped a tiled
		1353	* object through the GTT and then lose the fence register due to
		1354	* resource pressure. Similarly if the object has been moved out of the
		1355	* aperture, than pages mapped into userspace must be revoked. Removing the
		1356	* mapping will then trigger a page fault on the next user access, allowing
		1357	* fixup by i915_gem_fault().
		1358	*/
		1359	void
		1360	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1361	{
		1362	if (!obj->fault_mappable)
		1363	return;
		1364
4104	Serge	1365	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1366	obj->fault_mappable = false;
		1367	}
		1368
3480	Serge	1369	uint32_t
2332	Serge	1370	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1371	{
		1372	uint32_t gtt_size;
		1373
		1374	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1375	tiling_mode == I915_TILING_NONE)
		1376	return size;
		1377
		1378	/* Previous chips need a power-of-two fence region when tiling */
		1379	if (INTEL_INFO(dev)->gen == 3)
		1380	gtt_size = 1024*1024;
		1381	else
		1382	gtt_size = 512*1024;
		1383
		1384	while (gtt_size < size)
		1385	gtt_size <<= 1;
		1386
		1387	return gtt_size;
		1388	}
		1389
		1390	/**
		1391	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1392	* @obj: object to check
		1393	*
		1394	* Return the required GTT alignment for an object, taking into account
		1395	* potential fence register mapping.
		1396	*/
3480	Serge	1397	uint32_t
		1398	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1399	int tiling_mode, bool fenced)
2332	Serge	1400	{
		1401	/*
		1402	* Minimum alignment is 4k (GTT page size), but might be greater
		1403	* if a fence register is needed for the object.
		1404	*/
3480	Serge	1405	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1406	tiling_mode == I915_TILING_NONE)
		1407	return 4096;
		1408
		1409	/*
		1410	* Previous chips need to be aligned to the size of the smallest
		1411	* fence register that can contain the object.
		1412	*/
		1413	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1414	}
		1415
		1416
		1417
3480	Serge	1418	int
		1419	i915_gem_mmap_gtt(struct drm_file *file,
		1420	struct drm_device *dev,
		1421	uint32_t handle,
		1422	uint64_t *offset)
		1423	{
		1424	struct drm_i915_private *dev_priv = dev->dev_private;
		1425	struct drm_i915_gem_object *obj;
		1426	unsigned long pfn;
		1427	char mem, ptr;
		1428	int ret;
		1429
		1430	ret = i915_mutex_lock_interruptible(dev);
		1431	if (ret)
		1432	return ret;
		1433
		1434	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1435	if (&obj->base == NULL) {
		1436	ret = -ENOENT;
		1437	goto unlock;
		1438	}
		1439
		1440	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1441	ret = -E2BIG;
		1442	goto out;
		1443	}
		1444
		1445	if (obj->madv != I915_MADV_WILLNEED) {
		1446	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1447	ret = -EINVAL;
		1448	goto out;
		1449	}
		1450	/* Now bind it into the GTT if needed */
4104	Serge	1451	ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
3480	Serge	1452	if (ret)
		1453	goto out;
		1454
		1455	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1456	if (ret)
		1457	goto unpin;
		1458
		1459	ret = i915_gem_object_get_fence(obj);
		1460	if (ret)
		1461	goto unpin;
		1462
		1463	obj->fault_mappable = true;
		1464
4104	Serge	1465	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1466
		1467	/* Finally, remap it using the new GTT offset */
		1468
		1469	mem = UserAlloc(obj->base.size);
		1470	if(unlikely(mem == NULL))
		1471	{
		1472	ret = -ENOMEM;
		1473	goto unpin;
		1474	}
		1475
		1476	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1477	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1478
		1479	unpin:
		1480	i915_gem_object_unpin(obj);
		1481
		1482
4104	Serge	1483	*offset = mem;
3480	Serge	1484
		1485	out:
		1486	drm_gem_object_unreference(&obj->base);
		1487	unlock:
		1488	mutex_unlock(&dev->struct_mutex);
		1489	return ret;
		1490	}
		1491
		1492	/**
		1493	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1494	* @dev: DRM device
		1495	* @data: GTT mapping ioctl data
		1496	* @file: GEM object info
		1497	*
		1498	* Simply returns the fake offset to userspace so it can mmap it.
		1499	* The mmap call will end up in drm_gem_mmap(), which will set things
		1500	* up so we can get faults in the handler above.
		1501	*
		1502	* The fault handler will take care of binding the object into the GTT
		1503	* (since it may have been evicted to make room for something), allocating
		1504	* a fence register, and mapping the appropriate aperture address into
		1505	* userspace.
		1506	*/
		1507	int
		1508	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1509	struct drm_file *file)
		1510	{
		1511	struct drm_i915_gem_mmap_gtt *args = data;
		1512
		1513	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1514	}
		1515
3031	serge	1516	/* Immediately discard the backing storage */
		1517	static void
		1518	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1519	{
		1520	// struct inode *inode;
2332	Serge	1521
3031	serge	1522	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1523
3263	Serge	1524	if (obj->base.filp == NULL)
		1525	return;
2332	Serge	1526
3031	serge	1527	/* Our goal here is to return as much of the memory as
		1528	* is possible back to the system as we are called from OOM.
		1529	* To do this we must instruct the shmfs to drop all of its
		1530	* backing pages, now.
		1531	*/
		1532	// inode = obj->base.filp->f_path.dentry->d_inode;
		1533	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1534
3031	serge	1535	obj->madv = __I915_MADV_PURGED;
		1536	}
2332	Serge	1537
3031	serge	1538	static inline int
		1539	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1540	{
		1541	return obj->madv == I915_MADV_DONTNEED;
		1542	}
2332	Serge	1543
3031	serge	1544	static void
		1545	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1546	{
3746	Serge	1547	struct sg_page_iter sg_iter;
		1548	int ret;
2332	Serge	1549
3031	serge	1550	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1551
3031	serge	1552	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1553	if (ret) {
		1554	/* In the event of a disaster, abandon all caches and
		1555	* hope for the best.
		1556	*/
		1557	WARN_ON(ret != -EIO);
4104	Serge	1558	i915_gem_clflush_object(obj, true);
3031	serge	1559	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1560	}
2332	Serge	1561
3031	serge	1562	if (obj->madv == I915_MADV_DONTNEED)
		1563	obj->dirty = 0;
2332	Serge	1564
3746	Serge	1565	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1566	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1567
3290	Serge	1568	page_cache_release(page);
3243	Serge	1569	}
		1570	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1571
4104	Serge	1572	obj->dirty = 0;
3243	Serge	1573
		1574	sg_free_table(obj->pages);
		1575	kfree(obj->pages);
3031	serge	1576	}
2332	Serge	1577
3480	Serge	1578	int
3031	serge	1579	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1580	{
		1581	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1582
3243	Serge	1583	if (obj->pages == NULL)
3031	serge	1584	return 0;
2332	Serge	1585
3031	serge	1586	if (obj->pages_pin_count)
		1587	return -EBUSY;
		1588
4104	Serge	1589	BUG_ON(i915_gem_obj_bound_any(obj));
		1590
3243	Serge	1591	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1592	* array, hence protect them from being reaped by removing them from gtt
		1593	* lists early. */
4104	Serge	1594	list_del(&obj->global_list);
3243	Serge	1595
3031	serge	1596	ops->put_pages(obj);
3243	Serge	1597	obj->pages = NULL;
3031	serge	1598
		1599	if (i915_gem_object_is_purgeable(obj))
		1600	i915_gem_object_truncate(obj);
		1601
		1602	return 0;
		1603	}
		1604
		1605
		1606
		1607
		1608
		1609
		1610
		1611
2332	Serge	1612	static int
3031	serge	1613	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1614	{
3260	Serge	1615	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1616	int page_count, i;
4104	Serge	1617	struct sg_table *st;
3243	Serge	1618	struct scatterlist *sg;
3746	Serge	1619	struct sg_page_iter sg_iter;
3243	Serge	1620	struct page *page;
3746	Serge	1621	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1622	gfp_t gfp;
2332	Serge	1623
3243	Serge	1624	/* Assert that the object is not currently in any GPU domain. As it
		1625	* wasn't in the GTT, there shouldn't be any way it could have been in
		1626	* a GPU cache
2332	Serge	1627	*/
3243	Serge	1628	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1629	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1630
		1631	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1632	if (st == NULL)
		1633	return -ENOMEM;
		1634
2332	Serge	1635	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1636	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1637	kfree(st);
3746	Serge	1638	FAIL();
2332	Serge	1639	return -ENOMEM;
3243	Serge	1640	}
2332	Serge	1641
3243	Serge	1642	/* Get the list of pages out of our struct file. They'll be pinned
		1643	* at this point until we release them.
		1644	*
		1645	* Fail silently without starting the shrinker
		1646	*/
3746	Serge	1647	sg = st->sgl;
		1648	st->nents = 0;
		1649	for (i = 0; i < page_count; i++) {
4104	Serge	1650	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1651	if (IS_ERR(page)) {
		1652	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1653	goto err_pages;
		1654
3260	Serge	1655	}
3746	Serge	1656
		1657	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1658	if (i)
		1659	sg = sg_next(sg);
		1660	st->nents++;
3243	Serge	1661	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1662	} else {
		1663	sg->length += PAGE_SIZE;
		1664	}
		1665	last_pfn = page_to_pfn(page);
3243	Serge	1666	}
3031	serge	1667
3746	Serge	1668	sg_mark_end(sg);
3243	Serge	1669	obj->pages = st;
3031	serge	1670
2332	Serge	1671	return 0;
		1672
		1673	err_pages:
3746	Serge	1674	sg_mark_end(sg);
		1675	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1676	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1677	sg_free_table(st);
		1678	kfree(st);
3746	Serge	1679	FAIL();
3243	Serge	1680	return PTR_ERR(page);
2332	Serge	1681	}
		1682
3031	serge	1683	/* Ensure that the associated pages are gathered from the backing storage
		1684	* and pinned into our object. i915_gem_object_get_pages() may be called
		1685	* multiple times before they are released by a single call to
		1686	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1687	* either as a result of memory pressure (reaping pages under the shrinker)
		1688	* or as the object is itself released.
		1689	*/
		1690	int
		1691	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1692	{
3031	serge	1693	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1694	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1695	int ret;
2332	Serge	1696
3243	Serge	1697	if (obj->pages)
3031	serge	1698	return 0;
2332	Serge	1699
4392	Serge	1700	if (obj->madv != I915_MADV_WILLNEED) {
		1701	DRM_ERROR("Attempting to obtain a purgeable object\n");
		1702	return -EINVAL;
		1703	}
		1704
3031	serge	1705	BUG_ON(obj->pages_pin_count);
2332	Serge	1706
3031	serge	1707	ret = ops->get_pages(obj);
		1708	if (ret)
		1709	return ret;
2344	Serge	1710
4104	Serge	1711	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1712	return 0;
2332	Serge	1713	}
		1714
		1715	void
		1716	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1717	struct intel_ring_buffer *ring)
2332	Serge	1718	{
		1719	struct drm_device *dev = obj->base.dev;
		1720	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1721	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1722
		1723	BUG_ON(ring == NULL);
4104	Serge	1724	if (obj->ring != ring && obj->last_write_seqno) {
		1725	/* Keep the seqno relative to the current ring */
		1726	obj->last_write_seqno = seqno;
		1727	}
2332	Serge	1728	obj->ring = ring;
		1729
		1730	/* Add a reference if we're newly entering the active list. */
		1731	if (!obj->active) {
2344	Serge	1732	drm_gem_object_reference(&obj->base);
2332	Serge	1733	obj->active = 1;
		1734	}
		1735
		1736	list_move_tail(&obj->ring_list, &ring->active_list);
		1737
3031	serge	1738	obj->last_read_seqno = seqno;
		1739
2332	Serge	1740	if (obj->fenced_gpu_access) {
3031	serge	1741	obj->last_fenced_seqno = seqno;
		1742
		1743	/* Bump MRU to take account of the delayed flush */
		1744	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1745	struct drm_i915_fence_reg *reg;
		1746
		1747	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1748	list_move_tail(®->lru_list,
		1749	&dev_priv->mm.fence_list);
		1750	}
2332	Serge	1751	}
		1752	}
		1753
2344	Serge	1754	static void
3031	serge	1755	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1756	{
4104	Serge	1757	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1758	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
		1759	struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2332	Serge	1760
3031	serge	1761	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1762	BUG_ON(!obj->active);
2332	Serge	1763
4104	Serge	1764	list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
2344	Serge	1765
3031	serge	1766	list_del_init(&obj->ring_list);
2352	Serge	1767	obj->ring = NULL;
2344	Serge	1768
3031	serge	1769	obj->last_read_seqno = 0;
		1770	obj->last_write_seqno = 0;
		1771	obj->base.write_domain = 0;
		1772
		1773	obj->last_fenced_seqno = 0;
2352	Serge	1774	obj->fenced_gpu_access = false;
2344	Serge	1775
2352	Serge	1776	obj->active = 0;
		1777	drm_gem_object_unreference(&obj->base);
		1778
		1779	WARN_ON(i915_verify_lists(dev));
		1780	}
		1781
3243	Serge	1782	static int
3480	Serge	1783	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1784	{
3243	Serge	1785	struct drm_i915_private *dev_priv = dev->dev_private;
		1786	struct intel_ring_buffer *ring;
		1787	int ret, i, j;
2344	Serge	1788
3480	Serge	1789	/* Carefully retire all requests without writing to the rings */
3243	Serge	1790	for_each_ring(ring, dev_priv, i) {
3480	Serge	1791	ret = intel_ring_idle(ring);
3243	Serge	1792	if (ret)
		1793	return ret;
3480	Serge	1794	}
		1795	i915_gem_retire_requests(dev);
3243	Serge	1796
3480	Serge	1797	/* Finally reset hw state */
3243	Serge	1798	for_each_ring(ring, dev_priv, i) {
3480	Serge	1799	intel_ring_init_seqno(ring, seqno);
		1800
3243	Serge	1801	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1802	ring->sync_seqno[j] = 0;
		1803	}
		1804
		1805	return 0;
2344	Serge	1806	}
		1807
3480	Serge	1808	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1809	{
		1810	struct drm_i915_private *dev_priv = dev->dev_private;
		1811	int ret;
		1812
		1813	if (seqno == 0)
		1814	return -EINVAL;
		1815
		1816	/* HWS page needs to be set less than what we
		1817	* will inject to ring
		1818	*/
		1819	ret = i915_gem_init_seqno(dev, seqno - 1);
		1820	if (ret)
		1821	return ret;
		1822
		1823	/* Carefully set the last_seqno value so that wrap
		1824	* detection still works
		1825	*/
		1826	dev_priv->next_seqno = seqno;
		1827	dev_priv->last_seqno = seqno - 1;
		1828	if (dev_priv->last_seqno == 0)
		1829	dev_priv->last_seqno--;
		1830
		1831	return 0;
		1832	}
		1833
3243	Serge	1834	int
		1835	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1836	{
3243	Serge	1837	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1838
3243	Serge	1839	/* reserve 0 for non-seqno */
		1840	if (dev_priv->next_seqno == 0) {
3480	Serge	1841	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1842	if (ret)
		1843	return ret;
		1844
		1845	dev_priv->next_seqno = 1;
		1846	}
		1847
3480	Serge	1848	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1849	return 0;
2332	Serge	1850	}
		1851
4104	Serge	1852	int __i915_add_request(struct intel_ring_buffer *ring,
2352	Serge	1853	struct drm_file *file,
4104	Serge	1854	struct drm_i915_gem_object *obj,
3031	serge	1855	u32 *out_seqno)
2352	Serge	1856	{
		1857	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1858	struct drm_i915_gem_request *request;
4104	Serge	1859	u32 request_ring_position, request_start;
2352	Serge	1860	int was_empty;
		1861	int ret;
2332	Serge	1862
4104	Serge	1863	request_start = intel_ring_get_tail(ring);
3031	serge	1864	/*
		1865	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1866	* after having emitted the batchbuffer command. Hence we need to fix
		1867	* things up similar to emitting the lazy request. The difference here
		1868	* is that the flush _must_ happen before the next request, no matter
		1869	* what.
		1870	*/
4104	Serge	1871	ret = intel_ring_flush_all_caches(ring);
		1872	if (ret)
		1873	return ret;
2332	Serge	1874
3031	serge	1875	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1876	if (request == NULL)
		1877	return -ENOMEM;
		1878
		1879
		1880	/* Record the position of the start of the request so that
		1881	* should we detect the updated seqno part-way through the
4104	Serge	1882	* GPU processing the request, we never over-estimate the
3031	serge	1883	* position of the head.
		1884	*/
4104	Serge	1885	request_ring_position = intel_ring_get_tail(ring);
3031	serge	1886
3243	Serge	1887	ret = ring->add_request(ring);
3031	serge	1888	if (ret) {
		1889	kfree(request);
4104	Serge	1890	return ret;
3031	serge	1891	}
2332	Serge	1892
3243	Serge	1893	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1894	request->ring = ring;
4104	Serge	1895	request->head = request_start;
3031	serge	1896	request->tail = request_ring_position;
4104	Serge	1897	request->ctx = ring->last_context;
		1898	request->batch_obj = obj;
		1899
		1900	/* Whilst this request exists, batch_obj will be on the
		1901	* active_list, and so will hold the active reference. Only when this
		1902	* request is retired will the the batch_obj be moved onto the
		1903	* inactive_list and lose its active reference. Hence we do not need
		1904	* to explicitly hold another reference here.
		1905	*/
		1906
		1907	if (request->ctx)
		1908	i915_gem_context_reference(request->ctx);
		1909
3031	serge	1910	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1911	was_empty = list_empty(&ring->request_list);
		1912	list_add_tail(&request->list, &ring->request_list);
3031	serge	1913	request->file_priv = NULL;
2332	Serge	1914
3263	Serge	1915	if (file) {
		1916	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1917
3263	Serge	1918	spin_lock(&file_priv->mm.lock);
		1919	request->file_priv = file_priv;
		1920	list_add_tail(&request->client_list,
		1921	&file_priv->mm.request_list);
		1922	spin_unlock(&file_priv->mm.lock);
		1923	}
		1924
		1925	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1926	ring->outstanding_lazy_request = 0;
2332	Serge	1927
4104	Serge	1928	if (!dev_priv->ums.mm_suspended) {
		1929	// i915_queue_hangcheck(ring->dev);
		1930
		1931	if (was_empty) {
2360	Serge	1932	queue_delayed_work(dev_priv->wq,
3482	Serge	1933	&dev_priv->mm.retire_work,
		1934	round_jiffies_up_relative(HZ));
4104	Serge	1935	intel_mark_busy(dev_priv->dev);
		1936	}
		1937	}
3031	serge	1938
		1939	if (out_seqno)
3243	Serge	1940	*out_seqno = request->seqno;
2352	Serge	1941	return 0;
		1942	}
2332	Serge	1943
3263	Serge	1944	static inline void
		1945	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1946	{
		1947	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1948
3263	Serge	1949	if (!file_priv)
		1950	return;
2332	Serge	1951
3263	Serge	1952	spin_lock(&file_priv->mm.lock);
		1953	if (request->file_priv) {
		1954	list_del(&request->client_list);
		1955	request->file_priv = NULL;
		1956	}
		1957	spin_unlock(&file_priv->mm.lock);
		1958	}
2332	Serge	1959
4104	Serge	1960	static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
		1961	struct i915_address_space *vm)
		1962	{
		1963	if (acthd >= i915_gem_obj_offset(obj, vm) &&
		1964	acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
		1965	return true;
		1966
		1967	return false;
		1968	}
		1969
		1970	static bool i915_head_inside_request(const u32 acthd_unmasked,
		1971	const u32 request_start,
		1972	const u32 request_end)
		1973	{
		1974	const u32 acthd = acthd_unmasked & HEAD_ADDR;
		1975
		1976	if (request_start < request_end) {
		1977	if (acthd >= request_start && acthd < request_end)
		1978	return true;
		1979	} else if (request_start > request_end) {
		1980	if (acthd >= request_start \|\| acthd < request_end)
		1981	return true;
		1982	}
		1983
		1984	return false;
		1985	}
		1986
		1987	static struct i915_address_space *
		1988	request_to_vm(struct drm_i915_gem_request *request)
		1989	{
		1990	struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
		1991	struct i915_address_space *vm;
		1992
		1993	vm = &dev_priv->gtt.base;
		1994
		1995	return vm;
		1996	}
		1997
		1998	static bool i915_request_guilty(struct drm_i915_gem_request *request,
		1999	const u32 acthd, bool *inside)
		2000	{
		2001	/* There is a possibility that unmasked head address
		2002	* pointing inside the ring, matches the batch_obj address range.
		2003	* However this is extremely unlikely.
		2004	*/
		2005	if (request->batch_obj) {
		2006	if (i915_head_inside_object(acthd, request->batch_obj,
		2007	request_to_vm(request))) {
		2008	*inside = true;
		2009	return true;
		2010	}
		2011	}
		2012
		2013	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		2014	*inside = false;
		2015	return true;
		2016	}
		2017
		2018	return false;
		2019	}
		2020
		2021	static void i915_set_reset_status(struct intel_ring_buffer *ring,
		2022	struct drm_i915_gem_request *request,
		2023	u32 acthd)
		2024	{
		2025	struct i915_ctx_hang_stats *hs = NULL;
		2026	bool inside, guilty;
		2027	unsigned long offset = 0;
		2028
		2029	/* Innocent until proven guilty */
		2030	guilty = false;
		2031
		2032	if (request->batch_obj)
		2033	offset = i915_gem_obj_offset(request->batch_obj,
		2034	request_to_vm(request));
		2035
		2036	if (ring->hangcheck.action != HANGCHECK_WAIT &&
		2037	i915_request_guilty(request, acthd, &inside)) {
		2038	DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
		2039	ring->name,
		2040	inside ? "inside" : "flushing",
		2041	offset,
		2042	request->ctx ? request->ctx->id : 0,
		2043	acthd);
		2044
		2045	guilty = true;
		2046	}
		2047
		2048	/* If contexts are disabled or this is the default context, use
		2049	* file_priv->reset_state
		2050	*/
		2051	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		2052	hs = &request->ctx->hang_stats;
		2053	else if (request->file_priv)
		2054	hs = &request->file_priv->hang_stats;
		2055
		2056	if (hs) {
		2057	if (guilty)
		2058	hs->batch_active++;
		2059	else
		2060	hs->batch_pending++;
		2061	}
		2062	}
		2063
		2064	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2065	{
		2066	list_del(&request->list);
		2067	i915_gem_request_remove_from_client(request);
		2068
		2069	if (request->ctx)
		2070	i915_gem_context_unreference(request->ctx);
		2071
		2072	kfree(request);
		2073	}
		2074
4539	Serge	2075	static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
3031	serge	2076	struct intel_ring_buffer *ring)
		2077	{
4539	Serge	2078	u32 completed_seqno = ring->get_seqno(ring, false);
		2079	u32 acthd = intel_ring_get_active_head(ring);
		2080	struct drm_i915_gem_request *request;
4104	Serge	2081
4539	Serge	2082	list_for_each_entry(request, &ring->request_list, list) {
		2083	if (i915_seqno_passed(completed_seqno, request->seqno))
		2084	continue;
4104	Serge	2085
4539	Serge	2086	i915_set_reset_status(ring, request, acthd);
		2087	}
		2088	}
		2089
		2090	static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
		2091	struct intel_ring_buffer *ring)
		2092	{
3031	serge	2093	while (!list_empty(&ring->request_list)) {
		2094	struct drm_i915_gem_request *request;
2332	Serge	2095
3031	serge	2096	request = list_first_entry(&ring->request_list,
		2097	struct drm_i915_gem_request,
		2098	list);
2332	Serge	2099
4104	Serge	2100	i915_gem_free_request(request);
3031	serge	2101	}
2332	Serge	2102
3031	serge	2103	while (!list_empty(&ring->active_list)) {
		2104	struct drm_i915_gem_object *obj;
2332	Serge	2105
3031	serge	2106	obj = list_first_entry(&ring->active_list,
		2107	struct drm_i915_gem_object,
		2108	ring_list);
2332	Serge	2109
3031	serge	2110	i915_gem_object_move_to_inactive(obj);
		2111	}
		2112	}
2332	Serge	2113
3746	Serge	2114	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2115	{
		2116	struct drm_i915_private *dev_priv = dev->dev_private;
		2117	int i;
2332	Serge	2118
3031	serge	2119	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2120	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2121
		2122	/*
		2123	* Commit delayed tiling changes if we have an object still
		2124	* attached to the fence, otherwise just clear the fence.
		2125	*/
		2126	if (reg->obj) {
		2127	i915_gem_object_update_fence(reg->obj, reg,
		2128	reg->obj->tiling_mode);
		2129	} else {
		2130	i915_gem_write_fence(dev, i, NULL);
		2131	}
3031	serge	2132	}
		2133	}
2360	Serge	2134
3031	serge	2135	void i915_gem_reset(struct drm_device *dev)
		2136	{
		2137	struct drm_i915_private *dev_priv = dev->dev_private;
		2138	struct intel_ring_buffer *ring;
		2139	int i;
2360	Serge	2140
4539	Serge	2141	/*
		2142	* Before we free the objects from the requests, we need to inspect
		2143	* them for finding the guilty party. As the requests only borrow
		2144	* their reference to the objects, the inspection must be done first.
		2145	*/
3031	serge	2146	for_each_ring(ring, dev_priv, i)
4539	Serge	2147	i915_gem_reset_ring_status(dev_priv, ring);
2360	Serge	2148
4539	Serge	2149	for_each_ring(ring, dev_priv, i)
		2150	i915_gem_reset_ring_cleanup(dev_priv, ring);
		2151
3746	Serge	2152	i915_gem_restore_fences(dev);
3031	serge	2153	}
2360	Serge	2154
2352	Serge	2155	/**
		2156	* This function clears the request list as sequence numbers are passed.
		2157	*/
3031	serge	2158	void
2352	Serge	2159	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2160	{
		2161	uint32_t seqno;
2332	Serge	2162
2352	Serge	2163	if (list_empty(&ring->request_list))
		2164	return;
2332	Serge	2165
2352	Serge	2166	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2167
3031	serge	2168	seqno = ring->get_seqno(ring, true);
2332	Serge	2169
2352	Serge	2170	while (!list_empty(&ring->request_list)) {
		2171	struct drm_i915_gem_request *request;
2332	Serge	2172
2352	Serge	2173	request = list_first_entry(&ring->request_list,
		2174	struct drm_i915_gem_request,
		2175	list);
2332	Serge	2176
2352	Serge	2177	if (!i915_seqno_passed(seqno, request->seqno))
		2178	break;
2332	Serge	2179
2352	Serge	2180	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2181	/* We know the GPU must have read the request to have
		2182	* sent us the seqno + interrupt, so use the position
		2183	* of tail of the request to update the last known position
		2184	* of the GPU head.
		2185	*/
		2186	ring->last_retired_head = request->tail;
2332	Serge	2187
4104	Serge	2188	i915_gem_free_request(request);
2352	Serge	2189	}
2332	Serge	2190
2352	Serge	2191	/* Move any buffers on the active list that are no longer referenced
		2192	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2193	*/
		2194	while (!list_empty(&ring->active_list)) {
		2195	struct drm_i915_gem_object *obj;
2332	Serge	2196
2352	Serge	2197	obj = list_first_entry(&ring->active_list,
		2198	struct drm_i915_gem_object,
		2199	ring_list);
2332	Serge	2200
3031	serge	2201	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2202	break;
2332	Serge	2203
2352	Serge	2204	i915_gem_object_move_to_inactive(obj);
		2205	}
2332	Serge	2206
2352	Serge	2207	if (unlikely(ring->trace_irq_seqno &&
		2208	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2209	ring->irq_put(ring);
		2210	ring->trace_irq_seqno = 0;
		2211	}
2332	Serge	2212
2352	Serge	2213	WARN_ON(i915_verify_lists(ring->dev));
		2214	}
2332	Serge	2215
2352	Serge	2216	void
		2217	i915_gem_retire_requests(struct drm_device *dev)
		2218	{
		2219	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2220	struct intel_ring_buffer *ring;
2352	Serge	2221	int i;
2332	Serge	2222
3031	serge	2223	for_each_ring(ring, dev_priv, i)
		2224	i915_gem_retire_requests_ring(ring);
2352	Serge	2225	}
		2226
2360	Serge	2227	static void
		2228	i915_gem_retire_work_handler(struct work_struct *work)
		2229	{
		2230	drm_i915_private_t *dev_priv;
		2231	struct drm_device *dev;
3031	serge	2232	struct intel_ring_buffer *ring;
2360	Serge	2233	bool idle;
		2234	int i;
2352	Serge	2235
2360	Serge	2236	dev_priv = container_of(work, drm_i915_private_t,
		2237	mm.retire_work.work);
		2238	dev = dev_priv->dev;
2352	Serge	2239
2360	Serge	2240	/* Come back later if the device is busy... */
		2241	if (!mutex_trylock(&dev->struct_mutex)) {
3482	Serge	2242	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2243	round_jiffies_up_relative(HZ));
3243	Serge	2244	return;
2360	Serge	2245	}
2352	Serge	2246
2360	Serge	2247	i915_gem_retire_requests(dev);
2352	Serge	2248
2360	Serge	2249	/* Send a periodic flush down the ring so we don't hold onto GEM
		2250	* objects indefinitely.
		2251	*/
		2252	idle = true;
3031	serge	2253	for_each_ring(ring, dev_priv, i) {
		2254	if (ring->gpu_caches_dirty)
4104	Serge	2255	i915_add_request(ring, NULL);
2352	Serge	2256
2360	Serge	2257	idle &= list_empty(&ring->request_list);
		2258	}
2352	Serge	2259
4104	Serge	2260	if (!dev_priv->ums.mm_suspended && !idle)
3482	Serge	2261	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2262	round_jiffies_up_relative(HZ));
3031	serge	2263	if (idle)
		2264	intel_mark_idle(dev);
2360	Serge	2265
		2266	mutex_unlock(&dev->struct_mutex);
		2267	}
		2268
2344	Serge	2269	/**
3031	serge	2270	* Ensures that an object will eventually get non-busy by flushing any required
		2271	* write domains, emitting any outstanding lazy request and retiring and
		2272	* completed requests.
2352	Serge	2273	*/
3031	serge	2274	static int
		2275	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2276	{
3031	serge	2277	int ret;
2352	Serge	2278
3031	serge	2279	if (obj->active) {
		2280	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2281	if (ret)
		2282	return ret;
2352	Serge	2283
3031	serge	2284	i915_gem_retire_requests_ring(obj->ring);
		2285	}
2352	Serge	2286
3031	serge	2287	return 0;
		2288	}
2352	Serge	2289
3243	Serge	2290	/**
		2291	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2292	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2293	*
		2294	* Returns 0 if successful, else an error is returned with the remaining time in
		2295	* the timeout parameter.
		2296	* -ETIME: object is still busy after timeout
		2297	* -ERESTARTSYS: signal interrupted the wait
		2298	* -ENONENT: object doesn't exist
		2299	* Also possible, but rare:
		2300	* -EAGAIN: GPU wedged
		2301	* -ENOMEM: damn
		2302	* -ENODEV: Internal IRQ fail
		2303	* -E?: The add request failed
		2304	*
		2305	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2306	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2307	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2308	* without holding struct_mutex the object may become re-busied before this
		2309	* function completes. A similar but shorter * race condition exists in the busy
		2310	* ioctl
		2311	*/
4246	Serge	2312	int
		2313	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2314	{
		2315	drm_i915_private_t *dev_priv = dev->dev_private;
		2316	struct drm_i915_gem_wait *args = data;
		2317	struct drm_i915_gem_object *obj;
		2318	struct intel_ring_buffer *ring = NULL;
		2319	struct timespec timeout_stack, *timeout = NULL;
		2320	unsigned reset_counter;
		2321	u32 seqno = 0;
		2322	int ret = 0;
2352	Serge	2323
4246	Serge	2324	if (args->timeout_ns >= 0) {
		2325	timeout_stack = ns_to_timespec(args->timeout_ns);
		2326	timeout = &timeout_stack;
		2327	}
2352	Serge	2328
4246	Serge	2329	ret = i915_mutex_lock_interruptible(dev);
		2330	if (ret)
		2331	return ret;
2352	Serge	2332
4246	Serge	2333	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2334	if (&obj->base == NULL) {
		2335	mutex_unlock(&dev->struct_mutex);
		2336	return -ENOENT;
		2337	}
2352	Serge	2338
4246	Serge	2339	/* Need to make sure the object gets inactive eventually. */
		2340	ret = i915_gem_object_flush_active(obj);
		2341	if (ret)
		2342	goto out;
2352	Serge	2343
4246	Serge	2344	if (obj->active) {
		2345	seqno = obj->last_read_seqno;
		2346	ring = obj->ring;
		2347	}
2352	Serge	2348
4246	Serge	2349	if (seqno == 0)
		2350	goto out;
2352	Serge	2351
4246	Serge	2352	/* Do this after OLR check to make sure we make forward progress polling
		2353	* on this IOCTL with a 0 timeout (like busy ioctl)
		2354	*/
		2355	if (!args->timeout_ns) {
		2356	ret = -ETIME;
		2357	goto out;
		2358	}
2352	Serge	2359
4246	Serge	2360	drm_gem_object_unreference(&obj->base);
		2361	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		2362	mutex_unlock(&dev->struct_mutex);
2352	Serge	2363
4246	Serge	2364	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
		2365	if (timeout)
		2366	args->timeout_ns = timespec_to_ns(timeout);
		2367	return ret;
3243	Serge	2368
4246	Serge	2369	out:
		2370	drm_gem_object_unreference(&obj->base);
		2371	mutex_unlock(&dev->struct_mutex);
		2372	return ret;
		2373	}
3243	Serge	2374
2352	Serge	2375	/**
3031	serge	2376	* i915_gem_object_sync - sync an object to a ring.
		2377	*
		2378	* @obj: object which may be in use on another ring.
		2379	* @to: ring we wish to use the object on. May be NULL.
		2380	*
		2381	* This code is meant to abstract object synchronization with the GPU.
		2382	* Calling with NULL implies synchronizing the object with the CPU
		2383	* rather than a particular GPU ring.
		2384	*
		2385	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2386	*/
		2387	int
3031	serge	2388	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2389	struct intel_ring_buffer *to)
2344	Serge	2390	{
3031	serge	2391	struct intel_ring_buffer *from = obj->ring;
		2392	u32 seqno;
		2393	int ret, idx;
2332	Serge	2394
3031	serge	2395	if (from == NULL \|\| to == from)
		2396	return 0;
2332	Serge	2397
3031	serge	2398	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2399	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2400
3031	serge	2401	idx = intel_ring_sync_index(from, to);
		2402
		2403	seqno = obj->last_read_seqno;
		2404	if (seqno <= from->sync_seqno[idx])
		2405	return 0;
		2406
		2407	ret = i915_gem_check_olr(obj->ring, seqno);
		2408	if (ret)
		2409	return ret;
		2410
		2411	ret = to->sync_to(to, from, seqno);
		2412	if (!ret)
3243	Serge	2413	/* We use last_read_seqno because sync_to()
		2414	* might have just caused seqno wrap under
		2415	* the radar.
		2416	*/
		2417	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2418
		2419	return ret;
2344	Serge	2420	}
2332	Serge	2421
2344	Serge	2422	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2423	{
		2424	u32 old_write_domain, old_read_domains;
2332	Serge	2425
2344	Serge	2426	/* Force a pagefault for domain tracking on next user access */
		2427	// i915_gem_release_mmap(obj);
2332	Serge	2428
2344	Serge	2429	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2430	return;
2332	Serge	2431
3480	Serge	2432	/* Wait for any direct GTT access to complete */
		2433	mb();
		2434
2344	Serge	2435	old_read_domains = obj->base.read_domains;
		2436	old_write_domain = obj->base.write_domain;
2351	Serge	2437
2344	Serge	2438	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2439	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2440
2351	Serge	2441	trace_i915_gem_object_change_domain(obj,
		2442	old_read_domains,
		2443	old_write_domain);
2344	Serge	2444	}
2332	Serge	2445
4104	Serge	2446	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2447	{
4104	Serge	2448	struct drm_i915_gem_object *obj = vma->obj;
3031	serge	2449	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2450	int ret;
2332	Serge	2451
3263	Serge	2452	if(obj == get_fb_obj())
		2453	return 0;
		2454
4104	Serge	2455	if (list_empty(&vma->vma_link))
2344	Serge	2456	return 0;
2332	Serge	2457
4104	Serge	2458	if (!drm_mm_node_allocated(&vma->node))
		2459	goto destroy;
		2460
3031	serge	2461	if (obj->pin_count)
		2462	return -EBUSY;
2332	Serge	2463
3243	Serge	2464	BUG_ON(obj->pages == NULL);
3031	serge	2465
2344	Serge	2466	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2467	if (ret)
2344	Serge	2468	return ret;
		2469	/* Continue on if we fail due to EIO, the GPU is hung so we
		2470	* should be safe and we need to cleanup or else we might
		2471	* cause memory corruption through use-after-free.
		2472	*/
2332	Serge	2473
2344	Serge	2474	i915_gem_object_finish_gtt(obj);
2332	Serge	2475
2344	Serge	2476	/* release the fence reg _after_ flushing */
		2477	ret = i915_gem_object_put_fence(obj);
3031	serge	2478	if (ret)
2344	Serge	2479	return ret;
2332	Serge	2480
4104	Serge	2481	trace_i915_vma_unbind(vma);
2332	Serge	2482
3031	serge	2483	if (obj->has_global_gtt_mapping)
3243	Serge	2484	i915_gem_gtt_unbind_object(obj);
3031	serge	2485	if (obj->has_aliasing_ppgtt_mapping) {
		2486	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2487	obj->has_aliasing_ppgtt_mapping = 0;
		2488	}
		2489	i915_gem_gtt_finish_object(obj);
4104	Serge	2490	i915_gem_object_unpin_pages(obj);
2332	Serge	2491
4104	Serge	2492	list_del(&vma->mm_list);
2344	Serge	2493	/* Avoid an unnecessary call to unbind on rebind. */
4104	Serge	2494	if (i915_is_ggtt(vma->vm))
2344	Serge	2495	obj->map_and_fenceable = true;
2332	Serge	2496
4104	Serge	2497	drm_mm_remove_node(&vma->node);
2332	Serge	2498
4104	Serge	2499	destroy:
		2500	i915_gem_vma_destroy(vma);
		2501
		2502	/* Since the unbound list is global, only move to that list if
		2503	* no more VMAs exist.
		2504	* NB: Until we have real VMAs there will only ever be one */
		2505	WARN_ON(!list_empty(&obj->vma_list));
		2506	if (list_empty(&obj->vma_list))
		2507	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2508
2344	Serge	2509	return 0;
		2510	}
2332	Serge	2511
4104	Serge	2512	/**
		2513	* Unbinds an object from the global GTT aperture.
		2514	*/
		2515	int
		2516	i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
		2517	{
		2518	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2519	struct i915_address_space *ggtt = &dev_priv->gtt.base;
		2520
		2521	if (!i915_gem_obj_ggtt_bound(obj))
		2522	return 0;
		2523
		2524	if (obj->pin_count)
		2525	return -EBUSY;
		2526
		2527	BUG_ON(obj->pages == NULL);
		2528
		2529	return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
		2530	}
		2531
3031	serge	2532	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2533	{
		2534	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2535	struct intel_ring_buffer *ring;
2344	Serge	2536	int ret, i;
2332	Serge	2537
2344	Serge	2538	/* Flush everything onto the inactive list. */
3031	serge	2539	for_each_ring(ring, dev_priv, i) {
		2540	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2541	if (ret)
		2542	return ret;
3031	serge	2543
3243	Serge	2544	ret = intel_ring_idle(ring);
3031	serge	2545	if (ret)
		2546	return ret;
2344	Serge	2547	}
2332	Serge	2548
2344	Serge	2549	return 0;
		2550	}
2332	Serge	2551
3480	Serge	2552	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2553	struct drm_i915_gem_object *obj)
		2554	{
		2555	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2556	int fence_reg;
		2557	int fence_pitch_shift;
2332	Serge	2558
3480	Serge	2559	if (INTEL_INFO(dev)->gen >= 6) {
		2560	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2561	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2562	} else {
		2563	fence_reg = FENCE_REG_965_0;
		2564	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2565	}
2332	Serge	2566
4104	Serge	2567	fence_reg += reg * 8;
		2568
		2569	/* To w/a incoherency with non-atomic 64-bit register updates,
		2570	* we split the 64-bit update into two 32-bit writes. In order
		2571	* for a partial fence not to be evaluated between writes, we
		2572	* precede the update with write to turn off the fence register,
		2573	* and only enable the fence as the last step.
		2574	*
		2575	* For extra levels of paranoia, we make sure each step lands
		2576	* before applying the next step.
		2577	*/
		2578	I915_WRITE(fence_reg, 0);
		2579	POSTING_READ(fence_reg);
		2580
3031	serge	2581	if (obj) {
4104	Serge	2582	u32 size = i915_gem_obj_ggtt_size(obj);
		2583	uint64_t val;
2332	Serge	2584
4104	Serge	2585	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2586	0xfffff000) << 32;
4104	Serge	2587	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2588	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2589	if (obj->tiling_mode == I915_TILING_Y)
		2590	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2591	val \|= I965_FENCE_REG_VALID;
2332	Serge	2592
4104	Serge	2593	I915_WRITE(fence_reg + 4, val >> 32);
		2594	POSTING_READ(fence_reg + 4);
		2595
		2596	I915_WRITE(fence_reg + 0, val);
3480	Serge	2597	POSTING_READ(fence_reg);
4104	Serge	2598	} else {
		2599	I915_WRITE(fence_reg + 4, 0);
		2600	POSTING_READ(fence_reg + 4);
		2601	}
3031	serge	2602	}
2332	Serge	2603
3031	serge	2604	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2605	struct drm_i915_gem_object *obj)
		2606	{
		2607	drm_i915_private_t *dev_priv = dev->dev_private;
		2608	u32 val;
2332	Serge	2609
3031	serge	2610	if (obj) {
4104	Serge	2611	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2612	int pitch_val;
		2613	int tile_width;
2332	Serge	2614
4104	Serge	2615	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2616	(size & -size) != size \|\|
4104	Serge	2617	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2618	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2619	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2620
3031	serge	2621	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2622	tile_width = 128;
		2623	else
		2624	tile_width = 512;
2332	Serge	2625
3031	serge	2626	/* Note: pitch better be a power of two tile widths */
		2627	pitch_val = obj->stride / tile_width;
		2628	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2629
4104	Serge	2630	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2631	if (obj->tiling_mode == I915_TILING_Y)
		2632	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2633	val \|= I915_FENCE_SIZE_BITS(size);
		2634	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2635	val \|= I830_FENCE_REG_VALID;
		2636	} else
		2637	val = 0;
2332	Serge	2638
3031	serge	2639	if (reg < 8)
		2640	reg = FENCE_REG_830_0 + reg * 4;
		2641	else
		2642	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2643
3031	serge	2644	I915_WRITE(reg, val);
		2645	POSTING_READ(reg);
		2646	}
2332	Serge	2647
3031	serge	2648	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2649	struct drm_i915_gem_object *obj)
		2650	{
		2651	drm_i915_private_t *dev_priv = dev->dev_private;
		2652	uint32_t val;
2344	Serge	2653
3031	serge	2654	if (obj) {
4104	Serge	2655	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2656	uint32_t pitch_val;
2344	Serge	2657
4104	Serge	2658	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2659	(size & -size) != size \|\|
4104	Serge	2660	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2661	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2662	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2663
3031	serge	2664	pitch_val = obj->stride / 128;
		2665	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2666
4104	Serge	2667	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2668	if (obj->tiling_mode == I915_TILING_Y)
		2669	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2670	val \|= I830_FENCE_SIZE_BITS(size);
		2671	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2672	val \|= I830_FENCE_REG_VALID;
		2673	} else
		2674	val = 0;
		2675
		2676	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2677	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2678	}
		2679
3480	Serge	2680	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2681	{
		2682	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2683	}
		2684
3031	serge	2685	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2686	struct drm_i915_gem_object *obj)
2332	Serge	2687	{
3480	Serge	2688	struct drm_i915_private *dev_priv = dev->dev_private;
		2689
		2690	/* Ensure that all CPU reads are completed before installing a fence
		2691	* and all writes before removing the fence.
		2692	*/
		2693	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2694	mb();
		2695
4104	Serge	2696	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2697	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2698	obj->stride, obj->tiling_mode);
		2699
3031	serge	2700	switch (INTEL_INFO(dev)->gen) {
		2701	case 7:
3480	Serge	2702	case 6:
3031	serge	2703	case 5:
		2704	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2705	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2706	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2707	default: BUG();
3031	serge	2708	}
3480	Serge	2709
		2710	/* And similarly be paranoid that no direct access to this region
		2711	* is reordered to before the fence is installed.
		2712	*/
		2713	if (i915_gem_object_needs_mb(obj))
		2714	mb();
2344	Serge	2715	}
		2716
3031	serge	2717	static inline int fence_number(struct drm_i915_private *dev_priv,
		2718	struct drm_i915_fence_reg *fence)
2344	Serge	2719	{
3031	serge	2720	return fence - dev_priv->fence_regs;
		2721	}
2332	Serge	2722
3031	serge	2723	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2724	struct drm_i915_fence_reg *fence,
		2725	bool enable)
		2726	{
4104	Serge	2727	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2728	int reg = fence_number(dev_priv, fence);
2332	Serge	2729
4104	Serge	2730	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2731
		2732	if (enable) {
4104	Serge	2733	obj->fence_reg = reg;
3031	serge	2734	fence->obj = obj;
		2735	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2736	} else {
		2737	obj->fence_reg = I915_FENCE_REG_NONE;
		2738	fence->obj = NULL;
		2739	list_del_init(&fence->lru_list);
2344	Serge	2740	}
4104	Serge	2741	obj->fence_dirty = false;
3031	serge	2742	}
2344	Serge	2743
3031	serge	2744	static int
3480	Serge	2745	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2746	{
		2747	if (obj->last_fenced_seqno) {
		2748	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2749	if (ret)
		2750	return ret;
2344	Serge	2751
		2752	obj->last_fenced_seqno = 0;
		2753	}
		2754
3031	serge	2755	obj->fenced_gpu_access = false;
2332	Serge	2756	return 0;
		2757	}
		2758
		2759	int
2344	Serge	2760	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2761	{
3031	serge	2762	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2763	struct drm_i915_fence_reg *fence;
2332	Serge	2764	int ret;
		2765
3480	Serge	2766	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2767	if (ret)
		2768	return ret;
		2769
3031	serge	2770	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2771	return 0;
2332	Serge	2772
3746	Serge	2773	fence = &dev_priv->fence_regs[obj->fence_reg];
		2774
3031	serge	2775	i915_gem_object_fence_lost(obj);
3746	Serge	2776	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2777
2332	Serge	2778	return 0;
		2779	}
		2780
3031	serge	2781	static struct drm_i915_fence_reg *
		2782	i915_find_fence_reg(struct drm_device *dev)
		2783	{
		2784	struct drm_i915_private *dev_priv = dev->dev_private;
		2785	struct drm_i915_fence_reg reg, avail;
		2786	int i;
2332	Serge	2787
3031	serge	2788	/* First try to find a free reg */
		2789	avail = NULL;
		2790	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2791	reg = &dev_priv->fence_regs[i];
		2792	if (!reg->obj)
		2793	return reg;
2332	Serge	2794
3031	serge	2795	if (!reg->pin_count)
		2796	avail = reg;
		2797	}
2332	Serge	2798
3031	serge	2799	if (avail == NULL)
		2800	return NULL;
2332	Serge	2801
3031	serge	2802	/* None available, try to steal one or wait for a user to finish */
		2803	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2804	if (reg->pin_count)
		2805	continue;
2332	Serge	2806
3031	serge	2807	return reg;
		2808	}
2332	Serge	2809
3031	serge	2810	return NULL;
		2811	}
2332	Serge	2812
3031	serge	2813	/**
		2814	* i915_gem_object_get_fence - set up fencing for an object
		2815	* @obj: object to map through a fence reg
		2816	*
		2817	* When mapping objects through the GTT, userspace wants to be able to write
		2818	* to them without having to worry about swizzling if the object is tiled.
		2819	* This function walks the fence regs looking for a free one for @obj,
		2820	* stealing one if it can't find any.
		2821	*
		2822	* It then sets up the reg based on the object's properties: address, pitch
		2823	* and tiling format.
		2824	*
		2825	* For an untiled surface, this removes any existing fence.
		2826	*/
		2827	int
		2828	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2829	{
		2830	struct drm_device *dev = obj->base.dev;
		2831	struct drm_i915_private *dev_priv = dev->dev_private;
		2832	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2833	struct drm_i915_fence_reg *reg;
		2834	int ret;
2332	Serge	2835
3031	serge	2836	/* Have we updated the tiling parameters upon the object and so
		2837	* will need to serialise the write to the associated fence register?
		2838	*/
		2839	if (obj->fence_dirty) {
3480	Serge	2840	ret = i915_gem_object_wait_fence(obj);
3031	serge	2841	if (ret)
		2842	return ret;
		2843	}
2332	Serge	2844
3031	serge	2845	/* Just update our place in the LRU if our fence is getting reused. */
		2846	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2847	reg = &dev_priv->fence_regs[obj->fence_reg];
		2848	if (!obj->fence_dirty) {
		2849	list_move_tail(®->lru_list,
		2850	&dev_priv->mm.fence_list);
		2851	return 0;
		2852	}
		2853	} else if (enable) {
		2854	reg = i915_find_fence_reg(dev);
		2855	if (reg == NULL)
		2856	return -EDEADLK;
2332	Serge	2857
3031	serge	2858	if (reg->obj) {
		2859	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2860
3480	Serge	2861	ret = i915_gem_object_wait_fence(old);
3031	serge	2862	if (ret)
		2863	return ret;
2332	Serge	2864
3031	serge	2865	i915_gem_object_fence_lost(old);
		2866	}
		2867	} else
		2868	return 0;
2332	Serge	2869
3031	serge	2870	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2871
3031	serge	2872	return 0;
		2873	}
2332	Serge	2874
3031	serge	2875	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2876	struct drm_mm_node *gtt_space,
		2877	unsigned long cache_level)
		2878	{
		2879	struct drm_mm_node *other;
2332	Serge	2880
3031	serge	2881	/* On non-LLC machines we have to be careful when putting differing
		2882	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2883	* crossing memory domains and dying.
3031	serge	2884	*/
		2885	if (HAS_LLC(dev))
		2886	return true;
2332	Serge	2887
4104	Serge	2888	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2889	return true;
2332	Serge	2890
3031	serge	2891	if (list_empty(>t_space->node_list))
		2892	return true;
2332	Serge	2893
3031	serge	2894	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2895	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2896	return false;
2344	Serge	2897
3031	serge	2898	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2899	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2900	return false;
2344	Serge	2901
3031	serge	2902	return true;
		2903	}
2344	Serge	2904
3031	serge	2905	static void i915_gem_verify_gtt(struct drm_device *dev)
		2906	{
		2907	#if WATCH_GTT
		2908	struct drm_i915_private *dev_priv = dev->dev_private;
		2909	struct drm_i915_gem_object *obj;
		2910	int err = 0;
2344	Serge	2911
4104	Serge	2912	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3031	serge	2913	if (obj->gtt_space == NULL) {
		2914	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2915	err++;
		2916	continue;
		2917	}
2344	Serge	2918
3031	serge	2919	if (obj->cache_level != obj->gtt_space->color) {
		2920	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
4104	Serge	2921	i915_gem_obj_ggtt_offset(obj),
		2922	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2923	obj->cache_level,
		2924	obj->gtt_space->color);
		2925	err++;
		2926	continue;
		2927	}
2344	Serge	2928
3031	serge	2929	if (!i915_gem_valid_gtt_space(dev,
		2930	obj->gtt_space,
		2931	obj->cache_level)) {
		2932	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
4104	Serge	2933	i915_gem_obj_ggtt_offset(obj),
		2934	i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3031	serge	2935	obj->cache_level);
		2936	err++;
		2937	continue;
		2938	}
		2939	}
2344	Serge	2940
3031	serge	2941	WARN_ON(err);
		2942	#endif
2326	Serge	2943	}
		2944
2332	Serge	2945	/**
		2946	* Finds free space in the GTT aperture and binds the object there.
		2947	*/
		2948	static int
4104	Serge	2949	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2950	struct i915_address_space *vm,
2332	Serge	2951	unsigned alignment,
3031	serge	2952	bool map_and_fenceable,
		2953	bool nonblocking)
2332	Serge	2954	{
		2955	struct drm_device *dev = obj->base.dev;
		2956	drm_i915_private_t *dev_priv = dev->dev_private;
		2957	u32 size, fence_size, fence_alignment, unfenced_alignment;
4104	Serge	2958	size_t gtt_max =
		2959	map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
		2960	struct i915_vma *vma;
2332	Serge	2961	int ret;
2326	Serge	2962
2332	Serge	2963	fence_size = i915_gem_get_gtt_size(dev,
		2964	obj->base.size,
		2965	obj->tiling_mode);
		2966	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2967	obj->base.size,
3480	Serge	2968	obj->tiling_mode, true);
2332	Serge	2969	unfenced_alignment =
3480	Serge	2970	i915_gem_get_gtt_alignment(dev,
2332	Serge	2971	obj->base.size,
3480	Serge	2972	obj->tiling_mode, false);
2332	Serge	2973
		2974	if (alignment == 0)
		2975	alignment = map_and_fenceable ? fence_alignment :
		2976	unfenced_alignment;
		2977	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2978	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2979	return -EINVAL;
		2980	}
		2981
		2982	size = map_and_fenceable ? fence_size : obj->base.size;
		2983
		2984	/* If the object is bigger than the entire aperture, reject it early
		2985	* before evicting everything in a vain attempt to find space.
		2986	*/
4104	Serge	2987	if (obj->base.size > gtt_max) {
		2988	DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
		2989	obj->base.size,
		2990	map_and_fenceable ? "mappable" : "total",
		2991	gtt_max);
2332	Serge	2992	return -E2BIG;
		2993	}
		2994
3031	serge	2995	ret = i915_gem_object_get_pages(obj);
		2996	if (ret)
		2997	return ret;
		2998
3243	Serge	2999	i915_gem_object_pin_pages(obj);
		3000
4104	Serge	3001	BUG_ON(!i915_is_ggtt(vm));
		3002
		3003	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
		3004	if (IS_ERR(vma)) {
		3005	ret = PTR_ERR(vma);
		3006	goto err_unpin;
3243	Serge	3007	}
		3008
4104	Serge	3009	/* For now we only ever use 1 vma per object */
		3010	WARN_ON(!list_is_singular(&obj->vma_list));
		3011
		3012	search_free:
		3013	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3014	size, alignment,
		3015	obj->cache_level, 0, gtt_max,
		3016	DRM_MM_SEARCH_DEFAULT);
3243	Serge	3017	if (ret) {
2332	Serge	3018
4104	Serge	3019	goto err_free_vma;
2332	Serge	3020	}
4104	Serge	3021	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
		3022	obj->cache_level))) {
		3023	ret = -EINVAL;
		3024	goto err_remove_node;
3031	serge	3025	}
2332	Serge	3026
3031	serge	3027	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	3028	if (ret)
		3029	goto err_remove_node;
2332	Serge	3030
4104	Serge	3031	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3032	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3033
4104	Serge	3034	if (i915_is_ggtt(vm)) {
		3035	bool mappable, fenceable;
2332	Serge	3036
4104	Serge	3037	fenceable = (vma->node.size == fence_size &&
		3038	(vma->node.start & (fence_alignment - 1)) == 0);
2332	Serge	3039
4104	Serge	3040	mappable = (vma->node.start + obj->base.size <=
		3041	dev_priv->gtt.mappable_end);
2332	Serge	3042
		3043	obj->map_and_fenceable = mappable && fenceable;
4104	Serge	3044	}
2332	Serge	3045
4104	Serge	3046	WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
		3047
		3048	trace_i915_vma_bind(vma, map_and_fenceable);
3031	serge	3049	i915_gem_verify_gtt(dev);
2332	Serge	3050	return 0;
4104	Serge	3051
		3052	err_remove_node:
		3053	drm_mm_remove_node(&vma->node);
		3054	err_free_vma:
		3055	i915_gem_vma_destroy(vma);
		3056	err_unpin:
		3057	i915_gem_object_unpin_pages(obj);
		3058	return ret;
2332	Serge	3059	}
		3060
4104	Serge	3061	bool
		3062	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3063	bool force)
2332	Serge	3064	{
		3065	/* If we don't have a page list set up, then we're not pinned
		3066	* to GPU, and we can ignore the cache flush because it'll happen
		3067	* again at bind time.
		3068	*/
3243	Serge	3069	if (obj->pages == NULL)
4104	Serge	3070	return false;
2332	Serge	3071
3480	Serge	3072	/*
		3073	* Stolen memory is always coherent with the GPU as it is explicitly
		3074	* marked as wc by the system, or the system is cache-coherent.
		3075	*/
		3076	if (obj->stolen)
4104	Serge	3077	return false;
3480	Serge	3078
2332	Serge	3079	/* If the GPU is snooping the contents of the CPU cache,
		3080	* we do not need to manually clear the CPU cache lines. However,
		3081	* the caches are only snooped when the render cache is
		3082	* flushed/invalidated. As we always have to emit invalidations
		3083	* and flushes when moving into and out of the RENDER domain, correct
		3084	* snooping behaviour occurs naturally as the result of our domain
		3085	* tracking.
		3086	*/
4104	Serge	3087	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3088	return false;
2332	Serge	3089
4293	Serge	3090	trace_i915_gem_object_clflush(obj);
		3091	drm_clflush_sg(obj->pages);
2344	Serge	3092
4104	Serge	3093	return true;
2332	Serge	3094	}
		3095
2344	Serge	3096	/** Flushes the GTT write domain for the object if it's dirty. */
		3097	static void
		3098	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3099	{
		3100	uint32_t old_write_domain;
2332	Serge	3101
2344	Serge	3102	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3103	return;
2332	Serge	3104
2344	Serge	3105	/* No actual flushing is required for the GTT write domain. Writes
		3106	* to it immediately go to main memory as far as we know, so there's
		3107	* no chipset flush. It also doesn't land in render cache.
		3108	*
		3109	* However, we do have to enforce the order so that all writes through
		3110	* the GTT land before any writes to the device, such as updates to
		3111	* the GATT itself.
		3112	*/
		3113	wmb();
2332	Serge	3114
2344	Serge	3115	old_write_domain = obj->base.write_domain;
		3116	obj->base.write_domain = 0;
2332	Serge	3117
2351	Serge	3118	trace_i915_gem_object_change_domain(obj,
		3119	obj->base.read_domains,
		3120	old_write_domain);
2344	Serge	3121	}
2332	Serge	3122
		3123	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3124	static void
4104	Serge	3125	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3126	bool force)
2332	Serge	3127	{
		3128	uint32_t old_write_domain;
		3129
		3130	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3131	return;
		3132
4104	Serge	3133	if (i915_gem_clflush_object(obj, force))
3243	Serge	3134	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3135
2332	Serge	3136	old_write_domain = obj->base.write_domain;
		3137	obj->base.write_domain = 0;
		3138
2351	Serge	3139	trace_i915_gem_object_change_domain(obj,
		3140	obj->base.read_domains,
		3141	old_write_domain);
2332	Serge	3142	}
		3143
		3144	/**
		3145	* Moves a single object to the GTT read, and possibly write domain.
		3146	*
		3147	* This function returns when the move is complete, including waiting on
		3148	* flushes to occur.
		3149	*/
		3150	int
		3151	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3152	{
3031	serge	3153	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	3154	uint32_t old_write_domain, old_read_domains;
		3155	int ret;
		3156
		3157	/* Not valid to be called on unbound objects. */
4104	Serge	3158	if (!i915_gem_obj_bound_any(obj))
2332	Serge	3159	return -EINVAL;
		3160
		3161	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3162	return 0;
		3163
3031	serge	3164	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3165	if (ret)
		3166	return ret;
		3167
4104	Serge	3168	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3169
3480	Serge	3170	/* Serialise direct access to this object with the barriers for
		3171	* coherent writes from the GPU, by effectively invalidating the
		3172	* GTT domain upon first access.
		3173	*/
		3174	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3175	mb();
		3176
2332	Serge	3177	old_write_domain = obj->base.write_domain;
		3178	old_read_domains = obj->base.read_domains;
		3179
		3180	/* It should now be out of any other write domains, and we can update
		3181	* the domain values for our changes.
		3182	*/
		3183	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3184	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3185	if (write) {
		3186	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3187	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3188	obj->dirty = 1;
		3189	}
		3190
2351	Serge	3191	trace_i915_gem_object_change_domain(obj,
		3192	old_read_domains,
		3193	old_write_domain);
		3194
3031	serge	3195	/* And bump the LRU for this access */
4104	Serge	3196	if (i915_gem_object_is_inactive(obj)) {
		3197	struct i915_vma *vma = i915_gem_obj_to_vma(obj,
		3198	&dev_priv->gtt.base);
		3199	if (vma)
		3200	list_move_tail(&vma->mm_list,
		3201	&dev_priv->gtt.base.inactive_list);
3031	serge	3202
4104	Serge	3203	}
		3204
2332	Serge	3205	return 0;
		3206	}
		3207
2335	Serge	3208	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3209	enum i915_cache_level cache_level)
		3210	{
3031	serge	3211	struct drm_device *dev = obj->base.dev;
		3212	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3213	struct i915_vma *vma;
2335	Serge	3214	int ret;
2332	Serge	3215
2335	Serge	3216	if (obj->cache_level == cache_level)
		3217	return 0;
2332	Serge	3218
2335	Serge	3219	if (obj->pin_count) {
		3220	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3221	return -EBUSY;
		3222	}
2332	Serge	3223
4104	Serge	3224	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3225	if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
		3226	ret = i915_vma_unbind(vma);
3031	serge	3227	if (ret)
		3228	return ret;
4104	Serge	3229
		3230	break;
		3231	}
3031	serge	3232	}
		3233
4104	Serge	3234	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3235	ret = i915_gem_object_finish_gpu(obj);
		3236	if (ret)
		3237	return ret;
2332	Serge	3238
2335	Serge	3239	i915_gem_object_finish_gtt(obj);
2332	Serge	3240
2335	Serge	3241	/* Before SandyBridge, you could not use tiling or fence
		3242	* registers with snooped memory, so relinquish any fences
		3243	* currently pointing to our region in the aperture.
		3244	*/
3031	serge	3245	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3246	ret = i915_gem_object_put_fence(obj);
		3247	if (ret)
		3248	return ret;
		3249	}
2332	Serge	3250
3031	serge	3251	if (obj->has_global_gtt_mapping)
		3252	i915_gem_gtt_bind_object(obj, cache_level);
		3253	if (obj->has_aliasing_ppgtt_mapping)
		3254	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3255	obj, cache_level);
2335	Serge	3256	}
2332	Serge	3257
4104	Serge	3258	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3259	vma->node.color = cache_level;
		3260	obj->cache_level = cache_level;
		3261
		3262	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3263	u32 old_read_domains, old_write_domain;
2332	Serge	3264
2335	Serge	3265	/* If we're coming from LLC cached, then we haven't
		3266	* actually been tracking whether the data is in the
		3267	* CPU cache or not, since we only allow one bit set
		3268	* in obj->write_domain and have been skipping the clflushes.
		3269	* Just set it to the CPU cache for now.
		3270	*/
		3271	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3272
2335	Serge	3273	old_read_domains = obj->base.read_domains;
		3274	old_write_domain = obj->base.write_domain;
2332	Serge	3275
2335	Serge	3276	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3277	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3278
2351	Serge	3279	trace_i915_gem_object_change_domain(obj,
		3280	old_read_domains,
		3281	old_write_domain);
2344	Serge	3282	}
2332	Serge	3283
3031	serge	3284	i915_gem_verify_gtt(dev);
2335	Serge	3285	return 0;
		3286	}
2332	Serge	3287
3260	Serge	3288	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3289	struct drm_file *file)
		3290	{
		3291	struct drm_i915_gem_caching *args = data;
		3292	struct drm_i915_gem_object *obj;
		3293	int ret;
		3294
		3295	ret = i915_mutex_lock_interruptible(dev);
		3296	if (ret)
		3297	return ret;
		3298
		3299	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3300	if (&obj->base == NULL) {
		3301	ret = -ENOENT;
		3302	goto unlock;
		3303	}
		3304
4104	Serge	3305	switch (obj->cache_level) {
		3306	case I915_CACHE_LLC:
		3307	case I915_CACHE_L3_LLC:
		3308	args->caching = I915_CACHING_CACHED;
		3309	break;
3260	Serge	3310
4104	Serge	3311	case I915_CACHE_WT:
		3312	args->caching = I915_CACHING_DISPLAY;
		3313	break;
		3314
		3315	default:
		3316	args->caching = I915_CACHING_NONE;
		3317	break;
		3318	}
		3319
3260	Serge	3320	drm_gem_object_unreference(&obj->base);
		3321	unlock:
		3322	mutex_unlock(&dev->struct_mutex);
		3323	return ret;
		3324	}
		3325
		3326	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3327	struct drm_file *file)
		3328	{
		3329	struct drm_i915_gem_caching *args = data;
		3330	struct drm_i915_gem_object *obj;
		3331	enum i915_cache_level level;
		3332	int ret;
		3333
		3334	switch (args->caching) {
		3335	case I915_CACHING_NONE:
		3336	level = I915_CACHE_NONE;
		3337	break;
		3338	case I915_CACHING_CACHED:
		3339	level = I915_CACHE_LLC;
		3340	break;
4104	Serge	3341	case I915_CACHING_DISPLAY:
		3342	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3343	break;
3260	Serge	3344	default:
		3345	return -EINVAL;
		3346	}
		3347
		3348	ret = i915_mutex_lock_interruptible(dev);
		3349	if (ret)
		3350	return ret;
		3351
		3352	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3353	if (&obj->base == NULL) {
		3354	ret = -ENOENT;
		3355	goto unlock;
		3356	}
		3357
		3358	ret = i915_gem_object_set_cache_level(obj, level);
		3359
		3360	drm_gem_object_unreference(&obj->base);
		3361	unlock:
		3362	mutex_unlock(&dev->struct_mutex);
		3363	return ret;
		3364	}
		3365
4104	Serge	3366	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3367	{
		3368	/* There are 3 sources that pin objects:
		3369	* 1. The display engine (scanouts, sprites, cursors);
		3370	* 2. Reservations for execbuffer;
		3371	* 3. The user.
		3372	*
		3373	* We can ignore reservations as we hold the struct_mutex and
		3374	* are only called outside of the reservation path. The user
		3375	* can only increment pin_count once, and so if after
		3376	* subtracting the potential reference by the user, any pin_count
		3377	* remains, it must be due to another use by the display engine.
		3378	*/
		3379	return obj->pin_count - !!obj->user_pin_count;
		3380	}
		3381
2335	Serge	3382	/*
		3383	* Prepare buffer for display plane (scanout, cursors, etc).
		3384	* Can be called from an uninterruptible phase (modesetting) and allows
		3385	* any flushes to be pipelined (for pageflips).
		3386	*/
		3387	int
		3388	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3389	u32 alignment,
		3390	struct intel_ring_buffer *pipelined)
		3391	{
		3392	u32 old_read_domains, old_write_domain;
		3393	int ret;
2332	Serge	3394
3031	serge	3395	if (pipelined != obj->ring) {
		3396	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3397	if (ret)
		3398	return ret;
		3399	}
2332	Serge	3400
4104	Serge	3401	/* Mark the pin_display early so that we account for the
		3402	* display coherency whilst setting up the cache domains.
		3403	*/
		3404	obj->pin_display = true;
		3405
2335	Serge	3406	/* The display engine is not coherent with the LLC cache on gen6. As
		3407	* a result, we make sure that the pinning that is about to occur is
		3408	* done with uncached PTEs. This is lowest common denominator for all
		3409	* chipsets.
		3410	*
		3411	* However for gen6+, we could do better by using the GFDT bit instead
		3412	* of uncaching, which would allow us to flush all the LLC-cached data
		3413	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3414	*/
4104	Serge	3415	ret = i915_gem_object_set_cache_level(obj,
		3416	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3417	if (ret)
4104	Serge	3418	goto err_unpin_display;
2332	Serge	3419
2335	Serge	3420	/* As the user may map the buffer once pinned in the display plane
		3421	* (e.g. libkms for the bootup splash), we have to ensure that we
		3422	* always use map_and_fenceable for all scanout buffers.
		3423	*/
4104	Serge	3424	ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
2335	Serge	3425	if (ret)
4104	Serge	3426	goto err_unpin_display;
2332	Serge	3427
4104	Serge	3428	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3429
2335	Serge	3430	old_write_domain = obj->base.write_domain;
		3431	old_read_domains = obj->base.read_domains;
2332	Serge	3432
2335	Serge	3433	/* It should now be out of any other write domains, and we can update
		3434	* the domain values for our changes.
		3435	*/
3031	serge	3436	obj->base.write_domain = 0;
2335	Serge	3437	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3438
2351	Serge	3439	trace_i915_gem_object_change_domain(obj,
		3440	old_read_domains,
		3441	old_write_domain);
2332	Serge	3442
2335	Serge	3443	return 0;
4104	Serge	3444
		3445	err_unpin_display:
		3446	obj->pin_display = is_pin_display(obj);
		3447	return ret;
2335	Serge	3448	}
2332	Serge	3449
4104	Serge	3450	void
		3451	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3452	{
		3453	i915_gem_object_unpin(obj);
		3454	obj->pin_display = is_pin_display(obj);
		3455	}
		3456
2344	Serge	3457	int
		3458	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3459	{
		3460	int ret;
2332	Serge	3461
2344	Serge	3462	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3463	return 0;
2332	Serge	3464
3031	serge	3465	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3466	if (ret)
		3467	return ret;
2332	Serge	3468
2344	Serge	3469	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3470	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3471	return 0;
2344	Serge	3472	}
2332	Serge	3473
2344	Serge	3474	/**
		3475	* Moves a single object to the CPU read, and possibly write domain.
		3476	*
		3477	* This function returns when the move is complete, including waiting on
		3478	* flushes to occur.
		3479	*/
3031	serge	3480	int
2344	Serge	3481	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3482	{
		3483	uint32_t old_write_domain, old_read_domains;
		3484	int ret;
2332	Serge	3485
2344	Serge	3486	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3487	return 0;
2332	Serge	3488
3031	serge	3489	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3490	if (ret)
		3491	return ret;
2332	Serge	3492
2344	Serge	3493	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3494
2344	Serge	3495	old_write_domain = obj->base.write_domain;
		3496	old_read_domains = obj->base.read_domains;
2332	Serge	3497
2344	Serge	3498	/* Flush the CPU cache if it's still invalid. */
		3499	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3500	i915_gem_clflush_object(obj, false);
2332	Serge	3501
2344	Serge	3502	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3503	}
2332	Serge	3504
2344	Serge	3505	/* It should now be out of any other write domains, and we can update
		3506	* the domain values for our changes.
		3507	*/
		3508	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3509
2344	Serge	3510	/* If we're writing through the CPU, then the GPU read domains will
		3511	* need to be invalidated at next use.
		3512	*/
		3513	if (write) {
		3514	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3515	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3516	}
2332	Serge	3517
2351	Serge	3518	trace_i915_gem_object_change_domain(obj,
		3519	old_read_domains,
		3520	old_write_domain);
2332	Serge	3521
2344	Serge	3522	return 0;
		3523	}
2332	Serge	3524
3031	serge	3525	/* Throttle our rendering by waiting until the ring has completed our requests
		3526	* emitted over 20 msec ago.
2344	Serge	3527	*
3031	serge	3528	* Note that if we were to use the current jiffies each time around the loop,
		3529	* we wouldn't escape the function with any frames outstanding if the time to
		3530	* render a frame was over 20ms.
		3531	*
		3532	* This should get us reasonable parallelism between CPU and GPU but also
		3533	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3534	*/
3031	serge	3535	static int
		3536	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3537	{
3031	serge	3538	struct drm_i915_private *dev_priv = dev->dev_private;
		3539	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3540	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3541	struct drm_i915_gem_request *request;
		3542	struct intel_ring_buffer *ring = NULL;
3480	Serge	3543	unsigned reset_counter;
3031	serge	3544	u32 seqno = 0;
		3545	int ret;
2332	Serge	3546
3480	Serge	3547	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3548	if (ret)
		3549	return ret;
2332	Serge	3550
3480	Serge	3551	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3552	if (ret)
		3553	return ret;
		3554
3031	serge	3555	spin_lock(&file_priv->mm.lock);
		3556	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3557	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3558	break;
2332	Serge	3559
3031	serge	3560	ring = request->ring;
		3561	seqno = request->seqno;
		3562	}
3480	Serge	3563	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3564	spin_unlock(&file_priv->mm.lock);
2332	Serge	3565
3031	serge	3566	if (seqno == 0)
		3567	return 0;
2332	Serge	3568
3480	Serge	3569	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3570	if (ret == 0)
		3571	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3572
3031	serge	3573	return ret;
2352	Serge	3574	}
2332	Serge	3575
		3576	int
		3577	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3578	struct i915_address_space *vm,
2332	Serge	3579	uint32_t alignment,
3031	serge	3580	bool map_and_fenceable,
		3581	bool nonblocking)
2332	Serge	3582	{
4104	Serge	3583	struct i915_vma *vma;
2332	Serge	3584	int ret;
		3585
3031	serge	3586	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3587	return -EBUSY;
2332	Serge	3588
4104	Serge	3589	WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
		3590
		3591	vma = i915_gem_obj_to_vma(obj, vm);
		3592
		3593	if (vma) {
		3594	if ((alignment &&
		3595	vma->node.start & (alignment - 1)) \|\|
2332	Serge	3596	(map_and_fenceable && !obj->map_and_fenceable)) {
		3597	WARN(obj->pin_count,
		3598	"bo is already pinned with incorrect alignment:"
4104	Serge	3599	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3600	" obj->map_and_fenceable=%d\n",
4104	Serge	3601	i915_gem_obj_offset(obj, vm), alignment,
2332	Serge	3602	map_and_fenceable,
		3603	obj->map_and_fenceable);
4104	Serge	3604	ret = i915_vma_unbind(vma);
2332	Serge	3605	if (ret)
		3606	return ret;
		3607	}
		3608	}
		3609
4104	Serge	3610	if (!i915_gem_obj_bound(obj, vm)) {
3243	Serge	3611	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3612
4104	Serge	3613	ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
3031	serge	3614	map_and_fenceable,
		3615	nonblocking);
2332	Serge	3616	if (ret)
		3617	return ret;
3243	Serge	3618
		3619	if (!dev_priv->mm.aliasing_ppgtt)
		3620	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3621	}
		3622
3031	serge	3623	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3624	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3625
		3626	obj->pin_count++;
2332	Serge	3627	obj->pin_mappable \|= map_and_fenceable;
		3628
		3629	return 0;
		3630	}
		3631
2344	Serge	3632	void
		3633	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3634	{
		3635	BUG_ON(obj->pin_count == 0);
4104	Serge	3636	BUG_ON(!i915_gem_obj_bound_any(obj));
2332	Serge	3637
3031	serge	3638	if (--obj->pin_count == 0)
2344	Serge	3639	obj->pin_mappable = false;
		3640	}
2332	Serge	3641
3031	serge	3642	int
		3643	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3644	struct drm_file *file)
		3645	{
		3646	struct drm_i915_gem_pin *args = data;
		3647	struct drm_i915_gem_object *obj;
		3648	int ret;
2332	Serge	3649
3031	serge	3650	ret = i915_mutex_lock_interruptible(dev);
		3651	if (ret)
		3652	return ret;
2332	Serge	3653
3031	serge	3654	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3655	if (&obj->base == NULL) {
		3656	ret = -ENOENT;
		3657	goto unlock;
		3658	}
2332	Serge	3659
3031	serge	3660	if (obj->madv != I915_MADV_WILLNEED) {
		3661	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3662	ret = -EINVAL;
		3663	goto out;
		3664	}
2332	Serge	3665
3031	serge	3666	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3667	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3668	args->handle);
		3669	ret = -EINVAL;
		3670	goto out;
		3671	}
2332	Serge	3672
3243	Serge	3673	if (obj->user_pin_count == 0) {
4104	Serge	3674	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3031	serge	3675	if (ret)
		3676	goto out;
		3677	}
2332	Serge	3678
3243	Serge	3679	obj->user_pin_count++;
		3680	obj->pin_filp = file;
		3681
4104	Serge	3682	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3683	out:
		3684	drm_gem_object_unreference(&obj->base);
		3685	unlock:
		3686	mutex_unlock(&dev->struct_mutex);
		3687	return ret;
		3688	}
2332	Serge	3689
3031	serge	3690	int
		3691	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3692	struct drm_file *file)
		3693	{
		3694	struct drm_i915_gem_pin *args = data;
		3695	struct drm_i915_gem_object *obj;
		3696	int ret;
2332	Serge	3697
3031	serge	3698	ret = i915_mutex_lock_interruptible(dev);
		3699	if (ret)
		3700	return ret;
2332	Serge	3701
3031	serge	3702	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3703	if (&obj->base == NULL) {
		3704	ret = -ENOENT;
		3705	goto unlock;
		3706	}
2332	Serge	3707
3031	serge	3708	if (obj->pin_filp != file) {
		3709	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3710	args->handle);
		3711	ret = -EINVAL;
		3712	goto out;
		3713	}
		3714	obj->user_pin_count--;
		3715	if (obj->user_pin_count == 0) {
		3716	obj->pin_filp = NULL;
		3717	i915_gem_object_unpin(obj);
		3718	}
2332	Serge	3719
3031	serge	3720	out:
		3721	drm_gem_object_unreference(&obj->base);
		3722	unlock:
		3723	mutex_unlock(&dev->struct_mutex);
		3724	return ret;
		3725	}
2332	Serge	3726
3031	serge	3727	int
		3728	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3729	struct drm_file *file)
		3730	{
		3731	struct drm_i915_gem_busy *args = data;
		3732	struct drm_i915_gem_object *obj;
		3733	int ret;
2332	Serge	3734
3031	serge	3735	ret = i915_mutex_lock_interruptible(dev);
		3736	if (ret)
		3737	return ret;
2332	Serge	3738
4104	Serge	3739	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3740	if (&obj->base == NULL) {
		3741	ret = -ENOENT;
		3742	goto unlock;
		3743	}
2332	Serge	3744
3031	serge	3745	/* Count all active objects as busy, even if they are currently not used
		3746	* by the gpu. Users of this interface expect objects to eventually
		3747	* become non-busy without any further actions, therefore emit any
		3748	* necessary flushes here.
		3749	*/
		3750	ret = i915_gem_object_flush_active(obj);
2332	Serge	3751
3031	serge	3752	args->busy = obj->active;
		3753	if (obj->ring) {
		3754	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3755	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3756	}
2332	Serge	3757
3031	serge	3758	drm_gem_object_unreference(&obj->base);
		3759	unlock:
		3760	mutex_unlock(&dev->struct_mutex);
		3761	return ret;
		3762	}
2332	Serge	3763
3031	serge	3764	int
		3765	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3766	struct drm_file *file_priv)
		3767	{
		3768	return i915_gem_ring_throttle(dev, file_priv);
		3769	}
2332	Serge	3770
3263	Serge	3771	#if 0
		3772
3031	serge	3773	int
		3774	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3775	struct drm_file *file_priv)
		3776	{
		3777	struct drm_i915_gem_madvise *args = data;
		3778	struct drm_i915_gem_object *obj;
		3779	int ret;
2332	Serge	3780
3031	serge	3781	switch (args->madv) {
		3782	case I915_MADV_DONTNEED:
		3783	case I915_MADV_WILLNEED:
		3784	break;
		3785	default:
		3786	return -EINVAL;
		3787	}
2332	Serge	3788
3031	serge	3789	ret = i915_mutex_lock_interruptible(dev);
		3790	if (ret)
		3791	return ret;
2332	Serge	3792
3031	serge	3793	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3794	if (&obj->base == NULL) {
		3795	ret = -ENOENT;
		3796	goto unlock;
		3797	}
2332	Serge	3798
3031	serge	3799	if (obj->pin_count) {
		3800	ret = -EINVAL;
		3801	goto out;
		3802	}
2332	Serge	3803
3031	serge	3804	if (obj->madv != __I915_MADV_PURGED)
		3805	obj->madv = args->madv;
2332	Serge	3806
3031	serge	3807	/* if the object is no longer attached, discard its backing storage */
		3808	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3809	i915_gem_object_truncate(obj);
2332	Serge	3810
3031	serge	3811	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3812
3031	serge	3813	out:
		3814	drm_gem_object_unreference(&obj->base);
		3815	unlock:
		3816	mutex_unlock(&dev->struct_mutex);
		3817	return ret;
		3818	}
		3819	#endif
2332	Serge	3820
3031	serge	3821	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3822	const struct drm_i915_gem_object_ops *ops)
		3823	{
4104	Serge	3824	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3825	INIT_LIST_HEAD(&obj->ring_list);
		3826	INIT_LIST_HEAD(&obj->exec_list);
4104	Serge	3827	INIT_LIST_HEAD(&obj->obj_exec_link);
		3828	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3829
3031	serge	3830	obj->ops = ops;
		3831
		3832	obj->fence_reg = I915_FENCE_REG_NONE;
		3833	obj->madv = I915_MADV_WILLNEED;
		3834	/* Avoid an unnecessary call to unbind on the first bind. */
		3835	obj->map_and_fenceable = true;
		3836
		3837	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3838	}
		3839
		3840	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3841	.get_pages = i915_gem_object_get_pages_gtt,
		3842	.put_pages = i915_gem_object_put_pages_gtt,
		3843	};
		3844
2332	Serge	3845	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3846	size_t size)
		3847	{
		3848	struct drm_i915_gem_object *obj;
3031	serge	3849	struct address_space *mapping;
3480	Serge	3850	gfp_t mask;
2340	Serge	3851
3746	Serge	3852	obj = i915_gem_object_alloc(dev);
2332	Serge	3853	if (obj == NULL)
		3854	return NULL;
		3855
		3856	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3857	i915_gem_object_free(obj);
2332	Serge	3858	return NULL;
		3859	}
		3860
		3861
3031	serge	3862	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3863
		3864	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3865	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3866
3031	serge	3867	if (HAS_LLC(dev)) {
		3868	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3869	* cache) for about a 10% performance improvement
		3870	* compared to uncached. Graphics requests other than
		3871	* display scanout are coherent with the CPU in
		3872	* accessing this cache. This means in this mode we
		3873	* don't need to clflush on the CPU side, and on the
		3874	* GPU side we only need to flush internal caches to
		3875	* get data visible to the CPU.
		3876	*
		3877	* However, we maintain the display planes as UC, and so
		3878	* need to rebind when first used as such.
		3879	*/
		3880	obj->cache_level = I915_CACHE_LLC;
		3881	} else
		3882	obj->cache_level = I915_CACHE_NONE;
		3883
		3884	return obj;
		3885	}
		3886
2344	Serge	3887	int i915_gem_init_object(struct drm_gem_object *obj)
		3888	{
		3889	BUG();
2332	Serge	3890
2344	Serge	3891	return 0;
		3892	}
2332	Serge	3893
3031	serge	3894	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3895	{
3031	serge	3896	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3897	struct drm_device *dev = obj->base.dev;
		3898	drm_i915_private_t *dev_priv = dev->dev_private;
4104	Serge	3899	struct i915_vma vma, next;
2332	Serge	3900
3031	serge	3901	trace_i915_gem_object_destroy(obj);
		3902
		3903
		3904	obj->pin_count = 0;
4104	Serge	3905	/* NB: 0 or 1 elements */
		3906	WARN_ON(!list_empty(&obj->vma_list) &&
		3907	!list_is_singular(&obj->vma_list));
		3908	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3909	int ret = i915_vma_unbind(vma);
		3910	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	3911	bool was_interruptible;
		3912
		3913	was_interruptible = dev_priv->mm.interruptible;
		3914	dev_priv->mm.interruptible = false;
		3915
4104	Serge	3916	WARN_ON(i915_vma_unbind(vma));
3031	serge	3917
		3918	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3919	}
4104	Serge	3920	}
2332	Serge	3921
4104	Serge	3922	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		3923	* before progressing. */
		3924	if (obj->stolen)
		3925	i915_gem_object_unpin_pages(obj);
		3926
		3927	if (WARN_ON(obj->pages_pin_count))
3031	serge	3928	obj->pages_pin_count = 0;
		3929	i915_gem_object_put_pages(obj);
		3930	// i915_gem_object_free_mmap_offset(obj);
4104	Serge	3931	i915_gem_object_release_stolen(obj);
2332	Serge	3932
3243	Serge	3933	BUG_ON(obj->pages);
2332	Serge	3934
3031	serge	3935
3290	Serge	3936	if(obj->base.filp != NULL)
		3937	{
3298	Serge	3938	// printf("filp %p\n", obj->base.filp);
3290	Serge	3939	shmem_file_delete(obj->base.filp);
		3940	}
		3941
2344	Serge	3942	drm_gem_object_release(&obj->base);
		3943	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3944
2344	Serge	3945	kfree(obj->bit_17);
4104	Serge	3946	i915_gem_object_free(obj);
2344	Serge	3947	}
2332	Serge	3948
4104	Serge	3949	struct i915_vma i915_gem_vma_create(struct drm_i915_gem_object obj,
		3950	struct i915_address_space *vm)
		3951	{
		3952	struct i915_vma vma = kzalloc(sizeof(vma), GFP_KERNEL);
		3953	if (vma == NULL)
		3954	return ERR_PTR(-ENOMEM);
		3955
		3956	INIT_LIST_HEAD(&vma->vma_link);
		3957	INIT_LIST_HEAD(&vma->mm_list);
		3958	INIT_LIST_HEAD(&vma->exec_list);
		3959	vma->vm = vm;
		3960	vma->obj = obj;
		3961
		3962	/* Keep GGTT vmas first to make debug easier */
		3963	if (i915_is_ggtt(vm))
		3964	list_add(&vma->vma_link, &obj->vma_list);
		3965	else
		3966	list_add_tail(&vma->vma_link, &obj->vma_list);
		3967
		3968	return vma;
		3969	}
		3970
		3971	void i915_gem_vma_destroy(struct i915_vma *vma)
		3972	{
		3973	WARN_ON(vma->node.allocated);
		3974	list_del(&vma->vma_link);
		3975	kfree(vma);
		3976	}
		3977
3031	serge	3978	#if 0
		3979	int
		3980	i915_gem_idle(struct drm_device *dev)
2344	Serge	3981	{
3031	serge	3982	drm_i915_private_t *dev_priv = dev->dev_private;
		3983	int ret;
2332	Serge	3984
4104	Serge	3985	if (dev_priv->ums.mm_suspended) {
3031	serge	3986	mutex_unlock(&dev->struct_mutex);
		3987	return 0;
		3988	}
2332	Serge	3989
3031	serge	3990	ret = i915_gpu_idle(dev);
		3991	if (ret) {
		3992	mutex_unlock(&dev->struct_mutex);
		3993	return ret;
		3994	}
		3995	i915_gem_retire_requests(dev);
		3996
3480	Serge	3997	/* Under UMS, be paranoid and evict. */
		3998	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		3999	i915_gem_evict_everything(dev);
		4000
		4001	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	4002
		4003	i915_kernel_lost_context(dev);
		4004	i915_gem_cleanup_ringbuffer(dev);
		4005
		4006	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	4007	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	4008
		4009	return 0;
2344	Serge	4010	}
3031	serge	4011	#endif
2332	Serge	4012
3031	serge	4013	void i915_gem_l3_remap(struct drm_device *dev)
		4014	{
		4015	drm_i915_private_t *dev_priv = dev->dev_private;
		4016	u32 misccpctl;
		4017	int i;
2332	Serge	4018
3480	Serge	4019	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	4020	return;
2332	Serge	4021
3243	Serge	4022	if (!dev_priv->l3_parity.remap_info)
3031	serge	4023	return;
2332	Serge	4024
3031	serge	4025	misccpctl = I915_READ(GEN7_MISCCPCTL);
		4026	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		4027	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	4028
3031	serge	4029	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		4030	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	4031	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	4032	DRM_DEBUG("0x%x was already programmed to %x\n",
		4033	GEN7_L3LOG_BASE + i, remap);
3243	Serge	4034	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	4035	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	4036	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	4037	}
2332	Serge	4038
3031	serge	4039	/* Make sure all the writes land before disabling dop clock gating */
		4040	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	4041
3031	serge	4042	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		4043	}
2332	Serge	4044
3031	serge	4045	void i915_gem_init_swizzling(struct drm_device *dev)
		4046	{
		4047	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	4048
3031	serge	4049	if (INTEL_INFO(dev)->gen < 5 \|\|
		4050	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4051	return;
2332	Serge	4052
3031	serge	4053	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4054	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4055
3031	serge	4056	if (IS_GEN5(dev))
		4057	return;
2344	Serge	4058
3031	serge	4059	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4060	if (IS_GEN6(dev))
		4061	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4062	else if (IS_GEN7(dev))
		4063	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	4064	else
3480	Serge	4065	BUG();
3031	serge	4066	}
		4067
		4068	static bool
		4069	intel_enable_blt(struct drm_device *dev)
		4070	{
		4071	if (!HAS_BLT(dev))
		4072	return false;
		4073
		4074	/* The blitter was dysfunctional on early prototypes */
		4075	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4076	DRM_INFO("BLT not supported on this pre-production hardware;"
		4077	" graphics performance will be degraded.\n");
		4078	return false;
		4079	}
		4080
		4081	return true;
		4082	}
		4083
3480	Serge	4084	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	4085	{
3480	Serge	4086	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4087	int ret;
2351	Serge	4088
2332	Serge	4089	ret = intel_init_render_ring_buffer(dev);
		4090	if (ret)
		4091	return ret;
		4092
		4093	if (HAS_BSD(dev)) {
		4094	ret = intel_init_bsd_ring_buffer(dev);
		4095	if (ret)
		4096	goto cleanup_render_ring;
		4097	}
		4098
3031	serge	4099	if (intel_enable_blt(dev)) {
2332	Serge	4100	ret = intel_init_blt_ring_buffer(dev);
		4101	if (ret)
		4102	goto cleanup_bsd_ring;
		4103	}
		4104
4104	Serge	4105	if (HAS_VEBOX(dev)) {
		4106	ret = intel_init_vebox_ring_buffer(dev);
		4107	if (ret)
		4108	goto cleanup_blt_ring;
		4109	}
		4110
		4111
3480	Serge	4112	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4113	if (ret)
4104	Serge	4114	goto cleanup_vebox_ring;
2351	Serge	4115
2332	Serge	4116	return 0;
		4117
4104	Serge	4118	cleanup_vebox_ring:
		4119	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4120	cleanup_blt_ring:
		4121	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4122	cleanup_bsd_ring:
		4123	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4124	cleanup_render_ring:
		4125	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4126
2332	Serge	4127	return ret;
		4128	}
		4129
3480	Serge	4130	int
		4131	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4132	{
3480	Serge	4133	drm_i915_private_t *dev_priv = dev->dev_private;
		4134	int ret;
3031	serge	4135
3480	Serge	4136	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4137	return -EIO;
3031	serge	4138
4104	Serge	4139	if (dev_priv->ellc_size)
		4140	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4141
3746	Serge	4142	if (HAS_PCH_NOP(dev)) {
		4143	u32 temp = I915_READ(GEN7_MSG_CTL);
		4144	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4145	I915_WRITE(GEN7_MSG_CTL, temp);
		4146	}
		4147
3480	Serge	4148	i915_gem_l3_remap(dev);
		4149
		4150	i915_gem_init_swizzling(dev);
		4151
		4152	ret = i915_gem_init_rings(dev);
		4153	if (ret)
		4154	return ret;
		4155
		4156	/*
		4157	* XXX: There was some w/a described somewhere suggesting loading
		4158	* contexts before PPGTT.
		4159	*/
		4160	i915_gem_context_init(dev);
3746	Serge	4161	if (dev_priv->mm.aliasing_ppgtt) {
		4162	ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		4163	if (ret) {
		4164	i915_gem_cleanup_aliasing_ppgtt(dev);
		4165	DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		4166	}
		4167	}
3480	Serge	4168
		4169	return 0;
3031	serge	4170	}
		4171
		4172	int i915_gem_init(struct drm_device *dev)
		4173	{
		4174	struct drm_i915_private *dev_priv = dev->dev_private;
		4175	int ret;
		4176
		4177	mutex_lock(&dev->struct_mutex);
3746	Serge	4178
		4179	if (IS_VALLEYVIEW(dev)) {
		4180	/* VLVA0 (potential hack), BIOS isn't actually waking us */
		4181	I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		4182	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
		4183	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4184	}
		4185
3480	Serge	4186	i915_gem_init_global_gtt(dev);
3746	Serge	4187
3031	serge	4188	ret = i915_gem_init_hw(dev);
		4189	mutex_unlock(&dev->struct_mutex);
		4190	if (ret) {
		4191	i915_gem_cleanup_aliasing_ppgtt(dev);
		4192	return ret;
		4193	}
		4194
3746	Serge	4195
3031	serge	4196	return 0;
		4197	}
		4198
2332	Serge	4199	void
		4200	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4201	{
		4202	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	4203	struct intel_ring_buffer *ring;
2332	Serge	4204	int i;
		4205
3031	serge	4206	for_each_ring(ring, dev_priv, i)
		4207	intel_cleanup_ring_buffer(ring);
2332	Serge	4208	}
		4209
3031	serge	4210	#if 0
		4211
2332	Serge	4212	int
		4213	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		4214	struct drm_file *file_priv)
		4215	{
4104	Serge	4216	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4217	int ret;
2332	Serge	4218
		4219	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4220	return 0;
		4221
3480	Serge	4222	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	4223	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	4224	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	4225	}
		4226
		4227	mutex_lock(&dev->struct_mutex);
4104	Serge	4228	dev_priv->ums.mm_suspended = 0;
2332	Serge	4229
3031	serge	4230	ret = i915_gem_init_hw(dev);
2332	Serge	4231	if (ret != 0) {
		4232	mutex_unlock(&dev->struct_mutex);
		4233	return ret;
		4234	}
		4235
4104	Serge	4236	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
2332	Serge	4237	mutex_unlock(&dev->struct_mutex);
		4238
		4239	ret = drm_irq_install(dev);
		4240	if (ret)
		4241	goto cleanup_ringbuffer;
		4242
		4243	return 0;
		4244
		4245	cleanup_ringbuffer:
		4246	mutex_lock(&dev->struct_mutex);
		4247	i915_gem_cleanup_ringbuffer(dev);
4104	Serge	4248	dev_priv->ums.mm_suspended = 1;
2332	Serge	4249	mutex_unlock(&dev->struct_mutex);
		4250
		4251	return ret;
		4252	}
		4253
		4254	int
		4255	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		4256	struct drm_file *file_priv)
		4257	{
4104	Serge	4258	struct drm_i915_private *dev_priv = dev->dev_private;
		4259	int ret;
		4260
2332	Serge	4261	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4262	return 0;
		4263
		4264	drm_irq_uninstall(dev);
4104	Serge	4265
		4266	mutex_lock(&dev->struct_mutex);
		4267	ret = i915_gem_idle(dev);
		4268
		4269	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		4270	* We need to replace this with a semaphore, or something.
		4271	* And not confound ums.mm_suspended!
		4272	*/
		4273	if (ret != 0)
		4274	dev_priv->ums.mm_suspended = 1;
		4275	mutex_unlock(&dev->struct_mutex);
		4276
		4277	return ret;
2332	Serge	4278	}
		4279
		4280	void
		4281	i915_gem_lastclose(struct drm_device *dev)
		4282	{
		4283	int ret;
		4284
		4285	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4286	return;
		4287
4104	Serge	4288	mutex_lock(&dev->struct_mutex);
2332	Serge	4289	ret = i915_gem_idle(dev);
		4290	if (ret)
		4291	DRM_ERROR("failed to idle hardware: %d\n", ret);
4104	Serge	4292	mutex_unlock(&dev->struct_mutex);
2332	Serge	4293	}
		4294	#endif
		4295
		4296	static void
2326	Serge	4297	init_ring_lists(struct intel_ring_buffer *ring)
		4298	{
		4299	INIT_LIST_HEAD(&ring->active_list);
		4300	INIT_LIST_HEAD(&ring->request_list);
		4301	}
		4302
4104	Serge	4303	static void i915_init_vm(struct drm_i915_private *dev_priv,
		4304	struct i915_address_space *vm)
		4305	{
		4306	vm->dev = dev_priv->dev;
		4307	INIT_LIST_HEAD(&vm->active_list);
		4308	INIT_LIST_HEAD(&vm->inactive_list);
		4309	INIT_LIST_HEAD(&vm->global_link);
		4310	list_add(&vm->global_link, &dev_priv->vm_list);
		4311	}
		4312
2326	Serge	4313	void
		4314	i915_gem_load(struct drm_device *dev)
		4315	{
3480	Serge	4316	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4317	int i;
		4318
4104	Serge	4319	INIT_LIST_HEAD(&dev_priv->vm_list);
		4320	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4321
3031	serge	4322	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4323	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4324	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4325	for (i = 0; i < I915_NUM_RINGS; i++)
		4326	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4327	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4328	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4329	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4330	i915_gem_retire_work_handler);
3480	Serge	4331	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4332
		4333	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4334	if (IS_GEN3(dev)) {
3031	serge	4335	I915_WRITE(MI_ARB_STATE,
		4336	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4337	}
		4338
		4339	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4340
3746	Serge	4341	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4342	dev_priv->num_fence_regs = 32;
		4343	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4344	dev_priv->num_fence_regs = 16;
		4345	else
		4346	dev_priv->num_fence_regs = 8;
		4347
		4348	/* Initialize fence registers to zero */
3746	Serge	4349	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4350	i915_gem_restore_fences(dev);
2326	Serge	4351
		4352	i915_gem_detect_bit_6_swizzle(dev);
		4353
		4354	dev_priv->mm.interruptible = true;
		4355
		4356	}
		4357
4104	Serge	4358	#if 0
		4359	/*
		4360	* Create a physically contiguous memory object for this object
		4361	* e.g. for cursor + overlay regs
		4362	*/
		4363	static int i915_gem_init_phys_object(struct drm_device *dev,
		4364	int id, int size, int align)
		4365	{
		4366	drm_i915_private_t *dev_priv = dev->dev_private;
		4367	struct drm_i915_gem_phys_object *phys_obj;
		4368	int ret;
2326	Serge	4369
4104	Serge	4370	if (dev_priv->mm.phys_objs[id - 1] \|\| !size)
		4371	return 0;
		4372
		4373	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
		4374	if (!phys_obj)
		4375	return -ENOMEM;
		4376
		4377	phys_obj->id = id;
		4378
		4379	phys_obj->handle = drm_pci_alloc(dev, size, align);
		4380	if (!phys_obj->handle) {
		4381	ret = -ENOMEM;
		4382	goto kfree_obj;
		4383	}
		4384	#ifdef CONFIG_X86
		4385	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4386	#endif
		4387
		4388	dev_priv->mm.phys_objs[id - 1] = phys_obj;
		4389
		4390	return 0;
		4391	kfree_obj:
		4392	kfree(phys_obj);
		4393	return ret;
		4394	}
		4395
		4396	static void i915_gem_free_phys_object(struct drm_device *dev, int id)
		4397	{
		4398	drm_i915_private_t *dev_priv = dev->dev_private;
		4399	struct drm_i915_gem_phys_object *phys_obj;
		4400
		4401	if (!dev_priv->mm.phys_objs[id - 1])
		4402	return;
		4403
		4404	phys_obj = dev_priv->mm.phys_objs[id - 1];
		4405	if (phys_obj->cur_obj) {
		4406	i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
		4407	}
		4408
		4409	#ifdef CONFIG_X86
		4410	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
		4411	#endif
		4412	drm_pci_free(dev, phys_obj->handle);
		4413	kfree(phys_obj);
		4414	dev_priv->mm.phys_objs[id - 1] = NULL;
		4415	}
		4416
		4417	void i915_gem_free_all_phys_object(struct drm_device *dev)
		4418	{
		4419	int i;
		4420
		4421	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
		4422	i915_gem_free_phys_object(dev, i);
		4423	}
		4424
		4425	void i915_gem_detach_phys_object(struct drm_device *dev,
		4426	struct drm_i915_gem_object *obj)
		4427	{
		4428	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4429	char *vaddr;
		4430	int i;
		4431	int page_count;
		4432
		4433	if (!obj->phys_obj)
		4434	return;
		4435	vaddr = obj->phys_obj->handle->vaddr;
		4436
		4437	page_count = obj->base.size / PAGE_SIZE;
		4438	for (i = 0; i < page_count; i++) {
		4439	struct page *page = shmem_read_mapping_page(mapping, i);
		4440	if (!IS_ERR(page)) {
		4441	char *dst = kmap_atomic(page);
		4442	memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
		4443	kunmap_atomic(dst);
		4444
		4445	drm_clflush_pages(&page, 1);
		4446
		4447	set_page_dirty(page);
		4448	mark_page_accessed(page);
		4449	page_cache_release(page);
		4450	}
		4451	}
		4452	i915_gem_chipset_flush(dev);
		4453
		4454	obj->phys_obj->cur_obj = NULL;
		4455	obj->phys_obj = NULL;
		4456	}
		4457
		4458	int
		4459	i915_gem_attach_phys_object(struct drm_device *dev,
		4460	struct drm_i915_gem_object *obj,
		4461	int id,
		4462	int align)
		4463	{
		4464	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
		4465	drm_i915_private_t *dev_priv = dev->dev_private;
		4466	int ret = 0;
		4467	int page_count;
		4468	int i;
		4469
		4470	if (id > I915_MAX_PHYS_OBJECT)
		4471	return -EINVAL;
		4472
		4473	if (obj->phys_obj) {
		4474	if (obj->phys_obj->id == id)
		4475	return 0;
		4476	i915_gem_detach_phys_object(dev, obj);
		4477	}
		4478
		4479	/* create a new object */
		4480	if (!dev_priv->mm.phys_objs[id - 1]) {
		4481	ret = i915_gem_init_phys_object(dev, id,
		4482	obj->base.size, align);
		4483	if (ret) {
		4484	DRM_ERROR("failed to init phys object %d size: %zu\n",
		4485	id, obj->base.size);
		4486	return ret;
		4487	}
		4488	}
		4489
		4490	/* bind to the object */
		4491	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
		4492	obj->phys_obj->cur_obj = obj;
		4493
		4494	page_count = obj->base.size / PAGE_SIZE;
		4495
		4496	for (i = 0; i < page_count; i++) {
		4497	struct page *page;
		4498	char dst, src;
		4499
		4500	page = shmem_read_mapping_page(mapping, i);
		4501	if (IS_ERR(page))
		4502	return PTR_ERR(page);
		4503
		4504	src = kmap_atomic(page);
		4505	dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
		4506	memcpy(dst, src, PAGE_SIZE);
		4507	kunmap_atomic(src);
		4508
		4509	mark_page_accessed(page);
		4510	page_cache_release(page);
		4511	}
		4512
		4513	return 0;
		4514	}
		4515
		4516	static int
		4517	i915_gem_phys_pwrite(struct drm_device *dev,
		4518	struct drm_i915_gem_object *obj,
		4519	struct drm_i915_gem_pwrite *args,
		4520	struct drm_file *file_priv)
		4521	{
		4522	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
		4523	char __user *user_data = to_user_ptr(args->data_ptr);
		4524
		4525	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		4526	unsigned long unwritten;
		4527
		4528	/* The physical object once assigned is fixed for the lifetime
		4529	* of the obj, so we can safely drop the lock and continue
		4530	* to access vaddr.
		4531	*/
		4532	mutex_unlock(&dev->struct_mutex);
		4533	unwritten = copy_from_user(vaddr, user_data, args->size);
		4534	mutex_lock(&dev->struct_mutex);
		4535	if (unwritten)
		4536	return -EFAULT;
		4537	}
		4538
		4539	i915_gem_chipset_flush(dev);
		4540	return 0;
		4541	}
		4542
		4543	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4544	{
		4545	struct drm_i915_file_private *file_priv = file->driver_priv;
		4546
		4547	/* Clean up our request list when the client is going away, so that
		4548	* later retire_requests won't dereference our soon-to-be-gone
		4549	* file_priv.
		4550	*/
		4551	spin_lock(&file_priv->mm.lock);
		4552	while (!list_empty(&file_priv->mm.request_list)) {
		4553	struct drm_i915_gem_request *request;
		4554
		4555	request = list_first_entry(&file_priv->mm.request_list,
		4556	struct drm_i915_gem_request,
		4557	client_list);
		4558	list_del(&request->client_list);
		4559	request->file_priv = NULL;
		4560	}
		4561	spin_unlock(&file_priv->mm.lock);
		4562	}
		4563	#endif
		4564
		4565	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4566	{
		4567	if (!mutex_is_locked(mutex))
		4568	return false;
		4569
		4570	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4571	return mutex->owner == task;
		4572	#else
		4573	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4574	return false;
		4575	#endif
		4576	}
		4577
		4578	/* All the new VM stuff */
		4579	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4580	struct i915_address_space *vm)
		4581	{
		4582	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4583	struct i915_vma *vma;
		4584
		4585	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4586	vm = &dev_priv->gtt.base;
		4587
		4588	BUG_ON(list_empty(&o->vma_list));
		4589	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4590	if (vma->vm == vm)
		4591	return vma->node.start;
		4592
		4593	}
		4594	return 0; //-1;
		4595	}
		4596
		4597	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4598	struct i915_address_space *vm)
		4599	{
		4600	struct i915_vma *vma;
		4601
		4602	list_for_each_entry(vma, &o->vma_list, vma_link)
		4603	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4604	return true;
		4605
		4606	return false;
		4607	}
		4608
		4609	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4610	{
		4611	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4612	struct i915_address_space *vm;
		4613
		4614	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
		4615	if (i915_gem_obj_bound(o, vm))
		4616	return true;
		4617
		4618	return false;
		4619	}
		4620
		4621	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4622	struct i915_address_space *vm)
		4623	{
		4624	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4625	struct i915_vma *vma;
		4626
		4627	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		4628	vm = &dev_priv->gtt.base;
		4629
		4630	BUG_ON(list_empty(&o->vma_list));
		4631
		4632	list_for_each_entry(vma, &o->vma_list, vma_link)
		4633	if (vma->vm == vm)
		4634	return vma->node.size;
		4635
		4636	return 0;
		4637	}
		4638	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
		4639	struct i915_address_space *vm)
		4640	{
		4641	struct i915_vma *vma;
		4642	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4643	if (vma->vm == vm)
		4644	return vma;
		4645
		4646	return NULL;
		4647	}
		4648
		4649	struct i915_vma *
		4650	i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
		4651	struct i915_address_space *vm)
		4652	{
		4653	struct i915_vma *vma;
		4654
		4655	vma = i915_gem_obj_to_vma(obj, vm);
		4656	if (!vma)
		4657	vma = i915_gem_vma_create(obj, vm);
		4658
		4659	return vma;
		4660	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 5367 – Rev 4539