WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
3260	Serge	34	#include
2330	Serge	35	#include
2326	Serge	36	//#include
3746	Serge	37	#include
2326	Serge	38	#include
		39
2344	Serge	40	extern int x86_clflush_size;
2332	Serge	41
3263	Serge	42	#define PROT_READ 0x1 /* page can be read */
		43	#define PROT_WRITE 0x2 /* page can be written */
		44	#define MAP_SHARED 0x01 /* Share changes */
		45
2344	Serge	46
5060	serge	47	u64 nsecs_to_jiffies64(u64 n)
		48	{
		49	#if (NSEC_PER_SEC % HZ) == 0
		50	/* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */
		51	return div_u64(n, NSEC_PER_SEC / HZ);
		52	#elif (HZ % 512) == 0
		53	/* overflow after 292 years if HZ = 1024 */
		54	return div_u64(n * HZ / 512, NSEC_PER_SEC / 512);
		55	#else
		56	/*
		57	* Generic case - optimized for cases where HZ is a multiple of 3.
		58	* overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc.
		59	*/
		60	return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ);
		61	#endif
		62	}
		63
		64	unsigned long nsecs_to_jiffies(u64 n)
		65	{
		66	return (unsigned long)nsecs_to_jiffies64(n);
		67	}
		68
		69
3266	Serge	70	struct drm_i915_gem_object *get_fb_obj();
		71
3263	Serge	72	unsigned long vm_mmap(struct file *file, unsigned long addr,
		73	unsigned long len, unsigned long prot,
		74	unsigned long flag, unsigned long offset);
		75
2344	Serge	76
2332	Serge	77	#define MAX_ERRNO 4095
		78
		79	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		80
		81
		82	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
4104	Serge	83	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		84	bool force);
		85	static __must_check int
4560	Serge	86	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		87	bool readonly);
5060	serge	88	static void
		89	i915_gem_object_retire(struct drm_i915_gem_object *obj);
2326	Serge	90
3031	serge	91	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		92	struct drm_i915_gem_object *obj);
		93	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		94	struct drm_i915_fence_reg *fence,
		95	bool enable);
2332	Serge	96
5354	serge	97
4560	Serge	98	static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
3031	serge	99
4104	Serge	100	static bool cpu_cache_is_coherent(struct drm_device *dev,
		101	enum i915_cache_level level)
		102	{
		103	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		104	}
		105
		106	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		107	{
		108	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		109	return true;
		110
		111	return obj->pin_display;
		112	}
		113
3031	serge	114	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		115	{
		116	if (obj->tiling_mode)
		117	i915_gem_release_mmap(obj);
		118
		119	/* As we do not have an associated fence register, we will force
		120	* a tiling change if we ever need to acquire one.
		121	*/
		122	obj->fence_dirty = false;
		123	obj->fence_reg = I915_FENCE_REG_NONE;
		124	}
		125
2332	Serge	126	/* some bookkeeping */
		127	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		128	size_t size)
		129	{
4104	Serge	130	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	131	dev_priv->mm.object_count++;
		132	dev_priv->mm.object_memory += size;
4104	Serge	133	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	134	}
		135
		136	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		137	size_t size)
		138	{
4104	Serge	139	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	140	dev_priv->mm.object_count--;
		141	dev_priv->mm.object_memory -= size;
4104	Serge	142	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	143	}
		144
		145	static int
3480	Serge	146	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	147	{
		148	int ret;
		149
3480	Serge	150	#define EXIT_COND (!i915_reset_in_progress(error))
		151	if (EXIT_COND)
2332	Serge	152	return 0;
3255	Serge	153	#if 0
3031	serge	154	/*
		155	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		156	* userspace. If it takes that long something really bad is going on and
		157	* we should simply try to bail out and fail as gracefully as possible.
		158	*/
3480	Serge	159	ret = wait_event_interruptible_timeout(error->reset_queue,
		160	EXIT_COND,
		161	10*HZ);
3031	serge	162	if (ret == 0) {
		163	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		164	return -EIO;
		165	} else if (ret < 0) {
2332	Serge	166	return ret;
3031	serge	167	}
2332	Serge	168
3255	Serge	169	#endif
3480	Serge	170	#undef EXIT_COND
3255	Serge	171
2332	Serge	172	return 0;
		173	}
		174
		175	int i915_mutex_lock_interruptible(struct drm_device *dev)
		176	{
3480	Serge	177	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	178	int ret;
		179
3480	Serge	180	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	181	if (ret)
		182	return ret;
		183
3480	Serge	184	ret = mutex_lock_interruptible(&dev->struct_mutex);
		185	if (ret)
		186	return ret;
2332	Serge	187
		188	WARN_ON(i915_verify_lists(dev));
		189	return 0;
		190	}
		191
		192	static inline bool
		193	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		194	{
4104	Serge	195	return i915_gem_obj_bound_any(obj) && !obj->active;
2332	Serge	196	}
		197
		198	int
		199	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		200	struct drm_file *file)
		201	{
		202	struct drm_i915_private *dev_priv = dev->dev_private;
		203	struct drm_i915_gem_get_aperture *args = data;
		204	struct drm_i915_gem_object *obj;
		205	size_t pinned;
		206
		207	pinned = 0;
		208	mutex_lock(&dev->struct_mutex);
4104	Serge	209	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
5060	serge	210	if (i915_gem_obj_is_pinned(obj))
4104	Serge	211	pinned += i915_gem_obj_ggtt_size(obj);
2332	Serge	212	mutex_unlock(&dev->struct_mutex);
		213
4104	Serge	214	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	215	args->aper_available_size = args->aper_size - pinned;
2332	Serge	216
		217	return 0;
		218	}
		219
3480	Serge	220	void i915_gem_object_alloc(struct drm_device dev)
		221	{
		222	struct drm_i915_private *dev_priv = dev->dev_private;
5367	serge	223	return kzalloc(sizeof(struct drm_i915_gem_object), 0);
3480	Serge	224	}
		225
		226	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		227	{
		228	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		229	kfree(obj);
		230	}
		231
3031	serge	232	static int
		233	i915_gem_create(struct drm_file *file,
2332	Serge	234	struct drm_device *dev,
		235	uint64_t size,
		236	uint32_t *handle_p)
		237	{
		238	struct drm_i915_gem_object *obj;
		239	int ret;
		240	u32 handle;
		241
		242	size = roundup(size, PAGE_SIZE);
2342	Serge	243	if (size == 0)
		244	return -EINVAL;
2332	Serge	245
		246	/* Allocate the new object */
		247	obj = i915_gem_alloc_object(dev, size);
		248	if (obj == NULL)
		249	return -ENOMEM;
		250
		251	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	252	/* drop reference from allocate - handle holds it now */
		253	drm_gem_object_unreference_unlocked(&obj->base);
		254	if (ret)
2332	Serge	255	return ret;
		256
		257	*handle_p = handle;
		258	return 0;
		259	}
		260
		261	int
		262	i915_gem_dumb_create(struct drm_file *file,
		263	struct drm_device *dev,
		264	struct drm_mode_create_dumb *args)
		265	{
		266	/* have to work out size/pitch and return them */
4560	Serge	267	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
2332	Serge	268	args->size = args->pitch * args->height;
		269	return i915_gem_create(file, dev,
		270	args->size, &args->handle);
		271	}
		272
2326	Serge	273	/**
2332	Serge	274	* Creates a new mm object and returns a handle to it.
		275	*/
		276	int
		277	i915_gem_create_ioctl(struct drm_device dev, void data,
		278	struct drm_file *file)
		279	{
		280	struct drm_i915_gem_create *args = data;
3031	serge	281
2332	Serge	282	return i915_gem_create(file, dev,
		283	args->size, &args->handle);
		284	}
		285
		286
3260	Serge	287	#if 0
2332	Serge	288
3031	serge	289	static inline int
		290	__copy_to_user_swizzled(char __user *cpu_vaddr,
		291	const char *gpu_vaddr, int gpu_offset,
2332	Serge	292	int length)
		293	{
3031	serge	294	int ret, cpu_offset = 0;
2332	Serge	295
3031	serge	296	while (length > 0) {
		297	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		298	int this_length = min(cacheline_end - gpu_offset, length);
		299	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	300
3031	serge	301	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		302	gpu_vaddr + swizzled_gpu_offset,
		303	this_length);
		304	if (ret)
		305	return ret + length;
2332	Serge	306
3031	serge	307	cpu_offset += this_length;
		308	gpu_offset += this_length;
		309	length -= this_length;
		310	}
		311
		312	return 0;
2332	Serge	313	}
		314
3031	serge	315	static inline int
		316	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		317	const char __user *cpu_vaddr,
		318	int length)
2332	Serge	319	{
3031	serge	320	int ret, cpu_offset = 0;
2332	Serge	321
		322	while (length > 0) {
		323	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		324	int this_length = min(cacheline_end - gpu_offset, length);
		325	int swizzled_gpu_offset = gpu_offset ^ 64;
		326
3031	serge	327	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	328	cpu_vaddr + cpu_offset,
		329	this_length);
3031	serge	330	if (ret)
		331	return ret + length;
		332
2332	Serge	333	cpu_offset += this_length;
		334	gpu_offset += this_length;
		335	length -= this_length;
		336	}
		337
3031	serge	338	return 0;
2332	Serge	339	}
		340
3031	serge	341	/* Per-page copy function for the shmem pread fastpath.
		342	* Flushes invalid cachelines before reading the target if
		343	* needs_clflush is set. */
2332	Serge	344	static int
3031	serge	345	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		346	char __user *user_data,
		347	bool page_do_bit17_swizzling, bool needs_clflush)
		348	{
		349	char *vaddr;
		350	int ret;
		351
		352	if (unlikely(page_do_bit17_swizzling))
		353	return -EINVAL;
		354
		355	vaddr = kmap_atomic(page);
		356	if (needs_clflush)
		357	drm_clflush_virt_range(vaddr + shmem_page_offset,
		358	page_length);
		359	ret = __copy_to_user_inatomic(user_data,
		360	vaddr + shmem_page_offset,
		361	page_length);
		362	kunmap_atomic(vaddr);
		363
		364	return ret ? -EFAULT : 0;
		365	}
		366
		367	static void
		368	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		369	bool swizzled)
		370	{
		371	if (unlikely(swizzled)) {
		372	unsigned long start = (unsigned long) addr;
		373	unsigned long end = (unsigned long) addr + length;
		374
		375	/* For swizzling simply ensure that we always flush both
		376	* channels. Lame, but simple and it works. Swizzled
		377	* pwrite/pread is far from a hotpath - current userspace
		378	* doesn't use it at all. */
		379	start = round_down(start, 128);
		380	end = round_up(end, 128);
		381
		382	drm_clflush_virt_range((void *)start, end - start);
		383	} else {
		384	drm_clflush_virt_range(addr, length);
		385	}
		386
		387	}
		388
		389	/* Only difference to the fast-path function is that this can handle bit17
		390	* and uses non-atomic copy and kmap functions. */
		391	static int
		392	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		393	char __user *user_data,
		394	bool page_do_bit17_swizzling, bool needs_clflush)
		395	{
		396	char *vaddr;
		397	int ret;
		398
		399	vaddr = kmap(page);
		400	if (needs_clflush)
		401	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		402	page_length,
		403	page_do_bit17_swizzling);
		404
		405	if (page_do_bit17_swizzling)
		406	ret = __copy_to_user_swizzled(user_data,
		407	vaddr, shmem_page_offset,
		408	page_length);
		409	else
		410	ret = __copy_to_user(user_data,
		411	vaddr + shmem_page_offset,
		412	page_length);
		413	kunmap(page);
		414
		415	return ret ? - EFAULT : 0;
		416	}
		417
		418	static int
		419	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	420	struct drm_i915_gem_object *obj,
		421	struct drm_i915_gem_pread *args,
		422	struct drm_file *file)
		423	{
3031	serge	424	char __user *user_data;
2332	Serge	425	ssize_t remain;
		426	loff_t offset;
3031	serge	427	int shmem_page_offset, page_length, ret = 0;
		428	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		429	int prefaulted = 0;
		430	int needs_clflush = 0;
3746	Serge	431	struct sg_page_iter sg_iter;
2332	Serge	432
3746	Serge	433	user_data = to_user_ptr(args->data_ptr);
2332	Serge	434	remain = args->size;
		435
3031	serge	436	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		437
5060	serge	438	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
3031	serge	439	if (ret)
		440	return ret;
		441
2332	Serge	442	offset = args->offset;
		443
3746	Serge	444	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		445	offset >> PAGE_SHIFT) {
		446	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	447
3031	serge	448	if (remain <= 0)
		449	break;
		450
2332	Serge	451	/* Operation in this page
		452	*
3031	serge	453	* shmem_page_offset = offset within page in shmem file
2332	Serge	454	* page_length = bytes to copy for this page
		455	*/
3031	serge	456	shmem_page_offset = offset_in_page(offset);
2332	Serge	457	page_length = remain;
3031	serge	458	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		459	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	460
3031	serge	461	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		462	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	463
3031	serge	464	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		465	user_data, page_do_bit17_swizzling,
		466	needs_clflush);
		467	if (ret == 0)
		468	goto next_page;
2332	Serge	469
3031	serge	470	mutex_unlock(&dev->struct_mutex);
		471
5060	serge	472	if (likely(!i915.prefault_disable) && !prefaulted) {
3031	serge	473	ret = fault_in_multipages_writeable(user_data, remain);
		474	/* Userspace is tricking us, but we've already clobbered
		475	* its pages with the prefault and promised to write the
		476	* data up to the first fault. Hence ignore any errors
		477	* and just continue. */
		478	(void)ret;
		479	prefaulted = 1;
		480	}
		481
		482	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		483	user_data, page_do_bit17_swizzling,
		484	needs_clflush);
		485
		486	mutex_lock(&dev->struct_mutex);
		487
2332	Serge	488	if (ret)
3031	serge	489	goto out;
2332	Serge	490
5060	serge	491	next_page:
2332	Serge	492	remain -= page_length;
		493	user_data += page_length;
		494	offset += page_length;
		495	}
		496
3031	serge	497	out:
		498	i915_gem_object_unpin_pages(obj);
		499
		500	return ret;
2332	Serge	501	}
		502
		503	/**
3031	serge	504	* Reads data from the object referenced by handle.
		505	*
		506	* On error, the contents of *data are undefined.
2332	Serge	507	*/
3031	serge	508	int
		509	i915_gem_pread_ioctl(struct drm_device dev, void data,
		510	struct drm_file *file)
		511	{
		512	struct drm_i915_gem_pread *args = data;
		513	struct drm_i915_gem_object *obj;
		514	int ret = 0;
		515
		516	if (args->size == 0)
		517	return 0;
		518
		519	if (!access_ok(VERIFY_WRITE,
3746	Serge	520	to_user_ptr(args->data_ptr),
3031	serge	521	args->size))
		522	return -EFAULT;
		523
		524	ret = i915_mutex_lock_interruptible(dev);
		525	if (ret)
		526	return ret;
		527
		528	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		529	if (&obj->base == NULL) {
		530	ret = -ENOENT;
		531	goto unlock;
		532	}
		533
		534	/* Bounds check source. */
		535	if (args->offset > obj->base.size \|\|
		536	args->size > obj->base.size - args->offset) {
		537	ret = -EINVAL;
		538	goto out;
		539	}
		540
		541	/* prime objects have no backing filp to GEM pread/pwrite
		542	* pages from.
		543	*/
		544	if (!obj->base.filp) {
		545	ret = -EINVAL;
		546	goto out;
		547	}
		548
		549	trace_i915_gem_object_pread(obj, args->offset, args->size);
		550
		551	ret = i915_gem_shmem_pread(dev, obj, args, file);
		552
		553	out:
		554	drm_gem_object_unreference(&obj->base);
		555	unlock:
		556	mutex_unlock(&dev->struct_mutex);
		557	return ret;
		558	}
		559
		560	/* This is the fast write path which cannot handle
		561	* page faults in the source data
		562	*/
		563
		564	static inline int
		565	fast_user_write(struct io_mapping *mapping,
		566	loff_t page_base, int page_offset,
		567	char __user *user_data,
		568	int length)
		569	{
		570	void __iomem *vaddr_atomic;
		571	void *vaddr;
		572	unsigned long unwritten;
		573
		574	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		575	/* We can use the cpu mem copy function because this is X86. */
		576	vaddr = (void __force*)vaddr_atomic + page_offset;
		577	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		578	user_data, length);
		579	io_mapping_unmap_atomic(vaddr_atomic);
		580	return unwritten;
		581	}
3260	Serge	582	#endif
3031	serge	583
3260	Serge	584	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	585	/**
		586	* This is the fast pwrite path, where we copy the data directly from the
		587	* user into the GTT, uncached.
		588	*/
2332	Serge	589	static int
3031	serge	590	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		591	struct drm_i915_gem_object *obj,
		592	struct drm_i915_gem_pwrite *args,
		593	struct drm_file *file)
2332	Serge	594	{
5060	serge	595	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	596	ssize_t remain;
3031	serge	597	loff_t offset, page_base;
		598	char __user *user_data;
		599	int page_offset, page_length, ret;
2332	Serge	600
5060	serge	601	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3031	serge	602	if (ret)
		603	goto out;
		604
		605	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		606	if (ret)
		607	goto out_unpin;
		608
		609	ret = i915_gem_object_put_fence(obj);
		610	if (ret)
		611	goto out_unpin;
		612
4539	Serge	613	user_data = to_user_ptr(args->data_ptr);
2332	Serge	614	remain = args->size;
		615
4104	Serge	616	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	617
3031	serge	618	while (remain > 0) {
		619	/* Operation in this page
		620	*
		621	* page_base = page offset within aperture
		622	* page_offset = offset within page
		623	* page_length = bytes to copy for this page
		624	*/
		625	page_base = offset & PAGE_MASK;
		626	page_offset = offset_in_page(offset);
		627	page_length = remain;
		628	if ((page_offset + remain) > PAGE_SIZE)
		629	page_length = PAGE_SIZE - page_offset;
2332	Serge	630
4539	Serge	631	MapPage(dev_priv->gtt.mappable, dev_priv->gtt.mappable_base+page_base, PG_SW);
3031	serge	632
5060	serge	633	memcpy((char*)dev_priv->gtt.mappable+page_offset, user_data, page_length);
3260	Serge	634
3031	serge	635	remain -= page_length;
		636	user_data += page_length;
		637	offset += page_length;
2332	Serge	638	}
		639
3031	serge	640	out_unpin:
5060	serge	641	i915_gem_object_ggtt_unpin(obj);
3031	serge	642	out:
5060	serge	643	return ret;
3031	serge	644	}
		645
		646	/* Per-page copy function for the shmem pwrite fastpath.
		647	* Flushes invalid cachelines before writing to the target if
		648	* needs_clflush_before is set and flushes out any written cachelines after
		649	* writing if needs_clflush is set. */
		650	static int
		651	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		652	char __user *user_data,
		653	bool page_do_bit17_swizzling,
		654	bool needs_clflush_before,
		655	bool needs_clflush_after)
		656	{
		657	char *vaddr;
5354	serge	658	int ret;
3031	serge	659
		660	if (unlikely(page_do_bit17_swizzling))
		661	return -EINVAL;
		662
5354	serge	663	vaddr = kmap_atomic(page);
3031	serge	664	if (needs_clflush_before)
		665	drm_clflush_virt_range(vaddr + shmem_page_offset,
		666	page_length);
3260	Serge	667	memcpy(vaddr + shmem_page_offset,
3031	serge	668	user_data,
		669	page_length);
		670	if (needs_clflush_after)
		671	drm_clflush_virt_range(vaddr + shmem_page_offset,
		672	page_length);
5354	serge	673	kunmap_atomic(vaddr);
3031	serge	674
		675	return ret ? -EFAULT : 0;
		676	}
3260	Serge	677	#if 0
3031	serge	678
		679	/* Only difference to the fast-path function is that this can handle bit17
		680	* and uses non-atomic copy and kmap functions. */
		681	static int
		682	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		683	char __user *user_data,
		684	bool page_do_bit17_swizzling,
		685	bool needs_clflush_before,
		686	bool needs_clflush_after)
		687	{
		688	char *vaddr;
		689	int ret;
		690
		691	vaddr = kmap(page);
		692	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		693	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		694	page_length,
		695	page_do_bit17_swizzling);
		696	if (page_do_bit17_swizzling)
		697	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		698	user_data,
		699	page_length);
		700	else
		701	ret = __copy_from_user(vaddr + shmem_page_offset,
		702	user_data,
		703	page_length);
		704	if (needs_clflush_after)
		705	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		706	page_length,
		707	page_do_bit17_swizzling);
		708	kunmap(page);
		709
		710	return ret ? -EFAULT : 0;
		711	}
3260	Serge	712	#endif
3031	serge	713
3260	Serge	714
3031	serge	715	static int
		716	i915_gem_shmem_pwrite(struct drm_device *dev,
		717	struct drm_i915_gem_object *obj,
		718	struct drm_i915_gem_pwrite *args,
		719	struct drm_file *file)
		720	{
		721	ssize_t remain;
		722	loff_t offset;
		723	char __user *user_data;
		724	int shmem_page_offset, page_length, ret = 0;
		725	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		726	int hit_slowpath = 0;
		727	int needs_clflush_after = 0;
		728	int needs_clflush_before = 0;
3746	Serge	729	struct sg_page_iter sg_iter;
3031	serge	730
3746	Serge	731	user_data = to_user_ptr(args->data_ptr);
3031	serge	732	remain = args->size;
		733
		734	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		735
		736	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		737	/* If we're not in the cpu write domain, set ourself into the gtt
		738	* write domain and manually flush cachelines (if required). This
		739	* optimizes for the case when the gpu will use the data
		740	* right away and we therefore have to clflush anyway. */
4104	Serge	741	needs_clflush_after = cpu_write_needs_clflush(obj);
4560	Serge	742	ret = i915_gem_object_wait_rendering(obj, false);
3031	serge	743	if (ret)
		744	return ret;
5060	serge	745
		746	i915_gem_object_retire(obj);
3031	serge	747	}
4104	Serge	748	/* Same trick applies to invalidate partially written cachelines read
		749	* before writing. */
		750	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		751	needs_clflush_before =
		752	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	753
		754	ret = i915_gem_object_get_pages(obj);
2332	Serge	755	if (ret)
3031	serge	756	return ret;
2332	Serge	757
3031	serge	758	i915_gem_object_pin_pages(obj);
2332	Serge	759
		760	offset = args->offset;
3031	serge	761	obj->dirty = 1;
2332	Serge	762
3746	Serge	763	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		764	offset >> PAGE_SHIFT) {
		765	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	766	int partial_cacheline_write;
2332	Serge	767
3031	serge	768	if (remain <= 0)
		769	break;
		770
2332	Serge	771	/* Operation in this page
		772	*
		773	* shmem_page_offset = offset within page in shmem file
		774	* page_length = bytes to copy for this page
		775	*/
		776	shmem_page_offset = offset_in_page(offset);
		777
		778	page_length = remain;
		779	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		780	page_length = PAGE_SIZE - shmem_page_offset;
		781
3031	serge	782	/* If we don't overwrite a cacheline completely we need to be
		783	* careful to have up-to-date data by first clflushing. Don't
		784	* overcomplicate things and flush the entire patch. */
		785	partial_cacheline_write = needs_clflush_before &&
		786	((shmem_page_offset \| page_length)
3260	Serge	787	& (x86_clflush_size - 1));
2332	Serge	788
3031	serge	789	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		790	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	791
3031	serge	792	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		793	user_data, page_do_bit17_swizzling,
		794	partial_cacheline_write,
		795	needs_clflush_after);
		796	if (ret == 0)
		797	goto next_page;
		798
		799	hit_slowpath = 1;
		800	mutex_unlock(&dev->struct_mutex);
3260	Serge	801	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	802
3260	Serge	803	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		804	// user_data, page_do_bit17_swizzling,
		805	// partial_cacheline_write,
		806	// needs_clflush_after);
		807
3031	serge	808	mutex_lock(&dev->struct_mutex);
		809
		810	if (ret)
		811	goto out;
		812
5354	serge	813	next_page:
2332	Serge	814	remain -= page_length;
3031	serge	815	user_data += page_length;
2332	Serge	816	offset += page_length;
		817	}
		818
		819	out:
3031	serge	820	i915_gem_object_unpin_pages(obj);
		821
		822	if (hit_slowpath) {
3480	Serge	823	/*
		824	* Fixup: Flush cpu caches in case we didn't flush the dirty
		825	* cachelines in-line while writing and the object moved
		826	* out of the cpu write domain while we've dropped the lock.
		827	*/
		828	if (!needs_clflush_after &&
		829	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	830	if (i915_gem_clflush_object(obj, obj->pin_display))
3243	Serge	831	i915_gem_chipset_flush(dev);
3031	serge	832	}
2332	Serge	833	}
		834
3031	serge	835	if (needs_clflush_after)
3243	Serge	836	i915_gem_chipset_flush(dev);
3031	serge	837
2332	Serge	838	return ret;
		839	}
3031	serge	840
		841	/**
		842	* Writes data to the object referenced by handle.
		843	*
		844	* On error, the contents of the buffer that were to be modified are undefined.
		845	*/
		846	int
		847	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		848	struct drm_file *file)
		849	{
		850	struct drm_i915_gem_pwrite *args = data;
		851	struct drm_i915_gem_object *obj;
		852	int ret;
		853
4104	Serge	854	if (args->size == 0)
		855	return 0;
		856
3480	Serge	857
3031	serge	858	ret = i915_mutex_lock_interruptible(dev);
		859	if (ret)
		860	return ret;
		861
		862	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		863	if (&obj->base == NULL) {
		864	ret = -ENOENT;
		865	goto unlock;
		866	}
		867
		868	/* Bounds check destination. */
		869	if (args->offset > obj->base.size \|\|
		870	args->size > obj->base.size - args->offset) {
		871	ret = -EINVAL;
		872	goto out;
		873	}
		874
		875	/* prime objects have no backing filp to GEM pread/pwrite
		876	* pages from.
		877	*/
		878	if (!obj->base.filp) {
		879	ret = -EINVAL;
		880	goto out;
		881	}
		882
		883	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		884
		885	ret = -EFAULT;
		886	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		887	* it would end up going through the fenced access, and we'll get
		888	* different detiling behavior between reading and writing.
		889	* pread/pwrite currently are reading and writing from the CPU
		890	* perspective, requiring manual detiling by the client.
		891	*/
4104	Serge	892	if (obj->tiling_mode == I915_TILING_NONE &&
		893	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		894	cpu_write_needs_clflush(obj)) {
3031	serge	895	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		896	/* Note that the gtt paths might fail with non-page-backed user
		897	* pointers (e.g. gtt mappings when moving data between
		898	* textures). Fallback to the shmem path in that case. */
		899	}
		900
		901	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	902	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	903
		904	out:
		905	drm_gem_object_unreference(&obj->base);
		906	unlock:
		907	mutex_unlock(&dev->struct_mutex);
		908	return ret;
		909	}
		910
		911	int
3480	Serge	912	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	913	bool interruptible)
		914	{
3480	Serge	915	if (i915_reset_in_progress(error)) {
3031	serge	916	/* Non-interruptible callers can't handle -EAGAIN, hence return
		917	* -EIO unconditionally for these. */
		918	if (!interruptible)
		919	return -EIO;
2332	Serge	920
3480	Serge	921	/* Recovery complete, but the reset failed ... */
		922	if (i915_terminally_wedged(error))
3031	serge	923	return -EIO;
2332	Serge	924
3031	serge	925	return -EAGAIN;
		926	}
2332	Serge	927
3031	serge	928	return 0;
		929	}
2332	Serge	930
3031	serge	931	/*
		932	* Compare seqno against outstanding lazy request. Emit a request if they are
		933	* equal.
		934	*/
5060	serge	935	int
		936	i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno)
3031	serge	937	{
		938	int ret;
2332	Serge	939
3031	serge	940	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	941
3031	serge	942	ret = 0;
4560	Serge	943	if (seqno == ring->outstanding_lazy_seqno)
4104	Serge	944	ret = i915_add_request(ring, NULL);
2332	Serge	945
3031	serge	946	return ret;
		947	}
2332	Serge	948
4560	Serge	949	static void fake_irq(unsigned long data)
		950	{
		951	// wake_up_process((struct task_struct *)data);
		952	}
		953
		954	static bool missed_irq(struct drm_i915_private *dev_priv,
5060	serge	955	struct intel_engine_cs *ring)
4560	Serge	956	{
		957	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
		958	}
		959
		960	static bool can_wait_boost(struct drm_i915_file_private *file_priv)
		961	{
		962	if (file_priv == NULL)
		963	return true;
		964
		965	return !atomic_xchg(&file_priv->rps_wait_boost, true);
		966	}
		967
3031	serge	968	/**
5354	serge	969	* __i915_wait_seqno - wait until execution of seqno has finished
3031	serge	970	* @ring: the ring expected to report seqno
		971	* @seqno: duh!
3480	Serge	972	* @reset_counter: reset sequence associated with the given seqno
3031	serge	973	* @interruptible: do an interruptible wait (normally yes)
		974	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		975	*
3480	Serge	976	* Note: It is of utmost importance that the passed in seqno and reset_counter
		977	* values have been read by the caller in an smp safe manner. Where read-side
		978	* locks are involved, it is sufficient to read the reset_counter before
		979	* unlocking the lock that protects the seqno. For lockless tricks, the
		980	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		981	* inserted.
		982	*
3031	serge	983	* Returns 0 if the seqno was found within the alloted time. Else returns the
		984	* errno with remaining time filled in timeout argument.
		985	*/
5354	serge	986	int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
3480	Serge	987	unsigned reset_counter,
4560	Serge	988	bool interruptible,
5060	serge	989	s64 *timeout,
4560	Serge	990	struct drm_i915_file_private *file_priv)
3031	serge	991	{
5060	serge	992	struct drm_device *dev = ring->dev;
		993	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	994	const bool irq_test_in_progress =
		995	ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
5060	serge	996	unsigned long timeout_expire;
		997	s64 before, now;
		998
4560	Serge	999	wait_queue_t __wait;
3031	serge	1000	int ret;
2332	Serge	1001
5060	serge	1002	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
4104	Serge	1003
3031	serge	1004	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1005	return 0;
2332	Serge	1006
5354	serge	1007	timeout_expire = timeout ?
		1008	jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
2332	Serge	1009
5060	serge	1010	if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
4560	Serge	1011	gen6_rps_boost(dev_priv);
		1012	if (file_priv)
		1013	mod_delayed_work(dev_priv->wq,
		1014	&file_priv->mm.idle_work,
		1015	msecs_to_jiffies(100));
3031	serge	1016	}
2332	Serge	1017
4560	Serge	1018	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
3031	serge	1019	return -ENODEV;
2332	Serge	1020
4560	Serge	1021	INIT_LIST_HEAD(&__wait.task_list);
		1022	__wait.evnt = CreateEvent(NULL, MANUAL_DESTROY);
2332	Serge	1023
4560	Serge	1024	/* Record current time in case interrupted by signal, or wedged */
		1025	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1026
4560	Serge	1027	for (;;) {
		1028	unsigned long flags;
		1029
3480	Serge	1030	/* We need to check whether any gpu reset happened in between
		1031	* the caller grabbing the seqno and now ... */
4560	Serge	1032	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
		1033	/* ... but upgrade the -EAGAIN to an -EIO if the gpu
		1034	* is truely gone. */
		1035	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
		1036	if (ret == 0)
		1037	ret = -EAGAIN;
		1038	break;
		1039	}
3480	Serge	1040
4560	Serge	1041	if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
		1042	ret = 0;
		1043	break;
		1044	}
2332	Serge	1045
5060	serge	1046	if (timeout && time_after_eq(jiffies, timeout_expire)) {
4560	Serge	1047	ret = -ETIME;
		1048	break;
		1049	}
2332	Serge	1050
4560	Serge	1051	spin_lock_irqsave(&ring->irq_queue.lock, flags);
		1052	if (list_empty(&__wait.task_list))
		1053	__add_wait_queue(&ring->irq_queue, &__wait);
		1054	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1055
		1056	WaitEventTimeout(__wait.evnt, 1);
		1057
		1058	if (!list_empty(&__wait.task_list)) {
		1059	spin_lock_irqsave(&ring->irq_queue.lock, flags);
		1060	list_del_init(&__wait.task_list);
		1061	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1062	}
		1063	};
		1064	trace_i915_gem_request_wait_end(ring, seqno);
		1065
		1066	DestroyEvent(__wait.evnt);
		1067
		1068	if (!irq_test_in_progress)
5060	serge	1069	ring->irq_put(ring);
2332	Serge	1070
5060	serge	1071	// finish_wait(&ring->irq_queue, &wait);
4560	Serge	1072	return ret;
3031	serge	1073	}
2332	Serge	1074
3031	serge	1075	/**
		1076	* Waits for a sequence number to be signaled, and cleans up the
		1077	* request and object lists appropriately for that event.
		1078	*/
		1079	int
5060	serge	1080	i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
3031	serge	1081	{
		1082	struct drm_device *dev = ring->dev;
		1083	struct drm_i915_private *dev_priv = dev->dev_private;
		1084	bool interruptible = dev_priv->mm.interruptible;
5354	serge	1085	unsigned reset_counter;
3031	serge	1086	int ret;
2332	Serge	1087
3031	serge	1088	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1089	BUG_ON(seqno == 0);
2332	Serge	1090
3480	Serge	1091	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1092	if (ret)
		1093	return ret;
2332	Serge	1094
3031	serge	1095	ret = i915_gem_check_olr(ring, seqno);
		1096	if (ret)
		1097	return ret;
2332	Serge	1098
5354	serge	1099	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		1100	return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
		1101	NULL, NULL);
3031	serge	1102	}
2332	Serge	1103
4104	Serge	1104	static int
5354	serge	1105	i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
4104	Serge	1106	{
5060	serge	1107	if (!obj->active)
		1108	return 0;
4104	Serge	1109
		1110	/* Manually manage the write flush as we may have not yet
		1111	* retired the buffer.
		1112	*
		1113	* Note that the last_write_seqno is always the earlier of
		1114	* the two (read/write) seqno, so if we haved successfully waited,
		1115	* we know we have passed the last write.
		1116	*/
		1117	obj->last_write_seqno = 0;
		1118
		1119	return 0;
		1120	}
		1121
3031	serge	1122	/**
		1123	* Ensures that all rendering to the object has completed and the object is
		1124	* safe to unbind from the GTT or access from the CPU.
		1125	*/
		1126	static __must_check int
		1127	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1128	bool readonly)
		1129	{
5060	serge	1130	struct intel_engine_cs *ring = obj->ring;
3031	serge	1131	u32 seqno;
		1132	int ret;
2332	Serge	1133
3031	serge	1134	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1135	if (seqno == 0)
		1136	return 0;
2332	Serge	1137
3031	serge	1138	ret = i915_wait_seqno(ring, seqno);
4104	Serge	1139	if (ret)
		1140	return ret;
2332	Serge	1141
5354	serge	1142	return i915_gem_object_wait_rendering__tail(obj);
3031	serge	1143	}
2332	Serge	1144
3260	Serge	1145	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1146	* as the object state may change during this call.
		1147	*/
		1148	static __must_check int
		1149	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
5060	serge	1150	struct drm_i915_file_private *file_priv,
3260	Serge	1151	bool readonly)
		1152	{
		1153	struct drm_device *dev = obj->base.dev;
		1154	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	1155	struct intel_engine_cs *ring = obj->ring;
3480	Serge	1156	unsigned reset_counter;
3260	Serge	1157	u32 seqno;
		1158	int ret;
2332	Serge	1159
3260	Serge	1160	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1161	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1162
3260	Serge	1163	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1164	if (seqno == 0)
		1165	return 0;
2332	Serge	1166
3480	Serge	1167	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1168	if (ret)
		1169	return ret;
2332	Serge	1170
3260	Serge	1171	ret = i915_gem_check_olr(ring, seqno);
		1172	if (ret)
		1173	return ret;
2332	Serge	1174
3480	Serge	1175	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1176	mutex_unlock(&dev->struct_mutex);
5354	serge	1177	ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
		1178	file_priv);
3260	Serge	1179	mutex_lock(&dev->struct_mutex);
4104	Serge	1180	if (ret)
		1181	return ret;
2332	Serge	1182
5354	serge	1183	return i915_gem_object_wait_rendering__tail(obj);
3260	Serge	1184	}
2332	Serge	1185
3260	Serge	1186	/**
		1187	* Called when user space prepares to use an object with the CPU, either
		1188	* through the mmap ioctl's mapping or a GTT mapping.
		1189	*/
		1190	int
		1191	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1192	struct drm_file *file)
		1193	{
		1194	struct drm_i915_gem_set_domain *args = data;
		1195	struct drm_i915_gem_object *obj;
		1196	uint32_t read_domains = args->read_domains;
		1197	uint32_t write_domain = args->write_domain;
		1198	int ret;
2332	Serge	1199
3260	Serge	1200	/* Only handle setting domains to types used by the CPU. */
		1201	if (write_domain & I915_GEM_GPU_DOMAINS)
		1202	return -EINVAL;
2332	Serge	1203
3260	Serge	1204	if (read_domains & I915_GEM_GPU_DOMAINS)
		1205	return -EINVAL;
2332	Serge	1206
3260	Serge	1207	/* Having something in the write domain implies it's in the read
		1208	* domain, and only that read domain. Enforce that in the request.
		1209	*/
		1210	if (write_domain != 0 && read_domains != write_domain)
		1211	return -EINVAL;
2332	Serge	1212
3260	Serge	1213	ret = i915_mutex_lock_interruptible(dev);
		1214	if (ret)
		1215	return ret;
2332	Serge	1216
3260	Serge	1217	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1218	if (&obj->base == NULL) {
		1219	ret = -ENOENT;
		1220	goto unlock;
		1221	}
2332	Serge	1222
3260	Serge	1223	/* Try to flush the object off the GPU without holding the lock.
		1224	* We will repeat the flush holding the lock in the normal manner
		1225	* to catch cases where we are gazumped.
		1226	*/
5060	serge	1227	ret = i915_gem_object_wait_rendering__nonblocking(obj,
		1228	file->driver_priv,
		1229	!write_domain);
3260	Serge	1230	if (ret)
		1231	goto unref;
2332	Serge	1232
3260	Serge	1233	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1234	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1235
3260	Serge	1236	/* Silently promote "you're not bound, there was nothing to do"
		1237	* to success, since the client was just asking us to
		1238	* make sure everything was done.
		1239	*/
		1240	if (ret == -EINVAL)
		1241	ret = 0;
		1242	} else {
		1243	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1244	}
2332	Serge	1245
3260	Serge	1246	unref:
		1247	drm_gem_object_unreference(&obj->base);
		1248	unlock:
		1249	mutex_unlock(&dev->struct_mutex);
		1250	return ret;
		1251	}
2332	Serge	1252
4293	Serge	1253	/**
		1254	* Called when user space has done writes to this buffer
		1255	*/
		1256	int
		1257	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
		1258	struct drm_file *file)
		1259	{
		1260	struct drm_i915_gem_sw_finish *args = data;
		1261	struct drm_i915_gem_object *obj;
		1262	int ret = 0;
2332	Serge	1263
4293	Serge	1264	ret = i915_mutex_lock_interruptible(dev);
		1265	if (ret)
		1266	return ret;
2332	Serge	1267
4293	Serge	1268	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1269	if (&obj->base == NULL) {
		1270	ret = -ENOENT;
		1271	goto unlock;
		1272	}
2332	Serge	1273
4293	Serge	1274	/* Pinned buffers may be scanout, so flush the cache */
		1275	if (obj->pin_display)
		1276	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	1277
4293	Serge	1278	drm_gem_object_unreference(&obj->base);
		1279	unlock:
		1280	mutex_unlock(&dev->struct_mutex);
		1281	return ret;
		1282	}
		1283
3260	Serge	1284	/**
		1285	* Maps the contents of an object, returning the address it is mapped
		1286	* into.
		1287	*
		1288	* While the mapping holds a reference on the contents of the object, it doesn't
		1289	* imply a ref on the object itself.
5354	serge	1290	*
		1291	* IMPORTANT:
		1292	*
		1293	* DRM driver writers who look a this function as an example for how to do GEM
		1294	* mmap support, please don't implement mmap support like here. The modern way
		1295	* to implement DRM mmap support is with an mmap offset ioctl (like
		1296	* i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
		1297	* That way debug tooling like valgrind will understand what's going on, hiding
		1298	* the mmap call in a driver private ioctl will break that. The i915 driver only
		1299	* does cpu mmaps this way because we didn't know better.
3260	Serge	1300	*/
		1301	int
		1302	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1303	struct drm_file *file)
		1304	{
		1305	struct drm_i915_gem_mmap *args = data;
		1306	struct drm_gem_object *obj;
4392	Serge	1307	unsigned long addr;
2332	Serge	1308
3260	Serge	1309	obj = drm_gem_object_lookup(dev, file, args->handle);
		1310	if (obj == NULL)
		1311	return -ENOENT;
4104	Serge	1312
3260	Serge	1313	/* prime objects have no backing filp to GEM mmap
		1314	* pages from.
		1315	*/
		1316	if (!obj->filp) {
		1317	drm_gem_object_unreference_unlocked(obj);
		1318	return -EINVAL;
		1319	}
2332	Serge	1320
3263	Serge	1321	addr = vm_mmap(obj->filp, 0, args->size,
		1322	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1323	args->offset);
3260	Serge	1324	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1325	if (IS_ERR((void *)addr))
		1326	return addr;
2332	Serge	1327
3260	Serge	1328	args->addr_ptr = (uint64_t) addr;
2332	Serge	1329
3263	Serge	1330	return 0;
3260	Serge	1331	}
2332	Serge	1332
		1333
		1334
		1335
		1336
		1337
		1338
		1339
3031	serge	1340
		1341
		1342
		1343
		1344
		1345	/**
		1346	* i915_gem_release_mmap - remove physical page mappings
		1347	* @obj: obj in question
		1348	*
		1349	* Preserve the reservation of the mmapping with the DRM core code, but
		1350	* relinquish ownership of the pages back to the system.
		1351	*
		1352	* It is vital that we remove the page mapping if we have mapped a tiled
		1353	* object through the GTT and then lose the fence register due to
		1354	* resource pressure. Similarly if the object has been moved out of the
		1355	* aperture, than pages mapped into userspace must be revoked. Removing the
		1356	* mapping will then trigger a page fault on the next user access, allowing
		1357	* fixup by i915_gem_fault().
		1358	*/
		1359	void
		1360	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1361	{
		1362	if (!obj->fault_mappable)
		1363	return;
		1364
4104	Serge	1365	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1366	obj->fault_mappable = false;
		1367	}
		1368
3480	Serge	1369	uint32_t
2332	Serge	1370	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1371	{
		1372	uint32_t gtt_size;
		1373
		1374	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1375	tiling_mode == I915_TILING_NONE)
		1376	return size;
		1377
		1378	/* Previous chips need a power-of-two fence region when tiling */
		1379	if (INTEL_INFO(dev)->gen == 3)
		1380	gtt_size = 1024*1024;
		1381	else
		1382	gtt_size = 512*1024;
		1383
		1384	while (gtt_size < size)
		1385	gtt_size <<= 1;
		1386
		1387	return gtt_size;
		1388	}
		1389
		1390	/**
		1391	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1392	* @obj: object to check
		1393	*
		1394	* Return the required GTT alignment for an object, taking into account
		1395	* potential fence register mapping.
		1396	*/
3480	Serge	1397	uint32_t
		1398	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1399	int tiling_mode, bool fenced)
2332	Serge	1400	{
		1401	/*
		1402	* Minimum alignment is 4k (GTT page size), but might be greater
		1403	* if a fence register is needed for the object.
		1404	*/
3480	Serge	1405	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1406	tiling_mode == I915_TILING_NONE)
		1407	return 4096;
		1408
		1409	/*
		1410	* Previous chips need to be aligned to the size of the smallest
		1411	* fence register that can contain the object.
		1412	*/
		1413	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1414	}
		1415
		1416
		1417
3480	Serge	1418	int
		1419	i915_gem_mmap_gtt(struct drm_file *file,
		1420	struct drm_device *dev,
		1421	uint32_t handle,
		1422	uint64_t *offset)
		1423	{
		1424	struct drm_i915_private *dev_priv = dev->dev_private;
		1425	struct drm_i915_gem_object *obj;
		1426	unsigned long pfn;
		1427	char mem, ptr;
		1428	int ret;
		1429
		1430	ret = i915_mutex_lock_interruptible(dev);
		1431	if (ret)
		1432	return ret;
		1433
		1434	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1435	if (&obj->base == NULL) {
		1436	ret = -ENOENT;
		1437	goto unlock;
		1438	}
		1439
		1440	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1441	ret = -E2BIG;
		1442	goto out;
		1443	}
		1444
		1445	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1446	DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
		1447	ret = -EFAULT;
3480	Serge	1448	goto out;
		1449	}
		1450	/* Now bind it into the GTT if needed */
5060	serge	1451	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3480	Serge	1452	if (ret)
		1453	goto out;
		1454
		1455	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1456	if (ret)
		1457	goto unpin;
		1458
		1459	ret = i915_gem_object_get_fence(obj);
		1460	if (ret)
		1461	goto unpin;
		1462
		1463	obj->fault_mappable = true;
		1464
4104	Serge	1465	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1466
		1467	/* Finally, remap it using the new GTT offset */
		1468
		1469	mem = UserAlloc(obj->base.size);
		1470	if(unlikely(mem == NULL))
		1471	{
		1472	ret = -ENOMEM;
		1473	goto unpin;
		1474	}
		1475
		1476	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1477	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1478
		1479	unpin:
5060	serge	1480	i915_gem_object_unpin_pages(obj);
3480	Serge	1481
		1482
5367	serge	1483	*offset = (uint32_t)mem;
3480	Serge	1484
		1485	out:
		1486	drm_gem_object_unreference(&obj->base);
		1487	unlock:
		1488	mutex_unlock(&dev->struct_mutex);
		1489	return ret;
		1490	}
		1491
		1492	/**
		1493	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1494	* @dev: DRM device
		1495	* @data: GTT mapping ioctl data
		1496	* @file: GEM object info
		1497	*
		1498	* Simply returns the fake offset to userspace so it can mmap it.
		1499	* The mmap call will end up in drm_gem_mmap(), which will set things
		1500	* up so we can get faults in the handler above.
		1501	*
		1502	* The fault handler will take care of binding the object into the GTT
		1503	* (since it may have been evicted to make room for something), allocating
		1504	* a fence register, and mapping the appropriate aperture address into
		1505	* userspace.
		1506	*/
		1507	int
		1508	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1509	struct drm_file *file)
		1510	{
		1511	struct drm_i915_gem_mmap_gtt *args = data;
		1512
		1513	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1514	}
		1515
5060	serge	1516	static inline int
		1517	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1518	{
		1519	return obj->madv == I915_MADV_DONTNEED;
		1520	}
		1521
3031	serge	1522	/* Immediately discard the backing storage */
		1523	static void
		1524	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1525	{
		1526	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1527
3263	Serge	1528	if (obj->base.filp == NULL)
		1529	return;
2332	Serge	1530
3031	serge	1531	/* Our goal here is to return as much of the memory as
		1532	* is possible back to the system as we are called from OOM.
		1533	* To do this we must instruct the shmfs to drop all of its
		1534	* backing pages, now.
		1535	*/
5060	serge	1536	// shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
3031	serge	1537	obj->madv = __I915_MADV_PURGED;
		1538	}
2332	Serge	1539
5060	serge	1540	/* Try to discard unwanted pages */
		1541	static void
		1542	i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
3031	serge	1543	{
5060	serge	1544	struct address_space *mapping;
		1545
		1546	switch (obj->madv) {
		1547	case I915_MADV_DONTNEED:
		1548	i915_gem_object_truncate(obj);
		1549	case __I915_MADV_PURGED:
		1550	return;
		1551	}
		1552
		1553	if (obj->base.filp == NULL)
		1554	return;
		1555
3031	serge	1556	}
2332	Serge	1557
3031	serge	1558	static void
		1559	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1560	{
3746	Serge	1561	struct sg_page_iter sg_iter;
		1562	int ret;
2332	Serge	1563
3031	serge	1564	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1565
3031	serge	1566	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1567	if (ret) {
		1568	/* In the event of a disaster, abandon all caches and
		1569	* hope for the best.
		1570	*/
		1571	WARN_ON(ret != -EIO);
4104	Serge	1572	i915_gem_clflush_object(obj, true);
3031	serge	1573	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1574	}
2332	Serge	1575
3031	serge	1576	if (obj->madv == I915_MADV_DONTNEED)
		1577	obj->dirty = 0;
2332	Serge	1578
3746	Serge	1579	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1580	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1581
3290	Serge	1582	page_cache_release(page);
3243	Serge	1583	}
4104	Serge	1584	obj->dirty = 0;
3243	Serge	1585
		1586	sg_free_table(obj->pages);
		1587	kfree(obj->pages);
3031	serge	1588	}
2332	Serge	1589
3480	Serge	1590	int
3031	serge	1591	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1592	{
		1593	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1594
3243	Serge	1595	if (obj->pages == NULL)
3031	serge	1596	return 0;
2332	Serge	1597
3031	serge	1598	if (obj->pages_pin_count)
		1599	return -EBUSY;
		1600
4104	Serge	1601	BUG_ON(i915_gem_obj_bound_any(obj));
		1602
3243	Serge	1603	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1604	* array, hence protect them from being reaped by removing them from gtt
		1605	* lists early. */
4104	Serge	1606	list_del(&obj->global_list);
3243	Serge	1607
3031	serge	1608	ops->put_pages(obj);
3243	Serge	1609	obj->pages = NULL;
3031	serge	1610
5060	serge	1611	i915_gem_object_invalidate(obj);
3031	serge	1612
		1613	return 0;
		1614	}
		1615
		1616
		1617
		1618
		1619
		1620
		1621
		1622
2332	Serge	1623	static int
3031	serge	1624	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1625	{
3260	Serge	1626	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1627	int page_count, i;
4104	Serge	1628	struct sg_table *st;
3243	Serge	1629	struct scatterlist *sg;
3746	Serge	1630	struct sg_page_iter sg_iter;
3243	Serge	1631	struct page *page;
3746	Serge	1632	unsigned long last_pfn = 0; /* suppress gcc warning */
3243	Serge	1633	gfp_t gfp;
2332	Serge	1634
3243	Serge	1635	/* Assert that the object is not currently in any GPU domain. As it
		1636	* wasn't in the GTT, there shouldn't be any way it could have been in
		1637	* a GPU cache
2332	Serge	1638	*/
3243	Serge	1639	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1640	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1641
		1642	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1643	if (st == NULL)
		1644	return -ENOMEM;
		1645
2332	Serge	1646	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1647	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1648	kfree(st);
2332	Serge	1649	return -ENOMEM;
3243	Serge	1650	}
2332	Serge	1651
3243	Serge	1652	/* Get the list of pages out of our struct file. They'll be pinned
		1653	* at this point until we release them.
		1654	*
		1655	* Fail silently without starting the shrinker
		1656	*/
3746	Serge	1657	sg = st->sgl;
		1658	st->nents = 0;
		1659	for (i = 0; i < page_count; i++) {
4104	Serge	1660	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1661	if (IS_ERR(page)) {
		1662	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1663	goto err_pages;
3260	Serge	1664	}
5354	serge	1665	#ifdef CONFIG_SWIOTLB
		1666	if (swiotlb_nr_tbl()) {
		1667	st->nents++;
		1668	sg_set_page(sg, page, PAGE_SIZE, 0);
		1669	sg = sg_next(sg);
		1670	continue;
		1671	}
		1672	#endif
3746	Serge	1673	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1674	if (i)
		1675	sg = sg_next(sg);
		1676	st->nents++;
3243	Serge	1677	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1678	} else {
		1679	sg->length += PAGE_SIZE;
		1680	}
		1681	last_pfn = page_to_pfn(page);
3243	Serge	1682	}
5354	serge	1683	#ifdef CONFIG_SWIOTLB
		1684	if (!swiotlb_nr_tbl())
		1685	#endif
3746	Serge	1686	sg_mark_end(sg);
3243	Serge	1687	obj->pages = st;
3031	serge	1688
5367	serge	1689
		1690	if (obj->tiling_mode != I915_TILING_NONE &&
		1691	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
		1692	i915_gem_object_pin_pages(obj);
		1693
2332	Serge	1694	return 0;
		1695
		1696	err_pages:
3746	Serge	1697	sg_mark_end(sg);
		1698	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1699	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1700	sg_free_table(st);
		1701	kfree(st);
5367	serge	1702
3243	Serge	1703	return PTR_ERR(page);
2332	Serge	1704	}
		1705
3031	serge	1706	/* Ensure that the associated pages are gathered from the backing storage
		1707	* and pinned into our object. i915_gem_object_get_pages() may be called
		1708	* multiple times before they are released by a single call to
		1709	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1710	* either as a result of memory pressure (reaping pages under the shrinker)
		1711	* or as the object is itself released.
		1712	*/
		1713	int
		1714	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1715	{
3031	serge	1716	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1717	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1718	int ret;
2332	Serge	1719
3243	Serge	1720	if (obj->pages)
3031	serge	1721	return 0;
2332	Serge	1722
4392	Serge	1723	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1724	DRM_DEBUG("Attempting to obtain a purgeable object\n");
		1725	return -EFAULT;
4392	Serge	1726	}
		1727
3031	serge	1728	BUG_ON(obj->pages_pin_count);
2332	Serge	1729
3031	serge	1730	ret = ops->get_pages(obj);
		1731	if (ret)
		1732	return ret;
2344	Serge	1733
4104	Serge	1734	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3243	Serge	1735	return 0;
2332	Serge	1736	}
		1737
5060	serge	1738	static void
2332	Serge	1739	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
5060	serge	1740	struct intel_engine_cs *ring)
2332	Serge	1741	{
3243	Serge	1742	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1743
		1744	BUG_ON(ring == NULL);
4104	Serge	1745	if (obj->ring != ring && obj->last_write_seqno) {
		1746	/* Keep the seqno relative to the current ring */
		1747	obj->last_write_seqno = seqno;
		1748	}
2332	Serge	1749	obj->ring = ring;
		1750
		1751	/* Add a reference if we're newly entering the active list. */
		1752	if (!obj->active) {
2344	Serge	1753	drm_gem_object_reference(&obj->base);
2332	Serge	1754	obj->active = 1;
		1755	}
		1756
		1757	list_move_tail(&obj->ring_list, &ring->active_list);
		1758
3031	serge	1759	obj->last_read_seqno = seqno;
2332	Serge	1760	}
		1761
4560	Serge	1762	void i915_vma_move_to_active(struct i915_vma *vma,
5060	serge	1763	struct intel_engine_cs *ring)
4560	Serge	1764	{
		1765	list_move_tail(&vma->mm_list, &vma->vm->active_list);
		1766	return i915_gem_object_move_to_active(vma->obj, ring);
		1767	}
		1768
2344	Serge	1769	static void
3031	serge	1770	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1771	{
4104	Serge	1772	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
5060	serge	1773	struct i915_address_space *vm;
		1774	struct i915_vma *vma;
2332	Serge	1775
3031	serge	1776	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1777	BUG_ON(!obj->active);
2332	Serge	1778
5060	serge	1779	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
		1780	vma = i915_gem_obj_to_vma(obj, vm);
		1781	if (vma && !list_empty(&vma->mm_list))
		1782	list_move_tail(&vma->mm_list, &vm->inactive_list);
		1783	}
2344	Serge	1784
5354	serge	1785	intel_fb_obj_flush(obj, true);
		1786
3031	serge	1787	list_del_init(&obj->ring_list);
2352	Serge	1788	obj->ring = NULL;
2344	Serge	1789
3031	serge	1790	obj->last_read_seqno = 0;
		1791	obj->last_write_seqno = 0;
		1792	obj->base.write_domain = 0;
		1793
		1794	obj->last_fenced_seqno = 0;
2344	Serge	1795
2352	Serge	1796	obj->active = 0;
		1797	drm_gem_object_unreference(&obj->base);
		1798
		1799	WARN_ON(i915_verify_lists(dev));
		1800	}
		1801
5060	serge	1802	static void
		1803	i915_gem_object_retire(struct drm_i915_gem_object *obj)
		1804	{
		1805	struct intel_engine_cs *ring = obj->ring;
		1806
		1807	if (ring == NULL)
		1808	return;
		1809
		1810	if (i915_seqno_passed(ring->get_seqno(ring, true),
		1811	obj->last_read_seqno))
		1812	i915_gem_object_move_to_inactive(obj);
		1813	}
		1814
3243	Serge	1815	static int
3480	Serge	1816	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1817	{
3243	Serge	1818	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	1819	struct intel_engine_cs *ring;
3243	Serge	1820	int ret, i, j;
2344	Serge	1821
3480	Serge	1822	/* Carefully retire all requests without writing to the rings */
3243	Serge	1823	for_each_ring(ring, dev_priv, i) {
3480	Serge	1824	ret = intel_ring_idle(ring);
3243	Serge	1825	if (ret)
		1826	return ret;
3480	Serge	1827	}
		1828	i915_gem_retire_requests(dev);
3243	Serge	1829
3480	Serge	1830	/* Finally reset hw state */
3243	Serge	1831	for_each_ring(ring, dev_priv, i) {
3480	Serge	1832	intel_ring_init_seqno(ring, seqno);
		1833
5060	serge	1834	for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
		1835	ring->semaphore.sync_seqno[j] = 0;
3243	Serge	1836	}
		1837
		1838	return 0;
2344	Serge	1839	}
		1840
3480	Serge	1841	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1842	{
		1843	struct drm_i915_private *dev_priv = dev->dev_private;
		1844	int ret;
		1845
		1846	if (seqno == 0)
		1847	return -EINVAL;
		1848
		1849	/* HWS page needs to be set less than what we
		1850	* will inject to ring
		1851	*/
		1852	ret = i915_gem_init_seqno(dev, seqno - 1);
		1853	if (ret)
		1854	return ret;
		1855
		1856	/* Carefully set the last_seqno value so that wrap
		1857	* detection still works
		1858	*/
		1859	dev_priv->next_seqno = seqno;
		1860	dev_priv->last_seqno = seqno - 1;
		1861	if (dev_priv->last_seqno == 0)
		1862	dev_priv->last_seqno--;
		1863
		1864	return 0;
		1865	}
		1866
3243	Serge	1867	int
		1868	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1869	{
3243	Serge	1870	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1871
3243	Serge	1872	/* reserve 0 for non-seqno */
		1873	if (dev_priv->next_seqno == 0) {
3480	Serge	1874	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1875	if (ret)
		1876	return ret;
		1877
		1878	dev_priv->next_seqno = 1;
		1879	}
		1880
3480	Serge	1881	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1882	return 0;
2332	Serge	1883	}
		1884
5060	serge	1885	int __i915_add_request(struct intel_engine_cs *ring,
2352	Serge	1886	struct drm_file *file,
4104	Serge	1887	struct drm_i915_gem_object *obj,
3031	serge	1888	u32 *out_seqno)
2352	Serge	1889	{
5060	serge	1890	struct drm_i915_private *dev_priv = ring->dev->dev_private;
3031	serge	1891	struct drm_i915_gem_request *request;
5354	serge	1892	struct intel_ringbuffer *ringbuf;
4104	Serge	1893	u32 request_ring_position, request_start;
2352	Serge	1894	int ret;
2332	Serge	1895
5354	serge	1896	request = ring->preallocated_lazy_request;
		1897	if (WARN_ON(request == NULL))
		1898	return -ENOMEM;
		1899
		1900	if (i915.enable_execlists) {
		1901	struct intel_context *ctx = request->ctx;
		1902	ringbuf = ctx->engine[ring->id].ringbuf;
		1903	} else
		1904	ringbuf = ring->buffer;
		1905
		1906	request_start = intel_ring_get_tail(ringbuf);
3031	serge	1907	/*
		1908	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1909	* after having emitted the batchbuffer command. Hence we need to fix
		1910	* things up similar to emitting the lazy request. The difference here
		1911	* is that the flush _must_ happen before the next request, no matter
		1912	* what.
		1913	*/
5354	serge	1914	if (i915.enable_execlists) {
		1915	ret = logical_ring_flush_all_caches(ringbuf);
		1916	if (ret)
		1917	return ret;
		1918	} else {
4104	Serge	1919	ret = intel_ring_flush_all_caches(ring);
		1920	if (ret)
		1921	return ret;
5354	serge	1922	}
2332	Serge	1923
3031	serge	1924	/* Record the position of the start of the request so that
		1925	* should we detect the updated seqno part-way through the
4104	Serge	1926	* GPU processing the request, we never over-estimate the
3031	serge	1927	* position of the head.
		1928	*/
5354	serge	1929	request_ring_position = intel_ring_get_tail(ringbuf);
3031	serge	1930
5354	serge	1931	if (i915.enable_execlists) {
		1932	ret = ring->emit_request(ringbuf);
		1933	if (ret)
		1934	return ret;
		1935	} else {
3243	Serge	1936	ret = ring->add_request(ring);
4560	Serge	1937	if (ret)
4104	Serge	1938	return ret;
5354	serge	1939	}
2332	Serge	1940
3243	Serge	1941	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1942	request->ring = ring;
4104	Serge	1943	request->head = request_start;
3031	serge	1944	request->tail = request_ring_position;
4104	Serge	1945
		1946	/* Whilst this request exists, batch_obj will be on the
		1947	* active_list, and so will hold the active reference. Only when this
		1948	* request is retired will the the batch_obj be moved onto the
		1949	* inactive_list and lose its active reference. Hence we do not need
		1950	* to explicitly hold another reference here.
		1951	*/
4560	Serge	1952	request->batch_obj = obj;
4104	Serge	1953
5354	serge	1954	if (!i915.enable_execlists) {
4560	Serge	1955	/* Hold a reference to the current context so that we can inspect
		1956	* it later in case a hangcheck error event fires.
		1957	*/
		1958	request->ctx = ring->last_context;
4104	Serge	1959	if (request->ctx)
		1960	i915_gem_context_reference(request->ctx);
5354	serge	1961	}
4104	Serge	1962
5060	serge	1963	request->emitted_jiffies = jiffies;
2352	Serge	1964	list_add_tail(&request->list, &ring->request_list);
3031	serge	1965	request->file_priv = NULL;
2332	Serge	1966
3263	Serge	1967	if (file) {
		1968	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1969
3263	Serge	1970	spin_lock(&file_priv->mm.lock);
		1971	request->file_priv = file_priv;
		1972	list_add_tail(&request->client_list,
		1973	&file_priv->mm.request_list);
		1974	spin_unlock(&file_priv->mm.lock);
		1975	}
		1976
		1977	trace_i915_gem_request_add(ring, request->seqno);
4560	Serge	1978	ring->outstanding_lazy_seqno = 0;
		1979	ring->preallocated_lazy_request = NULL;
2332	Serge	1980
4104	Serge	1981	// i915_queue_hangcheck(ring->dev);
		1982
2360	Serge	1983	queue_delayed_work(dev_priv->wq,
3482	Serge	1984	&dev_priv->mm.retire_work,
		1985	round_jiffies_up_relative(HZ));
4104	Serge	1986	intel_mark_busy(dev_priv->dev);
3031	serge	1987
		1988	if (out_seqno)
3243	Serge	1989	*out_seqno = request->seqno;
2352	Serge	1990	return 0;
		1991	}
2332	Serge	1992
3263	Serge	1993	static inline void
		1994	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1995	{
		1996	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1997
3263	Serge	1998	if (!file_priv)
		1999	return;
2332	Serge	2000
3263	Serge	2001	spin_lock(&file_priv->mm.lock);
		2002	list_del(&request->client_list);
		2003	request->file_priv = NULL;
		2004	spin_unlock(&file_priv->mm.lock);
		2005	}
2332	Serge	2006
5060	serge	2007	static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
		2008	const struct intel_context *ctx)
4104	Serge	2009	{
5060	serge	2010	unsigned long elapsed;
4104	Serge	2011
5060	serge	2012	elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts;
4104	Serge	2013
5060	serge	2014	if (ctx->hang_stats.banned)
		2015	return true;
4104	Serge	2016
5060	serge	2017	if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
		2018	if (!i915_gem_context_is_default(ctx)) {
		2019	DRM_DEBUG("context hanging too fast, banning!\n");
4104	Serge	2020	return true;
5060	serge	2021	} else if (i915_stop_ring_allow_ban(dev_priv)) {
		2022	if (i915_stop_ring_allow_warn(dev_priv))
		2023	DRM_ERROR("gpu hanging too fast, banning!\n");
4104	Serge	2024	return true;
		2025	}
		2026	}
		2027
		2028	return false;
		2029	}
		2030
5060	serge	2031	static void i915_set_reset_status(struct drm_i915_private *dev_priv,
		2032	struct intel_context *ctx,
		2033	const bool guilty)
4560	Serge	2034	{
5060	serge	2035	struct i915_ctx_hang_stats *hs;
4560	Serge	2036
5060	serge	2037	if (WARN_ON(!ctx))
		2038	return;
4560	Serge	2039
5060	serge	2040	hs = &ctx->hang_stats;
4560	Serge	2041
5060	serge	2042	if (guilty) {
		2043	hs->banned = i915_context_is_banned(dev_priv, ctx);
		2044	hs->batch_active++;
		2045	hs->guilty_ts = GetTimerTicks()/100;
		2046	} else {
		2047	hs->batch_pending++;
4104	Serge	2048	}
		2049	}
		2050
		2051	static void i915_gem_free_request(struct drm_i915_gem_request *request)
		2052	{
5354	serge	2053	struct intel_context *ctx = request->ctx;
		2054
4104	Serge	2055	list_del(&request->list);
		2056	i915_gem_request_remove_from_client(request);
		2057
5354	serge	2058	if (ctx) {
		2059	if (i915.enable_execlists) {
		2060	struct intel_engine_cs *ring = request->ring;
4104	Serge	2061
5354	serge	2062	if (ctx != ring->default_context)
		2063	intel_lr_context_unpin(ring, ctx);
		2064	}
		2065	i915_gem_context_unreference(ctx);
		2066	}
4104	Serge	2067	kfree(request);
		2068	}
		2069
5060	serge	2070	struct drm_i915_gem_request *
		2071	i915_gem_find_active_request(struct intel_engine_cs *ring)
3031	serge	2072	{
4539	Serge	2073	struct drm_i915_gem_request *request;
5060	serge	2074	u32 completed_seqno;
4104	Serge	2075
5060	serge	2076	completed_seqno = ring->get_seqno(ring, false);
		2077
4539	Serge	2078	list_for_each_entry(request, &ring->request_list, list) {
		2079	if (i915_seqno_passed(completed_seqno, request->seqno))
		2080	continue;
4104	Serge	2081
5060	serge	2082	return request;
4539	Serge	2083	}
5060	serge	2084
		2085	return NULL;
4539	Serge	2086	}
		2087
5060	serge	2088	static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
		2089	struct intel_engine_cs *ring)
		2090	{
		2091	struct drm_i915_gem_request *request;
		2092	bool ring_hung;
		2093
		2094	request = i915_gem_find_active_request(ring);
		2095
		2096	if (request == NULL)
		2097	return;
		2098
		2099	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
		2100
		2101	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
		2102
		2103	list_for_each_entry_continue(request, &ring->request_list, list)
		2104	i915_set_reset_status(dev_priv, request->ctx, false);
		2105	}
		2106
4539	Serge	2107	static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
5060	serge	2108	struct intel_engine_cs *ring)
4539	Serge	2109	{
4560	Serge	2110	while (!list_empty(&ring->active_list)) {
		2111	struct drm_i915_gem_object *obj;
		2112
		2113	obj = list_first_entry(&ring->active_list,
		2114	struct drm_i915_gem_object,
		2115	ring_list);
		2116
		2117	i915_gem_object_move_to_inactive(obj);
		2118	}
		2119
		2120	/*
5354	serge	2121	* Clear the execlists queue up before freeing the requests, as those
		2122	* are the ones that keep the context and ringbuffer backing objects
		2123	* pinned in place.
		2124	*/
		2125	while (!list_empty(&ring->execlist_queue)) {
		2126	struct intel_ctx_submit_request *submit_req;
		2127
		2128	submit_req = list_first_entry(&ring->execlist_queue,
		2129	struct intel_ctx_submit_request,
		2130	execlist_link);
		2131	list_del(&submit_req->execlist_link);
		2132	intel_runtime_pm_put(dev_priv);
		2133	i915_gem_context_unreference(submit_req->ctx);
		2134	kfree(submit_req);
		2135	}
		2136
		2137	/*
4560	Serge	2138	* We must free the requests after all the corresponding objects have
		2139	* been moved off active lists. Which is the same order as the normal
		2140	* retire_requests function does. This is important if object hold
		2141	* implicit references on things like e.g. ppgtt address spaces through
		2142	* the request.
		2143	*/
3031	serge	2144	while (!list_empty(&ring->request_list)) {
		2145	struct drm_i915_gem_request *request;
2332	Serge	2146
3031	serge	2147	request = list_first_entry(&ring->request_list,
		2148	struct drm_i915_gem_request,
		2149	list);
2332	Serge	2150
4104	Serge	2151	i915_gem_free_request(request);
3031	serge	2152	}
5060	serge	2153
		2154	/* These may not have been flush before the reset, do so now */
		2155	kfree(ring->preallocated_lazy_request);
		2156	ring->preallocated_lazy_request = NULL;
		2157	ring->outstanding_lazy_seqno = 0;
3031	serge	2158	}
2332	Serge	2159
3746	Serge	2160	void i915_gem_restore_fences(struct drm_device *dev)
3031	serge	2161	{
		2162	struct drm_i915_private *dev_priv = dev->dev_private;
		2163	int i;
2332	Serge	2164
3031	serge	2165	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2166	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
4104	Serge	2167
		2168	/*
		2169	* Commit delayed tiling changes if we have an object still
		2170	* attached to the fence, otherwise just clear the fence.
		2171	*/
		2172	if (reg->obj) {
		2173	i915_gem_object_update_fence(reg->obj, reg,
		2174	reg->obj->tiling_mode);
		2175	} else {
		2176	i915_gem_write_fence(dev, i, NULL);
		2177	}
3031	serge	2178	}
		2179	}
2360	Serge	2180
3031	serge	2181	void i915_gem_reset(struct drm_device *dev)
		2182	{
		2183	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2184	struct intel_engine_cs *ring;
3031	serge	2185	int i;
2360	Serge	2186
4539	Serge	2187	/*
		2188	* Before we free the objects from the requests, we need to inspect
		2189	* them for finding the guilty party. As the requests only borrow
		2190	* their reference to the objects, the inspection must be done first.
		2191	*/
3031	serge	2192	for_each_ring(ring, dev_priv, i)
4539	Serge	2193	i915_gem_reset_ring_status(dev_priv, ring);
2360	Serge	2194
4539	Serge	2195	for_each_ring(ring, dev_priv, i)
		2196	i915_gem_reset_ring_cleanup(dev_priv, ring);
		2197
5060	serge	2198	i915_gem_context_reset(dev);
4560	Serge	2199
3746	Serge	2200	i915_gem_restore_fences(dev);
3031	serge	2201	}
2360	Serge	2202
2352	Serge	2203	/**
		2204	* This function clears the request list as sequence numbers are passed.
		2205	*/
3031	serge	2206	void
5060	serge	2207	i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2352	Serge	2208	{
		2209	uint32_t seqno;
2332	Serge	2210
2352	Serge	2211	if (list_empty(&ring->request_list))
		2212	return;
2332	Serge	2213
2352	Serge	2214	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2215
3031	serge	2216	seqno = ring->get_seqno(ring, true);
2332	Serge	2217
5060	serge	2218	/* Move any buffers on the active list that are no longer referenced
		2219	* by the ringbuffer to the flushing/inactive lists as appropriate,
		2220	* before we free the context associated with the requests.
		2221	*/
		2222	while (!list_empty(&ring->active_list)) {
		2223	struct drm_i915_gem_object *obj;
		2224
		2225	obj = list_first_entry(&ring->active_list,
		2226	struct drm_i915_gem_object,
		2227	ring_list);
		2228
		2229	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
		2230	break;
		2231
		2232	i915_gem_object_move_to_inactive(obj);
		2233	}
		2234
		2235
2352	Serge	2236	while (!list_empty(&ring->request_list)) {
		2237	struct drm_i915_gem_request *request;
5354	serge	2238	struct intel_ringbuffer *ringbuf;
2332	Serge	2239
2352	Serge	2240	request = list_first_entry(&ring->request_list,
		2241	struct drm_i915_gem_request,
		2242	list);
2332	Serge	2243
2352	Serge	2244	if (!i915_seqno_passed(seqno, request->seqno))
		2245	break;
2332	Serge	2246
2352	Serge	2247	trace_i915_gem_request_retire(ring, request->seqno);
5354	serge	2248
		2249	/* This is one of the few common intersection points
		2250	* between legacy ringbuffer submission and execlists:
		2251	* we need to tell them apart in order to find the correct
		2252	* ringbuffer to which the request belongs to.
		2253	*/
		2254	if (i915.enable_execlists) {
		2255	struct intel_context *ctx = request->ctx;
		2256	ringbuf = ctx->engine[ring->id].ringbuf;
		2257	} else
		2258	ringbuf = ring->buffer;
		2259
3031	serge	2260	/* We know the GPU must have read the request to have
		2261	* sent us the seqno + interrupt, so use the position
		2262	* of tail of the request to update the last known position
		2263	* of the GPU head.
		2264	*/
5354	serge	2265	ringbuf->last_retired_head = request->tail;
2332	Serge	2266
4104	Serge	2267	i915_gem_free_request(request);
2352	Serge	2268	}
2332	Serge	2269
2352	Serge	2270	if (unlikely(ring->trace_irq_seqno &&
		2271	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2272	ring->irq_put(ring);
		2273	ring->trace_irq_seqno = 0;
		2274	}
2332	Serge	2275
2352	Serge	2276	WARN_ON(i915_verify_lists(ring->dev));
		2277	}
2332	Serge	2278
4560	Serge	2279	bool
2352	Serge	2280	i915_gem_retire_requests(struct drm_device *dev)
		2281	{
5060	serge	2282	struct drm_i915_private *dev_priv = dev->dev_private;
		2283	struct intel_engine_cs *ring;
4560	Serge	2284	bool idle = true;
2352	Serge	2285	int i;
2332	Serge	2286
4560	Serge	2287	for_each_ring(ring, dev_priv, i) {
3031	serge	2288	i915_gem_retire_requests_ring(ring);
4560	Serge	2289	idle &= list_empty(&ring->request_list);
5354	serge	2290	if (i915.enable_execlists) {
		2291	unsigned long flags;
		2292
		2293	spin_lock_irqsave(&ring->execlist_lock, flags);
		2294	idle &= list_empty(&ring->execlist_queue);
		2295	spin_unlock_irqrestore(&ring->execlist_lock, flags);
		2296
		2297	intel_execlists_retire_requests(ring);
		2298	}
4560	Serge	2299	}
		2300
		2301	if (idle)
		2302	mod_delayed_work(dev_priv->wq,
		2303	&dev_priv->mm.idle_work,
		2304	msecs_to_jiffies(100));
		2305
		2306	return idle;
2352	Serge	2307	}
		2308
2360	Serge	2309	static void
		2310	i915_gem_retire_work_handler(struct work_struct *work)
		2311	{
4560	Serge	2312	struct drm_i915_private *dev_priv =
		2313	container_of(work, typeof(*dev_priv), mm.retire_work.work);
		2314	struct drm_device *dev = dev_priv->dev;
2360	Serge	2315	bool idle;
2352	Serge	2316
2360	Serge	2317	/* Come back later if the device is busy... */
4560	Serge	2318	idle = false;
		2319	if (mutex_trylock(&dev->struct_mutex)) {
		2320	idle = i915_gem_retire_requests(dev);
		2321	mutex_unlock(&dev->struct_mutex);
		2322	}
		2323	if (!idle)
3482	Serge	2324	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2325	round_jiffies_up_relative(HZ));
4560	Serge	2326	}
2352	Serge	2327
4560	Serge	2328	static void
		2329	i915_gem_idle_work_handler(struct work_struct *work)
		2330	{
		2331	struct drm_i915_private *dev_priv =
		2332	container_of(work, typeof(*dev_priv), mm.idle_work.work);
2352	Serge	2333
4560	Serge	2334	intel_mark_idle(dev_priv->dev);
2360	Serge	2335	}
		2336
2344	Serge	2337	/**
3031	serge	2338	* Ensures that an object will eventually get non-busy by flushing any required
		2339	* write domains, emitting any outstanding lazy request and retiring and
		2340	* completed requests.
2352	Serge	2341	*/
3031	serge	2342	static int
		2343	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2344	{
3031	serge	2345	int ret;
2352	Serge	2346
3031	serge	2347	if (obj->active) {
		2348	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2349	if (ret)
		2350	return ret;
2352	Serge	2351
3031	serge	2352	i915_gem_retire_requests_ring(obj->ring);
		2353	}
2352	Serge	2354
3031	serge	2355	return 0;
		2356	}
2352	Serge	2357
3243	Serge	2358	/**
		2359	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2360	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2361	*
		2362	* Returns 0 if successful, else an error is returned with the remaining time in
		2363	* the timeout parameter.
		2364	* -ETIME: object is still busy after timeout
		2365	* -ERESTARTSYS: signal interrupted the wait
		2366	* -ENONENT: object doesn't exist
		2367	* Also possible, but rare:
		2368	* -EAGAIN: GPU wedged
		2369	* -ENOMEM: damn
		2370	* -ENODEV: Internal IRQ fail
		2371	* -E?: The add request failed
		2372	*
		2373	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2374	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2375	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2376	* without holding struct_mutex the object may become re-busied before this
		2377	* function completes. A similar but shorter * race condition exists in the busy
		2378	* ioctl
		2379	*/
4246	Serge	2380	int
		2381	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2382	{
5060	serge	2383	struct drm_i915_private *dev_priv = dev->dev_private;
4246	Serge	2384	struct drm_i915_gem_wait *args = data;
		2385	struct drm_i915_gem_object *obj;
5060	serge	2386	struct intel_engine_cs *ring = NULL;
4246	Serge	2387	unsigned reset_counter;
		2388	u32 seqno = 0;
		2389	int ret = 0;
2352	Serge	2390
5354	serge	2391	if (args->flags != 0)
		2392	return -EINVAL;
		2393
4246	Serge	2394	ret = i915_mutex_lock_interruptible(dev);
		2395	if (ret)
		2396	return ret;
2352	Serge	2397
4246	Serge	2398	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2399	if (&obj->base == NULL) {
		2400	mutex_unlock(&dev->struct_mutex);
		2401	return -ENOENT;
		2402	}
2352	Serge	2403
4246	Serge	2404	/* Need to make sure the object gets inactive eventually. */
		2405	ret = i915_gem_object_flush_active(obj);
		2406	if (ret)
		2407	goto out;
2352	Serge	2408
4246	Serge	2409	if (obj->active) {
		2410	seqno = obj->last_read_seqno;
		2411	ring = obj->ring;
		2412	}
2352	Serge	2413
4246	Serge	2414	if (seqno == 0)
		2415	goto out;
2352	Serge	2416
4246	Serge	2417	/* Do this after OLR check to make sure we make forward progress polling
5060	serge	2418	* on this IOCTL with a timeout <=0 (like busy ioctl)
4246	Serge	2419	*/
5060	serge	2420	if (args->timeout_ns <= 0) {
4246	Serge	2421	ret = -ETIME;
		2422	goto out;
		2423	}
2352	Serge	2424
4246	Serge	2425	drm_gem_object_unreference(&obj->base);
		2426	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
		2427	mutex_unlock(&dev->struct_mutex);
2352	Serge	2428
5354	serge	2429	return __i915_wait_seqno(ring, seqno, reset_counter, true,
		2430	&args->timeout_ns, file->driver_priv);
3243	Serge	2431
4246	Serge	2432	out:
		2433	drm_gem_object_unreference(&obj->base);
		2434	mutex_unlock(&dev->struct_mutex);
		2435	return ret;
		2436	}
3243	Serge	2437
2352	Serge	2438	/**
3031	serge	2439	* i915_gem_object_sync - sync an object to a ring.
		2440	*
		2441	* @obj: object which may be in use on another ring.
		2442	* @to: ring we wish to use the object on. May be NULL.
		2443	*
		2444	* This code is meant to abstract object synchronization with the GPU.
		2445	* Calling with NULL implies synchronizing the object with the CPU
		2446	* rather than a particular GPU ring.
		2447	*
		2448	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2449	*/
		2450	int
3031	serge	2451	i915_gem_object_sync(struct drm_i915_gem_object *obj,
5060	serge	2452	struct intel_engine_cs *to)
2344	Serge	2453	{
5060	serge	2454	struct intel_engine_cs *from = obj->ring;
3031	serge	2455	u32 seqno;
		2456	int ret, idx;
2332	Serge	2457
3031	serge	2458	if (from == NULL \|\| to == from)
		2459	return 0;
2332	Serge	2460
3031	serge	2461	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2462	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2463
3031	serge	2464	idx = intel_ring_sync_index(from, to);
		2465
		2466	seqno = obj->last_read_seqno;
5060	serge	2467	/* Optimization: Avoid semaphore sync when we are sure we already
		2468	* waited for an object with higher seqno */
		2469	if (seqno <= from->semaphore.sync_seqno[idx])
3031	serge	2470	return 0;
		2471
		2472	ret = i915_gem_check_olr(obj->ring, seqno);
		2473	if (ret)
		2474	return ret;
		2475
4560	Serge	2476	trace_i915_gem_ring_sync_to(from, to, seqno);
5060	serge	2477	ret = to->semaphore.sync_to(to, from, seqno);
3031	serge	2478	if (!ret)
3243	Serge	2479	/* We use last_read_seqno because sync_to()
		2480	* might have just caused seqno wrap under
		2481	* the radar.
		2482	*/
5060	serge	2483	from->semaphore.sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2484
		2485	return ret;
2344	Serge	2486	}
2332	Serge	2487
2344	Serge	2488	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2489	{
		2490	u32 old_write_domain, old_read_domains;
2332	Serge	2491
2344	Serge	2492	/* Force a pagefault for domain tracking on next user access */
		2493	// i915_gem_release_mmap(obj);
2332	Serge	2494
2344	Serge	2495	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2496	return;
2332	Serge	2497
3480	Serge	2498	/* Wait for any direct GTT access to complete */
		2499	mb();
		2500
2344	Serge	2501	old_read_domains = obj->base.read_domains;
		2502	old_write_domain = obj->base.write_domain;
2351	Serge	2503
2344	Serge	2504	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2505	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2506
2351	Serge	2507	trace_i915_gem_object_change_domain(obj,
		2508	old_read_domains,
		2509	old_write_domain);
2344	Serge	2510	}
2332	Serge	2511
4104	Serge	2512	int i915_vma_unbind(struct i915_vma *vma)
2344	Serge	2513	{
4104	Serge	2514	struct drm_i915_gem_object *obj = vma->obj;
5060	serge	2515	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3480	Serge	2516	int ret;
3263	Serge	2517	if(obj == get_fb_obj())
5367	serge	2518	{
		2519	WARN(1,"attempt to unbind fb object\n");
3263	Serge	2520	return 0;
5367	serge	2521	};
3263	Serge	2522
4104	Serge	2523	if (list_empty(&vma->vma_link))
2344	Serge	2524	return 0;
2332	Serge	2525
4560	Serge	2526	if (!drm_mm_node_allocated(&vma->node)) {
		2527	i915_gem_vma_destroy(vma);
		2528	return 0;
		2529	}
		2530
5060	serge	2531	if (vma->pin_count)
3031	serge	2532	return -EBUSY;
2332	Serge	2533
3243	Serge	2534	BUG_ON(obj->pages == NULL);
3031	serge	2535
2344	Serge	2536	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2537	if (ret)
2344	Serge	2538	return ret;
		2539	/* Continue on if we fail due to EIO, the GPU is hung so we
		2540	* should be safe and we need to cleanup or else we might
		2541	* cause memory corruption through use-after-free.
		2542	*/
2332	Serge	2543
5354	serge	2544	/* Throw away the active reference before moving to the unbound list */
		2545	i915_gem_object_retire(obj);
		2546
5060	serge	2547	if (i915_is_ggtt(vma->vm)) {
2344	Serge	2548	i915_gem_object_finish_gtt(obj);
2332	Serge	2549
2344	Serge	2550	/* release the fence reg _after_ flushing */
		2551	ret = i915_gem_object_put_fence(obj);
3031	serge	2552	if (ret)
2344	Serge	2553	return ret;
5060	serge	2554	}
2332	Serge	2555
4104	Serge	2556	trace_i915_vma_unbind(vma);
2332	Serge	2557
5060	serge	2558	vma->unbind_vma(vma);
2332	Serge	2559
5060	serge	2560	list_del_init(&vma->mm_list);
4104	Serge	2561	if (i915_is_ggtt(vma->vm))
5354	serge	2562	obj->map_and_fenceable = false;
2332	Serge	2563
4104	Serge	2564	drm_mm_remove_node(&vma->node);
		2565	i915_gem_vma_destroy(vma);
		2566
		2567	/* Since the unbound list is global, only move to that list if
4560	Serge	2568	* no more VMAs exist. */
5060	serge	2569	if (list_empty(&obj->vma_list)) {
		2570	i915_gem_gtt_finish_object(obj);
4104	Serge	2571	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
5060	serge	2572	}
4104	Serge	2573
4560	Serge	2574	/* And finally now the object is completely decoupled from this vma,
		2575	* we can drop its hold on the backing storage and allow it to be
		2576	* reaped by the shrinker.
		2577	*/
		2578	i915_gem_object_unpin_pages(obj);
		2579
2344	Serge	2580	return 0;
		2581	}
2332	Serge	2582
3031	serge	2583	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2584	{
5060	serge	2585	struct drm_i915_private *dev_priv = dev->dev_private;
		2586	struct intel_engine_cs *ring;
2344	Serge	2587	int ret, i;
2332	Serge	2588
2344	Serge	2589	/* Flush everything onto the inactive list. */
3031	serge	2590	for_each_ring(ring, dev_priv, i) {
5354	serge	2591	if (!i915.enable_execlists) {
5060	serge	2592	ret = i915_switch_context(ring, ring->default_context);
2344	Serge	2593	if (ret)
		2594	return ret;
5354	serge	2595	}
3031	serge	2596
3243	Serge	2597	ret = intel_ring_idle(ring);
3031	serge	2598	if (ret)
		2599	return ret;
2344	Serge	2600	}
2332	Serge	2601
2344	Serge	2602	return 0;
		2603	}
2332	Serge	2604
3480	Serge	2605	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2606	struct drm_i915_gem_object *obj)
		2607	{
5060	serge	2608	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	2609	int fence_reg;
		2610	int fence_pitch_shift;
2332	Serge	2611
3480	Serge	2612	if (INTEL_INFO(dev)->gen >= 6) {
		2613	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2614	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2615	} else {
		2616	fence_reg = FENCE_REG_965_0;
		2617	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2618	}
2332	Serge	2619
4104	Serge	2620	fence_reg += reg * 8;
		2621
		2622	/* To w/a incoherency with non-atomic 64-bit register updates,
		2623	* we split the 64-bit update into two 32-bit writes. In order
		2624	* for a partial fence not to be evaluated between writes, we
		2625	* precede the update with write to turn off the fence register,
		2626	* and only enable the fence as the last step.
		2627	*
		2628	* For extra levels of paranoia, we make sure each step lands
		2629	* before applying the next step.
		2630	*/
		2631	I915_WRITE(fence_reg, 0);
		2632	POSTING_READ(fence_reg);
		2633
3031	serge	2634	if (obj) {
4104	Serge	2635	u32 size = i915_gem_obj_ggtt_size(obj);
		2636	uint64_t val;
2332	Serge	2637
4104	Serge	2638	val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
3031	serge	2639	0xfffff000) << 32;
4104	Serge	2640	val \|= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
3480	Serge	2641	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2642	if (obj->tiling_mode == I915_TILING_Y)
		2643	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2644	val \|= I965_FENCE_REG_VALID;
2332	Serge	2645
4104	Serge	2646	I915_WRITE(fence_reg + 4, val >> 32);
		2647	POSTING_READ(fence_reg + 4);
		2648
		2649	I915_WRITE(fence_reg + 0, val);
5060	serge	2650	POSTING_READ(fence_reg);
4104	Serge	2651	} else {
		2652	I915_WRITE(fence_reg + 4, 0);
		2653	POSTING_READ(fence_reg + 4);
		2654	}
3031	serge	2655	}
2332	Serge	2656
3031	serge	2657	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2658	struct drm_i915_gem_object *obj)
		2659	{
5060	serge	2660	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2661	u32 val;
2332	Serge	2662
3031	serge	2663	if (obj) {
4104	Serge	2664	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2665	int pitch_val;
		2666	int tile_width;
2332	Serge	2667
4104	Serge	2668	WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) \|\|
3031	serge	2669	(size & -size) != size \|\|
4104	Serge	2670	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2671	"object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2672	i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2332	Serge	2673
3031	serge	2674	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2675	tile_width = 128;
		2676	else
		2677	tile_width = 512;
2332	Serge	2678
3031	serge	2679	/* Note: pitch better be a power of two tile widths */
		2680	pitch_val = obj->stride / tile_width;
		2681	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2682
4104	Serge	2683	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2684	if (obj->tiling_mode == I915_TILING_Y)
		2685	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2686	val \|= I915_FENCE_SIZE_BITS(size);
		2687	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2688	val \|= I830_FENCE_REG_VALID;
		2689	} else
		2690	val = 0;
2332	Serge	2691
3031	serge	2692	if (reg < 8)
		2693	reg = FENCE_REG_830_0 + reg * 4;
		2694	else
		2695	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2696
3031	serge	2697	I915_WRITE(reg, val);
		2698	POSTING_READ(reg);
		2699	}
2332	Serge	2700
3031	serge	2701	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2702	struct drm_i915_gem_object *obj)
		2703	{
5060	serge	2704	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2705	uint32_t val;
2344	Serge	2706
3031	serge	2707	if (obj) {
4104	Serge	2708	u32 size = i915_gem_obj_ggtt_size(obj);
3031	serge	2709	uint32_t pitch_val;
2344	Serge	2710
4104	Serge	2711	WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) \|\|
3031	serge	2712	(size & -size) != size \|\|
4104	Serge	2713	(i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		2714	"object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		2715	i915_gem_obj_ggtt_offset(obj), size);
2344	Serge	2716
3031	serge	2717	pitch_val = obj->stride / 128;
		2718	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2719
4104	Serge	2720	val = i915_gem_obj_ggtt_offset(obj);
3031	serge	2721	if (obj->tiling_mode == I915_TILING_Y)
		2722	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2723	val \|= I830_FENCE_SIZE_BITS(size);
		2724	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2725	val \|= I830_FENCE_REG_VALID;
		2726	} else
		2727	val = 0;
		2728
		2729	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2730	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2731	}
		2732
3480	Serge	2733	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2734	{
		2735	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2736	}
		2737
3031	serge	2738	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2739	struct drm_i915_gem_object *obj)
2332	Serge	2740	{
3480	Serge	2741	struct drm_i915_private *dev_priv = dev->dev_private;
		2742
		2743	/* Ensure that all CPU reads are completed before installing a fence
		2744	* and all writes before removing the fence.
		2745	*/
		2746	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2747	mb();
		2748
4104	Serge	2749	WARN(obj && (!obj->stride \|\| !obj->tiling_mode),
		2750	"bogus fence setup with stride: 0x%x, tiling mode: %i\n",
		2751	obj->stride, obj->tiling_mode);
		2752
3031	serge	2753	switch (INTEL_INFO(dev)->gen) {
5354	serge	2754	case 9:
4560	Serge	2755	case 8:
3031	serge	2756	case 7:
3480	Serge	2757	case 6:
3031	serge	2758	case 5:
		2759	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2760	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2761	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2762	default: BUG();
3031	serge	2763	}
3480	Serge	2764
		2765	/* And similarly be paranoid that no direct access to this region
		2766	* is reordered to before the fence is installed.
		2767	*/
		2768	if (i915_gem_object_needs_mb(obj))
		2769	mb();
2344	Serge	2770	}
		2771
3031	serge	2772	static inline int fence_number(struct drm_i915_private *dev_priv,
		2773	struct drm_i915_fence_reg *fence)
2344	Serge	2774	{
3031	serge	2775	return fence - dev_priv->fence_regs;
		2776	}
2332	Serge	2777
3031	serge	2778	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2779	struct drm_i915_fence_reg *fence,
		2780	bool enable)
		2781	{
4104	Serge	2782	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2783	int reg = fence_number(dev_priv, fence);
2332	Serge	2784
4104	Serge	2785	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3031	serge	2786
		2787	if (enable) {
4104	Serge	2788	obj->fence_reg = reg;
3031	serge	2789	fence->obj = obj;
		2790	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2791	} else {
		2792	obj->fence_reg = I915_FENCE_REG_NONE;
		2793	fence->obj = NULL;
		2794	list_del_init(&fence->lru_list);
2344	Serge	2795	}
4104	Serge	2796	obj->fence_dirty = false;
3031	serge	2797	}
2344	Serge	2798
3031	serge	2799	static int
3480	Serge	2800	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2801	{
		2802	if (obj->last_fenced_seqno) {
		2803	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2804	if (ret)
		2805	return ret;
2344	Serge	2806
		2807	obj->last_fenced_seqno = 0;
		2808	}
		2809
2332	Serge	2810	return 0;
		2811	}
		2812
		2813	int
2344	Serge	2814	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2815	{
3031	serge	2816	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3746	Serge	2817	struct drm_i915_fence_reg *fence;
2332	Serge	2818	int ret;
		2819
3480	Serge	2820	ret = i915_gem_object_wait_fence(obj);
5060	serge	2821	if (ret)
		2822	return ret;
2332	Serge	2823
3031	serge	2824	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2825	return 0;
2332	Serge	2826
3746	Serge	2827	fence = &dev_priv->fence_regs[obj->fence_reg];
		2828
5060	serge	2829	if (WARN_ON(fence->pin_count))
		2830	return -EBUSY;
		2831
3031	serge	2832	i915_gem_object_fence_lost(obj);
3746	Serge	2833	i915_gem_object_update_fence(obj, fence, false);
2344	Serge	2834
2332	Serge	2835	return 0;
		2836	}
		2837
3031	serge	2838	static struct drm_i915_fence_reg *
		2839	i915_find_fence_reg(struct drm_device *dev)
		2840	{
		2841	struct drm_i915_private *dev_priv = dev->dev_private;
		2842	struct drm_i915_fence_reg reg, avail;
		2843	int i;
2332	Serge	2844
3031	serge	2845	/* First try to find a free reg */
		2846	avail = NULL;
		2847	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2848	reg = &dev_priv->fence_regs[i];
		2849	if (!reg->obj)
		2850	return reg;
2332	Serge	2851
3031	serge	2852	if (!reg->pin_count)
		2853	avail = reg;
		2854	}
2332	Serge	2855
3031	serge	2856	if (avail == NULL)
4560	Serge	2857	goto deadlock;
2332	Serge	2858
3031	serge	2859	/* None available, try to steal one or wait for a user to finish */
		2860	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2861	if (reg->pin_count)
		2862	continue;
2332	Serge	2863
3031	serge	2864	return reg;
		2865	}
2332	Serge	2866
4560	Serge	2867	deadlock:
		2868	/* Wait for completion of pending flips which consume fences */
		2869	// if (intel_has_pending_fb_unpin(dev))
		2870	// return ERR_PTR(-EAGAIN);
		2871
		2872	return ERR_PTR(-EDEADLK);
3031	serge	2873	}
2332	Serge	2874
3031	serge	2875	/**
		2876	* i915_gem_object_get_fence - set up fencing for an object
		2877	* @obj: object to map through a fence reg
		2878	*
		2879	* When mapping objects through the GTT, userspace wants to be able to write
		2880	* to them without having to worry about swizzling if the object is tiled.
		2881	* This function walks the fence regs looking for a free one for @obj,
		2882	* stealing one if it can't find any.
		2883	*
		2884	* It then sets up the reg based on the object's properties: address, pitch
		2885	* and tiling format.
		2886	*
		2887	* For an untiled surface, this removes any existing fence.
		2888	*/
		2889	int
		2890	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2891	{
		2892	struct drm_device *dev = obj->base.dev;
		2893	struct drm_i915_private *dev_priv = dev->dev_private;
		2894	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2895	struct drm_i915_fence_reg *reg;
		2896	int ret;
2332	Serge	2897
3031	serge	2898	/* Have we updated the tiling parameters upon the object and so
		2899	* will need to serialise the write to the associated fence register?
		2900	*/
		2901	if (obj->fence_dirty) {
3480	Serge	2902	ret = i915_gem_object_wait_fence(obj);
3031	serge	2903	if (ret)
		2904	return ret;
		2905	}
2332	Serge	2906
3031	serge	2907	/* Just update our place in the LRU if our fence is getting reused. */
		2908	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2909	reg = &dev_priv->fence_regs[obj->fence_reg];
		2910	if (!obj->fence_dirty) {
		2911	list_move_tail(®->lru_list,
		2912	&dev_priv->mm.fence_list);
		2913	return 0;
		2914	}
		2915	} else if (enable) {
5354	serge	2916	if (WARN_ON(!obj->map_and_fenceable))
		2917	return -EINVAL;
		2918
3031	serge	2919	reg = i915_find_fence_reg(dev);
4560	Serge	2920	if (IS_ERR(reg))
		2921	return PTR_ERR(reg);
2332	Serge	2922
3031	serge	2923	if (reg->obj) {
		2924	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2925
3480	Serge	2926	ret = i915_gem_object_wait_fence(old);
3031	serge	2927	if (ret)
		2928	return ret;
2332	Serge	2929
3031	serge	2930	i915_gem_object_fence_lost(old);
		2931	}
		2932	} else
		2933	return 0;
2332	Serge	2934
3031	serge	2935	i915_gem_object_update_fence(obj, reg, enable);
2332	Serge	2936
3031	serge	2937	return 0;
		2938	}
2332	Serge	2939
5354	serge	2940	static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3031	serge	2941	unsigned long cache_level)
		2942	{
5354	serge	2943	struct drm_mm_node *gtt_space = &vma->node;
3031	serge	2944	struct drm_mm_node *other;
2332	Serge	2945
5354	serge	2946	/*
		2947	* On some machines we have to be careful when putting differing types
		2948	* of snoopable memory together to avoid the prefetcher crossing memory
		2949	* domains and dying. During vm initialisation, we decide whether or not
		2950	* these constraints apply and set the drm_mm.color_adjust
		2951	* appropriately.
3031	serge	2952	*/
5354	serge	2953	if (vma->vm->mm.color_adjust == NULL)
3031	serge	2954	return true;
2332	Serge	2955
4104	Serge	2956	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2957	return true;
2332	Serge	2958
3031	serge	2959	if (list_empty(>t_space->node_list))
		2960	return true;
2332	Serge	2961
3031	serge	2962	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2963	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2964	return false;
2344	Serge	2965
3031	serge	2966	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2967	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2968	return false;
2344	Serge	2969
3031	serge	2970	return true;
		2971	}
2344	Serge	2972
2332	Serge	2973	/**
		2974	* Finds free space in the GTT aperture and binds the object there.
		2975	*/
5060	serge	2976	static struct i915_vma *
4104	Serge	2977	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2978	struct i915_address_space *vm,
2332	Serge	2979	unsigned alignment,
5060	serge	2980	uint64_t flags)
2332	Serge	2981	{
		2982	struct drm_device *dev = obj->base.dev;
5060	serge	2983	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	2984	u32 size, fence_size, fence_alignment, unfenced_alignment;
5060	serge	2985	unsigned long start =
		2986	flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
		2987	unsigned long end =
		2988	flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
4104	Serge	2989	struct i915_vma *vma;
2332	Serge	2990	int ret;
2326	Serge	2991
2332	Serge	2992	fence_size = i915_gem_get_gtt_size(dev,
		2993	obj->base.size,
		2994	obj->tiling_mode);
		2995	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2996	obj->base.size,
3480	Serge	2997	obj->tiling_mode, true);
2332	Serge	2998	unfenced_alignment =
3480	Serge	2999	i915_gem_get_gtt_alignment(dev,
2332	Serge	3000	obj->base.size,
3480	Serge	3001	obj->tiling_mode, false);
2332	Serge	3002
		3003	if (alignment == 0)
5060	serge	3004	alignment = flags & PIN_MAPPABLE ? fence_alignment :
2332	Serge	3005	unfenced_alignment;
5060	serge	3006	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
		3007	DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
		3008	return ERR_PTR(-EINVAL);
2332	Serge	3009	}
		3010
5060	serge	3011	size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
2332	Serge	3012
		3013	/* If the object is bigger than the entire aperture, reject it early
		3014	* before evicting everything in a vain attempt to find space.
		3015	*/
5060	serge	3016	if (obj->base.size > end) {
		3017	DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n",
4104	Serge	3018	obj->base.size,
5060	serge	3019	flags & PIN_MAPPABLE ? "mappable" : "total",
		3020	end);
		3021	return ERR_PTR(-E2BIG);
2332	Serge	3022	}
		3023
3031	serge	3024	ret = i915_gem_object_get_pages(obj);
		3025	if (ret)
5060	serge	3026	return ERR_PTR(ret);
3031	serge	3027
3243	Serge	3028	i915_gem_object_pin_pages(obj);
		3029
4104	Serge	3030	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
5060	serge	3031	if (IS_ERR(vma))
4104	Serge	3032	goto err_unpin;
3243	Serge	3033
4104	Serge	3034	search_free:
		3035	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3036	size, alignment,
5060	serge	3037	obj->cache_level,
		3038	start, end,
		3039	DRM_MM_SEARCH_DEFAULT,
		3040	DRM_MM_CREATE_DEFAULT);
3243	Serge	3041	if (ret) {
2332	Serge	3042
4104	Serge	3043	goto err_free_vma;
2332	Serge	3044	}
5354	serge	3045	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
4104	Serge	3046	ret = -EINVAL;
		3047	goto err_remove_node;
3031	serge	3048	}
2332	Serge	3049
3031	serge	3050	ret = i915_gem_gtt_prepare_object(obj);
4104	Serge	3051	if (ret)
		3052	goto err_remove_node;
2332	Serge	3053
4104	Serge	3054	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3055	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3056
5060	serge	3057	trace_i915_vma_bind(vma, flags);
		3058	vma->bind_vma(vma, obj->cache_level,
5354	serge	3059	flags & PIN_GLOBAL ? GLOBAL_BIND : 0);
5060	serge	3060
		3061	return vma;
4104	Serge	3062
		3063	err_remove_node:
		3064	drm_mm_remove_node(&vma->node);
		3065	err_free_vma:
		3066	i915_gem_vma_destroy(vma);
5060	serge	3067	vma = ERR_PTR(ret);
4104	Serge	3068	err_unpin:
		3069	i915_gem_object_unpin_pages(obj);
5060	serge	3070	return vma;
2332	Serge	3071	}
		3072
4104	Serge	3073	bool
		3074	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3075	bool force)
2332	Serge	3076	{
		3077	/* If we don't have a page list set up, then we're not pinned
		3078	* to GPU, and we can ignore the cache flush because it'll happen
		3079	* again at bind time.
		3080	*/
3243	Serge	3081	if (obj->pages == NULL)
4104	Serge	3082	return false;
2332	Serge	3083
3480	Serge	3084	/*
		3085	* Stolen memory is always coherent with the GPU as it is explicitly
		3086	* marked as wc by the system, or the system is cache-coherent.
		3087	*/
5354	serge	3088	if (obj->stolen \|\| obj->phys_handle)
4104	Serge	3089	return false;
3480	Serge	3090
2332	Serge	3091	/* If the GPU is snooping the contents of the CPU cache,
		3092	* we do not need to manually clear the CPU cache lines. However,
		3093	* the caches are only snooped when the render cache is
		3094	* flushed/invalidated. As we always have to emit invalidations
		3095	* and flushes when moving into and out of the RENDER domain, correct
		3096	* snooping behaviour occurs naturally as the result of our domain
		3097	* tracking.
		3098	*/
4104	Serge	3099	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		3100	return false;
2332	Serge	3101
4293	Serge	3102	trace_i915_gem_object_clflush(obj);
		3103	drm_clflush_sg(obj->pages);
2344	Serge	3104
4104	Serge	3105	return true;
2332	Serge	3106	}
		3107
2344	Serge	3108	/** Flushes the GTT write domain for the object if it's dirty. */
		3109	static void
		3110	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3111	{
		3112	uint32_t old_write_domain;
2332	Serge	3113
2344	Serge	3114	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3115	return;
2332	Serge	3116
2344	Serge	3117	/* No actual flushing is required for the GTT write domain. Writes
		3118	* to it immediately go to main memory as far as we know, so there's
		3119	* no chipset flush. It also doesn't land in render cache.
		3120	*
		3121	* However, we do have to enforce the order so that all writes through
		3122	* the GTT land before any writes to the device, such as updates to
		3123	* the GATT itself.
		3124	*/
		3125	wmb();
2332	Serge	3126
2344	Serge	3127	old_write_domain = obj->base.write_domain;
		3128	obj->base.write_domain = 0;
2332	Serge	3129
5354	serge	3130	intel_fb_obj_flush(obj, false);
		3131
2351	Serge	3132	trace_i915_gem_object_change_domain(obj,
		3133	obj->base.read_domains,
		3134	old_write_domain);
2344	Serge	3135	}
2332	Serge	3136
		3137	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3138	static void
4104	Serge	3139	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
		3140	bool force)
2332	Serge	3141	{
		3142	uint32_t old_write_domain;
		3143
		3144	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3145	return;
		3146
4104	Serge	3147	if (i915_gem_clflush_object(obj, force))
3243	Serge	3148	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3149
2332	Serge	3150	old_write_domain = obj->base.write_domain;
		3151	obj->base.write_domain = 0;
		3152
5354	serge	3153	intel_fb_obj_flush(obj, false);
		3154
2351	Serge	3155	trace_i915_gem_object_change_domain(obj,
		3156	obj->base.read_domains,
		3157	old_write_domain);
2332	Serge	3158	}
		3159
		3160	/**
		3161	* Moves a single object to the GTT read, and possibly write domain.
		3162	*
		3163	* This function returns when the move is complete, including waiting on
		3164	* flushes to occur.
		3165	*/
		3166	int
		3167	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3168	{
5060	serge	3169	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
5354	serge	3170	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
2332	Serge	3171	uint32_t old_write_domain, old_read_domains;
		3172	int ret;
		3173
		3174	/* Not valid to be called on unbound objects. */
5354	serge	3175	if (vma == NULL)
2332	Serge	3176	return -EINVAL;
		3177
		3178	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3179	return 0;
		3180
3031	serge	3181	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	3182	if (ret)
		3183	return ret;
		3184
5060	serge	3185	i915_gem_object_retire(obj);
4104	Serge	3186	i915_gem_object_flush_cpu_write_domain(obj, false);
2332	Serge	3187
3480	Serge	3188	/* Serialise direct access to this object with the barriers for
		3189	* coherent writes from the GPU, by effectively invalidating the
		3190	* GTT domain upon first access.
		3191	*/
		3192	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3193	mb();
		3194
2332	Serge	3195	old_write_domain = obj->base.write_domain;
		3196	old_read_domains = obj->base.read_domains;
		3197
		3198	/* It should now be out of any other write domains, and we can update
		3199	* the domain values for our changes.
		3200	*/
		3201	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3202	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3203	if (write) {
		3204	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3205	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3206	obj->dirty = 1;
		3207	}
		3208
5354	serge	3209	if (write)
		3210	intel_fb_obj_invalidate(obj, NULL);
		3211
2351	Serge	3212	trace_i915_gem_object_change_domain(obj,
		3213	old_read_domains,
		3214	old_write_domain);
		3215
3031	serge	3216	/* And bump the LRU for this access */
5354	serge	3217	if (i915_gem_object_is_inactive(obj))
4104	Serge	3218	list_move_tail(&vma->mm_list,
		3219	&dev_priv->gtt.base.inactive_list);
3031	serge	3220
2332	Serge	3221	return 0;
		3222	}
		3223
2335	Serge	3224	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3225	enum i915_cache_level cache_level)
		3226	{
3031	serge	3227	struct drm_device *dev = obj->base.dev;
5060	serge	3228	struct i915_vma vma, next;
2335	Serge	3229	int ret;
2332	Serge	3230
2335	Serge	3231	if (obj->cache_level == cache_level)
		3232	return 0;
2332	Serge	3233
5060	serge	3234	if (i915_gem_obj_is_pinned(obj)) {
2335	Serge	3235	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3236	return -EBUSY;
		3237	}
2332	Serge	3238
5060	serge	3239	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
5354	serge	3240	if (!i915_gem_valid_gtt_space(vma, cache_level)) {
4104	Serge	3241	ret = i915_vma_unbind(vma);
3031	serge	3242	if (ret)
		3243	return ret;
4104	Serge	3244	}
3031	serge	3245	}
		3246
4104	Serge	3247	if (i915_gem_obj_bound_any(obj)) {
2335	Serge	3248	ret = i915_gem_object_finish_gpu(obj);
		3249	if (ret)
		3250	return ret;
2332	Serge	3251
2335	Serge	3252	i915_gem_object_finish_gtt(obj);
2332	Serge	3253
2335	Serge	3254	/* Before SandyBridge, you could not use tiling or fence
		3255	* registers with snooped memory, so relinquish any fences
		3256	* currently pointing to our region in the aperture.
		3257	*/
3031	serge	3258	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3259	ret = i915_gem_object_put_fence(obj);
		3260	if (ret)
		3261	return ret;
5060	serge	3262	}
2332	Serge	3263
5060	serge	3264	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3265	if (drm_mm_node_allocated(&vma->node))
		3266	vma->bind_vma(vma, cache_level,
5354	serge	3267	vma->bound & GLOBAL_BIND);
2335	Serge	3268	}
2332	Serge	3269
4104	Serge	3270	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3271	vma->node.color = cache_level;
		3272	obj->cache_level = cache_level;
		3273
		3274	if (cpu_write_needs_clflush(obj)) {
2335	Serge	3275	u32 old_read_domains, old_write_domain;
2332	Serge	3276
2335	Serge	3277	/* If we're coming from LLC cached, then we haven't
		3278	* actually been tracking whether the data is in the
		3279	* CPU cache or not, since we only allow one bit set
		3280	* in obj->write_domain and have been skipping the clflushes.
		3281	* Just set it to the CPU cache for now.
		3282	*/
5060	serge	3283	i915_gem_object_retire(obj);
2335	Serge	3284	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3285
2335	Serge	3286	old_read_domains = obj->base.read_domains;
		3287	old_write_domain = obj->base.write_domain;
2332	Serge	3288
2335	Serge	3289	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3290	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3291
2351	Serge	3292	trace_i915_gem_object_change_domain(obj,
		3293	old_read_domains,
		3294	old_write_domain);
2344	Serge	3295	}
2332	Serge	3296
2335	Serge	3297	return 0;
		3298	}
2332	Serge	3299
3260	Serge	3300	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3301	struct drm_file *file)
		3302	{
		3303	struct drm_i915_gem_caching *args = data;
		3304	struct drm_i915_gem_object *obj;
		3305	int ret;
		3306
		3307	ret = i915_mutex_lock_interruptible(dev);
		3308	if (ret)
		3309	return ret;
		3310
		3311	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3312	if (&obj->base == NULL) {
		3313	ret = -ENOENT;
		3314	goto unlock;
		3315	}
		3316
4104	Serge	3317	switch (obj->cache_level) {
		3318	case I915_CACHE_LLC:
		3319	case I915_CACHE_L3_LLC:
		3320	args->caching = I915_CACHING_CACHED;
		3321	break;
3260	Serge	3322
4104	Serge	3323	case I915_CACHE_WT:
		3324	args->caching = I915_CACHING_DISPLAY;
		3325	break;
		3326
		3327	default:
		3328	args->caching = I915_CACHING_NONE;
		3329	break;
		3330	}
		3331
3260	Serge	3332	drm_gem_object_unreference(&obj->base);
		3333	unlock:
		3334	mutex_unlock(&dev->struct_mutex);
		3335	return ret;
		3336	}
		3337
		3338	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3339	struct drm_file *file)
		3340	{
		3341	struct drm_i915_gem_caching *args = data;
		3342	struct drm_i915_gem_object *obj;
		3343	enum i915_cache_level level;
		3344	int ret;
		3345
		3346	switch (args->caching) {
		3347	case I915_CACHING_NONE:
		3348	level = I915_CACHE_NONE;
		3349	break;
		3350	case I915_CACHING_CACHED:
		3351	level = I915_CACHE_LLC;
		3352	break;
4104	Serge	3353	case I915_CACHING_DISPLAY:
		3354	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3355	break;
3260	Serge	3356	default:
		3357	return -EINVAL;
		3358	}
		3359
		3360	ret = i915_mutex_lock_interruptible(dev);
		3361	if (ret)
		3362	return ret;
		3363
		3364	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3365	if (&obj->base == NULL) {
		3366	ret = -ENOENT;
		3367	goto unlock;
		3368	}
		3369
		3370	ret = i915_gem_object_set_cache_level(obj, level);
		3371
		3372	drm_gem_object_unreference(&obj->base);
		3373	unlock:
		3374	mutex_unlock(&dev->struct_mutex);
		3375	return ret;
		3376	}
		3377
4104	Serge	3378	static bool is_pin_display(struct drm_i915_gem_object *obj)
		3379	{
5060	serge	3380	struct i915_vma *vma;
		3381
		3382	vma = i915_gem_obj_to_ggtt(obj);
		3383	if (!vma)
		3384	return false;
		3385
4104	Serge	3386	/* There are 3 sources that pin objects:
		3387	* 1. The display engine (scanouts, sprites, cursors);
		3388	* 2. Reservations for execbuffer;
		3389	* 3. The user.
		3390	*
		3391	* We can ignore reservations as we hold the struct_mutex and
		3392	* are only called outside of the reservation path. The user
		3393	* can only increment pin_count once, and so if after
		3394	* subtracting the potential reference by the user, any pin_count
		3395	* remains, it must be due to another use by the display engine.
		3396	*/
5060	serge	3397	return vma->pin_count - !!obj->user_pin_count;
4104	Serge	3398	}
		3399
2335	Serge	3400	/*
		3401	* Prepare buffer for display plane (scanout, cursors, etc).
		3402	* Can be called from an uninterruptible phase (modesetting) and allows
		3403	* any flushes to be pipelined (for pageflips).
		3404	*/
		3405	int
		3406	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3407	u32 alignment,
5060	serge	3408	struct intel_engine_cs *pipelined)
2335	Serge	3409	{
		3410	u32 old_read_domains, old_write_domain;
5060	serge	3411	bool was_pin_display;
2335	Serge	3412	int ret;
2332	Serge	3413
3031	serge	3414	if (pipelined != obj->ring) {
		3415	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3416	if (ret)
		3417	return ret;
		3418	}
2332	Serge	3419
4104	Serge	3420	/* Mark the pin_display early so that we account for the
		3421	* display coherency whilst setting up the cache domains.
		3422	*/
5060	serge	3423	was_pin_display = obj->pin_display;
4104	Serge	3424	obj->pin_display = true;
		3425
2335	Serge	3426	/* The display engine is not coherent with the LLC cache on gen6. As
		3427	* a result, we make sure that the pinning that is about to occur is
		3428	* done with uncached PTEs. This is lowest common denominator for all
		3429	* chipsets.
		3430	*
		3431	* However for gen6+, we could do better by using the GFDT bit instead
		3432	* of uncaching, which would allow us to flush all the LLC-cached data
		3433	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3434	*/
4104	Serge	3435	ret = i915_gem_object_set_cache_level(obj,
		3436	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3437	if (ret)
4104	Serge	3438	goto err_unpin_display;
2332	Serge	3439
2335	Serge	3440	/* As the user may map the buffer once pinned in the display plane
		3441	* (e.g. libkms for the bootup splash), we have to ensure that we
		3442	* always use map_and_fenceable for all scanout buffers.
		3443	*/
5060	serge	3444	ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
2335	Serge	3445	if (ret)
4104	Serge	3446	goto err_unpin_display;
2332	Serge	3447
4104	Serge	3448	i915_gem_object_flush_cpu_write_domain(obj, true);
2332	Serge	3449
2335	Serge	3450	old_write_domain = obj->base.write_domain;
		3451	old_read_domains = obj->base.read_domains;
2332	Serge	3452
2335	Serge	3453	/* It should now be out of any other write domains, and we can update
		3454	* the domain values for our changes.
		3455	*/
3031	serge	3456	obj->base.write_domain = 0;
2335	Serge	3457	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3458
2351	Serge	3459	trace_i915_gem_object_change_domain(obj,
		3460	old_read_domains,
		3461	old_write_domain);
2332	Serge	3462
2335	Serge	3463	return 0;
4104	Serge	3464
		3465	err_unpin_display:
5060	serge	3466	WARN_ON(was_pin_display != is_pin_display(obj));
		3467	obj->pin_display = was_pin_display;
4104	Serge	3468	return ret;
2335	Serge	3469	}
2332	Serge	3470
4104	Serge	3471	void
		3472	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
		3473	{
5060	serge	3474	i915_gem_object_ggtt_unpin(obj);
4104	Serge	3475	obj->pin_display = is_pin_display(obj);
		3476	}
		3477
2344	Serge	3478	int
		3479	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3480	{
		3481	int ret;
2332	Serge	3482
2344	Serge	3483	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3484	return 0;
2332	Serge	3485
3031	serge	3486	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3487	if (ret)
		3488	return ret;
2332	Serge	3489
2344	Serge	3490	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3491	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3492	return 0;
2344	Serge	3493	}
2332	Serge	3494
2344	Serge	3495	/**
		3496	* Moves a single object to the CPU read, and possibly write domain.
		3497	*
		3498	* This function returns when the move is complete, including waiting on
		3499	* flushes to occur.
		3500	*/
3031	serge	3501	int
2344	Serge	3502	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3503	{
		3504	uint32_t old_write_domain, old_read_domains;
		3505	int ret;
2332	Serge	3506
2344	Serge	3507	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3508	return 0;
2332	Serge	3509
3031	serge	3510	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3511	if (ret)
		3512	return ret;
2332	Serge	3513
5060	serge	3514	i915_gem_object_retire(obj);
2344	Serge	3515	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3516
2344	Serge	3517	old_write_domain = obj->base.write_domain;
		3518	old_read_domains = obj->base.read_domains;
2332	Serge	3519
2344	Serge	3520	/* Flush the CPU cache if it's still invalid. */
		3521	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3522	i915_gem_clflush_object(obj, false);
2332	Serge	3523
2344	Serge	3524	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3525	}
2332	Serge	3526
2344	Serge	3527	/* It should now be out of any other write domains, and we can update
		3528	* the domain values for our changes.
		3529	*/
		3530	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3531
2344	Serge	3532	/* If we're writing through the CPU, then the GPU read domains will
		3533	* need to be invalidated at next use.
		3534	*/
		3535	if (write) {
		3536	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3537	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3538	}
2332	Serge	3539
5354	serge	3540	if (write)
		3541	intel_fb_obj_invalidate(obj, NULL);
		3542
2351	Serge	3543	trace_i915_gem_object_change_domain(obj,
		3544	old_read_domains,
		3545	old_write_domain);
2332	Serge	3546
2344	Serge	3547	return 0;
		3548	}
2332	Serge	3549
3031	serge	3550	/* Throttle our rendering by waiting until the ring has completed our requests
		3551	* emitted over 20 msec ago.
2344	Serge	3552	*
3031	serge	3553	* Note that if we were to use the current jiffies each time around the loop,
		3554	* we wouldn't escape the function with any frames outstanding if the time to
		3555	* render a frame was over 20ms.
		3556	*
		3557	* This should get us reasonable parallelism between CPU and GPU but also
		3558	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3559	*/
3031	serge	3560	static int
		3561	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3562	{
3031	serge	3563	struct drm_i915_private *dev_priv = dev->dev_private;
		3564	struct drm_i915_file_private *file_priv = file->driver_priv;
5060	serge	3565	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3031	serge	3566	struct drm_i915_gem_request *request;
5060	serge	3567	struct intel_engine_cs *ring = NULL;
3480	Serge	3568	unsigned reset_counter;
3031	serge	3569	u32 seqno = 0;
		3570	int ret;
2332	Serge	3571
3480	Serge	3572	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3573	if (ret)
		3574	return ret;
2332	Serge	3575
3480	Serge	3576	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3577	if (ret)
		3578	return ret;
		3579
3031	serge	3580	spin_lock(&file_priv->mm.lock);
		3581	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3582	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3583	break;
2332	Serge	3584
3031	serge	3585	ring = request->ring;
		3586	seqno = request->seqno;
		3587	}
3480	Serge	3588	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3589	spin_unlock(&file_priv->mm.lock);
2332	Serge	3590
3031	serge	3591	if (seqno == 0)
		3592	return 0;
2332	Serge	3593
5354	serge	3594	ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
3031	serge	3595	if (ret == 0)
		3596	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3597
3031	serge	3598	return ret;
2352	Serge	3599	}
2332	Serge	3600
5060	serge	3601	static bool
		3602	i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
		3603	{
		3604	struct drm_i915_gem_object *obj = vma->obj;
		3605
		3606	if (alignment &&
		3607	vma->node.start & (alignment - 1))
		3608	return true;
		3609
		3610	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
		3611	return true;
		3612
		3613	if (flags & PIN_OFFSET_BIAS &&
		3614	vma->node.start < (flags & PIN_OFFSET_MASK))
		3615	return true;
		3616
		3617	return false;
		3618	}
		3619
2332	Serge	3620	int
		3621	i915_gem_object_pin(struct drm_i915_gem_object *obj,
4104	Serge	3622	struct i915_address_space *vm,
2332	Serge	3623	uint32_t alignment,
5060	serge	3624	uint64_t flags)
2332	Serge	3625	{
5060	serge	3626	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4104	Serge	3627	struct i915_vma *vma;
5354	serge	3628	unsigned bound;
2332	Serge	3629	int ret;
		3630
5060	serge	3631	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
		3632	return -ENODEV;
2332	Serge	3633
5060	serge	3634	if (WARN_ON(flags & (PIN_GLOBAL \| PIN_MAPPABLE) && !i915_is_ggtt(vm)))
		3635	return -EINVAL;
4104	Serge	3636
5354	serge	3637	if (WARN_ON((flags & (PIN_MAPPABLE \| PIN_GLOBAL)) == PIN_MAPPABLE))
		3638	return -EINVAL;
		3639
4104	Serge	3640	vma = i915_gem_obj_to_vma(obj, vm);
5060	serge	3641	if (vma) {
		3642	if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3643	return -EBUSY;
4104	Serge	3644
5060	serge	3645	if (i915_vma_misplaced(vma, alignment, flags)) {
		3646	WARN(vma->pin_count,
2332	Serge	3647	"bo is already pinned with incorrect alignment:"
4104	Serge	3648	" offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3649	" obj->map_and_fenceable=%d\n",
4104	Serge	3650	i915_gem_obj_offset(obj, vm), alignment,
5060	serge	3651	!!(flags & PIN_MAPPABLE),
2332	Serge	3652	obj->map_and_fenceable);
4104	Serge	3653	ret = i915_vma_unbind(vma);
2332	Serge	3654	if (ret)
		3655	return ret;
5060	serge	3656
		3657	vma = NULL;
2332	Serge	3658	}
		3659	}
		3660
5354	serge	3661	bound = vma ? vma->bound : 0;
5060	serge	3662	if (vma == NULL \|\| !drm_mm_node_allocated(&vma->node)) {
		3663	vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
		3664	if (IS_ERR(vma))
		3665	return PTR_ERR(vma);
2332	Serge	3666	}
		3667
5354	serge	3668	if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND))
5060	serge	3669	vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
3031	serge	3670
5354	serge	3671	if ((bound ^ vma->bound) & GLOBAL_BIND) {
		3672	bool mappable, fenceable;
		3673	u32 fence_size, fence_alignment;
		3674
		3675	fence_size = i915_gem_get_gtt_size(obj->base.dev,
		3676	obj->base.size,
		3677	obj->tiling_mode);
		3678	fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
		3679	obj->base.size,
		3680	obj->tiling_mode,
		3681	true);
		3682
		3683	fenceable = (vma->node.size == fence_size &&
		3684	(vma->node.start & (fence_alignment - 1)) == 0);
		3685
		3686	mappable = (vma->node.start + obj->base.size <=
		3687	dev_priv->gtt.mappable_end);
		3688
		3689	obj->map_and_fenceable = mappable && fenceable;
		3690	}
		3691
		3692	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
		3693
5060	serge	3694	vma->pin_count++;
		3695	if (flags & PIN_MAPPABLE)
		3696	obj->pin_mappable \|= true;
2332	Serge	3697
		3698	return 0;
		3699	}
		3700
2344	Serge	3701	void
5060	serge	3702	i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
2344	Serge	3703	{
5060	serge	3704	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
2332	Serge	3705
5060	serge	3706	BUG_ON(!vma);
		3707	BUG_ON(vma->pin_count == 0);
		3708	BUG_ON(!i915_gem_obj_ggtt_bound(obj));
		3709
		3710	if (--vma->pin_count == 0)
2344	Serge	3711	obj->pin_mappable = false;
		3712	}
2332	Serge	3713
5060	serge	3714	bool
		3715	i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
		3716	{
		3717	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		3718	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3719	struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
		3720
		3721	WARN_ON(!ggtt_vma \|\|
		3722	dev_priv->fence_regs[obj->fence_reg].pin_count >
		3723	ggtt_vma->pin_count);
		3724	dev_priv->fence_regs[obj->fence_reg].pin_count++;
		3725	return true;
		3726	} else
		3727	return false;
		3728	}
		3729
		3730	void
		3731	i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
		3732	{
		3733	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		3734	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3735	WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
		3736	dev_priv->fence_regs[obj->fence_reg].pin_count--;
		3737	}
		3738	}
		3739
3031	serge	3740	int
		3741	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3742	struct drm_file *file)
		3743	{
		3744	struct drm_i915_gem_pin *args = data;
		3745	struct drm_i915_gem_object *obj;
		3746	int ret;
2332	Serge	3747
5354	serge	3748	if (drm_core_check_feature(dev, DRIVER_MODESET))
5060	serge	3749	return -ENODEV;
		3750
3031	serge	3751	ret = i915_mutex_lock_interruptible(dev);
		3752	if (ret)
		3753	return ret;
2332	Serge	3754
3031	serge	3755	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3756	if (&obj->base == NULL) {
		3757	ret = -ENOENT;
		3758	goto unlock;
		3759	}
2332	Serge	3760
3031	serge	3761	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	3762	DRM_DEBUG("Attempting to pin a purgeable buffer\n");
		3763	ret = -EFAULT;
3031	serge	3764	goto out;
		3765	}
2332	Serge	3766
3031	serge	3767	if (obj->pin_filp != NULL && obj->pin_filp != file) {
5060	serge	3768	DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
3031	serge	3769	args->handle);
		3770	ret = -EINVAL;
		3771	goto out;
		3772	}
2332	Serge	3773
4560	Serge	3774	if (obj->user_pin_count == ULONG_MAX) {
		3775	ret = -EBUSY;
		3776	goto out;
		3777	}
		3778
3243	Serge	3779	if (obj->user_pin_count == 0) {
5060	serge	3780	ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
3031	serge	3781	if (ret)
		3782	goto out;
		3783	}
2332	Serge	3784
3243	Serge	3785	obj->user_pin_count++;
		3786	obj->pin_filp = file;
		3787
4104	Serge	3788	args->offset = i915_gem_obj_ggtt_offset(obj);
3031	serge	3789	out:
		3790	drm_gem_object_unreference(&obj->base);
		3791	unlock:
		3792	mutex_unlock(&dev->struct_mutex);
		3793	return ret;
		3794	}
2332	Serge	3795
3031	serge	3796	int
		3797	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3798	struct drm_file *file)
		3799	{
		3800	struct drm_i915_gem_pin *args = data;
		3801	struct drm_i915_gem_object *obj;
		3802	int ret;
2332	Serge	3803
5354	serge	3804	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3805	return -ENODEV;
		3806
3031	serge	3807	ret = i915_mutex_lock_interruptible(dev);
		3808	if (ret)
		3809	return ret;
2332	Serge	3810
3031	serge	3811	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3812	if (&obj->base == NULL) {
		3813	ret = -ENOENT;
		3814	goto unlock;
		3815	}
2332	Serge	3816
3031	serge	3817	if (obj->pin_filp != file) {
5060	serge	3818	DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3031	serge	3819	args->handle);
		3820	ret = -EINVAL;
		3821	goto out;
		3822	}
		3823	obj->user_pin_count--;
		3824	if (obj->user_pin_count == 0) {
		3825	obj->pin_filp = NULL;
5060	serge	3826	i915_gem_object_ggtt_unpin(obj);
3031	serge	3827	}
2332	Serge	3828
3031	serge	3829	out:
		3830	drm_gem_object_unreference(&obj->base);
		3831	unlock:
		3832	mutex_unlock(&dev->struct_mutex);
		3833	return ret;
		3834	}
2332	Serge	3835
3031	serge	3836	int
		3837	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3838	struct drm_file *file)
		3839	{
		3840	struct drm_i915_gem_busy *args = data;
		3841	struct drm_i915_gem_object *obj;
		3842	int ret;
2332	Serge	3843
3031	serge	3844	ret = i915_mutex_lock_interruptible(dev);
		3845	if (ret)
		3846	return ret;
2332	Serge	3847
5060	serge	3848	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3849	if (&obj->base == NULL) {
		3850	ret = -ENOENT;
		3851	goto unlock;
		3852	}
2332	Serge	3853
3031	serge	3854	/* Count all active objects as busy, even if they are currently not used
		3855	* by the gpu. Users of this interface expect objects to eventually
		3856	* become non-busy without any further actions, therefore emit any
		3857	* necessary flushes here.
		3858	*/
		3859	ret = i915_gem_object_flush_active(obj);
2332	Serge	3860
3031	serge	3861	args->busy = obj->active;
		3862	if (obj->ring) {
		3863	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3864	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3865	}
2332	Serge	3866
3031	serge	3867	drm_gem_object_unreference(&obj->base);
		3868	unlock:
		3869	mutex_unlock(&dev->struct_mutex);
		3870	return ret;
		3871	}
2332	Serge	3872
3031	serge	3873	int
		3874	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3875	struct drm_file *file_priv)
		3876	{
		3877	return i915_gem_ring_throttle(dev, file_priv);
		3878	}
2332	Serge	3879
3263	Serge	3880	#if 0
		3881
3031	serge	3882	int
		3883	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3884	struct drm_file *file_priv)
		3885	{
5354	serge	3886	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3887	struct drm_i915_gem_madvise *args = data;
		3888	struct drm_i915_gem_object *obj;
		3889	int ret;
2332	Serge	3890
3031	serge	3891	switch (args->madv) {
		3892	case I915_MADV_DONTNEED:
		3893	case I915_MADV_WILLNEED:
		3894	break;
		3895	default:
		3896	return -EINVAL;
		3897	}
2332	Serge	3898
3031	serge	3899	ret = i915_mutex_lock_interruptible(dev);
		3900	if (ret)
		3901	return ret;
2332	Serge	3902
3031	serge	3903	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3904	if (&obj->base == NULL) {
		3905	ret = -ENOENT;
		3906	goto unlock;
		3907	}
2332	Serge	3908
5060	serge	3909	if (i915_gem_obj_is_pinned(obj)) {
3031	serge	3910	ret = -EINVAL;
		3911	goto out;
		3912	}
2332	Serge	3913
5354	serge	3914	if (obj->pages &&
		3915	obj->tiling_mode != I915_TILING_NONE &&
		3916	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		3917	if (obj->madv == I915_MADV_WILLNEED)
		3918	i915_gem_object_unpin_pages(obj);
		3919	if (args->madv == I915_MADV_WILLNEED)
		3920	i915_gem_object_pin_pages(obj);
		3921	}
		3922
3031	serge	3923	if (obj->madv != __I915_MADV_PURGED)
		3924	obj->madv = args->madv;
2332	Serge	3925
3031	serge	3926	/* if the object is no longer attached, discard its backing storage */
		3927	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3928	i915_gem_object_truncate(obj);
2332	Serge	3929
3031	serge	3930	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3931
3031	serge	3932	out:
		3933	drm_gem_object_unreference(&obj->base);
		3934	unlock:
		3935	mutex_unlock(&dev->struct_mutex);
		3936	return ret;
		3937	}
		3938	#endif
2332	Serge	3939
3031	serge	3940	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3941	const struct drm_i915_gem_object_ops *ops)
		3942	{
4104	Serge	3943	INIT_LIST_HEAD(&obj->global_list);
3031	serge	3944	INIT_LIST_HEAD(&obj->ring_list);
4104	Serge	3945	INIT_LIST_HEAD(&obj->obj_exec_link);
		3946	INIT_LIST_HEAD(&obj->vma_list);
2332	Serge	3947
3031	serge	3948	obj->ops = ops;
		3949
		3950	obj->fence_reg = I915_FENCE_REG_NONE;
		3951	obj->madv = I915_MADV_WILLNEED;
		3952
		3953	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3954	}
		3955
		3956	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3957	.get_pages = i915_gem_object_get_pages_gtt,
		3958	.put_pages = i915_gem_object_put_pages_gtt,
		3959	};
		3960
2332	Serge	3961	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3962	size_t size)
		3963	{
		3964	struct drm_i915_gem_object *obj;
3031	serge	3965	struct address_space *mapping;
3480	Serge	3966	gfp_t mask;
2340	Serge	3967
3746	Serge	3968	obj = i915_gem_object_alloc(dev);
2332	Serge	3969	if (obj == NULL)
		3970	return NULL;
		3971
		3972	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3973	i915_gem_object_free(obj);
2332	Serge	3974	return NULL;
		3975	}
		3976
		3977
3031	serge	3978	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3979
		3980	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3981	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3982
3031	serge	3983	if (HAS_LLC(dev)) {
		3984	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3985	* cache) for about a 10% performance improvement
		3986	* compared to uncached. Graphics requests other than
		3987	* display scanout are coherent with the CPU in
		3988	* accessing this cache. This means in this mode we
		3989	* don't need to clflush on the CPU side, and on the
		3990	* GPU side we only need to flush internal caches to
		3991	* get data visible to the CPU.
		3992	*
		3993	* However, we maintain the display planes as UC, and so
		3994	* need to rebind when first used as such.
		3995	*/
		3996	obj->cache_level = I915_CACHE_LLC;
		3997	} else
		3998	obj->cache_level = I915_CACHE_NONE;
		3999
4560	Serge	4000	trace_i915_gem_object_create(obj);
		4001
2332	Serge	4002	return obj;
		4003	}
		4004
3031	serge	4005	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	4006	{
3031	serge	4007	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	4008	struct drm_device *dev = obj->base.dev;
5060	serge	4009	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	4010	struct i915_vma vma, next;
2332	Serge	4011
4560	Serge	4012	intel_runtime_pm_get(dev_priv);
		4013
3031	serge	4014	trace_i915_gem_object_destroy(obj);
		4015
5060	serge	4016	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		4017	int ret;
3031	serge	4018
5060	serge	4019	vma->pin_count = 0;
		4020	ret = i915_vma_unbind(vma);
4104	Serge	4021	if (WARN_ON(ret == -ERESTARTSYS)) {
3031	serge	4022	bool was_interruptible;
		4023
		4024	was_interruptible = dev_priv->mm.interruptible;
		4025	dev_priv->mm.interruptible = false;
		4026
4104	Serge	4027	WARN_ON(i915_vma_unbind(vma));
3031	serge	4028
		4029	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	4030	}
4104	Serge	4031	}
2332	Serge	4032
4104	Serge	4033	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		4034	* before progressing. */
		4035	if (obj->stolen)
		4036	i915_gem_object_unpin_pages(obj);
		4037
5060	serge	4038	WARN_ON(obj->frontbuffer_bits);
		4039
5354	serge	4040	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
		4041	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
		4042	obj->tiling_mode != I915_TILING_NONE)
		4043	i915_gem_object_unpin_pages(obj);
		4044
4104	Serge	4045	if (WARN_ON(obj->pages_pin_count))
3031	serge	4046	obj->pages_pin_count = 0;
		4047	i915_gem_object_put_pages(obj);
		4048	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	4049
3243	Serge	4050	BUG_ON(obj->pages);
2332	Serge	4051
3031	serge	4052
3290	Serge	4053	if(obj->base.filp != NULL)
		4054	{
3298	Serge	4055	// printf("filp %p\n", obj->base.filp);
3290	Serge	4056	shmem_file_delete(obj->base.filp);
		4057	}
		4058
2344	Serge	4059	drm_gem_object_release(&obj->base);
		4060	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	4061
2344	Serge	4062	kfree(obj->bit_17);
4104	Serge	4063	i915_gem_object_free(obj);
4560	Serge	4064
		4065	intel_runtime_pm_put(dev_priv);
2344	Serge	4066	}
2332	Serge	4067
4560	Serge	4068	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
4104	Serge	4069	struct i915_address_space *vm)
		4070	{
4560	Serge	4071	struct i915_vma *vma;
		4072	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4073	if (vma->vm == vm)
		4074	return vma;
		4075
		4076	return NULL;
		4077	}
		4078
4104	Serge	4079	void i915_gem_vma_destroy(struct i915_vma *vma)
		4080	{
5354	serge	4081	struct i915_address_space *vm = NULL;
4104	Serge	4082	WARN_ON(vma->node.allocated);
4560	Serge	4083
		4084	/* Keep the vma as a placeholder in the execbuffer reservation lists */
		4085	if (!list_empty(&vma->exec_list))
		4086	return;
		4087
5354	serge	4088	vm = vma->vm;
		4089
		4090	if (!i915_is_ggtt(vm))
		4091	i915_ppgtt_put(i915_vm_to_ppgtt(vm));
		4092
4104	Serge	4093	list_del(&vma->vma_link);
4560	Serge	4094
4104	Serge	4095	kfree(vma);
		4096	}
		4097
3031	serge	4098	#if 0
		4099	int
4560	Serge	4100	i915_gem_suspend(struct drm_device *dev)
2344	Serge	4101	{
5060	serge	4102	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4103	int ret = 0;
2332	Serge	4104
4560	Serge	4105	mutex_lock(&dev->struct_mutex);
3031	serge	4106	ret = i915_gpu_idle(dev);
4560	Serge	4107	if (ret)
		4108	goto err;
		4109
3031	serge	4110	i915_gem_retire_requests(dev);
		4111
3480	Serge	4112	/* Under UMS, be paranoid and evict. */
		4113	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		4114	i915_gem_evict_everything(dev);
		4115
5060	serge	4116	i915_gem_stop_ringbuffers(dev);
4560	Serge	4117	mutex_unlock(&dev->struct_mutex);
		4118
		4119	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3263	Serge	4120	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
5060	serge	4121	flush_delayed_work(&dev_priv->mm.idle_work);
3031	serge	4122
		4123	return 0;
4560	Serge	4124
		4125	err:
		4126	mutex_unlock(&dev->struct_mutex);
		4127	return ret;
2344	Serge	4128	}
3031	serge	4129	#endif
2332	Serge	4130
5060	serge	4131	int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice)
3031	serge	4132	{
4560	Serge	4133	struct drm_device *dev = ring->dev;
5060	serge	4134	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4135	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
		4136	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
		4137	int i, ret;
2332	Serge	4138
4560	Serge	4139	if (!HAS_L3_DPF(dev) \|\| !remap_info)
		4140	return 0;
2332	Serge	4141
4560	Serge	4142	ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
		4143	if (ret)
		4144	return ret;
2332	Serge	4145
4560	Serge	4146	/*
		4147	* Note: We do not worry about the concurrent register cacheline hang
		4148	* here because no other code should access these registers other than
		4149	* at initialization time.
		4150	*/
3031	serge	4151	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4560	Serge	4152	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		4153	intel_ring_emit(ring, reg_base + i);
		4154	intel_ring_emit(ring, remap_info[i/4]);
3031	serge	4155	}
2332	Serge	4156
4560	Serge	4157	intel_ring_advance(ring);
2332	Serge	4158
4560	Serge	4159	return ret;
3031	serge	4160	}
2332	Serge	4161
3031	serge	4162	void i915_gem_init_swizzling(struct drm_device *dev)
		4163	{
5060	serge	4164	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4165
3031	serge	4166	if (INTEL_INFO(dev)->gen < 5 \|\|
		4167	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4168	return;
2332	Serge	4169
3031	serge	4170	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4171	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4172
3031	serge	4173	if (IS_GEN5(dev))
		4174	return;
2344	Serge	4175
3031	serge	4176	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4177	if (IS_GEN6(dev))
		4178	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4179	else if (IS_GEN7(dev))
		4180	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4560	Serge	4181	else if (IS_GEN8(dev))
		4182	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
3031	serge	4183	else
3480	Serge	4184	BUG();
3031	serge	4185	}
		4186
		4187	static bool
		4188	intel_enable_blt(struct drm_device *dev)
		4189	{
		4190	if (!HAS_BLT(dev))
		4191	return false;
		4192
		4193	/* The blitter was dysfunctional on early prototypes */
		4194	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		4195	DRM_INFO("BLT not supported on this pre-production hardware;"
		4196	" graphics performance will be degraded.\n");
		4197	return false;
		4198	}
		4199
		4200	return true;
		4201	}
		4202
5354	serge	4203	static void init_unused_ring(struct drm_device *dev, u32 base)
2332	Serge	4204	{
3480	Serge	4205	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4206
		4207	I915_WRITE(RING_CTL(base), 0);
		4208	I915_WRITE(RING_HEAD(base), 0);
		4209	I915_WRITE(RING_TAIL(base), 0);
		4210	I915_WRITE(RING_START(base), 0);
		4211	}
		4212
		4213	static void init_unused_rings(struct drm_device *dev)
		4214	{
		4215	if (IS_I830(dev)) {
		4216	init_unused_ring(dev, PRB1_BASE);
		4217	init_unused_ring(dev, SRB0_BASE);
		4218	init_unused_ring(dev, SRB1_BASE);
		4219	init_unused_ring(dev, SRB2_BASE);
		4220	init_unused_ring(dev, SRB3_BASE);
		4221	} else if (IS_GEN2(dev)) {
		4222	init_unused_ring(dev, SRB0_BASE);
		4223	init_unused_ring(dev, SRB1_BASE);
		4224	} else if (IS_GEN3(dev)) {
		4225	init_unused_ring(dev, PRB1_BASE);
		4226	init_unused_ring(dev, PRB2_BASE);
		4227	}
		4228	}
		4229
		4230	int i915_gem_init_rings(struct drm_device *dev)
		4231	{
		4232	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4233	int ret;
2351	Serge	4234
5354	serge	4235	/*
		4236	* At least 830 can leave some of the unused rings
		4237	* "active" (ie. head != tail) after resume which
		4238	* will prevent c3 entry. Makes sure all unused rings
		4239	* are totally idle.
		4240	*/
		4241	init_unused_rings(dev);
		4242
2332	Serge	4243	ret = intel_init_render_ring_buffer(dev);
		4244	if (ret)
		4245	return ret;
		4246
		4247	if (HAS_BSD(dev)) {
		4248	ret = intel_init_bsd_ring_buffer(dev);
		4249	if (ret)
		4250	goto cleanup_render_ring;
		4251	}
		4252
3031	serge	4253	if (intel_enable_blt(dev)) {
2332	Serge	4254	ret = intel_init_blt_ring_buffer(dev);
		4255	if (ret)
		4256	goto cleanup_bsd_ring;
		4257	}
		4258
4104	Serge	4259	if (HAS_VEBOX(dev)) {
		4260	ret = intel_init_vebox_ring_buffer(dev);
		4261	if (ret)
		4262	goto cleanup_blt_ring;
		4263	}
		4264
5060	serge	4265	if (HAS_BSD2(dev)) {
		4266	ret = intel_init_bsd2_ring_buffer(dev);
		4267	if (ret)
		4268	goto cleanup_vebox_ring;
		4269	}
4104	Serge	4270
3480	Serge	4271	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		4272	if (ret)
5060	serge	4273	goto cleanup_bsd2_ring;
2351	Serge	4274
2332	Serge	4275	return 0;
		4276
5060	serge	4277	cleanup_bsd2_ring:
		4278	intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
4104	Serge	4279	cleanup_vebox_ring:
		4280	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4281	cleanup_blt_ring:
		4282	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4283	cleanup_bsd_ring:
		4284	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4285	cleanup_render_ring:
		4286	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4287
2332	Serge	4288	return ret;
		4289	}
		4290
3480	Serge	4291	int
		4292	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4293	{
5060	serge	4294	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4295	int ret, i;
3031	serge	4296
3480	Serge	4297	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4298	return -EIO;
3031	serge	4299
4104	Serge	4300	if (dev_priv->ellc_size)
		4301	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4302
4560	Serge	4303	if (IS_HASWELL(dev))
		4304	I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
		4305	LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
		4306
3746	Serge	4307	if (HAS_PCH_NOP(dev)) {
5060	serge	4308	if (IS_IVYBRIDGE(dev)) {
3746	Serge	4309	u32 temp = I915_READ(GEN7_MSG_CTL);
		4310	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4311	I915_WRITE(GEN7_MSG_CTL, temp);
5060	serge	4312	} else if (INTEL_INFO(dev)->gen >= 7) {
		4313	u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
		4314	temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
		4315	I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
		4316	}
3746	Serge	4317	}
		4318
3480	Serge	4319	i915_gem_init_swizzling(dev);
		4320
5354	serge	4321	ret = dev_priv->gt.init_rings(dev);
3480	Serge	4322	if (ret)
		4323	return ret;
		4324
4560	Serge	4325	for (i = 0; i < NUM_L3_SLICES(dev); i++)
		4326	i915_gem_l3_remap(&dev_priv->ring[RCS], i);
		4327
3480	Serge	4328	/*
5060	serge	4329	* XXX: Contexts should only be initialized once. Doing a switch to the
		4330	* default context switch however is something we'd like to do after
		4331	* reset or thaw (the latter may not actually be necessary for HW, but
		4332	* goes with our code better). Context switching requires rings (for
		4333	* the do_switch), but before enabling PPGTT. So don't move this.
3480	Serge	4334	*/
5060	serge	4335	ret = i915_gem_context_enable(dev_priv);
		4336	if (ret && ret != -EIO) {
		4337	DRM_ERROR("Context enable failed %d\n", ret);
4560	Serge	4338	i915_gem_cleanup_ringbuffer(dev);
5354	serge	4339
		4340	return ret;
4560	Serge	4341	}
		4342
5354	serge	4343	ret = i915_ppgtt_init_hw(dev);
		4344	if (ret && ret != -EIO) {
		4345	DRM_ERROR("PPGTT enable failed %d\n", ret);
		4346	i915_gem_cleanup_ringbuffer(dev);
		4347	}
		4348
5060	serge	4349	return ret;
3031	serge	4350	}
		4351
		4352	int i915_gem_init(struct drm_device *dev)
		4353	{
		4354	struct drm_i915_private *dev_priv = dev->dev_private;
		4355	int ret;
		4356
5354	serge	4357	i915.enable_execlists = intel_sanitize_enable_execlists(dev,
		4358	i915.enable_execlists);
		4359
3031	serge	4360	mutex_lock(&dev->struct_mutex);
3746	Serge	4361
		4362	if (IS_VALLEYVIEW(dev)) {
		4363	/* VLVA0 (potential hack), BIOS isn't actually waking us */
5060	serge	4364	I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
		4365	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
		4366	VLV_GTLC_ALLOWWAKEACK), 10))
3746	Serge	4367	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4368	}
		4369
5354	serge	4370	if (!i915.enable_execlists) {
		4371	dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission;
		4372	dev_priv->gt.init_rings = i915_gem_init_rings;
		4373	dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
		4374	dev_priv->gt.stop_ring = intel_stop_ring_buffer;
		4375	} else {
		4376	dev_priv->gt.do_execbuf = intel_execlists_submission;
		4377	dev_priv->gt.init_rings = intel_logical_rings_init;
		4378	dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
		4379	dev_priv->gt.stop_ring = intel_logical_ring_stop;
		4380	}
		4381
		4382	// ret = i915_gem_init_userptr(dev);
		4383	// if (ret) {
		4384	// mutex_unlock(&dev->struct_mutex);
		4385	// return ret;
		4386	// }
		4387
5060	serge	4388	i915_gem_init_global_gtt(dev);
3746	Serge	4389
5060	serge	4390	ret = i915_gem_context_init(dev);
3031	serge	4391	if (ret) {
5060	serge	4392	mutex_unlock(&dev->struct_mutex);
3031	serge	4393	return ret;
		4394	}
		4395
5060	serge	4396	ret = i915_gem_init_hw(dev);
		4397	if (ret == -EIO) {
		4398	/* Allow ring initialisation to fail by marking the GPU as
		4399	* wedged. But we only want to do this where the GPU is angry,
		4400	* for all other failure, such as an allocation failure, bail.
		4401	*/
		4402	DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
		4403	atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
		4404	ret = 0;
		4405	}
		4406	mutex_unlock(&dev->struct_mutex);
3746	Serge	4407
5060	serge	4408	return ret;
3031	serge	4409	}
		4410
2332	Serge	4411	void
		4412	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4413	{
5060	serge	4414	struct drm_i915_private *dev_priv = dev->dev_private;
		4415	struct intel_engine_cs *ring;
2332	Serge	4416	int i;
		4417
3031	serge	4418	for_each_ring(ring, dev_priv, i)
5354	serge	4419	dev_priv->gt.cleanup_ring(ring);
2332	Serge	4420	}
		4421
		4422	static void
5060	serge	4423	init_ring_lists(struct intel_engine_cs *ring)
2326	Serge	4424	{
		4425	INIT_LIST_HEAD(&ring->active_list);
		4426	INIT_LIST_HEAD(&ring->request_list);
		4427	}
		4428
5060	serge	4429	void i915_init_vm(struct drm_i915_private *dev_priv,
4104	Serge	4430	struct i915_address_space *vm)
		4431	{
5060	serge	4432	if (!i915_is_ggtt(vm))
		4433	drm_mm_init(&vm->mm, vm->start, vm->total);
4104	Serge	4434	vm->dev = dev_priv->dev;
		4435	INIT_LIST_HEAD(&vm->active_list);
		4436	INIT_LIST_HEAD(&vm->inactive_list);
		4437	INIT_LIST_HEAD(&vm->global_link);
5060	serge	4438	list_add_tail(&vm->global_link, &dev_priv->vm_list);
4104	Serge	4439	}
		4440
2326	Serge	4441	void
		4442	i915_gem_load(struct drm_device *dev)
		4443	{
5060	serge	4444	struct drm_i915_private *dev_priv = dev->dev_private;
2326	Serge	4445	int i;
		4446
4104	Serge	4447	INIT_LIST_HEAD(&dev_priv->vm_list);
		4448	i915_init_vm(dev_priv, &dev_priv->gtt.base);
		4449
4560	Serge	4450	INIT_LIST_HEAD(&dev_priv->context_list);
3031	serge	4451	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4452	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4453	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4454	for (i = 0; i < I915_NUM_RINGS; i++)
		4455	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4456	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4457	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4458	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4459	i915_gem_retire_work_handler);
4560	Serge	4460	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
		4461	i915_gem_idle_work_handler);
3480	Serge	4462	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4463
		4464	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
5354	serge	4465	if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) {
3031	serge	4466	I915_WRITE(MI_ARB_STATE,
		4467	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4468	}
		4469
		4470	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4471
5354	serge	4472	/* Old X drivers will take 0-2 for front, back, depth buffers */
		4473	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		4474	dev_priv->fence_reg_start = 3;
		4475
3746	Serge	4476	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4477	dev_priv->num_fence_regs = 32;
		4478	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
2326	Serge	4479	dev_priv->num_fence_regs = 16;
		4480	else
		4481	dev_priv->num_fence_regs = 8;
		4482
		4483	/* Initialize fence registers to zero */
3746	Serge	4484	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4485	i915_gem_restore_fences(dev);
2326	Serge	4486
		4487	i915_gem_detect_bit_6_swizzle(dev);
		4488
		4489	dev_priv->mm.interruptible = true;
		4490
5060	serge	4491	mutex_init(&dev_priv->fb_tracking.lock);
2326	Serge	4492	}
		4493
5060	serge	4494	int i915_gem_open(struct drm_device dev, struct drm_file file)
4104	Serge	4495	{
5060	serge	4496	struct drm_i915_file_private *file_priv;
4104	Serge	4497	int ret;
2326	Serge	4498
5060	serge	4499	DRM_DEBUG_DRIVER("\n");
4104	Serge	4500
5060	serge	4501	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
		4502	if (!file_priv)
4104	Serge	4503	return -ENOMEM;
		4504
5060	serge	4505	file->driver_priv = file_priv;
		4506	file_priv->dev_priv = dev->dev_private;
		4507	file_priv->file = file;
4104	Serge	4508
5060	serge	4509	spin_lock_init(&file_priv->mm.lock);
		4510	INIT_LIST_HEAD(&file_priv->mm.request_list);
		4511	// INIT_DELAYED_WORK(&file_priv->mm.idle_work,
		4512	// i915_gem_file_idle_work_handler);
4104	Serge	4513
5060	serge	4514	ret = i915_gem_context_open(dev, file);
		4515	if (ret)
		4516	kfree(file_priv);
4104	Serge	4517
		4518	return ret;
		4519	}
		4520
5354	serge	4521	/**
		4522	* i915_gem_track_fb - update frontbuffer tracking
		4523	* old: current GEM buffer for the frontbuffer slots
		4524	* new: new GEM buffer for the frontbuffer slots
		4525	* frontbuffer_bits: bitmask of frontbuffer slots
		4526	*
		4527	* This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
		4528	* from @old and setting them in @new. Both @old and @new can be NULL.
		4529	*/
5060	serge	4530	void i915_gem_track_fb(struct drm_i915_gem_object *old,
		4531	struct drm_i915_gem_object *new,
		4532	unsigned frontbuffer_bits)
4104	Serge	4533	{
5060	serge	4534	if (old) {
		4535	WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
		4536	WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
		4537	old->frontbuffer_bits &= ~frontbuffer_bits;
4104	Serge	4538	}
		4539
5060	serge	4540	if (new) {
		4541	WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
		4542	WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
		4543	new->frontbuffer_bits \|= frontbuffer_bits;
4104	Serge	4544	}
		4545	}
		4546
		4547	static bool mutex_is_locked_by(struct mutex mutex, struct task_struct task)
		4548	{
		4549	if (!mutex_is_locked(mutex))
		4550	return false;
		4551
		4552	#if defined(CONFIG_SMP) \|\| defined(CONFIG_DEBUG_MUTEXES)
		4553	return mutex->owner == task;
		4554	#else
		4555	/* Since UP may be pre-empted, we cannot assume that we own the lock */
		4556	return false;
		4557	#endif
		4558	}
		4559
		4560	/* All the new VM stuff */
		4561	unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4562	struct i915_address_space *vm)
		4563	{
		4564	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4565	struct i915_vma *vma;
		4566
5354	serge	4567	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4104	Serge	4568
		4569	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4570	if (vma->vm == vm)
		4571	return vma->node.start;
		4572
		4573	}
5060	serge	4574	WARN(1, "%s vma for this object not found.\n",
		4575	i915_is_ggtt(vm) ? "global" : "ppgtt");
		4576	return -1;
4104	Serge	4577	}
		4578
		4579	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4580	struct i915_address_space *vm)
		4581	{
		4582	struct i915_vma *vma;
		4583
		4584	list_for_each_entry(vma, &o->vma_list, vma_link)
		4585	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4586	return true;
		4587
		4588	return false;
		4589	}
		4590
		4591	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4592	{
4560	Serge	4593	struct i915_vma *vma;
4104	Serge	4594
4560	Serge	4595	list_for_each_entry(vma, &o->vma_list, vma_link)
		4596	if (drm_mm_node_allocated(&vma->node))
4104	Serge	4597	return true;
		4598
		4599	return false;
		4600	}
		4601
		4602	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4603	struct i915_address_space *vm)
		4604	{
		4605	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4606	struct i915_vma *vma;
		4607
5354	serge	4608	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4104	Serge	4609
		4610	BUG_ON(list_empty(&o->vma_list));
		4611
		4612	list_for_each_entry(vma, &o->vma_list, vma_link)
		4613	if (vma->vm == vm)
		4614	return vma->node.size;
		4615
		4616	return 0;
		4617	}
4560	Serge	4618
		4619
5060	serge	4620
4560	Serge	4621	struct i915_vma i915_gem_obj_to_ggtt(struct drm_i915_gem_object obj)
4104	Serge	4622	{
		4623	struct i915_vma *vma;
		4624
4560	Serge	4625	vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
5354	serge	4626	if (vma->vm != i915_obj_to_ggtt(obj))
4560	Serge	4627	return NULL;
4104	Serge	4628
		4629	return vma;
		4630	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 5367