WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
6084	serge	2	* Copyright © 2008-2015 Intel Corporation
2326	Serge	3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
4280	Serge	29	#include
3031	serge	30	#include
2326	Serge	31	#include "i915_drv.h"
6084	serge	32	#include "i915_vgpu.h"
2351	Serge	33	#include "i915_trace.h"
2326	Serge	34	#include "intel_drv.h"
3260	Serge	35	#include
2330	Serge	36	#include
2326	Serge	37	//#include
3746	Serge	38	#include
2326	Serge	39	#include
6084	serge	40	#define RQ_BUG_ON(expr)
2326	Serge	41
2344	Serge	42	extern int x86_clflush_size;
2332	Serge	43
3263	Serge	44	#define PROT_READ 0x1 /* page can be read */
		45	#define PROT_WRITE 0x2 /* page can be written */
		46	#define MAP_SHARED 0x01 /* Share changes */
		47
2344	Serge	48
5060	serge	49
3266	Serge	50	struct drm_i915_gem_object *get_fb_obj();
		51
3263	Serge	52	unsigned long vm_mmap(struct file *file, unsigned long addr,
		53	unsigned long len, unsigned long prot,
		54	unsigned long flag, unsigned long offset);
		55
2344	Serge	56
2332	Serge	57	#define MAX_ERRNO 4095
		58
		59	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		60
		61
		62	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
6084	serge	63	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
5060	serge	64	static void
6084	serge	65	i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
		66	static void
		67	i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
2326	Serge	68
4104	Serge	69	static bool cpu_cache_is_coherent(struct drm_device *dev,
		70	enum i915_cache_level level)
		71	{
		72	return HAS_LLC(dev) \|\| level != I915_CACHE_NONE;
		73	}
		74
		75	static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
		76	{
		77	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		78	return true;
		79
		80	return obj->pin_display;
		81	}
		82
2332	Serge	83	/* some bookkeeping */
		84	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		85	size_t size)
		86	{
4104	Serge	87	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	88	dev_priv->mm.object_count++;
		89	dev_priv->mm.object_memory += size;
4104	Serge	90	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	91	}
		92
		93	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		94	size_t size)
		95	{
4104	Serge	96	spin_lock(&dev_priv->mm.object_stat_lock);
2332	Serge	97	dev_priv->mm.object_count--;
		98	dev_priv->mm.object_memory -= size;
4104	Serge	99	spin_unlock(&dev_priv->mm.object_stat_lock);
2332	Serge	100	}
		101
		102	static int
3480	Serge	103	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	104	{
		105	int ret;
		106
3480	Serge	107	#define EXIT_COND (!i915_reset_in_progress(error))
		108	if (EXIT_COND)
2332	Serge	109	return 0;
3255	Serge	110	#if 0
3031	serge	111	/*
		112	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		113	* userspace. If it takes that long something really bad is going on and
		114	* we should simply try to bail out and fail as gracefully as possible.
		115	*/
3480	Serge	116	ret = wait_event_interruptible_timeout(error->reset_queue,
		117	EXIT_COND,
		118	10*HZ);
3031	serge	119	if (ret == 0) {
		120	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		121	return -EIO;
		122	} else if (ret < 0) {
2332	Serge	123	return ret;
3031	serge	124	}
2332	Serge	125
3255	Serge	126	#endif
3480	Serge	127	#undef EXIT_COND
3255	Serge	128
2332	Serge	129	return 0;
		130	}
		131
		132	int i915_mutex_lock_interruptible(struct drm_device *dev)
		133	{
3480	Serge	134	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	135	int ret;
		136
3480	Serge	137	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	138	if (ret)
		139	return ret;
		140
3480	Serge	141	ret = mutex_lock_interruptible(&dev->struct_mutex);
		142	if (ret)
		143	return ret;
2332	Serge	144
		145	WARN_ON(i915_verify_lists(dev));
		146	return 0;
		147	}
		148
		149	int
		150	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		151	struct drm_file *file)
		152	{
		153	struct drm_i915_private *dev_priv = dev->dev_private;
		154	struct drm_i915_gem_get_aperture *args = data;
6084	serge	155	struct i915_gtt *ggtt = &dev_priv->gtt;
		156	struct i915_vma *vma;
2332	Serge	157	size_t pinned;
		158
		159	pinned = 0;
		160	mutex_lock(&dev->struct_mutex);
6084	serge	161	list_for_each_entry(vma, &ggtt->base.active_list, mm_list)
		162	if (vma->pin_count)
		163	pinned += vma->node.size;
		164	list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list)
		165	if (vma->pin_count)
		166	pinned += vma->node.size;
2332	Serge	167	mutex_unlock(&dev->struct_mutex);
		168
4104	Serge	169	args->aper_size = dev_priv->gtt.base.total;
2342	Serge	170	args->aper_available_size = args->aper_size - pinned;
2332	Serge	171
		172	return 0;
		173	}
		174
3480	Serge	175	void i915_gem_object_alloc(struct drm_device dev)
		176	{
		177	struct drm_i915_private *dev_priv = dev->dev_private;
5367	serge	178	return kzalloc(sizeof(struct drm_i915_gem_object), 0);
3480	Serge	179	}
		180
		181	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		182	{
		183	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		184	kfree(obj);
		185	}
		186
3031	serge	187	static int
		188	i915_gem_create(struct drm_file *file,
2332	Serge	189	struct drm_device *dev,
		190	uint64_t size,
		191	uint32_t *handle_p)
		192	{
		193	struct drm_i915_gem_object *obj;
		194	int ret;
		195	u32 handle;
		196
		197	size = roundup(size, PAGE_SIZE);
2342	Serge	198	if (size == 0)
		199	return -EINVAL;
2332	Serge	200
		201	/* Allocate the new object */
		202	obj = i915_gem_alloc_object(dev, size);
		203	if (obj == NULL)
		204	return -ENOMEM;
		205
		206	ret = drm_gem_handle_create(file, &obj->base, &handle);
4104	Serge	207	/* drop reference from allocate - handle holds it now */
		208	drm_gem_object_unreference_unlocked(&obj->base);
		209	if (ret)
2332	Serge	210	return ret;
		211
		212	*handle_p = handle;
		213	return 0;
		214	}
		215
		216	int
		217	i915_gem_dumb_create(struct drm_file *file,
		218	struct drm_device *dev,
		219	struct drm_mode_create_dumb *args)
		220	{
		221	/* have to work out size/pitch and return them */
4560	Serge	222	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
2332	Serge	223	args->size = args->pitch * args->height;
		224	return i915_gem_create(file, dev,
		225	args->size, &args->handle);
		226	}
		227
2326	Serge	228	/**
2332	Serge	229	* Creates a new mm object and returns a handle to it.
		230	*/
		231	int
		232	i915_gem_create_ioctl(struct drm_device dev, void data,
		233	struct drm_file *file)
		234	{
		235	struct drm_i915_gem_create *args = data;
3031	serge	236
2332	Serge	237	return i915_gem_create(file, dev,
		238	args->size, &args->handle);
		239	}
		240
		241
3260	Serge	242	#if 0
2332	Serge	243
3031	serge	244	static inline int
		245	__copy_to_user_swizzled(char __user *cpu_vaddr,
		246	const char *gpu_vaddr, int gpu_offset,
6084	serge	247	int length)
2332	Serge	248	{
3031	serge	249	int ret, cpu_offset = 0;
2332	Serge	250
3031	serge	251	while (length > 0) {
		252	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		253	int this_length = min(cacheline_end - gpu_offset, length);
		254	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	255
3031	serge	256	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		257	gpu_vaddr + swizzled_gpu_offset,
		258	this_length);
		259	if (ret)
		260	return ret + length;
2332	Serge	261
3031	serge	262	cpu_offset += this_length;
		263	gpu_offset += this_length;
		264	length -= this_length;
		265	}
		266
		267	return 0;
2332	Serge	268	}
		269
3031	serge	270	static inline int
		271	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		272	const char __user *cpu_vaddr,
		273	int length)
2332	Serge	274	{
3031	serge	275	int ret, cpu_offset = 0;
2332	Serge	276
		277	while (length > 0) {
		278	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		279	int this_length = min(cacheline_end - gpu_offset, length);
		280	int swizzled_gpu_offset = gpu_offset ^ 64;
		281
3031	serge	282	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
6084	serge	283	cpu_vaddr + cpu_offset,
		284	this_length);
3031	serge	285	if (ret)
		286	return ret + length;
		287
2332	Serge	288	cpu_offset += this_length;
		289	gpu_offset += this_length;
		290	length -= this_length;
		291	}
		292
3031	serge	293	return 0;
2332	Serge	294	}
		295
3031	serge	296	/* Per-page copy function for the shmem pread fastpath.
		297	* Flushes invalid cachelines before reading the target if
		298	* needs_clflush is set. */
2332	Serge	299	static int
3031	serge	300	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		301	char __user *user_data,
		302	bool page_do_bit17_swizzling, bool needs_clflush)
		303	{
6084	serge	304	char *vaddr;
		305	int ret;
3031	serge	306
		307	if (unlikely(page_do_bit17_swizzling))
		308	return -EINVAL;
		309
6084	serge	310	vaddr = kmap_atomic(page);
3031	serge	311	if (needs_clflush)
		312	drm_clflush_virt_range(vaddr + shmem_page_offset,
		313	page_length);
6084	serge	314	ret = __copy_to_user_inatomic(user_data,
3031	serge	315	vaddr + shmem_page_offset,
6084	serge	316	page_length);
		317	kunmap_atomic(vaddr);
3031	serge	318
		319	return ret ? -EFAULT : 0;
		320	}
		321
		322	static void
		323	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		324	bool swizzled)
		325	{
		326	if (unlikely(swizzled)) {
		327	unsigned long start = (unsigned long) addr;
		328	unsigned long end = (unsigned long) addr + length;
		329
		330	/* For swizzling simply ensure that we always flush both
		331	* channels. Lame, but simple and it works. Swizzled
		332	* pwrite/pread is far from a hotpath - current userspace
		333	* doesn't use it at all. */
		334	start = round_down(start, 128);
		335	end = round_up(end, 128);
		336
		337	drm_clflush_virt_range((void *)start, end - start);
		338	} else {
		339	drm_clflush_virt_range(addr, length);
		340	}
		341
		342	}
		343
		344	/* Only difference to the fast-path function is that this can handle bit17
		345	* and uses non-atomic copy and kmap functions. */
		346	static int
		347	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		348	char __user *user_data,
		349	bool page_do_bit17_swizzling, bool needs_clflush)
		350	{
		351	char *vaddr;
		352	int ret;
		353
		354	vaddr = kmap(page);
		355	if (needs_clflush)
		356	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		357	page_length,
		358	page_do_bit17_swizzling);
		359
		360	if (page_do_bit17_swizzling)
		361	ret = __copy_to_user_swizzled(user_data,
		362	vaddr, shmem_page_offset,
		363	page_length);
		364	else
		365	ret = __copy_to_user(user_data,
		366	vaddr + shmem_page_offset,
		367	page_length);
		368	kunmap(page);
		369
		370	return ret ? - EFAULT : 0;
		371	}
		372
		373	static int
		374	i915_gem_shmem_pread(struct drm_device *dev,
6084	serge	375	struct drm_i915_gem_object *obj,
		376	struct drm_i915_gem_pread *args,
		377	struct drm_file *file)
2332	Serge	378	{
3031	serge	379	char __user *user_data;
2332	Serge	380	ssize_t remain;
		381	loff_t offset;
3031	serge	382	int shmem_page_offset, page_length, ret = 0;
		383	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		384	int prefaulted = 0;
		385	int needs_clflush = 0;
3746	Serge	386	struct sg_page_iter sg_iter;
2332	Serge	387
3746	Serge	388	user_data = to_user_ptr(args->data_ptr);
2332	Serge	389	remain = args->size;
		390
3031	serge	391	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		392
5060	serge	393	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
3031	serge	394	if (ret)
		395	return ret;
		396
2332	Serge	397	offset = args->offset;
		398
3746	Serge	399	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		400	offset >> PAGE_SHIFT) {
		401	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	402
3031	serge	403	if (remain <= 0)
		404	break;
		405
2332	Serge	406	/* Operation in this page
		407	*
3031	serge	408	* shmem_page_offset = offset within page in shmem file
2332	Serge	409	* page_length = bytes to copy for this page
		410	*/
3031	serge	411	shmem_page_offset = offset_in_page(offset);
2332	Serge	412	page_length = remain;
3031	serge	413	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		414	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	415
3031	serge	416	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		417	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	418
3031	serge	419	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		420	user_data, page_do_bit17_swizzling,
		421	needs_clflush);
		422	if (ret == 0)
		423	goto next_page;
2332	Serge	424
3031	serge	425	mutex_unlock(&dev->struct_mutex);
		426
5060	serge	427	if (likely(!i915.prefault_disable) && !prefaulted) {
3031	serge	428	ret = fault_in_multipages_writeable(user_data, remain);
		429	/* Userspace is tricking us, but we've already clobbered
		430	* its pages with the prefault and promised to write the
		431	* data up to the first fault. Hence ignore any errors
		432	* and just continue. */
		433	(void)ret;
		434	prefaulted = 1;
		435	}
		436
		437	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		438	user_data, page_do_bit17_swizzling,
		439	needs_clflush);
		440
		441	mutex_lock(&dev->struct_mutex);
		442
2332	Serge	443	if (ret)
3031	serge	444	goto out;
2332	Serge	445
5060	serge	446	next_page:
2332	Serge	447	remain -= page_length;
		448	user_data += page_length;
		449	offset += page_length;
		450	}
		451
3031	serge	452	out:
		453	i915_gem_object_unpin_pages(obj);
		454
		455	return ret;
2332	Serge	456	}
		457
		458	/**
3031	serge	459	* Reads data from the object referenced by handle.
		460	*
		461	* On error, the contents of *data are undefined.
2332	Serge	462	*/
3031	serge	463	int
		464	i915_gem_pread_ioctl(struct drm_device dev, void data,
		465	struct drm_file *file)
		466	{
		467	struct drm_i915_gem_pread *args = data;
		468	struct drm_i915_gem_object *obj;
		469	int ret = 0;
		470
		471	if (args->size == 0)
		472	return 0;
		473
		474	if (!access_ok(VERIFY_WRITE,
3746	Serge	475	to_user_ptr(args->data_ptr),
3031	serge	476	args->size))
		477	return -EFAULT;
		478
		479	ret = i915_mutex_lock_interruptible(dev);
		480	if (ret)
		481	return ret;
		482
		483	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		484	if (&obj->base == NULL) {
		485	ret = -ENOENT;
		486	goto unlock;
		487	}
		488
		489	/* Bounds check source. */
		490	if (args->offset > obj->base.size \|\|
		491	args->size > obj->base.size - args->offset) {
		492	ret = -EINVAL;
		493	goto out;
		494	}
		495
		496	/* prime objects have no backing filp to GEM pread/pwrite
		497	* pages from.
		498	*/
		499	if (!obj->base.filp) {
		500	ret = -EINVAL;
		501	goto out;
		502	}
		503
		504	trace_i915_gem_object_pread(obj, args->offset, args->size);
		505
		506	ret = i915_gem_shmem_pread(dev, obj, args, file);
		507
		508	out:
		509	drm_gem_object_unreference(&obj->base);
		510	unlock:
		511	mutex_unlock(&dev->struct_mutex);
		512	return ret;
		513	}
		514
		515	/* This is the fast write path which cannot handle
		516	* page faults in the source data
		517	*/
		518
		519	static inline int
		520	fast_user_write(struct io_mapping *mapping,
		521	loff_t page_base, int page_offset,
		522	char __user *user_data,
		523	int length)
		524	{
		525	void __iomem *vaddr_atomic;
		526	void *vaddr;
		527	unsigned long unwritten;
		528
		529	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		530	/* We can use the cpu mem copy function because this is X86. */
		531	vaddr = (void __force*)vaddr_atomic + page_offset;
		532	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		533	user_data, length);
		534	io_mapping_unmap_atomic(vaddr_atomic);
		535	return unwritten;
		536	}
3260	Serge	537	#endif
3031	serge	538
3260	Serge	539	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	540	/**
		541	* This is the fast pwrite path, where we copy the data directly from the
		542	* user into the GTT, uncached.
		543	*/
2332	Serge	544	static int
3031	serge	545	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		546	struct drm_i915_gem_object *obj,
		547	struct drm_i915_gem_pwrite *args,
		548	struct drm_file *file)
2332	Serge	549	{
5060	serge	550	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	551	ssize_t remain;
3031	serge	552	loff_t offset, page_base;
		553	char __user *user_data;
		554	int page_offset, page_length, ret;
2332	Serge	555
5060	serge	556	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3031	serge	557	if (ret)
		558	goto out;
		559
		560	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		561	if (ret)
		562	goto out_unpin;
		563
		564	ret = i915_gem_object_put_fence(obj);
		565	if (ret)
		566	goto out_unpin;
		567
4539	Serge	568	user_data = to_user_ptr(args->data_ptr);
2332	Serge	569	remain = args->size;
		570
4104	Serge	571	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
2332	Serge	572
6084	serge	573	intel_fb_obj_invalidate(obj, ORIGIN_GTT);
		574
3031	serge	575	while (remain > 0) {
		576	/* Operation in this page
		577	*
		578	* page_base = page offset within aperture
		579	* page_offset = offset within page
		580	* page_length = bytes to copy for this page
		581	*/
		582	page_base = offset & PAGE_MASK;
		583	page_offset = offset_in_page(offset);
		584	page_length = remain;
		585	if ((page_offset + remain) > PAGE_SIZE)
		586	page_length = PAGE_SIZE - page_offset;
2332	Serge	587
4539	Serge	588	MapPage(dev_priv->gtt.mappable, dev_priv->gtt.mappable_base+page_base, PG_SW);
3031	serge	589
5060	serge	590	memcpy((char*)dev_priv->gtt.mappable+page_offset, user_data, page_length);
3260	Serge	591
3031	serge	592	remain -= page_length;
		593	user_data += page_length;
		594	offset += page_length;
2332	Serge	595	}
		596
6084	serge	597	out_flush:
		598	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3031	serge	599	out_unpin:
5060	serge	600	i915_gem_object_ggtt_unpin(obj);
3031	serge	601	out:
6084	serge	602	return ret;
3031	serge	603	}
		604
		605	/* Per-page copy function for the shmem pwrite fastpath.
		606	* Flushes invalid cachelines before writing to the target if
		607	* needs_clflush_before is set and flushes out any written cachelines after
		608	* writing if needs_clflush is set. */
		609	static int
		610	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		611	char __user *user_data,
		612	bool page_do_bit17_swizzling,
		613	bool needs_clflush_before,
		614	bool needs_clflush_after)
		615	{
		616	char *vaddr;
5354	serge	617	int ret;
3031	serge	618
		619	if (unlikely(page_do_bit17_swizzling))
		620	return -EINVAL;
		621
5354	serge	622	vaddr = kmap_atomic(page);
3031	serge	623	if (needs_clflush_before)
		624	drm_clflush_virt_range(vaddr + shmem_page_offset,
		625	page_length);
3260	Serge	626	memcpy(vaddr + shmem_page_offset,
3031	serge	627	user_data,
		628	page_length);
		629	if (needs_clflush_after)
		630	drm_clflush_virt_range(vaddr + shmem_page_offset,
		631	page_length);
5354	serge	632	kunmap_atomic(vaddr);
3031	serge	633
		634	return ret ? -EFAULT : 0;
		635	}
3260	Serge	636	#if 0
3031	serge	637
		638	/* Only difference to the fast-path function is that this can handle bit17
		639	* and uses non-atomic copy and kmap functions. */
		640	static int
		641	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		642	char __user *user_data,
		643	bool page_do_bit17_swizzling,
		644	bool needs_clflush_before,
		645	bool needs_clflush_after)
		646	{
		647	char *vaddr;
		648	int ret;
		649
		650	vaddr = kmap(page);
		651	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		652	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		653	page_length,
		654	page_do_bit17_swizzling);
		655	if (page_do_bit17_swizzling)
		656	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		657	user_data,
		658	page_length);
		659	else
		660	ret = __copy_from_user(vaddr + shmem_page_offset,
		661	user_data,
		662	page_length);
		663	if (needs_clflush_after)
		664	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		665	page_length,
		666	page_do_bit17_swizzling);
		667	kunmap(page);
		668
		669	return ret ? -EFAULT : 0;
		670	}
3260	Serge	671	#endif
3031	serge	672
3260	Serge	673
3031	serge	674	static int
		675	i915_gem_shmem_pwrite(struct drm_device *dev,
		676	struct drm_i915_gem_object *obj,
		677	struct drm_i915_gem_pwrite *args,
		678	struct drm_file *file)
		679	{
		680	ssize_t remain;
		681	loff_t offset;
		682	char __user *user_data;
		683	int shmem_page_offset, page_length, ret = 0;
		684	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		685	int hit_slowpath = 0;
		686	int needs_clflush_after = 0;
		687	int needs_clflush_before = 0;
3746	Serge	688	struct sg_page_iter sg_iter;
3031	serge	689
3746	Serge	690	user_data = to_user_ptr(args->data_ptr);
3031	serge	691	remain = args->size;
		692
		693	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		694
		695	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		696	/* If we're not in the cpu write domain, set ourself into the gtt
		697	* write domain and manually flush cachelines (if required). This
		698	* optimizes for the case when the gpu will use the data
		699	* right away and we therefore have to clflush anyway. */
4104	Serge	700	needs_clflush_after = cpu_write_needs_clflush(obj);
4560	Serge	701	ret = i915_gem_object_wait_rendering(obj, false);
6084	serge	702	if (ret)
		703	return ret;
		704	}
4104	Serge	705	/* Same trick applies to invalidate partially written cachelines read
		706	* before writing. */
		707	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		708	needs_clflush_before =
		709	!cpu_cache_is_coherent(dev, obj->cache_level);
3031	serge	710
		711	ret = i915_gem_object_get_pages(obj);
2332	Serge	712	if (ret)
3031	serge	713	return ret;
2332	Serge	714
6084	serge	715	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
		716
3031	serge	717	i915_gem_object_pin_pages(obj);
2332	Serge	718
		719	offset = args->offset;
3031	serge	720	obj->dirty = 1;
2332	Serge	721
3746	Serge	722	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
		723	offset >> PAGE_SHIFT) {
		724	struct page *page = sg_page_iter_page(&sg_iter);
3031	serge	725	int partial_cacheline_write;
2332	Serge	726
3031	serge	727	if (remain <= 0)
		728	break;
		729
2332	Serge	730	/* Operation in this page
		731	*
		732	* shmem_page_offset = offset within page in shmem file
		733	* page_length = bytes to copy for this page
		734	*/
		735	shmem_page_offset = offset_in_page(offset);
		736
		737	page_length = remain;
		738	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		739	page_length = PAGE_SIZE - shmem_page_offset;
		740
3031	serge	741	/* If we don't overwrite a cacheline completely we need to be
		742	* careful to have up-to-date data by first clflushing. Don't
		743	* overcomplicate things and flush the entire patch. */
		744	partial_cacheline_write = needs_clflush_before &&
		745	((shmem_page_offset \| page_length)
3260	Serge	746	& (x86_clflush_size - 1));
2332	Serge	747
3031	serge	748	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		749	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	750
3031	serge	751	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		752	user_data, page_do_bit17_swizzling,
		753	partial_cacheline_write,
		754	needs_clflush_after);
		755	if (ret == 0)
		756	goto next_page;
		757
		758	hit_slowpath = 1;
		759	mutex_unlock(&dev->struct_mutex);
3260	Serge	760	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	761
3260	Serge	762	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		763	// user_data, page_do_bit17_swizzling,
		764	// partial_cacheline_write,
		765	// needs_clflush_after);
		766
3031	serge	767	mutex_lock(&dev->struct_mutex);
		768
		769	if (ret)
		770	goto out;
		771
5354	serge	772	next_page:
2332	Serge	773	remain -= page_length;
3031	serge	774	user_data += page_length;
2332	Serge	775	offset += page_length;
		776	}
		777
		778	out:
3031	serge	779	i915_gem_object_unpin_pages(obj);
		780
		781	if (hit_slowpath) {
3480	Serge	782	/*
		783	* Fixup: Flush cpu caches in case we didn't flush the dirty
		784	* cachelines in-line while writing and the object moved
		785	* out of the cpu write domain while we've dropped the lock.
		786	*/
		787	if (!needs_clflush_after &&
		788	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
4104	Serge	789	if (i915_gem_clflush_object(obj, obj->pin_display))
6084	serge	790	needs_clflush_after = true;
3031	serge	791	}
2332	Serge	792	}
		793
3031	serge	794	if (needs_clflush_after)
3243	Serge	795	i915_gem_chipset_flush(dev);
6084	serge	796	else
		797	obj->cache_dirty = true;
3031	serge	798
6084	serge	799	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
2332	Serge	800	return ret;
		801	}
3031	serge	802
		803	/**
		804	* Writes data to the object referenced by handle.
		805	*
		806	* On error, the contents of the buffer that were to be modified are undefined.
		807	*/
		808	int
		809	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		810	struct drm_file *file)
		811	{
6084	serge	812	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	813	struct drm_i915_gem_pwrite *args = data;
		814	struct drm_i915_gem_object *obj;
		815	int ret;
		816
4104	Serge	817	if (args->size == 0)
		818	return 0;
		819
6084	serge	820	intel_runtime_pm_get(dev_priv);
3480	Serge	821
3031	serge	822	ret = i915_mutex_lock_interruptible(dev);
		823	if (ret)
6084	serge	824	goto put_rpm;
3031	serge	825
		826	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		827	if (&obj->base == NULL) {
		828	ret = -ENOENT;
		829	goto unlock;
		830	}
		831
		832	/* Bounds check destination. */
		833	if (args->offset > obj->base.size \|\|
		834	args->size > obj->base.size - args->offset) {
		835	ret = -EINVAL;
		836	goto out;
		837	}
		838
		839	/* prime objects have no backing filp to GEM pread/pwrite
		840	* pages from.
		841	*/
		842	if (!obj->base.filp) {
		843	ret = -EINVAL;
		844	goto out;
		845	}
		846
		847	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		848
		849	ret = -EFAULT;
		850	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		851	* it would end up going through the fenced access, and we'll get
		852	* different detiling behavior between reading and writing.
		853	* pread/pwrite currently are reading and writing from the CPU
		854	* perspective, requiring manual detiling by the client.
		855	*/
4104	Serge	856	if (obj->tiling_mode == I915_TILING_NONE &&
		857	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		858	cpu_write_needs_clflush(obj)) {
3031	serge	859	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		860	/* Note that the gtt paths might fail with non-page-backed user
		861	* pointers (e.g. gtt mappings when moving data between
		862	* textures). Fallback to the shmem path in that case. */
		863	}
		864
		865	if (ret == -EFAULT \|\| ret == -ENOSPC)
6084	serge	866	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	867
		868	out:
		869	drm_gem_object_unreference(&obj->base);
		870	unlock:
		871	mutex_unlock(&dev->struct_mutex);
6084	serge	872	put_rpm:
		873	intel_runtime_pm_put(dev_priv);
		874
3031	serge	875	return ret;
		876	}
		877
		878	int
3480	Serge	879	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	880	bool interruptible)
		881	{
3480	Serge	882	if (i915_reset_in_progress(error)) {
3031	serge	883	/* Non-interruptible callers can't handle -EAGAIN, hence return
		884	* -EIO unconditionally for these. */
		885	if (!interruptible)
		886	return -EIO;
2332	Serge	887
3480	Serge	888	/* Recovery complete, but the reset failed ... */
		889	if (i915_terminally_wedged(error))
3031	serge	890	return -EIO;
2332	Serge	891
6084	serge	892	/*
		893	* Check if GPU Reset is in progress - we need intel_ring_begin
		894	* to work properly to reinit the hw state while the gpu is
		895	* still marked as reset-in-progress. Handle this with a flag.
		896	*/
		897	if (!error->reload_in_reset)
		898	return -EAGAIN;
3031	serge	899	}
2332	Serge	900
3031	serge	901	return 0;
		902	}
2332	Serge	903
4560	Serge	904	static void fake_irq(unsigned long data)
		905	{
		906	// wake_up_process((struct task_struct *)data);
		907	}
		908
		909	static bool missed_irq(struct drm_i915_private *dev_priv,
5060	serge	910	struct intel_engine_cs *ring)
4560	Serge	911	{
		912	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
		913	}
		914
6084	serge	915	static unsigned long local_clock_us(unsigned *cpu)
4560	Serge	916	{
6084	serge	917	unsigned long t;
		918
		919	/* Cheaply and approximately convert from nanoseconds to microseconds.
		920	* The result and subsequent calculations are also defined in the same
		921	* approximate microseconds units. The principal source of timing
		922	* error here is from the simple truncation.
		923	*
		924	* Note that local_clock() is only defined wrt to the current CPU;
		925	* the comparisons are no longer valid if we switch CPUs. Instead of
		926	* blocking preemption for the entire busywait, we can detect the CPU
		927	* switch and use that as indicator of system load and a reason to
		928	* stop busywaiting, see busywait_stop().
		929	*/
		930	t = GetClockNs() >> 10;
		931
		932	return t;
		933	}
		934
		935	static bool busywait_stop(unsigned long timeout, unsigned cpu)
		936	{
		937	unsigned this_cpu = 0;
		938
		939	if (time_after(local_clock_us(&this_cpu), timeout))
4560	Serge	940	return true;
		941
6084	serge	942	return this_cpu != cpu;
4560	Serge	943	}
		944
6084	serge	945	static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
		946	{
		947	unsigned long timeout;
		948	unsigned cpu;
		949
		950	/* When waiting for high frequency requests, e.g. during synchronous
		951	* rendering split between the CPU and GPU, the finite amount of time
		952	* required to set up the irq and wait upon it limits the response
		953	* rate. By busywaiting on the request completion for a short while we
		954	* can service the high frequency waits as quick as possible. However,
		955	* if it is a slow request, we want to sleep as quickly as possible.
		956	* The tradeoff between waiting and sleeping is roughly the time it
		957	* takes to sleep on a request, on the order of a microsecond.
		958	*/
		959
		960	if (req->ring->irq_refcount)
		961	return -EBUSY;
		962
		963	/* Only spin if we know the GPU is processing this request */
		964	if (!i915_gem_request_started(req, true))
		965	return -EAGAIN;
		966
		967	timeout = local_clock_us(&cpu) + 5;
		968	while (1 /!need_resched()/) {
		969	if (i915_gem_request_completed(req, true))
		970	return 0;
		971
		972	if (busywait_stop(timeout, cpu))
		973	break;
		974
		975	cpu_relax_lowlatency();
		976	}
		977
		978	if (i915_gem_request_completed(req, false))
		979	return 0;
		980
		981	return -EAGAIN;
		982	}
		983
3031	serge	984	/**
6084	serge	985	* __i915_wait_request - wait until execution of request has finished
		986	* @req: duh!
		987	* @reset_counter: reset sequence associated with the given request
3031	serge	988	* @interruptible: do an interruptible wait (normally yes)
		989	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		990	*
3480	Serge	991	* Note: It is of utmost importance that the passed in seqno and reset_counter
		992	* values have been read by the caller in an smp safe manner. Where read-side
		993	* locks are involved, it is sufficient to read the reset_counter before
		994	* unlocking the lock that protects the seqno. For lockless tricks, the
		995	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		996	* inserted.
		997	*
6084	serge	998	* Returns 0 if the request was found within the alloted time. Else returns the
3031	serge	999	* errno with remaining time filled in timeout argument.
		1000	*/
6084	serge	1001	int __i915_wait_request(struct drm_i915_gem_request *req,
3480	Serge	1002	unsigned reset_counter,
4560	Serge	1003	bool interruptible,
5060	serge	1004	s64 *timeout,
6084	serge	1005	struct intel_rps_client *rps)
3031	serge	1006	{
6084	serge	1007	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
5060	serge	1008	struct drm_device *dev = ring->dev;
		1009	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1010	const bool irq_test_in_progress =
		1011	ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
6084	serge	1012	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
		1013	DEFINE_WAIT(wait);
5060	serge	1014	unsigned long timeout_expire;
		1015	s64 before, now;
		1016
4560	Serge	1017	wait_queue_t __wait;
3031	serge	1018	int ret;
2332	Serge	1019
5060	serge	1020	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
4104	Serge	1021
6084	serge	1022	if (list_empty(&req->list))
3031	serge	1023	return 0;
2332	Serge	1024
6084	serge	1025	if (i915_gem_request_completed(req, true))
		1026	return 0;
2332	Serge	1027
6084	serge	1028	timeout_expire = 0;
		1029	if (timeout) {
		1030	if (WARN_ON(*timeout < 0))
		1031	return -EINVAL;
		1032
		1033	if (*timeout == 0)
		1034	return -ETIME;
		1035
		1036	timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
3031	serge	1037	}
2332	Serge	1038
6084	serge	1039	if (INTEL_INFO(dev_priv)->gen >= 6)
		1040	gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
2332	Serge	1041
6084	serge	1042	/* Record current time in case interrupted by signal, or wedged */
		1043	trace_i915_gem_request_wait_begin(req);
		1044	before = ktime_get_raw_ns();
		1045
		1046	/* Optimistic spin for the next jiffie before touching IRQs */
		1047	ret = __i915_spin_request(req, state);
		1048	if (ret == 0)
		1049	goto out;
		1050
		1051	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
		1052	ret = -ENODEV;
		1053	goto out;
		1054	}
		1055
4560	Serge	1056	INIT_LIST_HEAD(&__wait.task_list);
		1057	__wait.evnt = CreateEvent(NULL, MANUAL_DESTROY);
2332	Serge	1058
		1059
4560	Serge	1060	for (;;) {
		1061	unsigned long flags;
		1062
3480	Serge	1063	/* We need to check whether any gpu reset happened in between
		1064	* the caller grabbing the seqno and now ... */
4560	Serge	1065	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
		1066	/* ... but upgrade the -EAGAIN to an -EIO if the gpu
		1067	* is truely gone. */
		1068	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
		1069	if (ret == 0)
		1070	ret = -EAGAIN;
		1071	break;
		1072	}
3480	Serge	1073
6084	serge	1074	if (i915_gem_request_completed(req, false)) {
4560	Serge	1075	ret = 0;
		1076	break;
		1077	}
2332	Serge	1078
5060	serge	1079	if (timeout && time_after_eq(jiffies, timeout_expire)) {
4560	Serge	1080	ret = -ETIME;
		1081	break;
		1082	}
2332	Serge	1083
4560	Serge	1084	spin_lock_irqsave(&ring->irq_queue.lock, flags);
		1085	if (list_empty(&__wait.task_list))
		1086	__add_wait_queue(&ring->irq_queue, &__wait);
		1087	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1088
		1089	WaitEventTimeout(__wait.evnt, 1);
		1090
		1091	if (!list_empty(&__wait.task_list)) {
		1092	spin_lock_irqsave(&ring->irq_queue.lock, flags);
		1093	list_del_init(&__wait.task_list);
		1094	spin_unlock_irqrestore(&ring->irq_queue.lock, flags);
		1095	}
		1096	};
		1097
		1098	DestroyEvent(__wait.evnt);
		1099
		1100	if (!irq_test_in_progress)
6084	serge	1101	ring->irq_put(ring);
2332	Serge	1102
5060	serge	1103	// finish_wait(&ring->irq_queue, &wait);
6084	serge	1104	out:
		1105	now = ktime_get_raw_ns();
		1106	trace_i915_gem_request_wait_end(req);
		1107
		1108	if (timeout) {
		1109	s64 tres = *timeout - (now - before);
		1110
		1111	*timeout = tres < 0 ? 0 : tres;
		1112
		1113	/*
		1114	* Apparently ktime isn't accurate enough and occasionally has a
		1115	* bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
		1116	* things up to make the test happy. We allow up to 1 jiffy.
		1117	*
		1118	* This is a regrssion from the timespec->ktime conversion.
		1119	*/
		1120	if (ret == -ETIME && timeout < jiffies_to_usecs(1)1000)
		1121	*timeout = 0;
		1122	}
		1123
4560	Serge	1124	return ret;
3031	serge	1125	}
2332	Serge	1126
6084	serge	1127	int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
		1128	struct drm_file *file)
		1129	{
		1130	struct drm_i915_private *dev_private;
		1131	struct drm_i915_file_private *file_priv;
		1132
		1133	WARN_ON(!req \|\| !file \|\| req->file_priv);
		1134
		1135	if (!req \|\| !file)
		1136	return -EINVAL;
		1137
		1138	if (req->file_priv)
		1139	return -EINVAL;
		1140
		1141	dev_private = req->ring->dev->dev_private;
		1142	file_priv = file->driver_priv;
		1143
		1144	spin_lock(&file_priv->mm.lock);
		1145	req->file_priv = file_priv;
		1146	list_add_tail(&req->client_list, &file_priv->mm.request_list);
		1147	spin_unlock(&file_priv->mm.lock);
		1148
		1149	req->pid = 1;
		1150
		1151	return 0;
		1152	}
		1153
		1154	static inline void
		1155	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1156	{
		1157	struct drm_i915_file_private *file_priv = request->file_priv;
		1158
		1159	if (!file_priv)
		1160	return;
		1161
		1162	spin_lock(&file_priv->mm.lock);
		1163	list_del(&request->client_list);
		1164	request->file_priv = NULL;
		1165	spin_unlock(&file_priv->mm.lock);
		1166	}
		1167
		1168	static void i915_gem_request_retire(struct drm_i915_gem_request *request)
		1169	{
		1170	trace_i915_gem_request_retire(request);
		1171
		1172	/* We know the GPU must have read the request to have
		1173	* sent us the seqno + interrupt, so use the position
		1174	* of tail of the request to update the last known position
		1175	* of the GPU head.
		1176	*
		1177	* Note this requires that we are always called in request
		1178	* completion order.
		1179	*/
		1180	request->ringbuf->last_retired_head = request->postfix;
		1181
		1182	list_del_init(&request->list);
		1183	i915_gem_request_remove_from_client(request);
		1184
		1185	i915_gem_request_unreference(request);
		1186	}
		1187
		1188	static void
		1189	__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
		1190	{
		1191	struct intel_engine_cs *engine = req->ring;
		1192	struct drm_i915_gem_request *tmp;
		1193
		1194
		1195	if (list_empty(&req->list))
		1196	return;
		1197
		1198	do {
		1199	tmp = list_first_entry(&engine->request_list,
		1200	typeof(*tmp), list);
		1201
		1202	i915_gem_request_retire(tmp);
		1203	} while (tmp != req);
		1204
		1205	WARN_ON(i915_verify_lists(engine->dev));
		1206	}
		1207
3031	serge	1208	/**
6084	serge	1209	* Waits for a request to be signaled, and cleans up the
3031	serge	1210	* request and object lists appropriately for that event.
		1211	*/
		1212	int
6084	serge	1213	i915_wait_request(struct drm_i915_gem_request *req)
3031	serge	1214	{
6084	serge	1215	struct drm_device *dev;
		1216	struct drm_i915_private *dev_priv;
		1217	bool interruptible;
3031	serge	1218	int ret;
2332	Serge	1219
6084	serge	1220	BUG_ON(req == NULL);
		1221
		1222	dev = req->ring->dev;
		1223	dev_priv = dev->dev_private;
		1224	interruptible = dev_priv->mm.interruptible;
		1225
3031	serge	1226	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2332	Serge	1227
3480	Serge	1228	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1229	if (ret)
		1230	return ret;
2332	Serge	1231
6084	serge	1232	ret = __i915_wait_request(req,
		1233	atomic_read(&dev_priv->gpu_error.reset_counter),
		1234	interruptible, NULL, NULL);
3031	serge	1235	if (ret)
		1236	return ret;
2332	Serge	1237
6084	serge	1238	__i915_gem_request_retire__upto(req);
4104	Serge	1239	return 0;
		1240	}
		1241
3031	serge	1242	/**
		1243	* Ensures that all rendering to the object has completed and the object is
		1244	* safe to unbind from the GTT or access from the CPU.
		1245	*/
6084	serge	1246	int
3031	serge	1247	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1248	bool readonly)
		1249	{
6084	serge	1250	int ret, i;
2332	Serge	1251
6084	serge	1252	if (!obj->active)
3031	serge	1253	return 0;
2332	Serge	1254
6084	serge	1255	if (readonly) {
		1256	if (obj->last_write_req != NULL) {
		1257	ret = i915_wait_request(obj->last_write_req);
		1258	if (ret)
		1259	return ret;
2332	Serge	1260
6084	serge	1261	i = obj->last_write_req->ring->id;
		1262	if (obj->last_read_req[i] == obj->last_write_req)
		1263	i915_gem_object_retire__read(obj, i);
		1264	else
		1265	i915_gem_object_retire__write(obj);
		1266	}
		1267	} else {
		1268	for (i = 0; i < I915_NUM_RINGS; i++) {
		1269	if (obj->last_read_req[i] == NULL)
		1270	continue;
		1271
		1272	ret = i915_wait_request(obj->last_read_req[i]);
		1273	if (ret)
		1274	return ret;
		1275
		1276	i915_gem_object_retire__read(obj, i);
		1277	}
		1278	RQ_BUG_ON(obj->active);
		1279	}
		1280
		1281	return 0;
3031	serge	1282	}
2332	Serge	1283
6084	serge	1284	static void
		1285	i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
		1286	struct drm_i915_gem_request *req)
		1287	{
		1288	int ring = req->ring->id;
		1289
		1290	if (obj->last_read_req[ring] == req)
		1291	i915_gem_object_retire__read(obj, ring);
		1292	else if (obj->last_write_req == req)
		1293	i915_gem_object_retire__write(obj);
		1294
		1295	__i915_gem_request_retire__upto(req);
		1296	}
		1297
3260	Serge	1298	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1299	* as the object state may change during this call.
		1300	*/
		1301	static __must_check int
		1302	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
6084	serge	1303	struct intel_rps_client *rps,
3260	Serge	1304	bool readonly)
		1305	{
		1306	struct drm_device *dev = obj->base.dev;
		1307	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1308	struct drm_i915_gem_request *requests[I915_NUM_RINGS];
3480	Serge	1309	unsigned reset_counter;
6084	serge	1310	int ret, i, n = 0;
2332	Serge	1311
3260	Serge	1312	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1313	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1314
6084	serge	1315	if (!obj->active)
3260	Serge	1316	return 0;
2332	Serge	1317
3480	Serge	1318	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1319	if (ret)
		1320	return ret;
2332	Serge	1321
6084	serge	1322	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2332	Serge	1323
6084	serge	1324	if (readonly) {
		1325	struct drm_i915_gem_request *req;
		1326
		1327	req = obj->last_write_req;
		1328	if (req == NULL)
		1329	return 0;
		1330
		1331	requests[n++] = i915_gem_request_reference(req);
		1332	} else {
		1333	for (i = 0; i < I915_NUM_RINGS; i++) {
		1334	struct drm_i915_gem_request *req;
		1335
		1336	req = obj->last_read_req[i];
		1337	if (req == NULL)
		1338	continue;
		1339
		1340	requests[n++] = i915_gem_request_reference(req);
		1341	}
		1342	}
		1343
3260	Serge	1344	mutex_unlock(&dev->struct_mutex);
6084	serge	1345	for (i = 0; ret == 0 && i < n; i++)
		1346	ret = __i915_wait_request(requests[i], reset_counter, true,
		1347	NULL, rps);
3260	Serge	1348	mutex_lock(&dev->struct_mutex);
2332	Serge	1349
6084	serge	1350	for (i = 0; i < n; i++) {
		1351	if (ret == 0)
		1352	i915_gem_object_retire_request(obj, requests[i]);
		1353	i915_gem_request_unreference(requests[i]);
		1354	}
		1355
		1356	return ret;
3260	Serge	1357	}
2332	Serge	1358
6084	serge	1359	static struct intel_rps_client to_rps_client(struct drm_file file)
		1360	{
		1361	struct drm_i915_file_private *fpriv = file->driver_priv;
		1362	return &fpriv->rps;
		1363	}
		1364
3260	Serge	1365	/**
		1366	* Called when user space prepares to use an object with the CPU, either
		1367	* through the mmap ioctl's mapping or a GTT mapping.
		1368	*/
		1369	int
		1370	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1371	struct drm_file *file)
		1372	{
		1373	struct drm_i915_gem_set_domain *args = data;
		1374	struct drm_i915_gem_object *obj;
		1375	uint32_t read_domains = args->read_domains;
		1376	uint32_t write_domain = args->write_domain;
		1377	int ret;
2332	Serge	1378
3260	Serge	1379	/* Only handle setting domains to types used by the CPU. */
		1380	if (write_domain & I915_GEM_GPU_DOMAINS)
		1381	return -EINVAL;
2332	Serge	1382
3260	Serge	1383	if (read_domains & I915_GEM_GPU_DOMAINS)
		1384	return -EINVAL;
2332	Serge	1385
3260	Serge	1386	/* Having something in the write domain implies it's in the read
		1387	* domain, and only that read domain. Enforce that in the request.
		1388	*/
		1389	if (write_domain != 0 && read_domains != write_domain)
		1390	return -EINVAL;
2332	Serge	1391
3260	Serge	1392	ret = i915_mutex_lock_interruptible(dev);
		1393	if (ret)
		1394	return ret;
2332	Serge	1395
3260	Serge	1396	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1397	if (&obj->base == NULL) {
		1398	ret = -ENOENT;
		1399	goto unlock;
		1400	}
2332	Serge	1401
3260	Serge	1402	/* Try to flush the object off the GPU without holding the lock.
		1403	* We will repeat the flush holding the lock in the normal manner
		1404	* to catch cases where we are gazumped.
		1405	*/
5060	serge	1406	ret = i915_gem_object_wait_rendering__nonblocking(obj,
6084	serge	1407	to_rps_client(file),
5060	serge	1408	!write_domain);
3260	Serge	1409	if (ret)
		1410	goto unref;
2332	Serge	1411
6084	serge	1412	if (read_domains & I915_GEM_DOMAIN_GTT)
3260	Serge	1413	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
6084	serge	1414	else
3260	Serge	1415	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2332	Serge	1416
6084	serge	1417	if (write_domain != 0)
		1418	intel_fb_obj_invalidate(obj,
		1419	write_domain == I915_GEM_DOMAIN_GTT ?
		1420	ORIGIN_GTT : ORIGIN_CPU);
		1421
3260	Serge	1422	unref:
		1423	drm_gem_object_unreference(&obj->base);
		1424	unlock:
		1425	mutex_unlock(&dev->struct_mutex);
		1426	return ret;
		1427	}
2332	Serge	1428
4293	Serge	1429	/**
		1430	* Called when user space has done writes to this buffer
		1431	*/
		1432	int
		1433	i915_gem_sw_finish_ioctl(struct drm_device dev, void data,
		1434	struct drm_file *file)
		1435	{
		1436	struct drm_i915_gem_sw_finish *args = data;
		1437	struct drm_i915_gem_object *obj;
		1438	int ret = 0;
2332	Serge	1439
4293	Serge	1440	ret = i915_mutex_lock_interruptible(dev);
		1441	if (ret)
		1442	return ret;
2332	Serge	1443
4293	Serge	1444	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1445	if (&obj->base == NULL) {
		1446	ret = -ENOENT;
		1447	goto unlock;
		1448	}
2332	Serge	1449
4293	Serge	1450	/* Pinned buffers may be scanout, so flush the cache */
		1451	if (obj->pin_display)
6084	serge	1452	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	1453
4293	Serge	1454	drm_gem_object_unreference(&obj->base);
		1455	unlock:
		1456	mutex_unlock(&dev->struct_mutex);
		1457	return ret;
		1458	}
		1459
3260	Serge	1460	/**
		1461	* Maps the contents of an object, returning the address it is mapped
		1462	* into.
		1463	*
		1464	* While the mapping holds a reference on the contents of the object, it doesn't
		1465	* imply a ref on the object itself.
5354	serge	1466	*
		1467	* IMPORTANT:
		1468	*
		1469	* DRM driver writers who look a this function as an example for how to do GEM
		1470	* mmap support, please don't implement mmap support like here. The modern way
		1471	* to implement DRM mmap support is with an mmap offset ioctl (like
		1472	* i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
		1473	* That way debug tooling like valgrind will understand what's going on, hiding
		1474	* the mmap call in a driver private ioctl will break that. The i915 driver only
		1475	* does cpu mmaps this way because we didn't know better.
3260	Serge	1476	*/
		1477	int
		1478	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1479	struct drm_file *file)
		1480	{
		1481	struct drm_i915_gem_mmap *args = data;
		1482	struct drm_gem_object *obj;
4392	Serge	1483	unsigned long addr;
2332	Serge	1484
6084	serge	1485	// if (args->flags & ~(I915_MMAP_WC))
		1486	// return -EINVAL;
3260	Serge	1487	obj = drm_gem_object_lookup(dev, file, args->handle);
		1488	if (obj == NULL)
		1489	return -ENOENT;
4104	Serge	1490
3260	Serge	1491	/* prime objects have no backing filp to GEM mmap
		1492	* pages from.
		1493	*/
		1494	if (!obj->filp) {
		1495	drm_gem_object_unreference_unlocked(obj);
		1496	return -EINVAL;
		1497	}
2332	Serge	1498
6084	serge	1499	addr = vm_mmap(obj->filp, 0, args->size,
		1500	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1501	args->offset);
3260	Serge	1502	drm_gem_object_unreference_unlocked(obj);
6084	serge	1503	if (IS_ERR((void *)addr))
		1504	return addr;
2332	Serge	1505
3260	Serge	1506	args->addr_ptr = (uint64_t) addr;
2332	Serge	1507
6084	serge	1508	return 0;
3260	Serge	1509	}
2332	Serge	1510
		1511
		1512
		1513
		1514
		1515
		1516
		1517
3031	serge	1518
		1519
		1520
		1521
		1522
		1523	/**
		1524	* i915_gem_release_mmap - remove physical page mappings
		1525	* @obj: obj in question
		1526	*
		1527	* Preserve the reservation of the mmapping with the DRM core code, but
		1528	* relinquish ownership of the pages back to the system.
		1529	*
		1530	* It is vital that we remove the page mapping if we have mapped a tiled
		1531	* object through the GTT and then lose the fence register due to
		1532	* resource pressure. Similarly if the object has been moved out of the
		1533	* aperture, than pages mapped into userspace must be revoked. Removing the
		1534	* mapping will then trigger a page fault on the next user access, allowing
		1535	* fixup by i915_gem_fault().
		1536	*/
		1537	void
		1538	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1539	{
		1540	if (!obj->fault_mappable)
		1541	return;
		1542
4104	Serge	1543	// drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
3031	serge	1544	obj->fault_mappable = false;
		1545	}
		1546
6084	serge	1547	void
		1548	i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
		1549	{
		1550	struct drm_i915_gem_object *obj;
		1551
		1552	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
		1553	i915_gem_release_mmap(obj);
		1554	}
		1555
3480	Serge	1556	uint32_t
2332	Serge	1557	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1558	{
		1559	uint32_t gtt_size;
		1560
		1561	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1562	tiling_mode == I915_TILING_NONE)
		1563	return size;
		1564
		1565	/* Previous chips need a power-of-two fence region when tiling */
		1566	if (INTEL_INFO(dev)->gen == 3)
		1567	gtt_size = 1024*1024;
		1568	else
		1569	gtt_size = 512*1024;
		1570
		1571	while (gtt_size < size)
		1572	gtt_size <<= 1;
		1573
		1574	return gtt_size;
		1575	}
		1576
		1577	/**
		1578	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1579	* @obj: object to check
		1580	*
		1581	* Return the required GTT alignment for an object, taking into account
		1582	* potential fence register mapping.
		1583	*/
3480	Serge	1584	uint32_t
		1585	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1586	int tiling_mode, bool fenced)
2332	Serge	1587	{
		1588	/*
		1589	* Minimum alignment is 4k (GTT page size), but might be greater
		1590	* if a fence register is needed for the object.
		1591	*/
3480	Serge	1592	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1593	tiling_mode == I915_TILING_NONE)
		1594	return 4096;
		1595
		1596	/*
		1597	* Previous chips need to be aligned to the size of the smallest
		1598	* fence register that can contain the object.
		1599	*/
		1600	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1601	}
		1602
		1603
		1604
3480	Serge	1605	int
		1606	i915_gem_mmap_gtt(struct drm_file *file,
		1607	struct drm_device *dev,
6084	serge	1608	uint32_t handle,
3480	Serge	1609	uint64_t *offset)
		1610	{
		1611	struct drm_i915_private *dev_priv = dev->dev_private;
		1612	struct drm_i915_gem_object *obj;
		1613	unsigned long pfn;
		1614	char mem, ptr;
		1615	int ret;
		1616
		1617	ret = i915_mutex_lock_interruptible(dev);
		1618	if (ret)
		1619	return ret;
		1620
		1621	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1622	if (&obj->base == NULL) {
		1623	ret = -ENOENT;
		1624	goto unlock;
		1625	}
		1626
		1627	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1628	DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
		1629	ret = -EFAULT;
3480	Serge	1630	goto out;
		1631	}
		1632	/* Now bind it into the GTT if needed */
5060	serge	1633	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE \| PIN_NONBLOCK);
3480	Serge	1634	if (ret)
		1635	goto out;
		1636
		1637	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1638	if (ret)
		1639	goto unpin;
		1640
		1641	ret = i915_gem_object_get_fence(obj);
		1642	if (ret)
		1643	goto unpin;
		1644
		1645	obj->fault_mappable = true;
		1646
4104	Serge	1647	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
3480	Serge	1648
		1649	/* Finally, remap it using the new GTT offset */
		1650
		1651	mem = UserAlloc(obj->base.size);
		1652	if(unlikely(mem == NULL))
		1653	{
		1654	ret = -ENOMEM;
		1655	goto unpin;
		1656	}
		1657
		1658	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1659	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1660
		1661	unpin:
5060	serge	1662	i915_gem_object_unpin_pages(obj);
3480	Serge	1663
		1664
5367	serge	1665	*offset = (uint32_t)mem;
3480	Serge	1666
		1667	out:
		1668	drm_gem_object_unreference(&obj->base);
		1669	unlock:
		1670	mutex_unlock(&dev->struct_mutex);
		1671	return ret;
		1672	}
		1673
		1674	/**
		1675	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1676	* @dev: DRM device
		1677	* @data: GTT mapping ioctl data
		1678	* @file: GEM object info
		1679	*
		1680	* Simply returns the fake offset to userspace so it can mmap it.
		1681	* The mmap call will end up in drm_gem_mmap(), which will set things
		1682	* up so we can get faults in the handler above.
		1683	*
		1684	* The fault handler will take care of binding the object into the GTT
		1685	* (since it may have been evicted to make room for something), allocating
		1686	* a fence register, and mapping the appropriate aperture address into
		1687	* userspace.
		1688	*/
		1689	int
		1690	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
6084	serge	1691	struct drm_file *file)
3480	Serge	1692	{
6084	serge	1693	struct drm_i915_gem_mmap_gtt *args = data;
3480	Serge	1694
6084	serge	1695	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
3480	Serge	1696	}
		1697
3031	serge	1698	/* Immediately discard the backing storage */
		1699	static void
		1700	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1701	{
		1702	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1703
3263	Serge	1704	if (obj->base.filp == NULL)
		1705	return;
2332	Serge	1706
3031	serge	1707	/* Our goal here is to return as much of the memory as
		1708	* is possible back to the system as we are called from OOM.
		1709	* To do this we must instruct the shmfs to drop all of its
		1710	* backing pages, now.
		1711	*/
5060	serge	1712	// shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
3031	serge	1713	obj->madv = __I915_MADV_PURGED;
		1714	}
2332	Serge	1715
5060	serge	1716	/* Try to discard unwanted pages */
		1717	static void
		1718	i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
3031	serge	1719	{
5060	serge	1720	struct address_space *mapping;
		1721
		1722	switch (obj->madv) {
		1723	case I915_MADV_DONTNEED:
		1724	i915_gem_object_truncate(obj);
		1725	case __I915_MADV_PURGED:
		1726	return;
		1727	}
		1728
		1729	if (obj->base.filp == NULL)
		1730	return;
		1731
3031	serge	1732	}
2332	Serge	1733
3031	serge	1734	static void
		1735	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1736	{
3746	Serge	1737	struct sg_page_iter sg_iter;
		1738	int ret;
2332	Serge	1739
3031	serge	1740	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1741
3031	serge	1742	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1743	if (ret) {
		1744	/* In the event of a disaster, abandon all caches and
		1745	* hope for the best.
		1746	*/
		1747	WARN_ON(ret != -EIO);
4104	Serge	1748	i915_gem_clflush_object(obj, true);
3031	serge	1749	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1750	}
2332	Serge	1751
6084	serge	1752	i915_gem_gtt_finish_object(obj);
3031	serge	1753	if (obj->madv == I915_MADV_DONTNEED)
		1754	obj->dirty = 0;
2332	Serge	1755
3746	Serge	1756	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		1757	struct page *page = sg_page_iter_page(&sg_iter);
2332	Serge	1758
6084	serge	1759	page_cache_release(page);
3243	Serge	1760	}
6084	serge	1761	obj->dirty = 0;
3243	Serge	1762
		1763	sg_free_table(obj->pages);
		1764	kfree(obj->pages);
3031	serge	1765	}
2332	Serge	1766
3480	Serge	1767	int
3031	serge	1768	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1769	{
		1770	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1771
3243	Serge	1772	if (obj->pages == NULL)
3031	serge	1773	return 0;
2332	Serge	1774
3031	serge	1775	if (obj->pages_pin_count)
		1776	return -EBUSY;
		1777
4104	Serge	1778	BUG_ON(i915_gem_obj_bound_any(obj));
		1779
3243	Serge	1780	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1781	* array, hence protect them from being reaped by removing them from gtt
		1782	* lists early. */
4104	Serge	1783	list_del(&obj->global_list);
3243	Serge	1784
3031	serge	1785	ops->put_pages(obj);
3243	Serge	1786	obj->pages = NULL;
3031	serge	1787
5060	serge	1788	i915_gem_object_invalidate(obj);
3031	serge	1789
		1790	return 0;
		1791	}
		1792
2332	Serge	1793	static int
3031	serge	1794	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1795	{
3260	Serge	1796	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
6084	serge	1797	int page_count, i;
		1798	struct address_space *mapping;
		1799	struct sg_table *st;
3243	Serge	1800	struct scatterlist *sg;
3746	Serge	1801	struct sg_page_iter sg_iter;
3243	Serge	1802	struct page *page;
3746	Serge	1803	unsigned long last_pfn = 0; /* suppress gcc warning */
6084	serge	1804	int ret;
3243	Serge	1805	gfp_t gfp;
2332	Serge	1806
3243	Serge	1807	/* Assert that the object is not currently in any GPU domain. As it
		1808	* wasn't in the GTT, there shouldn't be any way it could have been in
		1809	* a GPU cache
2332	Serge	1810	*/
3243	Serge	1811	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1812	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1813
		1814	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1815	if (st == NULL)
		1816	return -ENOMEM;
		1817
2332	Serge	1818	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1819	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1820	kfree(st);
2332	Serge	1821	return -ENOMEM;
3243	Serge	1822	}
2332	Serge	1823
3243	Serge	1824	/* Get the list of pages out of our struct file. They'll be pinned
		1825	* at this point until we release them.
		1826	*
		1827	* Fail silently without starting the shrinker
		1828	*/
3746	Serge	1829	sg = st->sgl;
		1830	st->nents = 0;
		1831	for (i = 0; i < page_count; i++) {
4104	Serge	1832	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
3260	Serge	1833	if (IS_ERR(page)) {
		1834	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1835	goto err_pages;
3260	Serge	1836	}
5354	serge	1837	#ifdef CONFIG_SWIOTLB
		1838	if (swiotlb_nr_tbl()) {
		1839	st->nents++;
		1840	sg_set_page(sg, page, PAGE_SIZE, 0);
		1841	sg = sg_next(sg);
		1842	continue;
		1843	}
		1844	#endif
3746	Serge	1845	if (!i \|\| page_to_pfn(page) != last_pfn + 1) {
		1846	if (i)
		1847	sg = sg_next(sg);
		1848	st->nents++;
6084	serge	1849	sg_set_page(sg, page, PAGE_SIZE, 0);
3746	Serge	1850	} else {
		1851	sg->length += PAGE_SIZE;
		1852	}
		1853	last_pfn = page_to_pfn(page);
3243	Serge	1854	}
5354	serge	1855	#ifdef CONFIG_SWIOTLB
		1856	if (!swiotlb_nr_tbl())
		1857	#endif
3746	Serge	1858	sg_mark_end(sg);
3243	Serge	1859	obj->pages = st;
3031	serge	1860
6084	serge	1861	ret = i915_gem_gtt_prepare_object(obj);
		1862	if (ret)
		1863	goto err_pages;
5367	serge	1864
		1865	if (obj->tiling_mode != I915_TILING_NONE &&
		1866	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
		1867	i915_gem_object_pin_pages(obj);
		1868
2332	Serge	1869	return 0;
		1870
		1871	err_pages:
3746	Serge	1872	sg_mark_end(sg);
		1873	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
		1874	page_cache_release(sg_page_iter_page(&sg_iter));
3243	Serge	1875	sg_free_table(st);
		1876	kfree(st);
6084	serge	1877
3243	Serge	1878	return PTR_ERR(page);
2332	Serge	1879	}
		1880
3031	serge	1881	/* Ensure that the associated pages are gathered from the backing storage
		1882	* and pinned into our object. i915_gem_object_get_pages() may be called
		1883	* multiple times before they are released by a single call to
		1884	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1885	* either as a result of memory pressure (reaping pages under the shrinker)
		1886	* or as the object is itself released.
		1887	*/
		1888	int
		1889	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1890	{
3031	serge	1891	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1892	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1893	int ret;
2332	Serge	1894
3243	Serge	1895	if (obj->pages)
3031	serge	1896	return 0;
2332	Serge	1897
4392	Serge	1898	if (obj->madv != I915_MADV_WILLNEED) {
5060	serge	1899	DRM_DEBUG("Attempting to obtain a purgeable object\n");
		1900	return -EFAULT;
4392	Serge	1901	}
		1902
3031	serge	1903	BUG_ON(obj->pages_pin_count);
2332	Serge	1904
3031	serge	1905	ret = ops->get_pages(obj);
		1906	if (ret)
		1907	return ret;
2344	Serge	1908
4104	Serge	1909	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
6084	serge	1910
		1911	obj->get_page.sg = obj->pages->sgl;
		1912	obj->get_page.last = 0;
		1913
		1914	return 0;
2332	Serge	1915	}
		1916
6084	serge	1917	void i915_vma_move_to_active(struct i915_vma *vma,
		1918	struct drm_i915_gem_request *req)
2332	Serge	1919	{
6084	serge	1920	struct drm_i915_gem_object *obj = vma->obj;
		1921	struct intel_engine_cs *ring;
2332	Serge	1922
6084	serge	1923	ring = i915_gem_request_get_ring(req);
2332	Serge	1924
		1925	/* Add a reference if we're newly entering the active list. */
6084	serge	1926	if (obj->active == 0)
2344	Serge	1927	drm_gem_object_reference(&obj->base);
6084	serge	1928	obj->active \|= intel_ring_flag(ring);
2332	Serge	1929
6084	serge	1930	list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
		1931	i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2332	Serge	1932
6084	serge	1933	list_move_tail(&vma->mm_list, &vma->vm->active_list);
2332	Serge	1934	}
		1935
6084	serge	1936	static void
		1937	i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
4560	Serge	1938	{
6084	serge	1939	RQ_BUG_ON(obj->last_write_req == NULL);
		1940	RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
		1941
		1942	i915_gem_request_assign(&obj->last_write_req, NULL);
		1943	intel_fb_obj_flush(obj, true, ORIGIN_CS);
4560	Serge	1944	}
		1945
2344	Serge	1946	static void
6084	serge	1947	i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2344	Serge	1948	{
5060	serge	1949	struct i915_vma *vma;
2332	Serge	1950
6084	serge	1951	RQ_BUG_ON(obj->last_read_req[ring] == NULL);
		1952	RQ_BUG_ON(!(obj->active & (1 << ring)));
2332	Serge	1953
6084	serge	1954	list_del_init(&obj->ring_list[ring]);
		1955	i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2344	Serge	1956
6084	serge	1957	if (obj->last_write_req && obj->last_write_req->ring->id == ring)
		1958	i915_gem_object_retire__write(obj);
5354	serge	1959
6084	serge	1960	obj->active &= ~(1 << ring);
		1961	if (obj->active)
		1962	return;
2344	Serge	1963
6084	serge	1964	/* Bump our place on the bound list to keep it roughly in LRU order
		1965	* so that we don't steal from recently used but inactive objects
		1966	* (unless we are forced to ofc!)
		1967	*/
		1968	list_move_tail(&obj->global_list,
		1969	&to_i915(obj->base.dev)->mm.bound_list);
3031	serge	1970
6084	serge	1971	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		1972	if (!list_empty(&vma->mm_list))
		1973	list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
		1974	}
2344	Serge	1975
6084	serge	1976	i915_gem_request_assign(&obj->last_fenced_req, NULL);
2352	Serge	1977	drm_gem_object_unreference(&obj->base);
		1978	}
		1979
3243	Serge	1980	static int
3480	Serge	1981	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1982	{
3243	Serge	1983	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	1984	struct intel_engine_cs *ring;
3243	Serge	1985	int ret, i, j;
2344	Serge	1986
3480	Serge	1987	/* Carefully retire all requests without writing to the rings */
3243	Serge	1988	for_each_ring(ring, dev_priv, i) {
3480	Serge	1989	ret = intel_ring_idle(ring);
6084	serge	1990	if (ret)
		1991	return ret;
3480	Serge	1992	}
		1993	i915_gem_retire_requests(dev);
3243	Serge	1994
3480	Serge	1995	/* Finally reset hw state */
3243	Serge	1996	for_each_ring(ring, dev_priv, i) {
3480	Serge	1997	intel_ring_init_seqno(ring, seqno);
		1998
5060	serge	1999	for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
		2000	ring->semaphore.sync_seqno[j] = 0;
3243	Serge	2001	}
		2002
		2003	return 0;
2344	Serge	2004	}
		2005
3480	Serge	2006	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		2007	{
		2008	struct drm_i915_private *dev_priv = dev->dev_private;
		2009	int ret;
		2010
		2011	if (seqno == 0)
		2012	return -EINVAL;
		2013
		2014	/* HWS page needs to be set less than what we
		2015	* will inject to ring
		2016	*/
		2017	ret = i915_gem_init_seqno(dev, seqno - 1);
		2018	if (ret)
		2019	return ret;
		2020
		2021	/* Carefully set the last_seqno value so that wrap
		2022	* detection still works
		2023	*/
		2024	dev_priv->next_seqno = seqno;
		2025	dev_priv->last_seqno = seqno - 1;
		2026	if (dev_priv->last_seqno == 0)
		2027	dev_priv->last_seqno--;
		2028
		2029	return 0;
		2030	}
		2031
3243	Serge	2032	int
		2033	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	2034	{
3243	Serge	2035	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	2036
3243	Serge	2037	/* reserve 0 for non-seqno */
		2038	if (dev_priv->next_seqno == 0) {
3480	Serge	2039	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	2040	if (ret)
		2041	return ret;
		2042
		2043	dev_priv->next_seqno = 1;
		2044	}
		2045
3480	Serge	2046	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	2047	return 0;
2332	Serge	2048	}
		2049
6084	serge	2050	/*
		2051	* NB: This function is not allowed to fail. Doing so would mean the the
		2052	* request is not being tracked for completion but the work itself is
		2053	* going to happen on the hardware. This would be a Bad Thing(tm).
		2054	*/
		2055	void __i915_add_request(struct drm_i915_gem_request *request,
		2056	struct drm_i915_gem_object *obj,
		2057	bool flush_caches)
2352	Serge	2058	{
6084	serge	2059	struct intel_engine_cs *ring;
		2060	struct drm_i915_private *dev_priv;
5354	serge	2061	struct intel_ringbuffer *ringbuf;
6084	serge	2062	u32 request_start;
2352	Serge	2063	int ret;
2332	Serge	2064
5354	serge	2065	if (WARN_ON(request == NULL))
6084	serge	2066	return;
5354	serge	2067
6084	serge	2068	ring = request->ring;
		2069	dev_priv = ring->dev->dev_private;
		2070	ringbuf = request->ringbuf;
5354	serge	2071
6084	serge	2072	/*
		2073	* To ensure that this call will not fail, space for its emissions
		2074	* should already have been reserved in the ring buffer. Let the ring
		2075	* know that it is time to use that space up.
		2076	*/
		2077	intel_ring_reserved_space_use(ringbuf);
		2078
5354	serge	2079	request_start = intel_ring_get_tail(ringbuf);
3031	serge	2080	/*
		2081	* Emit any outstanding flushes - execbuf can fail to emit the flush
		2082	* after having emitted the batchbuffer command. Hence we need to fix
		2083	* things up similar to emitting the lazy request. The difference here
		2084	* is that the flush _must_ happen before the next request, no matter
		2085	* what.
		2086	*/
6084	serge	2087	if (flush_caches) {
		2088	if (i915.enable_execlists)
		2089	ret = logical_ring_flush_all_caches(request);
		2090	else
		2091	ret = intel_ring_flush_all_caches(request);
		2092	/* Not allowed to fail! */
		2093	WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
5354	serge	2094	}
2332	Serge	2095
3031	serge	2096	/* Record the position of the start of the request so that
		2097	* should we detect the updated seqno part-way through the
6084	serge	2098	* GPU processing the request, we never over-estimate the
3031	serge	2099	* position of the head.
		2100	*/
6084	serge	2101	request->postfix = intel_ring_get_tail(ringbuf);
3031	serge	2102
6084	serge	2103	if (i915.enable_execlists)
		2104	ret = ring->emit_request(request);
		2105	else {
		2106	ret = ring->add_request(request);
		2107
		2108	request->tail = intel_ring_get_tail(ringbuf);
5354	serge	2109	}
6084	serge	2110	/* Not allowed to fail! */
		2111	WARN(ret, "emit\|add_request failed: %d!\n", ret);
2332	Serge	2112
4104	Serge	2113	request->head = request_start;
		2114
		2115	/* Whilst this request exists, batch_obj will be on the
		2116	* active_list, and so will hold the active reference. Only when this
		2117	* request is retired will the the batch_obj be moved onto the
		2118	* inactive_list and lose its active reference. Hence we do not need
		2119	* to explicitly hold another reference here.
		2120	*/
4560	Serge	2121	request->batch_obj = obj;
4104	Serge	2122
5060	serge	2123	request->emitted_jiffies = jiffies;
6084	serge	2124	request->previous_seqno = ring->last_submitted_seqno;
		2125	ring->last_submitted_seqno = request->seqno;
2352	Serge	2126	list_add_tail(&request->list, &ring->request_list);
2332	Serge	2127
6084	serge	2128	trace_i915_gem_request_add(request);
2332	Serge	2129
6084	serge	2130	// i915_queue_hangcheck(ring->dev);
3263	Serge	2131
6084	serge	2132	queue_delayed_work(dev_priv->wq,
		2133	&dev_priv->mm.retire_work,
		2134	round_jiffies_up_relative(HZ));
		2135	intel_mark_busy(dev_priv->dev);
2332	Serge	2136
6084	serge	2137	/* Sanity check that the reserved size was large enough. */
		2138	intel_ring_reserved_space_end(ringbuf);
2352	Serge	2139	}
2332	Serge	2140
5060	serge	2141	static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
		2142	const struct intel_context *ctx)
4104	Serge	2143	{
5060	serge	2144	unsigned long elapsed;
4104	Serge	2145
5060	serge	2146	elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts;
4104	Serge	2147
5060	serge	2148	if (ctx->hang_stats.banned)
		2149	return true;
4104	Serge	2150
6084	serge	2151	if (ctx->hang_stats.ban_period_seconds &&
		2152	elapsed <= ctx->hang_stats.ban_period_seconds) {
5060	serge	2153	if (!i915_gem_context_is_default(ctx)) {
		2154	DRM_DEBUG("context hanging too fast, banning!\n");
4104	Serge	2155	return true;
5060	serge	2156	} else if (i915_stop_ring_allow_ban(dev_priv)) {
		2157	if (i915_stop_ring_allow_warn(dev_priv))
6084	serge	2158	DRM_ERROR("gpu hanging too fast, banning!\n");
4104	Serge	2159	return true;
6084	serge	2160	}
4104	Serge	2161	}
		2162
		2163	return false;
		2164	}
		2165
5060	serge	2166	static void i915_set_reset_status(struct drm_i915_private *dev_priv,
		2167	struct intel_context *ctx,
		2168	const bool guilty)
4560	Serge	2169	{
5060	serge	2170	struct i915_ctx_hang_stats *hs;
4560	Serge	2171
5060	serge	2172	if (WARN_ON(!ctx))
		2173	return;
4560	Serge	2174
5060	serge	2175	hs = &ctx->hang_stats;
4560	Serge	2176
5060	serge	2177	if (guilty) {
		2178	hs->banned = i915_context_is_banned(dev_priv, ctx);
		2179	hs->batch_active++;
		2180	hs->guilty_ts = GetTimerTicks()/100;
		2181	} else {
		2182	hs->batch_pending++;
4104	Serge	2183	}
		2184	}
		2185
6084	serge	2186	void i915_gem_request_free(struct kref *req_ref)
4104	Serge	2187	{
6084	serge	2188	struct drm_i915_gem_request *req = container_of(req_ref,
		2189	typeof(*req), ref);
		2190	struct intel_context *ctx = req->ctx;
5354	serge	2191
6084	serge	2192	if (req->file_priv)
		2193	i915_gem_request_remove_from_client(req);
4104	Serge	2194
5354	serge	2195	if (ctx) {
		2196	if (i915.enable_execlists) {
6084	serge	2197	if (ctx != req->ring->default_context)
		2198	intel_lr_context_unpin(req);
		2199	}
4104	Serge	2200
5354	serge	2201	i915_gem_context_unreference(ctx);
		2202	}
6084	serge	2203
		2204	kfree(req);
4104	Serge	2205	}
		2206
6084	serge	2207	int i915_gem_request_alloc(struct intel_engine_cs *ring,
		2208	struct intel_context *ctx,
		2209	struct drm_i915_gem_request **req_out)
		2210	{
		2211	struct drm_i915_private *dev_priv = to_i915(ring->dev);
		2212	struct drm_i915_gem_request *req;
		2213	int ret;
		2214
		2215	if (!req_out)
		2216	return -EINVAL;
		2217
		2218	*req_out = NULL;
		2219
		2220	// req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
		2221	req = kzalloc(sizeof(*req),0);
		2222	if (req == NULL)
		2223	return -ENOMEM;
		2224
		2225	ret = i915_gem_get_seqno(ring->dev, &req->seqno);
		2226	if (ret)
		2227	goto err;
		2228
		2229	kref_init(&req->ref);
		2230	req->i915 = dev_priv;
		2231	req->ring = ring;
		2232	req->ctx = ctx;
		2233	i915_gem_context_reference(req->ctx);
		2234
		2235	if (i915.enable_execlists)
		2236	ret = intel_logical_ring_alloc_request_extras(req);
		2237	else
		2238	ret = intel_ring_alloc_request_extras(req);
		2239	if (ret) {
		2240	i915_gem_context_unreference(req->ctx);
		2241	goto err;
		2242	}
		2243
		2244	/*
		2245	* Reserve space in the ring buffer for all the commands required to
		2246	* eventually emit this request. This is to guarantee that the
		2247	* i915_add_request() call can't fail. Note that the reserve may need
		2248	* to be redone if the request is not actually submitted straight
		2249	* away, e.g. because a GPU scheduler has deferred it.
		2250	*/
		2251	if (i915.enable_execlists)
		2252	ret = intel_logical_ring_reserve_space(req);
		2253	else
		2254	ret = intel_ring_reserve_space(req);
		2255	if (ret) {
		2256	/*
		2257	* At this point, the request is fully allocated even if not
		2258	* fully prepared. Thus it can be cleaned up using the proper
		2259	* free code.
		2260	*/
		2261	i915_gem_request_cancel(req);
		2262	return ret;
		2263	}
		2264
		2265	*req_out = req;
		2266	return 0;
		2267
		2268	err:
		2269	kfree(req);
		2270	return ret;
		2271	}
		2272
		2273	void i915_gem_request_cancel(struct drm_i915_gem_request *req)
		2274	{
		2275	intel_ring_reserved_space_cancel(req->ringbuf);
		2276
		2277	i915_gem_request_unreference(req);
		2278	}
		2279
5060	serge	2280	struct drm_i915_gem_request *
		2281	i915_gem_find_active_request(struct intel_engine_cs *ring)
3031	serge	2282	{
4539	Serge	2283	struct drm_i915_gem_request *request;
4104	Serge	2284
4539	Serge	2285	list_for_each_entry(request, &ring->request_list, list) {
6084	serge	2286	if (i915_gem_request_completed(request, false))
4539	Serge	2287	continue;
4104	Serge	2288
5060	serge	2289	return request;
4539	Serge	2290	}
5060	serge	2291
		2292	return NULL;
4539	Serge	2293	}
		2294
5060	serge	2295	static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
		2296	struct intel_engine_cs *ring)
		2297	{
		2298	struct drm_i915_gem_request *request;
		2299	bool ring_hung;
		2300
		2301	request = i915_gem_find_active_request(ring);
		2302
		2303	if (request == NULL)
		2304	return;
		2305
		2306	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
		2307
		2308	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
		2309
		2310	list_for_each_entry_continue(request, &ring->request_list, list)
		2311	i915_set_reset_status(dev_priv, request->ctx, false);
		2312	}
		2313
4539	Serge	2314	static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
5060	serge	2315	struct intel_engine_cs *ring)
4539	Serge	2316	{
4560	Serge	2317	while (!list_empty(&ring->active_list)) {
		2318	struct drm_i915_gem_object *obj;
		2319
		2320	obj = list_first_entry(&ring->active_list,
		2321	struct drm_i915_gem_object,
6084	serge	2322	ring_list[ring->id]);
4560	Serge	2323
6084	serge	2324	i915_gem_object_retire__read(obj, ring->id);
4560	Serge	2325	}
		2326
		2327	/*
5354	serge	2328	* Clear the execlists queue up before freeing the requests, as those
		2329	* are the ones that keep the context and ringbuffer backing objects
		2330	* pinned in place.
		2331	*/
		2332	while (!list_empty(&ring->execlist_queue)) {
6084	serge	2333	struct drm_i915_gem_request *submit_req;
5354	serge	2334
		2335	submit_req = list_first_entry(&ring->execlist_queue,
6084	serge	2336	struct drm_i915_gem_request,
5354	serge	2337	execlist_link);
		2338	list_del(&submit_req->execlist_link);
6084	serge	2339
		2340	if (submit_req->ctx != ring->default_context)
		2341	intel_lr_context_unpin(submit_req);
		2342
		2343	i915_gem_request_unreference(submit_req);
5354	serge	2344	}
		2345
		2346	/*
4560	Serge	2347	* We must free the requests after all the corresponding objects have
		2348	* been moved off active lists. Which is the same order as the normal
		2349	* retire_requests function does. This is important if object hold
		2350	* implicit references on things like e.g. ppgtt address spaces through
		2351	* the request.
		2352	*/
3031	serge	2353	while (!list_empty(&ring->request_list)) {
		2354	struct drm_i915_gem_request *request;
2332	Serge	2355
3031	serge	2356	request = list_first_entry(&ring->request_list,
		2357	struct drm_i915_gem_request,
		2358	list);
2332	Serge	2359
6084	serge	2360	i915_gem_request_retire(request);
3031	serge	2361	}
		2362	}
2332	Serge	2363
3031	serge	2364	void i915_gem_reset(struct drm_device *dev)
		2365	{
		2366	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2367	struct intel_engine_cs *ring;
3031	serge	2368	int i;
2360	Serge	2369
4539	Serge	2370	/*
		2371	* Before we free the objects from the requests, we need to inspect
		2372	* them for finding the guilty party. As the requests only borrow
		2373	* their reference to the objects, the inspection must be done first.
		2374	*/
3031	serge	2375	for_each_ring(ring, dev_priv, i)
4539	Serge	2376	i915_gem_reset_ring_status(dev_priv, ring);
2360	Serge	2377
4539	Serge	2378	for_each_ring(ring, dev_priv, i)
		2379	i915_gem_reset_ring_cleanup(dev_priv, ring);
		2380
5060	serge	2381	i915_gem_context_reset(dev);
4560	Serge	2382
3746	Serge	2383	i915_gem_restore_fences(dev);
6084	serge	2384
		2385	WARN_ON(i915_verify_lists(dev));
3031	serge	2386	}
2360	Serge	2387
2352	Serge	2388	/**
		2389	* This function clears the request list as sequence numbers are passed.
		2390	*/
3031	serge	2391	void
5060	serge	2392	i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2352	Serge	2393	{
6084	serge	2394	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2395
6084	serge	2396	/* Retire requests first as we use it above for the early return.
		2397	* If we retire requests last, we may use a later seqno and so clear
		2398	* the requests lists without clearing the active list, leading to
		2399	* confusion.
		2400	*/
		2401	while (!list_empty(&ring->request_list)) {
		2402	struct drm_i915_gem_request *request;
2332	Serge	2403
6084	serge	2404	request = list_first_entry(&ring->request_list,
		2405	struct drm_i915_gem_request,
		2406	list);
2332	Serge	2407
6084	serge	2408	if (!i915_gem_request_completed(request, true))
		2409	break;
2332	Serge	2410
6084	serge	2411	i915_gem_request_retire(request);
		2412	}
		2413
5060	serge	2414	/* Move any buffers on the active list that are no longer referenced
		2415	* by the ringbuffer to the flushing/inactive lists as appropriate,
		2416	* before we free the context associated with the requests.
		2417	*/
		2418	while (!list_empty(&ring->active_list)) {
		2419	struct drm_i915_gem_object *obj;
		2420
		2421	obj = list_first_entry(&ring->active_list,
		2422	struct drm_i915_gem_object,
6084	serge	2423	ring_list[ring->id]);
5060	serge	2424
6084	serge	2425	if (!list_empty(&obj->last_read_req[ring->id]->list))
5060	serge	2426	break;
		2427
6084	serge	2428	i915_gem_object_retire__read(obj, ring->id);
5060	serge	2429	}
		2430
6084	serge	2431	if (unlikely(ring->trace_irq_req &&
		2432	i915_gem_request_completed(ring->trace_irq_req, true))) {
2352	Serge	2433	ring->irq_put(ring);
6084	serge	2434	i915_gem_request_assign(&ring->trace_irq_req, NULL);
2352	Serge	2435	}
2332	Serge	2436
2352	Serge	2437	WARN_ON(i915_verify_lists(ring->dev));
		2438	}
2332	Serge	2439
4560	Serge	2440	bool
2352	Serge	2441	i915_gem_retire_requests(struct drm_device *dev)
		2442	{
5060	serge	2443	struct drm_i915_private *dev_priv = dev->dev_private;
		2444	struct intel_engine_cs *ring;
4560	Serge	2445	bool idle = true;
2352	Serge	2446	int i;
2332	Serge	2447
4560	Serge	2448	for_each_ring(ring, dev_priv, i) {
3031	serge	2449	i915_gem_retire_requests_ring(ring);
4560	Serge	2450	idle &= list_empty(&ring->request_list);
5354	serge	2451	if (i915.enable_execlists) {
		2452	unsigned long flags;
		2453
		2454	spin_lock_irqsave(&ring->execlist_lock, flags);
		2455	idle &= list_empty(&ring->execlist_queue);
		2456	spin_unlock_irqrestore(&ring->execlist_lock, flags);
		2457
		2458	intel_execlists_retire_requests(ring);
		2459	}
4560	Serge	2460	}
		2461
		2462	if (idle)
		2463	mod_delayed_work(dev_priv->wq,
		2464	&dev_priv->mm.idle_work,
		2465	msecs_to_jiffies(100));
		2466
		2467	return idle;
2352	Serge	2468	}
		2469
2360	Serge	2470	static void
		2471	i915_gem_retire_work_handler(struct work_struct *work)
		2472	{
4560	Serge	2473	struct drm_i915_private *dev_priv =
		2474	container_of(work, typeof(*dev_priv), mm.retire_work.work);
		2475	struct drm_device *dev = dev_priv->dev;
2360	Serge	2476	bool idle;
2352	Serge	2477
2360	Serge	2478	/* Come back later if the device is busy... */
4560	Serge	2479	idle = false;
		2480	if (mutex_trylock(&dev->struct_mutex)) {
		2481	idle = i915_gem_retire_requests(dev);
		2482	mutex_unlock(&dev->struct_mutex);
		2483	}
		2484	if (!idle)
3482	Serge	2485	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
		2486	round_jiffies_up_relative(HZ));
4560	Serge	2487	}
2352	Serge	2488
4560	Serge	2489	static void
		2490	i915_gem_idle_work_handler(struct work_struct *work)
		2491	{
		2492	struct drm_i915_private *dev_priv =
		2493	container_of(work, typeof(*dev_priv), mm.idle_work.work);
6084	serge	2494	struct drm_device *dev = dev_priv->dev;
		2495	struct intel_engine_cs *ring;
		2496	int i;
2352	Serge	2497
6084	serge	2498	for_each_ring(ring, dev_priv, i)
		2499	if (!list_empty(&ring->request_list))
		2500	return;
		2501
		2502	intel_mark_idle(dev);
		2503
		2504	if (mutex_trylock(&dev->struct_mutex)) {
		2505	struct intel_engine_cs *ring;
		2506	int i;
		2507
		2508	for_each_ring(ring, dev_priv, i)
		2509	i915_gem_batch_pool_fini(&ring->batch_pool);
		2510
		2511	mutex_unlock(&dev->struct_mutex);
		2512	}
2360	Serge	2513	}
		2514
2344	Serge	2515	/**
3031	serge	2516	* Ensures that an object will eventually get non-busy by flushing any required
		2517	* write domains, emitting any outstanding lazy request and retiring and
		2518	* completed requests.
2352	Serge	2519	*/
3031	serge	2520	static int
		2521	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2522	{
6084	serge	2523	int i;
2352	Serge	2524
6084	serge	2525	if (!obj->active)
		2526	return 0;
2352	Serge	2527
6084	serge	2528	for (i = 0; i < I915_NUM_RINGS; i++) {
		2529	struct drm_i915_gem_request *req;
		2530
		2531	req = obj->last_read_req[i];
		2532	if (req == NULL)
		2533	continue;
		2534
		2535	if (list_empty(&req->list))
		2536	goto retire;
		2537
		2538	if (i915_gem_request_completed(req, true)) {
		2539	__i915_gem_request_retire__upto(req);
		2540	retire:
		2541	i915_gem_object_retire__read(obj, i);
		2542	}
3031	serge	2543	}
2352	Serge	2544
3031	serge	2545	return 0;
		2546	}
2352	Serge	2547
3243	Serge	2548	/**
		2549	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2550	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2551	*
		2552	* Returns 0 if successful, else an error is returned with the remaining time in
		2553	* the timeout parameter.
		2554	* -ETIME: object is still busy after timeout
		2555	* -ERESTARTSYS: signal interrupted the wait
		2556	* -ENONENT: object doesn't exist
		2557	* Also possible, but rare:
		2558	* -EAGAIN: GPU wedged
		2559	* -ENOMEM: damn
		2560	* -ENODEV: Internal IRQ fail
		2561	* -E?: The add request failed
		2562	*
		2563	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2564	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2565	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2566	* without holding struct_mutex the object may become re-busied before this
		2567	* function completes. A similar but shorter * race condition exists in the busy
		2568	* ioctl
		2569	*/
4246	Serge	2570	int
		2571	i915_gem_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
		2572	{
5060	serge	2573	struct drm_i915_private *dev_priv = dev->dev_private;
4246	Serge	2574	struct drm_i915_gem_wait *args = data;
		2575	struct drm_i915_gem_object *obj;
6084	serge	2576	struct drm_i915_gem_request *req[I915_NUM_RINGS];
4246	Serge	2577	unsigned reset_counter;
6084	serge	2578	int i, n = 0;
		2579	int ret;
2352	Serge	2580
5354	serge	2581	if (args->flags != 0)
		2582	return -EINVAL;
		2583
4246	Serge	2584	ret = i915_mutex_lock_interruptible(dev);
		2585	if (ret)
		2586	return ret;
2352	Serge	2587
4246	Serge	2588	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
		2589	if (&obj->base == NULL) {
		2590	mutex_unlock(&dev->struct_mutex);
		2591	return -ENOENT;
		2592	}
2352	Serge	2593
4246	Serge	2594	/* Need to make sure the object gets inactive eventually. */
		2595	ret = i915_gem_object_flush_active(obj);
		2596	if (ret)
		2597	goto out;
2352	Serge	2598
6084	serge	2599	if (!obj->active)
		2600	goto out;
2352	Serge	2601
4246	Serge	2602	/* Do this after OLR check to make sure we make forward progress polling
6084	serge	2603	* on this IOCTL with a timeout == 0 (like busy ioctl)
4246	Serge	2604	*/
6084	serge	2605	if (args->timeout_ns == 0) {
4246	Serge	2606	ret = -ETIME;
		2607	goto out;
		2608	}
2352	Serge	2609
4246	Serge	2610	drm_gem_object_unreference(&obj->base);
		2611	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
6084	serge	2612
		2613	for (i = 0; i < I915_NUM_RINGS; i++) {
		2614	if (obj->last_read_req[i] == NULL)
		2615	continue;
		2616
		2617	req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
		2618	}
		2619
4246	Serge	2620	mutex_unlock(&dev->struct_mutex);
2352	Serge	2621
6084	serge	2622	for (i = 0; i < n; i++) {
		2623	if (ret == 0)
		2624	ret = __i915_wait_request(req[i], reset_counter, true,
		2625	args->timeout_ns > 0 ? &args->timeout_ns : NULL,
		2626	file->driver_priv);
		2627	i915_gem_request_unreference__unlocked(req[i]);
		2628	}
		2629	return ret;
3243	Serge	2630
4246	Serge	2631	out:
		2632	drm_gem_object_unreference(&obj->base);
		2633	mutex_unlock(&dev->struct_mutex);
		2634	return ret;
		2635	}
3243	Serge	2636
6084	serge	2637	static int
		2638	__i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2639	struct intel_engine_cs *to,
		2640	struct drm_i915_gem_request *from_req,
		2641	struct drm_i915_gem_request **to_req)
		2642	{
		2643	struct intel_engine_cs *from;
		2644	int ret;
		2645
		2646	from = i915_gem_request_get_ring(from_req);
		2647	if (to == from)
		2648	return 0;
		2649
		2650	if (i915_gem_request_completed(from_req, true))
		2651	return 0;
		2652
		2653	if (!i915_semaphore_is_enabled(obj->base.dev)) {
		2654	struct drm_i915_private *i915 = to_i915(obj->base.dev);
		2655	ret = __i915_wait_request(from_req,
		2656	atomic_read(&i915->gpu_error.reset_counter),
		2657	i915->mm.interruptible,
		2658	NULL,
		2659	&i915->rps.semaphores);
		2660	if (ret)
		2661	return ret;
		2662
		2663	i915_gem_object_retire_request(obj, from_req);
		2664	} else {
		2665	int idx = intel_ring_sync_index(from, to);
		2666	u32 seqno = i915_gem_request_get_seqno(from_req);
		2667
		2668	WARN_ON(!to_req);
		2669
		2670	if (seqno <= from->semaphore.sync_seqno[idx])
		2671	return 0;
		2672
		2673	if (*to_req == NULL) {
		2674	ret = i915_gem_request_alloc(to, to->default_context, to_req);
		2675	if (ret)
		2676	return ret;
		2677	}
		2678
		2679	trace_i915_gem_ring_sync_to(*to_req, from, from_req);
		2680	ret = to->semaphore.sync_to(*to_req, from, seqno);
		2681	if (ret)
		2682	return ret;
		2683
		2684	/* We use last_read_req because sync_to()
		2685	* might have just caused seqno wrap under
		2686	* the radar.
		2687	*/
		2688	from->semaphore.sync_seqno[idx] =
		2689	i915_gem_request_get_seqno(obj->last_read_req[from->id]);
		2690	}
		2691
		2692	return 0;
		2693	}
		2694
2352	Serge	2695	/**
3031	serge	2696	* i915_gem_object_sync - sync an object to a ring.
		2697	*
		2698	* @obj: object which may be in use on another ring.
		2699	* @to: ring we wish to use the object on. May be NULL.
6084	serge	2700	* @to_req: request we wish to use the object for. See below.
		2701	* This will be allocated and returned if a request is
		2702	* required but not passed in.
3031	serge	2703	*
		2704	* This code is meant to abstract object synchronization with the GPU.
		2705	* Calling with NULL implies synchronizing the object with the CPU
6084	serge	2706	* rather than a particular GPU ring. Conceptually we serialise writes
		2707	* between engines inside the GPU. We only allow one engine to write
		2708	* into a buffer at any time, but multiple readers. To ensure each has
		2709	* a coherent view of memory, we must:
3031	serge	2710	*
6084	serge	2711	* - If there is an outstanding write request to the object, the new
		2712	* request must wait for it to complete (either CPU or in hw, requests
		2713	* on the same ring will be naturally ordered).
		2714	*
		2715	* - If we are a write request (pending_write_domain is set), the new
		2716	* request must wait for outstanding read requests to complete.
		2717	*
		2718	* For CPU synchronisation (NULL to) no request is required. For syncing with
		2719	* rings to_req must be non-NULL. However, a request does not have to be
		2720	* pre-allocated. If *to_req is NULL and sync commands will be emitted then a
		2721	* request will be allocated automatically and returned through *to_req. Note
		2722	* that it is not guaranteed that commands will be emitted (because the system
		2723	* might already be idle). Hence there is no need to create a request that
		2724	* might never have any work submitted. Note further that if a request is
		2725	* returned in *to_req, it is the responsibility of the caller to submit
		2726	* that request (after potentially adding more work to it).
		2727	*
3031	serge	2728	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2729	*/
		2730	int
3031	serge	2731	i915_gem_object_sync(struct drm_i915_gem_object *obj,
6084	serge	2732	struct intel_engine_cs *to,
		2733	struct drm_i915_gem_request **to_req)
2344	Serge	2734	{
6084	serge	2735	const bool readonly = obj->base.pending_write_domain == 0;
		2736	struct drm_i915_gem_request *req[I915_NUM_RINGS];
		2737	int ret, i, n;
2332	Serge	2738
6084	serge	2739	if (!obj->active)
3031	serge	2740	return 0;
2332	Serge	2741
6084	serge	2742	if (to == NULL)
		2743	return i915_gem_object_wait_rendering(obj, readonly);
2332	Serge	2744
6084	serge	2745	n = 0;
		2746	if (readonly) {
		2747	if (obj->last_write_req)
		2748	req[n++] = obj->last_write_req;
		2749	} else {
		2750	for (i = 0; i < I915_NUM_RINGS; i++)
		2751	if (obj->last_read_req[i])
		2752	req[n++] = obj->last_read_req[i];
		2753	}
		2754	for (i = 0; i < n; i++) {
		2755	ret = __i915_gem_object_sync(obj, to, req[i], to_req);
		2756	if (ret)
		2757	return ret;
		2758	}
3031	serge	2759
6084	serge	2760	return 0;
2344	Serge	2761	}
2332	Serge	2762
2344	Serge	2763	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2764	{
		2765	u32 old_write_domain, old_read_domains;
2332	Serge	2766
2344	Serge	2767	/* Force a pagefault for domain tracking on next user access */
6084	serge	2768	i915_gem_release_mmap(obj);
2332	Serge	2769
2344	Serge	2770	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2771	return;
2332	Serge	2772
3480	Serge	2773	/* Wait for any direct GTT access to complete */
		2774	mb();
		2775
2344	Serge	2776	old_read_domains = obj->base.read_domains;
		2777	old_write_domain = obj->base.write_domain;
2351	Serge	2778
2344	Serge	2779	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2780	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2781
2351	Serge	2782	trace_i915_gem_object_change_domain(obj,
		2783	old_read_domains,
		2784	old_write_domain);
2344	Serge	2785	}
2332	Serge	2786
6084	serge	2787	static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
2344	Serge	2788	{
4104	Serge	2789	struct drm_i915_gem_object *obj = vma->obj;
5060	serge	2790	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3480	Serge	2791	int ret;
3263	Serge	2792	if(obj == get_fb_obj())
5367	serge	2793	{
		2794	WARN(1,"attempt to unbind fb object\n");
3263	Serge	2795	return 0;
5367	serge	2796	};
3263	Serge	2797
4104	Serge	2798	if (list_empty(&vma->vma_link))
2344	Serge	2799	return 0;
2332	Serge	2800
4560	Serge	2801	if (!drm_mm_node_allocated(&vma->node)) {
		2802	i915_gem_vma_destroy(vma);
		2803	return 0;
		2804	}
		2805
5060	serge	2806	if (vma->pin_count)
3031	serge	2807	return -EBUSY;
2332	Serge	2808
3243	Serge	2809	BUG_ON(obj->pages == NULL);
3031	serge	2810
6084	serge	2811	if (wait) {
		2812	ret = i915_gem_object_wait_rendering(obj, false);
		2813	if (ret)
		2814	return ret;
		2815	}
2332	Serge	2816
6084	serge	2817	if (i915_is_ggtt(vma->vm) &&
		2818	vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
		2819	i915_gem_object_finish_gtt(obj);
5354	serge	2820
6084	serge	2821	/* release the fence reg _after_ flushing */
		2822	ret = i915_gem_object_put_fence(obj);
		2823	if (ret)
		2824	return ret;
5060	serge	2825	}
2332	Serge	2826
4104	Serge	2827	trace_i915_vma_unbind(vma);
2332	Serge	2828
6084	serge	2829	vma->vm->unbind_vma(vma);
		2830	vma->bound = 0;
2332	Serge	2831
5060	serge	2832	list_del_init(&vma->mm_list);
6084	serge	2833	if (i915_is_ggtt(vma->vm)) {
		2834	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
		2835	obj->map_and_fenceable = false;
		2836	} else if (vma->ggtt_view.pages) {
		2837	sg_free_table(vma->ggtt_view.pages);
		2838	kfree(vma->ggtt_view.pages);
		2839	}
		2840	vma->ggtt_view.pages = NULL;
		2841	}
2332	Serge	2842
4104	Serge	2843	drm_mm_remove_node(&vma->node);
		2844	i915_gem_vma_destroy(vma);
		2845
		2846	/* Since the unbound list is global, only move to that list if
4560	Serge	2847	* no more VMAs exist. */
6084	serge	2848	if (list_empty(&obj->vma_list))
4104	Serge	2849	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
		2850
4560	Serge	2851	/* And finally now the object is completely decoupled from this vma,
		2852	* we can drop its hold on the backing storage and allow it to be
		2853	* reaped by the shrinker.
		2854	*/
		2855	i915_gem_object_unpin_pages(obj);
		2856
2344	Serge	2857	return 0;
		2858	}
2332	Serge	2859
6084	serge	2860	int i915_vma_unbind(struct i915_vma *vma)
		2861	{
		2862	return __i915_vma_unbind(vma, true);
		2863	}
		2864
		2865	int __i915_vma_unbind_no_wait(struct i915_vma *vma)
		2866	{
		2867	return __i915_vma_unbind(vma, false);
		2868	}
		2869
3031	serge	2870	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2871	{
5060	serge	2872	struct drm_i915_private *dev_priv = dev->dev_private;
		2873	struct intel_engine_cs *ring;
2344	Serge	2874	int ret, i;
2332	Serge	2875
2344	Serge	2876	/* Flush everything onto the inactive list. */
3031	serge	2877	for_each_ring(ring, dev_priv, i) {
5354	serge	2878	if (!i915.enable_execlists) {
6084	serge	2879	struct drm_i915_gem_request *req;
3031	serge	2880
6084	serge	2881	ret = i915_gem_request_alloc(ring, ring->default_context, &req);
2352	Serge	2882	if (ret)
		2883	return ret;
2344	Serge	2884
6084	serge	2885	ret = i915_switch_context(req);
		2886	if (ret) {
		2887	i915_gem_request_cancel(req);
		2888	return ret;
		2889	}
2344	Serge	2890
6084	serge	2891	i915_add_request_no_flush(req);
		2892	}
2332	Serge	2893
6084	serge	2894	ret = intel_ring_idle(ring);
3031	serge	2895	if (ret)
		2896	return ret;
		2897	}
2332	Serge	2898
6084	serge	2899	WARN_ON(i915_verify_lists(dev));
3031	serge	2900	return 0;
		2901	}
2332	Serge	2902
5354	serge	2903	static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3031	serge	2904	unsigned long cache_level)
		2905	{
5354	serge	2906	struct drm_mm_node *gtt_space = &vma->node;
3031	serge	2907	struct drm_mm_node *other;
2332	Serge	2908
5354	serge	2909	/*
		2910	* On some machines we have to be careful when putting differing types
		2911	* of snoopable memory together to avoid the prefetcher crossing memory
		2912	* domains and dying. During vm initialisation, we decide whether or not
		2913	* these constraints apply and set the drm_mm.color_adjust
		2914	* appropriately.
3031	serge	2915	*/
5354	serge	2916	if (vma->vm->mm.color_adjust == NULL)
3031	serge	2917	return true;
2332	Serge	2918
4104	Serge	2919	if (!drm_mm_node_allocated(gtt_space))
3031	serge	2920	return true;
2332	Serge	2921
3031	serge	2922	if (list_empty(>t_space->node_list))
		2923	return true;
2332	Serge	2924
3031	serge	2925	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2926	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2927	return false;
2344	Serge	2928
3031	serge	2929	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2930	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2931	return false;
2344	Serge	2932
3031	serge	2933	return true;
		2934	}
2344	Serge	2935
2332	Serge	2936	/**
6084	serge	2937	* Finds free space in the GTT aperture and binds the object or a view of it
		2938	* there.
2332	Serge	2939	*/
5060	serge	2940	static struct i915_vma *
4104	Serge	2941	i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
		2942	struct i915_address_space *vm,
6084	serge	2943	const struct i915_ggtt_view *ggtt_view,
		2944	unsigned alignment,
5060	serge	2945	uint64_t flags)
2332	Serge	2946	{
		2947	struct drm_device *dev = obj->base.dev;
5060	serge	2948	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	2949	u32 fence_alignment, unfenced_alignment;
		2950	u32 search_flag, alloc_flag;
		2951	u64 start, end;
		2952	u64 size, fence_size;
4104	Serge	2953	struct i915_vma *vma;
2332	Serge	2954	int ret;
2326	Serge	2955
6084	serge	2956	if (i915_is_ggtt(vm)) {
		2957	u32 view_size;
2332	Serge	2958
6084	serge	2959	if (WARN_ON(!ggtt_view))
		2960	return ERR_PTR(-EINVAL);
		2961
		2962	view_size = i915_ggtt_view_size(obj, ggtt_view);
		2963
		2964	fence_size = i915_gem_get_gtt_size(dev,
		2965	view_size,
		2966	obj->tiling_mode);
		2967	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2968	view_size,
		2969	obj->tiling_mode,
		2970	true);
		2971	unfenced_alignment = i915_gem_get_gtt_alignment(dev,
		2972	view_size,
		2973	obj->tiling_mode,
		2974	false);
		2975	size = flags & PIN_MAPPABLE ? fence_size : view_size;
		2976	} else {
		2977	fence_size = i915_gem_get_gtt_size(dev,
		2978	obj->base.size,
		2979	obj->tiling_mode);
		2980	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2981	obj->base.size,
		2982	obj->tiling_mode,
		2983	true);
		2984	unfenced_alignment =
		2985	i915_gem_get_gtt_alignment(dev,
		2986	obj->base.size,
		2987	obj->tiling_mode,
		2988	false);
		2989	size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
		2990	}
		2991
		2992	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
		2993	end = vm->total;
		2994	if (flags & PIN_MAPPABLE)
		2995	end = min_t(u64, end, dev_priv->gtt.mappable_end);
		2996	if (flags & PIN_ZONE_4G)
		2997	end = min_t(u64, end, (1ULL << 32));
		2998
2332	Serge	2999	if (alignment == 0)
5060	serge	3000	alignment = flags & PIN_MAPPABLE ? fence_alignment :
2332	Serge	3001	unfenced_alignment;
5060	serge	3002	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
6084	serge	3003	DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
		3004	ggtt_view ? ggtt_view->type : 0,
		3005	alignment);
5060	serge	3006	return ERR_PTR(-EINVAL);
2332	Serge	3007	}
		3008
6084	serge	3009	/* If binding the object/GGTT view requires more space than the entire
		3010	* aperture has, reject it early before evicting everything in a vain
		3011	* attempt to find space.
2332	Serge	3012	*/
6084	serge	3013	if (size > end) {
		3014	DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
		3015	ggtt_view ? ggtt_view->type : 0,
		3016	size,
5060	serge	3017	flags & PIN_MAPPABLE ? "mappable" : "total",
		3018	end);
		3019	return ERR_PTR(-E2BIG);
2332	Serge	3020	}
		3021
3031	serge	3022	ret = i915_gem_object_get_pages(obj);
		3023	if (ret)
5060	serge	3024	return ERR_PTR(ret);
3031	serge	3025
3243	Serge	3026	i915_gem_object_pin_pages(obj);
		3027
6084	serge	3028	vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
		3029	i915_gem_obj_lookup_or_create_vma(obj, vm);
		3030
5060	serge	3031	if (IS_ERR(vma))
4104	Serge	3032	goto err_unpin;
3243	Serge	3033
6084	serge	3034	if (flags & PIN_HIGH) {
		3035	search_flag = DRM_MM_SEARCH_BELOW;
		3036	alloc_flag = DRM_MM_CREATE_TOP;
		3037	} else {
		3038	search_flag = DRM_MM_SEARCH_DEFAULT;
		3039	alloc_flag = DRM_MM_CREATE_DEFAULT;
		3040	}
		3041
4104	Serge	3042	search_free:
		3043	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
		3044	size, alignment,
5060	serge	3045	obj->cache_level,
		3046	start, end,
6084	serge	3047	search_flag,
		3048	alloc_flag);
3243	Serge	3049	if (ret) {
2332	Serge	3050
4104	Serge	3051	goto err_free_vma;
2332	Serge	3052	}
5354	serge	3053	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
4104	Serge	3054	ret = -EINVAL;
		3055	goto err_remove_node;
3031	serge	3056	}
2332	Serge	3057
6084	serge	3058	trace_i915_vma_bind(vma, flags);
		3059	ret = i915_vma_bind(vma, obj->cache_level, flags);
4104	Serge	3060	if (ret)
		3061	goto err_remove_node;
2332	Serge	3062
4104	Serge	3063	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
		3064	list_add_tail(&vma->mm_list, &vm->inactive_list);
2332	Serge	3065
5060	serge	3066	return vma;
4104	Serge	3067
		3068	err_remove_node:
		3069	drm_mm_remove_node(&vma->node);
		3070	err_free_vma:
		3071	i915_gem_vma_destroy(vma);
5060	serge	3072	vma = ERR_PTR(ret);
4104	Serge	3073	err_unpin:
		3074	i915_gem_object_unpin_pages(obj);
5060	serge	3075	return vma;
2332	Serge	3076	}
		3077
4104	Serge	3078	bool
		3079	i915_gem_clflush_object(struct drm_i915_gem_object *obj,
		3080	bool force)
2332	Serge	3081	{
		3082	/* If we don't have a page list set up, then we're not pinned
		3083	* to GPU, and we can ignore the cache flush because it'll happen
		3084	* again at bind time.
		3085	*/
3243	Serge	3086	if (obj->pages == NULL)
4104	Serge	3087	return false;
2332	Serge	3088
3480	Serge	3089	/*
		3090	* Stolen memory is always coherent with the GPU as it is explicitly
		3091	* marked as wc by the system, or the system is cache-coherent.
		3092	*/
5354	serge	3093	if (obj->stolen \|\| obj->phys_handle)
4104	Serge	3094	return false;
3480	Serge	3095
2332	Serge	3096	/* If the GPU is snooping the contents of the CPU cache,
		3097	* we do not need to manually clear the CPU cache lines. However,
		3098	* the caches are only snooped when the render cache is
		3099	* flushed/invalidated. As we always have to emit invalidations
		3100	* and flushes when moving into and out of the RENDER domain, correct
		3101	* snooping behaviour occurs naturally as the result of our domain
		3102	* tracking.
		3103	*/
6084	serge	3104	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
		3105	obj->cache_dirty = true;
4104	Serge	3106	return false;
6084	serge	3107	}
2332	Serge	3108
4293	Serge	3109	trace_i915_gem_object_clflush(obj);
		3110	drm_clflush_sg(obj->pages);
6084	serge	3111	obj->cache_dirty = false;
2344	Serge	3112
4104	Serge	3113	return true;
2332	Serge	3114	}
		3115
2344	Serge	3116	/** Flushes the GTT write domain for the object if it's dirty. */
		3117	static void
		3118	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		3119	{
		3120	uint32_t old_write_domain;
2332	Serge	3121
2344	Serge	3122	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		3123	return;
2332	Serge	3124
2344	Serge	3125	/* No actual flushing is required for the GTT write domain. Writes
		3126	* to it immediately go to main memory as far as we know, so there's
		3127	* no chipset flush. It also doesn't land in render cache.
		3128	*
		3129	* However, we do have to enforce the order so that all writes through
		3130	* the GTT land before any writes to the device, such as updates to
		3131	* the GATT itself.
		3132	*/
		3133	wmb();
2332	Serge	3134
2344	Serge	3135	old_write_domain = obj->base.write_domain;
		3136	obj->base.write_domain = 0;
2332	Serge	3137
6084	serge	3138	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
5354	serge	3139
2351	Serge	3140	trace_i915_gem_object_change_domain(obj,
		3141	obj->base.read_domains,
		3142	old_write_domain);
2344	Serge	3143	}
2332	Serge	3144
		3145	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	3146	static void
6084	serge	3147	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2332	Serge	3148	{
		3149	uint32_t old_write_domain;
		3150
		3151	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		3152	return;
		3153
6084	serge	3154	if (i915_gem_clflush_object(obj, obj->pin_display))
		3155	i915_gem_chipset_flush(obj->base.dev);
4104	Serge	3156
2332	Serge	3157	old_write_domain = obj->base.write_domain;
		3158	obj->base.write_domain = 0;
		3159
6084	serge	3160	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
5354	serge	3161
2351	Serge	3162	trace_i915_gem_object_change_domain(obj,
		3163	obj->base.read_domains,
		3164	old_write_domain);
2332	Serge	3165	}
		3166
		3167	/**
		3168	* Moves a single object to the GTT read, and possibly write domain.
		3169	*
		3170	* This function returns when the move is complete, including waiting on
		3171	* flushes to occur.
		3172	*/
		3173	int
		3174	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		3175	{
		3176	uint32_t old_write_domain, old_read_domains;
6084	serge	3177	struct i915_vma *vma;
2332	Serge	3178	int ret;
		3179
		3180	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		3181	return 0;
		3182
3031	serge	3183	ret = i915_gem_object_wait_rendering(obj, !write);
6084	serge	3184	if (ret)
		3185	return ret;
2332	Serge	3186
6084	serge	3187	/* Flush and acquire obj->pages so that we are coherent through
		3188	* direct access in memory with previous cached writes through
		3189	* shmemfs and that our cache domain tracking remains valid.
		3190	* For example, if the obj->filp was moved to swap without us
		3191	* being notified and releasing the pages, we would mistakenly
		3192	* continue to assume that the obj remained out of the CPU cached
		3193	* domain.
		3194	*/
		3195	ret = i915_gem_object_get_pages(obj);
		3196	if (ret)
		3197	return ret;
2332	Serge	3198
6084	serge	3199	i915_gem_object_flush_cpu_write_domain(obj);
		3200
3480	Serge	3201	/* Serialise direct access to this object with the barriers for
		3202	* coherent writes from the GPU, by effectively invalidating the
		3203	* GTT domain upon first access.
		3204	*/
		3205	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		3206	mb();
		3207
2332	Serge	3208	old_write_domain = obj->base.write_domain;
		3209	old_read_domains = obj->base.read_domains;
		3210
		3211	/* It should now be out of any other write domains, and we can update
		3212	* the domain values for our changes.
		3213	*/
		3214	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3215	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3216	if (write) {
		3217	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3218	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3219	obj->dirty = 1;
		3220	}
		3221
2351	Serge	3222	trace_i915_gem_object_change_domain(obj,
		3223	old_read_domains,
		3224	old_write_domain);
		3225
3031	serge	3226	/* And bump the LRU for this access */
6084	serge	3227	vma = i915_gem_obj_to_ggtt(obj);
		3228	if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
		3229	list_move_tail(&vma->mm_list,
		3230	&to_i915(obj->base.dev)->gtt.base.inactive_list);
3031	serge	3231
2332	Serge	3232	return 0;
		3233	}
		3234
6084	serge	3235	/**
		3236	* Changes the cache-level of an object across all VMA.
		3237	*
		3238	* After this function returns, the object will be in the new cache-level
		3239	* across all GTT and the contents of the backing storage will be coherent,
		3240	* with respect to the new cache-level. In order to keep the backing storage
		3241	* coherent for all users, we only allow a single cache level to be set
		3242	* globally on the object and prevent it from being changed whilst the
		3243	* hardware is reading from the object. That is if the object is currently
		3244	* on the scanout it will be set to uncached (or equivalent display
		3245	* cache coherency) and all non-MOCS GPU access will also be uncached so
		3246	* that all direct access to the scanout remains coherent.
		3247	*/
2335	Serge	3248	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3249	enum i915_cache_level cache_level)
		3250	{
3031	serge	3251	struct drm_device *dev = obj->base.dev;
5060	serge	3252	struct i915_vma vma, next;
6084	serge	3253	bool bound = false;
		3254	int ret = 0;
2332	Serge	3255
2335	Serge	3256	if (obj->cache_level == cache_level)
6084	serge	3257	goto out;
2332	Serge	3258
6084	serge	3259	/* Inspect the list of currently bound VMA and unbind any that would
		3260	* be invalid given the new cache-level. This is principally to
		3261	* catch the issue of the CS prefetch crossing page boundaries and
		3262	* reading an invalid PTE on older architectures.
		3263	*/
		3264	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3265	if (!drm_mm_node_allocated(&vma->node))
		3266	continue;
2332	Serge	3267
6084	serge	3268	if (vma->pin_count) {
		3269	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3270	return -EBUSY;
		3271	}
		3272
5354	serge	3273	if (!i915_gem_valid_gtt_space(vma, cache_level)) {
4104	Serge	3274	ret = i915_vma_unbind(vma);
6084	serge	3275	if (ret)
		3276	return ret;
		3277	} else
		3278	bound = true;
3031	serge	3279	}
		3280
6084	serge	3281	/* We can reuse the existing drm_mm nodes but need to change the
		3282	* cache-level on the PTE. We could simply unbind them all and
		3283	* rebind with the correct cache-level on next use. However since
		3284	* we already have a valid slot, dma mapping, pages etc, we may as
		3285	* rewrite the PTE in the belief that doing so tramples upon less
		3286	* state and so involves less work.
		3287	*/
		3288	if (bound) {
		3289	/* Before we change the PTE, the GPU must not be accessing it.
		3290	* If we wait upon the object, we know that all the bound
		3291	* VMA are no longer active.
		3292	*/
		3293	ret = i915_gem_object_wait_rendering(obj, false);
2335	Serge	3294	if (ret)
		3295	return ret;
2332	Serge	3296
6084	serge	3297	if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
		3298	/* Access to snoopable pages through the GTT is
		3299	* incoherent and on some machines causes a hard
		3300	* lockup. Relinquish the CPU mmaping to force
		3301	* userspace to refault in the pages and we can
		3302	* then double check if the GTT mapping is still
		3303	* valid for that pointer access.
		3304	*/
		3305	i915_gem_release_mmap(obj);
2332	Serge	3306
6084	serge	3307	/* As we no longer need a fence for GTT access,
		3308	* we can relinquish it now (and so prevent having
		3309	* to steal a fence from someone else on the next
		3310	* fence request). Note GPU activity would have
		3311	* dropped the fence as all snoopable access is
		3312	* supposed to be linear.
		3313	*/
2335	Serge	3314	ret = i915_gem_object_put_fence(obj);
		3315	if (ret)
		3316	return ret;
6084	serge	3317	} else {
		3318	/* We either have incoherent backing store and
		3319	* so no GTT access or the architecture is fully
		3320	* coherent. In such cases, existing GTT mmaps
		3321	* ignore the cache bit in the PTE and we can
		3322	* rewrite it without confusing the GPU or having
		3323	* to force userspace to fault back in its mmaps.
		3324	*/
		3325	}
2332	Serge	3326
6084	serge	3327	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		3328	if (!drm_mm_node_allocated(&vma->node))
		3329	continue;
		3330
		3331	ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
		3332	if (ret)
		3333	return ret;
		3334	}
2335	Serge	3335	}
2332	Serge	3336
4104	Serge	3337	list_for_each_entry(vma, &obj->vma_list, vma_link)
		3338	vma->node.color = cache_level;
		3339	obj->cache_level = cache_level;
		3340
6084	serge	3341	out:
		3342	/* Flush the dirty CPU caches to the backing storage so that the
		3343	* object is now coherent at its new cache level (with respect
		3344	* to the access domain).
		3345	*/
		3346	if (obj->cache_dirty &&
		3347	obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
		3348	cpu_write_needs_clflush(obj)) {
		3349	if (i915_gem_clflush_object(obj, true))
		3350	i915_gem_chipset_flush(obj->base.dev);
		3351	}
2332	Serge	3352
2335	Serge	3353	return 0;
		3354	}
2332	Serge	3355
3260	Serge	3356	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3357	struct drm_file *file)
		3358	{
		3359	struct drm_i915_gem_caching *args = data;
		3360	struct drm_i915_gem_object *obj;
		3361
		3362	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
6084	serge	3363	if (&obj->base == NULL)
		3364	return -ENOENT;
3260	Serge	3365
4104	Serge	3366	switch (obj->cache_level) {
		3367	case I915_CACHE_LLC:
		3368	case I915_CACHE_L3_LLC:
		3369	args->caching = I915_CACHING_CACHED;
		3370	break;
3260	Serge	3371
4104	Serge	3372	case I915_CACHE_WT:
		3373	args->caching = I915_CACHING_DISPLAY;
		3374	break;
		3375
		3376	default:
		3377	args->caching = I915_CACHING_NONE;
		3378	break;
		3379	}
		3380
6084	serge	3381	drm_gem_object_unreference_unlocked(&obj->base);
		3382	return 0;
3260	Serge	3383	}
		3384
		3385	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3386	struct drm_file *file)
		3387	{
6084	serge	3388	struct drm_i915_private *dev_priv = dev->dev_private;
3260	Serge	3389	struct drm_i915_gem_caching *args = data;
		3390	struct drm_i915_gem_object *obj;
		3391	enum i915_cache_level level;
		3392	int ret;
		3393
		3394	switch (args->caching) {
		3395	case I915_CACHING_NONE:
		3396	level = I915_CACHE_NONE;
		3397	break;
		3398	case I915_CACHING_CACHED:
6084	serge	3399	/*
		3400	* Due to a HW issue on BXT A stepping, GPU stores via a
		3401	* snooped mapping may leave stale data in a corresponding CPU
		3402	* cacheline, whereas normally such cachelines would get
		3403	* invalidated.
		3404	*/
		3405	if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
		3406	return -ENODEV;
		3407
3260	Serge	3408	level = I915_CACHE_LLC;
		3409	break;
4104	Serge	3410	case I915_CACHING_DISPLAY:
		3411	level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		3412	break;
3260	Serge	3413	default:
		3414	return -EINVAL;
		3415	}
		3416
6084	serge	3417	intel_runtime_pm_get(dev_priv);
		3418
3260	Serge	3419	ret = i915_mutex_lock_interruptible(dev);
		3420	if (ret)
6084	serge	3421	goto rpm_put;
3260	Serge	3422
		3423	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3424	if (&obj->base == NULL) {
		3425	ret = -ENOENT;
		3426	goto unlock;
		3427	}
		3428
		3429	ret = i915_gem_object_set_cache_level(obj, level);
		3430
		3431	drm_gem_object_unreference(&obj->base);
		3432	unlock:
		3433	mutex_unlock(&dev->struct_mutex);
6084	serge	3434	rpm_put:
		3435	intel_runtime_pm_put(dev_priv);
		3436
3260	Serge	3437	return ret;
		3438	}
		3439
2335	Serge	3440	/*
		3441	* Prepare buffer for display plane (scanout, cursors, etc).
		3442	* Can be called from an uninterruptible phase (modesetting) and allows
		3443	* any flushes to be pipelined (for pageflips).
		3444	*/
		3445	int
		3446	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3447	u32 alignment,
6084	serge	3448	struct intel_engine_cs *pipelined,
		3449	struct drm_i915_gem_request **pipelined_request,
		3450	const struct i915_ggtt_view *view)
2335	Serge	3451	{
		3452	u32 old_read_domains, old_write_domain;
		3453	int ret;
2332	Serge	3454
6084	serge	3455	ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
2335	Serge	3456	if (ret)
		3457	return ret;
2332	Serge	3458
4104	Serge	3459	/* Mark the pin_display early so that we account for the
		3460	* display coherency whilst setting up the cache domains.
		3461	*/
6084	serge	3462	obj->pin_display++;
4104	Serge	3463
2335	Serge	3464	/* The display engine is not coherent with the LLC cache on gen6. As
		3465	* a result, we make sure that the pinning that is about to occur is
		3466	* done with uncached PTEs. This is lowest common denominator for all
		3467	* chipsets.
		3468	*
		3469	* However for gen6+, we could do better by using the GFDT bit instead
		3470	* of uncaching, which would allow us to flush all the LLC-cached data
		3471	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3472	*/
4104	Serge	3473	ret = i915_gem_object_set_cache_level(obj,
		3474	HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
2360	Serge	3475	if (ret)
4104	Serge	3476	goto err_unpin_display;
2332	Serge	3477
2335	Serge	3478	/* As the user may map the buffer once pinned in the display plane
		3479	* (e.g. libkms for the bootup splash), we have to ensure that we
		3480	* always use map_and_fenceable for all scanout buffers.
		3481	*/
6084	serge	3482	ret = i915_gem_object_ggtt_pin(obj, view, alignment,
		3483	view->type == I915_GGTT_VIEW_NORMAL ?
		3484	PIN_MAPPABLE : 0);
2335	Serge	3485	if (ret)
4104	Serge	3486	goto err_unpin_display;
2332	Serge	3487
6084	serge	3488	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3489
2335	Serge	3490	old_write_domain = obj->base.write_domain;
		3491	old_read_domains = obj->base.read_domains;
2332	Serge	3492
2335	Serge	3493	/* It should now be out of any other write domains, and we can update
		3494	* the domain values for our changes.
		3495	*/
3031	serge	3496	obj->base.write_domain = 0;
2335	Serge	3497	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3498
2351	Serge	3499	trace_i915_gem_object_change_domain(obj,
		3500	old_read_domains,
		3501	old_write_domain);
2332	Serge	3502
2335	Serge	3503	return 0;
4104	Serge	3504
		3505	err_unpin_display:
6084	serge	3506	obj->pin_display--;
4104	Serge	3507	return ret;
2335	Serge	3508	}
2332	Serge	3509
4104	Serge	3510	void
6084	serge	3511	i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
		3512	const struct i915_ggtt_view *view)
4104	Serge	3513	{
6084	serge	3514	if (WARN_ON(obj->pin_display == 0))
		3515	return;
4104	Serge	3516
6084	serge	3517	i915_gem_object_ggtt_unpin_view(obj, view);
2332	Serge	3518
6084	serge	3519	obj->pin_display--;
2344	Serge	3520	}
2332	Serge	3521
2344	Serge	3522	/**
		3523	* Moves a single object to the CPU read, and possibly write domain.
		3524	*
		3525	* This function returns when the move is complete, including waiting on
		3526	* flushes to occur.
		3527	*/
3031	serge	3528	int
2344	Serge	3529	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3530	{
		3531	uint32_t old_write_domain, old_read_domains;
		3532	int ret;
2332	Serge	3533
2344	Serge	3534	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3535	return 0;
2332	Serge	3536
3031	serge	3537	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3538	if (ret)
		3539	return ret;
2332	Serge	3540
2344	Serge	3541	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3542
2344	Serge	3543	old_write_domain = obj->base.write_domain;
		3544	old_read_domains = obj->base.read_domains;
2332	Serge	3545
2344	Serge	3546	/* Flush the CPU cache if it's still invalid. */
		3547	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4104	Serge	3548	i915_gem_clflush_object(obj, false);
2332	Serge	3549
2344	Serge	3550	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3551	}
2332	Serge	3552
2344	Serge	3553	/* It should now be out of any other write domains, and we can update
		3554	* the domain values for our changes.
		3555	*/
		3556	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3557
2344	Serge	3558	/* If we're writing through the CPU, then the GPU read domains will
		3559	* need to be invalidated at next use.
		3560	*/
		3561	if (write) {
		3562	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3563	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3564	}
2332	Serge	3565
2351	Serge	3566	trace_i915_gem_object_change_domain(obj,
		3567	old_read_domains,
		3568	old_write_domain);
2332	Serge	3569
2344	Serge	3570	return 0;
		3571	}
2332	Serge	3572
3031	serge	3573	/* Throttle our rendering by waiting until the ring has completed our requests
		3574	* emitted over 20 msec ago.
2344	Serge	3575	*
3031	serge	3576	* Note that if we were to use the current jiffies each time around the loop,
		3577	* we wouldn't escape the function with any frames outstanding if the time to
		3578	* render a frame was over 20ms.
		3579	*
		3580	* This should get us reasonable parallelism between CPU and GPU but also
		3581	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3582	*/
3031	serge	3583	static int
		3584	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3585	{
3031	serge	3586	struct drm_i915_private *dev_priv = dev->dev_private;
		3587	struct drm_i915_file_private *file_priv = file->driver_priv;
6084	serge	3588	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
		3589	struct drm_i915_gem_request request, target = NULL;
3480	Serge	3590	unsigned reset_counter;
3031	serge	3591	int ret;
2332	Serge	3592
3480	Serge	3593	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3594	if (ret)
		3595	return ret;
2332	Serge	3596
3480	Serge	3597	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3598	if (ret)
		3599	return ret;
		3600
3031	serge	3601	spin_lock(&file_priv->mm.lock);
		3602	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3603	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3604	break;
2332	Serge	3605
6084	serge	3606	/*
		3607	* Note that the request might not have been submitted yet.
		3608	* In which case emitted_jiffies will be zero.
		3609	*/
		3610	if (!request->emitted_jiffies)
		3611	continue;
		3612
		3613	target = request;
3031	serge	3614	}
3480	Serge	3615	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
6084	serge	3616	if (target)
		3617	i915_gem_request_reference(target);
3031	serge	3618	spin_unlock(&file_priv->mm.lock);
2332	Serge	3619
6084	serge	3620	if (target == NULL)
3031	serge	3621	return 0;
2332	Serge	3622
6084	serge	3623	ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
3031	serge	3624	if (ret == 0)
		3625	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3626
6084	serge	3627	i915_gem_request_unreference__unlocked(target);
		3628
3031	serge	3629	return ret;
2352	Serge	3630	}
2332	Serge	3631
5060	serge	3632	static bool
		3633	i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
		3634	{
		3635	struct drm_i915_gem_object *obj = vma->obj;
		3636
		3637	if (alignment &&
		3638	vma->node.start & (alignment - 1))
		3639	return true;
		3640
		3641	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
		3642	return true;
		3643
		3644	if (flags & PIN_OFFSET_BIAS &&
		3645	vma->node.start < (flags & PIN_OFFSET_MASK))
		3646	return true;
		3647
		3648	return false;
		3649	}
		3650
6084	serge	3651	void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
2332	Serge	3652	{
6084	serge	3653	struct drm_i915_gem_object *obj = vma->obj;
		3654	bool mappable, fenceable;
		3655	u32 fence_size, fence_alignment;
		3656
		3657	fence_size = i915_gem_get_gtt_size(obj->base.dev,
		3658	obj->base.size,
		3659	obj->tiling_mode);
		3660	fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
		3661	obj->base.size,
		3662	obj->tiling_mode,
		3663	true);
		3664
		3665	fenceable = (vma->node.size == fence_size &&
		3666	(vma->node.start & (fence_alignment - 1)) == 0);
		3667
		3668	mappable = (vma->node.start + fence_size <=
		3669	to_i915(obj->base.dev)->gtt.mappable_end);
		3670
		3671	obj->map_and_fenceable = mappable && fenceable;
		3672	}
		3673
		3674	static int
		3675	i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
		3676	struct i915_address_space *vm,
		3677	const struct i915_ggtt_view *ggtt_view,
		3678	uint32_t alignment,
		3679	uint64_t flags)
		3680	{
5060	serge	3681	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4104	Serge	3682	struct i915_vma *vma;
5354	serge	3683	unsigned bound;
2332	Serge	3684	int ret;
		3685
5060	serge	3686	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
		3687	return -ENODEV;
2332	Serge	3688
5060	serge	3689	if (WARN_ON(flags & (PIN_GLOBAL \| PIN_MAPPABLE) && !i915_is_ggtt(vm)))
		3690	return -EINVAL;
4104	Serge	3691
5354	serge	3692	if (WARN_ON((flags & (PIN_MAPPABLE \| PIN_GLOBAL)) == PIN_MAPPABLE))
		3693	return -EINVAL;
		3694
6084	serge	3695	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
		3696	return -EINVAL;
		3697
		3698	vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
		3699	i915_gem_obj_to_vma(obj, vm);
		3700
		3701	if (IS_ERR(vma))
		3702	return PTR_ERR(vma);
		3703
5060	serge	3704	if (vma) {
		3705	if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3706	return -EBUSY;
4104	Serge	3707
5060	serge	3708	if (i915_vma_misplaced(vma, alignment, flags)) {
		3709	WARN(vma->pin_count,
6084	serge	3710	"bo is already pinned in %s with incorrect alignment:"
		3711	" offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
2332	Serge	3712	" obj->map_and_fenceable=%d\n",
6084	serge	3713	ggtt_view ? "ggtt" : "ppgtt",
		3714	upper_32_bits(vma->node.start),
		3715	lower_32_bits(vma->node.start),
		3716	alignment,
5060	serge	3717	!!(flags & PIN_MAPPABLE),
2332	Serge	3718	obj->map_and_fenceable);
4104	Serge	3719	ret = i915_vma_unbind(vma);
2332	Serge	3720	if (ret)
		3721	return ret;
5060	serge	3722
		3723	vma = NULL;
2332	Serge	3724	}
		3725	}
		3726
5354	serge	3727	bound = vma ? vma->bound : 0;
5060	serge	3728	if (vma == NULL \|\| !drm_mm_node_allocated(&vma->node)) {
6084	serge	3729	vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
		3730	flags);
5060	serge	3731	if (IS_ERR(vma))
		3732	return PTR_ERR(vma);
6084	serge	3733	} else {
		3734	ret = i915_vma_bind(vma, obj->cache_level, flags);
		3735	if (ret)
		3736	return ret;
2332	Serge	3737	}
		3738
6084	serge	3739	if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
		3740	(bound ^ vma->bound) & GLOBAL_BIND) {
		3741	__i915_vma_set_map_and_fenceable(vma);
		3742	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
5354	serge	3743	}
		3744
5060	serge	3745	vma->pin_count++;
2332	Serge	3746	return 0;
		3747	}
		3748
6084	serge	3749	int
		3750	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3751	struct i915_address_space *vm,
		3752	uint32_t alignment,
		3753	uint64_t flags)
2344	Serge	3754	{
6084	serge	3755	return i915_gem_object_do_pin(obj, vm,
		3756	i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
		3757	alignment, flags);
2344	Serge	3758	}
2332	Serge	3759
6084	serge	3760	int
		3761	i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
		3762	const struct i915_ggtt_view *view,
		3763	uint32_t alignment,
		3764	uint64_t flags)
5060	serge	3765	{
6084	serge	3766	if (WARN_ONCE(!view, "no view specified"))
		3767	return -EINVAL;
5060	serge	3768
6084	serge	3769	return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
		3770	alignment, flags \| PIN_GLOBAL);
5060	serge	3771	}
		3772
		3773	void
6084	serge	3774	i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
		3775	const struct i915_ggtt_view *view)
5060	serge	3776	{
6084	serge	3777	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
5060	serge	3778
6084	serge	3779	BUG_ON(!vma);
		3780	WARN_ON(vma->pin_count == 0);
		3781	WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
2332	Serge	3782
6084	serge	3783	--vma->pin_count;
3031	serge	3784	}
2332	Serge	3785
3031	serge	3786	int
		3787	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3788	struct drm_file *file)
		3789	{
		3790	struct drm_i915_gem_busy *args = data;
		3791	struct drm_i915_gem_object *obj;
		3792	int ret;
2332	Serge	3793
3031	serge	3794	ret = i915_mutex_lock_interruptible(dev);
		3795	if (ret)
		3796	return ret;
2332	Serge	3797
5060	serge	3798	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3031	serge	3799	if (&obj->base == NULL) {
		3800	ret = -ENOENT;
		3801	goto unlock;
		3802	}
2332	Serge	3803
3031	serge	3804	/* Count all active objects as busy, even if they are currently not used
		3805	* by the gpu. Users of this interface expect objects to eventually
		3806	* become non-busy without any further actions, therefore emit any
		3807	* necessary flushes here.
		3808	*/
		3809	ret = i915_gem_object_flush_active(obj);
6084	serge	3810	if (ret)
		3811	goto unref;
2332	Serge	3812
6084	serge	3813	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3814	args->busy = obj->active << 16;
		3815	if (obj->last_write_req)
		3816	args->busy \|= obj->last_write_req->ring->id;
2332	Serge	3817
6084	serge	3818	unref:
3031	serge	3819	drm_gem_object_unreference(&obj->base);
		3820	unlock:
		3821	mutex_unlock(&dev->struct_mutex);
		3822	return ret;
		3823	}
2332	Serge	3824
3031	serge	3825	int
		3826	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3827	struct drm_file *file_priv)
		3828	{
		3829	return i915_gem_ring_throttle(dev, file_priv);
		3830	}
2332	Serge	3831
3263	Serge	3832	#if 0
		3833
3031	serge	3834	int
		3835	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3836	struct drm_file *file_priv)
		3837	{
5354	serge	3838	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3839	struct drm_i915_gem_madvise *args = data;
		3840	struct drm_i915_gem_object *obj;
		3841	int ret;
2332	Serge	3842
3031	serge	3843	switch (args->madv) {
		3844	case I915_MADV_DONTNEED:
		3845	case I915_MADV_WILLNEED:
		3846	break;
		3847	default:
		3848	return -EINVAL;
		3849	}
2332	Serge	3850
3031	serge	3851	ret = i915_mutex_lock_interruptible(dev);
		3852	if (ret)
		3853	return ret;
2332	Serge	3854
3031	serge	3855	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3856	if (&obj->base == NULL) {
		3857	ret = -ENOENT;
		3858	goto unlock;
		3859	}
2332	Serge	3860
5060	serge	3861	if (i915_gem_obj_is_pinned(obj)) {
3031	serge	3862	ret = -EINVAL;
		3863	goto out;
		3864	}
2332	Serge	3865
5354	serge	3866	if (obj->pages &&
		3867	obj->tiling_mode != I915_TILING_NONE &&
		3868	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		3869	if (obj->madv == I915_MADV_WILLNEED)
		3870	i915_gem_object_unpin_pages(obj);
		3871	if (args->madv == I915_MADV_WILLNEED)
		3872	i915_gem_object_pin_pages(obj);
		3873	}
		3874
3031	serge	3875	if (obj->madv != __I915_MADV_PURGED)
		3876	obj->madv = args->madv;
2332	Serge	3877
3031	serge	3878	/* if the object is no longer attached, discard its backing storage */
6084	serge	3879	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
3031	serge	3880	i915_gem_object_truncate(obj);
2332	Serge	3881
3031	serge	3882	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3883
3031	serge	3884	out:
		3885	drm_gem_object_unreference(&obj->base);
		3886	unlock:
		3887	mutex_unlock(&dev->struct_mutex);
		3888	return ret;
		3889	}
		3890	#endif
2332	Serge	3891
3031	serge	3892	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3893	const struct drm_i915_gem_object_ops *ops)
		3894	{
6084	serge	3895	int i;
		3896
4104	Serge	3897	INIT_LIST_HEAD(&obj->global_list);
6084	serge	3898	for (i = 0; i < I915_NUM_RINGS; i++)
		3899	INIT_LIST_HEAD(&obj->ring_list[i]);
4104	Serge	3900	INIT_LIST_HEAD(&obj->obj_exec_link);
		3901	INIT_LIST_HEAD(&obj->vma_list);
6084	serge	3902	INIT_LIST_HEAD(&obj->batch_pool_link);
2332	Serge	3903
3031	serge	3904	obj->ops = ops;
		3905
		3906	obj->fence_reg = I915_FENCE_REG_NONE;
		3907	obj->madv = I915_MADV_WILLNEED;
		3908
		3909	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3910	}
		3911
		3912	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3913	.get_pages = i915_gem_object_get_pages_gtt,
		3914	.put_pages = i915_gem_object_put_pages_gtt,
		3915	};
		3916
2332	Serge	3917	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3918	size_t size)
		3919	{
		3920	struct drm_i915_gem_object *obj;
3031	serge	3921	struct address_space *mapping;
3480	Serge	3922	gfp_t mask;
2340	Serge	3923
3746	Serge	3924	obj = i915_gem_object_alloc(dev);
2332	Serge	3925	if (obj == NULL)
		3926	return NULL;
		3927
		3928	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4104	Serge	3929	i915_gem_object_free(obj);
2332	Serge	3930	return NULL;
		3931	}
		3932
		3933
3031	serge	3934	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3935
		3936	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3937	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3938
3031	serge	3939	if (HAS_LLC(dev)) {
		3940	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3941	* cache) for about a 10% performance improvement
		3942	* compared to uncached. Graphics requests other than
		3943	* display scanout are coherent with the CPU in
		3944	* accessing this cache. This means in this mode we
		3945	* don't need to clflush on the CPU side, and on the
		3946	* GPU side we only need to flush internal caches to
		3947	* get data visible to the CPU.
		3948	*
		3949	* However, we maintain the display planes as UC, and so
		3950	* need to rebind when first used as such.
		3951	*/
		3952	obj->cache_level = I915_CACHE_LLC;
		3953	} else
		3954	obj->cache_level = I915_CACHE_NONE;
		3955
4560	Serge	3956	trace_i915_gem_object_create(obj);
		3957
2332	Serge	3958	return obj;
		3959	}
		3960
3031	serge	3961	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3962	{
3031	serge	3963	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3964	struct drm_device *dev = obj->base.dev;
5060	serge	3965	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3966	struct i915_vma vma, next;
2332	Serge	3967
4560	Serge	3968	intel_runtime_pm_get(dev_priv);
		3969
3031	serge	3970	trace_i915_gem_object_destroy(obj);
		3971
5060	serge	3972	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		3973	int ret;
3031	serge	3974
5060	serge	3975	vma->pin_count = 0;
		3976	ret = i915_vma_unbind(vma);
4104	Serge	3977	if (WARN_ON(ret == -ERESTARTSYS)) {
6084	serge	3978	bool was_interruptible;
3031	serge	3979
6084	serge	3980	was_interruptible = dev_priv->mm.interruptible;
		3981	dev_priv->mm.interruptible = false;
3031	serge	3982
4104	Serge	3983	WARN_ON(i915_vma_unbind(vma));
3031	serge	3984
6084	serge	3985	dev_priv->mm.interruptible = was_interruptible;
		3986	}
2344	Serge	3987	}
2332	Serge	3988
4104	Serge	3989	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
		3990	* before progressing. */
		3991	if (obj->stolen)
		3992	i915_gem_object_unpin_pages(obj);
		3993
5060	serge	3994	WARN_ON(obj->frontbuffer_bits);
		3995
5354	serge	3996	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
		3997	dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
		3998	obj->tiling_mode != I915_TILING_NONE)
		3999	i915_gem_object_unpin_pages(obj);
		4000
4104	Serge	4001	if (WARN_ON(obj->pages_pin_count))
6084	serge	4002	obj->pages_pin_count = 0;
3031	serge	4003	i915_gem_object_put_pages(obj);
		4004	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	4005
3243	Serge	4006	BUG_ON(obj->pages);
2332	Serge	4007
3031	serge	4008
3290	Serge	4009	if(obj->base.filp != NULL)
		4010	{
3298	Serge	4011	// printf("filp %p\n", obj->base.filp);
3290	Serge	4012	shmem_file_delete(obj->base.filp);
		4013	}
		4014
2344	Serge	4015	drm_gem_object_release(&obj->base);
		4016	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	4017
2344	Serge	4018	kfree(obj->bit_17);
4104	Serge	4019	i915_gem_object_free(obj);
4560	Serge	4020
		4021	intel_runtime_pm_put(dev_priv);
2344	Serge	4022	}
2332	Serge	4023
4560	Serge	4024	struct i915_vma i915_gem_obj_to_vma(struct drm_i915_gem_object obj,
4104	Serge	4025	struct i915_address_space *vm)
		4026	{
4560	Serge	4027	struct i915_vma *vma;
6084	serge	4028	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		4029	if (i915_is_ggtt(vma->vm) &&
		4030	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4031	continue;
4560	Serge	4032	if (vma->vm == vm)
		4033	return vma;
6084	serge	4034	}
		4035	return NULL;
		4036	}
4560	Serge	4037
6084	serge	4038	struct i915_vma i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object obj,
		4039	const struct i915_ggtt_view *view)
		4040	{
		4041	struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
		4042	struct i915_vma *vma;
		4043
		4044	if (WARN_ONCE(!view, "no view specified"))
		4045	return ERR_PTR(-EINVAL);
		4046
		4047	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4048	if (vma->vm == ggtt &&
		4049	i915_ggtt_view_equal(&vma->ggtt_view, view))
		4050	return vma;
4560	Serge	4051	return NULL;
		4052	}
		4053
4104	Serge	4054	void i915_gem_vma_destroy(struct i915_vma *vma)
		4055	{
5354	serge	4056	struct i915_address_space *vm = NULL;
4104	Serge	4057	WARN_ON(vma->node.allocated);
4560	Serge	4058
		4059	/* Keep the vma as a placeholder in the execbuffer reservation lists */
		4060	if (!list_empty(&vma->exec_list))
		4061	return;
		4062
5354	serge	4063	vm = vma->vm;
		4064
		4065	if (!i915_is_ggtt(vm))
		4066	i915_ppgtt_put(i915_vm_to_ppgtt(vm));
		4067
4104	Serge	4068	list_del(&vma->vma_link);
4560	Serge	4069
4104	Serge	4070	kfree(vma);
		4071	}
		4072
6084	serge	4073	static void
		4074	i915_gem_stop_ringbuffers(struct drm_device *dev)
		4075	{
		4076	struct drm_i915_private *dev_priv = dev->dev_private;
		4077	struct intel_engine_cs *ring;
		4078	int i;
		4079
		4080	for_each_ring(ring, dev_priv, i)
		4081	dev_priv->gt.stop_ring(ring);
		4082	}
		4083
3031	serge	4084	#if 0
		4085	int
4560	Serge	4086	i915_gem_suspend(struct drm_device *dev)
2344	Serge	4087	{
5060	serge	4088	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4089	int ret = 0;
2332	Serge	4090
4560	Serge	4091	mutex_lock(&dev->struct_mutex);
3031	serge	4092	ret = i915_gpu_idle(dev);
4560	Serge	4093	if (ret)
		4094	goto err;
		4095
3031	serge	4096	i915_gem_retire_requests(dev);
		4097
5060	serge	4098	i915_gem_stop_ringbuffers(dev);
4560	Serge	4099	mutex_unlock(&dev->struct_mutex);
		4100
6084	serge	4101	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
3263	Serge	4102	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
5060	serge	4103	flush_delayed_work(&dev_priv->mm.idle_work);
3031	serge	4104
6084	serge	4105	/* Assert that we sucessfully flushed all the work and
		4106	* reset the GPU back to its idle, low power state.
		4107	*/
		4108	WARN_ON(dev_priv->mm.busy);
		4109
3031	serge	4110	return 0;
4560	Serge	4111
		4112	err:
		4113	mutex_unlock(&dev->struct_mutex);
		4114	return ret;
2344	Serge	4115	}
3031	serge	4116	#endif
2332	Serge	4117
6084	serge	4118	int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
3031	serge	4119	{
6084	serge	4120	struct intel_engine_cs *ring = req->ring;
4560	Serge	4121	struct drm_device *dev = ring->dev;
5060	serge	4122	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4123	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
		4124	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
		4125	int i, ret;
2332	Serge	4126
4560	Serge	4127	if (!HAS_L3_DPF(dev) \|\| !remap_info)
		4128	return 0;
2332	Serge	4129
6084	serge	4130	ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
4560	Serge	4131	if (ret)
		4132	return ret;
2332	Serge	4133
4560	Serge	4134	/*
		4135	* Note: We do not worry about the concurrent register cacheline hang
		4136	* here because no other code should access these registers other than
		4137	* at initialization time.
		4138	*/
3031	serge	4139	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4560	Serge	4140	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		4141	intel_ring_emit(ring, reg_base + i);
		4142	intel_ring_emit(ring, remap_info[i/4]);
3031	serge	4143	}
2332	Serge	4144
4560	Serge	4145	intel_ring_advance(ring);
2332	Serge	4146
4560	Serge	4147	return ret;
3031	serge	4148	}
2332	Serge	4149
3031	serge	4150	void i915_gem_init_swizzling(struct drm_device *dev)
		4151	{
5060	serge	4152	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4153
3031	serge	4154	if (INTEL_INFO(dev)->gen < 5 \|\|
		4155	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		4156	return;
2332	Serge	4157
3031	serge	4158	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		4159	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	4160
3031	serge	4161	if (IS_GEN5(dev))
		4162	return;
2344	Serge	4163
3031	serge	4164	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		4165	if (IS_GEN6(dev))
		4166	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	4167	else if (IS_GEN7(dev))
		4168	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4560	Serge	4169	else if (IS_GEN8(dev))
		4170	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
3031	serge	4171	else
3480	Serge	4172	BUG();
3031	serge	4173	}
		4174
5354	serge	4175	static void init_unused_ring(struct drm_device *dev, u32 base)
2332	Serge	4176	{
3480	Serge	4177	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4178
		4179	I915_WRITE(RING_CTL(base), 0);
		4180	I915_WRITE(RING_HEAD(base), 0);
		4181	I915_WRITE(RING_TAIL(base), 0);
		4182	I915_WRITE(RING_START(base), 0);
		4183	}
		4184
		4185	static void init_unused_rings(struct drm_device *dev)
		4186	{
		4187	if (IS_I830(dev)) {
		4188	init_unused_ring(dev, PRB1_BASE);
		4189	init_unused_ring(dev, SRB0_BASE);
		4190	init_unused_ring(dev, SRB1_BASE);
		4191	init_unused_ring(dev, SRB2_BASE);
		4192	init_unused_ring(dev, SRB3_BASE);
		4193	} else if (IS_GEN2(dev)) {
		4194	init_unused_ring(dev, SRB0_BASE);
		4195	init_unused_ring(dev, SRB1_BASE);
		4196	} else if (IS_GEN3(dev)) {
		4197	init_unused_ring(dev, PRB1_BASE);
		4198	init_unused_ring(dev, PRB2_BASE);
		4199	}
		4200	}
		4201
		4202	int i915_gem_init_rings(struct drm_device *dev)
		4203	{
		4204	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	4205	int ret;
2351	Serge	4206
2332	Serge	4207	ret = intel_init_render_ring_buffer(dev);
		4208	if (ret)
		4209	return ret;
		4210
6084	serge	4211	if (HAS_BSD(dev)) {
2332	Serge	4212	ret = intel_init_bsd_ring_buffer(dev);
		4213	if (ret)
		4214	goto cleanup_render_ring;
		4215	}
		4216
6084	serge	4217	if (HAS_BLT(dev)) {
2332	Serge	4218	ret = intel_init_blt_ring_buffer(dev);
		4219	if (ret)
		4220	goto cleanup_bsd_ring;
		4221	}
		4222
4104	Serge	4223	if (HAS_VEBOX(dev)) {
		4224	ret = intel_init_vebox_ring_buffer(dev);
		4225	if (ret)
		4226	goto cleanup_blt_ring;
		4227	}
		4228
5060	serge	4229	if (HAS_BSD2(dev)) {
		4230	ret = intel_init_bsd2_ring_buffer(dev);
		4231	if (ret)
		4232	goto cleanup_vebox_ring;
		4233	}
4104	Serge	4234
2332	Serge	4235	return 0;
		4236
4104	Serge	4237	cleanup_vebox_ring:
		4238	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
3480	Serge	4239	cleanup_blt_ring:
		4240	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	4241	cleanup_bsd_ring:
		4242	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		4243	cleanup_render_ring:
		4244	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	4245
2332	Serge	4246	return ret;
		4247	}
		4248
3480	Serge	4249	int
		4250	i915_gem_init_hw(struct drm_device *dev)
3031	serge	4251	{
5060	serge	4252	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4253	struct intel_engine_cs *ring;
		4254	int ret, i, j;
3031	serge	4255
3480	Serge	4256	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		4257	return -EIO;
3031	serge	4258
6084	serge	4259	/* Double layer security blanket, see i915_gem_init() */
		4260	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4261
4104	Serge	4262	if (dev_priv->ellc_size)
		4263	I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) \| IDIHASHMSK(0xf));
3480	Serge	4264
4560	Serge	4265	if (IS_HASWELL(dev))
		4266	I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
		4267	LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
		4268
3746	Serge	4269	if (HAS_PCH_NOP(dev)) {
5060	serge	4270	if (IS_IVYBRIDGE(dev)) {
6084	serge	4271	u32 temp = I915_READ(GEN7_MSG_CTL);
		4272	temp &= ~(WAIT_FOR_PCH_FLR_ACK \| WAIT_FOR_PCH_RESET_ACK);
		4273	I915_WRITE(GEN7_MSG_CTL, temp);
5060	serge	4274	} else if (INTEL_INFO(dev)->gen >= 7) {
		4275	u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
		4276	temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
		4277	I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
		4278	}
3746	Serge	4279	}
		4280
3480	Serge	4281	i915_gem_init_swizzling(dev);
		4282
6084	serge	4283	/*
		4284	* At least 830 can leave some of the unused rings
		4285	* "active" (ie. head != tail) after resume which
		4286	* will prevent c3 entry. Makes sure all unused rings
		4287	* are totally idle.
		4288	*/
		4289	init_unused_rings(dev);
3480	Serge	4290
6084	serge	4291	BUG_ON(!dev_priv->ring[RCS].default_context);
4560	Serge	4292
6084	serge	4293	ret = i915_ppgtt_init_hw(dev);
		4294	if (ret) {
		4295	DRM_ERROR("PPGTT enable HW failed %d\n", ret);
		4296	goto out;
		4297	}
		4298
		4299	/* Need to do basic initialisation of all rings first: */
		4300	for_each_ring(ring, dev_priv, i) {
		4301	ret = ring->init_hw(ring);
		4302	if (ret)
		4303	goto out;
		4304	}
		4305
		4306	/* We can't enable contexts until all firmware is loaded */
		4307	if (HAS_GUC_UCODE(dev)) {
		4308	ret = intel_guc_ucode_load(dev);
		4309	if (ret) {
		4310	/*
		4311	* If we got an error and GuC submission is enabled, map
		4312	* the error to -EIO so the GPU will be declared wedged.
		4313	* OTOH, if we didn't intend to use the GuC anyway, just
		4314	* discard the error and carry on.
		4315	*/
		4316	DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
		4317	i915.enable_guc_submission ? "" :
		4318	" (ignored)");
		4319	ret = i915.enable_guc_submission ? -EIO : 0;
		4320	if (ret)
		4321	goto out;
		4322	}
		4323	}
		4324
3480	Serge	4325	/*
6084	serge	4326	* Increment the next seqno by 0x100 so we have a visible break
		4327	* on re-initialisation
3480	Serge	4328	*/
6084	serge	4329	ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
		4330	if (ret)
		4331	goto out;
5354	serge	4332
6084	serge	4333	/* Now it is safe to go back round and do everything else: */
		4334	for_each_ring(ring, dev_priv, i) {
		4335	struct drm_i915_gem_request *req;
4560	Serge	4336
6084	serge	4337	WARN_ON(!ring->default_context);
		4338
		4339	ret = i915_gem_request_alloc(ring, ring->default_context, &req);
		4340	if (ret) {
		4341	i915_gem_cleanup_ringbuffer(dev);
		4342	goto out;
		4343	}
		4344
		4345	if (ring->id == RCS) {
		4346	for (j = 0; j < NUM_L3_SLICES(dev); j++)
		4347	i915_gem_l3_remap(req, j);
		4348	}
		4349
		4350	ret = i915_ppgtt_init_ring(req);
		4351	if (ret && ret != -EIO) {
		4352	DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
		4353	i915_gem_request_cancel(req);
		4354	i915_gem_cleanup_ringbuffer(dev);
		4355	goto out;
		4356	}
		4357
		4358	ret = i915_gem_context_enable(req);
		4359	if (ret && ret != -EIO) {
		4360	DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
		4361	i915_gem_request_cancel(req);
		4362	i915_gem_cleanup_ringbuffer(dev);
		4363	goto out;
		4364	}
		4365
		4366	i915_add_request_no_flush(req);
5354	serge	4367	}
		4368
6084	serge	4369	out:
		4370	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	4371	return ret;
3031	serge	4372	}
		4373
		4374	int i915_gem_init(struct drm_device *dev)
		4375	{
		4376	struct drm_i915_private *dev_priv = dev->dev_private;
		4377	int ret;
		4378
5354	serge	4379	i915.enable_execlists = intel_sanitize_enable_execlists(dev,
		4380	i915.enable_execlists);
		4381
3031	serge	4382	mutex_lock(&dev->struct_mutex);
3746	Serge	4383
		4384	if (IS_VALLEYVIEW(dev)) {
		4385	/* VLVA0 (potential hack), BIOS isn't actually waking us */
5060	serge	4386	I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
		4387	if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
		4388	VLV_GTLC_ALLOWWAKEACK), 10))
3746	Serge	4389	DRM_DEBUG_DRIVER("allow wake ack timed out\n");
		4390	}
		4391
5354	serge	4392	if (!i915.enable_execlists) {
6084	serge	4393	dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
5354	serge	4394	dev_priv->gt.init_rings = i915_gem_init_rings;
		4395	dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
		4396	dev_priv->gt.stop_ring = intel_stop_ring_buffer;
		4397	} else {
6084	serge	4398	dev_priv->gt.execbuf_submit = intel_execlists_submission;
5354	serge	4399	dev_priv->gt.init_rings = intel_logical_rings_init;
		4400	dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
		4401	dev_priv->gt.stop_ring = intel_logical_ring_stop;
		4402	}
		4403
6084	serge	4404	/* This is just a security blanket to placate dragons.
		4405	* On some systems, we very sporadically observe that the first TLBs
		4406	* used by the CS may be stale, despite us poking the TLB reset. If
		4407	* we hold the forcewake during initialisation these problems
		4408	* just magically go away.
		4409	*/
		4410	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4411
6084	serge	4412	// ret = i915_gem_init_userptr(dev);
		4413	// if (ret)
		4414	// goto out_unlock;
3746	Serge	4415
6084	serge	4416	i915_gem_init_global_gtt(dev);
		4417
5060	serge	4418	ret = i915_gem_context_init(dev);
6084	serge	4419	if (ret)
		4420	goto out_unlock;
3031	serge	4421
6084	serge	4422	ret = dev_priv->gt.init_rings(dev);
		4423	if (ret)
		4424	goto out_unlock;
		4425
5060	serge	4426	ret = i915_gem_init_hw(dev);
		4427	if (ret == -EIO) {
		4428	/* Allow ring initialisation to fail by marking the GPU as
		4429	* wedged. But we only want to do this where the GPU is angry,
		4430	* for all other failure, such as an allocation failure, bail.
		4431	*/
		4432	DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
6084	serge	4433	atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
5060	serge	4434	ret = 0;
		4435	}
6084	serge	4436
		4437	out_unlock:
		4438	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	4439	mutex_unlock(&dev->struct_mutex);
3746	Serge	4440
6084	serge	4441	return ret;
3031	serge	4442	}
		4443
2332	Serge	4444	void
		4445	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		4446	{
5060	serge	4447	struct drm_i915_private *dev_priv = dev->dev_private;
		4448	struct intel_engine_cs *ring;
2332	Serge	4449	int i;
		4450
3031	serge	4451	for_each_ring(ring, dev_priv, i)
5354	serge	4452	dev_priv->gt.cleanup_ring(ring);
2332	Serge	4453	}
		4454
		4455	static void
5060	serge	4456	init_ring_lists(struct intel_engine_cs *ring)
2326	Serge	4457	{
6084	serge	4458	INIT_LIST_HEAD(&ring->active_list);
		4459	INIT_LIST_HEAD(&ring->request_list);
2326	Serge	4460	}
		4461
		4462	void
		4463	i915_gem_load(struct drm_device *dev)
		4464	{
5060	serge	4465	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4466	int i;
2326	Serge	4467
4104	Serge	4468	INIT_LIST_HEAD(&dev_priv->vm_list);
4560	Serge	4469	INIT_LIST_HEAD(&dev_priv->context_list);
3031	serge	4470	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4471	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
6084	serge	4472	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4473	for (i = 0; i < I915_NUM_RINGS; i++)
		4474	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4475	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
6084	serge	4476	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4477	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4478	i915_gem_retire_work_handler);
4560	Serge	4479	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
		4480	i915_gem_idle_work_handler);
3480	Serge	4481	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4482
6084	serge	4483	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
2326	Serge	4484
3746	Serge	4485	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		4486	dev_priv->num_fence_regs = 32;
		4487	else if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
6084	serge	4488	dev_priv->num_fence_regs = 16;
		4489	else
		4490	dev_priv->num_fence_regs = 8;
2326	Serge	4491
6084	serge	4492	if (intel_vgpu_active(dev))
		4493	dev_priv->num_fence_regs =
		4494	I915_READ(vgtif_reg(avail_rs.fence_num));
		4495
		4496	/*
		4497	* Set initial sequence number for requests.
		4498	* Using this number allows the wraparound to happen early,
		4499	* catching any obvious problems.
		4500	*/
		4501	dev_priv->next_seqno = ((u32)~0 - 0x1100);
		4502	dev_priv->last_seqno = ((u32)~0 - 0x1101);
		4503
		4504	/* Initialize fence registers to zero */
3746	Serge	4505	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4506	i915_gem_restore_fences(dev);
2326	Serge	4507
6084	serge	4508	i915_gem_detect_bit_6_swizzle(dev);
2326	Serge	4509
6084	serge	4510	dev_priv->mm.interruptible = true;
2326	Serge	4511
5060	serge	4512	mutex_init(&dev_priv->fb_tracking.lock);
2326	Serge	4513	}
		4514
6084	serge	4515	void i915_gem_release(struct drm_device dev, struct drm_file file)
		4516	{
		4517	struct drm_i915_file_private *file_priv = file->driver_priv;
		4518
		4519	/* Clean up our request list when the client is going away, so that
		4520	* later retire_requests won't dereference our soon-to-be-gone
		4521	* file_priv.
		4522	*/
		4523	spin_lock(&file_priv->mm.lock);
		4524	while (!list_empty(&file_priv->mm.request_list)) {
		4525	struct drm_i915_gem_request *request;
		4526
		4527	request = list_first_entry(&file_priv->mm.request_list,
		4528	struct drm_i915_gem_request,
		4529	client_list);
		4530	list_del(&request->client_list);
		4531	request->file_priv = NULL;
		4532	}
		4533	spin_unlock(&file_priv->mm.lock);
		4534
		4535	if (!list_empty(&file_priv->rps.link)) {
		4536	spin_lock(&to_i915(dev)->rps.client_lock);
		4537	list_del(&file_priv->rps.link);
		4538	spin_unlock(&to_i915(dev)->rps.client_lock);
		4539	}
		4540	}
		4541
5060	serge	4542	int i915_gem_open(struct drm_device dev, struct drm_file file)
4104	Serge	4543	{
5060	serge	4544	struct drm_i915_file_private *file_priv;
4104	Serge	4545	int ret;
2326	Serge	4546
5060	serge	4547	DRM_DEBUG_DRIVER("\n");
4104	Serge	4548
5060	serge	4549	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
		4550	if (!file_priv)
4104	Serge	4551	return -ENOMEM;
		4552
5060	serge	4553	file->driver_priv = file_priv;
		4554	file_priv->dev_priv = dev->dev_private;
		4555	file_priv->file = file;
6084	serge	4556	INIT_LIST_HEAD(&file_priv->rps.link);
4104	Serge	4557
5060	serge	4558	spin_lock_init(&file_priv->mm.lock);
		4559	INIT_LIST_HEAD(&file_priv->mm.request_list);
4104	Serge	4560
5060	serge	4561	ret = i915_gem_context_open(dev, file);
		4562	if (ret)
		4563	kfree(file_priv);
4104	Serge	4564
		4565	return ret;
		4566	}
		4567
5354	serge	4568	/**
		4569	* i915_gem_track_fb - update frontbuffer tracking
6084	serge	4570	* @old: current GEM buffer for the frontbuffer slots
		4571	* @new: new GEM buffer for the frontbuffer slots
		4572	* @frontbuffer_bits: bitmask of frontbuffer slots
5354	serge	4573	*
		4574	* This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
		4575	* from @old and setting them in @new. Both @old and @new can be NULL.
		4576	*/
5060	serge	4577	void i915_gem_track_fb(struct drm_i915_gem_object *old,
		4578	struct drm_i915_gem_object *new,
		4579	unsigned frontbuffer_bits)
4104	Serge	4580	{
5060	serge	4581	if (old) {
		4582	WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
		4583	WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
		4584	old->frontbuffer_bits &= ~frontbuffer_bits;
4104	Serge	4585	}
		4586
5060	serge	4587	if (new) {
		4588	WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
		4589	WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
		4590	new->frontbuffer_bits \|= frontbuffer_bits;
4104	Serge	4591	}
		4592	}
		4593
		4594	/* All the new VM stuff */
6084	serge	4595	u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
		4596	struct i915_address_space *vm)
4104	Serge	4597	{
		4598	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4599	struct i915_vma *vma;
		4600
5354	serge	4601	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4104	Serge	4602
		4603	list_for_each_entry(vma, &o->vma_list, vma_link) {
6084	serge	4604	if (i915_is_ggtt(vma->vm) &&
		4605	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4606	continue;
4104	Serge	4607	if (vma->vm == vm)
		4608	return vma->node.start;
6084	serge	4609	}
4104	Serge	4610
5060	serge	4611	WARN(1, "%s vma for this object not found.\n",
		4612	i915_is_ggtt(vm) ? "global" : "ppgtt");
		4613	return -1;
4104	Serge	4614	}
		4615
6084	serge	4616	u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
		4617	const struct i915_ggtt_view *view)
		4618	{
		4619	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
		4620	struct i915_vma *vma;
		4621
		4622	list_for_each_entry(vma, &o->vma_list, vma_link)
		4623	if (vma->vm == ggtt &&
		4624	i915_ggtt_view_equal(&vma->ggtt_view, view))
		4625	return vma->node.start;
		4626
		4627	WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
		4628	return -1;
		4629	}
		4630
4104	Serge	4631	bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
		4632	struct i915_address_space *vm)
		4633	{
		4634	struct i915_vma *vma;
		4635
6084	serge	4636	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4637	if (i915_is_ggtt(vma->vm) &&
		4638	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4639	continue;
4104	Serge	4640	if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
		4641	return true;
6084	serge	4642	}
4104	Serge	4643
		4644	return false;
		4645	}
		4646
6084	serge	4647	bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
		4648	const struct i915_ggtt_view *view)
		4649	{
		4650	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
		4651	struct i915_vma *vma;
		4652
		4653	list_for_each_entry(vma, &o->vma_list, vma_link)
		4654	if (vma->vm == ggtt &&
		4655	i915_ggtt_view_equal(&vma->ggtt_view, view) &&
		4656	drm_mm_node_allocated(&vma->node))
		4657	return true;
		4658
		4659	return false;
		4660	}
		4661
4104	Serge	4662	bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
		4663	{
4560	Serge	4664	struct i915_vma *vma;
4104	Serge	4665
4560	Serge	4666	list_for_each_entry(vma, &o->vma_list, vma_link)
		4667	if (drm_mm_node_allocated(&vma->node))
4104	Serge	4668	return true;
		4669
		4670	return false;
		4671	}
		4672
		4673	unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
		4674	struct i915_address_space *vm)
		4675	{
		4676	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
		4677	struct i915_vma *vma;
		4678
5354	serge	4679	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4104	Serge	4680
		4681	BUG_ON(list_empty(&o->vma_list));
		4682
6084	serge	4683	list_for_each_entry(vma, &o->vma_list, vma_link) {
		4684	if (i915_is_ggtt(vma->vm) &&
		4685	vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
		4686	continue;
4104	Serge	4687	if (vma->vm == vm)
		4688	return vma->node.size;
6084	serge	4689	}
4104	Serge	4690	return 0;
		4691	}
4560	Serge	4692
6084	serge	4693	bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
		4694	{
		4695	struct i915_vma *vma;
		4696	list_for_each_entry(vma, &obj->vma_list, vma_link)
		4697	if (vma->pin_count > 0)
		4698	return true;
4560	Serge	4699
6084	serge	4700	return false;
		4701	}
5060	serge	4702
6084	serge	4703	/* Allocate a new GEM object and fill it with the supplied data */
		4704	struct drm_i915_gem_object *
		4705	i915_gem_object_create_from_data(struct drm_device *dev,
		4706	const void *data, size_t size)
4104	Serge	4707	{
6084	serge	4708	struct drm_i915_gem_object *obj;
		4709	struct sg_table *sg;
		4710	size_t bytes;
		4711	int ret;
4104	Serge	4712
6084	serge	4713	obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
		4714	if (IS_ERR_OR_NULL(obj))
		4715	return obj;
4104	Serge	4716
6084	serge	4717	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		4718	if (ret)
		4719	goto fail;
		4720
		4721	ret = i915_gem_object_get_pages(obj);
		4722	if (ret)
		4723	goto fail;
		4724
		4725	i915_gem_object_pin_pages(obj);
		4726	sg = obj->pages;
		4727	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
		4728	i915_gem_object_unpin_pages(obj);
		4729
		4730	if (WARN_ON(bytes != size)) {
		4731	DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
		4732	ret = -EFAULT;
		4733	goto fail;
		4734	}
		4735
		4736	return obj;
		4737
		4738	fail:
		4739	drm_gem_object_unreference(&obj->base);
		4740	return ERR_PTR(ret);
4104	Serge	4741	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 6084