WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
3263	Serge	40	#define PROT_READ 0x1 /* page can be read */
		41	#define PROT_WRITE 0x2 /* page can be written */
		42	#define MAP_SHARED 0x01 /* Share changes */
		43
2344	Serge	44	#undef mb
		45	#undef rmb
		46	#undef wmb
		47	#define mb() asm volatile("mfence")
		48	#define rmb() asm volatile ("lfence")
		49	#define wmb() asm volatile ("sfence")
		50
3266	Serge	51	struct drm_i915_gem_object *get_fb_obj();
		52
3263	Serge	53	unsigned long vm_mmap(struct file *file, unsigned long addr,
		54	unsigned long len, unsigned long prot,
		55	unsigned long flag, unsigned long offset);
		56
2344	Serge	57	static inline void clflush(volatile void *__p)
		58	{
		59	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		60	}
		61
2332	Serge	62	#define MAX_ERRNO 4095
		63
		64	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		65
		66
2326	Serge	67	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		68	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		69	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		70	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		71
2332	Serge	72	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		73	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		74	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		75	unsigned alignment,
3031	serge	76	bool map_and_fenceable,
		77	bool nonblocking);
2332	Serge	78	static int i915_gem_phys_pwrite(struct drm_device *dev,
		79	struct drm_i915_gem_object *obj,
		80	struct drm_i915_gem_pwrite *args,
		81	struct drm_file *file);
2326	Serge	82
3031	serge	83	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		84	struct drm_i915_gem_object *obj);
		85	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		86	struct drm_i915_fence_reg *fence,
		87	bool enable);
2332	Serge	88
3031	serge	89	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		90	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		91	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		92
		93	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		94	{
		95	if (obj->tiling_mode)
		96	i915_gem_release_mmap(obj);
		97
		98	/* As we do not have an associated fence register, we will force
		99	* a tiling change if we ever need to acquire one.
		100	*/
		101	obj->fence_dirty = false;
		102	obj->fence_reg = I915_FENCE_REG_NONE;
		103	}
		104
2332	Serge	105	/* some bookkeeping */
		106	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		107	size_t size)
		108	{
		109	dev_priv->mm.object_count++;
		110	dev_priv->mm.object_memory += size;
		111	}
		112
		113	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		114	size_t size)
		115	{
		116	dev_priv->mm.object_count--;
		117	dev_priv->mm.object_memory -= size;
		118	}
		119
		120	static int
3480	Serge	121	i915_gem_wait_for_error(struct i915_gpu_error *error)
2332	Serge	122	{
		123	int ret;
		124
3480	Serge	125	#define EXIT_COND (!i915_reset_in_progress(error))
		126	if (EXIT_COND)
2332	Serge	127	return 0;
3255	Serge	128	#if 0
3031	serge	129	/*
		130	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		131	* userspace. If it takes that long something really bad is going on and
		132	* we should simply try to bail out and fail as gracefully as possible.
		133	*/
3480	Serge	134	ret = wait_event_interruptible_timeout(error->reset_queue,
		135	EXIT_COND,
		136	10*HZ);
3031	serge	137	if (ret == 0) {
		138	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		139	return -EIO;
		140	} else if (ret < 0) {
2332	Serge	141	return ret;
3031	serge	142	}
2332	Serge	143
3255	Serge	144	#endif
3480	Serge	145	#undef EXIT_COND
3255	Serge	146
2332	Serge	147	return 0;
		148	}
		149
		150	int i915_mutex_lock_interruptible(struct drm_device *dev)
		151	{
3480	Serge	152	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	153	int ret;
		154
3480	Serge	155	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
2332	Serge	156	if (ret)
		157	return ret;
		158
3480	Serge	159	ret = mutex_lock_interruptible(&dev->struct_mutex);
		160	if (ret)
		161	return ret;
2332	Serge	162
		163	WARN_ON(i915_verify_lists(dev));
		164	return 0;
		165	}
		166
		167	static inline bool
		168	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		169	{
3031	serge	170	return obj->gtt_space && !obj->active;
2332	Serge	171	}
		172
		173
		174	#if 0
		175
		176	int
		177	i915_gem_init_ioctl(struct drm_device dev, void data,
		178	struct drm_file *file)
		179	{
3480	Serge	180	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	181	struct drm_i915_gem_init *args = data;
		182
3031	serge	183	if (drm_core_check_feature(dev, DRIVER_MODESET))
		184	return -ENODEV;
		185
2332	Serge	186	if (args->gtt_start >= args->gtt_end \|\|
		187	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		188	return -EINVAL;
		189
3031	serge	190	/* GEM with user mode setting was never supported on ilk and later. */
		191	if (INTEL_INFO(dev)->gen >= 5)
		192	return -ENODEV;
		193
2332	Serge	194	mutex_lock(&dev->struct_mutex);
3480	Serge	195	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
		196	args->gtt_end);
		197	dev_priv->gtt.mappable_end = args->gtt_end;
2332	Serge	198	mutex_unlock(&dev->struct_mutex);
		199
		200	return 0;
		201	}
2351	Serge	202	#endif
2332	Serge	203
		204	int
		205	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		206	struct drm_file *file)
		207	{
		208	struct drm_i915_private *dev_priv = dev->dev_private;
		209	struct drm_i915_gem_get_aperture *args = data;
		210	struct drm_i915_gem_object *obj;
		211	size_t pinned;
		212
		213	pinned = 0;
		214	mutex_lock(&dev->struct_mutex);
3031	serge	215	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		216	if (obj->pin_count)
2332	Serge	217	pinned += obj->gtt_space->size;
		218	mutex_unlock(&dev->struct_mutex);
		219
3480	Serge	220	args->aper_size = dev_priv->gtt.total;
2342	Serge	221	args->aper_available_size = args->aper_size - pinned;
2332	Serge	222
		223	return 0;
		224	}
		225
3480	Serge	226	void i915_gem_object_alloc(struct drm_device dev)
		227	{
		228	struct drm_i915_private *dev_priv = dev->dev_private;
		229	return kmalloc(sizeof(struct drm_i915_gem_object), 0);
		230	}
		231
		232	void i915_gem_object_free(struct drm_i915_gem_object *obj)
		233	{
		234	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		235	kfree(obj);
		236	}
		237
3031	serge	238	static int
		239	i915_gem_create(struct drm_file *file,
2332	Serge	240	struct drm_device *dev,
		241	uint64_t size,
		242	uint32_t *handle_p)
		243	{
		244	struct drm_i915_gem_object *obj;
		245	int ret;
		246	u32 handle;
		247
		248	size = roundup(size, PAGE_SIZE);
2342	Serge	249	if (size == 0)
		250	return -EINVAL;
2332	Serge	251
		252	/* Allocate the new object */
		253	obj = i915_gem_alloc_object(dev, size);
		254	if (obj == NULL)
		255	return -ENOMEM;
		256
		257	ret = drm_gem_handle_create(file, &obj->base, &handle);
		258	if (ret) {
		259	drm_gem_object_release(&obj->base);
		260	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		261	kfree(obj);
		262	return ret;
		263	}
		264
		265	/* drop reference from allocate - handle holds it now */
		266	drm_gem_object_unreference(&obj->base);
2351	Serge	267	trace_i915_gem_object_create(obj);
2332	Serge	268
		269	*handle_p = handle;
3290	Serge	270
3298	Serge	271	// printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle);
3290	Serge	272
2332	Serge	273	return 0;
		274	}
		275
		276	int
		277	i915_gem_dumb_create(struct drm_file *file,
		278	struct drm_device *dev,
		279	struct drm_mode_create_dumb *args)
		280	{
		281	/* have to work out size/pitch and return them */
		282	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		283	args->size = args->pitch * args->height;
		284	return i915_gem_create(file, dev,
		285	args->size, &args->handle);
		286	}
		287
		288	int i915_gem_dumb_destroy(struct drm_file *file,
		289	struct drm_device *dev,
		290	uint32_t handle)
		291	{
		292	return drm_gem_handle_delete(file, handle);
		293	}
		294
2326	Serge	295	/**
2332	Serge	296	* Creates a new mm object and returns a handle to it.
		297	*/
		298	int
		299	i915_gem_create_ioctl(struct drm_device dev, void data,
		300	struct drm_file *file)
		301	{
		302	struct drm_i915_gem_create *args = data;
3031	serge	303
2332	Serge	304	return i915_gem_create(file, dev,
		305	args->size, &args->handle);
		306	}
		307
		308
3260	Serge	309	#if 0
2332	Serge	310
3031	serge	311	static inline int
		312	__copy_to_user_swizzled(char __user *cpu_vaddr,
		313	const char *gpu_vaddr, int gpu_offset,
2332	Serge	314	int length)
		315	{
3031	serge	316	int ret, cpu_offset = 0;
2332	Serge	317
3031	serge	318	while (length > 0) {
		319	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		320	int this_length = min(cacheline_end - gpu_offset, length);
		321	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	322
3031	serge	323	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		324	gpu_vaddr + swizzled_gpu_offset,
		325	this_length);
		326	if (ret)
		327	return ret + length;
2332	Serge	328
3031	serge	329	cpu_offset += this_length;
		330	gpu_offset += this_length;
		331	length -= this_length;
		332	}
		333
		334	return 0;
2332	Serge	335	}
		336
3031	serge	337	static inline int
		338	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		339	const char __user *cpu_vaddr,
		340	int length)
2332	Serge	341	{
3031	serge	342	int ret, cpu_offset = 0;
2332	Serge	343
		344	while (length > 0) {
		345	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		346	int this_length = min(cacheline_end - gpu_offset, length);
		347	int swizzled_gpu_offset = gpu_offset ^ 64;
		348
3031	serge	349	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	350	cpu_vaddr + cpu_offset,
		351	this_length);
3031	serge	352	if (ret)
		353	return ret + length;
		354
2332	Serge	355	cpu_offset += this_length;
		356	gpu_offset += this_length;
		357	length -= this_length;
		358	}
		359
3031	serge	360	return 0;
2332	Serge	361	}
		362
3031	serge	363	/* Per-page copy function for the shmem pread fastpath.
		364	* Flushes invalid cachelines before reading the target if
		365	* needs_clflush is set. */
2332	Serge	366	static int
3031	serge	367	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		368	char __user *user_data,
		369	bool page_do_bit17_swizzling, bool needs_clflush)
		370	{
		371	char *vaddr;
		372	int ret;
		373
		374	if (unlikely(page_do_bit17_swizzling))
		375	return -EINVAL;
		376
		377	vaddr = kmap_atomic(page);
		378	if (needs_clflush)
		379	drm_clflush_virt_range(vaddr + shmem_page_offset,
		380	page_length);
		381	ret = __copy_to_user_inatomic(user_data,
		382	vaddr + shmem_page_offset,
		383	page_length);
		384	kunmap_atomic(vaddr);
		385
		386	return ret ? -EFAULT : 0;
		387	}
		388
		389	static void
		390	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		391	bool swizzled)
		392	{
		393	if (unlikely(swizzled)) {
		394	unsigned long start = (unsigned long) addr;
		395	unsigned long end = (unsigned long) addr + length;
		396
		397	/* For swizzling simply ensure that we always flush both
		398	* channels. Lame, but simple and it works. Swizzled
		399	* pwrite/pread is far from a hotpath - current userspace
		400	* doesn't use it at all. */
		401	start = round_down(start, 128);
		402	end = round_up(end, 128);
		403
		404	drm_clflush_virt_range((void *)start, end - start);
		405	} else {
		406	drm_clflush_virt_range(addr, length);
		407	}
		408
		409	}
		410
		411	/* Only difference to the fast-path function is that this can handle bit17
		412	* and uses non-atomic copy and kmap functions. */
		413	static int
		414	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		415	char __user *user_data,
		416	bool page_do_bit17_swizzling, bool needs_clflush)
		417	{
		418	char *vaddr;
		419	int ret;
		420
		421	vaddr = kmap(page);
		422	if (needs_clflush)
		423	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		424	page_length,
		425	page_do_bit17_swizzling);
		426
		427	if (page_do_bit17_swizzling)
		428	ret = __copy_to_user_swizzled(user_data,
		429	vaddr, shmem_page_offset,
		430	page_length);
		431	else
		432	ret = __copy_to_user(user_data,
		433	vaddr + shmem_page_offset,
		434	page_length);
		435	kunmap(page);
		436
		437	return ret ? - EFAULT : 0;
		438	}
		439
		440	static int
		441	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	442	struct drm_i915_gem_object *obj,
		443	struct drm_i915_gem_pread *args,
		444	struct drm_file *file)
		445	{
3031	serge	446	char __user *user_data;
2332	Serge	447	ssize_t remain;
		448	loff_t offset;
3031	serge	449	int shmem_page_offset, page_length, ret = 0;
		450	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		451	int prefaulted = 0;
		452	int needs_clflush = 0;
		453	struct scatterlist *sg;
		454	int i;
2332	Serge	455
		456	user_data = (char __user *) (uintptr_t) args->data_ptr;
		457	remain = args->size;
		458
3031	serge	459	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		460
		461	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		462	/* If we're not in the cpu read domain, set ourself into the gtt
		463	* read domain and manually flush cachelines (if required). This
		464	* optimizes for the case when the gpu will dirty the data
		465	* anyway again before the next pread happens. */
		466	if (obj->cache_level == I915_CACHE_NONE)
		467	needs_clflush = 1;
		468	if (obj->gtt_space) {
		469	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		470	if (ret)
		471	return ret;
		472	}
		473	}
		474
		475	ret = i915_gem_object_get_pages(obj);
		476	if (ret)
		477	return ret;
		478
		479	i915_gem_object_pin_pages(obj);
		480
2332	Serge	481	offset = args->offset;
		482
3031	serge	483	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	484	struct page *page;
		485
3031	serge	486	if (i < offset >> PAGE_SHIFT)
		487	continue;
		488
		489	if (remain <= 0)
		490	break;
		491
2332	Serge	492	/* Operation in this page
		493	*
3031	serge	494	* shmem_page_offset = offset within page in shmem file
2332	Serge	495	* page_length = bytes to copy for this page
		496	*/
3031	serge	497	shmem_page_offset = offset_in_page(offset);
2332	Serge	498	page_length = remain;
3031	serge	499	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		500	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	501
3031	serge	502	page = sg_page(sg);
		503	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		504	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	505
3031	serge	506	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		507	user_data, page_do_bit17_swizzling,
		508	needs_clflush);
		509	if (ret == 0)
		510	goto next_page;
2332	Serge	511
3031	serge	512	mutex_unlock(&dev->struct_mutex);
		513
		514	if (!prefaulted) {
		515	ret = fault_in_multipages_writeable(user_data, remain);
		516	/* Userspace is tricking us, but we've already clobbered
		517	* its pages with the prefault and promised to write the
		518	* data up to the first fault. Hence ignore any errors
		519	* and just continue. */
		520	(void)ret;
		521	prefaulted = 1;
		522	}
		523
		524	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		525	user_data, page_do_bit17_swizzling,
		526	needs_clflush);
		527
		528	mutex_lock(&dev->struct_mutex);
		529
		530	next_page:
2332	Serge	531	mark_page_accessed(page);
3031	serge	532
2332	Serge	533	if (ret)
3031	serge	534	goto out;
2332	Serge	535
		536	remain -= page_length;
		537	user_data += page_length;
		538	offset += page_length;
		539	}
		540
3031	serge	541	out:
		542	i915_gem_object_unpin_pages(obj);
		543
		544	return ret;
2332	Serge	545	}
		546
		547	/**
3031	serge	548	* Reads data from the object referenced by handle.
		549	*
		550	* On error, the contents of *data are undefined.
2332	Serge	551	*/
3031	serge	552	int
		553	i915_gem_pread_ioctl(struct drm_device dev, void data,
		554	struct drm_file *file)
		555	{
		556	struct drm_i915_gem_pread *args = data;
		557	struct drm_i915_gem_object *obj;
		558	int ret = 0;
		559
		560	if (args->size == 0)
		561	return 0;
		562
		563	if (!access_ok(VERIFY_WRITE,
		564	(char __user *)(uintptr_t)args->data_ptr,
		565	args->size))
		566	return -EFAULT;
		567
		568	ret = i915_mutex_lock_interruptible(dev);
		569	if (ret)
		570	return ret;
		571
		572	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		573	if (&obj->base == NULL) {
		574	ret = -ENOENT;
		575	goto unlock;
		576	}
		577
		578	/* Bounds check source. */
		579	if (args->offset > obj->base.size \|\|
		580	args->size > obj->base.size - args->offset) {
		581	ret = -EINVAL;
		582	goto out;
		583	}
		584
		585	/* prime objects have no backing filp to GEM pread/pwrite
		586	* pages from.
		587	*/
		588	if (!obj->base.filp) {
		589	ret = -EINVAL;
		590	goto out;
		591	}
		592
		593	trace_i915_gem_object_pread(obj, args->offset, args->size);
		594
		595	ret = i915_gem_shmem_pread(dev, obj, args, file);
		596
		597	out:
		598	drm_gem_object_unreference(&obj->base);
		599	unlock:
		600	mutex_unlock(&dev->struct_mutex);
		601	return ret;
		602	}
		603
		604	/* This is the fast write path which cannot handle
		605	* page faults in the source data
		606	*/
		607
		608	static inline int
		609	fast_user_write(struct io_mapping *mapping,
		610	loff_t page_base, int page_offset,
		611	char __user *user_data,
		612	int length)
		613	{
		614	void __iomem *vaddr_atomic;
		615	void *vaddr;
		616	unsigned long unwritten;
		617
		618	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		619	/* We can use the cpu mem copy function because this is X86. */
		620	vaddr = (void __force*)vaddr_atomic + page_offset;
		621	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		622	user_data, length);
		623	io_mapping_unmap_atomic(vaddr_atomic);
		624	return unwritten;
		625	}
3260	Serge	626	#endif
3031	serge	627
3260	Serge	628	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	629	/**
		630	* This is the fast pwrite path, where we copy the data directly from the
		631	* user into the GTT, uncached.
		632	*/
2332	Serge	633	static int
3031	serge	634	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		635	struct drm_i915_gem_object *obj,
		636	struct drm_i915_gem_pwrite *args,
		637	struct drm_file *file)
2332	Serge	638	{
3031	serge	639	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	640	ssize_t remain;
3031	serge	641	loff_t offset, page_base;
		642	char __user *user_data;
		643	int page_offset, page_length, ret;
3260	Serge	644	char *vaddr;
2332	Serge	645
3031	serge	646	ret = i915_gem_object_pin(obj, 0, true, true);
		647	if (ret)
		648	goto out;
		649
		650	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		651	if (ret)
		652	goto out_unpin;
		653
		654	ret = i915_gem_object_put_fence(obj);
		655	if (ret)
		656	goto out_unpin;
		657
3260	Serge	658	vaddr = AllocKernelSpace(4096);
		659	if(vaddr == NULL)
		660	{
		661	ret = -ENOSPC;
		662	goto out_unpin;
		663	};
		664
3031	serge	665	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	666	remain = args->size;
		667
3031	serge	668	offset = obj->gtt_offset + args->offset;
2332	Serge	669
3031	serge	670	while (remain > 0) {
		671	/* Operation in this page
		672	*
		673	* page_base = page offset within aperture
		674	* page_offset = offset within page
		675	* page_length = bytes to copy for this page
		676	*/
		677	page_base = offset & PAGE_MASK;
		678	page_offset = offset_in_page(offset);
		679	page_length = remain;
		680	if ((page_offset + remain) > PAGE_SIZE)
		681	page_length = PAGE_SIZE - page_offset;
2332	Serge	682
3260	Serge	683	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	684
3260	Serge	685	memcpy(vaddr+page_offset, user_data, page_length);
		686
3031	serge	687	remain -= page_length;
		688	user_data += page_length;
		689	offset += page_length;
2332	Serge	690	}
		691
3260	Serge	692	FreeKernelSpace(vaddr);
		693
3031	serge	694	out_unpin:
		695	i915_gem_object_unpin(obj);
		696	out:
3260	Serge	697	printf("% s ret = %d\n", __FUNCTION__, ret);
		698
3031	serge	699	return ret;
		700	}
		701
		702	/* Per-page copy function for the shmem pwrite fastpath.
		703	* Flushes invalid cachelines before writing to the target if
		704	* needs_clflush_before is set and flushes out any written cachelines after
		705	* writing if needs_clflush is set. */
		706	static int
		707	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		708	char __user *user_data,
		709	bool page_do_bit17_swizzling,
		710	bool needs_clflush_before,
		711	bool needs_clflush_after)
		712	{
		713	char *vaddr;
3260	Serge	714	int ret = 0;
3031	serge	715
		716	if (unlikely(page_do_bit17_swizzling))
		717	return -EINVAL;
		718
3260	Serge	719	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	720	if (needs_clflush_before)
		721	drm_clflush_virt_range(vaddr + shmem_page_offset,
		722	page_length);
3260	Serge	723	memcpy(vaddr + shmem_page_offset,
3031	serge	724	user_data,
		725	page_length);
		726	if (needs_clflush_after)
		727	drm_clflush_virt_range(vaddr + shmem_page_offset,
		728	page_length);
3260	Serge	729	FreeKernelSpace(vaddr);
3031	serge	730
		731	return ret ? -EFAULT : 0;
		732	}
3260	Serge	733	#if 0
3031	serge	734
		735	/* Only difference to the fast-path function is that this can handle bit17
		736	* and uses non-atomic copy and kmap functions. */
		737	static int
		738	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		739	char __user *user_data,
		740	bool page_do_bit17_swizzling,
		741	bool needs_clflush_before,
		742	bool needs_clflush_after)
		743	{
		744	char *vaddr;
		745	int ret;
		746
		747	vaddr = kmap(page);
		748	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		749	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		750	page_length,
		751	page_do_bit17_swizzling);
		752	if (page_do_bit17_swizzling)
		753	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		754	user_data,
		755	page_length);
		756	else
		757	ret = __copy_from_user(vaddr + shmem_page_offset,
		758	user_data,
		759	page_length);
		760	if (needs_clflush_after)
		761	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		762	page_length,
		763	page_do_bit17_swizzling);
		764	kunmap(page);
		765
		766	return ret ? -EFAULT : 0;
		767	}
3260	Serge	768	#endif
3031	serge	769
3260	Serge	770
3031	serge	771	static int
		772	i915_gem_shmem_pwrite(struct drm_device *dev,
		773	struct drm_i915_gem_object *obj,
		774	struct drm_i915_gem_pwrite *args,
		775	struct drm_file *file)
		776	{
		777	ssize_t remain;
		778	loff_t offset;
		779	char __user *user_data;
		780	int shmem_page_offset, page_length, ret = 0;
		781	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		782	int hit_slowpath = 0;
		783	int needs_clflush_after = 0;
		784	int needs_clflush_before = 0;
		785	int i;
		786	struct scatterlist *sg;
		787
		788	user_data = (char __user *) (uintptr_t) args->data_ptr;
		789	remain = args->size;
		790
		791	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		792
		793	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		794	/* If we're not in the cpu write domain, set ourself into the gtt
		795	* write domain and manually flush cachelines (if required). This
		796	* optimizes for the case when the gpu will use the data
		797	* right away and we therefore have to clflush anyway. */
		798	if (obj->cache_level == I915_CACHE_NONE)
		799	needs_clflush_after = 1;
		800	if (obj->gtt_space) {
		801	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		802	if (ret)
		803	return ret;
		804	}
		805	}
		806	/* Same trick applies for invalidate partially written cachelines before
		807	* writing. */
		808	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		809	&& obj->cache_level == I915_CACHE_NONE)
		810	needs_clflush_before = 1;
		811
		812	ret = i915_gem_object_get_pages(obj);
2332	Serge	813	if (ret)
3031	serge	814	return ret;
2332	Serge	815
3031	serge	816	i915_gem_object_pin_pages(obj);
2332	Serge	817
		818	offset = args->offset;
3031	serge	819	obj->dirty = 1;
2332	Serge	820
3031	serge	821	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	822	struct page *page;
3031	serge	823	int partial_cacheline_write;
2332	Serge	824
3031	serge	825	if (i < offset >> PAGE_SHIFT)
		826	continue;
		827
		828	if (remain <= 0)
		829	break;
		830
2332	Serge	831	/* Operation in this page
		832	*
		833	* shmem_page_offset = offset within page in shmem file
		834	* page_length = bytes to copy for this page
		835	*/
		836	shmem_page_offset = offset_in_page(offset);
		837
		838	page_length = remain;
		839	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		840	page_length = PAGE_SIZE - shmem_page_offset;
		841
3031	serge	842	/* If we don't overwrite a cacheline completely we need to be
		843	* careful to have up-to-date data by first clflushing. Don't
		844	* overcomplicate things and flush the entire patch. */
		845	partial_cacheline_write = needs_clflush_before &&
		846	((shmem_page_offset \| page_length)
3260	Serge	847	& (x86_clflush_size - 1));
2332	Serge	848
3031	serge	849	page = sg_page(sg);
		850	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		851	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	852
3031	serge	853	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		854	user_data, page_do_bit17_swizzling,
		855	partial_cacheline_write,
		856	needs_clflush_after);
		857	if (ret == 0)
		858	goto next_page;
		859
		860	hit_slowpath = 1;
		861	mutex_unlock(&dev->struct_mutex);
3260	Serge	862	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	863
3260	Serge	864	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		865	// user_data, page_do_bit17_swizzling,
		866	// partial_cacheline_write,
		867	// needs_clflush_after);
		868
3031	serge	869	mutex_lock(&dev->struct_mutex);
		870
		871	next_page:
2332	Serge	872
3031	serge	873	if (ret)
		874	goto out;
		875
2332	Serge	876	remain -= page_length;
3031	serge	877	user_data += page_length;
2332	Serge	878	offset += page_length;
		879	}
		880
		881	out:
3031	serge	882	i915_gem_object_unpin_pages(obj);
		883
		884	if (hit_slowpath) {
3480	Serge	885	/*
		886	* Fixup: Flush cpu caches in case we didn't flush the dirty
		887	* cachelines in-line while writing and the object moved
		888	* out of the cpu write domain while we've dropped the lock.
		889	*/
		890	if (!needs_clflush_after &&
		891	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3031	serge	892	i915_gem_clflush_object(obj);
3243	Serge	893	i915_gem_chipset_flush(dev);
3031	serge	894	}
2332	Serge	895	}
		896
3031	serge	897	if (needs_clflush_after)
3243	Serge	898	i915_gem_chipset_flush(dev);
3031	serge	899
2332	Serge	900	return ret;
		901	}
3031	serge	902
		903	/**
		904	* Writes data to the object referenced by handle.
		905	*
		906	* On error, the contents of the buffer that were to be modified are undefined.
		907	*/
		908	int
		909	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		910	struct drm_file *file)
		911	{
		912	struct drm_i915_gem_pwrite *args = data;
		913	struct drm_i915_gem_object *obj;
		914	int ret;
		915
3480	Serge	916	if(args->handle == -2)
		917	{
		918	printf("%s handle %d\n", __FUNCTION__, args->handle);
		919	return 0;
		920	}
		921
3031	serge	922	if (args->size == 0)
		923	return 0;
		924
		925	ret = i915_mutex_lock_interruptible(dev);
		926	if (ret)
		927	return ret;
		928
		929	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		930	if (&obj->base == NULL) {
		931	ret = -ENOENT;
		932	goto unlock;
		933	}
		934
		935	/* Bounds check destination. */
		936	if (args->offset > obj->base.size \|\|
		937	args->size > obj->base.size - args->offset) {
		938	ret = -EINVAL;
		939	goto out;
		940	}
		941
		942	/* prime objects have no backing filp to GEM pread/pwrite
		943	* pages from.
		944	*/
		945	if (!obj->base.filp) {
		946	ret = -EINVAL;
		947	goto out;
		948	}
		949
		950	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		951
		952	ret = -EFAULT;
		953	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		954	* it would end up going through the fenced access, and we'll get
		955	* different detiling behavior between reading and writing.
		956	* pread/pwrite currently are reading and writing from the CPU
		957	* perspective, requiring manual detiling by the client.
		958	*/
3260	Serge	959	// if (obj->phys_obj) {
		960	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		961	// goto out;
		962	// }
3031	serge	963
		964	if (obj->cache_level == I915_CACHE_NONE &&
		965	obj->tiling_mode == I915_TILING_NONE &&
		966	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		967	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		968	/* Note that the gtt paths might fail with non-page-backed user
		969	* pointers (e.g. gtt mappings when moving data between
		970	* textures). Fallback to the shmem path in that case. */
		971	}
		972
		973	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	974	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	975
		976	out:
		977	drm_gem_object_unreference(&obj->base);
		978	unlock:
		979	mutex_unlock(&dev->struct_mutex);
		980	return ret;
		981	}
		982
		983	int
3480	Serge	984	i915_gem_check_wedge(struct i915_gpu_error *error,
3031	serge	985	bool interruptible)
		986	{
3480	Serge	987	if (i915_reset_in_progress(error)) {
3031	serge	988	/* Non-interruptible callers can't handle -EAGAIN, hence return
		989	* -EIO unconditionally for these. */
		990	if (!interruptible)
		991	return -EIO;
2332	Serge	992
3480	Serge	993	/* Recovery complete, but the reset failed ... */
		994	if (i915_terminally_wedged(error))
3031	serge	995	return -EIO;
2332	Serge	996
3031	serge	997	return -EAGAIN;
		998	}
2332	Serge	999
3031	serge	1000	return 0;
		1001	}
2332	Serge	1002
3031	serge	1003	/*
		1004	* Compare seqno against outstanding lazy request. Emit a request if they are
		1005	* equal.
		1006	*/
		1007	static int
		1008	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1009	{
		1010	int ret;
2332	Serge	1011
3031	serge	1012	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1013
3031	serge	1014	ret = 0;
		1015	if (seqno == ring->outstanding_lazy_request)
		1016	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1017
3031	serge	1018	return ret;
		1019	}
2332	Serge	1020
3031	serge	1021	/**
		1022	* __wait_seqno - wait until execution of seqno has finished
		1023	* @ring: the ring expected to report seqno
		1024	* @seqno: duh!
3480	Serge	1025	* @reset_counter: reset sequence associated with the given seqno
3031	serge	1026	* @interruptible: do an interruptible wait (normally yes)
		1027	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1028	*
3480	Serge	1029	* Note: It is of utmost importance that the passed in seqno and reset_counter
		1030	* values have been read by the caller in an smp safe manner. Where read-side
		1031	* locks are involved, it is sufficient to read the reset_counter before
		1032	* unlocking the lock that protects the seqno. For lockless tricks, the
		1033	* reset_counter _must_ be read before, and an appropriate smp_rmb must be
		1034	* inserted.
		1035	*
3031	serge	1036	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1037	* errno with remaining time filled in timeout argument.
		1038	*/
		1039	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3480	Serge	1040	unsigned reset_counter,
3031	serge	1041	bool interruptible, struct timespec *timeout)
		1042	{
		1043	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1044	struct timespec before, now, wait_time={1,0};
		1045	unsigned long timeout_jiffies;
		1046	long end;
		1047	bool wait_forever = true;
		1048	int ret;
2332	Serge	1049
3031	serge	1050	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1051	return 0;
2332	Serge	1052
3031	serge	1053	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1054
3031	serge	1055	if (timeout != NULL) {
		1056	wait_time = *timeout;
		1057	wait_forever = false;
		1058	}
2332	Serge	1059
3266	Serge	1060	timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1061
3031	serge	1062	if (WARN_ON(!ring->irq_get(ring)))
		1063	return -ENODEV;
2332	Serge	1064
3031	serge	1065	/* Record current time in case interrupted by signal, or wedged * */
		1066	getrawmonotonic(&before);
2332	Serge	1067
3031	serge	1068	#define EXIT_COND \
		1069	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
3480	Serge	1070	i915_reset_in_progress(&dev_priv->gpu_error) \|\| \
		1071	reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3031	serge	1072	do {
3266	Serge	1073	if (interruptible)
		1074	end = wait_event_interruptible_timeout(ring->irq_queue,
		1075	EXIT_COND,
		1076	timeout_jiffies);
		1077	else
3031	serge	1078	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1079	timeout_jiffies);
2332	Serge	1080
3480	Serge	1081	/* We need to check whether any gpu reset happened in between
		1082	* the caller grabbing the seqno and now ... */
		1083	if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
		1084	end = -EAGAIN;
		1085
		1086	/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		1087	* gone. */
		1088	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1089	if (ret)
		1090	end = ret;
		1091	} while (end == 0 && wait_forever);
2332	Serge	1092
3031	serge	1093	getrawmonotonic(&now);
2332	Serge	1094
3031	serge	1095	ring->irq_put(ring);
		1096	trace_i915_gem_request_wait_end(ring, seqno);
		1097	#undef EXIT_COND
2332	Serge	1098
3031	serge	1099	if (timeout) {
		1100	// struct timespec sleep_time = timespec_sub(now, before);
		1101	// timeout = timespec_sub(timeout, sleep_time);
		1102	}
2332	Serge	1103
3031	serge	1104	switch (end) {
		1105	case -EIO:
		1106	case -EAGAIN: /* Wedged */
		1107	case -ERESTARTSYS: /* Signal */
		1108	return (int)end;
		1109	case 0: /* Timeout */
3266	Serge	1110	if (timeout)
		1111	set_normalized_timespec(timeout, 0, 0);
3031	serge	1112	return -ETIME;
		1113	default: /* Completed */
		1114	WARN_ON(end < 0); /* We're not aware of other errors */
		1115	return 0;
		1116	}
		1117	}
2332	Serge	1118
3031	serge	1119	/**
		1120	* Waits for a sequence number to be signaled, and cleans up the
		1121	* request and object lists appropriately for that event.
		1122	*/
		1123	int
		1124	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1125	{
		1126	struct drm_device *dev = ring->dev;
		1127	struct drm_i915_private *dev_priv = dev->dev_private;
		1128	bool interruptible = dev_priv->mm.interruptible;
		1129	int ret;
2332	Serge	1130
3031	serge	1131	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1132	BUG_ON(seqno == 0);
2332	Serge	1133
3480	Serge	1134	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
3031	serge	1135	if (ret)
		1136	return ret;
2332	Serge	1137
3031	serge	1138	ret = i915_gem_check_olr(ring, seqno);
		1139	if (ret)
		1140	return ret;
2332	Serge	1141
3480	Serge	1142	return __wait_seqno(ring, seqno,
		1143	atomic_read(&dev_priv->gpu_error.reset_counter),
		1144	interruptible, NULL);
3031	serge	1145	}
2332	Serge	1146
3031	serge	1147	/**
		1148	* Ensures that all rendering to the object has completed and the object is
		1149	* safe to unbind from the GTT or access from the CPU.
		1150	*/
		1151	static __must_check int
		1152	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1153	bool readonly)
		1154	{
		1155	struct intel_ring_buffer *ring = obj->ring;
		1156	u32 seqno;
		1157	int ret;
2332	Serge	1158
3031	serge	1159	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1160	if (seqno == 0)
		1161	return 0;
2332	Serge	1162
3031	serge	1163	ret = i915_wait_seqno(ring, seqno);
		1164	if (ret)
		1165	return ret;
2332	Serge	1166
3031	serge	1167	i915_gem_retire_requests_ring(ring);
2332	Serge	1168
3031	serge	1169	/* Manually manage the write flush as we may have not yet
		1170	* retired the buffer.
		1171	*/
		1172	if (obj->last_write_seqno &&
		1173	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1174	obj->last_write_seqno = 0;
		1175	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1176	}
2332	Serge	1177
3031	serge	1178	return 0;
		1179	}
2332	Serge	1180
3260	Serge	1181	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1182	* as the object state may change during this call.
		1183	*/
		1184	static __must_check int
		1185	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1186	bool readonly)
		1187	{
		1188	struct drm_device *dev = obj->base.dev;
		1189	struct drm_i915_private *dev_priv = dev->dev_private;
		1190	struct intel_ring_buffer *ring = obj->ring;
3480	Serge	1191	unsigned reset_counter;
3260	Serge	1192	u32 seqno;
		1193	int ret;
2332	Serge	1194
3260	Serge	1195	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1196	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1197
3260	Serge	1198	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1199	if (seqno == 0)
		1200	return 0;
2332	Serge	1201
3480	Serge	1202	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
3260	Serge	1203	if (ret)
		1204	return ret;
2332	Serge	1205
3260	Serge	1206	ret = i915_gem_check_olr(ring, seqno);
		1207	if (ret)
		1208	return ret;
2332	Serge	1209
3480	Serge	1210	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3260	Serge	1211	mutex_unlock(&dev->struct_mutex);
3480	Serge	1212	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3260	Serge	1213	mutex_lock(&dev->struct_mutex);
2332	Serge	1214
3260	Serge	1215	i915_gem_retire_requests_ring(ring);
2332	Serge	1216
3260	Serge	1217	/* Manually manage the write flush as we may have not yet
		1218	* retired the buffer.
		1219	*/
		1220	if (obj->last_write_seqno &&
		1221	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1222	obj->last_write_seqno = 0;
		1223	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1224	}
2332	Serge	1225
3260	Serge	1226	return ret;
		1227	}
2332	Serge	1228
3260	Serge	1229	/**
		1230	* Called when user space prepares to use an object with the CPU, either
		1231	* through the mmap ioctl's mapping or a GTT mapping.
		1232	*/
		1233	int
		1234	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1235	struct drm_file *file)
		1236	{
		1237	struct drm_i915_gem_set_domain *args = data;
		1238	struct drm_i915_gem_object *obj;
		1239	uint32_t read_domains = args->read_domains;
		1240	uint32_t write_domain = args->write_domain;
		1241	int ret;
2332	Serge	1242
3480	Serge	1243
		1244	if(args->handle == -2)
		1245	{
		1246	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1247	return 0;
		1248	}
		1249
3260	Serge	1250	/* Only handle setting domains to types used by the CPU. */
		1251	if (write_domain & I915_GEM_GPU_DOMAINS)
		1252	return -EINVAL;
2332	Serge	1253
3260	Serge	1254	if (read_domains & I915_GEM_GPU_DOMAINS)
		1255	return -EINVAL;
2332	Serge	1256
3260	Serge	1257	/* Having something in the write domain implies it's in the read
		1258	* domain, and only that read domain. Enforce that in the request.
		1259	*/
		1260	if (write_domain != 0 && read_domains != write_domain)
		1261	return -EINVAL;
2332	Serge	1262
3260	Serge	1263	ret = i915_mutex_lock_interruptible(dev);
		1264	if (ret)
		1265	return ret;
2332	Serge	1266
3260	Serge	1267	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1268	if (&obj->base == NULL) {
		1269	ret = -ENOENT;
		1270	goto unlock;
		1271	}
2332	Serge	1272
3260	Serge	1273	/* Try to flush the object off the GPU without holding the lock.
		1274	* We will repeat the flush holding the lock in the normal manner
		1275	* to catch cases where we are gazumped.
		1276	*/
		1277	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1278	if (ret)
		1279	goto unref;
2332	Serge	1280
3260	Serge	1281	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1282	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1283
3260	Serge	1284	/* Silently promote "you're not bound, there was nothing to do"
		1285	* to success, since the client was just asking us to
		1286	* make sure everything was done.
		1287	*/
		1288	if (ret == -EINVAL)
		1289	ret = 0;
		1290	} else {
		1291	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1292	}
2332	Serge	1293
3260	Serge	1294	unref:
		1295	drm_gem_object_unreference(&obj->base);
		1296	unlock:
		1297	mutex_unlock(&dev->struct_mutex);
		1298	return ret;
		1299	}
2332	Serge	1300
		1301
		1302
		1303
		1304
		1305
3260	Serge	1306	/**
		1307	* Maps the contents of an object, returning the address it is mapped
		1308	* into.
		1309	*
		1310	* While the mapping holds a reference on the contents of the object, it doesn't
		1311	* imply a ref on the object itself.
		1312	*/
		1313	int
		1314	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1315	struct drm_file *file)
		1316	{
		1317	struct drm_i915_gem_mmap *args = data;
		1318	struct drm_gem_object *obj;
		1319	unsigned long addr = 0;
2332	Serge	1320
3480	Serge	1321	if(args->handle == -2)
		1322	{
		1323	printf("%s handle %d\n", __FUNCTION__, args->handle);
		1324	return 0;
		1325	}
		1326
3260	Serge	1327	obj = drm_gem_object_lookup(dev, file, args->handle);
		1328	if (obj == NULL)
		1329	return -ENOENT;
		1330
3298	Serge	1331	//dbgprintf("%s offset %lld size %lld\n",
		1332	// __FUNCTION__, args->offset, args->size);
3260	Serge	1333	/* prime objects have no backing filp to GEM mmap
		1334	* pages from.
		1335	*/
		1336	if (!obj->filp) {
		1337	drm_gem_object_unreference_unlocked(obj);
		1338	return -EINVAL;
		1339	}
2332	Serge	1340
3263	Serge	1341	addr = vm_mmap(obj->filp, 0, args->size,
		1342	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1343	args->offset);
3260	Serge	1344	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1345	if (IS_ERR((void *)addr))
		1346	return addr;
2332	Serge	1347
3260	Serge	1348	args->addr_ptr = (uint64_t) addr;
2332	Serge	1349
3263	Serge	1350	return 0;
3260	Serge	1351	}
2332	Serge	1352
		1353
		1354
		1355
		1356
		1357
		1358
		1359
3031	serge	1360
		1361
		1362
		1363
		1364
		1365	/**
		1366	* i915_gem_release_mmap - remove physical page mappings
		1367	* @obj: obj in question
		1368	*
		1369	* Preserve the reservation of the mmapping with the DRM core code, but
		1370	* relinquish ownership of the pages back to the system.
		1371	*
		1372	* It is vital that we remove the page mapping if we have mapped a tiled
		1373	* object through the GTT and then lose the fence register due to
		1374	* resource pressure. Similarly if the object has been moved out of the
		1375	* aperture, than pages mapped into userspace must be revoked. Removing the
		1376	* mapping will then trigger a page fault on the next user access, allowing
		1377	* fixup by i915_gem_fault().
		1378	*/
		1379	void
		1380	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1381	{
		1382	if (!obj->fault_mappable)
		1383	return;
		1384
		1385	if (obj->base.dev->dev_mapping)
		1386	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1387	// (loff_t)obj->base.map_list.hash.key<
		1388	// obj->base.size, 1);
		1389
		1390	obj->fault_mappable = false;
		1391	}
		1392
3480	Serge	1393	uint32_t
2332	Serge	1394	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1395	{
		1396	uint32_t gtt_size;
		1397
		1398	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1399	tiling_mode == I915_TILING_NONE)
		1400	return size;
		1401
		1402	/* Previous chips need a power-of-two fence region when tiling */
		1403	if (INTEL_INFO(dev)->gen == 3)
		1404	gtt_size = 1024*1024;
		1405	else
		1406	gtt_size = 512*1024;
		1407
		1408	while (gtt_size < size)
		1409	gtt_size <<= 1;
		1410
		1411	return gtt_size;
		1412	}
		1413
		1414	/**
		1415	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1416	* @obj: object to check
		1417	*
		1418	* Return the required GTT alignment for an object, taking into account
		1419	* potential fence register mapping.
		1420	*/
3480	Serge	1421	uint32_t
		1422	i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
		1423	int tiling_mode, bool fenced)
2332	Serge	1424	{
		1425	/*
		1426	* Minimum alignment is 4k (GTT page size), but might be greater
		1427	* if a fence register is needed for the object.
		1428	*/
3480	Serge	1429	if (INTEL_INFO(dev)->gen >= 4 \|\| (!fenced && IS_G33(dev)) \|\|
2332	Serge	1430	tiling_mode == I915_TILING_NONE)
		1431	return 4096;
		1432
		1433	/*
		1434	* Previous chips need to be aligned to the size of the smallest
		1435	* fence register that can contain the object.
		1436	*/
		1437	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1438	}
		1439
		1440	/**
		1441	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1442	* unfenced object
		1443	* @dev: the device
		1444	* @size: size of the object
		1445	* @tiling_mode: tiling mode of the object
		1446	*
		1447	* Return the required GTT alignment for an object, only taking into account
		1448	* unfenced tiled surface requirements.
		1449	*/
		1450	uint32_t
		1451	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1452	uint32_t size,
		1453	int tiling_mode)
		1454	{
		1455	/*
		1456	* Minimum alignment is 4k (GTT page size) for sane hw.
		1457	*/
		1458	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1459	tiling_mode == I915_TILING_NONE)
		1460	return 4096;
		1461
		1462	/* Previous hardware however needs to be aligned to a power-of-two
		1463	* tile height. The simplest method for determining this is to reuse
		1464	* the power-of-tile object size.
		1465	*/
		1466	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1467	}
		1468
3480	Serge	1469	int
		1470	i915_gem_mmap_gtt(struct drm_file *file,
		1471	struct drm_device *dev,
		1472	uint32_t handle,
		1473	uint64_t *offset)
		1474	{
		1475	struct drm_i915_private *dev_priv = dev->dev_private;
		1476	struct drm_i915_gem_object *obj;
		1477	unsigned long pfn;
		1478	char mem, ptr;
		1479	int ret;
		1480
		1481	ret = i915_mutex_lock_interruptible(dev);
		1482	if (ret)
		1483	return ret;
		1484
		1485	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
		1486	if (&obj->base == NULL) {
		1487	ret = -ENOENT;
		1488	goto unlock;
		1489	}
		1490
		1491	if (obj->base.size > dev_priv->gtt.mappable_end) {
		1492	ret = -E2BIG;
		1493	goto out;
		1494	}
		1495
		1496	if (obj->madv != I915_MADV_WILLNEED) {
		1497	DRM_ERROR("Attempting to mmap a purgeable buffer\n");
		1498	ret = -EINVAL;
		1499	goto out;
		1500	}
		1501	/* Now bind it into the GTT if needed */
		1502	ret = i915_gem_object_pin(obj, 0, true, false);
		1503	if (ret)
		1504	goto out;
		1505
		1506	ret = i915_gem_object_set_to_gtt_domain(obj, 1);
		1507	if (ret)
		1508	goto unpin;
		1509
		1510	ret = i915_gem_object_get_fence(obj);
		1511	if (ret)
		1512	goto unpin;
		1513
		1514	obj->fault_mappable = true;
		1515
		1516	pfn = dev_priv->gtt.mappable_base + obj->gtt_offset;
		1517
		1518	/* Finally, remap it using the new GTT offset */
		1519
		1520	mem = UserAlloc(obj->base.size);
		1521	if(unlikely(mem == NULL))
		1522	{
		1523	ret = -ENOMEM;
		1524	goto unpin;
		1525	}
		1526
		1527	for(ptr = mem; ptr < mem + obj->base.size; ptr+= 4096, pfn+= 4096)
		1528	MapPage(ptr, pfn, PG_SHARED\|PG_UW);
		1529
		1530	unpin:
		1531	i915_gem_object_unpin(obj);
		1532
		1533
		1534	*offset = (u64)mem;
		1535
		1536	out:
		1537	drm_gem_object_unreference(&obj->base);
		1538	unlock:
		1539	mutex_unlock(&dev->struct_mutex);
		1540	return ret;
		1541	}
		1542
		1543	/**
		1544	* i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
		1545	* @dev: DRM device
		1546	* @data: GTT mapping ioctl data
		1547	* @file: GEM object info
		1548	*
		1549	* Simply returns the fake offset to userspace so it can mmap it.
		1550	* The mmap call will end up in drm_gem_mmap(), which will set things
		1551	* up so we can get faults in the handler above.
		1552	*
		1553	* The fault handler will take care of binding the object into the GTT
		1554	* (since it may have been evicted to make room for something), allocating
		1555	* a fence register, and mapping the appropriate aperture address into
		1556	* userspace.
		1557	*/
		1558	int
		1559	i915_gem_mmap_gtt_ioctl(struct drm_device dev, void data,
		1560	struct drm_file *file)
		1561	{
		1562	struct drm_i915_gem_mmap_gtt *args = data;
		1563
		1564	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
		1565	}
		1566
3031	serge	1567	/* Immediately discard the backing storage */
		1568	static void
		1569	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1570	{
		1571	// struct inode *inode;
2332	Serge	1572
3031	serge	1573	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1574
3263	Serge	1575	if (obj->base.filp == NULL)
		1576	return;
2332	Serge	1577
3031	serge	1578	/* Our goal here is to return as much of the memory as
		1579	* is possible back to the system as we are called from OOM.
		1580	* To do this we must instruct the shmfs to drop all of its
		1581	* backing pages, now.
		1582	*/
		1583	// inode = obj->base.filp->f_path.dentry->d_inode;
		1584	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1585
3031	serge	1586	obj->madv = __I915_MADV_PURGED;
		1587	}
2332	Serge	1588
3031	serge	1589	static inline int
		1590	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1591	{
		1592	return obj->madv == I915_MADV_DONTNEED;
		1593	}
2332	Serge	1594
3031	serge	1595	static void
		1596	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1597	{
3243	Serge	1598	int page_count = obj->base.size / PAGE_SIZE;
		1599	struct scatterlist *sg;
3031	serge	1600	int ret, i;
2332	Serge	1601
3031	serge	1602	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1603
3031	serge	1604	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1605	if (ret) {
		1606	/* In the event of a disaster, abandon all caches and
		1607	* hope for the best.
		1608	*/
		1609	WARN_ON(ret != -EIO);
		1610	i915_gem_clflush_object(obj);
		1611	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1612	}
2332	Serge	1613
3031	serge	1614	if (obj->madv == I915_MADV_DONTNEED)
		1615	obj->dirty = 0;
2332	Serge	1616
3243	Serge	1617	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1618	struct page *page = sg_page(sg);
2332	Serge	1619
3290	Serge	1620	page_cache_release(page);
3243	Serge	1621	}
		1622	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1623
3031	serge	1624	obj->dirty = 0;
3243	Serge	1625
		1626	sg_free_table(obj->pages);
		1627	kfree(obj->pages);
3031	serge	1628	}
2332	Serge	1629
3480	Serge	1630	int
3031	serge	1631	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1632	{
		1633	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1634
3243	Serge	1635	if (obj->pages == NULL)
3031	serge	1636	return 0;
2332	Serge	1637
3031	serge	1638	BUG_ON(obj->gtt_space);
		1639
		1640	if (obj->pages_pin_count)
		1641	return -EBUSY;
		1642
3243	Serge	1643	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1644	* array, hence protect them from being reaped by removing them from gtt
		1645	* lists early. */
		1646	list_del(&obj->gtt_list);
		1647
3031	serge	1648	ops->put_pages(obj);
3243	Serge	1649	obj->pages = NULL;
3031	serge	1650
		1651	if (i915_gem_object_is_purgeable(obj))
		1652	i915_gem_object_truncate(obj);
		1653
		1654	return 0;
		1655	}
		1656
		1657
		1658
		1659
		1660
		1661
		1662
		1663
2332	Serge	1664	static int
3031	serge	1665	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1666	{
3260	Serge	1667	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1668	int page_count, i;
3260	Serge	1669	struct address_space *mapping;
3243	Serge	1670	struct sg_table *st;
		1671	struct scatterlist *sg;
		1672	struct page *page;
		1673	gfp_t gfp;
2332	Serge	1674
3243	Serge	1675	/* Assert that the object is not currently in any GPU domain. As it
		1676	* wasn't in the GTT, there shouldn't be any way it could have been in
		1677	* a GPU cache
2332	Serge	1678	*/
3243	Serge	1679	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1680	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1681
		1682	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1683	if (st == NULL)
		1684	return -ENOMEM;
		1685
2332	Serge	1686	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1687	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1688	sg_free_table(st);
		1689	kfree(st);
2332	Serge	1690	return -ENOMEM;
3243	Serge	1691	}
2332	Serge	1692
3243	Serge	1693	/* Get the list of pages out of our struct file. They'll be pinned
		1694	* at this point until we release them.
		1695	*
		1696	* Fail silently without starting the shrinker
		1697	*/
		1698	for_each_sg(st->sgl, sg, page_count, i) {
3260	Serge	1699	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1700	if (IS_ERR(page)) {
		1701	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1702	goto err_pages;
		1703
3260	Serge	1704	}
3243	Serge	1705	sg_set_page(sg, page, PAGE_SIZE, 0);
		1706	}
3031	serge	1707
3243	Serge	1708	obj->pages = st;
3031	serge	1709
3298	Serge	1710	// DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1711
		1712	return 0;
		1713
		1714	err_pages:
3243	Serge	1715	for_each_sg(st->sgl, sg, i, page_count)
		1716	page_cache_release(sg_page(sg));
		1717	sg_free_table(st);
		1718	kfree(st);
		1719	return PTR_ERR(page);
2332	Serge	1720	}
		1721
3031	serge	1722	/* Ensure that the associated pages are gathered from the backing storage
		1723	* and pinned into our object. i915_gem_object_get_pages() may be called
		1724	* multiple times before they are released by a single call to
		1725	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1726	* either as a result of memory pressure (reaping pages under the shrinker)
		1727	* or as the object is itself released.
		1728	*/
		1729	int
		1730	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1731	{
3031	serge	1732	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1733	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1734	int ret;
2332	Serge	1735
3243	Serge	1736	if (obj->pages)
3031	serge	1737	return 0;
2332	Serge	1738
3031	serge	1739	BUG_ON(obj->pages_pin_count);
2332	Serge	1740
3031	serge	1741	ret = ops->get_pages(obj);
		1742	if (ret)
		1743	return ret;
2344	Serge	1744
3031	serge	1745	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1746	return 0;
2332	Serge	1747	}
		1748
		1749	void
		1750	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1751	struct intel_ring_buffer *ring)
2332	Serge	1752	{
		1753	struct drm_device *dev = obj->base.dev;
		1754	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1755	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1756
		1757	BUG_ON(ring == NULL);
		1758	obj->ring = ring;
		1759
		1760	/* Add a reference if we're newly entering the active list. */
		1761	if (!obj->active) {
2344	Serge	1762	drm_gem_object_reference(&obj->base);
2332	Serge	1763	obj->active = 1;
		1764	}
		1765
		1766	/* Move from whatever list we were on to the tail of execution. */
		1767	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1768	list_move_tail(&obj->ring_list, &ring->active_list);
		1769
3031	serge	1770	obj->last_read_seqno = seqno;
		1771
2332	Serge	1772	if (obj->fenced_gpu_access) {
3031	serge	1773	obj->last_fenced_seqno = seqno;
		1774
		1775	/* Bump MRU to take account of the delayed flush */
		1776	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1777	struct drm_i915_fence_reg *reg;
		1778
		1779	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1780	list_move_tail(®->lru_list,
		1781	&dev_priv->mm.fence_list);
		1782	}
2332	Serge	1783	}
		1784	}
		1785
2344	Serge	1786	static void
3031	serge	1787	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1788	{
		1789	struct drm_device *dev = obj->base.dev;
3031	serge	1790	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1791
3031	serge	1792	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1793	BUG_ON(!obj->active);
2332	Serge	1794
2352	Serge	1795	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1796
3031	serge	1797	list_del_init(&obj->ring_list);
2352	Serge	1798	obj->ring = NULL;
2344	Serge	1799
3031	serge	1800	obj->last_read_seqno = 0;
		1801	obj->last_write_seqno = 0;
		1802	obj->base.write_domain = 0;
		1803
		1804	obj->last_fenced_seqno = 0;
2352	Serge	1805	obj->fenced_gpu_access = false;
2344	Serge	1806
2352	Serge	1807	obj->active = 0;
		1808	drm_gem_object_unreference(&obj->base);
		1809
		1810	WARN_ON(i915_verify_lists(dev));
		1811	}
		1812
3243	Serge	1813	static int
3480	Serge	1814	i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2344	Serge	1815	{
3243	Serge	1816	struct drm_i915_private *dev_priv = dev->dev_private;
		1817	struct intel_ring_buffer *ring;
		1818	int ret, i, j;
2344	Serge	1819
3480	Serge	1820	/* Carefully retire all requests without writing to the rings */
3243	Serge	1821	for_each_ring(ring, dev_priv, i) {
3480	Serge	1822	ret = intel_ring_idle(ring);
3243	Serge	1823	if (ret)
		1824	return ret;
3480	Serge	1825	}
		1826	i915_gem_retire_requests(dev);
3243	Serge	1827
3480	Serge	1828	/* Finally reset hw state */
3243	Serge	1829	for_each_ring(ring, dev_priv, i) {
3480	Serge	1830	intel_ring_init_seqno(ring, seqno);
		1831
3243	Serge	1832	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1833	ring->sync_seqno[j] = 0;
		1834	}
		1835
		1836	return 0;
2344	Serge	1837	}
		1838
3480	Serge	1839	int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
		1840	{
		1841	struct drm_i915_private *dev_priv = dev->dev_private;
		1842	int ret;
		1843
		1844	if (seqno == 0)
		1845	return -EINVAL;
		1846
		1847	/* HWS page needs to be set less than what we
		1848	* will inject to ring
		1849	*/
		1850	ret = i915_gem_init_seqno(dev, seqno - 1);
		1851	if (ret)
		1852	return ret;
		1853
		1854	/* Carefully set the last_seqno value so that wrap
		1855	* detection still works
		1856	*/
		1857	dev_priv->next_seqno = seqno;
		1858	dev_priv->last_seqno = seqno - 1;
		1859	if (dev_priv->last_seqno == 0)
		1860	dev_priv->last_seqno--;
		1861
		1862	return 0;
		1863	}
		1864
3243	Serge	1865	int
		1866	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1867	{
3243	Serge	1868	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1869
3243	Serge	1870	/* reserve 0 for non-seqno */
		1871	if (dev_priv->next_seqno == 0) {
3480	Serge	1872	int ret = i915_gem_init_seqno(dev, 0);
3243	Serge	1873	if (ret)
		1874	return ret;
		1875
		1876	dev_priv->next_seqno = 1;
		1877	}
		1878
3480	Serge	1879	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3243	Serge	1880	return 0;
2332	Serge	1881	}
		1882
2352	Serge	1883	int
		1884	i915_add_request(struct intel_ring_buffer *ring,
		1885	struct drm_file *file,
3031	serge	1886	u32 *out_seqno)
2352	Serge	1887	{
		1888	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1889	struct drm_i915_gem_request *request;
		1890	u32 request_ring_position;
2352	Serge	1891	int was_empty;
		1892	int ret;
2332	Serge	1893
3031	serge	1894	/*
		1895	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1896	* after having emitted the batchbuffer command. Hence we need to fix
		1897	* things up similar to emitting the lazy request. The difference here
		1898	* is that the flush _must_ happen before the next request, no matter
		1899	* what.
		1900	*/
		1901	ret = intel_ring_flush_all_caches(ring);
		1902	if (ret)
		1903	return ret;
2332	Serge	1904
3031	serge	1905	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1906	if (request == NULL)
		1907	return -ENOMEM;
		1908
		1909
		1910	/* Record the position of the start of the request so that
		1911	* should we detect the updated seqno part-way through the
		1912	* GPU processing the request, we never over-estimate the
		1913	* position of the head.
		1914	*/
		1915	request_ring_position = intel_ring_get_tail(ring);
		1916
3243	Serge	1917	ret = ring->add_request(ring);
3031	serge	1918	if (ret) {
		1919	kfree(request);
2352	Serge	1920	return ret;
3031	serge	1921	}
2332	Serge	1922
3243	Serge	1923	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1924	request->ring = ring;
3031	serge	1925	request->tail = request_ring_position;
		1926	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1927	was_empty = list_empty(&ring->request_list);
		1928	list_add_tail(&request->list, &ring->request_list);
3031	serge	1929	request->file_priv = NULL;
2332	Serge	1930
3263	Serge	1931	if (file) {
		1932	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1933
3263	Serge	1934	spin_lock(&file_priv->mm.lock);
		1935	request->file_priv = file_priv;
		1936	list_add_tail(&request->client_list,
		1937	&file_priv->mm.request_list);
		1938	spin_unlock(&file_priv->mm.lock);
		1939	}
		1940
		1941	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1942	ring->outstanding_lazy_request = 0;
2332	Serge	1943
2360	Serge	1944	if (!dev_priv->mm.suspended) {
		1945	if (i915_enable_hangcheck) {
2352	Serge	1946	// mod_timer(&dev_priv->hangcheck_timer,
		1947	// jiffies +
		1948	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1949	}
3031	serge	1950	if (was_empty) {
2360	Serge	1951	queue_delayed_work(dev_priv->wq,
		1952	&dev_priv->mm.retire_work, HZ);
3031	serge	1953	intel_mark_busy(dev_priv->dev);
		1954	}
2360	Serge	1955	}
3031	serge	1956
		1957	if (out_seqno)
3243	Serge	1958	*out_seqno = request->seqno;
2352	Serge	1959	return 0;
		1960	}
2332	Serge	1961
3263	Serge	1962	static inline void
		1963	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1964	{
		1965	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1966
3263	Serge	1967	if (!file_priv)
		1968	return;
2332	Serge	1969
3263	Serge	1970	spin_lock(&file_priv->mm.lock);
		1971	if (request->file_priv) {
		1972	list_del(&request->client_list);
		1973	request->file_priv = NULL;
		1974	}
		1975	spin_unlock(&file_priv->mm.lock);
		1976	}
2332	Serge	1977
3031	serge	1978	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1979	struct intel_ring_buffer *ring)
		1980	{
		1981	while (!list_empty(&ring->request_list)) {
		1982	struct drm_i915_gem_request *request;
2332	Serge	1983
3031	serge	1984	request = list_first_entry(&ring->request_list,
		1985	struct drm_i915_gem_request,
		1986	list);
2332	Serge	1987
3031	serge	1988	list_del(&request->list);
3263	Serge	1989	i915_gem_request_remove_from_client(request);
3031	serge	1990	kfree(request);
		1991	}
2332	Serge	1992
3031	serge	1993	while (!list_empty(&ring->active_list)) {
		1994	struct drm_i915_gem_object *obj;
2332	Serge	1995
3031	serge	1996	obj = list_first_entry(&ring->active_list,
		1997	struct drm_i915_gem_object,
		1998	ring_list);
2332	Serge	1999
3031	serge	2000	i915_gem_object_move_to_inactive(obj);
		2001	}
		2002	}
2332	Serge	2003
3031	serge	2004	static void i915_gem_reset_fences(struct drm_device *dev)
		2005	{
		2006	struct drm_i915_private *dev_priv = dev->dev_private;
		2007	int i;
2332	Serge	2008
3031	serge	2009	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2010	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	2011
3031	serge	2012	i915_gem_write_fence(dev, i, NULL);
2360	Serge	2013
3031	serge	2014	if (reg->obj)
		2015	i915_gem_object_fence_lost(reg->obj);
2360	Serge	2016
3031	serge	2017	reg->pin_count = 0;
		2018	reg->obj = NULL;
		2019	INIT_LIST_HEAD(®->lru_list);
		2020	}
2360	Serge	2021
3031	serge	2022	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		2023	}
2360	Serge	2024
3031	serge	2025	void i915_gem_reset(struct drm_device *dev)
		2026	{
		2027	struct drm_i915_private *dev_priv = dev->dev_private;
		2028	struct drm_i915_gem_object *obj;
		2029	struct intel_ring_buffer *ring;
		2030	int i;
2360	Serge	2031
3031	serge	2032	for_each_ring(ring, dev_priv, i)
		2033	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	2034
3031	serge	2035	/* Move everything out of the GPU domains to ensure we do any
		2036	* necessary invalidation upon reuse.
		2037	*/
		2038	list_for_each_entry(obj,
		2039	&dev_priv->mm.inactive_list,
		2040	mm_list)
		2041	{
		2042	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		2043	}
2360	Serge	2044
3031	serge	2045	/* The fence registers are invalidated so clear them out */
		2046	i915_gem_reset_fences(dev);
		2047	}
2360	Serge	2048
2352	Serge	2049	/**
		2050	* This function clears the request list as sequence numbers are passed.
		2051	*/
3031	serge	2052	void
2352	Serge	2053	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		2054	{
		2055	uint32_t seqno;
2332	Serge	2056
2352	Serge	2057	if (list_empty(&ring->request_list))
		2058	return;
2332	Serge	2059
2352	Serge	2060	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	2061
3031	serge	2062	seqno = ring->get_seqno(ring, true);
2332	Serge	2063
2352	Serge	2064	while (!list_empty(&ring->request_list)) {
		2065	struct drm_i915_gem_request *request;
2332	Serge	2066
2352	Serge	2067	request = list_first_entry(&ring->request_list,
		2068	struct drm_i915_gem_request,
		2069	list);
2332	Serge	2070
2352	Serge	2071	if (!i915_seqno_passed(seqno, request->seqno))
		2072	break;
2332	Serge	2073
2352	Serge	2074	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	2075	/* We know the GPU must have read the request to have
		2076	* sent us the seqno + interrupt, so use the position
		2077	* of tail of the request to update the last known position
		2078	* of the GPU head.
		2079	*/
		2080	ring->last_retired_head = request->tail;
2332	Serge	2081
2352	Serge	2082	list_del(&request->list);
3263	Serge	2083	i915_gem_request_remove_from_client(request);
2352	Serge	2084	kfree(request);
		2085	}
2332	Serge	2086
2352	Serge	2087	/* Move any buffers on the active list that are no longer referenced
		2088	* by the ringbuffer to the flushing/inactive lists as appropriate.
		2089	*/
		2090	while (!list_empty(&ring->active_list)) {
		2091	struct drm_i915_gem_object *obj;
2332	Serge	2092
2352	Serge	2093	obj = list_first_entry(&ring->active_list,
		2094	struct drm_i915_gem_object,
		2095	ring_list);
2332	Serge	2096
3031	serge	2097	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	2098	break;
2332	Serge	2099
2352	Serge	2100	i915_gem_object_move_to_inactive(obj);
		2101	}
2332	Serge	2102
2352	Serge	2103	if (unlikely(ring->trace_irq_seqno &&
		2104	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		2105	ring->irq_put(ring);
		2106	ring->trace_irq_seqno = 0;
		2107	}
2332	Serge	2108
2352	Serge	2109	WARN_ON(i915_verify_lists(ring->dev));
		2110	}
2332	Serge	2111
2352	Serge	2112	void
		2113	i915_gem_retire_requests(struct drm_device *dev)
		2114	{
		2115	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2116	struct intel_ring_buffer *ring;
2352	Serge	2117	int i;
2332	Serge	2118
3031	serge	2119	for_each_ring(ring, dev_priv, i)
		2120	i915_gem_retire_requests_ring(ring);
2352	Serge	2121	}
		2122
2360	Serge	2123	static void
		2124	i915_gem_retire_work_handler(struct work_struct *work)
		2125	{
		2126	drm_i915_private_t *dev_priv;
		2127	struct drm_device *dev;
3031	serge	2128	struct intel_ring_buffer *ring;
2360	Serge	2129	bool idle;
		2130	int i;
2352	Serge	2131
2360	Serge	2132	dev_priv = container_of(work, drm_i915_private_t,
		2133	mm.retire_work.work);
		2134	dev = dev_priv->dev;
2352	Serge	2135
2360	Serge	2136	/* Come back later if the device is busy... */
		2137	if (!mutex_trylock(&dev->struct_mutex)) {
		2138	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	2139	return;
2360	Serge	2140	}
2352	Serge	2141
2360	Serge	2142	i915_gem_retire_requests(dev);
2352	Serge	2143
2360	Serge	2144	/* Send a periodic flush down the ring so we don't hold onto GEM
		2145	* objects indefinitely.
		2146	*/
		2147	idle = true;
3031	serge	2148	for_each_ring(ring, dev_priv, i) {
		2149	if (ring->gpu_caches_dirty)
		2150	i915_add_request(ring, NULL, NULL);
2352	Serge	2151
2360	Serge	2152	idle &= list_empty(&ring->request_list);
		2153	}
2352	Serge	2154
2360	Serge	2155	if (!dev_priv->mm.suspended && !idle)
		2156	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	2157	if (idle)
		2158	intel_mark_idle(dev);
2360	Serge	2159
		2160	mutex_unlock(&dev->struct_mutex);
		2161	}
		2162
2344	Serge	2163	/**
3031	serge	2164	* Ensures that an object will eventually get non-busy by flushing any required
		2165	* write domains, emitting any outstanding lazy request and retiring and
		2166	* completed requests.
2352	Serge	2167	*/
3031	serge	2168	static int
		2169	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2170	{
3031	serge	2171	int ret;
2352	Serge	2172
3031	serge	2173	if (obj->active) {
		2174	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2175	if (ret)
		2176	return ret;
2352	Serge	2177
3031	serge	2178	i915_gem_retire_requests_ring(obj->ring);
		2179	}
2352	Serge	2180
3031	serge	2181	return 0;
		2182	}
2352	Serge	2183
3243	Serge	2184	/**
		2185	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2186	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2187	*
		2188	* Returns 0 if successful, else an error is returned with the remaining time in
		2189	* the timeout parameter.
		2190	* -ETIME: object is still busy after timeout
		2191	* -ERESTARTSYS: signal interrupted the wait
		2192	* -ENONENT: object doesn't exist
		2193	* Also possible, but rare:
		2194	* -EAGAIN: GPU wedged
		2195	* -ENOMEM: damn
		2196	* -ENODEV: Internal IRQ fail
		2197	* -E?: The add request failed
		2198	*
		2199	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2200	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2201	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2202	* without holding struct_mutex the object may become re-busied before this
		2203	* function completes. A similar but shorter * race condition exists in the busy
		2204	* ioctl
		2205	*/
2352	Serge	2206
		2207
		2208
		2209
		2210
		2211
		2212
		2213
		2214
3243	Serge	2215
		2216
		2217
2352	Serge	2218	/**
3031	serge	2219	* i915_gem_object_sync - sync an object to a ring.
		2220	*
		2221	* @obj: object which may be in use on another ring.
		2222	* @to: ring we wish to use the object on. May be NULL.
		2223	*
		2224	* This code is meant to abstract object synchronization with the GPU.
		2225	* Calling with NULL implies synchronizing the object with the CPU
		2226	* rather than a particular GPU ring.
		2227	*
		2228	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2229	*/
		2230	int
3031	serge	2231	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2232	struct intel_ring_buffer *to)
2344	Serge	2233	{
3031	serge	2234	struct intel_ring_buffer *from = obj->ring;
		2235	u32 seqno;
		2236	int ret, idx;
2332	Serge	2237
3031	serge	2238	if (from == NULL \|\| to == from)
		2239	return 0;
2332	Serge	2240
3031	serge	2241	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2242	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2243
3031	serge	2244	idx = intel_ring_sync_index(from, to);
		2245
		2246	seqno = obj->last_read_seqno;
		2247	if (seqno <= from->sync_seqno[idx])
		2248	return 0;
		2249
		2250	ret = i915_gem_check_olr(obj->ring, seqno);
		2251	if (ret)
		2252	return ret;
		2253
		2254	ret = to->sync_to(to, from, seqno);
		2255	if (!ret)
3243	Serge	2256	/* We use last_read_seqno because sync_to()
		2257	* might have just caused seqno wrap under
		2258	* the radar.
		2259	*/
		2260	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2261
		2262	return ret;
2344	Serge	2263	}
2332	Serge	2264
2344	Serge	2265	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2266	{
		2267	u32 old_write_domain, old_read_domains;
2332	Serge	2268
2344	Serge	2269	/* Force a pagefault for domain tracking on next user access */
		2270	// i915_gem_release_mmap(obj);
2332	Serge	2271
2344	Serge	2272	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2273	return;
2332	Serge	2274
3480	Serge	2275	/* Wait for any direct GTT access to complete */
		2276	mb();
		2277
2344	Serge	2278	old_read_domains = obj->base.read_domains;
		2279	old_write_domain = obj->base.write_domain;
2351	Serge	2280
2344	Serge	2281	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2282	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2283
2351	Serge	2284	trace_i915_gem_object_change_domain(obj,
		2285	old_read_domains,
		2286	old_write_domain);
2344	Serge	2287	}
2332	Serge	2288
2344	Serge	2289	/**
		2290	* Unbinds an object from the GTT aperture.
		2291	*/
		2292	int
		2293	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2294	{
3031	serge	2295	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
3480	Serge	2296	int ret;
2332	Serge	2297
3263	Serge	2298	if(obj == get_fb_obj())
		2299	return 0;
		2300
2344	Serge	2301	if (obj->gtt_space == NULL)
		2302	return 0;
2332	Serge	2303
3031	serge	2304	if (obj->pin_count)
		2305	return -EBUSY;
2332	Serge	2306
3243	Serge	2307	BUG_ON(obj->pages == NULL);
3031	serge	2308
2344	Serge	2309	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2310	if (ret)
2344	Serge	2311	return ret;
		2312	/* Continue on if we fail due to EIO, the GPU is hung so we
		2313	* should be safe and we need to cleanup or else we might
		2314	* cause memory corruption through use-after-free.
		2315	*/
2332	Serge	2316
2344	Serge	2317	i915_gem_object_finish_gtt(obj);
2332	Serge	2318
2344	Serge	2319	/* release the fence reg _after_ flushing */
		2320	ret = i915_gem_object_put_fence(obj);
3031	serge	2321	if (ret)
2344	Serge	2322	return ret;
2332	Serge	2323
2351	Serge	2324	trace_i915_gem_object_unbind(obj);
2332	Serge	2325
3031	serge	2326	if (obj->has_global_gtt_mapping)
3243	Serge	2327	i915_gem_gtt_unbind_object(obj);
3031	serge	2328	if (obj->has_aliasing_ppgtt_mapping) {
		2329	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2330	obj->has_aliasing_ppgtt_mapping = 0;
		2331	}
		2332	i915_gem_gtt_finish_object(obj);
2332	Serge	2333
3031	serge	2334	list_del(&obj->mm_list);
		2335	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2336	/* Avoid an unnecessary call to unbind on rebind. */
		2337	obj->map_and_fenceable = true;
2332	Serge	2338
2344	Serge	2339	drm_mm_put_block(obj->gtt_space);
		2340	obj->gtt_space = NULL;
		2341	obj->gtt_offset = 0;
2332	Serge	2342
2344	Serge	2343	return 0;
		2344	}
2332	Serge	2345
3031	serge	2346	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2347	{
		2348	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2349	struct intel_ring_buffer *ring;
2344	Serge	2350	int ret, i;
2332	Serge	2351
2344	Serge	2352	/* Flush everything onto the inactive list. */
3031	serge	2353	for_each_ring(ring, dev_priv, i) {
		2354	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2355	if (ret)
		2356	return ret;
3031	serge	2357
3243	Serge	2358	ret = intel_ring_idle(ring);
3031	serge	2359	if (ret)
		2360	return ret;
2344	Serge	2361	}
2332	Serge	2362
2344	Serge	2363	return 0;
		2364	}
2332	Serge	2365
3480	Serge	2366	static void i965_write_fence_reg(struct drm_device *dev, int reg,
3031	serge	2367	struct drm_i915_gem_object *obj)
		2368	{
		2369	drm_i915_private_t *dev_priv = dev->dev_private;
3480	Serge	2370	int fence_reg;
		2371	int fence_pitch_shift;
3031	serge	2372	uint64_t val;
2332	Serge	2373
3480	Serge	2374	if (INTEL_INFO(dev)->gen >= 6) {
		2375	fence_reg = FENCE_REG_SANDYBRIDGE_0;
		2376	fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
		2377	} else {
		2378	fence_reg = FENCE_REG_965_0;
		2379	fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
		2380	}
2332	Serge	2381
3031	serge	2382	if (obj) {
		2383	u32 size = obj->gtt_space->size;
2332	Serge	2384
3031	serge	2385	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2386	0xfffff000) << 32;
		2387	val \|= obj->gtt_offset & 0xfffff000;
3480	Serge	2388	val \|= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
3031	serge	2389	if (obj->tiling_mode == I915_TILING_Y)
		2390	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2391	val \|= I965_FENCE_REG_VALID;
		2392	} else
		2393	val = 0;
2332	Serge	2394
3480	Serge	2395	fence_reg += reg * 8;
		2396	I915_WRITE64(fence_reg, val);
		2397	POSTING_READ(fence_reg);
3031	serge	2398	}
2332	Serge	2399
3031	serge	2400	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2401	struct drm_i915_gem_object *obj)
		2402	{
		2403	drm_i915_private_t *dev_priv = dev->dev_private;
		2404	u32 val;
2332	Serge	2405
3031	serge	2406	if (obj) {
		2407	u32 size = obj->gtt_space->size;
		2408	int pitch_val;
		2409	int tile_width;
2332	Serge	2410
3031	serge	2411	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2412	(size & -size) != size \|\|
		2413	(obj->gtt_offset & (size - 1)),
		2414	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2415	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2416
3031	serge	2417	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2418	tile_width = 128;
		2419	else
		2420	tile_width = 512;
2332	Serge	2421
3031	serge	2422	/* Note: pitch better be a power of two tile widths */
		2423	pitch_val = obj->stride / tile_width;
		2424	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2425
3031	serge	2426	val = obj->gtt_offset;
		2427	if (obj->tiling_mode == I915_TILING_Y)
		2428	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2429	val \|= I915_FENCE_SIZE_BITS(size);
		2430	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2431	val \|= I830_FENCE_REG_VALID;
		2432	} else
		2433	val = 0;
2332	Serge	2434
3031	serge	2435	if (reg < 8)
		2436	reg = FENCE_REG_830_0 + reg * 4;
		2437	else
		2438	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2439
3031	serge	2440	I915_WRITE(reg, val);
		2441	POSTING_READ(reg);
		2442	}
2332	Serge	2443
3031	serge	2444	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2445	struct drm_i915_gem_object *obj)
		2446	{
		2447	drm_i915_private_t *dev_priv = dev->dev_private;
		2448	uint32_t val;
2344	Serge	2449
3031	serge	2450	if (obj) {
		2451	u32 size = obj->gtt_space->size;
		2452	uint32_t pitch_val;
2344	Serge	2453
3031	serge	2454	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2455	(size & -size) != size \|\|
		2456	(obj->gtt_offset & (size - 1)),
		2457	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2458	obj->gtt_offset, size);
2344	Serge	2459
3031	serge	2460	pitch_val = obj->stride / 128;
		2461	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2462
3031	serge	2463	val = obj->gtt_offset;
		2464	if (obj->tiling_mode == I915_TILING_Y)
		2465	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2466	val \|= I830_FENCE_SIZE_BITS(size);
		2467	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2468	val \|= I830_FENCE_REG_VALID;
		2469	} else
		2470	val = 0;
		2471
		2472	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2473	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2474	}
		2475
3480	Serge	2476	inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
		2477	{
		2478	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
		2479	}
		2480
3031	serge	2481	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2482	struct drm_i915_gem_object *obj)
2332	Serge	2483	{
3480	Serge	2484	struct drm_i915_private *dev_priv = dev->dev_private;
		2485
		2486	/* Ensure that all CPU reads are completed before installing a fence
		2487	* and all writes before removing the fence.
		2488	*/
		2489	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		2490	mb();
		2491
3031	serge	2492	switch (INTEL_INFO(dev)->gen) {
		2493	case 7:
3480	Serge	2494	case 6:
3031	serge	2495	case 5:
		2496	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2497	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2498	case 2: i830_write_fence_reg(dev, reg, obj); break;
3480	Serge	2499	default: BUG();
3031	serge	2500	}
3480	Serge	2501
		2502	/* And similarly be paranoid that no direct access to this region
		2503	* is reordered to before the fence is installed.
		2504	*/
		2505	if (i915_gem_object_needs_mb(obj))
		2506	mb();
2344	Serge	2507	}
		2508
3031	serge	2509	static inline int fence_number(struct drm_i915_private *dev_priv,
		2510	struct drm_i915_fence_reg *fence)
2344	Serge	2511	{
3031	serge	2512	return fence - dev_priv->fence_regs;
		2513	}
2332	Serge	2514
3031	serge	2515	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2516	struct drm_i915_fence_reg *fence,
		2517	bool enable)
		2518	{
		2519	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2520	int reg = fence_number(dev_priv, fence);
2332	Serge	2521
3031	serge	2522	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2523
		2524	if (enable) {
		2525	obj->fence_reg = reg;
		2526	fence->obj = obj;
		2527	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2528	} else {
		2529	obj->fence_reg = I915_FENCE_REG_NONE;
		2530	fence->obj = NULL;
		2531	list_del_init(&fence->lru_list);
2344	Serge	2532	}
3031	serge	2533	}
2344	Serge	2534
3031	serge	2535	static int
3480	Serge	2536	i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
3031	serge	2537	{
		2538	if (obj->last_fenced_seqno) {
		2539	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2540	if (ret)
		2541	return ret;
2344	Serge	2542
		2543	obj->last_fenced_seqno = 0;
		2544	}
		2545
3031	serge	2546	obj->fenced_gpu_access = false;
2332	Serge	2547	return 0;
		2548	}
		2549
		2550	int
2344	Serge	2551	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2552	{
3031	serge	2553	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2554	int ret;
		2555
3480	Serge	2556	ret = i915_gem_object_wait_fence(obj);
2332	Serge	2557	if (ret)
		2558	return ret;
		2559
3031	serge	2560	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2561	return 0;
2332	Serge	2562
3031	serge	2563	i915_gem_object_update_fence(obj,
		2564	&dev_priv->fence_regs[obj->fence_reg],
		2565	false);
		2566	i915_gem_object_fence_lost(obj);
2344	Serge	2567
2332	Serge	2568	return 0;
		2569	}
		2570
3031	serge	2571	static struct drm_i915_fence_reg *
		2572	i915_find_fence_reg(struct drm_device *dev)
		2573	{
		2574	struct drm_i915_private *dev_priv = dev->dev_private;
		2575	struct drm_i915_fence_reg reg, avail;
		2576	int i;
2332	Serge	2577
3031	serge	2578	/* First try to find a free reg */
		2579	avail = NULL;
		2580	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2581	reg = &dev_priv->fence_regs[i];
		2582	if (!reg->obj)
		2583	return reg;
2332	Serge	2584
3031	serge	2585	if (!reg->pin_count)
		2586	avail = reg;
		2587	}
2332	Serge	2588
3031	serge	2589	if (avail == NULL)
		2590	return NULL;
2332	Serge	2591
3031	serge	2592	/* None available, try to steal one or wait for a user to finish */
		2593	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2594	if (reg->pin_count)
		2595	continue;
2332	Serge	2596
3031	serge	2597	return reg;
		2598	}
2332	Serge	2599
3031	serge	2600	return NULL;
		2601	}
2332	Serge	2602
3031	serge	2603	/**
		2604	* i915_gem_object_get_fence - set up fencing for an object
		2605	* @obj: object to map through a fence reg
		2606	*
		2607	* When mapping objects through the GTT, userspace wants to be able to write
		2608	* to them without having to worry about swizzling if the object is tiled.
		2609	* This function walks the fence regs looking for a free one for @obj,
		2610	* stealing one if it can't find any.
		2611	*
		2612	* It then sets up the reg based on the object's properties: address, pitch
		2613	* and tiling format.
		2614	*
		2615	* For an untiled surface, this removes any existing fence.
		2616	*/
		2617	int
		2618	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2619	{
		2620	struct drm_device *dev = obj->base.dev;
		2621	struct drm_i915_private *dev_priv = dev->dev_private;
		2622	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2623	struct drm_i915_fence_reg *reg;
		2624	int ret;
2332	Serge	2625
3031	serge	2626	/* Have we updated the tiling parameters upon the object and so
		2627	* will need to serialise the write to the associated fence register?
		2628	*/
		2629	if (obj->fence_dirty) {
3480	Serge	2630	ret = i915_gem_object_wait_fence(obj);
3031	serge	2631	if (ret)
		2632	return ret;
		2633	}
2332	Serge	2634
3031	serge	2635	/* Just update our place in the LRU if our fence is getting reused. */
		2636	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2637	reg = &dev_priv->fence_regs[obj->fence_reg];
		2638	if (!obj->fence_dirty) {
		2639	list_move_tail(®->lru_list,
		2640	&dev_priv->mm.fence_list);
		2641	return 0;
		2642	}
		2643	} else if (enable) {
		2644	reg = i915_find_fence_reg(dev);
		2645	if (reg == NULL)
		2646	return -EDEADLK;
2332	Serge	2647
3031	serge	2648	if (reg->obj) {
		2649	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2650
3480	Serge	2651	ret = i915_gem_object_wait_fence(old);
3031	serge	2652	if (ret)
		2653	return ret;
2332	Serge	2654
3031	serge	2655	i915_gem_object_fence_lost(old);
		2656	}
		2657	} else
		2658	return 0;
2332	Serge	2659
3031	serge	2660	i915_gem_object_update_fence(obj, reg, enable);
		2661	obj->fence_dirty = false;
2332	Serge	2662
3031	serge	2663	return 0;
		2664	}
2332	Serge	2665
3031	serge	2666	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2667	struct drm_mm_node *gtt_space,
		2668	unsigned long cache_level)
		2669	{
		2670	struct drm_mm_node *other;
2332	Serge	2671
3031	serge	2672	/* On non-LLC machines we have to be careful when putting differing
		2673	* types of snoopable memory together to avoid the prefetcher
3480	Serge	2674	* crossing memory domains and dying.
3031	serge	2675	*/
		2676	if (HAS_LLC(dev))
		2677	return true;
2332	Serge	2678
3031	serge	2679	if (gtt_space == NULL)
		2680	return true;
2332	Serge	2681
3031	serge	2682	if (list_empty(>t_space->node_list))
		2683	return true;
2332	Serge	2684
3031	serge	2685	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2686	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2687	return false;
2344	Serge	2688
3031	serge	2689	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2690	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2691	return false;
2344	Serge	2692
3031	serge	2693	return true;
		2694	}
2344	Serge	2695
3031	serge	2696	static void i915_gem_verify_gtt(struct drm_device *dev)
		2697	{
		2698	#if WATCH_GTT
		2699	struct drm_i915_private *dev_priv = dev->dev_private;
		2700	struct drm_i915_gem_object *obj;
		2701	int err = 0;
2344	Serge	2702
3031	serge	2703	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2704	if (obj->gtt_space == NULL) {
		2705	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2706	err++;
		2707	continue;
		2708	}
2344	Serge	2709
3031	serge	2710	if (obj->cache_level != obj->gtt_space->color) {
		2711	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2712	obj->gtt_space->start,
		2713	obj->gtt_space->start + obj->gtt_space->size,
		2714	obj->cache_level,
		2715	obj->gtt_space->color);
		2716	err++;
		2717	continue;
		2718	}
2344	Serge	2719
3031	serge	2720	if (!i915_gem_valid_gtt_space(dev,
		2721	obj->gtt_space,
		2722	obj->cache_level)) {
		2723	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2724	obj->gtt_space->start,
		2725	obj->gtt_space->start + obj->gtt_space->size,
		2726	obj->cache_level);
		2727	err++;
		2728	continue;
		2729	}
		2730	}
2344	Serge	2731
3031	serge	2732	WARN_ON(err);
		2733	#endif
2326	Serge	2734	}
		2735
2332	Serge	2736	/**
		2737	* Finds free space in the GTT aperture and binds the object there.
		2738	*/
		2739	static int
		2740	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2741	unsigned alignment,
3031	serge	2742	bool map_and_fenceable,
		2743	bool nonblocking)
2332	Serge	2744	{
		2745	struct drm_device *dev = obj->base.dev;
		2746	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2747	struct drm_mm_node *node;
2332	Serge	2748	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2749	bool mappable, fenceable;
		2750	int ret;
2326	Serge	2751
2332	Serge	2752	fence_size = i915_gem_get_gtt_size(dev,
		2753	obj->base.size,
		2754	obj->tiling_mode);
		2755	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2756	obj->base.size,
3480	Serge	2757	obj->tiling_mode, true);
2332	Serge	2758	unfenced_alignment =
3480	Serge	2759	i915_gem_get_gtt_alignment(dev,
2332	Serge	2760	obj->base.size,
3480	Serge	2761	obj->tiling_mode, false);
2332	Serge	2762
		2763	if (alignment == 0)
		2764	alignment = map_and_fenceable ? fence_alignment :
		2765	unfenced_alignment;
		2766	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2767	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2768	return -EINVAL;
		2769	}
		2770
		2771	size = map_and_fenceable ? fence_size : obj->base.size;
		2772
		2773	/* If the object is bigger than the entire aperture, reject it early
		2774	* before evicting everything in a vain attempt to find space.
		2775	*/
		2776	if (obj->base.size >
3480	Serge	2777	(map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) {
2332	Serge	2778	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2779	return -E2BIG;
		2780	}
		2781
3031	serge	2782	ret = i915_gem_object_get_pages(obj);
		2783	if (ret)
		2784	return ret;
		2785
3243	Serge	2786	i915_gem_object_pin_pages(obj);
		2787
		2788	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2789	if (node == NULL) {
		2790	i915_gem_object_unpin_pages(obj);
		2791	return -ENOMEM;
		2792	}
		2793
2332	Serge	2794	search_free:
		2795	if (map_and_fenceable)
3243	Serge	2796	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2797	size, alignment, obj->cache_level,
3480	Serge	2798	0, dev_priv->gtt.mappable_end);
2332	Serge	2799	else
3243	Serge	2800	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2801	size, alignment, obj->cache_level);
		2802	if (ret) {
2332	Serge	2803
3243	Serge	2804	i915_gem_object_unpin_pages(obj);
		2805	kfree(node);
2332	Serge	2806	return ret;
		2807	}
3243	Serge	2808	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2809	i915_gem_object_unpin_pages(obj);
		2810	drm_mm_put_block(node);
3031	serge	2811	return -EINVAL;
		2812	}
2332	Serge	2813
3031	serge	2814	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2815	if (ret) {
3243	Serge	2816	i915_gem_object_unpin_pages(obj);
		2817	drm_mm_put_block(node);
2332	Serge	2818	return ret;
		2819	}
		2820
3031	serge	2821	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2822	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2823
3243	Serge	2824	obj->gtt_space = node;
		2825	obj->gtt_offset = node->start;
2332	Serge	2826
		2827	fenceable =
3243	Serge	2828	node->size == fence_size &&
		2829	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2830
		2831	mappable =
3480	Serge	2832	obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end;
2332	Serge	2833
		2834	obj->map_and_fenceable = mappable && fenceable;
		2835
3243	Serge	2836	i915_gem_object_unpin_pages(obj);
2351	Serge	2837	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2838	i915_gem_verify_gtt(dev);
2332	Serge	2839	return 0;
		2840	}
		2841
		2842	void
		2843	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2844	{
		2845	/* If we don't have a page list set up, then we're not pinned
		2846	* to GPU, and we can ignore the cache flush because it'll happen
		2847	* again at bind time.
		2848	*/
3243	Serge	2849	if (obj->pages == NULL)
2332	Serge	2850	return;
		2851
3480	Serge	2852	/*
		2853	* Stolen memory is always coherent with the GPU as it is explicitly
		2854	* marked as wc by the system, or the system is cache-coherent.
		2855	*/
		2856	if (obj->stolen)
		2857	return;
		2858
2332	Serge	2859	/* If the GPU is snooping the contents of the CPU cache,
		2860	* we do not need to manually clear the CPU cache lines. However,
		2861	* the caches are only snooped when the render cache is
		2862	* flushed/invalidated. As we always have to emit invalidations
		2863	* and flushes when moving into and out of the RENDER domain, correct
		2864	* snooping behaviour occurs naturally as the result of our domain
		2865	* tracking.
		2866	*/
		2867	if (obj->cache_level != I915_CACHE_NONE)
		2868	return;
3243	Serge	2869	#if 0
2344	Serge	2870	if(obj->mapped != NULL)
		2871	{
		2872	uint8_t *page_virtual;
		2873	unsigned int i;
2332	Serge	2874
2344	Serge	2875	page_virtual = obj->mapped;
		2876	asm volatile("mfence");
		2877	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2878	clflush(page_virtual + i);
		2879	asm volatile("mfence");
		2880	}
		2881	else
		2882	{
		2883	uint8_t *page_virtual;
		2884	unsigned int i;
		2885	page_virtual = AllocKernelSpace(obj->base.size);
		2886	if(page_virtual != NULL)
		2887	{
3031	serge	2888	dma_addr_t src, dst;
2344	Serge	2889	u32 count;
		2890
		2891	#define page_tabs 0xFDC00000 /* really dirty hack */
		2892
3031	serge	2893	src = obj->pages.page;
		2894	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2895	count = obj->base.size/4096;
		2896
		2897	while(count--)
		2898	{
		2899	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2900	};
		2901
		2902	asm volatile("mfence");
		2903	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2904	clflush(page_virtual + i);
		2905	asm volatile("mfence");
		2906	FreeKernelSpace(page_virtual);
		2907	}
		2908	else
		2909	{
		2910	asm volatile (
		2911	"mfence \n"
		2912	"wbinvd \n" /* this is really ugly */
		2913	"mfence");
		2914	}
		2915	}
3243	Serge	2916	#endif
		2917
2332	Serge	2918	}
		2919
2344	Serge	2920	/** Flushes the GTT write domain for the object if it's dirty. */
		2921	static void
		2922	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2923	{
		2924	uint32_t old_write_domain;
2332	Serge	2925
2344	Serge	2926	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2927	return;
2332	Serge	2928
2344	Serge	2929	/* No actual flushing is required for the GTT write domain. Writes
		2930	* to it immediately go to main memory as far as we know, so there's
		2931	* no chipset flush. It also doesn't land in render cache.
		2932	*
		2933	* However, we do have to enforce the order so that all writes through
		2934	* the GTT land before any writes to the device, such as updates to
		2935	* the GATT itself.
		2936	*/
		2937	wmb();
2332	Serge	2938
2344	Serge	2939	old_write_domain = obj->base.write_domain;
		2940	obj->base.write_domain = 0;
2332	Serge	2941
2351	Serge	2942	trace_i915_gem_object_change_domain(obj,
		2943	obj->base.read_domains,
		2944	old_write_domain);
2344	Serge	2945	}
2332	Serge	2946
		2947	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2948	static void
2332	Serge	2949	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2950	{
		2951	uint32_t old_write_domain;
		2952
		2953	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2954	return;
		2955
		2956	i915_gem_clflush_object(obj);
3243	Serge	2957	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2958	old_write_domain = obj->base.write_domain;
		2959	obj->base.write_domain = 0;
		2960
2351	Serge	2961	trace_i915_gem_object_change_domain(obj,
		2962	obj->base.read_domains,
		2963	old_write_domain);
2332	Serge	2964	}
		2965
		2966	/**
		2967	* Moves a single object to the GTT read, and possibly write domain.
		2968	*
		2969	* This function returns when the move is complete, including waiting on
		2970	* flushes to occur.
		2971	*/
		2972	int
		2973	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2974	{
3031	serge	2975	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2976	uint32_t old_write_domain, old_read_domains;
		2977	int ret;
		2978
		2979	/* Not valid to be called on unbound objects. */
		2980	if (obj->gtt_space == NULL)
		2981	return -EINVAL;
		2982
		2983	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2984	return 0;
		2985
3031	serge	2986	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2987	if (ret)
		2988	return ret;
		2989
		2990	i915_gem_object_flush_cpu_write_domain(obj);
		2991
3480	Serge	2992	/* Serialise direct access to this object with the barriers for
		2993	* coherent writes from the GPU, by effectively invalidating the
		2994	* GTT domain upon first access.
		2995	*/
		2996	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2997	mb();
		2998
2332	Serge	2999	old_write_domain = obj->base.write_domain;
		3000	old_read_domains = obj->base.read_domains;
		3001
		3002	/* It should now be out of any other write domains, and we can update
		3003	* the domain values for our changes.
		3004	*/
		3005	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		3006	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		3007	if (write) {
		3008	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		3009	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		3010	obj->dirty = 1;
		3011	}
		3012
2351	Serge	3013	trace_i915_gem_object_change_domain(obj,
		3014	old_read_domains,
		3015	old_write_domain);
		3016
3031	serge	3017	/* And bump the LRU for this access */
		3018	if (i915_gem_object_is_inactive(obj))
		3019	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		3020
2332	Serge	3021	return 0;
		3022	}
		3023
2335	Serge	3024	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		3025	enum i915_cache_level cache_level)
		3026	{
3031	serge	3027	struct drm_device *dev = obj->base.dev;
		3028	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	3029	int ret;
2332	Serge	3030
2335	Serge	3031	if (obj->cache_level == cache_level)
		3032	return 0;
2332	Serge	3033
2335	Serge	3034	if (obj->pin_count) {
		3035	DRM_DEBUG("can not change the cache level of pinned objects\n");
		3036	return -EBUSY;
		3037	}
2332	Serge	3038
3031	serge	3039	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		3040	ret = i915_gem_object_unbind(obj);
		3041	if (ret)
		3042	return ret;
		3043	}
		3044
2335	Serge	3045	if (obj->gtt_space) {
		3046	ret = i915_gem_object_finish_gpu(obj);
		3047	if (ret)
		3048	return ret;
2332	Serge	3049
2335	Serge	3050	i915_gem_object_finish_gtt(obj);
2332	Serge	3051
2335	Serge	3052	/* Before SandyBridge, you could not use tiling or fence
		3053	* registers with snooped memory, so relinquish any fences
		3054	* currently pointing to our region in the aperture.
		3055	*/
3031	serge	3056	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	3057	ret = i915_gem_object_put_fence(obj);
		3058	if (ret)
		3059	return ret;
		3060	}
2332	Serge	3061
3031	serge	3062	if (obj->has_global_gtt_mapping)
		3063	i915_gem_gtt_bind_object(obj, cache_level);
		3064	if (obj->has_aliasing_ppgtt_mapping)
		3065	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		3066	obj, cache_level);
		3067
		3068	obj->gtt_space->color = cache_level;
2335	Serge	3069	}
2332	Serge	3070
2335	Serge	3071	if (cache_level == I915_CACHE_NONE) {
		3072	u32 old_read_domains, old_write_domain;
2332	Serge	3073
2335	Serge	3074	/* If we're coming from LLC cached, then we haven't
		3075	* actually been tracking whether the data is in the
		3076	* CPU cache or not, since we only allow one bit set
		3077	* in obj->write_domain and have been skipping the clflushes.
		3078	* Just set it to the CPU cache for now.
		3079	*/
		3080	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		3081	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	3082
2335	Serge	3083	old_read_domains = obj->base.read_domains;
		3084	old_write_domain = obj->base.write_domain;
2332	Serge	3085
2335	Serge	3086	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3087	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	3088
2351	Serge	3089	trace_i915_gem_object_change_domain(obj,
		3090	old_read_domains,
		3091	old_write_domain);
2344	Serge	3092	}
2332	Serge	3093
2335	Serge	3094	obj->cache_level = cache_level;
3031	serge	3095	i915_gem_verify_gtt(dev);
2335	Serge	3096	return 0;
		3097	}
2332	Serge	3098
3260	Serge	3099	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		3100	struct drm_file *file)
		3101	{
		3102	struct drm_i915_gem_caching *args = data;
		3103	struct drm_i915_gem_object *obj;
		3104	int ret;
		3105
3480	Serge	3106	if(args->handle == -2)
		3107	{
		3108	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3109	return 0;
		3110	}
		3111
3260	Serge	3112	ret = i915_mutex_lock_interruptible(dev);
		3113	if (ret)
		3114	return ret;
		3115
		3116	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3117	if (&obj->base == NULL) {
		3118	ret = -ENOENT;
		3119	goto unlock;
		3120	}
		3121
		3122	args->caching = obj->cache_level != I915_CACHE_NONE;
		3123
		3124	drm_gem_object_unreference(&obj->base);
		3125	unlock:
		3126	mutex_unlock(&dev->struct_mutex);
		3127	return ret;
		3128	}
		3129
		3130	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		3131	struct drm_file *file)
		3132	{
		3133	struct drm_i915_gem_caching *args = data;
		3134	struct drm_i915_gem_object *obj;
		3135	enum i915_cache_level level;
		3136	int ret;
		3137
3480	Serge	3138	if(args->handle == -2)
		3139	{
		3140	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3141	return 0;
		3142	}
		3143
3260	Serge	3144	switch (args->caching) {
		3145	case I915_CACHING_NONE:
		3146	level = I915_CACHE_NONE;
		3147	break;
		3148	case I915_CACHING_CACHED:
		3149	level = I915_CACHE_LLC;
		3150	break;
		3151	default:
		3152	return -EINVAL;
		3153	}
		3154
		3155	ret = i915_mutex_lock_interruptible(dev);
		3156	if (ret)
		3157	return ret;
		3158
		3159	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3160	if (&obj->base == NULL) {
		3161	ret = -ENOENT;
		3162	goto unlock;
		3163	}
		3164
		3165	ret = i915_gem_object_set_cache_level(obj, level);
		3166
		3167	drm_gem_object_unreference(&obj->base);
		3168	unlock:
		3169	mutex_unlock(&dev->struct_mutex);
		3170	return ret;
		3171	}
		3172
2335	Serge	3173	/*
		3174	* Prepare buffer for display plane (scanout, cursors, etc).
		3175	* Can be called from an uninterruptible phase (modesetting) and allows
		3176	* any flushes to be pipelined (for pageflips).
		3177	*/
		3178	int
		3179	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3180	u32 alignment,
		3181	struct intel_ring_buffer *pipelined)
		3182	{
		3183	u32 old_read_domains, old_write_domain;
		3184	int ret;
2332	Serge	3185
3031	serge	3186	if (pipelined != obj->ring) {
		3187	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3188	if (ret)
		3189	return ret;
		3190	}
2332	Serge	3191
2335	Serge	3192	/* The display engine is not coherent with the LLC cache on gen6. As
		3193	* a result, we make sure that the pinning that is about to occur is
		3194	* done with uncached PTEs. This is lowest common denominator for all
		3195	* chipsets.
		3196	*
		3197	* However for gen6+, we could do better by using the GFDT bit instead
		3198	* of uncaching, which would allow us to flush all the LLC-cached data
		3199	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3200	*/
2360	Serge	3201	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3202	if (ret)
		3203	return ret;
2332	Serge	3204
2335	Serge	3205	/* As the user may map the buffer once pinned in the display plane
		3206	* (e.g. libkms for the bootup splash), we have to ensure that we
		3207	* always use map_and_fenceable for all scanout buffers.
		3208	*/
3031	serge	3209	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3210	if (ret)
		3211	return ret;
2332	Serge	3212
2335	Serge	3213	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3214
2335	Serge	3215	old_write_domain = obj->base.write_domain;
		3216	old_read_domains = obj->base.read_domains;
2332	Serge	3217
2335	Serge	3218	/* It should now be out of any other write domains, and we can update
		3219	* the domain values for our changes.
		3220	*/
3031	serge	3221	obj->base.write_domain = 0;
2335	Serge	3222	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3223
2351	Serge	3224	trace_i915_gem_object_change_domain(obj,
		3225	old_read_domains,
		3226	old_write_domain);
2332	Serge	3227
2335	Serge	3228	return 0;
		3229	}
2332	Serge	3230
2344	Serge	3231	int
		3232	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3233	{
		3234	int ret;
2332	Serge	3235
2344	Serge	3236	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3237	return 0;
2332	Serge	3238
3031	serge	3239	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3240	if (ret)
		3241	return ret;
2332	Serge	3242
2344	Serge	3243	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3244	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3245	return 0;
2344	Serge	3246	}
2332	Serge	3247
2344	Serge	3248	/**
		3249	* Moves a single object to the CPU read, and possibly write domain.
		3250	*
		3251	* This function returns when the move is complete, including waiting on
		3252	* flushes to occur.
		3253	*/
3031	serge	3254	int
2344	Serge	3255	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3256	{
		3257	uint32_t old_write_domain, old_read_domains;
		3258	int ret;
2332	Serge	3259
2344	Serge	3260	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3261	return 0;
2332	Serge	3262
3031	serge	3263	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3264	if (ret)
		3265	return ret;
2332	Serge	3266
2344	Serge	3267	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3268
2344	Serge	3269	old_write_domain = obj->base.write_domain;
		3270	old_read_domains = obj->base.read_domains;
2332	Serge	3271
2344	Serge	3272	/* Flush the CPU cache if it's still invalid. */
		3273	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3274	i915_gem_clflush_object(obj);
2332	Serge	3275
2344	Serge	3276	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3277	}
2332	Serge	3278
2344	Serge	3279	/* It should now be out of any other write domains, and we can update
		3280	* the domain values for our changes.
		3281	*/
		3282	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3283
2344	Serge	3284	/* If we're writing through the CPU, then the GPU read domains will
		3285	* need to be invalidated at next use.
		3286	*/
		3287	if (write) {
		3288	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3289	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3290	}
2332	Serge	3291
2351	Serge	3292	trace_i915_gem_object_change_domain(obj,
		3293	old_read_domains,
		3294	old_write_domain);
2332	Serge	3295
2344	Serge	3296	return 0;
		3297	}
2332	Serge	3298
3031	serge	3299	/* Throttle our rendering by waiting until the ring has completed our requests
		3300	* emitted over 20 msec ago.
2344	Serge	3301	*
3031	serge	3302	* Note that if we were to use the current jiffies each time around the loop,
		3303	* we wouldn't escape the function with any frames outstanding if the time to
		3304	* render a frame was over 20ms.
		3305	*
		3306	* This should get us reasonable parallelism between CPU and GPU but also
		3307	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3308	*/
3031	serge	3309	static int
		3310	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3311	{
3031	serge	3312	struct drm_i915_private *dev_priv = dev->dev_private;
		3313	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3314	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3315	struct drm_i915_gem_request *request;
		3316	struct intel_ring_buffer *ring = NULL;
3480	Serge	3317	unsigned reset_counter;
3031	serge	3318	u32 seqno = 0;
		3319	int ret;
2332	Serge	3320
3480	Serge	3321	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
		3322	if (ret)
		3323	return ret;
2332	Serge	3324
3480	Serge	3325	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
		3326	if (ret)
		3327	return ret;
		3328
3031	serge	3329	spin_lock(&file_priv->mm.lock);
		3330	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3331	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3332	break;
2332	Serge	3333
3031	serge	3334	ring = request->ring;
		3335	seqno = request->seqno;
		3336	}
3480	Serge	3337	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3031	serge	3338	spin_unlock(&file_priv->mm.lock);
2332	Serge	3339
3031	serge	3340	if (seqno == 0)
		3341	return 0;
2332	Serge	3342
3480	Serge	3343	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3031	serge	3344	if (ret == 0)
		3345	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3346
3031	serge	3347	return ret;
2352	Serge	3348	}
2332	Serge	3349
		3350	int
		3351	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3352	uint32_t alignment,
3031	serge	3353	bool map_and_fenceable,
		3354	bool nonblocking)
2332	Serge	3355	{
		3356	int ret;
		3357
3031	serge	3358	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3359	return -EBUSY;
2332	Serge	3360
		3361	if (obj->gtt_space != NULL) {
		3362	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3363	(map_and_fenceable && !obj->map_and_fenceable)) {
		3364	WARN(obj->pin_count,
		3365	"bo is already pinned with incorrect alignment:"
		3366	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3367	" obj->map_and_fenceable=%d\n",
		3368	obj->gtt_offset, alignment,
		3369	map_and_fenceable,
		3370	obj->map_and_fenceable);
		3371	ret = i915_gem_object_unbind(obj);
		3372	if (ret)
		3373	return ret;
		3374	}
		3375	}
		3376
		3377	if (obj->gtt_space == NULL) {
3243	Serge	3378	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3379
2332	Serge	3380	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3381	map_and_fenceable,
		3382	nonblocking);
2332	Serge	3383	if (ret)
		3384	return ret;
3243	Serge	3385
		3386	if (!dev_priv->mm.aliasing_ppgtt)
		3387	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3388	}
		3389
3031	serge	3390	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3391	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3392
		3393	obj->pin_count++;
2332	Serge	3394	obj->pin_mappable \|= map_and_fenceable;
		3395
		3396	return 0;
		3397	}
		3398
2344	Serge	3399	void
		3400	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3401	{
		3402	BUG_ON(obj->pin_count == 0);
		3403	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3404
3031	serge	3405	if (--obj->pin_count == 0)
2344	Serge	3406	obj->pin_mappable = false;
		3407	}
2332	Serge	3408
3031	serge	3409	int
		3410	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3411	struct drm_file *file)
		3412	{
		3413	struct drm_i915_gem_pin *args = data;
		3414	struct drm_i915_gem_object *obj;
		3415	int ret;
2332	Serge	3416
3480	Serge	3417	if(args->handle == -2)
		3418	{
		3419	printf("%s handle %d\n", __FUNCTION__, args->handle);
		3420	return 0;
		3421	}
		3422
3031	serge	3423	ret = i915_mutex_lock_interruptible(dev);
		3424	if (ret)
		3425	return ret;
2332	Serge	3426
3031	serge	3427	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3428	if (&obj->base == NULL) {
		3429	ret = -ENOENT;
		3430	goto unlock;
		3431	}
2332	Serge	3432
3031	serge	3433	if (obj->madv != I915_MADV_WILLNEED) {
		3434	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3435	ret = -EINVAL;
		3436	goto out;
		3437	}
2332	Serge	3438
3031	serge	3439	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3440	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3441	args->handle);
		3442	ret = -EINVAL;
		3443	goto out;
		3444	}
2332	Serge	3445
3243	Serge	3446	if (obj->user_pin_count == 0) {
3031	serge	3447	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3448	if (ret)
		3449	goto out;
		3450	}
2332	Serge	3451
3243	Serge	3452	obj->user_pin_count++;
		3453	obj->pin_filp = file;
		3454
3031	serge	3455	/* XXX - flush the CPU caches for pinned objects
		3456	* as the X server doesn't manage domains yet
		3457	*/
		3458	i915_gem_object_flush_cpu_write_domain(obj);
		3459	args->offset = obj->gtt_offset;
		3460	out:
		3461	drm_gem_object_unreference(&obj->base);
		3462	unlock:
		3463	mutex_unlock(&dev->struct_mutex);
		3464	return ret;
		3465	}
2332	Serge	3466
3255	Serge	3467	#if 0
		3468
3031	serge	3469	int
		3470	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3471	struct drm_file *file)
		3472	{
		3473	struct drm_i915_gem_pin *args = data;
		3474	struct drm_i915_gem_object *obj;
		3475	int ret;
2332	Serge	3476
3031	serge	3477	ret = i915_mutex_lock_interruptible(dev);
		3478	if (ret)
		3479	return ret;
2332	Serge	3480
3031	serge	3481	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3482	if (&obj->base == NULL) {
		3483	ret = -ENOENT;
		3484	goto unlock;
		3485	}
2332	Serge	3486
3031	serge	3487	if (obj->pin_filp != file) {
		3488	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3489	args->handle);
		3490	ret = -EINVAL;
		3491	goto out;
		3492	}
		3493	obj->user_pin_count--;
		3494	if (obj->user_pin_count == 0) {
		3495	obj->pin_filp = NULL;
		3496	i915_gem_object_unpin(obj);
		3497	}
2332	Serge	3498
3031	serge	3499	out:
		3500	drm_gem_object_unreference(&obj->base);
		3501	unlock:
		3502	mutex_unlock(&dev->struct_mutex);
		3503	return ret;
		3504	}
2332	Serge	3505
3260	Serge	3506	#endif
		3507
3031	serge	3508	int
		3509	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3510	struct drm_file *file)
		3511	{
		3512	struct drm_i915_gem_busy *args = data;
		3513	struct drm_i915_gem_object *obj;
		3514	int ret;
2332	Serge	3515
3031	serge	3516	ret = i915_mutex_lock_interruptible(dev);
		3517	if (ret)
		3518	return ret;
2332	Serge	3519
3480	Serge	3520	if(args->handle == -2)
		3521	{
		3522	obj = get_fb_obj();
		3523	drm_gem_object_reference(&obj->base);
		3524	}
		3525	else
3031	serge	3526	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3527	if (&obj->base == NULL) {
		3528	ret = -ENOENT;
		3529	goto unlock;
		3530	}
2332	Serge	3531
3031	serge	3532	/* Count all active objects as busy, even if they are currently not used
		3533	* by the gpu. Users of this interface expect objects to eventually
		3534	* become non-busy without any further actions, therefore emit any
		3535	* necessary flushes here.
		3536	*/
		3537	ret = i915_gem_object_flush_active(obj);
2332	Serge	3538
3031	serge	3539	args->busy = obj->active;
		3540	if (obj->ring) {
		3541	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3542	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3543	}
2332	Serge	3544
3031	serge	3545	drm_gem_object_unreference(&obj->base);
		3546	unlock:
		3547	mutex_unlock(&dev->struct_mutex);
		3548	return ret;
		3549	}
2332	Serge	3550
3031	serge	3551	int
		3552	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3553	struct drm_file *file_priv)
		3554	{
		3555	return i915_gem_ring_throttle(dev, file_priv);
		3556	}
2332	Serge	3557
3263	Serge	3558	#if 0
		3559
3031	serge	3560	int
		3561	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3562	struct drm_file *file_priv)
		3563	{
		3564	struct drm_i915_gem_madvise *args = data;
		3565	struct drm_i915_gem_object *obj;
		3566	int ret;
2332	Serge	3567
3031	serge	3568	switch (args->madv) {
		3569	case I915_MADV_DONTNEED:
		3570	case I915_MADV_WILLNEED:
		3571	break;
		3572	default:
		3573	return -EINVAL;
		3574	}
2332	Serge	3575
3031	serge	3576	ret = i915_mutex_lock_interruptible(dev);
		3577	if (ret)
		3578	return ret;
2332	Serge	3579
3031	serge	3580	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3581	if (&obj->base == NULL) {
		3582	ret = -ENOENT;
		3583	goto unlock;
		3584	}
2332	Serge	3585
3031	serge	3586	if (obj->pin_count) {
		3587	ret = -EINVAL;
		3588	goto out;
		3589	}
2332	Serge	3590
3031	serge	3591	if (obj->madv != __I915_MADV_PURGED)
		3592	obj->madv = args->madv;
2332	Serge	3593
3031	serge	3594	/* if the object is no longer attached, discard its backing storage */
		3595	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3596	i915_gem_object_truncate(obj);
2332	Serge	3597
3031	serge	3598	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3599
3031	serge	3600	out:
		3601	drm_gem_object_unreference(&obj->base);
		3602	unlock:
		3603	mutex_unlock(&dev->struct_mutex);
		3604	return ret;
		3605	}
		3606	#endif
2332	Serge	3607
3031	serge	3608	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3609	const struct drm_i915_gem_object_ops *ops)
		3610	{
		3611	INIT_LIST_HEAD(&obj->mm_list);
		3612	INIT_LIST_HEAD(&obj->gtt_list);
		3613	INIT_LIST_HEAD(&obj->ring_list);
		3614	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3615
3031	serge	3616	obj->ops = ops;
		3617
		3618	obj->fence_reg = I915_FENCE_REG_NONE;
		3619	obj->madv = I915_MADV_WILLNEED;
		3620	/* Avoid an unnecessary call to unbind on the first bind. */
		3621	obj->map_and_fenceable = true;
		3622
		3623	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3624	}
		3625
		3626	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3627	.get_pages = i915_gem_object_get_pages_gtt,
		3628	.put_pages = i915_gem_object_put_pages_gtt,
		3629	};
		3630
2332	Serge	3631	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3632	size_t size)
		3633	{
		3634	struct drm_i915_gem_object *obj;
3031	serge	3635	struct address_space *mapping;
3480	Serge	3636	gfp_t mask;
2340	Serge	3637
2332	Serge	3638	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3639	if (obj == NULL)
		3640	return NULL;
		3641
		3642	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3643	kfree(obj);
		3644	return NULL;
		3645	}
		3646
		3647
3031	serge	3648	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3649
		3650	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3651	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3652
3031	serge	3653	if (HAS_LLC(dev)) {
		3654	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3655	* cache) for about a 10% performance improvement
		3656	* compared to uncached. Graphics requests other than
		3657	* display scanout are coherent with the CPU in
		3658	* accessing this cache. This means in this mode we
		3659	* don't need to clflush on the CPU side, and on the
		3660	* GPU side we only need to flush internal caches to
		3661	* get data visible to the CPU.
		3662	*
		3663	* However, we maintain the display planes as UC, and so
		3664	* need to rebind when first used as such.
		3665	*/
		3666	obj->cache_level = I915_CACHE_LLC;
		3667	} else
		3668	obj->cache_level = I915_CACHE_NONE;
		3669
		3670	return obj;
		3671	}
		3672
2344	Serge	3673	int i915_gem_init_object(struct drm_gem_object *obj)
		3674	{
		3675	BUG();
2332	Serge	3676
2344	Serge	3677	return 0;
		3678	}
2332	Serge	3679
3031	serge	3680	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3681	{
3031	serge	3682	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3683	struct drm_device *dev = obj->base.dev;
		3684	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3685
3031	serge	3686	trace_i915_gem_object_destroy(obj);
		3687
		3688	// if (obj->phys_obj)
		3689	// i915_gem_detach_phys_object(dev, obj);
3298	Serge	3690	// printf("%s obj %p\n", __FUNCTION__, obj);
3031	serge	3691
		3692	obj->pin_count = 0;
		3693	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3694	bool was_interruptible;
		3695
		3696	was_interruptible = dev_priv->mm.interruptible;
		3697	dev_priv->mm.interruptible = false;
		3698
		3699	WARN_ON(i915_gem_object_unbind(obj));
		3700
		3701	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3702	}
2332	Serge	3703
3031	serge	3704	obj->pages_pin_count = 0;
		3705	i915_gem_object_put_pages(obj);
		3706	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3707
3243	Serge	3708	BUG_ON(obj->pages);
2332	Serge	3709
3031	serge	3710	// if (obj->base.import_attach)
		3711	// drm_prime_gem_destroy(&obj->base, NULL);
		3712
3290	Serge	3713	if(obj->base.filp != NULL)
		3714	{
3298	Serge	3715	// printf("filp %p\n", obj->base.filp);
3290	Serge	3716	shmem_file_delete(obj->base.filp);
		3717	}
		3718
2344	Serge	3719	drm_gem_object_release(&obj->base);
		3720	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3721
2344	Serge	3722	kfree(obj->bit_17);
		3723	kfree(obj);
		3724	}
2332	Serge	3725
3031	serge	3726	#if 0
		3727	int
		3728	i915_gem_idle(struct drm_device *dev)
2344	Serge	3729	{
3031	serge	3730	drm_i915_private_t *dev_priv = dev->dev_private;
		3731	int ret;
2332	Serge	3732
3031	serge	3733	mutex_lock(&dev->struct_mutex);
2332	Serge	3734
3031	serge	3735	if (dev_priv->mm.suspended) {
		3736	mutex_unlock(&dev->struct_mutex);
		3737	return 0;
		3738	}
2332	Serge	3739
3031	serge	3740	ret = i915_gpu_idle(dev);
		3741	if (ret) {
		3742	mutex_unlock(&dev->struct_mutex);
		3743	return ret;
		3744	}
		3745	i915_gem_retire_requests(dev);
		3746
3480	Serge	3747	/* Under UMS, be paranoid and evict. */
		3748	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		3749	i915_gem_evict_everything(dev);
		3750
3031	serge	3751	i915_gem_reset_fences(dev);
		3752
		3753	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3754	* We need to replace this with a semaphore, or something.
		3755	* And not confound mm.suspended!
		3756	*/
		3757	dev_priv->mm.suspended = 1;
3480	Serge	3758	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3031	serge	3759
		3760	i915_kernel_lost_context(dev);
		3761	i915_gem_cleanup_ringbuffer(dev);
		3762
		3763	mutex_unlock(&dev->struct_mutex);
		3764
		3765	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	3766	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	3767
		3768	return 0;
2344	Serge	3769	}
3031	serge	3770	#endif
2332	Serge	3771
3031	serge	3772	void i915_gem_l3_remap(struct drm_device *dev)
		3773	{
		3774	drm_i915_private_t *dev_priv = dev->dev_private;
		3775	u32 misccpctl;
		3776	int i;
2332	Serge	3777
3480	Serge	3778	if (!HAS_L3_GPU_CACHE(dev))
3031	serge	3779	return;
2332	Serge	3780
3243	Serge	3781	if (!dev_priv->l3_parity.remap_info)
3031	serge	3782	return;
2332	Serge	3783
3031	serge	3784	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3785	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3786	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3787
3031	serge	3788	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3789	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3790	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3791	DRM_DEBUG("0x%x was already programmed to %x\n",
		3792	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3793	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3794	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3795	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3796	}
2332	Serge	3797
3031	serge	3798	/* Make sure all the writes land before disabling dop clock gating */
		3799	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3800
3031	serge	3801	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3802	}
2332	Serge	3803
3031	serge	3804	void i915_gem_init_swizzling(struct drm_device *dev)
		3805	{
		3806	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3807
3031	serge	3808	if (INTEL_INFO(dev)->gen < 5 \|\|
		3809	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3810	return;
2332	Serge	3811
3031	serge	3812	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3813	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3814
3031	serge	3815	if (IS_GEN5(dev))
		3816	return;
2344	Serge	3817
3031	serge	3818	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3819	if (IS_GEN6(dev))
		3820	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3480	Serge	3821	else if (IS_GEN7(dev))
		3822	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3031	serge	3823	else
3480	Serge	3824	BUG();
3031	serge	3825	}
		3826
		3827	static bool
		3828	intel_enable_blt(struct drm_device *dev)
		3829	{
		3830	if (!HAS_BLT(dev))
		3831	return false;
		3832
		3833	/* The blitter was dysfunctional on early prototypes */
		3834	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3835	DRM_INFO("BLT not supported on this pre-production hardware;"
		3836	" graphics performance will be degraded.\n");
		3837	return false;
		3838	}
		3839
		3840	return true;
		3841	}
		3842
3480	Serge	3843	static int i915_gem_init_rings(struct drm_device *dev)
2332	Serge	3844	{
3480	Serge	3845	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	3846	int ret;
2351	Serge	3847
2332	Serge	3848	ret = intel_init_render_ring_buffer(dev);
		3849	if (ret)
		3850	return ret;
		3851
		3852	if (HAS_BSD(dev)) {
		3853	ret = intel_init_bsd_ring_buffer(dev);
		3854	if (ret)
		3855	goto cleanup_render_ring;
		3856	}
		3857
3031	serge	3858	if (intel_enable_blt(dev)) {
2332	Serge	3859	ret = intel_init_blt_ring_buffer(dev);
		3860	if (ret)
		3861	goto cleanup_bsd_ring;
		3862	}
		3863
3480	Serge	3864	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
		3865	if (ret)
		3866	goto cleanup_blt_ring;
2351	Serge	3867
2332	Serge	3868	return 0;
		3869
3480	Serge	3870	cleanup_blt_ring:
		3871	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
2332	Serge	3872	cleanup_bsd_ring:
		3873	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3874	cleanup_render_ring:
		3875	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3480	Serge	3876
2332	Serge	3877	return ret;
		3878	}
		3879
3480	Serge	3880	int
		3881	i915_gem_init_hw(struct drm_device *dev)
3031	serge	3882	{
3480	Serge	3883	drm_i915_private_t *dev_priv = dev->dev_private;
		3884	int ret;
3031	serge	3885
3480	Serge	3886	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		3887	return -EIO;
3031	serge	3888
3480	Serge	3889	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3890	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3891
		3892	i915_gem_l3_remap(dev);
		3893
		3894	i915_gem_init_swizzling(dev);
		3895
		3896	ret = i915_gem_init_rings(dev);
		3897	if (ret)
		3898	return ret;
		3899
		3900	/*
		3901	* XXX: There was some w/a described somewhere suggesting loading
		3902	* contexts before PPGTT.
		3903	*/
		3904	i915_gem_context_init(dev);
		3905	i915_gem_init_ppgtt(dev);
		3906
		3907	return 0;
3031	serge	3908	}
		3909
		3910	#define LFB_SIZE 0xC00000
		3911
		3912	int i915_gem_init(struct drm_device *dev)
		3913	{
		3914	struct drm_i915_private *dev_priv = dev->dev_private;
		3915	int ret;
		3916
		3917	mutex_lock(&dev->struct_mutex);
3480	Serge	3918	i915_gem_init_global_gtt(dev);
3031	serge	3919	ret = i915_gem_init_hw(dev);
		3920	mutex_unlock(&dev->struct_mutex);
		3921	if (ret) {
		3922	i915_gem_cleanup_aliasing_ppgtt(dev);
		3923	return ret;
		3924	}
		3925
		3926	return 0;
		3927	}
		3928
2332	Serge	3929	void
		3930	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3931	{
		3932	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3933	struct intel_ring_buffer *ring;
2332	Serge	3934	int i;
		3935
3031	serge	3936	for_each_ring(ring, dev_priv, i)
		3937	intel_cleanup_ring_buffer(ring);
2332	Serge	3938	}
		3939
3031	serge	3940	#if 0
		3941
2332	Serge	3942	int
		3943	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3944	struct drm_file *file_priv)
		3945	{
		3946	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3947	int ret;
2332	Serge	3948
		3949	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3950	return 0;
		3951
3480	Serge	3952	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2332	Serge	3953	DRM_ERROR("Reenabling wedged hardware, good luck\n");
3480	Serge	3954	atomic_set(&dev_priv->gpu_error.reset_counter, 0);
2332	Serge	3955	}
		3956
		3957	mutex_lock(&dev->struct_mutex);
		3958	dev_priv->mm.suspended = 0;
		3959
3031	serge	3960	ret = i915_gem_init_hw(dev);
2332	Serge	3961	if (ret != 0) {
		3962	mutex_unlock(&dev->struct_mutex);
		3963	return ret;
		3964	}
		3965
		3966	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3967	mutex_unlock(&dev->struct_mutex);
		3968
		3969	ret = drm_irq_install(dev);
		3970	if (ret)
		3971	goto cleanup_ringbuffer;
		3972
		3973	return 0;
		3974
		3975	cleanup_ringbuffer:
		3976	mutex_lock(&dev->struct_mutex);
		3977	i915_gem_cleanup_ringbuffer(dev);
		3978	dev_priv->mm.suspended = 1;
		3979	mutex_unlock(&dev->struct_mutex);
		3980
		3981	return ret;
		3982	}
		3983
		3984	int
		3985	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3986	struct drm_file *file_priv)
		3987	{
		3988	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3989	return 0;
		3990
		3991	drm_irq_uninstall(dev);
		3992	return i915_gem_idle(dev);
		3993	}
		3994
		3995	void
		3996	i915_gem_lastclose(struct drm_device *dev)
		3997	{
		3998	int ret;
		3999
		4000	if (drm_core_check_feature(dev, DRIVER_MODESET))
		4001	return;
		4002
		4003	ret = i915_gem_idle(dev);
		4004	if (ret)
		4005	DRM_ERROR("failed to idle hardware: %d\n", ret);
		4006	}
		4007	#endif
		4008
		4009	static void
2326	Serge	4010	init_ring_lists(struct intel_ring_buffer *ring)
		4011	{
		4012	INIT_LIST_HEAD(&ring->active_list);
		4013	INIT_LIST_HEAD(&ring->request_list);
		4014	}
		4015
		4016	void
		4017	i915_gem_load(struct drm_device *dev)
		4018	{
3480	Serge	4019	drm_i915_private_t *dev_priv = dev->dev_private;
2326	Serge	4020	int i;
		4021
		4022	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		4023	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	4024	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		4025	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	4026	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		4027	for (i = 0; i < I915_NUM_RINGS; i++)
		4028	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	4029	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	4030	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	4031	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		4032	i915_gem_retire_work_handler);
3480	Serge	4033	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
2326	Serge	4034
		4035	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		4036	if (IS_GEN3(dev)) {
3031	serge	4037	I915_WRITE(MI_ARB_STATE,
		4038	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	4039	}
		4040
		4041	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		4042
		4043	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		4044	dev_priv->num_fence_regs = 16;
		4045	else
		4046	dev_priv->num_fence_regs = 8;
		4047
		4048	/* Initialize fence registers to zero */
3031	serge	4049	i915_gem_reset_fences(dev);
2326	Serge	4050
		4051	i915_gem_detect_bit_6_swizzle(dev);
		4052
		4053	dev_priv->mm.interruptible = true;
		4054
		4055	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		4056	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		4057	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		4058	}
		4059
		4060

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3480