WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
3260	Serge	33	#include
2330	Serge	34	#include
2326	Serge	35	//#include
		36	#include
		37
2344	Serge	38	extern int x86_clflush_size;
2332	Serge	39
3263	Serge	40	#define PROT_READ 0x1 /* page can be read */
		41	#define PROT_WRITE 0x2 /* page can be written */
		42	#define MAP_SHARED 0x01 /* Share changes */
		43
2344	Serge	44	#undef mb
		45	#undef rmb
		46	#undef wmb
		47	#define mb() asm volatile("mfence")
		48	#define rmb() asm volatile ("lfence")
		49	#define wmb() asm volatile ("sfence")
		50
3266	Serge	51	struct drm_i915_gem_object *get_fb_obj();
		52
3263	Serge	53	unsigned long vm_mmap(struct file *file, unsigned long addr,
		54	unsigned long len, unsigned long prot,
		55	unsigned long flag, unsigned long offset);
		56
2344	Serge	57	static inline void clflush(volatile void *__p)
		58	{
		59	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		60	}
		61
2332	Serge	62	#define MAX_ERRNO 4095
		63
		64	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		65
		66
2326	Serge	67	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		68	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		69	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		70	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		71
2332	Serge	72	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		73	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		74	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		75	unsigned alignment,
3031	serge	76	bool map_and_fenceable,
		77	bool nonblocking);
2332	Serge	78	static int i915_gem_phys_pwrite(struct drm_device *dev,
		79	struct drm_i915_gem_object *obj,
		80	struct drm_i915_gem_pwrite *args,
		81	struct drm_file *file);
2326	Serge	82
3031	serge	83	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		84	struct drm_i915_gem_object *obj);
		85	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		86	struct drm_i915_fence_reg *fence,
		87	bool enable);
2332	Serge	88
3031	serge	89	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		90	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		91	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		92
		93	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		94	{
		95	if (obj->tiling_mode)
		96	i915_gem_release_mmap(obj);
		97
		98	/* As we do not have an associated fence register, we will force
		99	* a tiling change if we ever need to acquire one.
		100	*/
		101	obj->fence_dirty = false;
		102	obj->fence_reg = I915_FENCE_REG_NONE;
		103	}
		104
2332	Serge	105	/* some bookkeeping */
		106	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		107	size_t size)
		108	{
		109	dev_priv->mm.object_count++;
		110	dev_priv->mm.object_memory += size;
		111	}
		112
		113	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		114	size_t size)
		115	{
		116	dev_priv->mm.object_count--;
		117	dev_priv->mm.object_memory -= size;
		118	}
		119
		120	static int
		121	i915_gem_wait_for_error(struct drm_device *dev)
		122	{
		123	struct drm_i915_private *dev_priv = dev->dev_private;
		124	struct completion *x = &dev_priv->error_completion;
		125	unsigned long flags;
		126	int ret;
		127
		128	if (!atomic_read(&dev_priv->mm.wedged))
		129	return 0;
3255	Serge	130	#if 0
3031	serge	131	/*
		132	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		133	* userspace. If it takes that long something really bad is going on and
		134	* we should simply try to bail out and fail as gracefully as possible.
		135	*/
		136	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		137	if (ret == 0) {
		138	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		139	return -EIO;
		140	} else if (ret < 0) {
2332	Serge	141	return ret;
3031	serge	142	}
2332	Serge	143
		144	if (atomic_read(&dev_priv->mm.wedged)) {
		145	/* GPU is hung, bump the completion count to account for
		146	* the token we just consumed so that we never hit zero and
		147	* end up waiting upon a subsequent completion event that
		148	* will never happen.
		149	*/
		150	spin_lock_irqsave(&x->wait.lock, flags);
		151	x->done++;
		152	spin_unlock_irqrestore(&x->wait.lock, flags);
		153	}
3255	Serge	154	#endif
		155
2332	Serge	156	return 0;
		157	}
		158
		159	int i915_mutex_lock_interruptible(struct drm_device *dev)
		160	{
		161	int ret;
		162
		163	ret = i915_gem_wait_for_error(dev);
		164	if (ret)
		165	return ret;
		166
3255	Serge	167	mutex_lock(&dev->struct_mutex);
2332	Serge	168
		169	WARN_ON(i915_verify_lists(dev));
		170	return 0;
		171	}
		172
		173	static inline bool
		174	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		175	{
3031	serge	176	return obj->gtt_space && !obj->active;
2332	Serge	177	}
		178
		179
		180	#if 0
		181
		182	int
		183	i915_gem_init_ioctl(struct drm_device dev, void data,
		184	struct drm_file *file)
		185	{
		186	struct drm_i915_gem_init *args = data;
		187
3031	serge	188	if (drm_core_check_feature(dev, DRIVER_MODESET))
		189	return -ENODEV;
		190
2332	Serge	191	if (args->gtt_start >= args->gtt_end \|\|
		192	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		193	return -EINVAL;
		194
3031	serge	195	/* GEM with user mode setting was never supported on ilk and later. */
		196	if (INTEL_INFO(dev)->gen >= 5)
		197	return -ENODEV;
		198
2332	Serge	199	mutex_lock(&dev->struct_mutex);
3031	serge	200	i915_gem_init_global_gtt(dev, args->gtt_start,
		201	args->gtt_end, args->gtt_end);
2332	Serge	202	mutex_unlock(&dev->struct_mutex);
		203
		204	return 0;
		205	}
2351	Serge	206	#endif
2332	Serge	207
		208	int
		209	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		210	struct drm_file *file)
		211	{
		212	struct drm_i915_private *dev_priv = dev->dev_private;
		213	struct drm_i915_gem_get_aperture *args = data;
		214	struct drm_i915_gem_object *obj;
		215	size_t pinned;
		216
		217	pinned = 0;
		218	mutex_lock(&dev->struct_mutex);
3031	serge	219	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		220	if (obj->pin_count)
2332	Serge	221	pinned += obj->gtt_space->size;
		222	mutex_unlock(&dev->struct_mutex);
		223
		224	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	225	args->aper_available_size = args->aper_size - pinned;
2332	Serge	226
		227	return 0;
		228	}
		229
3031	serge	230	static int
		231	i915_gem_create(struct drm_file *file,
2332	Serge	232	struct drm_device *dev,
		233	uint64_t size,
		234	uint32_t *handle_p)
		235	{
		236	struct drm_i915_gem_object *obj;
		237	int ret;
		238	u32 handle;
		239
		240	size = roundup(size, PAGE_SIZE);
2342	Serge	241	if (size == 0)
		242	return -EINVAL;
2332	Serge	243
		244	/* Allocate the new object */
		245	obj = i915_gem_alloc_object(dev, size);
		246	if (obj == NULL)
		247	return -ENOMEM;
		248
		249	ret = drm_gem_handle_create(file, &obj->base, &handle);
		250	if (ret) {
		251	drm_gem_object_release(&obj->base);
		252	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		253	kfree(obj);
		254	return ret;
		255	}
		256
		257	/* drop reference from allocate - handle holds it now */
		258	drm_gem_object_unreference(&obj->base);
2351	Serge	259	trace_i915_gem_object_create(obj);
2332	Serge	260
		261	*handle_p = handle;
3290	Serge	262
3298	Serge	263	// printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle);
3290	Serge	264
2332	Serge	265	return 0;
		266	}
		267
		268	int
		269	i915_gem_dumb_create(struct drm_file *file,
		270	struct drm_device *dev,
		271	struct drm_mode_create_dumb *args)
		272	{
		273	/* have to work out size/pitch and return them */
		274	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		275	args->size = args->pitch * args->height;
		276	return i915_gem_create(file, dev,
		277	args->size, &args->handle);
		278	}
		279
		280	int i915_gem_dumb_destroy(struct drm_file *file,
		281	struct drm_device *dev,
		282	uint32_t handle)
		283	{
		284	return drm_gem_handle_delete(file, handle);
		285	}
		286
2326	Serge	287	/**
2332	Serge	288	* Creates a new mm object and returns a handle to it.
		289	*/
		290	int
		291	i915_gem_create_ioctl(struct drm_device dev, void data,
		292	struct drm_file *file)
		293	{
		294	struct drm_i915_gem_create *args = data;
3031	serge	295
2332	Serge	296	return i915_gem_create(file, dev,
		297	args->size, &args->handle);
		298	}
		299
		300	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		301	{
		302	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		303
		304	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		305	obj->tiling_mode != I915_TILING_NONE;
		306	}
3260	Serge	307	#if 0
2332	Serge	308
3031	serge	309	static inline int
		310	__copy_to_user_swizzled(char __user *cpu_vaddr,
		311	const char *gpu_vaddr, int gpu_offset,
2332	Serge	312	int length)
		313	{
3031	serge	314	int ret, cpu_offset = 0;
2332	Serge	315
3031	serge	316	while (length > 0) {
		317	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		318	int this_length = min(cacheline_end - gpu_offset, length);
		319	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	320
3031	serge	321	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		322	gpu_vaddr + swizzled_gpu_offset,
		323	this_length);
		324	if (ret)
		325	return ret + length;
2332	Serge	326
3031	serge	327	cpu_offset += this_length;
		328	gpu_offset += this_length;
		329	length -= this_length;
		330	}
		331
		332	return 0;
2332	Serge	333	}
		334
3031	serge	335	static inline int
		336	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		337	const char __user *cpu_vaddr,
		338	int length)
2332	Serge	339	{
3031	serge	340	int ret, cpu_offset = 0;
2332	Serge	341
		342	while (length > 0) {
		343	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		344	int this_length = min(cacheline_end - gpu_offset, length);
		345	int swizzled_gpu_offset = gpu_offset ^ 64;
		346
3031	serge	347	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	348	cpu_vaddr + cpu_offset,
		349	this_length);
3031	serge	350	if (ret)
		351	return ret + length;
		352
2332	Serge	353	cpu_offset += this_length;
		354	gpu_offset += this_length;
		355	length -= this_length;
		356	}
		357
3031	serge	358	return 0;
2332	Serge	359	}
		360
3031	serge	361	/* Per-page copy function for the shmem pread fastpath.
		362	* Flushes invalid cachelines before reading the target if
		363	* needs_clflush is set. */
2332	Serge	364	static int
3031	serge	365	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		366	char __user *user_data,
		367	bool page_do_bit17_swizzling, bool needs_clflush)
		368	{
		369	char *vaddr;
		370	int ret;
		371
		372	if (unlikely(page_do_bit17_swizzling))
		373	return -EINVAL;
		374
		375	vaddr = kmap_atomic(page);
		376	if (needs_clflush)
		377	drm_clflush_virt_range(vaddr + shmem_page_offset,
		378	page_length);
		379	ret = __copy_to_user_inatomic(user_data,
		380	vaddr + shmem_page_offset,
		381	page_length);
		382	kunmap_atomic(vaddr);
		383
		384	return ret ? -EFAULT : 0;
		385	}
		386
		387	static void
		388	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		389	bool swizzled)
		390	{
		391	if (unlikely(swizzled)) {
		392	unsigned long start = (unsigned long) addr;
		393	unsigned long end = (unsigned long) addr + length;
		394
		395	/* For swizzling simply ensure that we always flush both
		396	* channels. Lame, but simple and it works. Swizzled
		397	* pwrite/pread is far from a hotpath - current userspace
		398	* doesn't use it at all. */
		399	start = round_down(start, 128);
		400	end = round_up(end, 128);
		401
		402	drm_clflush_virt_range((void *)start, end - start);
		403	} else {
		404	drm_clflush_virt_range(addr, length);
		405	}
		406
		407	}
		408
		409	/* Only difference to the fast-path function is that this can handle bit17
		410	* and uses non-atomic copy and kmap functions. */
		411	static int
		412	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		413	char __user *user_data,
		414	bool page_do_bit17_swizzling, bool needs_clflush)
		415	{
		416	char *vaddr;
		417	int ret;
		418
		419	vaddr = kmap(page);
		420	if (needs_clflush)
		421	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		422	page_length,
		423	page_do_bit17_swizzling);
		424
		425	if (page_do_bit17_swizzling)
		426	ret = __copy_to_user_swizzled(user_data,
		427	vaddr, shmem_page_offset,
		428	page_length);
		429	else
		430	ret = __copy_to_user(user_data,
		431	vaddr + shmem_page_offset,
		432	page_length);
		433	kunmap(page);
		434
		435	return ret ? - EFAULT : 0;
		436	}
		437
		438	static int
		439	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	440	struct drm_i915_gem_object *obj,
		441	struct drm_i915_gem_pread *args,
		442	struct drm_file *file)
		443	{
3031	serge	444	char __user *user_data;
2332	Serge	445	ssize_t remain;
		446	loff_t offset;
3031	serge	447	int shmem_page_offset, page_length, ret = 0;
		448	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		449	int hit_slowpath = 0;
		450	int prefaulted = 0;
		451	int needs_clflush = 0;
		452	struct scatterlist *sg;
		453	int i;
2332	Serge	454
		455	user_data = (char __user *) (uintptr_t) args->data_ptr;
		456	remain = args->size;
		457
3031	serge	458	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		459
		460	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		461	/* If we're not in the cpu read domain, set ourself into the gtt
		462	* read domain and manually flush cachelines (if required). This
		463	* optimizes for the case when the gpu will dirty the data
		464	* anyway again before the next pread happens. */
		465	if (obj->cache_level == I915_CACHE_NONE)
		466	needs_clflush = 1;
		467	if (obj->gtt_space) {
		468	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		469	if (ret)
		470	return ret;
		471	}
		472	}
		473
		474	ret = i915_gem_object_get_pages(obj);
		475	if (ret)
		476	return ret;
		477
		478	i915_gem_object_pin_pages(obj);
		479
2332	Serge	480	offset = args->offset;
		481
3031	serge	482	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	483	struct page *page;
		484
3031	serge	485	if (i < offset >> PAGE_SHIFT)
		486	continue;
		487
		488	if (remain <= 0)
		489	break;
		490
2332	Serge	491	/* Operation in this page
		492	*
3031	serge	493	* shmem_page_offset = offset within page in shmem file
2332	Serge	494	* page_length = bytes to copy for this page
		495	*/
3031	serge	496	shmem_page_offset = offset_in_page(offset);
2332	Serge	497	page_length = remain;
3031	serge	498	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		499	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	500
3031	serge	501	page = sg_page(sg);
		502	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		503	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	504
3031	serge	505	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		506	user_data, page_do_bit17_swizzling,
		507	needs_clflush);
		508	if (ret == 0)
		509	goto next_page;
2332	Serge	510
3031	serge	511	hit_slowpath = 1;
		512	mutex_unlock(&dev->struct_mutex);
		513
		514	if (!prefaulted) {
		515	ret = fault_in_multipages_writeable(user_data, remain);
		516	/* Userspace is tricking us, but we've already clobbered
		517	* its pages with the prefault and promised to write the
		518	* data up to the first fault. Hence ignore any errors
		519	* and just continue. */
		520	(void)ret;
		521	prefaulted = 1;
		522	}
		523
		524	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		525	user_data, page_do_bit17_swizzling,
		526	needs_clflush);
		527
		528	mutex_lock(&dev->struct_mutex);
		529
		530	next_page:
2332	Serge	531	mark_page_accessed(page);
3031	serge	532
2332	Serge	533	if (ret)
3031	serge	534	goto out;
2332	Serge	535
		536	remain -= page_length;
		537	user_data += page_length;
		538	offset += page_length;
		539	}
		540
3031	serge	541	out:
		542	i915_gem_object_unpin_pages(obj);
		543
		544	if (hit_slowpath) {
		545	/* Fixup: Kill any reinstated backing storage pages */
		546	if (obj->madv == __I915_MADV_PURGED)
		547	i915_gem_object_truncate(obj);
		548	}
		549
		550	return ret;
2332	Serge	551	}
		552
		553	/**
3031	serge	554	* Reads data from the object referenced by handle.
		555	*
		556	* On error, the contents of *data are undefined.
2332	Serge	557	*/
3031	serge	558	int
		559	i915_gem_pread_ioctl(struct drm_device dev, void data,
		560	struct drm_file *file)
		561	{
		562	struct drm_i915_gem_pread *args = data;
		563	struct drm_i915_gem_object *obj;
		564	int ret = 0;
		565
		566	if (args->size == 0)
		567	return 0;
		568
		569	if (!access_ok(VERIFY_WRITE,
		570	(char __user *)(uintptr_t)args->data_ptr,
		571	args->size))
		572	return -EFAULT;
		573
		574	ret = i915_mutex_lock_interruptible(dev);
		575	if (ret)
		576	return ret;
		577
		578	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		579	if (&obj->base == NULL) {
		580	ret = -ENOENT;
		581	goto unlock;
		582	}
		583
		584	/* Bounds check source. */
		585	if (args->offset > obj->base.size \|\|
		586	args->size > obj->base.size - args->offset) {
		587	ret = -EINVAL;
		588	goto out;
		589	}
		590
		591	/* prime objects have no backing filp to GEM pread/pwrite
		592	* pages from.
		593	*/
		594	if (!obj->base.filp) {
		595	ret = -EINVAL;
		596	goto out;
		597	}
		598
		599	trace_i915_gem_object_pread(obj, args->offset, args->size);
		600
		601	ret = i915_gem_shmem_pread(dev, obj, args, file);
		602
		603	out:
		604	drm_gem_object_unreference(&obj->base);
		605	unlock:
		606	mutex_unlock(&dev->struct_mutex);
		607	return ret;
		608	}
		609
		610	/* This is the fast write path which cannot handle
		611	* page faults in the source data
		612	*/
		613
		614	static inline int
		615	fast_user_write(struct io_mapping *mapping,
		616	loff_t page_base, int page_offset,
		617	char __user *user_data,
		618	int length)
		619	{
		620	void __iomem *vaddr_atomic;
		621	void *vaddr;
		622	unsigned long unwritten;
		623
		624	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		625	/* We can use the cpu mem copy function because this is X86. */
		626	vaddr = (void __force*)vaddr_atomic + page_offset;
		627	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		628	user_data, length);
		629	io_mapping_unmap_atomic(vaddr_atomic);
		630	return unwritten;
		631	}
3260	Serge	632	#endif
3031	serge	633
3260	Serge	634	#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
3031	serge	635	/**
		636	* This is the fast pwrite path, where we copy the data directly from the
		637	* user into the GTT, uncached.
		638	*/
2332	Serge	639	static int
3031	serge	640	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		641	struct drm_i915_gem_object *obj,
		642	struct drm_i915_gem_pwrite *args,
		643	struct drm_file *file)
2332	Serge	644	{
3031	serge	645	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	646	ssize_t remain;
3031	serge	647	loff_t offset, page_base;
		648	char __user *user_data;
		649	int page_offset, page_length, ret;
3260	Serge	650	char *vaddr;
2332	Serge	651
3031	serge	652	ret = i915_gem_object_pin(obj, 0, true, true);
		653	if (ret)
		654	goto out;
		655
		656	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		657	if (ret)
		658	goto out_unpin;
		659
		660	ret = i915_gem_object_put_fence(obj);
		661	if (ret)
		662	goto out_unpin;
		663
3260	Serge	664	vaddr = AllocKernelSpace(4096);
		665	if(vaddr == NULL)
		666	{
		667	ret = -ENOSPC;
		668	goto out_unpin;
		669	};
		670
3031	serge	671	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	672	remain = args->size;
		673
3031	serge	674	offset = obj->gtt_offset + args->offset;
2332	Serge	675
3031	serge	676	while (remain > 0) {
		677	/* Operation in this page
		678	*
		679	* page_base = page offset within aperture
		680	* page_offset = offset within page
		681	* page_length = bytes to copy for this page
		682	*/
		683	page_base = offset & PAGE_MASK;
		684	page_offset = offset_in_page(offset);
		685	page_length = remain;
		686	if ((page_offset + remain) > PAGE_SIZE)
		687	page_length = PAGE_SIZE - page_offset;
2332	Serge	688
3260	Serge	689	MapPage(vaddr, page_base, PG_SW\|PG_NOCACHE);
3031	serge	690
3260	Serge	691	memcpy(vaddr+page_offset, user_data, page_length);
		692
3031	serge	693	remain -= page_length;
		694	user_data += page_length;
		695	offset += page_length;
2332	Serge	696	}
		697
3260	Serge	698	FreeKernelSpace(vaddr);
		699
3031	serge	700	out_unpin:
		701	i915_gem_object_unpin(obj);
		702	out:
3260	Serge	703	printf("% s ret = %d\n", __FUNCTION__, ret);
		704
3031	serge	705	return ret;
		706	}
		707
		708	/* Per-page copy function for the shmem pwrite fastpath.
		709	* Flushes invalid cachelines before writing to the target if
		710	* needs_clflush_before is set and flushes out any written cachelines after
		711	* writing if needs_clflush is set. */
		712	static int
		713	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		714	char __user *user_data,
		715	bool page_do_bit17_swizzling,
		716	bool needs_clflush_before,
		717	bool needs_clflush_after)
		718	{
		719	char *vaddr;
3260	Serge	720	int ret = 0;
3031	serge	721
		722	if (unlikely(page_do_bit17_swizzling))
		723	return -EINVAL;
		724
3260	Serge	725	vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW);
3031	serge	726	if (needs_clflush_before)
		727	drm_clflush_virt_range(vaddr + shmem_page_offset,
		728	page_length);
3260	Serge	729	memcpy(vaddr + shmem_page_offset,
3031	serge	730	user_data,
		731	page_length);
		732	if (needs_clflush_after)
		733	drm_clflush_virt_range(vaddr + shmem_page_offset,
		734	page_length);
3260	Serge	735	FreeKernelSpace(vaddr);
3031	serge	736
		737	return ret ? -EFAULT : 0;
		738	}
3260	Serge	739	#if 0
3031	serge	740
		741	/* Only difference to the fast-path function is that this can handle bit17
		742	* and uses non-atomic copy and kmap functions. */
		743	static int
		744	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		745	char __user *user_data,
		746	bool page_do_bit17_swizzling,
		747	bool needs_clflush_before,
		748	bool needs_clflush_after)
		749	{
		750	char *vaddr;
		751	int ret;
		752
		753	vaddr = kmap(page);
		754	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		755	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		756	page_length,
		757	page_do_bit17_swizzling);
		758	if (page_do_bit17_swizzling)
		759	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		760	user_data,
		761	page_length);
		762	else
		763	ret = __copy_from_user(vaddr + shmem_page_offset,
		764	user_data,
		765	page_length);
		766	if (needs_clflush_after)
		767	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		768	page_length,
		769	page_do_bit17_swizzling);
		770	kunmap(page);
		771
		772	return ret ? -EFAULT : 0;
		773	}
3260	Serge	774	#endif
3031	serge	775
3260	Serge	776
3031	serge	777	static int
		778	i915_gem_shmem_pwrite(struct drm_device *dev,
		779	struct drm_i915_gem_object *obj,
		780	struct drm_i915_gem_pwrite *args,
		781	struct drm_file *file)
		782	{
		783	ssize_t remain;
		784	loff_t offset;
		785	char __user *user_data;
		786	int shmem_page_offset, page_length, ret = 0;
		787	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		788	int hit_slowpath = 0;
		789	int needs_clflush_after = 0;
		790	int needs_clflush_before = 0;
		791	int i;
		792	struct scatterlist *sg;
		793
		794	user_data = (char __user *) (uintptr_t) args->data_ptr;
		795	remain = args->size;
		796
		797	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		798
		799	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		800	/* If we're not in the cpu write domain, set ourself into the gtt
		801	* write domain and manually flush cachelines (if required). This
		802	* optimizes for the case when the gpu will use the data
		803	* right away and we therefore have to clflush anyway. */
		804	if (obj->cache_level == I915_CACHE_NONE)
		805	needs_clflush_after = 1;
		806	if (obj->gtt_space) {
		807	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		808	if (ret)
		809	return ret;
		810	}
		811	}
		812	/* Same trick applies for invalidate partially written cachelines before
		813	* writing. */
		814	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		815	&& obj->cache_level == I915_CACHE_NONE)
		816	needs_clflush_before = 1;
		817
		818	ret = i915_gem_object_get_pages(obj);
2332	Serge	819	if (ret)
3031	serge	820	return ret;
2332	Serge	821
3031	serge	822	i915_gem_object_pin_pages(obj);
2332	Serge	823
		824	offset = args->offset;
3031	serge	825	obj->dirty = 1;
2332	Serge	826
3031	serge	827	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	828	struct page *page;
3031	serge	829	int partial_cacheline_write;
2332	Serge	830
3031	serge	831	if (i < offset >> PAGE_SHIFT)
		832	continue;
		833
		834	if (remain <= 0)
		835	break;
		836
2332	Serge	837	/* Operation in this page
		838	*
		839	* shmem_page_offset = offset within page in shmem file
		840	* page_length = bytes to copy for this page
		841	*/
		842	shmem_page_offset = offset_in_page(offset);
		843
		844	page_length = remain;
		845	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		846	page_length = PAGE_SIZE - shmem_page_offset;
		847
3031	serge	848	/* If we don't overwrite a cacheline completely we need to be
		849	* careful to have up-to-date data by first clflushing. Don't
		850	* overcomplicate things and flush the entire patch. */
		851	partial_cacheline_write = needs_clflush_before &&
		852	((shmem_page_offset \| page_length)
3260	Serge	853	& (x86_clflush_size - 1));
2332	Serge	854
3031	serge	855	page = sg_page(sg);
		856	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		857	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	858
3031	serge	859	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		860	user_data, page_do_bit17_swizzling,
		861	partial_cacheline_write,
		862	needs_clflush_after);
		863	if (ret == 0)
		864	goto next_page;
		865
		866	hit_slowpath = 1;
		867	mutex_unlock(&dev->struct_mutex);
3260	Serge	868	dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__);
3031	serge	869
3260	Serge	870	// ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		871	// user_data, page_do_bit17_swizzling,
		872	// partial_cacheline_write,
		873	// needs_clflush_after);
		874
3031	serge	875	mutex_lock(&dev->struct_mutex);
		876
		877	next_page:
2332	Serge	878
3031	serge	879	if (ret)
		880	goto out;
		881
2332	Serge	882	remain -= page_length;
3031	serge	883	user_data += page_length;
2332	Serge	884	offset += page_length;
		885	}
		886
		887	out:
3031	serge	888	i915_gem_object_unpin_pages(obj);
		889
		890	if (hit_slowpath) {
		891	/* Fixup: Kill any reinstated backing storage pages */
		892	if (obj->madv == __I915_MADV_PURGED)
		893	i915_gem_object_truncate(obj);
		894	/* and flush dirty cachelines in case the object isn't in the cpu write
		895	* domain anymore. */
		896	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		897	i915_gem_clflush_object(obj);
3243	Serge	898	i915_gem_chipset_flush(dev);
3031	serge	899	}
2332	Serge	900	}
		901
3031	serge	902	if (needs_clflush_after)
3243	Serge	903	i915_gem_chipset_flush(dev);
3031	serge	904
2332	Serge	905	return ret;
		906	}
3031	serge	907
		908	/**
		909	* Writes data to the object referenced by handle.
		910	*
		911	* On error, the contents of the buffer that were to be modified are undefined.
		912	*/
		913	int
		914	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		915	struct drm_file *file)
		916	{
		917	struct drm_i915_gem_pwrite *args = data;
		918	struct drm_i915_gem_object *obj;
		919	int ret;
		920
		921	if (args->size == 0)
		922	return 0;
		923
		924	ret = i915_mutex_lock_interruptible(dev);
		925	if (ret)
		926	return ret;
		927
		928	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		929	if (&obj->base == NULL) {
		930	ret = -ENOENT;
		931	goto unlock;
		932	}
		933
		934	/* Bounds check destination. */
		935	if (args->offset > obj->base.size \|\|
		936	args->size > obj->base.size - args->offset) {
		937	ret = -EINVAL;
		938	goto out;
		939	}
		940
		941	/* prime objects have no backing filp to GEM pread/pwrite
		942	* pages from.
		943	*/
		944	if (!obj->base.filp) {
		945	ret = -EINVAL;
		946	goto out;
		947	}
		948
		949	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		950
		951	ret = -EFAULT;
		952	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		953	* it would end up going through the fenced access, and we'll get
		954	* different detiling behavior between reading and writing.
		955	* pread/pwrite currently are reading and writing from the CPU
		956	* perspective, requiring manual detiling by the client.
		957	*/
3260	Serge	958	// if (obj->phys_obj) {
		959	// ret = i915_gem_phys_pwrite(dev, obj, args, file);
		960	// goto out;
		961	// }
3031	serge	962
		963	if (obj->cache_level == I915_CACHE_NONE &&
		964	obj->tiling_mode == I915_TILING_NONE &&
		965	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		966	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		967	/* Note that the gtt paths might fail with non-page-backed user
		968	* pointers (e.g. gtt mappings when moving data between
		969	* textures). Fallback to the shmem path in that case. */
		970	}
		971
		972	if (ret == -EFAULT \|\| ret == -ENOSPC)
3260	Serge	973	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
3031	serge	974
		975	out:
		976	drm_gem_object_unreference(&obj->base);
		977	unlock:
		978	mutex_unlock(&dev->struct_mutex);
		979	return ret;
		980	}
		981
		982	int
		983	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		984	bool interruptible)
		985	{
		986	if (atomic_read(&dev_priv->mm.wedged)) {
		987	struct completion *x = &dev_priv->error_completion;
		988	bool recovery_complete;
		989	unsigned long flags;
2332	Serge	990
3031	serge	991	/* Give the error handler a chance to run. */
		992	spin_lock_irqsave(&x->wait.lock, flags);
		993	recovery_complete = x->done > 0;
		994	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	995
3031	serge	996	/* Non-interruptible callers can't handle -EAGAIN, hence return
		997	* -EIO unconditionally for these. */
		998	if (!interruptible)
		999	return -EIO;
2332	Serge	1000
3031	serge	1001	/* Recovery complete, but still wedged means reset failure. */
		1002	if (recovery_complete)
		1003	return -EIO;
2332	Serge	1004
3031	serge	1005	return -EAGAIN;
		1006	}
2332	Serge	1007
3031	serge	1008	return 0;
		1009	}
2332	Serge	1010
3031	serge	1011	/*
		1012	* Compare seqno against outstanding lazy request. Emit a request if they are
		1013	* equal.
		1014	*/
		1015	static int
		1016	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1017	{
		1018	int ret;
2332	Serge	1019
3031	serge	1020	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1021
3031	serge	1022	ret = 0;
		1023	if (seqno == ring->outstanding_lazy_request)
		1024	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1025
3031	serge	1026	return ret;
		1027	}
2332	Serge	1028
3031	serge	1029	/**
		1030	* __wait_seqno - wait until execution of seqno has finished
		1031	* @ring: the ring expected to report seqno
		1032	* @seqno: duh!
		1033	* @interruptible: do an interruptible wait (normally yes)
		1034	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1035	*
		1036	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1037	* errno with remaining time filled in timeout argument.
		1038	*/
		1039	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1040	bool interruptible, struct timespec *timeout)
		1041	{
		1042	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1043	struct timespec before, now, wait_time={1,0};
		1044	unsigned long timeout_jiffies;
		1045	long end;
		1046	bool wait_forever = true;
		1047	int ret;
2332	Serge	1048
3031	serge	1049	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1050	return 0;
2332	Serge	1051
3031	serge	1052	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1053
3031	serge	1054	if (timeout != NULL) {
		1055	wait_time = *timeout;
		1056	wait_forever = false;
		1057	}
2332	Serge	1058
3266	Serge	1059	timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1060
3031	serge	1061	if (WARN_ON(!ring->irq_get(ring)))
		1062	return -ENODEV;
2332	Serge	1063
3031	serge	1064	/* Record current time in case interrupted by signal, or wedged * */
		1065	getrawmonotonic(&before);
2332	Serge	1066
3031	serge	1067	#define EXIT_COND \
		1068	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1069	atomic_read(&dev_priv->mm.wedged))
		1070	do {
3266	Serge	1071	if (interruptible)
		1072	end = wait_event_interruptible_timeout(ring->irq_queue,
		1073	EXIT_COND,
		1074	timeout_jiffies);
		1075	else
3031	serge	1076	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1077	timeout_jiffies);
2332	Serge	1078
3031	serge	1079	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1080	if (ret)
		1081	end = ret;
		1082	} while (end == 0 && wait_forever);
2332	Serge	1083
3031	serge	1084	getrawmonotonic(&now);
2332	Serge	1085
3031	serge	1086	ring->irq_put(ring);
		1087	trace_i915_gem_request_wait_end(ring, seqno);
		1088	#undef EXIT_COND
2332	Serge	1089
3031	serge	1090	if (timeout) {
		1091	// struct timespec sleep_time = timespec_sub(now, before);
		1092	// timeout = timespec_sub(timeout, sleep_time);
		1093	}
2332	Serge	1094
3031	serge	1095	switch (end) {
		1096	case -EIO:
		1097	case -EAGAIN: /* Wedged */
		1098	case -ERESTARTSYS: /* Signal */
		1099	return (int)end;
		1100	case 0: /* Timeout */
3266	Serge	1101	if (timeout)
		1102	set_normalized_timespec(timeout, 0, 0);
3031	serge	1103	return -ETIME;
		1104	default: /* Completed */
		1105	WARN_ON(end < 0); /* We're not aware of other errors */
		1106	return 0;
		1107	}
		1108	}
2332	Serge	1109
3031	serge	1110	/**
		1111	* Waits for a sequence number to be signaled, and cleans up the
		1112	* request and object lists appropriately for that event.
		1113	*/
		1114	int
		1115	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1116	{
		1117	struct drm_device *dev = ring->dev;
		1118	struct drm_i915_private *dev_priv = dev->dev_private;
		1119	bool interruptible = dev_priv->mm.interruptible;
		1120	int ret;
2332	Serge	1121
3031	serge	1122	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1123	BUG_ON(seqno == 0);
2332	Serge	1124
3031	serge	1125	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1126	if (ret)
		1127	return ret;
2332	Serge	1128
3031	serge	1129	ret = i915_gem_check_olr(ring, seqno);
		1130	if (ret)
		1131	return ret;
2332	Serge	1132
3031	serge	1133	return __wait_seqno(ring, seqno, interruptible, NULL);
		1134	}
2332	Serge	1135
3031	serge	1136	/**
		1137	* Ensures that all rendering to the object has completed and the object is
		1138	* safe to unbind from the GTT or access from the CPU.
		1139	*/
		1140	static __must_check int
		1141	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1142	bool readonly)
		1143	{
		1144	struct intel_ring_buffer *ring = obj->ring;
		1145	u32 seqno;
		1146	int ret;
2332	Serge	1147
3031	serge	1148	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1149	if (seqno == 0)
		1150	return 0;
2332	Serge	1151
3031	serge	1152	ret = i915_wait_seqno(ring, seqno);
		1153	if (ret)
		1154	return ret;
2332	Serge	1155
3031	serge	1156	i915_gem_retire_requests_ring(ring);
2332	Serge	1157
3031	serge	1158	/* Manually manage the write flush as we may have not yet
		1159	* retired the buffer.
		1160	*/
		1161	if (obj->last_write_seqno &&
		1162	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1163	obj->last_write_seqno = 0;
		1164	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1165	}
2332	Serge	1166
3031	serge	1167	return 0;
		1168	}
2332	Serge	1169
3260	Serge	1170	/* A nonblocking variant of the above wait. This is a highly dangerous routine
		1171	* as the object state may change during this call.
		1172	*/
		1173	static __must_check int
		1174	i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
		1175	bool readonly)
		1176	{
		1177	struct drm_device *dev = obj->base.dev;
		1178	struct drm_i915_private *dev_priv = dev->dev_private;
		1179	struct intel_ring_buffer *ring = obj->ring;
		1180	u32 seqno;
		1181	int ret;
2332	Serge	1182
3260	Serge	1183	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1184	BUG_ON(!dev_priv->mm.interruptible);
2332	Serge	1185
3260	Serge	1186	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1187	if (seqno == 0)
		1188	return 0;
2332	Serge	1189
3260	Serge	1190	ret = i915_gem_check_wedge(dev_priv, true);
		1191	if (ret)
		1192	return ret;
2332	Serge	1193
3260	Serge	1194	ret = i915_gem_check_olr(ring, seqno);
		1195	if (ret)
		1196	return ret;
2332	Serge	1197
3260	Serge	1198	mutex_unlock(&dev->struct_mutex);
		1199	ret = __wait_seqno(ring, seqno, true, NULL);
		1200	mutex_lock(&dev->struct_mutex);
2332	Serge	1201
3260	Serge	1202	i915_gem_retire_requests_ring(ring);
2332	Serge	1203
3260	Serge	1204	/* Manually manage the write flush as we may have not yet
		1205	* retired the buffer.
		1206	*/
		1207	if (obj->last_write_seqno &&
		1208	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1209	obj->last_write_seqno = 0;
		1210	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1211	}
2332	Serge	1212
3260	Serge	1213	return ret;
		1214	}
2332	Serge	1215
3260	Serge	1216	/**
		1217	* Called when user space prepares to use an object with the CPU, either
		1218	* through the mmap ioctl's mapping or a GTT mapping.
		1219	*/
		1220	int
		1221	i915_gem_set_domain_ioctl(struct drm_device dev, void data,
		1222	struct drm_file *file)
		1223	{
		1224	struct drm_i915_gem_set_domain *args = data;
		1225	struct drm_i915_gem_object *obj;
		1226	uint32_t read_domains = args->read_domains;
		1227	uint32_t write_domain = args->write_domain;
		1228	int ret;
2332	Serge	1229
3260	Serge	1230	/* Only handle setting domains to types used by the CPU. */
		1231	if (write_domain & I915_GEM_GPU_DOMAINS)
		1232	return -EINVAL;
2332	Serge	1233
3260	Serge	1234	if (read_domains & I915_GEM_GPU_DOMAINS)
		1235	return -EINVAL;
2332	Serge	1236
3260	Serge	1237	/* Having something in the write domain implies it's in the read
		1238	* domain, and only that read domain. Enforce that in the request.
		1239	*/
		1240	if (write_domain != 0 && read_domains != write_domain)
		1241	return -EINVAL;
2332	Serge	1242
3260	Serge	1243	ret = i915_mutex_lock_interruptible(dev);
		1244	if (ret)
		1245	return ret;
2332	Serge	1246
3260	Serge	1247	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		1248	if (&obj->base == NULL) {
		1249	ret = -ENOENT;
		1250	goto unlock;
		1251	}
2332	Serge	1252
3260	Serge	1253	/* Try to flush the object off the GPU without holding the lock.
		1254	* We will repeat the flush holding the lock in the normal manner
		1255	* to catch cases where we are gazumped.
		1256	*/
		1257	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
		1258	if (ret)
		1259	goto unref;
2332	Serge	1260
3260	Serge	1261	if (read_domains & I915_GEM_DOMAIN_GTT) {
		1262	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
2332	Serge	1263
3260	Serge	1264	/* Silently promote "you're not bound, there was nothing to do"
		1265	* to success, since the client was just asking us to
		1266	* make sure everything was done.
		1267	*/
		1268	if (ret == -EINVAL)
		1269	ret = 0;
		1270	} else {
		1271	ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
		1272	}
2332	Serge	1273
3260	Serge	1274	unref:
		1275	drm_gem_object_unreference(&obj->base);
		1276	unlock:
		1277	mutex_unlock(&dev->struct_mutex);
		1278	return ret;
		1279	}
2332	Serge	1280
		1281
		1282
		1283
		1284
		1285
3260	Serge	1286	/**
		1287	* Maps the contents of an object, returning the address it is mapped
		1288	* into.
		1289	*
		1290	* While the mapping holds a reference on the contents of the object, it doesn't
		1291	* imply a ref on the object itself.
		1292	*/
		1293	int
		1294	i915_gem_mmap_ioctl(struct drm_device dev, void data,
		1295	struct drm_file *file)
		1296	{
		1297	struct drm_i915_gem_mmap *args = data;
		1298	struct drm_gem_object *obj;
		1299	unsigned long addr = 0;
2332	Serge	1300
3260	Serge	1301	obj = drm_gem_object_lookup(dev, file, args->handle);
		1302	if (obj == NULL)
		1303	return -ENOENT;
		1304
3298	Serge	1305	//dbgprintf("%s offset %lld size %lld\n",
		1306	// __FUNCTION__, args->offset, args->size);
3260	Serge	1307	/* prime objects have no backing filp to GEM mmap
		1308	* pages from.
		1309	*/
		1310	if (!obj->filp) {
		1311	drm_gem_object_unreference_unlocked(obj);
		1312	return -EINVAL;
		1313	}
2332	Serge	1314
3263	Serge	1315	addr = vm_mmap(obj->filp, 0, args->size,
		1316	PROT_READ \| PROT_WRITE, MAP_SHARED,
		1317	args->offset);
3260	Serge	1318	drm_gem_object_unreference_unlocked(obj);
3263	Serge	1319	if (IS_ERR((void *)addr))
		1320	return addr;
2332	Serge	1321
3260	Serge	1322	args->addr_ptr = (uint64_t) addr;
2332	Serge	1323
3263	Serge	1324	return 0;
3260	Serge	1325	}
2332	Serge	1326
		1327
		1328
		1329
		1330
		1331
		1332
		1333
3031	serge	1334
		1335
		1336
		1337
		1338
		1339	/**
		1340	* i915_gem_release_mmap - remove physical page mappings
		1341	* @obj: obj in question
		1342	*
		1343	* Preserve the reservation of the mmapping with the DRM core code, but
		1344	* relinquish ownership of the pages back to the system.
		1345	*
		1346	* It is vital that we remove the page mapping if we have mapped a tiled
		1347	* object through the GTT and then lose the fence register due to
		1348	* resource pressure. Similarly if the object has been moved out of the
		1349	* aperture, than pages mapped into userspace must be revoked. Removing the
		1350	* mapping will then trigger a page fault on the next user access, allowing
		1351	* fixup by i915_gem_fault().
		1352	*/
		1353	void
		1354	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1355	{
		1356	if (!obj->fault_mappable)
		1357	return;
		1358
		1359	if (obj->base.dev->dev_mapping)
		1360	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1361	// (loff_t)obj->base.map_list.hash.key<
		1362	// obj->base.size, 1);
		1363
		1364	obj->fault_mappable = false;
		1365	}
		1366
2332	Serge	1367	static uint32_t
		1368	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1369	{
		1370	uint32_t gtt_size;
		1371
		1372	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1373	tiling_mode == I915_TILING_NONE)
		1374	return size;
		1375
		1376	/* Previous chips need a power-of-two fence region when tiling */
		1377	if (INTEL_INFO(dev)->gen == 3)
		1378	gtt_size = 1024*1024;
		1379	else
		1380	gtt_size = 512*1024;
		1381
		1382	while (gtt_size < size)
		1383	gtt_size <<= 1;
		1384
		1385	return gtt_size;
		1386	}
		1387
		1388	/**
		1389	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1390	* @obj: object to check
		1391	*
		1392	* Return the required GTT alignment for an object, taking into account
		1393	* potential fence register mapping.
		1394	*/
		1395	static uint32_t
		1396	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1397	uint32_t size,
		1398	int tiling_mode)
		1399	{
		1400	/*
		1401	* Minimum alignment is 4k (GTT page size), but might be greater
		1402	* if a fence register is needed for the object.
		1403	*/
		1404	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1405	tiling_mode == I915_TILING_NONE)
		1406	return 4096;
		1407
		1408	/*
		1409	* Previous chips need to be aligned to the size of the smallest
		1410	* fence register that can contain the object.
		1411	*/
		1412	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1413	}
		1414
		1415	/**
		1416	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1417	* unfenced object
		1418	* @dev: the device
		1419	* @size: size of the object
		1420	* @tiling_mode: tiling mode of the object
		1421	*
		1422	* Return the required GTT alignment for an object, only taking into account
		1423	* unfenced tiled surface requirements.
		1424	*/
		1425	uint32_t
		1426	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1427	uint32_t size,
		1428	int tiling_mode)
		1429	{
		1430	/*
		1431	* Minimum alignment is 4k (GTT page size) for sane hw.
		1432	*/
		1433	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1434	tiling_mode == I915_TILING_NONE)
		1435	return 4096;
		1436
		1437	/* Previous hardware however needs to be aligned to a power-of-two
		1438	* tile height. The simplest method for determining this is to reuse
		1439	* the power-of-tile object size.
		1440	*/
		1441	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1442	}
		1443
3031	serge	1444	/* Immediately discard the backing storage */
		1445	static void
		1446	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1447	{
		1448	// struct inode *inode;
2332	Serge	1449
3031	serge	1450	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1451
3263	Serge	1452	if (obj->base.filp == NULL)
		1453	return;
2332	Serge	1454
3031	serge	1455	/* Our goal here is to return as much of the memory as
		1456	* is possible back to the system as we are called from OOM.
		1457	* To do this we must instruct the shmfs to drop all of its
		1458	* backing pages, now.
		1459	*/
		1460	// inode = obj->base.filp->f_path.dentry->d_inode;
		1461	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1462
3031	serge	1463	obj->madv = __I915_MADV_PURGED;
		1464	}
2332	Serge	1465
3031	serge	1466	static inline int
		1467	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1468	{
		1469	return obj->madv == I915_MADV_DONTNEED;
		1470	}
2332	Serge	1471
3031	serge	1472	static void
		1473	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1474	{
3243	Serge	1475	int page_count = obj->base.size / PAGE_SIZE;
		1476	struct scatterlist *sg;
3031	serge	1477	int ret, i;
2332	Serge	1478
3031	serge	1479	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1480
3031	serge	1481	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1482	if (ret) {
		1483	/* In the event of a disaster, abandon all caches and
		1484	* hope for the best.
		1485	*/
		1486	WARN_ON(ret != -EIO);
		1487	i915_gem_clflush_object(obj);
		1488	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1489	}
2332	Serge	1490
3031	serge	1491	if (obj->madv == I915_MADV_DONTNEED)
		1492	obj->dirty = 0;
2332	Serge	1493
3243	Serge	1494	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1495	struct page *page = sg_page(sg);
2332	Serge	1496
3290	Serge	1497	page_cache_release(page);
3243	Serge	1498	}
		1499	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3290	Serge	1500
3031	serge	1501	obj->dirty = 0;
3243	Serge	1502
		1503	sg_free_table(obj->pages);
		1504	kfree(obj->pages);
3031	serge	1505	}
2332	Serge	1506
3031	serge	1507	static int
		1508	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1509	{
		1510	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1511
3243	Serge	1512	if (obj->pages == NULL)
3031	serge	1513	return 0;
2332	Serge	1514
3031	serge	1515	BUG_ON(obj->gtt_space);
		1516
		1517	if (obj->pages_pin_count)
		1518	return -EBUSY;
		1519
3243	Serge	1520	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1521	* array, hence protect them from being reaped by removing them from gtt
		1522	* lists early. */
		1523	list_del(&obj->gtt_list);
		1524
3031	serge	1525	ops->put_pages(obj);
3243	Serge	1526	obj->pages = NULL;
3031	serge	1527
		1528	if (i915_gem_object_is_purgeable(obj))
		1529	i915_gem_object_truncate(obj);
		1530
		1531	return 0;
		1532	}
		1533
		1534
		1535
		1536
		1537
		1538
		1539
		1540
2332	Serge	1541	static int
3031	serge	1542	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1543	{
3260	Serge	1544	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3243	Serge	1545	int page_count, i;
3260	Serge	1546	struct address_space *mapping;
3243	Serge	1547	struct sg_table *st;
		1548	struct scatterlist *sg;
		1549	struct page *page;
		1550	gfp_t gfp;
2332	Serge	1551
3243	Serge	1552	/* Assert that the object is not currently in any GPU domain. As it
		1553	* wasn't in the GTT, there shouldn't be any way it could have been in
		1554	* a GPU cache
2332	Serge	1555	*/
3243	Serge	1556	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1557	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1558
		1559	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1560	if (st == NULL)
		1561	return -ENOMEM;
		1562
2332	Serge	1563	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1564	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1565	sg_free_table(st);
		1566	kfree(st);
2332	Serge	1567	return -ENOMEM;
3243	Serge	1568	}
2332	Serge	1569
3243	Serge	1570	/* Get the list of pages out of our struct file. They'll be pinned
		1571	* at this point until we release them.
		1572	*
		1573	* Fail silently without starting the shrinker
		1574	*/
		1575	for_each_sg(st->sgl, sg, page_count, i) {
3260	Serge	1576	page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp);
		1577	if (IS_ERR(page)) {
		1578	dbgprintf("%s invalid page %p\n", __FUNCTION__, page);
2332	Serge	1579	goto err_pages;
		1580
3260	Serge	1581	}
3243	Serge	1582	sg_set_page(sg, page, PAGE_SIZE, 0);
		1583	}
3031	serge	1584
3243	Serge	1585	obj->pages = st;
3031	serge	1586
3298	Serge	1587	// DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1588
		1589	return 0;
		1590
		1591	err_pages:
3243	Serge	1592	for_each_sg(st->sgl, sg, i, page_count)
		1593	page_cache_release(sg_page(sg));
		1594	sg_free_table(st);
		1595	kfree(st);
		1596	return PTR_ERR(page);
2332	Serge	1597	}
		1598
3031	serge	1599	/* Ensure that the associated pages are gathered from the backing storage
		1600	* and pinned into our object. i915_gem_object_get_pages() may be called
		1601	* multiple times before they are released by a single call to
		1602	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1603	* either as a result of memory pressure (reaping pages under the shrinker)
		1604	* or as the object is itself released.
		1605	*/
		1606	int
		1607	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1608	{
3031	serge	1609	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1610	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1611	int ret;
2332	Serge	1612
3243	Serge	1613	if (obj->pages)
3031	serge	1614	return 0;
2332	Serge	1615
3031	serge	1616	BUG_ON(obj->pages_pin_count);
2332	Serge	1617
3031	serge	1618	ret = ops->get_pages(obj);
		1619	if (ret)
		1620	return ret;
2344	Serge	1621
3031	serge	1622	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1623	return 0;
2332	Serge	1624	}
		1625
		1626	void
		1627	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1628	struct intel_ring_buffer *ring)
2332	Serge	1629	{
		1630	struct drm_device *dev = obj->base.dev;
		1631	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1632	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1633
		1634	BUG_ON(ring == NULL);
		1635	obj->ring = ring;
		1636
		1637	/* Add a reference if we're newly entering the active list. */
		1638	if (!obj->active) {
2344	Serge	1639	drm_gem_object_reference(&obj->base);
2332	Serge	1640	obj->active = 1;
		1641	}
		1642
		1643	/* Move from whatever list we were on to the tail of execution. */
		1644	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1645	list_move_tail(&obj->ring_list, &ring->active_list);
		1646
3031	serge	1647	obj->last_read_seqno = seqno;
		1648
2332	Serge	1649	if (obj->fenced_gpu_access) {
3031	serge	1650	obj->last_fenced_seqno = seqno;
		1651
		1652	/* Bump MRU to take account of the delayed flush */
		1653	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1654	struct drm_i915_fence_reg *reg;
		1655
		1656	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1657	list_move_tail(®->lru_list,
		1658	&dev_priv->mm.fence_list);
		1659	}
2332	Serge	1660	}
		1661	}
		1662
2344	Serge	1663	static void
3031	serge	1664	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1665	{
		1666	struct drm_device *dev = obj->base.dev;
3031	serge	1667	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1668
3031	serge	1669	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1670	BUG_ON(!obj->active);
2332	Serge	1671
3031	serge	1672	if (obj->pin_count) /* are we a framebuffer? */
		1673	intel_mark_fb_idle(obj);
2344	Serge	1674
2352	Serge	1675	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1676
3031	serge	1677	list_del_init(&obj->ring_list);
2352	Serge	1678	obj->ring = NULL;
2344	Serge	1679
3031	serge	1680	obj->last_read_seqno = 0;
		1681	obj->last_write_seqno = 0;
		1682	obj->base.write_domain = 0;
		1683
		1684	obj->last_fenced_seqno = 0;
2352	Serge	1685	obj->fenced_gpu_access = false;
2344	Serge	1686
2352	Serge	1687	obj->active = 0;
		1688	drm_gem_object_unreference(&obj->base);
		1689
		1690	WARN_ON(i915_verify_lists(dev));
		1691	}
		1692
3243	Serge	1693	static int
		1694	i915_gem_handle_seqno_wrap(struct drm_device *dev)
2344	Serge	1695	{
3243	Serge	1696	struct drm_i915_private *dev_priv = dev->dev_private;
		1697	struct intel_ring_buffer *ring;
		1698	int ret, i, j;
2344	Serge	1699
3243	Serge	1700	/* The hardware uses various monotonic 32-bit counters, if we
		1701	* detect that they will wraparound we need to idle the GPU
		1702	* and reset those counters.
		1703	*/
		1704	ret = 0;
		1705	for_each_ring(ring, dev_priv, i) {
		1706	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1707	ret \|= ring->sync_seqno[j] != 0;
		1708	}
		1709	if (ret == 0)
		1710	return ret;
2344	Serge	1711
3243	Serge	1712	ret = i915_gpu_idle(dev);
		1713	if (ret)
		1714	return ret;
		1715
		1716	i915_gem_retire_requests(dev);
		1717	for_each_ring(ring, dev_priv, i) {
		1718	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1719	ring->sync_seqno[j] = 0;
		1720	}
		1721
		1722	return 0;
2344	Serge	1723	}
		1724
3243	Serge	1725	int
		1726	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1727	{
3243	Serge	1728	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1729
3243	Serge	1730	/* reserve 0 for non-seqno */
		1731	if (dev_priv->next_seqno == 0) {
		1732	int ret = i915_gem_handle_seqno_wrap(dev);
		1733	if (ret)
		1734	return ret;
		1735
		1736	dev_priv->next_seqno = 1;
		1737	}
		1738
		1739	*seqno = dev_priv->next_seqno++;
		1740	return 0;
2332	Serge	1741	}
		1742
2352	Serge	1743	int
		1744	i915_add_request(struct intel_ring_buffer *ring,
		1745	struct drm_file *file,
3031	serge	1746	u32 *out_seqno)
2352	Serge	1747	{
		1748	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1749	struct drm_i915_gem_request *request;
		1750	u32 request_ring_position;
2352	Serge	1751	int was_empty;
		1752	int ret;
2332	Serge	1753
3031	serge	1754	/*
		1755	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1756	* after having emitted the batchbuffer command. Hence we need to fix
		1757	* things up similar to emitting the lazy request. The difference here
		1758	* is that the flush _must_ happen before the next request, no matter
		1759	* what.
		1760	*/
		1761	ret = intel_ring_flush_all_caches(ring);
		1762	if (ret)
		1763	return ret;
2332	Serge	1764
3031	serge	1765	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1766	if (request == NULL)
		1767	return -ENOMEM;
		1768
		1769
		1770	/* Record the position of the start of the request so that
		1771	* should we detect the updated seqno part-way through the
		1772	* GPU processing the request, we never over-estimate the
		1773	* position of the head.
		1774	*/
		1775	request_ring_position = intel_ring_get_tail(ring);
		1776
3243	Serge	1777	ret = ring->add_request(ring);
3031	serge	1778	if (ret) {
		1779	kfree(request);
2352	Serge	1780	return ret;
3031	serge	1781	}
2332	Serge	1782
3243	Serge	1783	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1784	request->ring = ring;
3031	serge	1785	request->tail = request_ring_position;
		1786	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1787	was_empty = list_empty(&ring->request_list);
		1788	list_add_tail(&request->list, &ring->request_list);
3031	serge	1789	request->file_priv = NULL;
2332	Serge	1790
3263	Serge	1791	if (file) {
		1792	struct drm_i915_file_private *file_priv = file->driver_priv;
2332	Serge	1793
3263	Serge	1794	spin_lock(&file_priv->mm.lock);
		1795	request->file_priv = file_priv;
		1796	list_add_tail(&request->client_list,
		1797	&file_priv->mm.request_list);
		1798	spin_unlock(&file_priv->mm.lock);
		1799	}
		1800
		1801	trace_i915_gem_request_add(ring, request->seqno);
3031	serge	1802	ring->outstanding_lazy_request = 0;
2332	Serge	1803
2360	Serge	1804	if (!dev_priv->mm.suspended) {
		1805	if (i915_enable_hangcheck) {
2352	Serge	1806	// mod_timer(&dev_priv->hangcheck_timer,
		1807	// jiffies +
		1808	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1809	}
3031	serge	1810	if (was_empty) {
2360	Serge	1811	queue_delayed_work(dev_priv->wq,
		1812	&dev_priv->mm.retire_work, HZ);
3031	serge	1813	intel_mark_busy(dev_priv->dev);
		1814	}
2360	Serge	1815	}
3031	serge	1816
		1817	if (out_seqno)
3243	Serge	1818	*out_seqno = request->seqno;
2352	Serge	1819	return 0;
		1820	}
2332	Serge	1821
3263	Serge	1822	static inline void
		1823	i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
		1824	{
		1825	struct drm_i915_file_private *file_priv = request->file_priv;
2332	Serge	1826
3263	Serge	1827	if (!file_priv)
		1828	return;
2332	Serge	1829
3263	Serge	1830	spin_lock(&file_priv->mm.lock);
		1831	if (request->file_priv) {
		1832	list_del(&request->client_list);
		1833	request->file_priv = NULL;
		1834	}
		1835	spin_unlock(&file_priv->mm.lock);
		1836	}
2332	Serge	1837
3031	serge	1838	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1839	struct intel_ring_buffer *ring)
		1840	{
		1841	while (!list_empty(&ring->request_list)) {
		1842	struct drm_i915_gem_request *request;
2332	Serge	1843
3031	serge	1844	request = list_first_entry(&ring->request_list,
		1845	struct drm_i915_gem_request,
		1846	list);
2332	Serge	1847
3031	serge	1848	list_del(&request->list);
3263	Serge	1849	i915_gem_request_remove_from_client(request);
3031	serge	1850	kfree(request);
		1851	}
2332	Serge	1852
3031	serge	1853	while (!list_empty(&ring->active_list)) {
		1854	struct drm_i915_gem_object *obj;
2332	Serge	1855
3031	serge	1856	obj = list_first_entry(&ring->active_list,
		1857	struct drm_i915_gem_object,
		1858	ring_list);
2332	Serge	1859
3031	serge	1860	i915_gem_object_move_to_inactive(obj);
		1861	}
		1862	}
2332	Serge	1863
3031	serge	1864	static void i915_gem_reset_fences(struct drm_device *dev)
		1865	{
		1866	struct drm_i915_private *dev_priv = dev->dev_private;
		1867	int i;
2332	Serge	1868
3031	serge	1869	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1870	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1871
3031	serge	1872	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1873
3031	serge	1874	if (reg->obj)
		1875	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1876
3031	serge	1877	reg->pin_count = 0;
		1878	reg->obj = NULL;
		1879	INIT_LIST_HEAD(®->lru_list);
		1880	}
2360	Serge	1881
3031	serge	1882	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1883	}
2360	Serge	1884
3031	serge	1885	void i915_gem_reset(struct drm_device *dev)
		1886	{
		1887	struct drm_i915_private *dev_priv = dev->dev_private;
		1888	struct drm_i915_gem_object *obj;
		1889	struct intel_ring_buffer *ring;
		1890	int i;
2360	Serge	1891
3031	serge	1892	for_each_ring(ring, dev_priv, i)
		1893	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1894
3031	serge	1895	/* Move everything out of the GPU domains to ensure we do any
		1896	* necessary invalidation upon reuse.
		1897	*/
		1898	list_for_each_entry(obj,
		1899	&dev_priv->mm.inactive_list,
		1900	mm_list)
		1901	{
		1902	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1903	}
2360	Serge	1904
3031	serge	1905	/* The fence registers are invalidated so clear them out */
		1906	i915_gem_reset_fences(dev);
		1907	}
2360	Serge	1908
2352	Serge	1909	/**
		1910	* This function clears the request list as sequence numbers are passed.
		1911	*/
3031	serge	1912	void
2352	Serge	1913	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1914	{
		1915	uint32_t seqno;
2332	Serge	1916
2352	Serge	1917	if (list_empty(&ring->request_list))
		1918	return;
2332	Serge	1919
2352	Serge	1920	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1921
3031	serge	1922	seqno = ring->get_seqno(ring, true);
2332	Serge	1923
2352	Serge	1924	while (!list_empty(&ring->request_list)) {
		1925	struct drm_i915_gem_request *request;
2332	Serge	1926
2352	Serge	1927	request = list_first_entry(&ring->request_list,
		1928	struct drm_i915_gem_request,
		1929	list);
2332	Serge	1930
2352	Serge	1931	if (!i915_seqno_passed(seqno, request->seqno))
		1932	break;
2332	Serge	1933
2352	Serge	1934	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1935	/* We know the GPU must have read the request to have
		1936	* sent us the seqno + interrupt, so use the position
		1937	* of tail of the request to update the last known position
		1938	* of the GPU head.
		1939	*/
		1940	ring->last_retired_head = request->tail;
2332	Serge	1941
2352	Serge	1942	list_del(&request->list);
3263	Serge	1943	i915_gem_request_remove_from_client(request);
2352	Serge	1944	kfree(request);
		1945	}
2332	Serge	1946
2352	Serge	1947	/* Move any buffers on the active list that are no longer referenced
		1948	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1949	*/
		1950	while (!list_empty(&ring->active_list)) {
		1951	struct drm_i915_gem_object *obj;
2332	Serge	1952
2352	Serge	1953	obj = list_first_entry(&ring->active_list,
		1954	struct drm_i915_gem_object,
		1955	ring_list);
2332	Serge	1956
3031	serge	1957	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1958	break;
2332	Serge	1959
2352	Serge	1960	i915_gem_object_move_to_inactive(obj);
		1961	}
2332	Serge	1962
2352	Serge	1963	if (unlikely(ring->trace_irq_seqno &&
		1964	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1965	ring->irq_put(ring);
		1966	ring->trace_irq_seqno = 0;
		1967	}
2332	Serge	1968
2352	Serge	1969	WARN_ON(i915_verify_lists(ring->dev));
		1970	}
2332	Serge	1971
2352	Serge	1972	void
		1973	i915_gem_retire_requests(struct drm_device *dev)
		1974	{
		1975	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1976	struct intel_ring_buffer *ring;
2352	Serge	1977	int i;
2332	Serge	1978
3031	serge	1979	for_each_ring(ring, dev_priv, i)
		1980	i915_gem_retire_requests_ring(ring);
2352	Serge	1981	}
		1982
2360	Serge	1983	static void
		1984	i915_gem_retire_work_handler(struct work_struct *work)
		1985	{
		1986	drm_i915_private_t *dev_priv;
		1987	struct drm_device *dev;
3031	serge	1988	struct intel_ring_buffer *ring;
2360	Serge	1989	bool idle;
		1990	int i;
2352	Serge	1991
2360	Serge	1992	dev_priv = container_of(work, drm_i915_private_t,
		1993	mm.retire_work.work);
		1994	dev = dev_priv->dev;
2352	Serge	1995
2360	Serge	1996	/* Come back later if the device is busy... */
		1997	if (!mutex_trylock(&dev->struct_mutex)) {
		1998	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	1999	return;
2360	Serge	2000	}
2352	Serge	2001
2360	Serge	2002	i915_gem_retire_requests(dev);
2352	Serge	2003
2360	Serge	2004	/* Send a periodic flush down the ring so we don't hold onto GEM
		2005	* objects indefinitely.
		2006	*/
		2007	idle = true;
3031	serge	2008	for_each_ring(ring, dev_priv, i) {
		2009	if (ring->gpu_caches_dirty)
		2010	i915_add_request(ring, NULL, NULL);
2352	Serge	2011
2360	Serge	2012	idle &= list_empty(&ring->request_list);
		2013	}
2352	Serge	2014
2360	Serge	2015	if (!dev_priv->mm.suspended && !idle)
		2016	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	2017	if (idle)
		2018	intel_mark_idle(dev);
2360	Serge	2019
		2020	mutex_unlock(&dev->struct_mutex);
		2021	}
		2022
2344	Serge	2023	/**
3031	serge	2024	* Ensures that an object will eventually get non-busy by flushing any required
		2025	* write domains, emitting any outstanding lazy request and retiring and
		2026	* completed requests.
2352	Serge	2027	*/
3031	serge	2028	static int
		2029	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	2030	{
3031	serge	2031	int ret;
2352	Serge	2032
3031	serge	2033	if (obj->active) {
		2034	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		2035	if (ret)
		2036	return ret;
2352	Serge	2037
3031	serge	2038	i915_gem_retire_requests_ring(obj->ring);
		2039	}
2352	Serge	2040
3031	serge	2041	return 0;
		2042	}
2352	Serge	2043
3243	Serge	2044	/**
		2045	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		2046	* @DRM_IOCTL_ARGS: standard ioctl arguments
		2047	*
		2048	* Returns 0 if successful, else an error is returned with the remaining time in
		2049	* the timeout parameter.
		2050	* -ETIME: object is still busy after timeout
		2051	* -ERESTARTSYS: signal interrupted the wait
		2052	* -ENONENT: object doesn't exist
		2053	* Also possible, but rare:
		2054	* -EAGAIN: GPU wedged
		2055	* -ENOMEM: damn
		2056	* -ENODEV: Internal IRQ fail
		2057	* -E?: The add request failed
		2058	*
		2059	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		2060	* non-zero timeout parameter the wait ioctl will wait for the given number of
		2061	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		2062	* without holding struct_mutex the object may become re-busied before this
		2063	* function completes. A similar but shorter * race condition exists in the busy
		2064	* ioctl
		2065	*/
2352	Serge	2066
		2067
		2068
		2069
		2070
		2071
		2072
		2073
		2074
3243	Serge	2075
		2076
		2077
2352	Serge	2078	/**
3031	serge	2079	* i915_gem_object_sync - sync an object to a ring.
		2080	*
		2081	* @obj: object which may be in use on another ring.
		2082	* @to: ring we wish to use the object on. May be NULL.
		2083	*
		2084	* This code is meant to abstract object synchronization with the GPU.
		2085	* Calling with NULL implies synchronizing the object with the CPU
		2086	* rather than a particular GPU ring.
		2087	*
		2088	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	2089	*/
		2090	int
3031	serge	2091	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		2092	struct intel_ring_buffer *to)
2344	Serge	2093	{
3031	serge	2094	struct intel_ring_buffer *from = obj->ring;
		2095	u32 seqno;
		2096	int ret, idx;
2332	Serge	2097
3031	serge	2098	if (from == NULL \|\| to == from)
		2099	return 0;
2332	Serge	2100
3031	serge	2101	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		2102	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	2103
3031	serge	2104	idx = intel_ring_sync_index(from, to);
		2105
		2106	seqno = obj->last_read_seqno;
		2107	if (seqno <= from->sync_seqno[idx])
		2108	return 0;
		2109
		2110	ret = i915_gem_check_olr(obj->ring, seqno);
		2111	if (ret)
		2112	return ret;
		2113
		2114	ret = to->sync_to(to, from, seqno);
		2115	if (!ret)
3243	Serge	2116	/* We use last_read_seqno because sync_to()
		2117	* might have just caused seqno wrap under
		2118	* the radar.
		2119	*/
		2120	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	2121
		2122	return ret;
2344	Serge	2123	}
2332	Serge	2124
2344	Serge	2125	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2126	{
		2127	u32 old_write_domain, old_read_domains;
2332	Serge	2128
2344	Serge	2129	/* Act a barrier for all accesses through the GTT */
		2130	mb();
2332	Serge	2131
2344	Serge	2132	/* Force a pagefault for domain tracking on next user access */
		2133	// i915_gem_release_mmap(obj);
2332	Serge	2134
2344	Serge	2135	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2136	return;
2332	Serge	2137
2344	Serge	2138	old_read_domains = obj->base.read_domains;
		2139	old_write_domain = obj->base.write_domain;
2351	Serge	2140
2344	Serge	2141	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2142	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2143
2351	Serge	2144	trace_i915_gem_object_change_domain(obj,
		2145	old_read_domains,
		2146	old_write_domain);
2344	Serge	2147	}
2332	Serge	2148
2344	Serge	2149	/**
		2150	* Unbinds an object from the GTT aperture.
		2151	*/
		2152	int
		2153	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2154	{
3031	serge	2155	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	2156	int ret = 0;
2332	Serge	2157
3263	Serge	2158	if(obj == get_fb_obj())
		2159	return 0;
		2160
2344	Serge	2161	if (obj->gtt_space == NULL)
		2162	return 0;
2332	Serge	2163
3031	serge	2164	if (obj->pin_count)
		2165	return -EBUSY;
2332	Serge	2166
3243	Serge	2167	BUG_ON(obj->pages == NULL);
3031	serge	2168
2344	Serge	2169	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2170	if (ret)
2344	Serge	2171	return ret;
		2172	/* Continue on if we fail due to EIO, the GPU is hung so we
		2173	* should be safe and we need to cleanup or else we might
		2174	* cause memory corruption through use-after-free.
		2175	*/
2332	Serge	2176
2344	Serge	2177	i915_gem_object_finish_gtt(obj);
2332	Serge	2178
2344	Serge	2179	/* release the fence reg _after_ flushing */
		2180	ret = i915_gem_object_put_fence(obj);
3031	serge	2181	if (ret)
2344	Serge	2182	return ret;
2332	Serge	2183
2351	Serge	2184	trace_i915_gem_object_unbind(obj);
2332	Serge	2185
3031	serge	2186	if (obj->has_global_gtt_mapping)
3243	Serge	2187	i915_gem_gtt_unbind_object(obj);
3031	serge	2188	if (obj->has_aliasing_ppgtt_mapping) {
		2189	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2190	obj->has_aliasing_ppgtt_mapping = 0;
		2191	}
		2192	i915_gem_gtt_finish_object(obj);
2332	Serge	2193
3031	serge	2194	list_del(&obj->mm_list);
		2195	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2196	/* Avoid an unnecessary call to unbind on rebind. */
		2197	obj->map_and_fenceable = true;
2332	Serge	2198
2344	Serge	2199	drm_mm_put_block(obj->gtt_space);
		2200	obj->gtt_space = NULL;
		2201	obj->gtt_offset = 0;
2332	Serge	2202
2344	Serge	2203	return 0;
		2204	}
2332	Serge	2205
3031	serge	2206	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2207	{
		2208	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2209	struct intel_ring_buffer *ring;
2344	Serge	2210	int ret, i;
2332	Serge	2211
2344	Serge	2212	/* Flush everything onto the inactive list. */
3031	serge	2213	for_each_ring(ring, dev_priv, i) {
		2214	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2215	if (ret)
		2216	return ret;
3031	serge	2217
3243	Serge	2218	ret = intel_ring_idle(ring);
3031	serge	2219	if (ret)
		2220	return ret;
2344	Serge	2221	}
2332	Serge	2222
2344	Serge	2223	return 0;
		2224	}
2332	Serge	2225
3031	serge	2226	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2227	struct drm_i915_gem_object *obj)
		2228	{
		2229	drm_i915_private_t *dev_priv = dev->dev_private;
		2230	uint64_t val;
2332	Serge	2231
3031	serge	2232	if (obj) {
		2233	u32 size = obj->gtt_space->size;
2332	Serge	2234
3031	serge	2235	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2236	0xfffff000) << 32;
		2237	val \|= obj->gtt_offset & 0xfffff000;
		2238	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2239	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2240
3031	serge	2241	if (obj->tiling_mode == I915_TILING_Y)
		2242	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2243	val \|= I965_FENCE_REG_VALID;
		2244	} else
		2245	val = 0;
2332	Serge	2246
3031	serge	2247	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2248	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2249	}
2332	Serge	2250
3031	serge	2251	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2252	struct drm_i915_gem_object *obj)
		2253	{
		2254	drm_i915_private_t *dev_priv = dev->dev_private;
		2255	uint64_t val;
2332	Serge	2256
3031	serge	2257	if (obj) {
		2258	u32 size = obj->gtt_space->size;
2332	Serge	2259
3031	serge	2260	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2261	0xfffff000) << 32;
		2262	val \|= obj->gtt_offset & 0xfffff000;
		2263	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2264	if (obj->tiling_mode == I915_TILING_Y)
		2265	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2266	val \|= I965_FENCE_REG_VALID;
		2267	} else
		2268	val = 0;
2332	Serge	2269
3031	serge	2270	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2271	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2272	}
2332	Serge	2273
3031	serge	2274	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2275	struct drm_i915_gem_object *obj)
		2276	{
		2277	drm_i915_private_t *dev_priv = dev->dev_private;
		2278	u32 val;
2332	Serge	2279
3031	serge	2280	if (obj) {
		2281	u32 size = obj->gtt_space->size;
		2282	int pitch_val;
		2283	int tile_width;
2332	Serge	2284
3031	serge	2285	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2286	(size & -size) != size \|\|
		2287	(obj->gtt_offset & (size - 1)),
		2288	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2289	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2290
3031	serge	2291	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2292	tile_width = 128;
		2293	else
		2294	tile_width = 512;
2332	Serge	2295
3031	serge	2296	/* Note: pitch better be a power of two tile widths */
		2297	pitch_val = obj->stride / tile_width;
		2298	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2299
3031	serge	2300	val = obj->gtt_offset;
		2301	if (obj->tiling_mode == I915_TILING_Y)
		2302	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2303	val \|= I915_FENCE_SIZE_BITS(size);
		2304	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2305	val \|= I830_FENCE_REG_VALID;
		2306	} else
		2307	val = 0;
2332	Serge	2308
3031	serge	2309	if (reg < 8)
		2310	reg = FENCE_REG_830_0 + reg * 4;
		2311	else
		2312	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2313
3031	serge	2314	I915_WRITE(reg, val);
		2315	POSTING_READ(reg);
		2316	}
2332	Serge	2317
3031	serge	2318	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2319	struct drm_i915_gem_object *obj)
		2320	{
		2321	drm_i915_private_t *dev_priv = dev->dev_private;
		2322	uint32_t val;
2344	Serge	2323
3031	serge	2324	if (obj) {
		2325	u32 size = obj->gtt_space->size;
		2326	uint32_t pitch_val;
2344	Serge	2327
3031	serge	2328	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2329	(size & -size) != size \|\|
		2330	(obj->gtt_offset & (size - 1)),
		2331	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2332	obj->gtt_offset, size);
2344	Serge	2333
3031	serge	2334	pitch_val = obj->stride / 128;
		2335	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2336
3031	serge	2337	val = obj->gtt_offset;
		2338	if (obj->tiling_mode == I915_TILING_Y)
		2339	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2340	val \|= I830_FENCE_SIZE_BITS(size);
		2341	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2342	val \|= I830_FENCE_REG_VALID;
		2343	} else
		2344	val = 0;
		2345
		2346	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2347	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2348	}
		2349
		2350	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2351	struct drm_i915_gem_object *obj)
2332	Serge	2352	{
3031	serge	2353	switch (INTEL_INFO(dev)->gen) {
		2354	case 7:
		2355	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2356	case 5:
		2357	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2358	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2359	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2360	default: break;
		2361	}
2344	Serge	2362	}
		2363
3031	serge	2364	static inline int fence_number(struct drm_i915_private *dev_priv,
		2365	struct drm_i915_fence_reg *fence)
2344	Serge	2366	{
3031	serge	2367	return fence - dev_priv->fence_regs;
		2368	}
2332	Serge	2369
3031	serge	2370	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2371	struct drm_i915_fence_reg *fence,
		2372	bool enable)
		2373	{
		2374	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2375	int reg = fence_number(dev_priv, fence);
2332	Serge	2376
3031	serge	2377	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2378
		2379	if (enable) {
		2380	obj->fence_reg = reg;
		2381	fence->obj = obj;
		2382	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2383	} else {
		2384	obj->fence_reg = I915_FENCE_REG_NONE;
		2385	fence->obj = NULL;
		2386	list_del_init(&fence->lru_list);
2344	Serge	2387	}
3031	serge	2388	}
2344	Serge	2389
3031	serge	2390	static int
		2391	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2392	{
		2393	if (obj->last_fenced_seqno) {
		2394	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2395	if (ret)
		2396	return ret;
2344	Serge	2397
		2398	obj->last_fenced_seqno = 0;
		2399	}
		2400
		2401	/* Ensure that all CPU reads are completed before installing a fence
		2402	* and all writes before removing the fence.
2332	Serge	2403	*/
2344	Serge	2404	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2405	mb();
2332	Serge	2406
3031	serge	2407	obj->fenced_gpu_access = false;
2332	Serge	2408	return 0;
		2409	}
		2410
		2411	int
2344	Serge	2412	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2413	{
3031	serge	2414	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2415	int ret;
		2416
3031	serge	2417	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2418	if (ret)
		2419	return ret;
		2420
3031	serge	2421	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2422	return 0;
2332	Serge	2423
3031	serge	2424	i915_gem_object_update_fence(obj,
		2425	&dev_priv->fence_regs[obj->fence_reg],
		2426	false);
		2427	i915_gem_object_fence_lost(obj);
2344	Serge	2428
2332	Serge	2429	return 0;
		2430	}
		2431
3031	serge	2432	static struct drm_i915_fence_reg *
		2433	i915_find_fence_reg(struct drm_device *dev)
		2434	{
		2435	struct drm_i915_private *dev_priv = dev->dev_private;
		2436	struct drm_i915_fence_reg reg, avail;
		2437	int i;
2332	Serge	2438
3031	serge	2439	/* First try to find a free reg */
		2440	avail = NULL;
		2441	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2442	reg = &dev_priv->fence_regs[i];
		2443	if (!reg->obj)
		2444	return reg;
2332	Serge	2445
3031	serge	2446	if (!reg->pin_count)
		2447	avail = reg;
		2448	}
2332	Serge	2449
3031	serge	2450	if (avail == NULL)
		2451	return NULL;
2332	Serge	2452
3031	serge	2453	/* None available, try to steal one or wait for a user to finish */
		2454	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2455	if (reg->pin_count)
		2456	continue;
2332	Serge	2457
3031	serge	2458	return reg;
		2459	}
2332	Serge	2460
3031	serge	2461	return NULL;
		2462	}
2332	Serge	2463
3031	serge	2464	/**
		2465	* i915_gem_object_get_fence - set up fencing for an object
		2466	* @obj: object to map through a fence reg
		2467	*
		2468	* When mapping objects through the GTT, userspace wants to be able to write
		2469	* to them without having to worry about swizzling if the object is tiled.
		2470	* This function walks the fence regs looking for a free one for @obj,
		2471	* stealing one if it can't find any.
		2472	*
		2473	* It then sets up the reg based on the object's properties: address, pitch
		2474	* and tiling format.
		2475	*
		2476	* For an untiled surface, this removes any existing fence.
		2477	*/
		2478	int
		2479	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2480	{
		2481	struct drm_device *dev = obj->base.dev;
		2482	struct drm_i915_private *dev_priv = dev->dev_private;
		2483	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2484	struct drm_i915_fence_reg *reg;
		2485	int ret;
2332	Serge	2486
3031	serge	2487	/* Have we updated the tiling parameters upon the object and so
		2488	* will need to serialise the write to the associated fence register?
		2489	*/
		2490	if (obj->fence_dirty) {
		2491	ret = i915_gem_object_flush_fence(obj);
		2492	if (ret)
		2493	return ret;
		2494	}
2332	Serge	2495
3031	serge	2496	/* Just update our place in the LRU if our fence is getting reused. */
		2497	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2498	reg = &dev_priv->fence_regs[obj->fence_reg];
		2499	if (!obj->fence_dirty) {
		2500	list_move_tail(®->lru_list,
		2501	&dev_priv->mm.fence_list);
		2502	return 0;
		2503	}
		2504	} else if (enable) {
		2505	reg = i915_find_fence_reg(dev);
		2506	if (reg == NULL)
		2507	return -EDEADLK;
2332	Serge	2508
3031	serge	2509	if (reg->obj) {
		2510	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2511
3031	serge	2512	ret = i915_gem_object_flush_fence(old);
		2513	if (ret)
		2514	return ret;
2332	Serge	2515
3031	serge	2516	i915_gem_object_fence_lost(old);
		2517	}
		2518	} else
		2519	return 0;
2332	Serge	2520
3031	serge	2521	i915_gem_object_update_fence(obj, reg, enable);
		2522	obj->fence_dirty = false;
2332	Serge	2523
3031	serge	2524	return 0;
		2525	}
2332	Serge	2526
3031	serge	2527	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2528	struct drm_mm_node *gtt_space,
		2529	unsigned long cache_level)
		2530	{
		2531	struct drm_mm_node *other;
2332	Serge	2532
3031	serge	2533	/* On non-LLC machines we have to be careful when putting differing
		2534	* types of snoopable memory together to avoid the prefetcher
		2535	* crossing memory domains and dieing.
		2536	*/
		2537	if (HAS_LLC(dev))
		2538	return true;
2332	Serge	2539
3031	serge	2540	if (gtt_space == NULL)
		2541	return true;
2332	Serge	2542
3031	serge	2543	if (list_empty(>t_space->node_list))
		2544	return true;
2332	Serge	2545
3031	serge	2546	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2547	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2548	return false;
2344	Serge	2549
3031	serge	2550	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2551	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2552	return false;
2344	Serge	2553
3031	serge	2554	return true;
		2555	}
2344	Serge	2556
3031	serge	2557	static void i915_gem_verify_gtt(struct drm_device *dev)
		2558	{
		2559	#if WATCH_GTT
		2560	struct drm_i915_private *dev_priv = dev->dev_private;
		2561	struct drm_i915_gem_object *obj;
		2562	int err = 0;
2344	Serge	2563
3031	serge	2564	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2565	if (obj->gtt_space == NULL) {
		2566	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2567	err++;
		2568	continue;
		2569	}
2344	Serge	2570
3031	serge	2571	if (obj->cache_level != obj->gtt_space->color) {
		2572	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2573	obj->gtt_space->start,
		2574	obj->gtt_space->start + obj->gtt_space->size,
		2575	obj->cache_level,
		2576	obj->gtt_space->color);
		2577	err++;
		2578	continue;
		2579	}
2344	Serge	2580
3031	serge	2581	if (!i915_gem_valid_gtt_space(dev,
		2582	obj->gtt_space,
		2583	obj->cache_level)) {
		2584	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2585	obj->gtt_space->start,
		2586	obj->gtt_space->start + obj->gtt_space->size,
		2587	obj->cache_level);
		2588	err++;
		2589	continue;
		2590	}
		2591	}
2344	Serge	2592
3031	serge	2593	WARN_ON(err);
		2594	#endif
2326	Serge	2595	}
		2596
2332	Serge	2597	/**
		2598	* Finds free space in the GTT aperture and binds the object there.
		2599	*/
		2600	static int
		2601	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2602	unsigned alignment,
3031	serge	2603	bool map_and_fenceable,
		2604	bool nonblocking)
2332	Serge	2605	{
		2606	struct drm_device *dev = obj->base.dev;
		2607	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2608	struct drm_mm_node *node;
2332	Serge	2609	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2610	bool mappable, fenceable;
		2611	int ret;
2326	Serge	2612
2332	Serge	2613	if (obj->madv != I915_MADV_WILLNEED) {
3243	Serge	2614	DRM_ERROR("Attempting to bind a purgeable object\n");
2332	Serge	2615	return -EINVAL;
		2616	}
		2617
		2618	fence_size = i915_gem_get_gtt_size(dev,
		2619	obj->base.size,
		2620	obj->tiling_mode);
		2621	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2622	obj->base.size,
		2623	obj->tiling_mode);
		2624	unfenced_alignment =
		2625	i915_gem_get_unfenced_gtt_alignment(dev,
		2626	obj->base.size,
		2627	obj->tiling_mode);
		2628
		2629	if (alignment == 0)
		2630	alignment = map_and_fenceable ? fence_alignment :
		2631	unfenced_alignment;
		2632	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2633	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2634	return -EINVAL;
		2635	}
		2636
		2637	size = map_and_fenceable ? fence_size : obj->base.size;
		2638
		2639	/* If the object is bigger than the entire aperture, reject it early
		2640	* before evicting everything in a vain attempt to find space.
		2641	*/
		2642	if (obj->base.size >
		2643	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2644	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2645	return -E2BIG;
		2646	}
		2647
3031	serge	2648	ret = i915_gem_object_get_pages(obj);
		2649	if (ret)
		2650	return ret;
		2651
3243	Serge	2652	i915_gem_object_pin_pages(obj);
		2653
		2654	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2655	if (node == NULL) {
		2656	i915_gem_object_unpin_pages(obj);
		2657	return -ENOMEM;
		2658	}
		2659
2332	Serge	2660	search_free:
		2661	if (map_and_fenceable)
3243	Serge	2662	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2663	size, alignment, obj->cache_level,
3243	Serge	2664	0, dev_priv->mm.gtt_mappable_end);
2332	Serge	2665	else
3243	Serge	2666	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2667	size, alignment, obj->cache_level);
		2668	if (ret) {
2332	Serge	2669
3243	Serge	2670	i915_gem_object_unpin_pages(obj);
		2671	kfree(node);
2332	Serge	2672	return ret;
		2673	}
3243	Serge	2674	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2675	i915_gem_object_unpin_pages(obj);
		2676	drm_mm_put_block(node);
3031	serge	2677	return -EINVAL;
		2678	}
2332	Serge	2679
3031	serge	2680	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2681	if (ret) {
3243	Serge	2682	i915_gem_object_unpin_pages(obj);
		2683	drm_mm_put_block(node);
2332	Serge	2684	return ret;
		2685	}
		2686
3031	serge	2687	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2688	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2689
3243	Serge	2690	obj->gtt_space = node;
		2691	obj->gtt_offset = node->start;
2332	Serge	2692
		2693	fenceable =
3243	Serge	2694	node->size == fence_size &&
		2695	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2696
		2697	mappable =
		2698	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2699
		2700	obj->map_and_fenceable = mappable && fenceable;
		2701
3243	Serge	2702	i915_gem_object_unpin_pages(obj);
2351	Serge	2703	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2704	i915_gem_verify_gtt(dev);
2332	Serge	2705	return 0;
		2706	}
		2707
		2708	void
		2709	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2710	{
		2711	/* If we don't have a page list set up, then we're not pinned
		2712	* to GPU, and we can ignore the cache flush because it'll happen
		2713	* again at bind time.
		2714	*/
3243	Serge	2715	if (obj->pages == NULL)
2332	Serge	2716	return;
		2717
		2718	/* If the GPU is snooping the contents of the CPU cache,
		2719	* we do not need to manually clear the CPU cache lines. However,
		2720	* the caches are only snooped when the render cache is
		2721	* flushed/invalidated. As we always have to emit invalidations
		2722	* and flushes when moving into and out of the RENDER domain, correct
		2723	* snooping behaviour occurs naturally as the result of our domain
		2724	* tracking.
		2725	*/
		2726	if (obj->cache_level != I915_CACHE_NONE)
		2727	return;
3243	Serge	2728	#if 0
2344	Serge	2729	if(obj->mapped != NULL)
		2730	{
		2731	uint8_t *page_virtual;
		2732	unsigned int i;
2332	Serge	2733
2344	Serge	2734	page_virtual = obj->mapped;
		2735	asm volatile("mfence");
		2736	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2737	clflush(page_virtual + i);
		2738	asm volatile("mfence");
		2739	}
		2740	else
		2741	{
		2742	uint8_t *page_virtual;
		2743	unsigned int i;
		2744	page_virtual = AllocKernelSpace(obj->base.size);
		2745	if(page_virtual != NULL)
		2746	{
3031	serge	2747	dma_addr_t src, dst;
2344	Serge	2748	u32 count;
		2749
		2750	#define page_tabs 0xFDC00000 /* really dirty hack */
		2751
3031	serge	2752	src = obj->pages.page;
		2753	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2754	count = obj->base.size/4096;
		2755
		2756	while(count--)
		2757	{
		2758	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2759	};
		2760
		2761	asm volatile("mfence");
		2762	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2763	clflush(page_virtual + i);
		2764	asm volatile("mfence");
		2765	FreeKernelSpace(page_virtual);
		2766	}
		2767	else
		2768	{
		2769	asm volatile (
		2770	"mfence \n"
		2771	"wbinvd \n" /* this is really ugly */
		2772	"mfence");
		2773	}
		2774	}
3243	Serge	2775	#endif
		2776
2332	Serge	2777	}
		2778
2344	Serge	2779	/** Flushes the GTT write domain for the object if it's dirty. */
		2780	static void
		2781	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2782	{
		2783	uint32_t old_write_domain;
2332	Serge	2784
2344	Serge	2785	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2786	return;
2332	Serge	2787
2344	Serge	2788	/* No actual flushing is required for the GTT write domain. Writes
		2789	* to it immediately go to main memory as far as we know, so there's
		2790	* no chipset flush. It also doesn't land in render cache.
		2791	*
		2792	* However, we do have to enforce the order so that all writes through
		2793	* the GTT land before any writes to the device, such as updates to
		2794	* the GATT itself.
		2795	*/
		2796	wmb();
2332	Serge	2797
2344	Serge	2798	old_write_domain = obj->base.write_domain;
		2799	obj->base.write_domain = 0;
2332	Serge	2800
2351	Serge	2801	trace_i915_gem_object_change_domain(obj,
		2802	obj->base.read_domains,
		2803	old_write_domain);
2344	Serge	2804	}
2332	Serge	2805
		2806	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2807	static void
2332	Serge	2808	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2809	{
		2810	uint32_t old_write_domain;
		2811
		2812	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2813	return;
		2814
		2815	i915_gem_clflush_object(obj);
3243	Serge	2816	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2817	old_write_domain = obj->base.write_domain;
		2818	obj->base.write_domain = 0;
		2819
2351	Serge	2820	trace_i915_gem_object_change_domain(obj,
		2821	obj->base.read_domains,
		2822	old_write_domain);
2332	Serge	2823	}
		2824
		2825	/**
		2826	* Moves a single object to the GTT read, and possibly write domain.
		2827	*
		2828	* This function returns when the move is complete, including waiting on
		2829	* flushes to occur.
		2830	*/
		2831	int
		2832	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2833	{
3031	serge	2834	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2835	uint32_t old_write_domain, old_read_domains;
		2836	int ret;
		2837
		2838	/* Not valid to be called on unbound objects. */
		2839	if (obj->gtt_space == NULL)
		2840	return -EINVAL;
		2841
		2842	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2843	return 0;
		2844
3031	serge	2845	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2846	if (ret)
		2847	return ret;
		2848
		2849	i915_gem_object_flush_cpu_write_domain(obj);
		2850
		2851	old_write_domain = obj->base.write_domain;
		2852	old_read_domains = obj->base.read_domains;
		2853
		2854	/* It should now be out of any other write domains, and we can update
		2855	* the domain values for our changes.
		2856	*/
		2857	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2858	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2859	if (write) {
		2860	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2861	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2862	obj->dirty = 1;
		2863	}
		2864
2351	Serge	2865	trace_i915_gem_object_change_domain(obj,
		2866	old_read_domains,
		2867	old_write_domain);
		2868
3031	serge	2869	/* And bump the LRU for this access */
		2870	if (i915_gem_object_is_inactive(obj))
		2871	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2872
2332	Serge	2873	return 0;
		2874	}
		2875
2335	Serge	2876	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2877	enum i915_cache_level cache_level)
		2878	{
3031	serge	2879	struct drm_device *dev = obj->base.dev;
		2880	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2881	int ret;
2332	Serge	2882
2335	Serge	2883	if (obj->cache_level == cache_level)
		2884	return 0;
2332	Serge	2885
2335	Serge	2886	if (obj->pin_count) {
		2887	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2888	return -EBUSY;
		2889	}
2332	Serge	2890
3031	serge	2891	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2892	ret = i915_gem_object_unbind(obj);
		2893	if (ret)
		2894	return ret;
		2895	}
		2896
2335	Serge	2897	if (obj->gtt_space) {
		2898	ret = i915_gem_object_finish_gpu(obj);
		2899	if (ret)
		2900	return ret;
2332	Serge	2901
2335	Serge	2902	i915_gem_object_finish_gtt(obj);
2332	Serge	2903
2335	Serge	2904	/* Before SandyBridge, you could not use tiling or fence
		2905	* registers with snooped memory, so relinquish any fences
		2906	* currently pointing to our region in the aperture.
		2907	*/
3031	serge	2908	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2909	ret = i915_gem_object_put_fence(obj);
		2910	if (ret)
		2911	return ret;
		2912	}
2332	Serge	2913
3031	serge	2914	if (obj->has_global_gtt_mapping)
		2915	i915_gem_gtt_bind_object(obj, cache_level);
		2916	if (obj->has_aliasing_ppgtt_mapping)
		2917	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2918	obj, cache_level);
		2919
		2920	obj->gtt_space->color = cache_level;
2335	Serge	2921	}
2332	Serge	2922
2335	Serge	2923	if (cache_level == I915_CACHE_NONE) {
		2924	u32 old_read_domains, old_write_domain;
2332	Serge	2925
2335	Serge	2926	/* If we're coming from LLC cached, then we haven't
		2927	* actually been tracking whether the data is in the
		2928	* CPU cache or not, since we only allow one bit set
		2929	* in obj->write_domain and have been skipping the clflushes.
		2930	* Just set it to the CPU cache for now.
		2931	*/
		2932	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2933	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2934
2335	Serge	2935	old_read_domains = obj->base.read_domains;
		2936	old_write_domain = obj->base.write_domain;
2332	Serge	2937
2335	Serge	2938	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2939	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2940
2351	Serge	2941	trace_i915_gem_object_change_domain(obj,
		2942	old_read_domains,
		2943	old_write_domain);
2344	Serge	2944	}
2332	Serge	2945
2335	Serge	2946	obj->cache_level = cache_level;
3031	serge	2947	i915_gem_verify_gtt(dev);
2335	Serge	2948	return 0;
		2949	}
2332	Serge	2950
3260	Serge	2951	int i915_gem_get_caching_ioctl(struct drm_device dev, void data,
		2952	struct drm_file *file)
		2953	{
		2954	struct drm_i915_gem_caching *args = data;
		2955	struct drm_i915_gem_object *obj;
		2956	int ret;
		2957
		2958	ret = i915_mutex_lock_interruptible(dev);
		2959	if (ret)
		2960	return ret;
		2961
		2962	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		2963	if (&obj->base == NULL) {
		2964	ret = -ENOENT;
		2965	goto unlock;
		2966	}
		2967
		2968	args->caching = obj->cache_level != I915_CACHE_NONE;
		2969
		2970	drm_gem_object_unreference(&obj->base);
		2971	unlock:
		2972	mutex_unlock(&dev->struct_mutex);
		2973	return ret;
		2974	}
		2975
		2976	int i915_gem_set_caching_ioctl(struct drm_device dev, void data,
		2977	struct drm_file *file)
		2978	{
		2979	struct drm_i915_gem_caching *args = data;
		2980	struct drm_i915_gem_object *obj;
		2981	enum i915_cache_level level;
		2982	int ret;
		2983
		2984	switch (args->caching) {
		2985	case I915_CACHING_NONE:
		2986	level = I915_CACHE_NONE;
		2987	break;
		2988	case I915_CACHING_CACHED:
		2989	level = I915_CACHE_LLC;
		2990	break;
		2991	default:
		2992	return -EINVAL;
		2993	}
		2994
		2995	ret = i915_mutex_lock_interruptible(dev);
		2996	if (ret)
		2997	return ret;
		2998
		2999	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3000	if (&obj->base == NULL) {
		3001	ret = -ENOENT;
		3002	goto unlock;
		3003	}
		3004
		3005	ret = i915_gem_object_set_cache_level(obj, level);
		3006
		3007	drm_gem_object_unreference(&obj->base);
		3008	unlock:
		3009	mutex_unlock(&dev->struct_mutex);
		3010	return ret;
		3011	}
		3012
2335	Serge	3013	/*
		3014	* Prepare buffer for display plane (scanout, cursors, etc).
		3015	* Can be called from an uninterruptible phase (modesetting) and allows
		3016	* any flushes to be pipelined (for pageflips).
		3017	*/
		3018	int
		3019	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		3020	u32 alignment,
		3021	struct intel_ring_buffer *pipelined)
		3022	{
		3023	u32 old_read_domains, old_write_domain;
		3024	int ret;
2332	Serge	3025
3031	serge	3026	if (pipelined != obj->ring) {
		3027	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	3028	if (ret)
		3029	return ret;
		3030	}
2332	Serge	3031
2335	Serge	3032	/* The display engine is not coherent with the LLC cache on gen6. As
		3033	* a result, we make sure that the pinning that is about to occur is
		3034	* done with uncached PTEs. This is lowest common denominator for all
		3035	* chipsets.
		3036	*
		3037	* However for gen6+, we could do better by using the GFDT bit instead
		3038	* of uncaching, which would allow us to flush all the LLC-cached data
		3039	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		3040	*/
2360	Serge	3041	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		3042	if (ret)
		3043	return ret;
2332	Serge	3044
2335	Serge	3045	/* As the user may map the buffer once pinned in the display plane
		3046	* (e.g. libkms for the bootup splash), we have to ensure that we
		3047	* always use map_and_fenceable for all scanout buffers.
		3048	*/
3031	serge	3049	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	3050	if (ret)
		3051	return ret;
2332	Serge	3052
2335	Serge	3053	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	3054
2335	Serge	3055	old_write_domain = obj->base.write_domain;
		3056	old_read_domains = obj->base.read_domains;
2332	Serge	3057
2335	Serge	3058	/* It should now be out of any other write domains, and we can update
		3059	* the domain values for our changes.
		3060	*/
3031	serge	3061	obj->base.write_domain = 0;
2335	Serge	3062	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	3063
2351	Serge	3064	trace_i915_gem_object_change_domain(obj,
		3065	old_read_domains,
		3066	old_write_domain);
2332	Serge	3067
2335	Serge	3068	return 0;
		3069	}
2332	Serge	3070
2344	Serge	3071	int
		3072	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		3073	{
		3074	int ret;
2332	Serge	3075
2344	Serge	3076	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		3077	return 0;
2332	Serge	3078
3031	serge	3079	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	3080	if (ret)
		3081	return ret;
2332	Serge	3082
2344	Serge	3083	/* Ensure that we invalidate the GPU's caches and TLBs. */
		3084	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	3085	return 0;
2344	Serge	3086	}
2332	Serge	3087
2344	Serge	3088	/**
		3089	* Moves a single object to the CPU read, and possibly write domain.
		3090	*
		3091	* This function returns when the move is complete, including waiting on
		3092	* flushes to occur.
		3093	*/
3031	serge	3094	int
2344	Serge	3095	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		3096	{
		3097	uint32_t old_write_domain, old_read_domains;
		3098	int ret;
2332	Serge	3099
2344	Serge	3100	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		3101	return 0;
2332	Serge	3102
3031	serge	3103	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	3104	if (ret)
		3105	return ret;
2332	Serge	3106
2344	Serge	3107	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	3108
2344	Serge	3109	old_write_domain = obj->base.write_domain;
		3110	old_read_domains = obj->base.read_domains;
2332	Serge	3111
2344	Serge	3112	/* Flush the CPU cache if it's still invalid. */
		3113	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		3114	i915_gem_clflush_object(obj);
2332	Serge	3115
2344	Serge	3116	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		3117	}
2332	Serge	3118
2344	Serge	3119	/* It should now be out of any other write domains, and we can update
		3120	* the domain values for our changes.
		3121	*/
		3122	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	3123
2344	Serge	3124	/* If we're writing through the CPU, then the GPU read domains will
		3125	* need to be invalidated at next use.
		3126	*/
		3127	if (write) {
		3128	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3129	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3130	}
2332	Serge	3131
2351	Serge	3132	trace_i915_gem_object_change_domain(obj,
		3133	old_read_domains,
		3134	old_write_domain);
2332	Serge	3135
2344	Serge	3136	return 0;
		3137	}
2332	Serge	3138
3031	serge	3139	/* Throttle our rendering by waiting until the ring has completed our requests
		3140	* emitted over 20 msec ago.
2344	Serge	3141	*
3031	serge	3142	* Note that if we were to use the current jiffies each time around the loop,
		3143	* we wouldn't escape the function with any frames outstanding if the time to
		3144	* render a frame was over 20ms.
		3145	*
		3146	* This should get us reasonable parallelism between CPU and GPU but also
		3147	* relatively low latency when blocking on a particular request to finish.
2344	Serge	3148	*/
3031	serge	3149	static int
		3150	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	3151	{
3031	serge	3152	struct drm_i915_private *dev_priv = dev->dev_private;
		3153	struct drm_i915_file_private *file_priv = file->driver_priv;
3263	Serge	3154	unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20);
3031	serge	3155	struct drm_i915_gem_request *request;
		3156	struct intel_ring_buffer *ring = NULL;
		3157	u32 seqno = 0;
		3158	int ret;
2332	Serge	3159
3031	serge	3160	if (atomic_read(&dev_priv->mm.wedged))
		3161	return -EIO;
2332	Serge	3162
3031	serge	3163	spin_lock(&file_priv->mm.lock);
		3164	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		3165	if (time_after_eq(request->emitted_jiffies, recent_enough))
		3166	break;
2332	Serge	3167
3031	serge	3168	ring = request->ring;
		3169	seqno = request->seqno;
		3170	}
		3171	spin_unlock(&file_priv->mm.lock);
2332	Serge	3172
3031	serge	3173	if (seqno == 0)
		3174	return 0;
2332	Serge	3175
3031	serge	3176	ret = __wait_seqno(ring, seqno, true, NULL);
		3177	if (ret == 0)
		3178	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	3179
3031	serge	3180	return ret;
2352	Serge	3181	}
2332	Serge	3182
		3183	int
		3184	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		3185	uint32_t alignment,
3031	serge	3186	bool map_and_fenceable,
		3187	bool nonblocking)
2332	Serge	3188	{
		3189	int ret;
		3190
3031	serge	3191	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3192	return -EBUSY;
2332	Serge	3193
		3194	if (obj->gtt_space != NULL) {
		3195	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3196	(map_and_fenceable && !obj->map_and_fenceable)) {
		3197	WARN(obj->pin_count,
		3198	"bo is already pinned with incorrect alignment:"
		3199	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3200	" obj->map_and_fenceable=%d\n",
		3201	obj->gtt_offset, alignment,
		3202	map_and_fenceable,
		3203	obj->map_and_fenceable);
		3204	ret = i915_gem_object_unbind(obj);
		3205	if (ret)
		3206	return ret;
		3207	}
		3208	}
		3209
		3210	if (obj->gtt_space == NULL) {
3243	Serge	3211	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3212
2332	Serge	3213	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3214	map_and_fenceable,
		3215	nonblocking);
2332	Serge	3216	if (ret)
		3217	return ret;
3243	Serge	3218
		3219	if (!dev_priv->mm.aliasing_ppgtt)
		3220	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3221	}
		3222
3031	serge	3223	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3224	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3225
		3226	obj->pin_count++;
2332	Serge	3227	obj->pin_mappable \|= map_and_fenceable;
		3228
		3229	return 0;
		3230	}
		3231
2344	Serge	3232	void
		3233	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3234	{
		3235	BUG_ON(obj->pin_count == 0);
		3236	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3237
3031	serge	3238	if (--obj->pin_count == 0)
2344	Serge	3239	obj->pin_mappable = false;
		3240	}
2332	Serge	3241
3031	serge	3242	int
		3243	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3244	struct drm_file *file)
		3245	{
		3246	struct drm_i915_gem_pin *args = data;
		3247	struct drm_i915_gem_object *obj;
		3248	int ret;
2332	Serge	3249
3031	serge	3250	ret = i915_mutex_lock_interruptible(dev);
		3251	if (ret)
		3252	return ret;
2332	Serge	3253
3031	serge	3254	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3255	if (&obj->base == NULL) {
		3256	ret = -ENOENT;
		3257	goto unlock;
		3258	}
2332	Serge	3259
3031	serge	3260	if (obj->madv != I915_MADV_WILLNEED) {
		3261	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3262	ret = -EINVAL;
		3263	goto out;
		3264	}
2332	Serge	3265
3031	serge	3266	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3267	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3268	args->handle);
		3269	ret = -EINVAL;
		3270	goto out;
		3271	}
2332	Serge	3272
3243	Serge	3273	if (obj->user_pin_count == 0) {
3031	serge	3274	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3275	if (ret)
		3276	goto out;
		3277	}
2332	Serge	3278
3243	Serge	3279	obj->user_pin_count++;
		3280	obj->pin_filp = file;
		3281
3031	serge	3282	/* XXX - flush the CPU caches for pinned objects
		3283	* as the X server doesn't manage domains yet
		3284	*/
		3285	i915_gem_object_flush_cpu_write_domain(obj);
		3286	args->offset = obj->gtt_offset;
		3287	out:
		3288	drm_gem_object_unreference(&obj->base);
		3289	unlock:
		3290	mutex_unlock(&dev->struct_mutex);
		3291	return ret;
		3292	}
2332	Serge	3293
3255	Serge	3294	#if 0
		3295
3031	serge	3296	int
		3297	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3298	struct drm_file *file)
		3299	{
		3300	struct drm_i915_gem_pin *args = data;
		3301	struct drm_i915_gem_object *obj;
		3302	int ret;
2332	Serge	3303
3031	serge	3304	ret = i915_mutex_lock_interruptible(dev);
		3305	if (ret)
		3306	return ret;
2332	Serge	3307
3031	serge	3308	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3309	if (&obj->base == NULL) {
		3310	ret = -ENOENT;
		3311	goto unlock;
		3312	}
2332	Serge	3313
3031	serge	3314	if (obj->pin_filp != file) {
		3315	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3316	args->handle);
		3317	ret = -EINVAL;
		3318	goto out;
		3319	}
		3320	obj->user_pin_count--;
		3321	if (obj->user_pin_count == 0) {
		3322	obj->pin_filp = NULL;
		3323	i915_gem_object_unpin(obj);
		3324	}
2332	Serge	3325
3031	serge	3326	out:
		3327	drm_gem_object_unreference(&obj->base);
		3328	unlock:
		3329	mutex_unlock(&dev->struct_mutex);
		3330	return ret;
		3331	}
2332	Serge	3332
3260	Serge	3333	#endif
		3334
3031	serge	3335	int
		3336	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3337	struct drm_file *file)
		3338	{
		3339	struct drm_i915_gem_busy *args = data;
		3340	struct drm_i915_gem_object *obj;
		3341	int ret;
2332	Serge	3342
3031	serge	3343	ret = i915_mutex_lock_interruptible(dev);
		3344	if (ret)
		3345	return ret;
2332	Serge	3346
3031	serge	3347	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3348	if (&obj->base == NULL) {
		3349	ret = -ENOENT;
		3350	goto unlock;
		3351	}
2332	Serge	3352
3031	serge	3353	/* Count all active objects as busy, even if they are currently not used
		3354	* by the gpu. Users of this interface expect objects to eventually
		3355	* become non-busy without any further actions, therefore emit any
		3356	* necessary flushes here.
		3357	*/
		3358	ret = i915_gem_object_flush_active(obj);
2332	Serge	3359
3031	serge	3360	args->busy = obj->active;
		3361	if (obj->ring) {
		3362	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3363	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3364	}
2332	Serge	3365
3031	serge	3366	drm_gem_object_unreference(&obj->base);
		3367	unlock:
		3368	mutex_unlock(&dev->struct_mutex);
		3369	return ret;
		3370	}
2332	Serge	3371
3031	serge	3372	int
		3373	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3374	struct drm_file *file_priv)
		3375	{
		3376	return i915_gem_ring_throttle(dev, file_priv);
		3377	}
2332	Serge	3378
3263	Serge	3379	#if 0
		3380
3031	serge	3381	int
		3382	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3383	struct drm_file *file_priv)
		3384	{
		3385	struct drm_i915_gem_madvise *args = data;
		3386	struct drm_i915_gem_object *obj;
		3387	int ret;
2332	Serge	3388
3031	serge	3389	switch (args->madv) {
		3390	case I915_MADV_DONTNEED:
		3391	case I915_MADV_WILLNEED:
		3392	break;
		3393	default:
		3394	return -EINVAL;
		3395	}
2332	Serge	3396
3031	serge	3397	ret = i915_mutex_lock_interruptible(dev);
		3398	if (ret)
		3399	return ret;
2332	Serge	3400
3031	serge	3401	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3402	if (&obj->base == NULL) {
		3403	ret = -ENOENT;
		3404	goto unlock;
		3405	}
2332	Serge	3406
3031	serge	3407	if (obj->pin_count) {
		3408	ret = -EINVAL;
		3409	goto out;
		3410	}
2332	Serge	3411
3031	serge	3412	if (obj->madv != __I915_MADV_PURGED)
		3413	obj->madv = args->madv;
2332	Serge	3414
3031	serge	3415	/* if the object is no longer attached, discard its backing storage */
		3416	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3417	i915_gem_object_truncate(obj);
2332	Serge	3418
3031	serge	3419	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3420
3031	serge	3421	out:
		3422	drm_gem_object_unreference(&obj->base);
		3423	unlock:
		3424	mutex_unlock(&dev->struct_mutex);
		3425	return ret;
		3426	}
		3427	#endif
2332	Serge	3428
3031	serge	3429	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3430	const struct drm_i915_gem_object_ops *ops)
		3431	{
		3432	INIT_LIST_HEAD(&obj->mm_list);
		3433	INIT_LIST_HEAD(&obj->gtt_list);
		3434	INIT_LIST_HEAD(&obj->ring_list);
		3435	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3436
3031	serge	3437	obj->ops = ops;
		3438
		3439	obj->fence_reg = I915_FENCE_REG_NONE;
		3440	obj->madv = I915_MADV_WILLNEED;
		3441	/* Avoid an unnecessary call to unbind on the first bind. */
		3442	obj->map_and_fenceable = true;
		3443
		3444	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3445	}
		3446
		3447	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3448	.get_pages = i915_gem_object_get_pages_gtt,
		3449	.put_pages = i915_gem_object_put_pages_gtt,
		3450	};
		3451
2332	Serge	3452	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3453	size_t size)
		3454	{
		3455	struct drm_i915_gem_object *obj;
3031	serge	3456	struct address_space *mapping;
		3457	u32 mask;
2340	Serge	3458
2332	Serge	3459	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3460	if (obj == NULL)
		3461	return NULL;
		3462
		3463	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3464	kfree(obj);
		3465	return NULL;
		3466	}
		3467
		3468
3031	serge	3469	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3470
		3471	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3472	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3473
3031	serge	3474	if (HAS_LLC(dev)) {
		3475	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3476	* cache) for about a 10% performance improvement
		3477	* compared to uncached. Graphics requests other than
		3478	* display scanout are coherent with the CPU in
		3479	* accessing this cache. This means in this mode we
		3480	* don't need to clflush on the CPU side, and on the
		3481	* GPU side we only need to flush internal caches to
		3482	* get data visible to the CPU.
		3483	*
		3484	* However, we maintain the display planes as UC, and so
		3485	* need to rebind when first used as such.
		3486	*/
		3487	obj->cache_level = I915_CACHE_LLC;
		3488	} else
		3489	obj->cache_level = I915_CACHE_NONE;
		3490
		3491	return obj;
		3492	}
		3493
2344	Serge	3494	int i915_gem_init_object(struct drm_gem_object *obj)
		3495	{
		3496	BUG();
2332	Serge	3497
2344	Serge	3498	return 0;
		3499	}
2332	Serge	3500
3031	serge	3501	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3502	{
3031	serge	3503	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3504	struct drm_device *dev = obj->base.dev;
		3505	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3506
3031	serge	3507	trace_i915_gem_object_destroy(obj);
		3508
		3509	// if (obj->phys_obj)
		3510	// i915_gem_detach_phys_object(dev, obj);
3298	Serge	3511	// printf("%s obj %p\n", __FUNCTION__, obj);
3031	serge	3512
		3513	obj->pin_count = 0;
		3514	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3515	bool was_interruptible;
		3516
		3517	was_interruptible = dev_priv->mm.interruptible;
		3518	dev_priv->mm.interruptible = false;
		3519
		3520	WARN_ON(i915_gem_object_unbind(obj));
		3521
		3522	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3523	}
2332	Serge	3524
3031	serge	3525	obj->pages_pin_count = 0;
		3526	i915_gem_object_put_pages(obj);
		3527	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3528
3243	Serge	3529	BUG_ON(obj->pages);
2332	Serge	3530
3031	serge	3531	// if (obj->base.import_attach)
		3532	// drm_prime_gem_destroy(&obj->base, NULL);
		3533
3290	Serge	3534	if(obj->base.filp != NULL)
		3535	{
3298	Serge	3536	// printf("filp %p\n", obj->base.filp);
3290	Serge	3537	shmem_file_delete(obj->base.filp);
		3538	}
		3539
2344	Serge	3540	drm_gem_object_release(&obj->base);
		3541	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3542
2344	Serge	3543	kfree(obj->bit_17);
		3544	kfree(obj);
		3545	}
2332	Serge	3546
3031	serge	3547	#if 0
		3548	int
		3549	i915_gem_idle(struct drm_device *dev)
2344	Serge	3550	{
3031	serge	3551	drm_i915_private_t *dev_priv = dev->dev_private;
		3552	int ret;
2332	Serge	3553
3031	serge	3554	mutex_lock(&dev->struct_mutex);
2332	Serge	3555
3031	serge	3556	if (dev_priv->mm.suspended) {
		3557	mutex_unlock(&dev->struct_mutex);
		3558	return 0;
		3559	}
2332	Serge	3560
3031	serge	3561	ret = i915_gpu_idle(dev);
		3562	if (ret) {
		3563	mutex_unlock(&dev->struct_mutex);
		3564	return ret;
		3565	}
		3566	i915_gem_retire_requests(dev);
		3567
		3568	i915_gem_reset_fences(dev);
		3569
		3570	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3571	* We need to replace this with a semaphore, or something.
		3572	* And not confound mm.suspended!
		3573	*/
		3574	dev_priv->mm.suspended = 1;
		3575	del_timer_sync(&dev_priv->hangcheck_timer);
		3576
		3577	i915_kernel_lost_context(dev);
		3578	i915_gem_cleanup_ringbuffer(dev);
		3579
		3580	mutex_unlock(&dev->struct_mutex);
		3581
		3582	/* Cancel the retire work handler, which should be idle now. */
3263	Serge	3583	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3031	serge	3584
		3585	return 0;
2344	Serge	3586	}
3031	serge	3587	#endif
2332	Serge	3588
3031	serge	3589	void i915_gem_l3_remap(struct drm_device *dev)
		3590	{
		3591	drm_i915_private_t *dev_priv = dev->dev_private;
		3592	u32 misccpctl;
		3593	int i;
2332	Serge	3594
3031	serge	3595	if (!IS_IVYBRIDGE(dev))
		3596	return;
2332	Serge	3597
3243	Serge	3598	if (!dev_priv->l3_parity.remap_info)
3031	serge	3599	return;
2332	Serge	3600
3031	serge	3601	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3602	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3603	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3604
3031	serge	3605	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3606	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3607	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3608	DRM_DEBUG("0x%x was already programmed to %x\n",
		3609	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3610	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3611	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3612	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3613	}
2332	Serge	3614
3031	serge	3615	/* Make sure all the writes land before disabling dop clock gating */
		3616	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3617
3031	serge	3618	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3619	}
2332	Serge	3620
3031	serge	3621	void i915_gem_init_swizzling(struct drm_device *dev)
		3622	{
		3623	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3624
3031	serge	3625	if (INTEL_INFO(dev)->gen < 5 \|\|
		3626	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3627	return;
2332	Serge	3628
3031	serge	3629	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3630	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3631
3031	serge	3632	if (IS_GEN5(dev))
		3633	return;
2344	Serge	3634
3031	serge	3635	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3636	if (IS_GEN6(dev))
		3637	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3638	else
		3639	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3640	}
		3641
		3642	static bool
		3643	intel_enable_blt(struct drm_device *dev)
		3644	{
		3645	if (!HAS_BLT(dev))
		3646	return false;
		3647
		3648	/* The blitter was dysfunctional on early prototypes */
		3649	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3650	DRM_INFO("BLT not supported on this pre-production hardware;"
		3651	" graphics performance will be degraded.\n");
		3652	return false;
		3653	}
		3654
		3655	return true;
		3656	}
		3657
2332	Serge	3658	int
3031	serge	3659	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3660	{
		3661	drm_i915_private_t *dev_priv = dev->dev_private;
		3662	int ret;
2351	Serge	3663
3243	Serge	3664	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
3031	serge	3665	return -EIO;
		3666
		3667	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3668	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3669
		3670	i915_gem_l3_remap(dev);
		3671
		3672	i915_gem_init_swizzling(dev);
		3673
2332	Serge	3674	ret = intel_init_render_ring_buffer(dev);
		3675	if (ret)
		3676	return ret;
		3677
		3678	if (HAS_BSD(dev)) {
		3679	ret = intel_init_bsd_ring_buffer(dev);
		3680	if (ret)
		3681	goto cleanup_render_ring;
		3682	}
		3683
3031	serge	3684	if (intel_enable_blt(dev)) {
2332	Serge	3685	ret = intel_init_blt_ring_buffer(dev);
		3686	if (ret)
		3687	goto cleanup_bsd_ring;
		3688	}
		3689
		3690	dev_priv->next_seqno = 1;
2351	Serge	3691
3031	serge	3692	/*
		3693	* XXX: There was some w/a described somewhere suggesting loading
		3694	* contexts before PPGTT.
		3695	*/
		3696	i915_gem_context_init(dev);
		3697	i915_gem_init_ppgtt(dev);
		3698
2332	Serge	3699	return 0;
		3700
		3701	cleanup_bsd_ring:
		3702	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3703	cleanup_render_ring:
		3704	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3705	return ret;
		3706	}
		3707
3031	serge	3708	static bool
		3709	intel_enable_ppgtt(struct drm_device *dev)
		3710	{
		3711	if (i915_enable_ppgtt >= 0)
		3712	return i915_enable_ppgtt;
		3713
		3714	#ifdef CONFIG_INTEL_IOMMU
		3715	/* Disable ppgtt on SNB if VT-d is on. */
		3716	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3717	return false;
		3718	#endif
		3719
		3720	return true;
		3721	}
		3722
		3723	#define LFB_SIZE 0xC00000
		3724
		3725	int i915_gem_init(struct drm_device *dev)
		3726	{
		3727	struct drm_i915_private *dev_priv = dev->dev_private;
		3728	unsigned long gtt_size, mappable_size;
		3729	int ret;
		3730
		3731	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3732	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3733
		3734	mutex_lock(&dev->struct_mutex);
		3735	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3736	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3737	* aperture accordingly when using aliasing ppgtt. */
		3738	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3739
		3740	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3741
		3742	ret = i915_gem_init_aliasing_ppgtt(dev);
		3743	if (ret) {
		3744	mutex_unlock(&dev->struct_mutex);
		3745	return ret;
		3746	}
		3747	} else {
		3748	/* Let GEM Manage all of the aperture.
		3749	*
		3750	* However, leave one page at the end still bound to the scratch
		3751	* page. There are a number of places where the hardware
		3752	* apparently prefetches past the end of the object, and we've
		3753	* seen multiple hangs with the GPU head pointer stuck in a
		3754	* batchbuffer bound at the last page of the aperture. One page
		3755	* should be enough to keep any prefetching inside of the
		3756	* aperture.
		3757	*/
		3758	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3759	}
		3760
		3761	ret = i915_gem_init_hw(dev);
		3762	mutex_unlock(&dev->struct_mutex);
		3763	if (ret) {
		3764	i915_gem_cleanup_aliasing_ppgtt(dev);
		3765	return ret;
		3766	}
		3767
		3768	return 0;
		3769	}
		3770
2332	Serge	3771	void
		3772	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3773	{
		3774	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3775	struct intel_ring_buffer *ring;
2332	Serge	3776	int i;
		3777
3031	serge	3778	for_each_ring(ring, dev_priv, i)
		3779	intel_cleanup_ring_buffer(ring);
2332	Serge	3780	}
		3781
3031	serge	3782	#if 0
		3783
2332	Serge	3784	int
		3785	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3786	struct drm_file *file_priv)
		3787	{
		3788	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3789	int ret;
2332	Serge	3790
		3791	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3792	return 0;
		3793
		3794	if (atomic_read(&dev_priv->mm.wedged)) {
		3795	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3796	atomic_set(&dev_priv->mm.wedged, 0);
		3797	}
		3798
		3799	mutex_lock(&dev->struct_mutex);
		3800	dev_priv->mm.suspended = 0;
		3801
3031	serge	3802	ret = i915_gem_init_hw(dev);
2332	Serge	3803	if (ret != 0) {
		3804	mutex_unlock(&dev->struct_mutex);
		3805	return ret;
		3806	}
		3807
		3808	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3809	mutex_unlock(&dev->struct_mutex);
		3810
		3811	ret = drm_irq_install(dev);
		3812	if (ret)
		3813	goto cleanup_ringbuffer;
		3814
		3815	return 0;
		3816
		3817	cleanup_ringbuffer:
		3818	mutex_lock(&dev->struct_mutex);
		3819	i915_gem_cleanup_ringbuffer(dev);
		3820	dev_priv->mm.suspended = 1;
		3821	mutex_unlock(&dev->struct_mutex);
		3822
		3823	return ret;
		3824	}
		3825
		3826	int
		3827	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3828	struct drm_file *file_priv)
		3829	{
		3830	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3831	return 0;
		3832
		3833	drm_irq_uninstall(dev);
		3834	return i915_gem_idle(dev);
		3835	}
		3836
		3837	void
		3838	i915_gem_lastclose(struct drm_device *dev)
		3839	{
		3840	int ret;
		3841
		3842	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3843	return;
		3844
		3845	ret = i915_gem_idle(dev);
		3846	if (ret)
		3847	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3848	}
		3849	#endif
		3850
		3851	static void
2326	Serge	3852	init_ring_lists(struct intel_ring_buffer *ring)
		3853	{
		3854	INIT_LIST_HEAD(&ring->active_list);
		3855	INIT_LIST_HEAD(&ring->request_list);
		3856	}
		3857
		3858	void
		3859	i915_gem_load(struct drm_device *dev)
		3860	{
		3861	int i;
		3862	drm_i915_private_t *dev_priv = dev->dev_private;
		3863
		3864	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3865	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3866	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3867	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3868	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3869	for (i = 0; i < I915_NUM_RINGS; i++)
		3870	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3871	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3872	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3873	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3874	i915_gem_retire_work_handler);
2326	Serge	3875
		3876	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3877	if (IS_GEN3(dev)) {
3031	serge	3878	I915_WRITE(MI_ARB_STATE,
		3879	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3880	}
		3881
		3882	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3883
		3884	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3885	dev_priv->num_fence_regs = 16;
		3886	else
		3887	dev_priv->num_fence_regs = 8;
		3888
		3889	/* Initialize fence registers to zero */
3031	serge	3890	i915_gem_reset_fences(dev);
2326	Serge	3891
		3892	i915_gem_detect_bit_6_swizzle(dev);
		3893
		3894	dev_priv->mm.interruptible = true;
		3895
		3896	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3897	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3898	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3899	}
		3900
		3901

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3298