WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
3031	serge	28	#include
		29	#include
2326	Serge	30	#include "i915_drv.h"
2351	Serge	31	#include "i915_trace.h"
2326	Serge	32	#include "intel_drv.h"
2330	Serge	33	#include
2326	Serge	34	//#include
		35	#include
		36
2344	Serge	37	extern int x86_clflush_size;
2332	Serge	38
2344	Serge	39	#undef mb
		40	#undef rmb
		41	#undef wmb
		42	#define mb() asm volatile("mfence")
		43	#define rmb() asm volatile ("lfence")
		44	#define wmb() asm volatile ("sfence")
		45
		46	static inline void clflush(volatile void *__p)
		47	{
		48	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		49	}
		50
2332	Serge	51	#define MAX_ERRNO 4095
		52
		53	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		54
2344	Serge	55	void
		56	drm_gem_object_free(struct kref *kref)
		57	{
		58	struct drm_gem_object obj = (struct drm_gem_object ) kref;
		59	struct drm_device *dev = obj->dev;
2332	Serge	60
2344	Serge	61	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		62
		63	i915_gem_free_object(obj);
		64	}
		65
2332	Serge	66	/**
		67	* Initialize an already allocated GEM object of the specified size with
		68	* shmfs backing store.
		69	*/
		70	int drm_gem_object_init(struct drm_device *dev,
		71	struct drm_gem_object *obj, size_t size)
		72	{
		73	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		74
		75	obj->dev = dev;
2344	Serge	76	kref_init(&obj->refcount);
2332	Serge	77	atomic_set(&obj->handle_count, 0);
		78	obj->size = size;
		79
		80	return 0;
		81	}
		82
2344	Serge	83	void
		84	drm_gem_object_release(struct drm_gem_object *obj)
		85	{ }
2332	Serge	86
		87
2326	Serge	88	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		89	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		90	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		91	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		92
2332	Serge	93	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		94	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		95	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		96	unsigned alignment,
3031	serge	97	bool map_and_fenceable,
		98	bool nonblocking);
2332	Serge	99	static int i915_gem_phys_pwrite(struct drm_device *dev,
		100	struct drm_i915_gem_object *obj,
		101	struct drm_i915_gem_pwrite *args,
		102	struct drm_file *file);
2326	Serge	103
3031	serge	104	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		105	struct drm_i915_gem_object *obj);
		106	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		107	struct drm_i915_fence_reg *fence,
		108	bool enable);
2332	Serge	109
3031	serge	110	static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
		111	static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
		112	static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
		113
		114	static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
		115	{
		116	if (obj->tiling_mode)
		117	i915_gem_release_mmap(obj);
		118
		119	/* As we do not have an associated fence register, we will force
		120	* a tiling change if we ever need to acquire one.
		121	*/
		122	obj->fence_dirty = false;
		123	obj->fence_reg = I915_FENCE_REG_NONE;
		124	}
		125
2332	Serge	126	/* some bookkeeping */
		127	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		128	size_t size)
		129	{
		130	dev_priv->mm.object_count++;
		131	dev_priv->mm.object_memory += size;
		132	}
		133
		134	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		135	size_t size)
		136	{
		137	dev_priv->mm.object_count--;
		138	dev_priv->mm.object_memory -= size;
		139	}
		140
		141
		142	static int
		143	i915_gem_wait_for_error(struct drm_device *dev)
		144	{
		145	struct drm_i915_private *dev_priv = dev->dev_private;
		146	struct completion *x = &dev_priv->error_completion;
		147	unsigned long flags;
		148	int ret;
		149
		150	if (!atomic_read(&dev_priv->mm.wedged))
		151	return 0;
3255	Serge	152	#if 0
3031	serge	153	/*
		154	* Only wait 10 seconds for the gpu reset to complete to avoid hanging
		155	* userspace. If it takes that long something really bad is going on and
		156	* we should simply try to bail out and fail as gracefully as possible.
		157	*/
		158	ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
		159	if (ret == 0) {
		160	DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		161	return -EIO;
		162	} else if (ret < 0) {
2332	Serge	163	return ret;
3031	serge	164	}
2332	Serge	165
		166	if (atomic_read(&dev_priv->mm.wedged)) {
		167	/* GPU is hung, bump the completion count to account for
		168	* the token we just consumed so that we never hit zero and
		169	* end up waiting upon a subsequent completion event that
		170	* will never happen.
		171	*/
		172	spin_lock_irqsave(&x->wait.lock, flags);
		173	x->done++;
		174	spin_unlock_irqrestore(&x->wait.lock, flags);
		175	}
3255	Serge	176	#endif
		177
2332	Serge	178	return 0;
		179	}
		180
		181	int i915_mutex_lock_interruptible(struct drm_device *dev)
		182	{
		183	int ret;
		184
		185	ret = i915_gem_wait_for_error(dev);
		186	if (ret)
		187	return ret;
		188
3255	Serge	189	mutex_lock(&dev->struct_mutex);
2332	Serge	190
		191	WARN_ON(i915_verify_lists(dev));
		192	return 0;
		193	}
		194
		195	static inline bool
		196	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		197	{
3031	serge	198	return obj->gtt_space && !obj->active;
2332	Serge	199	}
		200
		201
		202	#if 0
		203
		204	int
		205	i915_gem_init_ioctl(struct drm_device dev, void data,
		206	struct drm_file *file)
		207	{
		208	struct drm_i915_gem_init *args = data;
		209
3031	serge	210	if (drm_core_check_feature(dev, DRIVER_MODESET))
		211	return -ENODEV;
		212
2332	Serge	213	if (args->gtt_start >= args->gtt_end \|\|
		214	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		215	return -EINVAL;
		216
3031	serge	217	/* GEM with user mode setting was never supported on ilk and later. */
		218	if (INTEL_INFO(dev)->gen >= 5)
		219	return -ENODEV;
		220
2332	Serge	221	mutex_lock(&dev->struct_mutex);
3031	serge	222	i915_gem_init_global_gtt(dev, args->gtt_start,
		223	args->gtt_end, args->gtt_end);
2332	Serge	224	mutex_unlock(&dev->struct_mutex);
		225
		226	return 0;
		227	}
2351	Serge	228	#endif
2332	Serge	229
		230	int
		231	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		232	struct drm_file *file)
		233	{
		234	struct drm_i915_private *dev_priv = dev->dev_private;
		235	struct drm_i915_gem_get_aperture *args = data;
		236	struct drm_i915_gem_object *obj;
		237	size_t pinned;
		238
		239	pinned = 0;
		240	mutex_lock(&dev->struct_mutex);
3031	serge	241	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
		242	if (obj->pin_count)
2332	Serge	243	pinned += obj->gtt_space->size;
		244	mutex_unlock(&dev->struct_mutex);
		245
		246	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	247	args->aper_available_size = args->aper_size - pinned;
2332	Serge	248
		249	return 0;
		250	}
		251
3031	serge	252	static int
		253	i915_gem_create(struct drm_file *file,
2332	Serge	254	struct drm_device *dev,
		255	uint64_t size,
		256	uint32_t *handle_p)
		257	{
		258	struct drm_i915_gem_object *obj;
		259	int ret;
		260	u32 handle;
		261
		262	size = roundup(size, PAGE_SIZE);
2342	Serge	263	if (size == 0)
		264	return -EINVAL;
2332	Serge	265
		266	/* Allocate the new object */
		267	obj = i915_gem_alloc_object(dev, size);
		268	if (obj == NULL)
		269	return -ENOMEM;
		270
		271	ret = drm_gem_handle_create(file, &obj->base, &handle);
		272	if (ret) {
		273	drm_gem_object_release(&obj->base);
		274	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		275	kfree(obj);
		276	return ret;
		277	}
		278
		279	/* drop reference from allocate - handle holds it now */
		280	drm_gem_object_unreference(&obj->base);
2351	Serge	281	trace_i915_gem_object_create(obj);
2332	Serge	282
		283	*handle_p = handle;
3255	Serge	284
2332	Serge	285	return 0;
		286	}
		287
		288	int
		289	i915_gem_dumb_create(struct drm_file *file,
		290	struct drm_device *dev,
		291	struct drm_mode_create_dumb *args)
		292	{
		293	/* have to work out size/pitch and return them */
		294	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		295	args->size = args->pitch * args->height;
		296	return i915_gem_create(file, dev,
		297	args->size, &args->handle);
		298	}
		299
		300	int i915_gem_dumb_destroy(struct drm_file *file,
		301	struct drm_device *dev,
		302	uint32_t handle)
		303	{
		304	return drm_gem_handle_delete(file, handle);
		305	}
		306
2326	Serge	307	/**
2332	Serge	308	* Creates a new mm object and returns a handle to it.
		309	*/
		310	int
		311	i915_gem_create_ioctl(struct drm_device dev, void data,
		312	struct drm_file *file)
		313	{
		314	struct drm_i915_gem_create *args = data;
3031	serge	315
2332	Serge	316	return i915_gem_create(file, dev,
		317	args->size, &args->handle);
		318	}
		319
3255	Serge	320	#if 0
		321
2332	Serge	322	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		323	{
		324	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		325
		326	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		327	obj->tiling_mode != I915_TILING_NONE;
		328	}
		329
3031	serge	330	static inline int
		331	__copy_to_user_swizzled(char __user *cpu_vaddr,
		332	const char *gpu_vaddr, int gpu_offset,
2332	Serge	333	int length)
		334	{
3031	serge	335	int ret, cpu_offset = 0;
2332	Serge	336
3031	serge	337	while (length > 0) {
		338	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		339	int this_length = min(cacheline_end - gpu_offset, length);
		340	int swizzled_gpu_offset = gpu_offset ^ 64;
2332	Serge	341
3031	serge	342	ret = __copy_to_user(cpu_vaddr + cpu_offset,
		343	gpu_vaddr + swizzled_gpu_offset,
		344	this_length);
		345	if (ret)
		346	return ret + length;
2332	Serge	347
3031	serge	348	cpu_offset += this_length;
		349	gpu_offset += this_length;
		350	length -= this_length;
		351	}
		352
		353	return 0;
2332	Serge	354	}
		355
3031	serge	356	static inline int
		357	__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
		358	const char __user *cpu_vaddr,
		359	int length)
2332	Serge	360	{
3031	serge	361	int ret, cpu_offset = 0;
2332	Serge	362
		363	while (length > 0) {
		364	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		365	int this_length = min(cacheline_end - gpu_offset, length);
		366	int swizzled_gpu_offset = gpu_offset ^ 64;
		367
3031	serge	368	ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
2332	Serge	369	cpu_vaddr + cpu_offset,
		370	this_length);
3031	serge	371	if (ret)
		372	return ret + length;
		373
2332	Serge	374	cpu_offset += this_length;
		375	gpu_offset += this_length;
		376	length -= this_length;
		377	}
		378
3031	serge	379	return 0;
2332	Serge	380	}
		381
3031	serge	382	/* Per-page copy function for the shmem pread fastpath.
		383	* Flushes invalid cachelines before reading the target if
		384	* needs_clflush is set. */
2332	Serge	385	static int
3031	serge	386	shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		387	char __user *user_data,
		388	bool page_do_bit17_swizzling, bool needs_clflush)
		389	{
		390	char *vaddr;
		391	int ret;
		392
		393	if (unlikely(page_do_bit17_swizzling))
		394	return -EINVAL;
		395
		396	vaddr = kmap_atomic(page);
		397	if (needs_clflush)
		398	drm_clflush_virt_range(vaddr + shmem_page_offset,
		399	page_length);
		400	ret = __copy_to_user_inatomic(user_data,
		401	vaddr + shmem_page_offset,
		402	page_length);
		403	kunmap_atomic(vaddr);
		404
		405	return ret ? -EFAULT : 0;
		406	}
		407
		408	static void
		409	shmem_clflush_swizzled_range(char *addr, unsigned long length,
		410	bool swizzled)
		411	{
		412	if (unlikely(swizzled)) {
		413	unsigned long start = (unsigned long) addr;
		414	unsigned long end = (unsigned long) addr + length;
		415
		416	/* For swizzling simply ensure that we always flush both
		417	* channels. Lame, but simple and it works. Swizzled
		418	* pwrite/pread is far from a hotpath - current userspace
		419	* doesn't use it at all. */
		420	start = round_down(start, 128);
		421	end = round_up(end, 128);
		422
		423	drm_clflush_virt_range((void *)start, end - start);
		424	} else {
		425	drm_clflush_virt_range(addr, length);
		426	}
		427
		428	}
		429
		430	/* Only difference to the fast-path function is that this can handle bit17
		431	* and uses non-atomic copy and kmap functions. */
		432	static int
		433	shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		434	char __user *user_data,
		435	bool page_do_bit17_swizzling, bool needs_clflush)
		436	{
		437	char *vaddr;
		438	int ret;
		439
		440	vaddr = kmap(page);
		441	if (needs_clflush)
		442	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		443	page_length,
		444	page_do_bit17_swizzling);
		445
		446	if (page_do_bit17_swizzling)
		447	ret = __copy_to_user_swizzled(user_data,
		448	vaddr, shmem_page_offset,
		449	page_length);
		450	else
		451	ret = __copy_to_user(user_data,
		452	vaddr + shmem_page_offset,
		453	page_length);
		454	kunmap(page);
		455
		456	return ret ? - EFAULT : 0;
		457	}
		458
		459	static int
		460	i915_gem_shmem_pread(struct drm_device *dev,
2332	Serge	461	struct drm_i915_gem_object *obj,
		462	struct drm_i915_gem_pread *args,
		463	struct drm_file *file)
		464	{
3031	serge	465	char __user *user_data;
2332	Serge	466	ssize_t remain;
		467	loff_t offset;
3031	serge	468	int shmem_page_offset, page_length, ret = 0;
		469	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		470	int hit_slowpath = 0;
		471	int prefaulted = 0;
		472	int needs_clflush = 0;
		473	struct scatterlist *sg;
		474	int i;
2332	Serge	475
		476	user_data = (char __user *) (uintptr_t) args->data_ptr;
		477	remain = args->size;
		478
3031	serge	479	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		480
		481	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		482	/* If we're not in the cpu read domain, set ourself into the gtt
		483	* read domain and manually flush cachelines (if required). This
		484	* optimizes for the case when the gpu will dirty the data
		485	* anyway again before the next pread happens. */
		486	if (obj->cache_level == I915_CACHE_NONE)
		487	needs_clflush = 1;
		488	if (obj->gtt_space) {
		489	ret = i915_gem_object_set_to_gtt_domain(obj, false);
		490	if (ret)
		491	return ret;
		492	}
		493	}
		494
		495	ret = i915_gem_object_get_pages(obj);
		496	if (ret)
		497	return ret;
		498
		499	i915_gem_object_pin_pages(obj);
		500
2332	Serge	501	offset = args->offset;
		502
3031	serge	503	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	504	struct page *page;
		505
3031	serge	506	if (i < offset >> PAGE_SHIFT)
		507	continue;
		508
		509	if (remain <= 0)
		510	break;
		511
2332	Serge	512	/* Operation in this page
		513	*
3031	serge	514	* shmem_page_offset = offset within page in shmem file
2332	Serge	515	* page_length = bytes to copy for this page
		516	*/
3031	serge	517	shmem_page_offset = offset_in_page(offset);
2332	Serge	518	page_length = remain;
3031	serge	519	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		520	page_length = PAGE_SIZE - shmem_page_offset;
2332	Serge	521
3031	serge	522	page = sg_page(sg);
		523	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		524	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	525
3031	serge	526	ret = shmem_pread_fast(page, shmem_page_offset, page_length,
		527	user_data, page_do_bit17_swizzling,
		528	needs_clflush);
		529	if (ret == 0)
		530	goto next_page;
2332	Serge	531
3031	serge	532	hit_slowpath = 1;
		533	mutex_unlock(&dev->struct_mutex);
		534
		535	if (!prefaulted) {
		536	ret = fault_in_multipages_writeable(user_data, remain);
		537	/* Userspace is tricking us, but we've already clobbered
		538	* its pages with the prefault and promised to write the
		539	* data up to the first fault. Hence ignore any errors
		540	* and just continue. */
		541	(void)ret;
		542	prefaulted = 1;
		543	}
		544
		545	ret = shmem_pread_slow(page, shmem_page_offset, page_length,
		546	user_data, page_do_bit17_swizzling,
		547	needs_clflush);
		548
		549	mutex_lock(&dev->struct_mutex);
		550
		551	next_page:
2332	Serge	552	mark_page_accessed(page);
3031	serge	553
2332	Serge	554	if (ret)
3031	serge	555	goto out;
2332	Serge	556
		557	remain -= page_length;
		558	user_data += page_length;
		559	offset += page_length;
		560	}
		561
3031	serge	562	out:
		563	i915_gem_object_unpin_pages(obj);
		564
		565	if (hit_slowpath) {
		566	/* Fixup: Kill any reinstated backing storage pages */
		567	if (obj->madv == __I915_MADV_PURGED)
		568	i915_gem_object_truncate(obj);
		569	}
		570
		571	return ret;
2332	Serge	572	}
		573
		574	/**
3031	serge	575	* Reads data from the object referenced by handle.
		576	*
		577	* On error, the contents of *data are undefined.
2332	Serge	578	*/
3031	serge	579	int
		580	i915_gem_pread_ioctl(struct drm_device dev, void data,
		581	struct drm_file *file)
		582	{
		583	struct drm_i915_gem_pread *args = data;
		584	struct drm_i915_gem_object *obj;
		585	int ret = 0;
		586
		587	if (args->size == 0)
		588	return 0;
		589
		590	if (!access_ok(VERIFY_WRITE,
		591	(char __user *)(uintptr_t)args->data_ptr,
		592	args->size))
		593	return -EFAULT;
		594
		595	ret = i915_mutex_lock_interruptible(dev);
		596	if (ret)
		597	return ret;
		598
		599	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		600	if (&obj->base == NULL) {
		601	ret = -ENOENT;
		602	goto unlock;
		603	}
		604
		605	/* Bounds check source. */
		606	if (args->offset > obj->base.size \|\|
		607	args->size > obj->base.size - args->offset) {
		608	ret = -EINVAL;
		609	goto out;
		610	}
		611
		612	/* prime objects have no backing filp to GEM pread/pwrite
		613	* pages from.
		614	*/
		615	if (!obj->base.filp) {
		616	ret = -EINVAL;
		617	goto out;
		618	}
		619
		620	trace_i915_gem_object_pread(obj, args->offset, args->size);
		621
		622	ret = i915_gem_shmem_pread(dev, obj, args, file);
		623
		624	out:
		625	drm_gem_object_unreference(&obj->base);
		626	unlock:
		627	mutex_unlock(&dev->struct_mutex);
		628	return ret;
		629	}
		630
		631	/* This is the fast write path which cannot handle
		632	* page faults in the source data
		633	*/
		634
		635	static inline int
		636	fast_user_write(struct io_mapping *mapping,
		637	loff_t page_base, int page_offset,
		638	char __user *user_data,
		639	int length)
		640	{
		641	void __iomem *vaddr_atomic;
		642	void *vaddr;
		643	unsigned long unwritten;
		644
		645	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
		646	/* We can use the cpu mem copy function because this is X86. */
		647	vaddr = (void __force*)vaddr_atomic + page_offset;
		648	unwritten = __copy_from_user_inatomic_nocache(vaddr,
		649	user_data, length);
		650	io_mapping_unmap_atomic(vaddr_atomic);
		651	return unwritten;
		652	}
		653
		654	/**
		655	* This is the fast pwrite path, where we copy the data directly from the
		656	* user into the GTT, uncached.
		657	*/
2332	Serge	658	static int
3031	serge	659	i915_gem_gtt_pwrite_fast(struct drm_device *dev,
		660	struct drm_i915_gem_object *obj,
		661	struct drm_i915_gem_pwrite *args,
		662	struct drm_file *file)
2332	Serge	663	{
3031	serge	664	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	665	ssize_t remain;
3031	serge	666	loff_t offset, page_base;
		667	char __user *user_data;
		668	int page_offset, page_length, ret;
2332	Serge	669
3031	serge	670	ret = i915_gem_object_pin(obj, 0, true, true);
		671	if (ret)
		672	goto out;
		673
		674	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		675	if (ret)
		676	goto out_unpin;
		677
		678	ret = i915_gem_object_put_fence(obj);
		679	if (ret)
		680	goto out_unpin;
		681
		682	user_data = (char __user *) (uintptr_t) args->data_ptr;
2332	Serge	683	remain = args->size;
		684
3031	serge	685	offset = obj->gtt_offset + args->offset;
2332	Serge	686
3031	serge	687	while (remain > 0) {
		688	/* Operation in this page
		689	*
		690	* page_base = page offset within aperture
		691	* page_offset = offset within page
		692	* page_length = bytes to copy for this page
		693	*/
		694	page_base = offset & PAGE_MASK;
		695	page_offset = offset_in_page(offset);
		696	page_length = remain;
		697	if ((page_offset + remain) > PAGE_SIZE)
		698	page_length = PAGE_SIZE - page_offset;
2332	Serge	699
3031	serge	700	/* If we get a fault while copying data, then (presumably) our
		701	* source page isn't available. Return the error and we'll
		702	* retry in the slow path.
		703	*/
		704	if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
		705	page_offset, user_data, page_length)) {
		706	ret = -EFAULT;
		707	goto out_unpin;
		708	}
		709
		710	remain -= page_length;
		711	user_data += page_length;
		712	offset += page_length;
2332	Serge	713	}
		714
3031	serge	715	out_unpin:
		716	i915_gem_object_unpin(obj);
		717	out:
		718	return ret;
		719	}
		720
		721	/* Per-page copy function for the shmem pwrite fastpath.
		722	* Flushes invalid cachelines before writing to the target if
		723	* needs_clflush_before is set and flushes out any written cachelines after
		724	* writing if needs_clflush is set. */
		725	static int
		726	shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		727	char __user *user_data,
		728	bool page_do_bit17_swizzling,
		729	bool needs_clflush_before,
		730	bool needs_clflush_after)
		731	{
		732	char *vaddr;
		733	int ret;
		734
		735	if (unlikely(page_do_bit17_swizzling))
		736	return -EINVAL;
		737
		738	vaddr = kmap_atomic(page);
		739	if (needs_clflush_before)
		740	drm_clflush_virt_range(vaddr + shmem_page_offset,
		741	page_length);
		742	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
		743	user_data,
		744	page_length);
		745	if (needs_clflush_after)
		746	drm_clflush_virt_range(vaddr + shmem_page_offset,
		747	page_length);
		748	kunmap_atomic(vaddr);
		749
		750	return ret ? -EFAULT : 0;
		751	}
		752
		753	/* Only difference to the fast-path function is that this can handle bit17
		754	* and uses non-atomic copy and kmap functions. */
		755	static int
		756	shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		757	char __user *user_data,
		758	bool page_do_bit17_swizzling,
		759	bool needs_clflush_before,
		760	bool needs_clflush_after)
		761	{
		762	char *vaddr;
		763	int ret;
		764
		765	vaddr = kmap(page);
		766	if (unlikely(needs_clflush_before \|\| page_do_bit17_swizzling))
		767	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		768	page_length,
		769	page_do_bit17_swizzling);
		770	if (page_do_bit17_swizzling)
		771	ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
		772	user_data,
		773	page_length);
		774	else
		775	ret = __copy_from_user(vaddr + shmem_page_offset,
		776	user_data,
		777	page_length);
		778	if (needs_clflush_after)
		779	shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
		780	page_length,
		781	page_do_bit17_swizzling);
		782	kunmap(page);
		783
		784	return ret ? -EFAULT : 0;
		785	}
		786
		787	static int
		788	i915_gem_shmem_pwrite(struct drm_device *dev,
		789	struct drm_i915_gem_object *obj,
		790	struct drm_i915_gem_pwrite *args,
		791	struct drm_file *file)
		792	{
		793	ssize_t remain;
		794	loff_t offset;
		795	char __user *user_data;
		796	int shmem_page_offset, page_length, ret = 0;
		797	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
		798	int hit_slowpath = 0;
		799	int needs_clflush_after = 0;
		800	int needs_clflush_before = 0;
		801	int i;
		802	struct scatterlist *sg;
		803
		804	user_data = (char __user *) (uintptr_t) args->data_ptr;
		805	remain = args->size;
		806
		807	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		808
		809	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		810	/* If we're not in the cpu write domain, set ourself into the gtt
		811	* write domain and manually flush cachelines (if required). This
		812	* optimizes for the case when the gpu will use the data
		813	* right away and we therefore have to clflush anyway. */
		814	if (obj->cache_level == I915_CACHE_NONE)
		815	needs_clflush_after = 1;
		816	if (obj->gtt_space) {
		817	ret = i915_gem_object_set_to_gtt_domain(obj, true);
		818	if (ret)
		819	return ret;
		820	}
		821	}
		822	/* Same trick applies for invalidate partially written cachelines before
		823	* writing. */
		824	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
		825	&& obj->cache_level == I915_CACHE_NONE)
		826	needs_clflush_before = 1;
		827
		828	ret = i915_gem_object_get_pages(obj);
2332	Serge	829	if (ret)
3031	serge	830	return ret;
2332	Serge	831
3031	serge	832	i915_gem_object_pin_pages(obj);
2332	Serge	833
		834	offset = args->offset;
3031	serge	835	obj->dirty = 1;
2332	Serge	836
3031	serge	837	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
2332	Serge	838	struct page *page;
3031	serge	839	int partial_cacheline_write;
2332	Serge	840
3031	serge	841	if (i < offset >> PAGE_SHIFT)
		842	continue;
		843
		844	if (remain <= 0)
		845	break;
		846
2332	Serge	847	/* Operation in this page
		848	*
		849	* shmem_page_offset = offset within page in shmem file
		850	* page_length = bytes to copy for this page
		851	*/
		852	shmem_page_offset = offset_in_page(offset);
		853
		854	page_length = remain;
		855	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		856	page_length = PAGE_SIZE - shmem_page_offset;
		857
3031	serge	858	/* If we don't overwrite a cacheline completely we need to be
		859	* careful to have up-to-date data by first clflushing. Don't
		860	* overcomplicate things and flush the entire patch. */
		861	partial_cacheline_write = needs_clflush_before &&
		862	((shmem_page_offset \| page_length)
		863	& (boot_cpu_data.x86_clflush_size - 1));
2332	Serge	864
3031	serge	865	page = sg_page(sg);
		866	page_do_bit17_swizzling = obj_do_bit17_swizzling &&
		867	(page_to_phys(page) & (1 << 17)) != 0;
2332	Serge	868
3031	serge	869	ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
		870	user_data, page_do_bit17_swizzling,
		871	partial_cacheline_write,
		872	needs_clflush_after);
		873	if (ret == 0)
		874	goto next_page;
		875
		876	hit_slowpath = 1;
		877	mutex_unlock(&dev->struct_mutex);
		878	ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
		879	user_data, page_do_bit17_swizzling,
		880	partial_cacheline_write,
		881	needs_clflush_after);
		882
		883	mutex_lock(&dev->struct_mutex);
		884
		885	next_page:
		886	set_page_dirty(page);
2332	Serge	887	mark_page_accessed(page);
		888
3031	serge	889	if (ret)
		890	goto out;
		891
2332	Serge	892	remain -= page_length;
3031	serge	893	user_data += page_length;
2332	Serge	894	offset += page_length;
		895	}
		896
		897	out:
3031	serge	898	i915_gem_object_unpin_pages(obj);
		899
		900	if (hit_slowpath) {
		901	/* Fixup: Kill any reinstated backing storage pages */
		902	if (obj->madv == __I915_MADV_PURGED)
		903	i915_gem_object_truncate(obj);
		904	/* and flush dirty cachelines in case the object isn't in the cpu write
		905	* domain anymore. */
		906	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		907	i915_gem_clflush_object(obj);
3243	Serge	908	i915_gem_chipset_flush(dev);
3031	serge	909	}
2332	Serge	910	}
		911
3031	serge	912	if (needs_clflush_after)
3243	Serge	913	i915_gem_chipset_flush(dev);
3031	serge	914
2332	Serge	915	return ret;
		916	}
3031	serge	917
		918	/**
		919	* Writes data to the object referenced by handle.
		920	*
		921	* On error, the contents of the buffer that were to be modified are undefined.
		922	*/
		923	int
		924	i915_gem_pwrite_ioctl(struct drm_device dev, void data,
		925	struct drm_file *file)
		926	{
		927	struct drm_i915_gem_pwrite *args = data;
		928	struct drm_i915_gem_object *obj;
		929	int ret;
		930
		931	if (args->size == 0)
		932	return 0;
		933
		934	if (!access_ok(VERIFY_READ,
		935	(char __user *)(uintptr_t)args->data_ptr,
		936	args->size))
		937	return -EFAULT;
		938
		939	ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
		940	args->size);
		941	if (ret)
		942	return -EFAULT;
		943
		944	ret = i915_mutex_lock_interruptible(dev);
		945	if (ret)
		946	return ret;
		947
		948	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		949	if (&obj->base == NULL) {
		950	ret = -ENOENT;
		951	goto unlock;
		952	}
		953
		954	/* Bounds check destination. */
		955	if (args->offset > obj->base.size \|\|
		956	args->size > obj->base.size - args->offset) {
		957	ret = -EINVAL;
		958	goto out;
		959	}
		960
		961	/* prime objects have no backing filp to GEM pread/pwrite
		962	* pages from.
		963	*/
		964	if (!obj->base.filp) {
		965	ret = -EINVAL;
		966	goto out;
		967	}
		968
		969	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
		970
		971	ret = -EFAULT;
		972	/* We can only do the GTT pwrite on untiled buffers, as otherwise
		973	* it would end up going through the fenced access, and we'll get
		974	* different detiling behavior between reading and writing.
		975	* pread/pwrite currently are reading and writing from the CPU
		976	* perspective, requiring manual detiling by the client.
		977	*/
		978	if (obj->phys_obj) {
		979	ret = i915_gem_phys_pwrite(dev, obj, args, file);
		980	goto out;
		981	}
		982
		983	if (obj->cache_level == I915_CACHE_NONE &&
		984	obj->tiling_mode == I915_TILING_NONE &&
		985	obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		986	ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
		987	/* Note that the gtt paths might fail with non-page-backed user
		988	* pointers (e.g. gtt mappings when moving data between
		989	* textures). Fallback to the shmem path in that case. */
		990	}
		991
		992	if (ret == -EFAULT \|\| ret == -ENOSPC)
		993	ret = i915_gem_shmem_pwrite(dev, obj, args, file);
		994
		995	out:
		996	drm_gem_object_unreference(&obj->base);
		997	unlock:
		998	mutex_unlock(&dev->struct_mutex);
		999	return ret;
		1000	}
		1001
2332	Serge	1002	#endif
		1003
3031	serge	1004	int
		1005	i915_gem_check_wedge(struct drm_i915_private *dev_priv,
		1006	bool interruptible)
		1007	{
		1008	if (atomic_read(&dev_priv->mm.wedged)) {
		1009	struct completion *x = &dev_priv->error_completion;
		1010	bool recovery_complete;
		1011	unsigned long flags;
2332	Serge	1012
3031	serge	1013	/* Give the error handler a chance to run. */
		1014	spin_lock_irqsave(&x->wait.lock, flags);
		1015	recovery_complete = x->done > 0;
		1016	spin_unlock_irqrestore(&x->wait.lock, flags);
2332	Serge	1017
3031	serge	1018	/* Non-interruptible callers can't handle -EAGAIN, hence return
		1019	* -EIO unconditionally for these. */
		1020	if (!interruptible)
		1021	return -EIO;
2332	Serge	1022
3031	serge	1023	/* Recovery complete, but still wedged means reset failure. */
		1024	if (recovery_complete)
		1025	return -EIO;
2332	Serge	1026
3031	serge	1027	return -EAGAIN;
		1028	}
2332	Serge	1029
3031	serge	1030	return 0;
		1031	}
2332	Serge	1032
3031	serge	1033	/*
		1034	* Compare seqno against outstanding lazy request. Emit a request if they are
		1035	* equal.
		1036	*/
		1037	static int
		1038	i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
		1039	{
		1040	int ret;
2332	Serge	1041
3031	serge	1042	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
2332	Serge	1043
3031	serge	1044	ret = 0;
		1045	if (seqno == ring->outstanding_lazy_request)
		1046	ret = i915_add_request(ring, NULL, NULL);
2332	Serge	1047
3031	serge	1048	return ret;
		1049	}
2332	Serge	1050
3031	serge	1051	/**
		1052	* __wait_seqno - wait until execution of seqno has finished
		1053	* @ring: the ring expected to report seqno
		1054	* @seqno: duh!
		1055	* @interruptible: do an interruptible wait (normally yes)
		1056	* @timeout: in - how long to wait (NULL forever); out - how much time remaining
		1057	*
		1058	* Returns 0 if the seqno was found within the alloted time. Else returns the
		1059	* errno with remaining time filled in timeout argument.
		1060	*/
		1061	static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
		1062	bool interruptible, struct timespec *timeout)
		1063	{
		1064	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1065	struct timespec before, now, wait_time={1,0};
		1066	unsigned long timeout_jiffies;
		1067	long end;
		1068	bool wait_forever = true;
		1069	int ret;
2332	Serge	1070
3031	serge	1071	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		1072	return 0;
2332	Serge	1073
3031	serge	1074	trace_i915_gem_request_wait_begin(ring, seqno);
2332	Serge	1075
3031	serge	1076	if (timeout != NULL) {
		1077	wait_time = *timeout;
		1078	wait_forever = false;
		1079	}
2332	Serge	1080
3031	serge	1081	// timeout_jiffies = timespec_to_jiffies(&wait_time);
2332	Serge	1082
3031	serge	1083	if (WARN_ON(!ring->irq_get(ring)))
		1084	return -ENODEV;
		1085	#if 0
2332	Serge	1086
3031	serge	1087	/* Record current time in case interrupted by signal, or wedged * */
		1088	getrawmonotonic(&before);
2332	Serge	1089
3031	serge	1090	#define EXIT_COND \
		1091	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1092	atomic_read(&dev_priv->mm.wedged))
		1093	do {
		1094	end = wait_event_timeout(ring->irq_queue, EXIT_COND,
		1095	timeout_jiffies);
2332	Serge	1096
3031	serge	1097	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1098	if (ret)
		1099	end = ret;
		1100	} while (end == 0 && wait_forever);
2332	Serge	1101
3031	serge	1102	getrawmonotonic(&now);
2332	Serge	1103
3031	serge	1104	ring->irq_put(ring);
		1105	trace_i915_gem_request_wait_end(ring, seqno);
		1106	#undef EXIT_COND
2332	Serge	1107
3031	serge	1108	if (timeout) {
		1109	// struct timespec sleep_time = timespec_sub(now, before);
		1110	// timeout = timespec_sub(timeout, sleep_time);
		1111	}
2332	Serge	1112
3031	serge	1113	switch (end) {
		1114	case -EIO:
		1115	case -EAGAIN: /* Wedged */
		1116	case -ERESTARTSYS: /* Signal */
		1117	return (int)end;
		1118	case 0: /* Timeout */
		1119	// if (timeout)
		1120	// set_normalized_timespec(timeout, 0, 0);
		1121	return -ETIME;
		1122	default: /* Completed */
		1123	WARN_ON(end < 0); /* We're not aware of other errors */
		1124	return 0;
		1125	}
		1126	#endif
2332	Serge	1127
3031	serge	1128	#define EXIT_COND \
		1129	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) \|\| \
		1130	atomic_read(&dev_priv->mm.wedged))
		1131	wait_event(ring->irq_queue, EXIT_COND);
		1132	#undef EXIT_COND
		1133	ring->irq_put(ring);
2332	Serge	1134
3031	serge	1135	return 0;
		1136	}
2332	Serge	1137
3031	serge	1138	/**
		1139	* Waits for a sequence number to be signaled, and cleans up the
		1140	* request and object lists appropriately for that event.
		1141	*/
		1142	int
		1143	i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
		1144	{
		1145	struct drm_device *dev = ring->dev;
		1146	struct drm_i915_private *dev_priv = dev->dev_private;
		1147	bool interruptible = dev_priv->mm.interruptible;
		1148	int ret;
2332	Serge	1149
3031	serge	1150	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		1151	BUG_ON(seqno == 0);
2332	Serge	1152
3031	serge	1153	ret = i915_gem_check_wedge(dev_priv, interruptible);
		1154	if (ret)
		1155	return ret;
2332	Serge	1156
3031	serge	1157	ret = i915_gem_check_olr(ring, seqno);
		1158	if (ret)
		1159	return ret;
2332	Serge	1160
3031	serge	1161	return __wait_seqno(ring, seqno, interruptible, NULL);
		1162	}
2332	Serge	1163
3031	serge	1164	/**
		1165	* Ensures that all rendering to the object has completed and the object is
		1166	* safe to unbind from the GTT or access from the CPU.
		1167	*/
		1168	static __must_check int
		1169	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
		1170	bool readonly)
		1171	{
		1172	struct intel_ring_buffer *ring = obj->ring;
		1173	u32 seqno;
		1174	int ret;
2332	Serge	1175
3031	serge	1176	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
		1177	if (seqno == 0)
		1178	return 0;
2332	Serge	1179
3031	serge	1180	ret = i915_wait_seqno(ring, seqno);
		1181	if (ret)
		1182	return ret;
2332	Serge	1183
3031	serge	1184	i915_gem_retire_requests_ring(ring);
2332	Serge	1185
3031	serge	1186	/* Manually manage the write flush as we may have not yet
		1187	* retired the buffer.
		1188	*/
		1189	if (obj->last_write_seqno &&
		1190	i915_seqno_passed(seqno, obj->last_write_seqno)) {
		1191	obj->last_write_seqno = 0;
		1192	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
		1193	}
2332	Serge	1194
3031	serge	1195	return 0;
		1196	}
2332	Serge	1197
		1198
		1199
		1200
		1201
		1202
		1203
		1204
		1205
		1206
		1207
		1208
		1209
		1210
		1211
		1212
		1213
		1214
		1215
		1216
		1217
		1218
		1219
		1220
		1221
		1222
		1223
		1224
		1225
		1226
		1227
		1228
		1229
		1230
		1231
		1232
		1233
3031	serge	1234
		1235
		1236
		1237
		1238
		1239
		1240	/**
		1241	* i915_gem_release_mmap - remove physical page mappings
		1242	* @obj: obj in question
		1243	*
		1244	* Preserve the reservation of the mmapping with the DRM core code, but
		1245	* relinquish ownership of the pages back to the system.
		1246	*
		1247	* It is vital that we remove the page mapping if we have mapped a tiled
		1248	* object through the GTT and then lose the fence register due to
		1249	* resource pressure. Similarly if the object has been moved out of the
		1250	* aperture, than pages mapped into userspace must be revoked. Removing the
		1251	* mapping will then trigger a page fault on the next user access, allowing
		1252	* fixup by i915_gem_fault().
		1253	*/
		1254	void
		1255	i915_gem_release_mmap(struct drm_i915_gem_object *obj)
		1256	{
		1257	if (!obj->fault_mappable)
		1258	return;
		1259
		1260	if (obj->base.dev->dev_mapping)
		1261	// unmap_mapping_range(obj->base.dev->dev_mapping,
		1262	// (loff_t)obj->base.map_list.hash.key<
		1263	// obj->base.size, 1);
		1264
		1265	obj->fault_mappable = false;
		1266	}
		1267
2332	Serge	1268	static uint32_t
		1269	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		1270	{
		1271	uint32_t gtt_size;
		1272
		1273	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1274	tiling_mode == I915_TILING_NONE)
		1275	return size;
		1276
		1277	/* Previous chips need a power-of-two fence region when tiling */
		1278	if (INTEL_INFO(dev)->gen == 3)
		1279	gtt_size = 1024*1024;
		1280	else
		1281	gtt_size = 512*1024;
		1282
		1283	while (gtt_size < size)
		1284	gtt_size <<= 1;
		1285
		1286	return gtt_size;
		1287	}
		1288
		1289	/**
		1290	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		1291	* @obj: object to check
		1292	*
		1293	* Return the required GTT alignment for an object, taking into account
		1294	* potential fence register mapping.
		1295	*/
		1296	static uint32_t
		1297	i915_gem_get_gtt_alignment(struct drm_device *dev,
		1298	uint32_t size,
		1299	int tiling_mode)
		1300	{
		1301	/*
		1302	* Minimum alignment is 4k (GTT page size), but might be greater
		1303	* if a fence register is needed for the object.
		1304	*/
		1305	if (INTEL_INFO(dev)->gen >= 4 \|\|
		1306	tiling_mode == I915_TILING_NONE)
		1307	return 4096;
		1308
		1309	/*
		1310	* Previous chips need to be aligned to the size of the smallest
		1311	* fence register that can contain the object.
		1312	*/
		1313	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1314	}
		1315
		1316	/**
		1317	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		1318	* unfenced object
		1319	* @dev: the device
		1320	* @size: size of the object
		1321	* @tiling_mode: tiling mode of the object
		1322	*
		1323	* Return the required GTT alignment for an object, only taking into account
		1324	* unfenced tiled surface requirements.
		1325	*/
		1326	uint32_t
		1327	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		1328	uint32_t size,
		1329	int tiling_mode)
		1330	{
		1331	/*
		1332	* Minimum alignment is 4k (GTT page size) for sane hw.
		1333	*/
		1334	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		1335	tiling_mode == I915_TILING_NONE)
		1336	return 4096;
		1337
		1338	/* Previous hardware however needs to be aligned to a power-of-two
		1339	* tile height. The simplest method for determining this is to reuse
		1340	* the power-of-tile object size.
		1341	*/
		1342	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		1343	}
		1344
3031	serge	1345	/* Immediately discard the backing storage */
		1346	static void
		1347	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		1348	{
		1349	// struct inode *inode;
2332	Serge	1350
3031	serge	1351	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	1352
3031	serge	1353	// if (obj->base.filp == NULL)
		1354	// return;
2332	Serge	1355
3031	serge	1356	/* Our goal here is to return as much of the memory as
		1357	* is possible back to the system as we are called from OOM.
		1358	* To do this we must instruct the shmfs to drop all of its
		1359	* backing pages, now.
		1360	*/
		1361	// inode = obj->base.filp->f_path.dentry->d_inode;
		1362	// shmem_truncate_range(inode, 0, (loff_t)-1);
2332	Serge	1363
3031	serge	1364	obj->madv = __I915_MADV_PURGED;
		1365	}
2332	Serge	1366
3031	serge	1367	static inline int
		1368	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		1369	{
		1370	return obj->madv == I915_MADV_DONTNEED;
		1371	}
2332	Serge	1372
3031	serge	1373	static void
		1374	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		1375	{
3243	Serge	1376	int page_count = obj->base.size / PAGE_SIZE;
		1377	struct scatterlist *sg;
3031	serge	1378	int ret, i;
2332	Serge	1379
3031	serge	1380	BUG_ON(obj->madv == __I915_MADV_PURGED);
2332	Serge	1381
3031	serge	1382	ret = i915_gem_object_set_to_cpu_domain(obj, true);
		1383	if (ret) {
		1384	/* In the event of a disaster, abandon all caches and
		1385	* hope for the best.
		1386	*/
		1387	WARN_ON(ret != -EIO);
		1388	i915_gem_clflush_object(obj);
		1389	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1390	}
2332	Serge	1391
3031	serge	1392	if (obj->madv == I915_MADV_DONTNEED)
		1393	obj->dirty = 0;
2332	Serge	1394
3243	Serge	1395	for_each_sg(obj->pages->sgl, sg, page_count, i) {
		1396	struct page *page = sg_page(sg);
2332	Serge	1397
3243	Serge	1398
		1399
		1400	page_cache_release(page);
		1401	}
		1402	//DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count);
3031	serge	1403	obj->dirty = 0;
3243	Serge	1404
		1405	sg_free_table(obj->pages);
		1406	kfree(obj->pages);
3031	serge	1407	}
2332	Serge	1408
3031	serge	1409	static int
		1410	i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
		1411	{
		1412	const struct drm_i915_gem_object_ops *ops = obj->ops;
2332	Serge	1413
3243	Serge	1414	if (obj->pages == NULL)
3031	serge	1415	return 0;
2332	Serge	1416
3031	serge	1417	BUG_ON(obj->gtt_space);
		1418
		1419	if (obj->pages_pin_count)
		1420	return -EBUSY;
		1421
3243	Serge	1422	/* ->put_pages might need to allocate memory for the bit17 swizzle
		1423	* array, hence protect them from being reaped by removing them from gtt
		1424	* lists early. */
		1425	list_del(&obj->gtt_list);
		1426
3031	serge	1427	ops->put_pages(obj);
3243	Serge	1428	obj->pages = NULL;
3031	serge	1429
		1430	if (i915_gem_object_is_purgeable(obj))
		1431	i915_gem_object_truncate(obj);
		1432
		1433	return 0;
		1434	}
		1435
		1436
		1437
		1438
		1439
		1440
		1441
		1442
2332	Serge	1443	static int
3031	serge	1444	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2332	Serge	1445	{
3243	Serge	1446	int page_count, i;
		1447	struct sg_table *st;
		1448	struct scatterlist *sg;
		1449	struct page *page;
		1450	gfp_t gfp;
2332	Serge	1451
3243	Serge	1452	/* Assert that the object is not currently in any GPU domain. As it
		1453	* wasn't in the GTT, there shouldn't be any way it could have been in
		1454	* a GPU cache
2332	Serge	1455	*/
3243	Serge	1456	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1457	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1458
		1459	st = kmalloc(sizeof(*st), GFP_KERNEL);
		1460	if (st == NULL)
		1461	return -ENOMEM;
		1462
2332	Serge	1463	page_count = obj->base.size / PAGE_SIZE;
3243	Serge	1464	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		1465	sg_free_table(st);
		1466	kfree(st);
2332	Serge	1467	return -ENOMEM;
3243	Serge	1468	}
2332	Serge	1469
3243	Serge	1470	/* Get the list of pages out of our struct file. They'll be pinned
		1471	* at this point until we release them.
		1472	*
		1473	* Fail silently without starting the shrinker
		1474	*/
		1475	for_each_sg(st->sgl, sg, page_count, i) {
3255	Serge	1476	page = (struct page *)AllocPage(); // oh-oh
3031	serge	1477	if ( page == 0 )
2332	Serge	1478	goto err_pages;
		1479
3243	Serge	1480	sg_set_page(sg, page, PAGE_SIZE, 0);
		1481	}
3031	serge	1482
3243	Serge	1483	obj->pages = st;
3031	serge	1484
3243	Serge	1485	// DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count);
2332	Serge	1486
		1487	return 0;
		1488
		1489	err_pages:
3243	Serge	1490	for_each_sg(st->sgl, sg, i, page_count)
		1491	page_cache_release(sg_page(sg));
		1492	sg_free_table(st);
		1493	kfree(st);
		1494	return PTR_ERR(page);
2332	Serge	1495	}
		1496
3031	serge	1497	/* Ensure that the associated pages are gathered from the backing storage
		1498	* and pinned into our object. i915_gem_object_get_pages() may be called
		1499	* multiple times before they are released by a single call to
		1500	* i915_gem_object_put_pages() - once the pages are no longer referenced
		1501	* either as a result of memory pressure (reaping pages under the shrinker)
		1502	* or as the object is itself released.
		1503	*/
		1504	int
		1505	i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2332	Serge	1506	{
3031	serge	1507	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1508	const struct drm_i915_gem_object_ops *ops = obj->ops;
		1509	int ret;
2332	Serge	1510
3243	Serge	1511	if (obj->pages)
3031	serge	1512	return 0;
2332	Serge	1513
3031	serge	1514	BUG_ON(obj->pages_pin_count);
2332	Serge	1515
3031	serge	1516	ret = ops->get_pages(obj);
		1517	if (ret)
		1518	return ret;
2344	Serge	1519
3031	serge	1520	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
3243	Serge	1521	return 0;
2332	Serge	1522	}
		1523
		1524	void
		1525	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3243	Serge	1526	struct intel_ring_buffer *ring)
2332	Serge	1527	{
		1528	struct drm_device *dev = obj->base.dev;
		1529	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	1530	u32 seqno = intel_ring_get_seqno(ring);
2332	Serge	1531
		1532	BUG_ON(ring == NULL);
		1533	obj->ring = ring;
		1534
		1535	/* Add a reference if we're newly entering the active list. */
		1536	if (!obj->active) {
2344	Serge	1537	drm_gem_object_reference(&obj->base);
2332	Serge	1538	obj->active = 1;
		1539	}
		1540
		1541	/* Move from whatever list we were on to the tail of execution. */
		1542	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		1543	list_move_tail(&obj->ring_list, &ring->active_list);
		1544
3031	serge	1545	obj->last_read_seqno = seqno;
		1546
2332	Serge	1547	if (obj->fenced_gpu_access) {
3031	serge	1548	obj->last_fenced_seqno = seqno;
		1549
		1550	/* Bump MRU to take account of the delayed flush */
		1551	if (obj->fence_reg != I915_FENCE_REG_NONE) {
2332	Serge	1552	struct drm_i915_fence_reg *reg;
		1553
		1554	reg = &dev_priv->fence_regs[obj->fence_reg];
3031	serge	1555	list_move_tail(®->lru_list,
		1556	&dev_priv->mm.fence_list);
		1557	}
2332	Serge	1558	}
		1559	}
		1560
2344	Serge	1561	static void
3031	serge	1562	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2344	Serge	1563	{
		1564	struct drm_device *dev = obj->base.dev;
3031	serge	1565	struct drm_i915_private *dev_priv = dev->dev_private;
2332	Serge	1566
3031	serge	1567	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
2344	Serge	1568	BUG_ON(!obj->active);
2332	Serge	1569
3031	serge	1570	if (obj->pin_count) /* are we a framebuffer? */
		1571	intel_mark_fb_idle(obj);
2344	Serge	1572
2352	Serge	1573	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	1574
3031	serge	1575	list_del_init(&obj->ring_list);
2352	Serge	1576	obj->ring = NULL;
2344	Serge	1577
3031	serge	1578	obj->last_read_seqno = 0;
		1579	obj->last_write_seqno = 0;
		1580	obj->base.write_domain = 0;
		1581
		1582	obj->last_fenced_seqno = 0;
2352	Serge	1583	obj->fenced_gpu_access = false;
2344	Serge	1584
2352	Serge	1585	obj->active = 0;
		1586	drm_gem_object_unreference(&obj->base);
		1587
		1588	WARN_ON(i915_verify_lists(dev));
		1589	}
		1590
3243	Serge	1591	static int
		1592	i915_gem_handle_seqno_wrap(struct drm_device *dev)
2344	Serge	1593	{
3243	Serge	1594	struct drm_i915_private *dev_priv = dev->dev_private;
		1595	struct intel_ring_buffer *ring;
		1596	int ret, i, j;
2344	Serge	1597
3243	Serge	1598	/* The hardware uses various monotonic 32-bit counters, if we
		1599	* detect that they will wraparound we need to idle the GPU
		1600	* and reset those counters.
		1601	*/
		1602	ret = 0;
		1603	for_each_ring(ring, dev_priv, i) {
		1604	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1605	ret \|= ring->sync_seqno[j] != 0;
		1606	}
		1607	if (ret == 0)
		1608	return ret;
2344	Serge	1609
3243	Serge	1610	ret = i915_gpu_idle(dev);
		1611	if (ret)
		1612	return ret;
		1613
		1614	i915_gem_retire_requests(dev);
		1615	for_each_ring(ring, dev_priv, i) {
		1616	for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
		1617	ring->sync_seqno[j] = 0;
		1618	}
		1619
		1620	return 0;
2344	Serge	1621	}
		1622
3243	Serge	1623	int
		1624	i915_gem_get_seqno(struct drm_device dev, u32 seqno)
2344	Serge	1625	{
3243	Serge	1626	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	1627
3243	Serge	1628	/* reserve 0 for non-seqno */
		1629	if (dev_priv->next_seqno == 0) {
		1630	int ret = i915_gem_handle_seqno_wrap(dev);
		1631	if (ret)
		1632	return ret;
		1633
		1634	dev_priv->next_seqno = 1;
		1635	}
		1636
		1637	*seqno = dev_priv->next_seqno++;
		1638	return 0;
2332	Serge	1639	}
		1640
2352	Serge	1641	int
		1642	i915_add_request(struct intel_ring_buffer *ring,
		1643	struct drm_file *file,
3031	serge	1644	u32 *out_seqno)
2352	Serge	1645	{
		1646	drm_i915_private_t *dev_priv = ring->dev->dev_private;
3031	serge	1647	struct drm_i915_gem_request *request;
		1648	u32 request_ring_position;
2352	Serge	1649	int was_empty;
		1650	int ret;
2332	Serge	1651
3031	serge	1652	/*
		1653	* Emit any outstanding flushes - execbuf can fail to emit the flush
		1654	* after having emitted the batchbuffer command. Hence we need to fix
		1655	* things up similar to emitting the lazy request. The difference here
		1656	* is that the flush _must_ happen before the next request, no matter
		1657	* what.
		1658	*/
		1659	ret = intel_ring_flush_all_caches(ring);
		1660	if (ret)
		1661	return ret;
2332	Serge	1662
3031	serge	1663	request = kmalloc(sizeof(*request), GFP_KERNEL);
		1664	if (request == NULL)
		1665	return -ENOMEM;
		1666
		1667
		1668	/* Record the position of the start of the request so that
		1669	* should we detect the updated seqno part-way through the
		1670	* GPU processing the request, we never over-estimate the
		1671	* position of the head.
		1672	*/
		1673	request_ring_position = intel_ring_get_tail(ring);
		1674
3243	Serge	1675	ret = ring->add_request(ring);
3031	serge	1676	if (ret) {
		1677	kfree(request);
2352	Serge	1678	return ret;
3031	serge	1679	}
2332	Serge	1680
3243	Serge	1681	request->seqno = intel_ring_get_seqno(ring);
2352	Serge	1682	request->ring = ring;
3031	serge	1683	request->tail = request_ring_position;
		1684	request->emitted_jiffies = GetTimerTicks();
2352	Serge	1685	was_empty = list_empty(&ring->request_list);
		1686	list_add_tail(&request->list, &ring->request_list);
3031	serge	1687	request->file_priv = NULL;
2332	Serge	1688
		1689
3031	serge	1690	ring->outstanding_lazy_request = 0;
2332	Serge	1691
2360	Serge	1692	if (!dev_priv->mm.suspended) {
		1693	if (i915_enable_hangcheck) {
2352	Serge	1694	// mod_timer(&dev_priv->hangcheck_timer,
		1695	// jiffies +
		1696	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	1697	}
3031	serge	1698	if (was_empty) {
2360	Serge	1699	queue_delayed_work(dev_priv->wq,
		1700	&dev_priv->mm.retire_work, HZ);
3031	serge	1701	intel_mark_busy(dev_priv->dev);
		1702	}
2360	Serge	1703	}
3031	serge	1704
		1705	if (out_seqno)
3243	Serge	1706	*out_seqno = request->seqno;
2352	Serge	1707	return 0;
		1708	}
2332	Serge	1709
		1710
		1711
		1712
3031	serge	1713	static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
		1714	struct intel_ring_buffer *ring)
		1715	{
		1716	while (!list_empty(&ring->request_list)) {
		1717	struct drm_i915_gem_request *request;
2332	Serge	1718
3031	serge	1719	request = list_first_entry(&ring->request_list,
		1720	struct drm_i915_gem_request,
		1721	list);
2332	Serge	1722
3031	serge	1723	list_del(&request->list);
		1724	// i915_gem_request_remove_from_client(request);
		1725	kfree(request);
		1726	}
2332	Serge	1727
3031	serge	1728	while (!list_empty(&ring->active_list)) {
		1729	struct drm_i915_gem_object *obj;
2332	Serge	1730
3031	serge	1731	obj = list_first_entry(&ring->active_list,
		1732	struct drm_i915_gem_object,
		1733	ring_list);
2332	Serge	1734
3031	serge	1735	i915_gem_object_move_to_inactive(obj);
		1736	}
		1737	}
2332	Serge	1738
3031	serge	1739	static void i915_gem_reset_fences(struct drm_device *dev)
		1740	{
		1741	struct drm_i915_private *dev_priv = dev->dev_private;
		1742	int i;
2332	Serge	1743
3031	serge	1744	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1745	struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2332	Serge	1746
3031	serge	1747	i915_gem_write_fence(dev, i, NULL);
2360	Serge	1748
3031	serge	1749	if (reg->obj)
		1750	i915_gem_object_fence_lost(reg->obj);
2360	Serge	1751
3031	serge	1752	reg->pin_count = 0;
		1753	reg->obj = NULL;
		1754	INIT_LIST_HEAD(®->lru_list);
		1755	}
2360	Serge	1756
3031	serge	1757	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1758	}
2360	Serge	1759
3031	serge	1760	void i915_gem_reset(struct drm_device *dev)
		1761	{
		1762	struct drm_i915_private *dev_priv = dev->dev_private;
		1763	struct drm_i915_gem_object *obj;
		1764	struct intel_ring_buffer *ring;
		1765	int i;
2360	Serge	1766
3031	serge	1767	for_each_ring(ring, dev_priv, i)
		1768	i915_gem_reset_ring_lists(dev_priv, ring);
2360	Serge	1769
3031	serge	1770	/* Move everything out of the GPU domains to ensure we do any
		1771	* necessary invalidation upon reuse.
		1772	*/
		1773	list_for_each_entry(obj,
		1774	&dev_priv->mm.inactive_list,
		1775	mm_list)
		1776	{
		1777	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
		1778	}
2360	Serge	1779
3031	serge	1780	/* The fence registers are invalidated so clear them out */
		1781	i915_gem_reset_fences(dev);
		1782	}
2360	Serge	1783
2352	Serge	1784	/**
		1785	* This function clears the request list as sequence numbers are passed.
		1786	*/
3031	serge	1787	void
2352	Serge	1788	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1789	{
		1790	uint32_t seqno;
2332	Serge	1791
2352	Serge	1792	if (list_empty(&ring->request_list))
		1793	return;
2332	Serge	1794
2352	Serge	1795	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1796
3031	serge	1797	seqno = ring->get_seqno(ring, true);
2332	Serge	1798
2352	Serge	1799	while (!list_empty(&ring->request_list)) {
		1800	struct drm_i915_gem_request *request;
2332	Serge	1801
2352	Serge	1802	request = list_first_entry(&ring->request_list,
		1803	struct drm_i915_gem_request,
		1804	list);
2332	Serge	1805
2352	Serge	1806	if (!i915_seqno_passed(seqno, request->seqno))
		1807	break;
2332	Serge	1808
2352	Serge	1809	trace_i915_gem_request_retire(ring, request->seqno);
3031	serge	1810	/* We know the GPU must have read the request to have
		1811	* sent us the seqno + interrupt, so use the position
		1812	* of tail of the request to update the last known position
		1813	* of the GPU head.
		1814	*/
		1815	ring->last_retired_head = request->tail;
2332	Serge	1816
2352	Serge	1817	list_del(&request->list);
		1818	kfree(request);
		1819	}
2332	Serge	1820
2352	Serge	1821	/* Move any buffers on the active list that are no longer referenced
		1822	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1823	*/
		1824	while (!list_empty(&ring->active_list)) {
		1825	struct drm_i915_gem_object *obj;
2332	Serge	1826
2352	Serge	1827	obj = list_first_entry(&ring->active_list,
		1828	struct drm_i915_gem_object,
		1829	ring_list);
2332	Serge	1830
3031	serge	1831	if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2352	Serge	1832	break;
2332	Serge	1833
2352	Serge	1834	i915_gem_object_move_to_inactive(obj);
		1835	}
2332	Serge	1836
2352	Serge	1837	if (unlikely(ring->trace_irq_seqno &&
		1838	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1839	ring->irq_put(ring);
		1840	ring->trace_irq_seqno = 0;
		1841	}
2332	Serge	1842
2352	Serge	1843	WARN_ON(i915_verify_lists(ring->dev));
		1844	}
2332	Serge	1845
2352	Serge	1846	void
		1847	i915_gem_retire_requests(struct drm_device *dev)
		1848	{
		1849	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	1850	struct intel_ring_buffer *ring;
2352	Serge	1851	int i;
2332	Serge	1852
3031	serge	1853	for_each_ring(ring, dev_priv, i)
		1854	i915_gem_retire_requests_ring(ring);
2352	Serge	1855	}
		1856
2360	Serge	1857	static void
		1858	i915_gem_retire_work_handler(struct work_struct *work)
		1859	{
		1860	drm_i915_private_t *dev_priv;
		1861	struct drm_device *dev;
3031	serge	1862	struct intel_ring_buffer *ring;
2360	Serge	1863	bool idle;
		1864	int i;
2352	Serge	1865
2360	Serge	1866	dev_priv = container_of(work, drm_i915_private_t,
		1867	mm.retire_work.work);
		1868	dev = dev_priv->dev;
2352	Serge	1869
2360	Serge	1870	/* Come back later if the device is busy... */
		1871	if (!mutex_trylock(&dev->struct_mutex)) {
		1872	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3243	Serge	1873	return;
2360	Serge	1874	}
2352	Serge	1875
2360	Serge	1876	i915_gem_retire_requests(dev);
2352	Serge	1877
2360	Serge	1878	/* Send a periodic flush down the ring so we don't hold onto GEM
		1879	* objects indefinitely.
		1880	*/
		1881	idle = true;
3031	serge	1882	for_each_ring(ring, dev_priv, i) {
		1883	if (ring->gpu_caches_dirty)
		1884	i915_add_request(ring, NULL, NULL);
2352	Serge	1885
2360	Serge	1886	idle &= list_empty(&ring->request_list);
		1887	}
2352	Serge	1888
2360	Serge	1889	if (!dev_priv->mm.suspended && !idle)
		1890	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
3031	serge	1891	if (idle)
		1892	intel_mark_idle(dev);
2360	Serge	1893
		1894	mutex_unlock(&dev->struct_mutex);
		1895	}
		1896
2344	Serge	1897	/**
3031	serge	1898	* Ensures that an object will eventually get non-busy by flushing any required
		1899	* write domains, emitting any outstanding lazy request and retiring and
		1900	* completed requests.
2352	Serge	1901	*/
3031	serge	1902	static int
		1903	i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2352	Serge	1904	{
3031	serge	1905	int ret;
2352	Serge	1906
3031	serge	1907	if (obj->active) {
		1908	ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
		1909	if (ret)
		1910	return ret;
2352	Serge	1911
3031	serge	1912	i915_gem_retire_requests_ring(obj->ring);
		1913	}
2352	Serge	1914
3031	serge	1915	return 0;
		1916	}
2352	Serge	1917
3243	Serge	1918	/**
		1919	* i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
		1920	* @DRM_IOCTL_ARGS: standard ioctl arguments
		1921	*
		1922	* Returns 0 if successful, else an error is returned with the remaining time in
		1923	* the timeout parameter.
		1924	* -ETIME: object is still busy after timeout
		1925	* -ERESTARTSYS: signal interrupted the wait
		1926	* -ENONENT: object doesn't exist
		1927	* Also possible, but rare:
		1928	* -EAGAIN: GPU wedged
		1929	* -ENOMEM: damn
		1930	* -ENODEV: Internal IRQ fail
		1931	* -E?: The add request failed
		1932	*
		1933	* The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
		1934	* non-zero timeout parameter the wait ioctl will wait for the given number of
		1935	* nanoseconds on an object becoming unbusy. Since the wait itself does so
		1936	* without holding struct_mutex the object may become re-busied before this
		1937	* function completes. A similar but shorter * race condition exists in the busy
		1938	* ioctl
		1939	*/
2352	Serge	1940
		1941
		1942
		1943
		1944
		1945
		1946
		1947
		1948
3243	Serge	1949
		1950
		1951
		1952
		1953
2352	Serge	1954	/**
3031	serge	1955	* i915_gem_object_sync - sync an object to a ring.
		1956	*
		1957	* @obj: object which may be in use on another ring.
		1958	* @to: ring we wish to use the object on. May be NULL.
		1959	*
		1960	* This code is meant to abstract object synchronization with the GPU.
		1961	* Calling with NULL implies synchronizing the object with the CPU
		1962	* rather than a particular GPU ring.
		1963	*
		1964	* Returns 0 if successful, else propagates up the lower layer error.
2344	Serge	1965	*/
		1966	int
3031	serge	1967	i915_gem_object_sync(struct drm_i915_gem_object *obj,
		1968	struct intel_ring_buffer *to)
2344	Serge	1969	{
3031	serge	1970	struct intel_ring_buffer *from = obj->ring;
		1971	u32 seqno;
		1972	int ret, idx;
2332	Serge	1973
3031	serge	1974	if (from == NULL \|\| to == from)
		1975	return 0;
2332	Serge	1976
3031	serge	1977	if (to == NULL \|\| !i915_semaphore_is_enabled(obj->base.dev))
		1978	return i915_gem_object_wait_rendering(obj, false);
2332	Serge	1979
3031	serge	1980	idx = intel_ring_sync_index(from, to);
		1981
		1982	seqno = obj->last_read_seqno;
		1983	if (seqno <= from->sync_seqno[idx])
		1984	return 0;
		1985
		1986	ret = i915_gem_check_olr(obj->ring, seqno);
		1987	if (ret)
		1988	return ret;
		1989
		1990	ret = to->sync_to(to, from, seqno);
		1991	if (!ret)
3243	Serge	1992	/* We use last_read_seqno because sync_to()
		1993	* might have just caused seqno wrap under
		1994	* the radar.
		1995	*/
		1996	from->sync_seqno[idx] = obj->last_read_seqno;
3031	serge	1997
		1998	return ret;
2344	Serge	1999	}
2332	Serge	2000
2344	Serge	2001	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		2002	{
		2003	u32 old_write_domain, old_read_domains;
2332	Serge	2004
2344	Serge	2005	/* Act a barrier for all accesses through the GTT */
		2006	mb();
2332	Serge	2007
2344	Serge	2008	/* Force a pagefault for domain tracking on next user access */
		2009	// i915_gem_release_mmap(obj);
2332	Serge	2010
2344	Serge	2011	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		2012	return;
2332	Serge	2013
2344	Serge	2014	old_read_domains = obj->base.read_domains;
		2015	old_write_domain = obj->base.write_domain;
2351	Serge	2016
2344	Serge	2017	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		2018	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	2019
2351	Serge	2020	trace_i915_gem_object_change_domain(obj,
		2021	old_read_domains,
		2022	old_write_domain);
2344	Serge	2023	}
2332	Serge	2024
2344	Serge	2025	/**
		2026	* Unbinds an object from the GTT aperture.
		2027	*/
		2028	int
		2029	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		2030	{
3031	serge	2031	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2344	Serge	2032	int ret = 0;
2332	Serge	2033
2344	Serge	2034	if (obj->gtt_space == NULL)
		2035	return 0;
2332	Serge	2036
3031	serge	2037	if (obj->pin_count)
		2038	return -EBUSY;
2332	Serge	2039
3243	Serge	2040	BUG_ON(obj->pages == NULL);
3031	serge	2041
2344	Serge	2042	ret = i915_gem_object_finish_gpu(obj);
3031	serge	2043	if (ret)
2344	Serge	2044	return ret;
		2045	/* Continue on if we fail due to EIO, the GPU is hung so we
		2046	* should be safe and we need to cleanup or else we might
		2047	* cause memory corruption through use-after-free.
		2048	*/
2332	Serge	2049
2344	Serge	2050	i915_gem_object_finish_gtt(obj);
2332	Serge	2051
2344	Serge	2052	/* release the fence reg _after_ flushing */
		2053	ret = i915_gem_object_put_fence(obj);
3031	serge	2054	if (ret)
2344	Serge	2055	return ret;
2332	Serge	2056
2351	Serge	2057	trace_i915_gem_object_unbind(obj);
2332	Serge	2058
3031	serge	2059	if (obj->has_global_gtt_mapping)
3243	Serge	2060	i915_gem_gtt_unbind_object(obj);
3031	serge	2061	if (obj->has_aliasing_ppgtt_mapping) {
		2062	i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		2063	obj->has_aliasing_ppgtt_mapping = 0;
		2064	}
		2065	i915_gem_gtt_finish_object(obj);
2332	Serge	2066
3031	serge	2067	list_del(&obj->mm_list);
		2068	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
2344	Serge	2069	/* Avoid an unnecessary call to unbind on rebind. */
		2070	obj->map_and_fenceable = true;
2332	Serge	2071
2344	Serge	2072	drm_mm_put_block(obj->gtt_space);
		2073	obj->gtt_space = NULL;
		2074	obj->gtt_offset = 0;
2332	Serge	2075
2344	Serge	2076	return 0;
		2077	}
2332	Serge	2078
3031	serge	2079	int i915_gpu_idle(struct drm_device *dev)
2344	Serge	2080	{
		2081	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	2082	struct intel_ring_buffer *ring;
2344	Serge	2083	int ret, i;
2332	Serge	2084
2344	Serge	2085	/* Flush everything onto the inactive list. */
3031	serge	2086	for_each_ring(ring, dev_priv, i) {
		2087	ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2344	Serge	2088	if (ret)
		2089	return ret;
3031	serge	2090
3243	Serge	2091	ret = intel_ring_idle(ring);
3031	serge	2092	if (ret)
		2093	return ret;
2344	Serge	2094	}
2332	Serge	2095
2344	Serge	2096	return 0;
		2097	}
2332	Serge	2098
3031	serge	2099	static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
		2100	struct drm_i915_gem_object *obj)
		2101	{
		2102	drm_i915_private_t *dev_priv = dev->dev_private;
		2103	uint64_t val;
2332	Serge	2104
3031	serge	2105	if (obj) {
		2106	u32 size = obj->gtt_space->size;
2332	Serge	2107
3031	serge	2108	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2109	0xfffff000) << 32;
		2110	val \|= obj->gtt_offset & 0xfffff000;
		2111	val \|= (uint64_t)((obj->stride / 128) - 1) <<
		2112	SANDYBRIDGE_FENCE_PITCH_SHIFT;
2332	Serge	2113
3031	serge	2114	if (obj->tiling_mode == I915_TILING_Y)
		2115	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2116	val \|= I965_FENCE_REG_VALID;
		2117	} else
		2118	val = 0;
2332	Serge	2119
3031	serge	2120	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
		2121	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
		2122	}
2332	Serge	2123
3031	serge	2124	static void i965_write_fence_reg(struct drm_device *dev, int reg,
		2125	struct drm_i915_gem_object *obj)
		2126	{
		2127	drm_i915_private_t *dev_priv = dev->dev_private;
		2128	uint64_t val;
2332	Serge	2129
3031	serge	2130	if (obj) {
		2131	u32 size = obj->gtt_space->size;
2332	Serge	2132
3031	serge	2133	val = (uint64_t)((obj->gtt_offset + size - 4096) &
		2134	0xfffff000) << 32;
		2135	val \|= obj->gtt_offset & 0xfffff000;
		2136	val \|= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
		2137	if (obj->tiling_mode == I915_TILING_Y)
		2138	val \|= 1 << I965_FENCE_TILING_Y_SHIFT;
		2139	val \|= I965_FENCE_REG_VALID;
		2140	} else
		2141	val = 0;
2332	Serge	2142
3031	serge	2143	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
		2144	POSTING_READ(FENCE_REG_965_0 + reg * 8);
		2145	}
2332	Serge	2146
3031	serge	2147	static void i915_write_fence_reg(struct drm_device *dev, int reg,
		2148	struct drm_i915_gem_object *obj)
		2149	{
		2150	drm_i915_private_t *dev_priv = dev->dev_private;
		2151	u32 val;
2332	Serge	2152
3031	serge	2153	if (obj) {
		2154	u32 size = obj->gtt_space->size;
		2155	int pitch_val;
		2156	int tile_width;
2332	Serge	2157
3031	serge	2158	WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) \|\|
		2159	(size & -size) != size \|\|
		2160	(obj->gtt_offset & (size - 1)),
		2161	"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		2162	obj->gtt_offset, obj->map_and_fenceable, size);
2332	Serge	2163
3031	serge	2164	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
		2165	tile_width = 128;
		2166	else
		2167	tile_width = 512;
2332	Serge	2168
3031	serge	2169	/* Note: pitch better be a power of two tile widths */
		2170	pitch_val = obj->stride / tile_width;
		2171	pitch_val = ffs(pitch_val) - 1;
2332	Serge	2172
3031	serge	2173	val = obj->gtt_offset;
		2174	if (obj->tiling_mode == I915_TILING_Y)
		2175	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2176	val \|= I915_FENCE_SIZE_BITS(size);
		2177	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2178	val \|= I830_FENCE_REG_VALID;
		2179	} else
		2180	val = 0;
2332	Serge	2181
3031	serge	2182	if (reg < 8)
		2183	reg = FENCE_REG_830_0 + reg * 4;
		2184	else
		2185	reg = FENCE_REG_945_8 + (reg - 8) * 4;
2332	Serge	2186
3031	serge	2187	I915_WRITE(reg, val);
		2188	POSTING_READ(reg);
		2189	}
2332	Serge	2190
3031	serge	2191	static void i830_write_fence_reg(struct drm_device *dev, int reg,
		2192	struct drm_i915_gem_object *obj)
		2193	{
		2194	drm_i915_private_t *dev_priv = dev->dev_private;
		2195	uint32_t val;
2344	Serge	2196
3031	serge	2197	if (obj) {
		2198	u32 size = obj->gtt_space->size;
		2199	uint32_t pitch_val;
2344	Serge	2200
3031	serge	2201	WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) \|\|
		2202	(size & -size) != size \|\|
		2203	(obj->gtt_offset & (size - 1)),
		2204	"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		2205	obj->gtt_offset, size);
2344	Serge	2206
3031	serge	2207	pitch_val = obj->stride / 128;
		2208	pitch_val = ffs(pitch_val) - 1;
2344	Serge	2209
3031	serge	2210	val = obj->gtt_offset;
		2211	if (obj->tiling_mode == I915_TILING_Y)
		2212	val \|= 1 << I830_FENCE_TILING_Y_SHIFT;
		2213	val \|= I830_FENCE_SIZE_BITS(size);
		2214	val \|= pitch_val << I830_FENCE_PITCH_SHIFT;
		2215	val \|= I830_FENCE_REG_VALID;
		2216	} else
		2217	val = 0;
		2218
		2219	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
		2220	POSTING_READ(FENCE_REG_830_0 + reg * 4);
		2221	}
		2222
		2223	static void i915_gem_write_fence(struct drm_device *dev, int reg,
		2224	struct drm_i915_gem_object *obj)
2332	Serge	2225	{
3031	serge	2226	switch (INTEL_INFO(dev)->gen) {
		2227	case 7:
		2228	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
		2229	case 5:
		2230	case 4: i965_write_fence_reg(dev, reg, obj); break;
		2231	case 3: i915_write_fence_reg(dev, reg, obj); break;
		2232	case 2: i830_write_fence_reg(dev, reg, obj); break;
		2233	default: break;
		2234	}
2344	Serge	2235	}
		2236
3031	serge	2237	static inline int fence_number(struct drm_i915_private *dev_priv,
		2238	struct drm_i915_fence_reg *fence)
2344	Serge	2239	{
3031	serge	2240	return fence - dev_priv->fence_regs;
		2241	}
2332	Serge	2242
3031	serge	2243	static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
		2244	struct drm_i915_fence_reg *fence,
		2245	bool enable)
		2246	{
		2247	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		2248	int reg = fence_number(dev_priv, fence);
2332	Serge	2249
3031	serge	2250	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
		2251
		2252	if (enable) {
		2253	obj->fence_reg = reg;
		2254	fence->obj = obj;
		2255	list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
		2256	} else {
		2257	obj->fence_reg = I915_FENCE_REG_NONE;
		2258	fence->obj = NULL;
		2259	list_del_init(&fence->lru_list);
2344	Serge	2260	}
3031	serge	2261	}
2344	Serge	2262
3031	serge	2263	static int
		2264	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
		2265	{
		2266	if (obj->last_fenced_seqno) {
		2267	int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2352	Serge	2268	if (ret)
		2269	return ret;
2344	Serge	2270
		2271	obj->last_fenced_seqno = 0;
		2272	}
		2273
		2274	/* Ensure that all CPU reads are completed before installing a fence
		2275	* and all writes before removing the fence.
2332	Serge	2276	*/
2344	Serge	2277	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		2278	mb();
2332	Serge	2279
3031	serge	2280	obj->fenced_gpu_access = false;
2332	Serge	2281	return 0;
		2282	}
		2283
		2284	int
2344	Serge	2285	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	2286	{
3031	serge	2287	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2332	Serge	2288	int ret;
		2289
3031	serge	2290	ret = i915_gem_object_flush_fence(obj);
2332	Serge	2291	if (ret)
		2292	return ret;
		2293
3031	serge	2294	if (obj->fence_reg == I915_FENCE_REG_NONE)
		2295	return 0;
2332	Serge	2296
3031	serge	2297	i915_gem_object_update_fence(obj,
		2298	&dev_priv->fence_regs[obj->fence_reg],
		2299	false);
		2300	i915_gem_object_fence_lost(obj);
2344	Serge	2301
2332	Serge	2302	return 0;
		2303	}
		2304
3031	serge	2305	static struct drm_i915_fence_reg *
		2306	i915_find_fence_reg(struct drm_device *dev)
		2307	{
		2308	struct drm_i915_private *dev_priv = dev->dev_private;
		2309	struct drm_i915_fence_reg reg, avail;
		2310	int i;
2332	Serge	2311
3031	serge	2312	/* First try to find a free reg */
		2313	avail = NULL;
		2314	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		2315	reg = &dev_priv->fence_regs[i];
		2316	if (!reg->obj)
		2317	return reg;
2332	Serge	2318
3031	serge	2319	if (!reg->pin_count)
		2320	avail = reg;
		2321	}
2332	Serge	2322
3031	serge	2323	if (avail == NULL)
		2324	return NULL;
2332	Serge	2325
3031	serge	2326	/* None available, try to steal one or wait for a user to finish */
		2327	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
		2328	if (reg->pin_count)
		2329	continue;
2332	Serge	2330
3031	serge	2331	return reg;
		2332	}
2332	Serge	2333
3031	serge	2334	return NULL;
		2335	}
2332	Serge	2336
3031	serge	2337	/**
		2338	* i915_gem_object_get_fence - set up fencing for an object
		2339	* @obj: object to map through a fence reg
		2340	*
		2341	* When mapping objects through the GTT, userspace wants to be able to write
		2342	* to them without having to worry about swizzling if the object is tiled.
		2343	* This function walks the fence regs looking for a free one for @obj,
		2344	* stealing one if it can't find any.
		2345	*
		2346	* It then sets up the reg based on the object's properties: address, pitch
		2347	* and tiling format.
		2348	*
		2349	* For an untiled surface, this removes any existing fence.
		2350	*/
		2351	int
		2352	i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
		2353	{
		2354	struct drm_device *dev = obj->base.dev;
		2355	struct drm_i915_private *dev_priv = dev->dev_private;
		2356	bool enable = obj->tiling_mode != I915_TILING_NONE;
		2357	struct drm_i915_fence_reg *reg;
		2358	int ret;
2332	Serge	2359
3031	serge	2360	/* Have we updated the tiling parameters upon the object and so
		2361	* will need to serialise the write to the associated fence register?
		2362	*/
		2363	if (obj->fence_dirty) {
		2364	ret = i915_gem_object_flush_fence(obj);
		2365	if (ret)
		2366	return ret;
		2367	}
2332	Serge	2368
3031	serge	2369	/* Just update our place in the LRU if our fence is getting reused. */
		2370	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		2371	reg = &dev_priv->fence_regs[obj->fence_reg];
		2372	if (!obj->fence_dirty) {
		2373	list_move_tail(®->lru_list,
		2374	&dev_priv->mm.fence_list);
		2375	return 0;
		2376	}
		2377	} else if (enable) {
		2378	reg = i915_find_fence_reg(dev);
		2379	if (reg == NULL)
		2380	return -EDEADLK;
2332	Serge	2381
3031	serge	2382	if (reg->obj) {
		2383	struct drm_i915_gem_object *old = reg->obj;
2332	Serge	2384
3031	serge	2385	ret = i915_gem_object_flush_fence(old);
		2386	if (ret)
		2387	return ret;
2332	Serge	2388
3031	serge	2389	i915_gem_object_fence_lost(old);
		2390	}
		2391	} else
		2392	return 0;
2332	Serge	2393
3031	serge	2394	i915_gem_object_update_fence(obj, reg, enable);
		2395	obj->fence_dirty = false;
2332	Serge	2396
3031	serge	2397	return 0;
		2398	}
2332	Serge	2399
3031	serge	2400	static bool i915_gem_valid_gtt_space(struct drm_device *dev,
		2401	struct drm_mm_node *gtt_space,
		2402	unsigned long cache_level)
		2403	{
		2404	struct drm_mm_node *other;
2332	Serge	2405
3031	serge	2406	/* On non-LLC machines we have to be careful when putting differing
		2407	* types of snoopable memory together to avoid the prefetcher
		2408	* crossing memory domains and dieing.
		2409	*/
		2410	if (HAS_LLC(dev))
		2411	return true;
2332	Serge	2412
3031	serge	2413	if (gtt_space == NULL)
		2414	return true;
2332	Serge	2415
3031	serge	2416	if (list_empty(>t_space->node_list))
		2417	return true;
2332	Serge	2418
3031	serge	2419	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
		2420	if (other->allocated && !other->hole_follows && other->color != cache_level)
		2421	return false;
2344	Serge	2422
3031	serge	2423	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
		2424	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		2425	return false;
2344	Serge	2426
3031	serge	2427	return true;
		2428	}
2344	Serge	2429
3031	serge	2430	static void i915_gem_verify_gtt(struct drm_device *dev)
		2431	{
		2432	#if WATCH_GTT
		2433	struct drm_i915_private *dev_priv = dev->dev_private;
		2434	struct drm_i915_gem_object *obj;
		2435	int err = 0;
2344	Serge	2436
3031	serge	2437	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
		2438	if (obj->gtt_space == NULL) {
		2439	printk(KERN_ERR "object found on GTT list with no space reserved\n");
		2440	err++;
		2441	continue;
		2442	}
2344	Serge	2443
3031	serge	2444	if (obj->cache_level != obj->gtt_space->color) {
		2445	printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
		2446	obj->gtt_space->start,
		2447	obj->gtt_space->start + obj->gtt_space->size,
		2448	obj->cache_level,
		2449	obj->gtt_space->color);
		2450	err++;
		2451	continue;
		2452	}
2344	Serge	2453
3031	serge	2454	if (!i915_gem_valid_gtt_space(dev,
		2455	obj->gtt_space,
		2456	obj->cache_level)) {
		2457	printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
		2458	obj->gtt_space->start,
		2459	obj->gtt_space->start + obj->gtt_space->size,
		2460	obj->cache_level);
		2461	err++;
		2462	continue;
		2463	}
		2464	}
2344	Serge	2465
3031	serge	2466	WARN_ON(err);
		2467	#endif
2326	Serge	2468	}
		2469
2332	Serge	2470	/**
		2471	* Finds free space in the GTT aperture and binds the object there.
		2472	*/
		2473	static int
		2474	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		2475	unsigned alignment,
3031	serge	2476	bool map_and_fenceable,
		2477	bool nonblocking)
2332	Serge	2478	{
		2479	struct drm_device *dev = obj->base.dev;
		2480	drm_i915_private_t *dev_priv = dev->dev_private;
3243	Serge	2481	struct drm_mm_node *node;
2332	Serge	2482	u32 size, fence_size, fence_alignment, unfenced_alignment;
		2483	bool mappable, fenceable;
		2484	int ret;
2326	Serge	2485
2332	Serge	2486	if (obj->madv != I915_MADV_WILLNEED) {
3243	Serge	2487	DRM_ERROR("Attempting to bind a purgeable object\n");
2332	Serge	2488	return -EINVAL;
		2489	}
		2490
		2491	fence_size = i915_gem_get_gtt_size(dev,
		2492	obj->base.size,
		2493	obj->tiling_mode);
		2494	fence_alignment = i915_gem_get_gtt_alignment(dev,
		2495	obj->base.size,
		2496	obj->tiling_mode);
		2497	unfenced_alignment =
		2498	i915_gem_get_unfenced_gtt_alignment(dev,
		2499	obj->base.size,
		2500	obj->tiling_mode);
		2501
		2502	if (alignment == 0)
		2503	alignment = map_and_fenceable ? fence_alignment :
		2504	unfenced_alignment;
		2505	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		2506	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		2507	return -EINVAL;
		2508	}
		2509
		2510	size = map_and_fenceable ? fence_size : obj->base.size;
		2511
		2512	/* If the object is bigger than the entire aperture, reject it early
		2513	* before evicting everything in a vain attempt to find space.
		2514	*/
		2515	if (obj->base.size >
		2516	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		2517	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		2518	return -E2BIG;
		2519	}
		2520
3031	serge	2521	ret = i915_gem_object_get_pages(obj);
		2522	if (ret)
		2523	return ret;
		2524
3243	Serge	2525	i915_gem_object_pin_pages(obj);
		2526
		2527	node = kzalloc(sizeof(*node), GFP_KERNEL);
		2528	if (node == NULL) {
		2529	i915_gem_object_unpin_pages(obj);
		2530	return -ENOMEM;
		2531	}
		2532
2332	Serge	2533	search_free:
		2534	if (map_and_fenceable)
3243	Serge	2535	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3031	serge	2536	size, alignment, obj->cache_level,
3243	Serge	2537	0, dev_priv->mm.gtt_mappable_end);
2332	Serge	2538	else
3243	Serge	2539	ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
		2540	size, alignment, obj->cache_level);
		2541	if (ret) {
2332	Serge	2542
3243	Serge	2543	i915_gem_object_unpin_pages(obj);
		2544	kfree(node);
2332	Serge	2545	return ret;
		2546	}
3243	Serge	2547	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
		2548	i915_gem_object_unpin_pages(obj);
		2549	drm_mm_put_block(node);
3031	serge	2550	return -EINVAL;
		2551	}
2332	Serge	2552
3031	serge	2553	ret = i915_gem_gtt_prepare_object(obj);
2332	Serge	2554	if (ret) {
3243	Serge	2555	i915_gem_object_unpin_pages(obj);
		2556	drm_mm_put_block(node);
2332	Serge	2557	return ret;
		2558	}
		2559
3031	serge	2560	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
2332	Serge	2561	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2562
3243	Serge	2563	obj->gtt_space = node;
		2564	obj->gtt_offset = node->start;
2332	Serge	2565
		2566	fenceable =
3243	Serge	2567	node->size == fence_size &&
		2568	(node->start & (fence_alignment - 1)) == 0;
2332	Serge	2569
		2570	mappable =
		2571	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		2572
		2573	obj->map_and_fenceable = mappable && fenceable;
		2574
3243	Serge	2575	i915_gem_object_unpin_pages(obj);
2351	Serge	2576	trace_i915_gem_object_bind(obj, map_and_fenceable);
3031	serge	2577	i915_gem_verify_gtt(dev);
2332	Serge	2578	return 0;
		2579	}
		2580
		2581	void
		2582	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		2583	{
		2584	/* If we don't have a page list set up, then we're not pinned
		2585	* to GPU, and we can ignore the cache flush because it'll happen
		2586	* again at bind time.
		2587	*/
3243	Serge	2588	if (obj->pages == NULL)
2332	Serge	2589	return;
		2590
		2591	/* If the GPU is snooping the contents of the CPU cache,
		2592	* we do not need to manually clear the CPU cache lines. However,
		2593	* the caches are only snooped when the render cache is
		2594	* flushed/invalidated. As we always have to emit invalidations
		2595	* and flushes when moving into and out of the RENDER domain, correct
		2596	* snooping behaviour occurs naturally as the result of our domain
		2597	* tracking.
		2598	*/
		2599	if (obj->cache_level != I915_CACHE_NONE)
		2600	return;
3243	Serge	2601	#if 0
2344	Serge	2602	if(obj->mapped != NULL)
		2603	{
		2604	uint8_t *page_virtual;
		2605	unsigned int i;
2332	Serge	2606
2344	Serge	2607	page_virtual = obj->mapped;
		2608	asm volatile("mfence");
		2609	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2610	clflush(page_virtual + i);
		2611	asm volatile("mfence");
		2612	}
		2613	else
		2614	{
		2615	uint8_t *page_virtual;
		2616	unsigned int i;
		2617	page_virtual = AllocKernelSpace(obj->base.size);
		2618	if(page_virtual != NULL)
		2619	{
3031	serge	2620	dma_addr_t src, dst;
2344	Serge	2621	u32 count;
		2622
		2623	#define page_tabs 0xFDC00000 /* really dirty hack */
		2624
3031	serge	2625	src = obj->pages.page;
		2626	dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12];
2344	Serge	2627	count = obj->base.size/4096;
		2628
		2629	while(count--)
		2630	{
		2631	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		2632	};
		2633
		2634	asm volatile("mfence");
		2635	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		2636	clflush(page_virtual + i);
		2637	asm volatile("mfence");
		2638	FreeKernelSpace(page_virtual);
		2639	}
		2640	else
		2641	{
		2642	asm volatile (
		2643	"mfence \n"
		2644	"wbinvd \n" /* this is really ugly */
		2645	"mfence");
		2646	}
		2647	}
3243	Serge	2648	#endif
		2649
2332	Serge	2650	}
		2651
2344	Serge	2652	/** Flushes the GTT write domain for the object if it's dirty. */
		2653	static void
		2654	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		2655	{
		2656	uint32_t old_write_domain;
2332	Serge	2657
2344	Serge	2658	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		2659	return;
2332	Serge	2660
2344	Serge	2661	/* No actual flushing is required for the GTT write domain. Writes
		2662	* to it immediately go to main memory as far as we know, so there's
		2663	* no chipset flush. It also doesn't land in render cache.
		2664	*
		2665	* However, we do have to enforce the order so that all writes through
		2666	* the GTT land before any writes to the device, such as updates to
		2667	* the GATT itself.
		2668	*/
		2669	wmb();
2332	Serge	2670
2344	Serge	2671	old_write_domain = obj->base.write_domain;
		2672	obj->base.write_domain = 0;
2332	Serge	2673
2351	Serge	2674	trace_i915_gem_object_change_domain(obj,
		2675	obj->base.read_domains,
		2676	old_write_domain);
2344	Serge	2677	}
2332	Serge	2678
		2679	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	2680	static void
2332	Serge	2681	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		2682	{
		2683	uint32_t old_write_domain;
		2684
		2685	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		2686	return;
		2687
		2688	i915_gem_clflush_object(obj);
3243	Serge	2689	i915_gem_chipset_flush(obj->base.dev);
2332	Serge	2690	old_write_domain = obj->base.write_domain;
		2691	obj->base.write_domain = 0;
		2692
2351	Serge	2693	trace_i915_gem_object_change_domain(obj,
		2694	obj->base.read_domains,
		2695	old_write_domain);
2332	Serge	2696	}
		2697
		2698	/**
		2699	* Moves a single object to the GTT read, and possibly write domain.
		2700	*
		2701	* This function returns when the move is complete, including waiting on
		2702	* flushes to occur.
		2703	*/
		2704	int
		2705	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		2706	{
3031	serge	2707	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2332	Serge	2708	uint32_t old_write_domain, old_read_domains;
		2709	int ret;
		2710
		2711	/* Not valid to be called on unbound objects. */
		2712	if (obj->gtt_space == NULL)
		2713	return -EINVAL;
		2714
		2715	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		2716	return 0;
		2717
3031	serge	2718	ret = i915_gem_object_wait_rendering(obj, !write);
2332	Serge	2719	if (ret)
		2720	return ret;
		2721
		2722	i915_gem_object_flush_cpu_write_domain(obj);
		2723
		2724	old_write_domain = obj->base.write_domain;
		2725	old_read_domains = obj->base.read_domains;
		2726
		2727	/* It should now be out of any other write domains, and we can update
		2728	* the domain values for our changes.
		2729	*/
		2730	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2731	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		2732	if (write) {
		2733	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		2734	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		2735	obj->dirty = 1;
		2736	}
		2737
2351	Serge	2738	trace_i915_gem_object_change_domain(obj,
		2739	old_read_domains,
		2740	old_write_domain);
		2741
3031	serge	2742	/* And bump the LRU for this access */
		2743	if (i915_gem_object_is_inactive(obj))
		2744	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		2745
2332	Serge	2746	return 0;
		2747	}
		2748
2335	Serge	2749	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		2750	enum i915_cache_level cache_level)
		2751	{
3031	serge	2752	struct drm_device *dev = obj->base.dev;
		2753	drm_i915_private_t *dev_priv = dev->dev_private;
2335	Serge	2754	int ret;
2332	Serge	2755
2335	Serge	2756	if (obj->cache_level == cache_level)
		2757	return 0;
2332	Serge	2758
2335	Serge	2759	if (obj->pin_count) {
		2760	DRM_DEBUG("can not change the cache level of pinned objects\n");
		2761	return -EBUSY;
		2762	}
2332	Serge	2763
3031	serge	2764	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		2765	ret = i915_gem_object_unbind(obj);
		2766	if (ret)
		2767	return ret;
		2768	}
		2769
2335	Serge	2770	if (obj->gtt_space) {
		2771	ret = i915_gem_object_finish_gpu(obj);
		2772	if (ret)
		2773	return ret;
2332	Serge	2774
2335	Serge	2775	i915_gem_object_finish_gtt(obj);
2332	Serge	2776
2335	Serge	2777	/* Before SandyBridge, you could not use tiling or fence
		2778	* registers with snooped memory, so relinquish any fences
		2779	* currently pointing to our region in the aperture.
		2780	*/
3031	serge	2781	if (INTEL_INFO(dev)->gen < 6) {
2335	Serge	2782	ret = i915_gem_object_put_fence(obj);
		2783	if (ret)
		2784	return ret;
		2785	}
2332	Serge	2786
3031	serge	2787	if (obj->has_global_gtt_mapping)
		2788	i915_gem_gtt_bind_object(obj, cache_level);
		2789	if (obj->has_aliasing_ppgtt_mapping)
		2790	i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
		2791	obj, cache_level);
		2792
		2793	obj->gtt_space->color = cache_level;
2335	Serge	2794	}
2332	Serge	2795
2335	Serge	2796	if (cache_level == I915_CACHE_NONE) {
		2797	u32 old_read_domains, old_write_domain;
2332	Serge	2798
2335	Serge	2799	/* If we're coming from LLC cached, then we haven't
		2800	* actually been tracking whether the data is in the
		2801	* CPU cache or not, since we only allow one bit set
		2802	* in obj->write_domain and have been skipping the clflushes.
		2803	* Just set it to the CPU cache for now.
		2804	*/
		2805	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		2806	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	2807
2335	Serge	2808	old_read_domains = obj->base.read_domains;
		2809	old_write_domain = obj->base.write_domain;
2332	Serge	2810
2335	Serge	2811	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2812	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	2813
2351	Serge	2814	trace_i915_gem_object_change_domain(obj,
		2815	old_read_domains,
		2816	old_write_domain);
2344	Serge	2817	}
2332	Serge	2818
2335	Serge	2819	obj->cache_level = cache_level;
3031	serge	2820	i915_gem_verify_gtt(dev);
2335	Serge	2821	return 0;
		2822	}
2332	Serge	2823
2335	Serge	2824	/*
		2825	* Prepare buffer for display plane (scanout, cursors, etc).
		2826	* Can be called from an uninterruptible phase (modesetting) and allows
		2827	* any flushes to be pipelined (for pageflips).
		2828	*/
		2829	int
		2830	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		2831	u32 alignment,
		2832	struct intel_ring_buffer *pipelined)
		2833	{
		2834	u32 old_read_domains, old_write_domain;
		2835	int ret;
2332	Serge	2836
3031	serge	2837	if (pipelined != obj->ring) {
		2838	ret = i915_gem_object_sync(obj, pipelined);
2335	Serge	2839	if (ret)
		2840	return ret;
		2841	}
2332	Serge	2842
2335	Serge	2843	/* The display engine is not coherent with the LLC cache on gen6. As
		2844	* a result, we make sure that the pinning that is about to occur is
		2845	* done with uncached PTEs. This is lowest common denominator for all
		2846	* chipsets.
		2847	*
		2848	* However for gen6+, we could do better by using the GFDT bit instead
		2849	* of uncaching, which would allow us to flush all the LLC-cached data
		2850	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		2851	*/
2360	Serge	2852	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		2853	if (ret)
		2854	return ret;
2332	Serge	2855
2335	Serge	2856	/* As the user may map the buffer once pinned in the display plane
		2857	* (e.g. libkms for the bootup splash), we have to ensure that we
		2858	* always use map_and_fenceable for all scanout buffers.
		2859	*/
3031	serge	2860	ret = i915_gem_object_pin(obj, alignment, true, false);
2335	Serge	2861	if (ret)
		2862	return ret;
2332	Serge	2863
2335	Serge	2864	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	2865
2335	Serge	2866	old_write_domain = obj->base.write_domain;
		2867	old_read_domains = obj->base.read_domains;
2332	Serge	2868
2335	Serge	2869	/* It should now be out of any other write domains, and we can update
		2870	* the domain values for our changes.
		2871	*/
3031	serge	2872	obj->base.write_domain = 0;
2335	Serge	2873	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	2874
2351	Serge	2875	trace_i915_gem_object_change_domain(obj,
		2876	old_read_domains,
		2877	old_write_domain);
2332	Serge	2878
2335	Serge	2879	return 0;
		2880	}
2332	Serge	2881
2344	Serge	2882	int
		2883	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		2884	{
		2885	int ret;
2332	Serge	2886
2344	Serge	2887	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		2888	return 0;
2332	Serge	2889
3031	serge	2890	ret = i915_gem_object_wait_rendering(obj, false);
3243	Serge	2891	if (ret)
		2892	return ret;
2332	Serge	2893
2344	Serge	2894	/* Ensure that we invalidate the GPU's caches and TLBs. */
		2895	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3031	serge	2896	return 0;
2344	Serge	2897	}
2332	Serge	2898
2344	Serge	2899	/**
		2900	* Moves a single object to the CPU read, and possibly write domain.
		2901	*
		2902	* This function returns when the move is complete, including waiting on
		2903	* flushes to occur.
		2904	*/
3031	serge	2905	int
2344	Serge	2906	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		2907	{
		2908	uint32_t old_write_domain, old_read_domains;
		2909	int ret;
2332	Serge	2910
2344	Serge	2911	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		2912	return 0;
2332	Serge	2913
3031	serge	2914	ret = i915_gem_object_wait_rendering(obj, !write);
2344	Serge	2915	if (ret)
		2916	return ret;
2332	Serge	2917
2344	Serge	2918	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	2919
2344	Serge	2920	old_write_domain = obj->base.write_domain;
		2921	old_read_domains = obj->base.read_domains;
2332	Serge	2922
2344	Serge	2923	/* Flush the CPU cache if it's still invalid. */
		2924	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		2925	i915_gem_clflush_object(obj);
2332	Serge	2926
2344	Serge	2927	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		2928	}
2332	Serge	2929
2344	Serge	2930	/* It should now be out of any other write domains, and we can update
		2931	* the domain values for our changes.
		2932	*/
		2933	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	2934
2344	Serge	2935	/* If we're writing through the CPU, then the GPU read domains will
		2936	* need to be invalidated at next use.
		2937	*/
		2938	if (write) {
		2939	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2940	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2941	}
2332	Serge	2942
2351	Serge	2943	trace_i915_gem_object_change_domain(obj,
		2944	old_read_domains,
		2945	old_write_domain);
2332	Serge	2946
2344	Serge	2947	return 0;
		2948	}
2332	Serge	2949
3031	serge	2950	#if 0
		2951	/* Throttle our rendering by waiting until the ring has completed our requests
		2952	* emitted over 20 msec ago.
2344	Serge	2953	*
3031	serge	2954	* Note that if we were to use the current jiffies each time around the loop,
		2955	* we wouldn't escape the function with any frames outstanding if the time to
		2956	* render a frame was over 20ms.
		2957	*
		2958	* This should get us reasonable parallelism between CPU and GPU but also
		2959	* relatively low latency when blocking on a particular request to finish.
2344	Serge	2960	*/
3031	serge	2961	static int
		2962	i915_gem_ring_throttle(struct drm_device dev, struct drm_file file)
2344	Serge	2963	{
3031	serge	2964	struct drm_i915_private *dev_priv = dev->dev_private;
		2965	struct drm_i915_file_private *file_priv = file->driver_priv;
		2966	unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20);
		2967	struct drm_i915_gem_request *request;
		2968	struct intel_ring_buffer *ring = NULL;
		2969	u32 seqno = 0;
		2970	int ret;
2332	Serge	2971
3031	serge	2972	if (atomic_read(&dev_priv->mm.wedged))
		2973	return -EIO;
2332	Serge	2974
3031	serge	2975	spin_lock(&file_priv->mm.lock);
		2976	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
		2977	if (time_after_eq(request->emitted_jiffies, recent_enough))
		2978	break;
2332	Serge	2979
3031	serge	2980	ring = request->ring;
		2981	seqno = request->seqno;
		2982	}
		2983	spin_unlock(&file_priv->mm.lock);
2332	Serge	2984
3031	serge	2985	if (seqno == 0)
		2986	return 0;
2332	Serge	2987
3031	serge	2988	ret = __wait_seqno(ring, seqno, true, NULL);
		2989	if (ret == 0)
		2990	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2332	Serge	2991
3031	serge	2992	return ret;
2352	Serge	2993	}
3031	serge	2994	#endif
2332	Serge	2995
		2996	int
		2997	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		2998	uint32_t alignment,
3031	serge	2999	bool map_and_fenceable,
		3000	bool nonblocking)
2332	Serge	3001	{
		3002	int ret;
		3003
3031	serge	3004	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		3005	return -EBUSY;
2332	Serge	3006
		3007	#if 0
		3008	if (obj->gtt_space != NULL) {
		3009	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		3010	(map_and_fenceable && !obj->map_and_fenceable)) {
		3011	WARN(obj->pin_count,
		3012	"bo is already pinned with incorrect alignment:"
		3013	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		3014	" obj->map_and_fenceable=%d\n",
		3015	obj->gtt_offset, alignment,
		3016	map_and_fenceable,
		3017	obj->map_and_fenceable);
		3018	ret = i915_gem_object_unbind(obj);
		3019	if (ret)
		3020	return ret;
		3021	}
		3022	}
		3023	#endif
		3024
		3025	if (obj->gtt_space == NULL) {
3243	Serge	3026	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		3027
2332	Serge	3028	ret = i915_gem_object_bind_to_gtt(obj, alignment,
3031	serge	3029	map_and_fenceable,
		3030	nonblocking);
2332	Serge	3031	if (ret)
		3032	return ret;
3243	Serge	3033
		3034	if (!dev_priv->mm.aliasing_ppgtt)
		3035	i915_gem_gtt_bind_object(obj, obj->cache_level);
2332	Serge	3036	}
		3037
3031	serge	3038	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		3039	i915_gem_gtt_bind_object(obj, obj->cache_level);
		3040
		3041	obj->pin_count++;
2332	Serge	3042	obj->pin_mappable \|= map_and_fenceable;
		3043
		3044	return 0;
		3045	}
		3046
2344	Serge	3047	void
		3048	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		3049	{
		3050	BUG_ON(obj->pin_count == 0);
		3051	BUG_ON(obj->gtt_space == NULL);
2332	Serge	3052
3031	serge	3053	if (--obj->pin_count == 0)
2344	Serge	3054	obj->pin_mappable = false;
		3055	}
2332	Serge	3056
3031	serge	3057	int
		3058	i915_gem_pin_ioctl(struct drm_device dev, void data,
		3059	struct drm_file *file)
		3060	{
		3061	struct drm_i915_gem_pin *args = data;
		3062	struct drm_i915_gem_object *obj;
		3063	int ret;
2332	Serge	3064
3031	serge	3065	ret = i915_mutex_lock_interruptible(dev);
		3066	if (ret)
		3067	return ret;
2332	Serge	3068
3031	serge	3069	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3070	if (&obj->base == NULL) {
		3071	ret = -ENOENT;
		3072	goto unlock;
		3073	}
2332	Serge	3074
3031	serge	3075	if (obj->madv != I915_MADV_WILLNEED) {
		3076	DRM_ERROR("Attempting to pin a purgeable buffer\n");
		3077	ret = -EINVAL;
		3078	goto out;
		3079	}
2332	Serge	3080
3031	serge	3081	if (obj->pin_filp != NULL && obj->pin_filp != file) {
		3082	DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
		3083	args->handle);
		3084	ret = -EINVAL;
		3085	goto out;
		3086	}
2332	Serge	3087
3243	Serge	3088	if (obj->user_pin_count == 0) {
3031	serge	3089	ret = i915_gem_object_pin(obj, args->alignment, true, false);
		3090	if (ret)
		3091	goto out;
		3092	}
2332	Serge	3093
3243	Serge	3094	obj->user_pin_count++;
		3095	obj->pin_filp = file;
		3096
3031	serge	3097	/* XXX - flush the CPU caches for pinned objects
		3098	* as the X server doesn't manage domains yet
		3099	*/
		3100	i915_gem_object_flush_cpu_write_domain(obj);
		3101	args->offset = obj->gtt_offset;
		3102	out:
		3103	drm_gem_object_unreference(&obj->base);
		3104	unlock:
		3105	mutex_unlock(&dev->struct_mutex);
		3106	return ret;
		3107	}
2332	Serge	3108
3255	Serge	3109	#if 0
		3110
3031	serge	3111	int
		3112	i915_gem_unpin_ioctl(struct drm_device dev, void data,
		3113	struct drm_file *file)
		3114	{
		3115	struct drm_i915_gem_pin *args = data;
		3116	struct drm_i915_gem_object *obj;
		3117	int ret;
2332	Serge	3118
3031	serge	3119	ret = i915_mutex_lock_interruptible(dev);
		3120	if (ret)
		3121	return ret;
2332	Serge	3122
3031	serge	3123	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3124	if (&obj->base == NULL) {
		3125	ret = -ENOENT;
		3126	goto unlock;
		3127	}
2332	Serge	3128
3031	serge	3129	if (obj->pin_filp != file) {
		3130	DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
		3131	args->handle);
		3132	ret = -EINVAL;
		3133	goto out;
		3134	}
		3135	obj->user_pin_count--;
		3136	if (obj->user_pin_count == 0) {
		3137	obj->pin_filp = NULL;
		3138	i915_gem_object_unpin(obj);
		3139	}
2332	Serge	3140
3031	serge	3141	out:
		3142	drm_gem_object_unreference(&obj->base);
		3143	unlock:
		3144	mutex_unlock(&dev->struct_mutex);
		3145	return ret;
		3146	}
2332	Serge	3147
3031	serge	3148	int
		3149	i915_gem_busy_ioctl(struct drm_device dev, void data,
		3150	struct drm_file *file)
		3151	{
		3152	struct drm_i915_gem_busy *args = data;
		3153	struct drm_i915_gem_object *obj;
		3154	int ret;
2332	Serge	3155
3031	serge	3156	ret = i915_mutex_lock_interruptible(dev);
		3157	if (ret)
		3158	return ret;
2332	Serge	3159
3031	serge	3160	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
		3161	if (&obj->base == NULL) {
		3162	ret = -ENOENT;
		3163	goto unlock;
		3164	}
2332	Serge	3165
3031	serge	3166	/* Count all active objects as busy, even if they are currently not used
		3167	* by the gpu. Users of this interface expect objects to eventually
		3168	* become non-busy without any further actions, therefore emit any
		3169	* necessary flushes here.
		3170	*/
		3171	ret = i915_gem_object_flush_active(obj);
2332	Serge	3172
3031	serge	3173	args->busy = obj->active;
		3174	if (obj->ring) {
		3175	BUILD_BUG_ON(I915_NUM_RINGS > 16);
		3176	args->busy \|= intel_ring_flag(obj->ring) << 16;
		3177	}
2332	Serge	3178
3031	serge	3179	drm_gem_object_unreference(&obj->base);
		3180	unlock:
		3181	mutex_unlock(&dev->struct_mutex);
		3182	return ret;
		3183	}
2332	Serge	3184
3031	serge	3185	int
		3186	i915_gem_throttle_ioctl(struct drm_device dev, void data,
		3187	struct drm_file *file_priv)
		3188	{
		3189	return i915_gem_ring_throttle(dev, file_priv);
		3190	}
2332	Serge	3191
3031	serge	3192	int
		3193	i915_gem_madvise_ioctl(struct drm_device dev, void data,
		3194	struct drm_file *file_priv)
		3195	{
		3196	struct drm_i915_gem_madvise *args = data;
		3197	struct drm_i915_gem_object *obj;
		3198	int ret;
2332	Serge	3199
3031	serge	3200	switch (args->madv) {
		3201	case I915_MADV_DONTNEED:
		3202	case I915_MADV_WILLNEED:
		3203	break;
		3204	default:
		3205	return -EINVAL;
		3206	}
2332	Serge	3207
3031	serge	3208	ret = i915_mutex_lock_interruptible(dev);
		3209	if (ret)
		3210	return ret;
2332	Serge	3211
3031	serge	3212	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
		3213	if (&obj->base == NULL) {
		3214	ret = -ENOENT;
		3215	goto unlock;
		3216	}
2332	Serge	3217
3031	serge	3218	if (obj->pin_count) {
		3219	ret = -EINVAL;
		3220	goto out;
		3221	}
2332	Serge	3222
3031	serge	3223	if (obj->madv != __I915_MADV_PURGED)
		3224	obj->madv = args->madv;
2332	Serge	3225
3031	serge	3226	/* if the object is no longer attached, discard its backing storage */
		3227	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
		3228	i915_gem_object_truncate(obj);
2332	Serge	3229
3031	serge	3230	args->retained = obj->madv != __I915_MADV_PURGED;
2332	Serge	3231
3031	serge	3232	out:
		3233	drm_gem_object_unreference(&obj->base);
		3234	unlock:
		3235	mutex_unlock(&dev->struct_mutex);
		3236	return ret;
		3237	}
		3238	#endif
2332	Serge	3239
3031	serge	3240	void i915_gem_object_init(struct drm_i915_gem_object *obj,
		3241	const struct drm_i915_gem_object_ops *ops)
		3242	{
		3243	INIT_LIST_HEAD(&obj->mm_list);
		3244	INIT_LIST_HEAD(&obj->gtt_list);
		3245	INIT_LIST_HEAD(&obj->ring_list);
		3246	INIT_LIST_HEAD(&obj->exec_list);
2332	Serge	3247
3031	serge	3248	obj->ops = ops;
		3249
		3250	obj->fence_reg = I915_FENCE_REG_NONE;
		3251	obj->madv = I915_MADV_WILLNEED;
		3252	/* Avoid an unnecessary call to unbind on the first bind. */
		3253	obj->map_and_fenceable = true;
		3254
		3255	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
		3256	}
		3257
		3258	static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
		3259	.get_pages = i915_gem_object_get_pages_gtt,
		3260	.put_pages = i915_gem_object_put_pages_gtt,
		3261	};
		3262
2332	Serge	3263	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		3264	size_t size)
		3265	{
		3266	struct drm_i915_gem_object *obj;
3031	serge	3267	struct address_space *mapping;
		3268	u32 mask;
2340	Serge	3269
2332	Serge	3270	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		3271	if (obj == NULL)
		3272	return NULL;
		3273
		3274	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		3275	kfree(obj);
		3276	return NULL;
		3277	}
		3278
		3279
3031	serge	3280	i915_gem_object_init(obj, &i915_gem_object_ops);
2332	Serge	3281
		3282	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		3283	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		3284
3031	serge	3285	if (HAS_LLC(dev)) {
		3286	/* On some devices, we can have the GPU use the LLC (the CPU
2332	Serge	3287	* cache) for about a 10% performance improvement
		3288	* compared to uncached. Graphics requests other than
		3289	* display scanout are coherent with the CPU in
		3290	* accessing this cache. This means in this mode we
		3291	* don't need to clflush on the CPU side, and on the
		3292	* GPU side we only need to flush internal caches to
		3293	* get data visible to the CPU.
		3294	*
		3295	* However, we maintain the display planes as UC, and so
		3296	* need to rebind when first used as such.
		3297	*/
		3298	obj->cache_level = I915_CACHE_LLC;
		3299	} else
		3300	obj->cache_level = I915_CACHE_NONE;
		3301
		3302	return obj;
		3303	}
		3304
2344	Serge	3305	int i915_gem_init_object(struct drm_gem_object *obj)
		3306	{
		3307	BUG();
2332	Serge	3308
2344	Serge	3309	return 0;
		3310	}
2332	Serge	3311
3031	serge	3312	void i915_gem_free_object(struct drm_gem_object *gem_obj)
2344	Serge	3313	{
3031	serge	3314	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2344	Serge	3315	struct drm_device *dev = obj->base.dev;
		3316	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3317
3031	serge	3318	trace_i915_gem_object_destroy(obj);
		3319
		3320	// if (obj->phys_obj)
		3321	// i915_gem_detach_phys_object(dev, obj);
		3322
		3323	obj->pin_count = 0;
		3324	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		3325	bool was_interruptible;
		3326
		3327	was_interruptible = dev_priv->mm.interruptible;
		3328	dev_priv->mm.interruptible = false;
		3329
		3330	WARN_ON(i915_gem_object_unbind(obj));
		3331
		3332	dev_priv->mm.interruptible = was_interruptible;
2344	Serge	3333	}
2332	Serge	3334
3031	serge	3335	obj->pages_pin_count = 0;
		3336	i915_gem_object_put_pages(obj);
		3337	// i915_gem_object_free_mmap_offset(obj);
2332	Serge	3338
3243	Serge	3339	BUG_ON(obj->pages);
2332	Serge	3340
3031	serge	3341	// if (obj->base.import_attach)
		3342	// drm_prime_gem_destroy(&obj->base, NULL);
		3343
2344	Serge	3344	drm_gem_object_release(&obj->base);
		3345	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	3346
2344	Serge	3347	kfree(obj->bit_17);
		3348	kfree(obj);
		3349	}
2332	Serge	3350
3031	serge	3351	#if 0
		3352	int
		3353	i915_gem_idle(struct drm_device *dev)
2344	Serge	3354	{
3031	serge	3355	drm_i915_private_t *dev_priv = dev->dev_private;
		3356	int ret;
2332	Serge	3357
3031	serge	3358	mutex_lock(&dev->struct_mutex);
2332	Serge	3359
3031	serge	3360	if (dev_priv->mm.suspended) {
		3361	mutex_unlock(&dev->struct_mutex);
		3362	return 0;
		3363	}
2332	Serge	3364
3031	serge	3365	ret = i915_gpu_idle(dev);
		3366	if (ret) {
		3367	mutex_unlock(&dev->struct_mutex);
		3368	return ret;
		3369	}
		3370	i915_gem_retire_requests(dev);
		3371
		3372	i915_gem_reset_fences(dev);
		3373
		3374	/* Hack! Don't let anybody do execbuf while we don't control the chip.
		3375	* We need to replace this with a semaphore, or something.
		3376	* And not confound mm.suspended!
		3377	*/
		3378	dev_priv->mm.suspended = 1;
		3379	del_timer_sync(&dev_priv->hangcheck_timer);
		3380
		3381	i915_kernel_lost_context(dev);
		3382	i915_gem_cleanup_ringbuffer(dev);
		3383
		3384	mutex_unlock(&dev->struct_mutex);
		3385
		3386	/* Cancel the retire work handler, which should be idle now. */
		3387	// cancel_delayed_work_sync(&dev_priv->mm.retire_work);
		3388
		3389	return 0;
2344	Serge	3390	}
3031	serge	3391	#endif
2332	Serge	3392
3031	serge	3393	void i915_gem_l3_remap(struct drm_device *dev)
		3394	{
		3395	drm_i915_private_t *dev_priv = dev->dev_private;
		3396	u32 misccpctl;
		3397	int i;
2332	Serge	3398
3031	serge	3399	if (!IS_IVYBRIDGE(dev))
		3400	return;
2332	Serge	3401
3243	Serge	3402	if (!dev_priv->l3_parity.remap_info)
3031	serge	3403	return;
2332	Serge	3404
3031	serge	3405	misccpctl = I915_READ(GEN7_MISCCPCTL);
		3406	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		3407	POSTING_READ(GEN7_MISCCPCTL);
2332	Serge	3408
3031	serge	3409	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		3410	u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3243	Serge	3411	if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
3031	serge	3412	DRM_DEBUG("0x%x was already programmed to %x\n",
		3413	GEN7_L3LOG_BASE + i, remap);
3243	Serge	3414	if (remap && !dev_priv->l3_parity.remap_info[i/4])
3031	serge	3415	DRM_DEBUG_DRIVER("Clearing remapped register\n");
3243	Serge	3416	I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
3031	serge	3417	}
2332	Serge	3418
3031	serge	3419	/* Make sure all the writes land before disabling dop clock gating */
		3420	POSTING_READ(GEN7_L3LOG_BASE);
2332	Serge	3421
3031	serge	3422	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		3423	}
2332	Serge	3424
3031	serge	3425	void i915_gem_init_swizzling(struct drm_device *dev)
		3426	{
		3427	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	3428
3031	serge	3429	if (INTEL_INFO(dev)->gen < 5 \|\|
		3430	dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		3431	return;
2332	Serge	3432
3031	serge	3433	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) \|
		3434	DISP_TILE_SURFACE_SWIZZLING);
2332	Serge	3435
3031	serge	3436	if (IS_GEN5(dev))
		3437	return;
2344	Serge	3438
3031	serge	3439	I915_WRITE(TILECTL, I915_READ(TILECTL) \| TILECTL_SWZCTL);
		3440	if (IS_GEN6(dev))
		3441	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
		3442	else
		3443	I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
		3444	}
		3445
		3446	static bool
		3447	intel_enable_blt(struct drm_device *dev)
		3448	{
		3449	if (!HAS_BLT(dev))
		3450	return false;
		3451
		3452	/* The blitter was dysfunctional on early prototypes */
		3453	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		3454	DRM_INFO("BLT not supported on this pre-production hardware;"
		3455	" graphics performance will be degraded.\n");
		3456	return false;
		3457	}
		3458
		3459	return true;
		3460	}
		3461
2332	Serge	3462	int
3031	serge	3463	i915_gem_init_hw(struct drm_device *dev)
2332	Serge	3464	{
		3465	drm_i915_private_t *dev_priv = dev->dev_private;
		3466	int ret;
2351	Serge	3467
3243	Serge	3468	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
3031	serge	3469	return -EIO;
		3470
		3471	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		3472	I915_WRITE(0x9008, I915_READ(0x9008) \| 0xf0000);
		3473
		3474	i915_gem_l3_remap(dev);
		3475
		3476	i915_gem_init_swizzling(dev);
		3477
2332	Serge	3478	ret = intel_init_render_ring_buffer(dev);
		3479	if (ret)
		3480	return ret;
		3481
		3482	if (HAS_BSD(dev)) {
		3483	ret = intel_init_bsd_ring_buffer(dev);
		3484	if (ret)
		3485	goto cleanup_render_ring;
		3486	}
		3487
3031	serge	3488	if (intel_enable_blt(dev)) {
2332	Serge	3489	ret = intel_init_blt_ring_buffer(dev);
		3490	if (ret)
		3491	goto cleanup_bsd_ring;
		3492	}
		3493
		3494	dev_priv->next_seqno = 1;
2351	Serge	3495
3031	serge	3496	/*
		3497	* XXX: There was some w/a described somewhere suggesting loading
		3498	* contexts before PPGTT.
		3499	*/
		3500	i915_gem_context_init(dev);
		3501	i915_gem_init_ppgtt(dev);
		3502
2332	Serge	3503	return 0;
		3504
		3505	cleanup_bsd_ring:
		3506	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		3507	cleanup_render_ring:
		3508	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		3509	return ret;
		3510	}
		3511
3031	serge	3512	static bool
		3513	intel_enable_ppgtt(struct drm_device *dev)
		3514	{
		3515	if (i915_enable_ppgtt >= 0)
		3516	return i915_enable_ppgtt;
		3517
		3518	#ifdef CONFIG_INTEL_IOMMU
		3519	/* Disable ppgtt on SNB if VT-d is on. */
		3520	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
		3521	return false;
		3522	#endif
		3523
		3524	return true;
		3525	}
		3526
		3527	#define LFB_SIZE 0xC00000
		3528
		3529	int i915_gem_init(struct drm_device *dev)
		3530	{
		3531	struct drm_i915_private *dev_priv = dev->dev_private;
		3532	unsigned long gtt_size, mappable_size;
		3533	int ret;
		3534
		3535	gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
		3536	mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
		3537
		3538	mutex_lock(&dev->struct_mutex);
		3539	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
		3540	/* PPGTT pdes are stolen from global gtt ptes, so shrink the
		3541	* aperture accordingly when using aliasing ppgtt. */
		3542	gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
		3543
		3544	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3545
		3546	ret = i915_gem_init_aliasing_ppgtt(dev);
		3547	if (ret) {
		3548	mutex_unlock(&dev->struct_mutex);
		3549	return ret;
		3550	}
		3551	} else {
		3552	/* Let GEM Manage all of the aperture.
		3553	*
		3554	* However, leave one page at the end still bound to the scratch
		3555	* page. There are a number of places where the hardware
		3556	* apparently prefetches past the end of the object, and we've
		3557	* seen multiple hangs with the GPU head pointer stuck in a
		3558	* batchbuffer bound at the last page of the aperture. One page
		3559	* should be enough to keep any prefetching inside of the
		3560	* aperture.
		3561	*/
		3562	i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE);
		3563	}
		3564
		3565	ret = i915_gem_init_hw(dev);
		3566	mutex_unlock(&dev->struct_mutex);
		3567	if (ret) {
		3568	i915_gem_cleanup_aliasing_ppgtt(dev);
		3569	return ret;
		3570	}
		3571
		3572	return 0;
		3573	}
		3574
2332	Serge	3575	void
		3576	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		3577	{
		3578	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3579	struct intel_ring_buffer *ring;
2332	Serge	3580	int i;
		3581
3031	serge	3582	for_each_ring(ring, dev_priv, i)
		3583	intel_cleanup_ring_buffer(ring);
2332	Serge	3584	}
		3585
3031	serge	3586	#if 0
		3587
2332	Serge	3588	int
		3589	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		3590	struct drm_file *file_priv)
		3591	{
		3592	drm_i915_private_t *dev_priv = dev->dev_private;
3031	serge	3593	int ret;
2332	Serge	3594
		3595	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3596	return 0;
		3597
		3598	if (atomic_read(&dev_priv->mm.wedged)) {
		3599	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		3600	atomic_set(&dev_priv->mm.wedged, 0);
		3601	}
		3602
		3603	mutex_lock(&dev->struct_mutex);
		3604	dev_priv->mm.suspended = 0;
		3605
3031	serge	3606	ret = i915_gem_init_hw(dev);
2332	Serge	3607	if (ret != 0) {
		3608	mutex_unlock(&dev->struct_mutex);
		3609	return ret;
		3610	}
		3611
		3612	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		3613	mutex_unlock(&dev->struct_mutex);
		3614
		3615	ret = drm_irq_install(dev);
		3616	if (ret)
		3617	goto cleanup_ringbuffer;
		3618
		3619	return 0;
		3620
		3621	cleanup_ringbuffer:
		3622	mutex_lock(&dev->struct_mutex);
		3623	i915_gem_cleanup_ringbuffer(dev);
		3624	dev_priv->mm.suspended = 1;
		3625	mutex_unlock(&dev->struct_mutex);
		3626
		3627	return ret;
		3628	}
		3629
		3630	int
		3631	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		3632	struct drm_file *file_priv)
		3633	{
		3634	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3635	return 0;
		3636
		3637	drm_irq_uninstall(dev);
		3638	return i915_gem_idle(dev);
		3639	}
		3640
		3641	void
		3642	i915_gem_lastclose(struct drm_device *dev)
		3643	{
		3644	int ret;
		3645
		3646	if (drm_core_check_feature(dev, DRIVER_MODESET))
		3647	return;
		3648
		3649	ret = i915_gem_idle(dev);
		3650	if (ret)
		3651	DRM_ERROR("failed to idle hardware: %d\n", ret);
		3652	}
		3653	#endif
		3654
		3655	static void
2326	Serge	3656	init_ring_lists(struct intel_ring_buffer *ring)
		3657	{
		3658	INIT_LIST_HEAD(&ring->active_list);
		3659	INIT_LIST_HEAD(&ring->request_list);
		3660	}
		3661
		3662	void
		3663	i915_gem_load(struct drm_device *dev)
		3664	{
		3665	int i;
		3666	drm_i915_private_t *dev_priv = dev->dev_private;
		3667
		3668	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		3669	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3031	serge	3670	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
		3671	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
2326	Serge	3672	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		3673	for (i = 0; i < I915_NUM_RINGS; i++)
		3674	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	3675	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	3676	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	3677	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		3678	i915_gem_retire_work_handler);
2326	Serge	3679
		3680	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		3681	if (IS_GEN3(dev)) {
3031	serge	3682	I915_WRITE(MI_ARB_STATE,
		3683	_MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2326	Serge	3684	}
		3685
		3686	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		3687
		3688	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		3689	dev_priv->num_fence_regs = 16;
		3690	else
		3691	dev_priv->num_fence_regs = 8;
		3692
		3693	/* Initialize fence registers to zero */
3031	serge	3694	i915_gem_reset_fences(dev);
2326	Serge	3695
		3696	i915_gem_detect_bit_6_swizzle(dev);
		3697
		3698	dev_priv->mm.interruptible = true;
		3699
		3700	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		3701	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		3702	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		3703	}
		3704
		3705

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 3255