WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
		28	#include "drmP.h"
		29	#include "drm.h"
2330	Serge	30	#include "i915_drm.h"
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
		34	//#include
2330	Serge	35	#include
2326	Serge	36	//#include
		37	#include
		38
2344	Serge	39	extern int x86_clflush_size;
2332	Serge	40
2344	Serge	41	#undef mb
		42	#undef rmb
		43	#undef wmb
		44	#define mb() asm volatile("mfence")
		45	#define rmb() asm volatile ("lfence")
		46	#define wmb() asm volatile ("sfence")
		47
		48	static inline void clflush(volatile void *__p)
		49	{
		50	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		51	}
		52
2332	Serge	53	#define MAX_ERRNO 4095
		54
		55	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		56
		57	static inline long IS_ERR(const void *ptr)
		58	{
		59	return IS_ERR_VALUE((unsigned long)ptr);
		60	}
		61
		62	static inline void *ERR_PTR(long error)
		63	{
		64	return (void *) error;
		65	}
		66
		67	static inline long PTR_ERR(const void *ptr)
		68	{
		69	return (long) ptr;
		70	}
		71
2344	Serge	72	void
		73	drm_gem_object_free(struct kref *kref)
		74	{
		75	struct drm_gem_object obj = (struct drm_gem_object ) kref;
		76	struct drm_device *dev = obj->dev;
2332	Serge	77
2344	Serge	78	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		79
		80	i915_gem_free_object(obj);
		81	}
		82
2332	Serge	83	/**
		84	* Initialize an already allocated GEM object of the specified size with
		85	* shmfs backing store.
		86	*/
		87	int drm_gem_object_init(struct drm_device *dev,
		88	struct drm_gem_object *obj, size_t size)
		89	{
		90	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		91
		92	obj->dev = dev;
2344	Serge	93	kref_init(&obj->refcount);
2332	Serge	94	atomic_set(&obj->handle_count, 0);
		95	obj->size = size;
		96
		97	return 0;
		98	}
		99
2344	Serge	100	void
		101	drm_gem_object_release(struct drm_gem_object *obj)
		102	{ }
2332	Serge	103
		104
2326	Serge	105	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		106	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		107	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		108	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		109
2332	Serge	110	static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
		111	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		112	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		113	static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
		114	bool write);
		115	static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
		116	uint64_t offset,
		117	uint64_t size);
		118	static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
		119	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		120	unsigned alignment,
		121	bool map_and_fenceable);
		122	static void i915_gem_clear_fence_reg(struct drm_device *dev,
		123	struct drm_i915_fence_reg *reg);
		124	static int i915_gem_phys_pwrite(struct drm_device *dev,
		125	struct drm_i915_gem_object *obj,
		126	struct drm_i915_gem_pwrite *args,
		127	struct drm_file *file);
		128	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
2326	Serge	129
2352	Serge	130	//static int i915_gem_inactive_shrink(struct shrinker *shrinker,
		131	// struct shrink_control *sc);
2332	Serge	132
		133	/* some bookkeeping */
		134	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		135	size_t size)
		136	{
		137	dev_priv->mm.object_count++;
		138	dev_priv->mm.object_memory += size;
		139	}
		140
		141	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		142	size_t size)
		143	{
		144	dev_priv->mm.object_count--;
		145	dev_priv->mm.object_memory -= size;
		146	}
		147
		148	#if 0
		149
		150	static int
		151	i915_gem_wait_for_error(struct drm_device *dev)
		152	{
		153	struct drm_i915_private *dev_priv = dev->dev_private;
		154	struct completion *x = &dev_priv->error_completion;
		155	unsigned long flags;
		156	int ret;
		157
		158	if (!atomic_read(&dev_priv->mm.wedged))
		159	return 0;
		160
		161	ret = wait_for_completion_interruptible(x);
		162	if (ret)
		163	return ret;
		164
		165	if (atomic_read(&dev_priv->mm.wedged)) {
		166	/* GPU is hung, bump the completion count to account for
		167	* the token we just consumed so that we never hit zero and
		168	* end up waiting upon a subsequent completion event that
		169	* will never happen.
		170	*/
		171	spin_lock_irqsave(&x->wait.lock, flags);
		172	x->done++;
		173	spin_unlock_irqrestore(&x->wait.lock, flags);
		174	}
		175	return 0;
		176	}
		177
		178	int i915_mutex_lock_interruptible(struct drm_device *dev)
		179	{
		180	int ret;
		181
		182	ret = i915_gem_wait_for_error(dev);
		183	if (ret)
		184	return ret;
		185
		186	ret = mutex_lock_interruptible(&dev->struct_mutex);
		187	if (ret)
		188	return ret;
		189
		190	WARN_ON(i915_verify_lists(dev));
		191	return 0;
		192	}
2352	Serge	193	#endif
2332	Serge	194
		195	static inline bool
		196	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		197	{
		198	return obj->gtt_space && !obj->active && obj->pin_count == 0;
		199	}
		200
		201	void i915_gem_do_init(struct drm_device *dev,
		202	unsigned long start,
		203	unsigned long mappable_end,
		204	unsigned long end)
		205	{
		206	drm_i915_private_t *dev_priv = dev->dev_private;
		207
		208	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
		209
		210	dev_priv->mm.gtt_start = start;
		211	dev_priv->mm.gtt_mappable_end = mappable_end;
		212	dev_priv->mm.gtt_end = end;
		213	dev_priv->mm.gtt_total = end - start;
		214	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
		215
		216	/* Take over this portion of the GTT */
		217	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
		218	}
		219
		220	#if 0
		221
		222	int
		223	i915_gem_init_ioctl(struct drm_device dev, void data,
		224	struct drm_file *file)
		225	{
		226	struct drm_i915_gem_init *args = data;
		227
		228	if (args->gtt_start >= args->gtt_end \|\|
		229	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		230	return -EINVAL;
		231
		232	mutex_lock(&dev->struct_mutex);
		233	i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
		234	mutex_unlock(&dev->struct_mutex);
		235
		236	return 0;
		237	}
2351	Serge	238	#endif
2332	Serge	239
		240	int
		241	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		242	struct drm_file *file)
		243	{
		244	struct drm_i915_private *dev_priv = dev->dev_private;
		245	struct drm_i915_gem_get_aperture *args = data;
		246	struct drm_i915_gem_object *obj;
		247	size_t pinned;
		248
		249
		250	pinned = 0;
		251	mutex_lock(&dev->struct_mutex);
		252	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
		253	pinned += obj->gtt_space->size;
		254	mutex_unlock(&dev->struct_mutex);
		255
		256	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	257	args->aper_available_size = args->aper_size - pinned;
2332	Serge	258
		259	return 0;
		260	}
		261
2351	Serge	262	#if 0
		263
		264	int i915_gem_create(struct drm_file *file,
2332	Serge	265	struct drm_device *dev,
		266	uint64_t size,
		267	uint32_t *handle_p)
		268	{
		269	struct drm_i915_gem_object *obj;
		270	int ret;
		271	u32 handle;
		272
		273	size = roundup(size, PAGE_SIZE);
2342	Serge	274	if (size == 0)
		275	return -EINVAL;
2332	Serge	276
		277	/* Allocate the new object */
		278	obj = i915_gem_alloc_object(dev, size);
		279	if (obj == NULL)
		280	return -ENOMEM;
		281
		282	ret = drm_gem_handle_create(file, &obj->base, &handle);
		283	if (ret) {
		284	drm_gem_object_release(&obj->base);
		285	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		286	kfree(obj);
		287	return ret;
		288	}
		289
		290	/* drop reference from allocate - handle holds it now */
		291	drm_gem_object_unreference(&obj->base);
2351	Serge	292	trace_i915_gem_object_create(obj);
2332	Serge	293
		294	*handle_p = handle;
		295	return 0;
		296	}
		297
		298	int
		299	i915_gem_dumb_create(struct drm_file *file,
		300	struct drm_device *dev,
		301	struct drm_mode_create_dumb *args)
		302	{
		303	/* have to work out size/pitch and return them */
		304	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		305	args->size = args->pitch * args->height;
		306	return i915_gem_create(file, dev,
		307	args->size, &args->handle);
		308	}
		309
		310	int i915_gem_dumb_destroy(struct drm_file *file,
		311	struct drm_device *dev,
		312	uint32_t handle)
		313	{
		314	return drm_gem_handle_delete(file, handle);
		315	}
		316
2326	Serge	317	/**
2332	Serge	318	* Creates a new mm object and returns a handle to it.
		319	*/
		320	int
		321	i915_gem_create_ioctl(struct drm_device dev, void data,
		322	struct drm_file *file)
		323	{
		324	struct drm_i915_gem_create *args = data;
		325	return i915_gem_create(file, dev,
		326	args->size, &args->handle);
		327	}
		328
		329	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		330	{
		331	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		332
		333	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		334	obj->tiling_mode != I915_TILING_NONE;
		335	}
		336
		337	static inline void
		338	slow_shmem_copy(struct page *dst_page,
		339	int dst_offset,
		340	struct page *src_page,
		341	int src_offset,
		342	int length)
		343	{
		344	char dst_vaddr, src_vaddr;
		345
		346	dst_vaddr = kmap(dst_page);
		347	src_vaddr = kmap(src_page);
		348
		349	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
		350
		351	kunmap(src_page);
		352	kunmap(dst_page);
		353	}
		354
		355	static inline void
		356	slow_shmem_bit17_copy(struct page *gpu_page,
		357	int gpu_offset,
		358	struct page *cpu_page,
		359	int cpu_offset,
		360	int length,
		361	int is_read)
		362	{
		363	char gpu_vaddr, cpu_vaddr;
		364
		365	/* Use the unswizzled path if this page isn't affected. */
		366	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
		367	if (is_read)
		368	return slow_shmem_copy(cpu_page, cpu_offset,
		369	gpu_page, gpu_offset, length);
		370	else
		371	return slow_shmem_copy(gpu_page, gpu_offset,
		372	cpu_page, cpu_offset, length);
		373	}
		374
		375	gpu_vaddr = kmap(gpu_page);
		376	cpu_vaddr = kmap(cpu_page);
		377
		378	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
		379	* XORing with the other bits (A9 for Y, A9 and A10 for X)
		380	*/
		381	while (length > 0) {
		382	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		383	int this_length = min(cacheline_end - gpu_offset, length);
		384	int swizzled_gpu_offset = gpu_offset ^ 64;
		385
		386	if (is_read) {
		387	memcpy(cpu_vaddr + cpu_offset,
		388	gpu_vaddr + swizzled_gpu_offset,
		389	this_length);
		390	} else {
		391	memcpy(gpu_vaddr + swizzled_gpu_offset,
		392	cpu_vaddr + cpu_offset,
		393	this_length);
		394	}
		395	cpu_offset += this_length;
		396	gpu_offset += this_length;
		397	length -= this_length;
		398	}
		399
		400	kunmap(cpu_page);
		401	kunmap(gpu_page);
		402	}
		403
		404	/**
		405	* This is the fast shmem pread path, which attempts to copy_from_user directly
		406	* from the backing pages of the object to the user's address space. On a
		407	* fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
		408	*/
		409	static int
		410	i915_gem_shmem_pread_fast(struct drm_device *dev,
		411	struct drm_i915_gem_object *obj,
		412	struct drm_i915_gem_pread *args,
		413	struct drm_file *file)
		414	{
		415	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		416	ssize_t remain;
		417	loff_t offset;
		418	char __user *user_data;
		419	int page_offset, page_length;
		420
		421	user_data = (char __user *) (uintptr_t) args->data_ptr;
		422	remain = args->size;
		423
		424	offset = args->offset;
		425
		426	while (remain > 0) {
		427	struct page *page;
		428	char *vaddr;
		429	int ret;
		430
		431	/* Operation in this page
		432	*
		433	* page_offset = offset within page
		434	* page_length = bytes to copy for this page
		435	*/
		436	page_offset = offset_in_page(offset);
		437	page_length = remain;
		438	if ((page_offset + remain) > PAGE_SIZE)
		439	page_length = PAGE_SIZE - page_offset;
		440
		441	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		442	if (IS_ERR(page))
		443	return PTR_ERR(page);
		444
		445	vaddr = kmap_atomic(page);
		446	ret = __copy_to_user_inatomic(user_data,
		447	vaddr + page_offset,
		448	page_length);
		449	kunmap_atomic(vaddr);
		450
		451	mark_page_accessed(page);
		452	page_cache_release(page);
		453	if (ret)
		454	return -EFAULT;
		455
		456	remain -= page_length;
		457	user_data += page_length;
		458	offset += page_length;
		459	}
		460
		461	return 0;
		462	}
		463
		464	/**
		465	* This is the fallback shmem pread path, which allocates temporary storage
		466	* in kernel space to copy_to_user into outside of the struct_mutex, so we
		467	* can copy out of the object's backing pages while holding the struct mutex
		468	* and not take page faults.
		469	*/
		470	static int
		471	i915_gem_shmem_pread_slow(struct drm_device *dev,
		472	struct drm_i915_gem_object *obj,
		473	struct drm_i915_gem_pread *args,
		474	struct drm_file *file)
		475	{
		476	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		477	struct mm_struct *mm = current->mm;
		478	struct page **user_pages;
		479	ssize_t remain;
		480	loff_t offset, pinned_pages, i;
		481	loff_t first_data_page, last_data_page, num_pages;
		482	int shmem_page_offset;
		483	int data_page_index, data_page_offset;
		484	int page_length;
		485	int ret;
		486	uint64_t data_ptr = args->data_ptr;
		487	int do_bit17_swizzling;
		488
		489	remain = args->size;
		490
		491	/* Pin the user pages containing the data. We can't fault while
		492	* holding the struct mutex, yet we want to hold it while
		493	* dereferencing the user data.
		494	*/
		495	first_data_page = data_ptr / PAGE_SIZE;
		496	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
		497	num_pages = last_data_page - first_data_page + 1;
		498
		499	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
		500	if (user_pages == NULL)
		501	return -ENOMEM;
		502
		503	mutex_unlock(&dev->struct_mutex);
		504	down_read(&mm->mmap_sem);
		505	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
		506	num_pages, 1, 0, user_pages, NULL);
		507	up_read(&mm->mmap_sem);
		508	mutex_lock(&dev->struct_mutex);
		509	if (pinned_pages < num_pages) {
		510	ret = -EFAULT;
		511	goto out;
		512	}
		513
		514	ret = i915_gem_object_set_cpu_read_domain_range(obj,
		515	args->offset,
		516	args->size);
		517	if (ret)
		518	goto out;
		519
		520	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		521
		522	offset = args->offset;
		523
		524	while (remain > 0) {
		525	struct page *page;
		526
		527	/* Operation in this page
		528	*
		529	* shmem_page_offset = offset within page in shmem file
		530	* data_page_index = page number in get_user_pages return
		531	* data_page_offset = offset with data_page_index page.
		532	* page_length = bytes to copy for this page
		533	*/
		534	shmem_page_offset = offset_in_page(offset);
		535	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		536	data_page_offset = offset_in_page(data_ptr);
		537
		538	page_length = remain;
		539	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		540	page_length = PAGE_SIZE - shmem_page_offset;
		541	if ((data_page_offset + page_length) > PAGE_SIZE)
		542	page_length = PAGE_SIZE - data_page_offset;
		543
		544	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		545	if (IS_ERR(page)) {
		546	ret = PTR_ERR(page);
		547	goto out;
		548	}
		549
		550	if (do_bit17_swizzling) {
		551	slow_shmem_bit17_copy(page,
		552	shmem_page_offset,
		553	user_pages[data_page_index],
		554	data_page_offset,
		555	page_length,
		556	1);
		557	} else {
		558	slow_shmem_copy(user_pages[data_page_index],
		559	data_page_offset,
		560	page,
		561	shmem_page_offset,
		562	page_length);
		563	}
		564
		565	mark_page_accessed(page);
		566	page_cache_release(page);
		567
		568	remain -= page_length;
		569	data_ptr += page_length;
		570	offset += page_length;
		571	}
		572
		573	out:
		574	for (i = 0; i < pinned_pages; i++) {
		575	SetPageDirty(user_pages[i]);
		576	mark_page_accessed(user_pages[i]);
		577	page_cache_release(user_pages[i]);
		578	}
		579	drm_free_large(user_pages);
		580
		581	return ret;
		582	}
		583	#endif
		584
		585
		586
		587
		588
		589
		590
		591
		592
		593
		594
		595
		596
		597
		598
		599
		600
		601
		602
		603
		604
		605
		606
		607
		608
		609
		610
		611
		612
		613
		614
		615
		616
		617
		618
		619
		620
		621
		622
		623
		624
		625
		626
		627
		628
		629
		630
		631
		632
		633
		634
		635
		636
		637
		638
		639
		640
		641
		642
		643
		644
		645
		646
		647
		648
		649
		650
		651
		652
		653
		654
		655
		656
		657	static uint32_t
		658	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		659	{
		660	uint32_t gtt_size;
		661
		662	if (INTEL_INFO(dev)->gen >= 4 \|\|
		663	tiling_mode == I915_TILING_NONE)
		664	return size;
		665
		666	/* Previous chips need a power-of-two fence region when tiling */
		667	if (INTEL_INFO(dev)->gen == 3)
		668	gtt_size = 1024*1024;
		669	else
		670	gtt_size = 512*1024;
		671
		672	while (gtt_size < size)
		673	gtt_size <<= 1;
		674
		675	return gtt_size;
		676	}
		677
		678	/**
		679	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		680	* @obj: object to check
		681	*
		682	* Return the required GTT alignment for an object, taking into account
		683	* potential fence register mapping.
		684	*/
		685	static uint32_t
		686	i915_gem_get_gtt_alignment(struct drm_device *dev,
		687	uint32_t size,
		688	int tiling_mode)
		689	{
		690	/*
		691	* Minimum alignment is 4k (GTT page size), but might be greater
		692	* if a fence register is needed for the object.
		693	*/
		694	if (INTEL_INFO(dev)->gen >= 4 \|\|
		695	tiling_mode == I915_TILING_NONE)
		696	return 4096;
		697
		698	/*
		699	* Previous chips need to be aligned to the size of the smallest
		700	* fence register that can contain the object.
		701	*/
		702	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		703	}
		704
		705	/**
		706	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		707	* unfenced object
		708	* @dev: the device
		709	* @size: size of the object
		710	* @tiling_mode: tiling mode of the object
		711	*
		712	* Return the required GTT alignment for an object, only taking into account
		713	* unfenced tiled surface requirements.
		714	*/
		715	uint32_t
		716	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		717	uint32_t size,
		718	int tiling_mode)
		719	{
		720	/*
		721	* Minimum alignment is 4k (GTT page size) for sane hw.
		722	*/
		723	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		724	tiling_mode == I915_TILING_NONE)
		725	return 4096;
		726
		727	/* Previous hardware however needs to be aligned to a power-of-two
		728	* tile height. The simplest method for determining this is to reuse
		729	* the power-of-tile object size.
		730	*/
		731	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		732	}
		733
		734
		735
		736
		737
		738
		739
		740
		741
		742
		743
		744
		745
		746
		747
		748	static int
		749	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
		750	gfp_t gfpmask)
		751	{
		752	int page_count, i;
		753	struct page *page;
		754
		755	/* Get the list of pages out of our struct file. They'll be pinned
		756	* at this point until we release them.
		757	*/
		758	page_count = obj->base.size / PAGE_SIZE;
		759	BUG_ON(obj->pages != NULL);
		760	obj->pages = malloc(page_count * sizeof(struct page *));
		761	if (obj->pages == NULL)
		762	return -ENOMEM;
		763
		764
		765	for (i = 0; i < page_count; i++) {
		766	page = (struct page*)AllocPage(); // oh-oh
		767	if (IS_ERR(page))
		768	goto err_pages;
		769
		770	obj->pages[i] = page;
		771	}
		772
		773	// if (obj->tiling_mode != I915_TILING_NONE)
		774	// i915_gem_object_do_bit_17_swizzle(obj);
		775
		776
2340	Serge	777
2332	Serge	778	return 0;
		779
		780	err_pages:
2344	Serge	781	while (i--)
2352	Serge	782	FreePage((addr_t)obj->pages[i]);
2332	Serge	783
		784	free(obj->pages);
		785	obj->pages = NULL;
		786	return PTR_ERR(page);
		787	}
		788
		789	static void
		790	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		791	{
		792	int page_count = obj->base.size / PAGE_SIZE;
		793	int i;
		794
		795	BUG_ON(obj->madv == __I915_MADV_PURGED);
		796
		797	// if (obj->tiling_mode != I915_TILING_NONE)
		798	// i915_gem_object_save_bit_17_swizzle(obj);
		799
		800	if (obj->madv == I915_MADV_DONTNEED)
		801	obj->dirty = 0;
2344	Serge	802
2332	Serge	803	for (i = 0; i < page_count; i++) {
2352	Serge	804	FreePage((addr_t)obj->pages[i]);
2332	Serge	805	}
		806	obj->dirty = 0;
		807
		808	free(obj->pages);
		809	obj->pages = NULL;
		810	}
		811
		812	void
		813	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
		814	struct intel_ring_buffer *ring,
		815	u32 seqno)
		816	{
		817	struct drm_device *dev = obj->base.dev;
		818	struct drm_i915_private *dev_priv = dev->dev_private;
		819
		820	BUG_ON(ring == NULL);
		821	obj->ring = ring;
		822
		823	/* Add a reference if we're newly entering the active list. */
		824	if (!obj->active) {
2344	Serge	825	drm_gem_object_reference(&obj->base);
2332	Serge	826	obj->active = 1;
		827	}
		828
		829	/* Move from whatever list we were on to the tail of execution. */
		830	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		831	list_move_tail(&obj->ring_list, &ring->active_list);
		832
		833	obj->last_rendering_seqno = seqno;
		834	if (obj->fenced_gpu_access) {
		835	struct drm_i915_fence_reg *reg;
		836
		837	BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
		838
		839	obj->last_fenced_seqno = seqno;
		840	obj->last_fenced_ring = ring;
		841
		842	reg = &dev_priv->fence_regs[obj->fence_reg];
		843	list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
		844	}
		845	}
		846
2344	Serge	847	static void
		848	i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
		849	{
		850	list_del_init(&obj->ring_list);
		851	obj->last_rendering_seqno = 0;
		852	}
2332	Serge	853
2344	Serge	854	static void
		855	i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
		856	{
		857	struct drm_device *dev = obj->base.dev;
		858	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	859
2344	Serge	860	BUG_ON(!obj->active);
		861	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
2332	Serge	862
2344	Serge	863	i915_gem_object_move_off_active(obj);
		864	}
		865
2352	Serge	866	static void
		867	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
		868	{
		869	struct drm_device *dev = obj->base.dev;
		870	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	871
2352	Serge	872	if (obj->pin_count != 0)
		873	list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
		874	else
		875	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	876
2352	Serge	877	BUG_ON(!list_empty(&obj->gpu_write_list));
		878	BUG_ON(!obj->active);
		879	obj->ring = NULL;
2344	Serge	880
2352	Serge	881	i915_gem_object_move_off_active(obj);
		882	obj->fenced_gpu_access = false;
2344	Serge	883
2352	Serge	884	obj->active = 0;
		885	obj->pending_gpu_write = false;
		886	drm_gem_object_unreference(&obj->base);
		887
		888	WARN_ON(i915_verify_lists(dev));
		889	}
		890
2344	Serge	891	/* Immediately discard the backing storage */
2332	Serge	892	static void
2344	Serge	893	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		894	{
		895	struct inode *inode;
		896
		897	/* Our goal here is to return as much of the memory as
		898	* is possible back to the system as we are called from OOM.
		899	* To do this we must instruct the shmfs to drop all of its
		900	* backing pages, now.
		901	*/
		902
		903	obj->madv = __I915_MADV_PURGED;
		904	}
		905
		906	static inline int
		907	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		908	{
		909	return obj->madv == I915_MADV_DONTNEED;
		910	}
		911
		912	static void
2332	Serge	913	i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
		914	uint32_t flush_domains)
		915	{
		916	struct drm_i915_gem_object obj, next;
		917
		918	list_for_each_entry_safe(obj, next,
		919	&ring->gpu_write_list,
		920	gpu_write_list) {
		921	if (obj->base.write_domain & flush_domains) {
		922	uint32_t old_write_domain = obj->base.write_domain;
		923
		924	obj->base.write_domain = 0;
		925	list_del_init(&obj->gpu_write_list);
		926	i915_gem_object_move_to_active(obj, ring,
		927	i915_gem_next_request_seqno(ring));
		928
2352	Serge	929	trace_i915_gem_object_change_domain(obj,
		930	obj->base.read_domains,
		931	old_write_domain);
2332	Serge	932	}
		933	}
		934	}
		935
2352	Serge	936	int
		937	i915_add_request(struct intel_ring_buffer *ring,
		938	struct drm_file *file,
		939	struct drm_i915_gem_request *request)
		940	{
		941	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		942	uint32_t seqno;
		943	int was_empty;
		944	int ret;
2332	Serge	945
2352	Serge	946	BUG_ON(request == NULL);
2332	Serge	947
2352	Serge	948	ret = ring->add_request(ring, &seqno);
		949	if (ret)
		950	return ret;
2332	Serge	951
2352	Serge	952	trace_i915_gem_request_add(ring, seqno);
2332	Serge	953
2352	Serge	954	request->seqno = seqno;
		955	request->ring = ring;
		956	request->emitted_jiffies = jiffies;
		957	was_empty = list_empty(&ring->request_list);
		958	list_add_tail(&request->list, &ring->request_list);
2332	Serge	959
		960
2352	Serge	961	ring->outstanding_lazy_request = false;
2332	Serge	962
2360	Serge	963	if (!dev_priv->mm.suspended) {
		964	if (i915_enable_hangcheck) {
2352	Serge	965	// mod_timer(&dev_priv->hangcheck_timer,
		966	// jiffies +
		967	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
2360	Serge	968	}
		969	if (was_empty)
		970	queue_delayed_work(dev_priv->wq,
		971	&dev_priv->mm.retire_work, HZ);
		972	}
2352	Serge	973	return 0;
		974	}
2332	Serge	975
		976
		977
		978
		979
		980
		981
		982
		983
		984
		985
		986
2360	Serge	987
		988
		989
		990
		991
		992
		993
		994
		995
		996
2352	Serge	997	/**
		998	* This function clears the request list as sequence numbers are passed.
		999	*/
		1000	static void
		1001	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		1002	{
		1003	uint32_t seqno;
		1004	int i;
2332	Serge	1005
2352	Serge	1006	if (list_empty(&ring->request_list))
		1007	return;
2332	Serge	1008
2352	Serge	1009	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1010
2352	Serge	1011	seqno = ring->get_seqno(ring);
2332	Serge	1012
2352	Serge	1013	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
		1014	if (seqno >= ring->sync_seqno[i])
		1015	ring->sync_seqno[i] = 0;
2332	Serge	1016
2352	Serge	1017	while (!list_empty(&ring->request_list)) {
		1018	struct drm_i915_gem_request *request;
2332	Serge	1019
2352	Serge	1020	request = list_first_entry(&ring->request_list,
		1021	struct drm_i915_gem_request,
		1022	list);
2332	Serge	1023
2352	Serge	1024	if (!i915_seqno_passed(seqno, request->seqno))
		1025	break;
2332	Serge	1026
2352	Serge	1027	trace_i915_gem_request_retire(ring, request->seqno);
2332	Serge	1028
2352	Serge	1029	list_del(&request->list);
		1030	kfree(request);
		1031	}
2332	Serge	1032
2352	Serge	1033	/* Move any buffers on the active list that are no longer referenced
		1034	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1035	*/
		1036	while (!list_empty(&ring->active_list)) {
		1037	struct drm_i915_gem_object *obj;
2332	Serge	1038
2352	Serge	1039	obj = list_first_entry(&ring->active_list,
		1040	struct drm_i915_gem_object,
		1041	ring_list);
2332	Serge	1042
2352	Serge	1043	if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
		1044	break;
2332	Serge	1045
2352	Serge	1046	if (obj->base.write_domain != 0)
		1047	i915_gem_object_move_to_flushing(obj);
		1048	else
		1049	i915_gem_object_move_to_inactive(obj);
		1050	}
2332	Serge	1051
2352	Serge	1052	if (unlikely(ring->trace_irq_seqno &&
		1053	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1054	ring->irq_put(ring);
		1055	ring->trace_irq_seqno = 0;
		1056	}
2332	Serge	1057
2352	Serge	1058	WARN_ON(i915_verify_lists(ring->dev));
		1059	}
2332	Serge	1060
2352	Serge	1061	void
		1062	i915_gem_retire_requests(struct drm_device *dev)
		1063	{
		1064	drm_i915_private_t *dev_priv = dev->dev_private;
		1065	int i;
2332	Serge	1066
2352	Serge	1067	if (!list_empty(&dev_priv->mm.deferred_free_list)) {
		1068	struct drm_i915_gem_object obj, next;
2332	Serge	1069
2352	Serge	1070	/* We must be careful that during unbind() we do not
		1071	* accidentally infinitely recurse into retire requests.
		1072	* Currently:
		1073	* retire -> free -> unbind -> wait -> retire_ring
		1074	*/
		1075	list_for_each_entry_safe(obj, next,
		1076	&dev_priv->mm.deferred_free_list,
		1077	mm_list)
		1078	i915_gem_free_object_tail(obj);
		1079	}
2332	Serge	1080
2352	Serge	1081	for (i = 0; i < I915_NUM_RINGS; i++)
		1082	i915_gem_retire_requests_ring(&dev_priv->ring[i]);
		1083	}
		1084
2360	Serge	1085	static void
		1086	i915_gem_retire_work_handler(struct work_struct *work)
		1087	{
		1088	drm_i915_private_t *dev_priv;
		1089	struct drm_device *dev;
		1090	bool idle;
		1091	int i;
2352	Serge	1092
2360	Serge	1093	// ENTER();
2352	Serge	1094
2360	Serge	1095	dev_priv = container_of(work, drm_i915_private_t,
		1096	mm.retire_work.work);
		1097	dev = dev_priv->dev;
2352	Serge	1098
2360	Serge	1099	/* Come back later if the device is busy... */
		1100	if (!mutex_trylock(&dev->struct_mutex)) {
		1101	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
		1102	// LEAVE();
		1103	return;
		1104	}
2352	Serge	1105
2360	Serge	1106	i915_gem_retire_requests(dev);
2352	Serge	1107
2360	Serge	1108	/* Send a periodic flush down the ring so we don't hold onto GEM
		1109	* objects indefinitely.
		1110	*/
		1111	idle = true;
		1112	for (i = 0; i < I915_NUM_RINGS; i++) {
		1113	struct intel_ring_buffer *ring = &dev_priv->ring[i];
2352	Serge	1114
2360	Serge	1115	if (!list_empty(&ring->gpu_write_list)) {
		1116	struct drm_i915_gem_request *request;
		1117	int ret;
2352	Serge	1118
2360	Serge	1119	ret = i915_gem_flush_ring(ring,
		1120	0, I915_GEM_GPU_DOMAINS);
		1121	request = kzalloc(sizeof(*request), GFP_KERNEL);
		1122	if (ret \|\| request == NULL \|\|
		1123	i915_add_request(ring, NULL, request))
		1124	kfree(request);
		1125	}
2352	Serge	1126
2360	Serge	1127	idle &= list_empty(&ring->request_list);
		1128	}
2352	Serge	1129
2360	Serge	1130	if (!dev_priv->mm.suspended && !idle)
		1131	queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
		1132
		1133	mutex_unlock(&dev->struct_mutex);
		1134	// LEAVE();
		1135	}
		1136
2344	Serge	1137	/**
2352	Serge	1138	* Waits for a sequence number to be signaled, and cleans up the
		1139	* request and object lists appropriately for that event.
		1140	*/
		1141	int
		1142	i915_wait_request(struct intel_ring_buffer *ring,
		1143	uint32_t seqno)
		1144	{
		1145	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1146	u32 ier;
		1147	int ret = 0;
		1148
		1149	BUG_ON(seqno == 0);
		1150
		1151	// if (atomic_read(&dev_priv->mm.wedged)) {
		1152	// struct completion *x = &dev_priv->error_completion;
		1153	// bool recovery_complete;
		1154	// unsigned long flags;
		1155
		1156	/* Give the error handler a chance to run. */
		1157	// spin_lock_irqsave(&x->wait.lock, flags);
		1158	// recovery_complete = x->done > 0;
		1159	// spin_unlock_irqrestore(&x->wait.lock, flags);
		1160	//
		1161	// return recovery_complete ? -EIO : -EAGAIN;
		1162	// }
		1163
		1164	if (seqno == ring->outstanding_lazy_request) {
		1165	struct drm_i915_gem_request *request;
		1166
		1167	request = kzalloc(sizeof(*request), GFP_KERNEL);
		1168	if (request == NULL)
		1169	return -ENOMEM;
		1170
		1171	ret = i915_add_request(ring, NULL, request);
		1172	if (ret) {
		1173	kfree(request);
		1174	return ret;
		1175	}
		1176
		1177	seqno = request->seqno;
		1178	}
		1179
		1180	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
		1181	if (HAS_PCH_SPLIT(ring->dev))
		1182	ier = I915_READ(DEIER) \| I915_READ(GTIER);
		1183	else
		1184	ier = I915_READ(IER);
		1185	if (!ier) {
		1186	DRM_ERROR("something (likely vbetool) disabled "
		1187	"interrupts, re-enabling\n");
		1188	// ring->dev->driver->irq_preinstall(ring->dev);
		1189	// ring->dev->driver->irq_postinstall(ring->dev);
		1190	}
		1191
		1192	trace_i915_gem_request_wait_begin(ring, seqno);
		1193
		1194	ring->waiting_seqno = seqno;
		1195	if (ring->irq_get(ring)) {
		1196	// printf("enter wait\n");
		1197	wait_event(ring->irq_queue,
		1198	i915_seqno_passed(ring->get_seqno(ring), seqno)
		1199	\|\| atomic_read(&dev_priv->mm.wedged));
		1200
		1201	ring->irq_put(ring);
		1202	} else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
		1203	seqno) \|\|
		1204	atomic_read(&dev_priv->mm.wedged), 3000))
		1205	ret = -EBUSY;
		1206	ring->waiting_seqno = 0;
		1207
		1208	trace_i915_gem_request_wait_end(ring, seqno);
		1209	}
		1210	if (atomic_read(&dev_priv->mm.wedged))
		1211	ret = -EAGAIN;
		1212
		1213	if (ret && ret != -ERESTARTSYS)
		1214	DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
		1215	__func__, ret, seqno, ring->get_seqno(ring),
		1216	dev_priv->next_seqno);
		1217
		1218	/* Directly dispatch request retiring. While we have the work queue
		1219	* to handle this, the waiter on a request often wants an associated
		1220	* buffer to have made it to the inactive list, and we would need
		1221	* a separate wait queue to handle that.
		1222	*/
		1223	if (ret == 0)
		1224	i915_gem_retire_requests_ring(ring);
		1225
		1226	return ret;
		1227	}
		1228
		1229	/**
2344	Serge	1230	* Ensures that all rendering to the object has completed and the object is
		1231	* safe to unbind from the GTT or access from the CPU.
		1232	*/
		1233	int
		1234	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
		1235	{
		1236	int ret;
2332	Serge	1237
2344	Serge	1238	/* This function only exists to support waiting for existing rendering,
		1239	* not for emitting required flushes.
		1240	*/
		1241	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2332	Serge	1242
2344	Serge	1243	/* If there is rendering queued on the buffer being evicted, wait for
		1244	* it.
		1245	*/
		1246	if (obj->active) {
2352	Serge	1247	ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
		1248	if (ret)
		1249	return ret;
2344	Serge	1250	}
2332	Serge	1251
2344	Serge	1252	return 0;
		1253	}
2332	Serge	1254
2344	Serge	1255	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		1256	{
		1257	u32 old_write_domain, old_read_domains;
2332	Serge	1258
2344	Serge	1259	/* Act a barrier for all accesses through the GTT */
		1260	mb();
2332	Serge	1261
2344	Serge	1262	/* Force a pagefault for domain tracking on next user access */
		1263	// i915_gem_release_mmap(obj);
2332	Serge	1264
2344	Serge	1265	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		1266	return;
2332	Serge	1267
2344	Serge	1268	old_read_domains = obj->base.read_domains;
		1269	old_write_domain = obj->base.write_domain;
2351	Serge	1270
2344	Serge	1271	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		1272	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	1273
2351	Serge	1274	trace_i915_gem_object_change_domain(obj,
		1275	old_read_domains,
		1276	old_write_domain);
2344	Serge	1277	}
2332	Serge	1278
2344	Serge	1279	/**
		1280	* Unbinds an object from the GTT aperture.
		1281	*/
		1282	int
		1283	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		1284	{
		1285	int ret = 0;
2332	Serge	1286
2344	Serge	1287	if (obj->gtt_space == NULL)
		1288	return 0;
2332	Serge	1289
2344	Serge	1290	if (obj->pin_count != 0) {
		1291	DRM_ERROR("Attempting to unbind pinned buffer\n");
		1292	return -EINVAL;
		1293	}
2332	Serge	1294
2344	Serge	1295	ret = i915_gem_object_finish_gpu(obj);
		1296	if (ret == -ERESTARTSYS)
		1297	return ret;
		1298	/* Continue on if we fail due to EIO, the GPU is hung so we
		1299	* should be safe and we need to cleanup or else we might
		1300	* cause memory corruption through use-after-free.
		1301	*/
2332	Serge	1302
2344	Serge	1303	i915_gem_object_finish_gtt(obj);
2332	Serge	1304
2344	Serge	1305	/* Move the object to the CPU domain to ensure that
		1306	* any possible CPU writes while it's not in the GTT
		1307	* are flushed when we go to remap it.
		1308	*/
		1309	if (ret == 0)
		1310	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
		1311	if (ret == -ERESTARTSYS)
		1312	return ret;
		1313	if (ret) {
		1314	/* In the event of a disaster, abandon all caches and
		1315	* hope for the best.
		1316	*/
		1317	i915_gem_clflush_object(obj);
		1318	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1319	}
2332	Serge	1320
2344	Serge	1321	/* release the fence reg _after_ flushing */
		1322	ret = i915_gem_object_put_fence(obj);
		1323	if (ret == -ERESTARTSYS)
		1324	return ret;
2332	Serge	1325
2351	Serge	1326	trace_i915_gem_object_unbind(obj);
2332	Serge	1327
2344	Serge	1328	i915_gem_gtt_unbind_object(obj);
		1329	i915_gem_object_put_pages_gtt(obj);
2332	Serge	1330
2344	Serge	1331	list_del_init(&obj->gtt_list);
		1332	list_del_init(&obj->mm_list);
		1333	/* Avoid an unnecessary call to unbind on rebind. */
		1334	obj->map_and_fenceable = true;
2332	Serge	1335
2344	Serge	1336	drm_mm_put_block(obj->gtt_space);
		1337	obj->gtt_space = NULL;
		1338	obj->gtt_offset = 0;
2332	Serge	1339
2344	Serge	1340	if (i915_gem_object_is_purgeable(obj))
		1341	i915_gem_object_truncate(obj);
2332	Serge	1342
2344	Serge	1343	return ret;
		1344	}
2332	Serge	1345
2344	Serge	1346	int
		1347	i915_gem_flush_ring(struct intel_ring_buffer *ring,
		1348	uint32_t invalidate_domains,
		1349	uint32_t flush_domains)
		1350	{
		1351	int ret;
2332	Serge	1352
2344	Serge	1353	if (((invalidate_domains \| flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
		1354	return 0;
2332	Serge	1355
2351	Serge	1356	trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2332	Serge	1357
2344	Serge	1358	ret = ring->flush(ring, invalidate_domains, flush_domains);
		1359	if (ret)
		1360	return ret;
2332	Serge	1361
2344	Serge	1362	if (flush_domains & I915_GEM_GPU_DOMAINS)
		1363	i915_gem_process_flushing_list(ring, flush_domains);
2332	Serge	1364
2344	Serge	1365	return 0;
		1366	}
2332	Serge	1367
2344	Serge	1368	static int i915_ring_idle(struct intel_ring_buffer *ring)
		1369	{
		1370	int ret;
2332	Serge	1371
2344	Serge	1372	if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
		1373	return 0;
2332	Serge	1374
2344	Serge	1375	if (!list_empty(&ring->gpu_write_list)) {
		1376	ret = i915_gem_flush_ring(ring,
		1377	I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
		1378	if (ret)
		1379	return ret;
		1380	}
2332	Serge	1381
2360	Serge	1382	return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2344	Serge	1383	}
2332	Serge	1384
2344	Serge	1385	int
		1386	i915_gpu_idle(struct drm_device *dev)
		1387	{
		1388	drm_i915_private_t *dev_priv = dev->dev_private;
		1389	int ret, i;
2332	Serge	1390
2344	Serge	1391	/* Flush everything onto the inactive list. */
		1392	for (i = 0; i < I915_NUM_RINGS; i++) {
		1393	ret = i915_ring_idle(&dev_priv->ring[i]);
		1394	if (ret)
		1395	return ret;
		1396	}
2332	Serge	1397
2344	Serge	1398	return 0;
		1399	}
2332	Serge	1400
		1401
		1402
		1403
		1404
		1405
		1406
		1407
		1408
		1409
		1410
		1411
		1412
		1413
		1414
		1415
		1416
		1417
2344	Serge	1418
		1419
		1420
		1421
		1422	static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332	Serge	1423	{
2344	Serge	1424	return i915_seqno_passed(ring->get_seqno(ring), seqno);
		1425	}
		1426
		1427	static int
		1428	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
		1429	struct intel_ring_buffer *pipelined)
		1430	{
2332	Serge	1431	int ret;
		1432
2344	Serge	1433	if (obj->fenced_gpu_access) {
		1434	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
		1435	ret = i915_gem_flush_ring(obj->last_fenced_ring,
		1436	0, obj->base.write_domain);
		1437	if (ret)
		1438	return ret;
		1439	}
2332	Serge	1440
2344	Serge	1441	obj->fenced_gpu_access = false;
		1442	}
		1443
		1444	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
		1445	if (!ring_passed_seqno(obj->last_fenced_ring,
		1446	obj->last_fenced_seqno)) {
2352	Serge	1447	ret = i915_wait_request(obj->last_fenced_ring,
		1448	obj->last_fenced_seqno);
		1449	if (ret)
		1450	return ret;
2344	Serge	1451	}
		1452
		1453	obj->last_fenced_seqno = 0;
		1454	obj->last_fenced_ring = NULL;
		1455	}
		1456
		1457	/* Ensure that all CPU reads are completed before installing a fence
		1458	* and all writes before removing the fence.
2332	Serge	1459	*/
2344	Serge	1460	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		1461	mb();
2332	Serge	1462
		1463	return 0;
		1464	}
		1465
		1466	int
2344	Serge	1467	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	1468	{
		1469	int ret;
		1470
2344	Serge	1471	// if (obj->tiling_mode)
		1472	// i915_gem_release_mmap(obj);
2332	Serge	1473
2344	Serge	1474	ret = i915_gem_object_flush_fence(obj, NULL);
2332	Serge	1475	if (ret)
		1476	return ret;
		1477
2344	Serge	1478	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		1479	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1480	i915_gem_clear_fence_reg(obj->base.dev,
		1481	&dev_priv->fence_regs[obj->fence_reg]);
2332	Serge	1482
2344	Serge	1483	obj->fence_reg = I915_FENCE_REG_NONE;
		1484	}
		1485
2332	Serge	1486	return 0;
		1487	}
		1488
		1489
		1490
		1491
		1492
		1493
		1494
		1495
		1496
		1497
		1498
		1499
		1500
		1501
		1502
		1503
		1504
		1505
		1506
		1507
2344	Serge	1508
		1509
		1510
		1511
		1512
		1513
		1514
		1515
		1516
		1517
		1518
2332	Serge	1519	/**
2326	Serge	1520	* i915_gem_clear_fence_reg - clear out fence register info
		1521	* @obj: object to clear
		1522	*
		1523	* Zeroes out the fence register itself and clears out the associated
		1524	* data structures in dev_priv and obj.
		1525	*/
		1526	static void
		1527	i915_gem_clear_fence_reg(struct drm_device *dev,
		1528	struct drm_i915_fence_reg *reg)
		1529	{
		1530	drm_i915_private_t *dev_priv = dev->dev_private;
		1531	uint32_t fence_reg = reg - dev_priv->fence_regs;
		1532
		1533	switch (INTEL_INFO(dev)->gen) {
		1534	case 7:
		1535	case 6:
		1536	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
		1537	break;
		1538	case 5:
		1539	case 4:
		1540	I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
		1541	break;
		1542	case 3:
		1543	if (fence_reg >= 8)
		1544	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
		1545	else
		1546	case 2:
		1547	fence_reg = FENCE_REG_830_0 + fence_reg * 4;
		1548
		1549	I915_WRITE(fence_reg, 0);
		1550	break;
		1551	}
		1552
		1553	list_del_init(®->lru_list);
		1554	reg->obj = NULL;
		1555	reg->setup_seqno = 0;
		1556	}
		1557
2332	Serge	1558	/**
		1559	* Finds free space in the GTT aperture and binds the object there.
		1560	*/
		1561	static int
		1562	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		1563	unsigned alignment,
		1564	bool map_and_fenceable)
		1565	{
		1566	struct drm_device *dev = obj->base.dev;
		1567	drm_i915_private_t *dev_priv = dev->dev_private;
		1568	struct drm_mm_node *free_space;
		1569	gfp_t gfpmask = 0; //__GFP_NORETRY \| __GFP_NOWARN;
		1570	u32 size, fence_size, fence_alignment, unfenced_alignment;
		1571	bool mappable, fenceable;
		1572	int ret;
2326	Serge	1573
2332	Serge	1574	if (obj->madv != I915_MADV_WILLNEED) {
		1575	DRM_ERROR("Attempting to bind a purgeable object\n");
		1576	return -EINVAL;
		1577	}
		1578
		1579	fence_size = i915_gem_get_gtt_size(dev,
		1580	obj->base.size,
		1581	obj->tiling_mode);
		1582	fence_alignment = i915_gem_get_gtt_alignment(dev,
		1583	obj->base.size,
		1584	obj->tiling_mode);
		1585	unfenced_alignment =
		1586	i915_gem_get_unfenced_gtt_alignment(dev,
		1587	obj->base.size,
		1588	obj->tiling_mode);
		1589
		1590	if (alignment == 0)
		1591	alignment = map_and_fenceable ? fence_alignment :
		1592	unfenced_alignment;
		1593	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		1594	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		1595	return -EINVAL;
		1596	}
		1597
		1598	size = map_and_fenceable ? fence_size : obj->base.size;
		1599
		1600	/* If the object is bigger than the entire aperture, reject it early
		1601	* before evicting everything in a vain attempt to find space.
		1602	*/
		1603	if (obj->base.size >
		1604	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		1605	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		1606	return -E2BIG;
		1607	}
		1608
		1609	search_free:
		1610	if (map_and_fenceable)
		1611	free_space =
		1612	drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
		1613	size, alignment, 0,
		1614	dev_priv->mm.gtt_mappable_end,
		1615	0);
		1616	else
		1617	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
		1618	size, alignment, 0);
		1619
		1620	if (free_space != NULL) {
		1621	if (map_and_fenceable)
		1622	obj->gtt_space =
		1623	drm_mm_get_block_range_generic(free_space,
		1624	size, alignment, 0,
		1625	dev_priv->mm.gtt_mappable_end,
		1626	0);
		1627	else
		1628	obj->gtt_space =
		1629	drm_mm_get_block(free_space, size, alignment);
		1630	}
		1631	if (obj->gtt_space == NULL) {
		1632	/* If the gtt is empty and we're still having trouble
		1633	* fitting our object in, we're out of memory.
		1634	*/
		1635	ret = 1; //i915_gem_evict_something(dev, size, alignment,
		1636	// map_and_fenceable);
		1637	if (ret)
		1638	return ret;
		1639
		1640	goto search_free;
		1641	}
		1642
		1643	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
		1644	if (ret) {
		1645	drm_mm_put_block(obj->gtt_space);
		1646	obj->gtt_space = NULL;
		1647	#if 0
		1648	if (ret == -ENOMEM) {
		1649	/* first try to reclaim some memory by clearing the GTT */
		1650	ret = i915_gem_evict_everything(dev, false);
		1651	if (ret) {
		1652	/* now try to shrink everyone else */
		1653	if (gfpmask) {
		1654	gfpmask = 0;
		1655	goto search_free;
		1656	}
		1657
		1658	return -ENOMEM;
		1659	}
		1660
		1661	goto search_free;
		1662	}
		1663	#endif
		1664	return ret;
		1665	}
		1666
		1667	ret = i915_gem_gtt_bind_object(obj);
		1668	if (ret) {
2344	Serge	1669	i915_gem_object_put_pages_gtt(obj);
2332	Serge	1670	drm_mm_put_block(obj->gtt_space);
		1671	obj->gtt_space = NULL;
		1672
		1673	// if (i915_gem_evict_everything(dev, false))
		1674	return ret;
		1675
		1676	// goto search_free;
		1677	}
		1678
		1679	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
		1680	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		1681
		1682	/* Assert that the object is not currently in any GPU domain. As it
		1683	* wasn't in the GTT, there shouldn't be any way it could have been in
		1684	* a GPU cache
		1685	*/
		1686	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1687	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1688
		1689	obj->gtt_offset = obj->gtt_space->start;
		1690
		1691	fenceable =
		1692	obj->gtt_space->size == fence_size &&
2342	Serge	1693	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2332	Serge	1694
		1695	mappable =
		1696	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		1697
		1698	obj->map_and_fenceable = mappable && fenceable;
		1699
2351	Serge	1700	trace_i915_gem_object_bind(obj, map_and_fenceable);
2332	Serge	1701	return 0;
		1702	}
		1703
		1704	void
		1705	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		1706	{
		1707	/* If we don't have a page list set up, then we're not pinned
		1708	* to GPU, and we can ignore the cache flush because it'll happen
		1709	* again at bind time.
		1710	*/
		1711	if (obj->pages == NULL)
		1712	return;
		1713
		1714	/* If the GPU is snooping the contents of the CPU cache,
		1715	* we do not need to manually clear the CPU cache lines. However,
		1716	* the caches are only snooped when the render cache is
		1717	* flushed/invalidated. As we always have to emit invalidations
		1718	* and flushes when moving into and out of the RENDER domain, correct
		1719	* snooping behaviour occurs naturally as the result of our domain
		1720	* tracking.
		1721	*/
		1722	if (obj->cache_level != I915_CACHE_NONE)
		1723	return;
		1724
2344	Serge	1725	if(obj->mapped != NULL)
		1726	{
		1727	uint8_t *page_virtual;
		1728	unsigned int i;
2332	Serge	1729
2344	Serge	1730	page_virtual = obj->mapped;
		1731	asm volatile("mfence");
		1732	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		1733	clflush(page_virtual + i);
		1734	asm volatile("mfence");
		1735	}
		1736	else
		1737	{
		1738	uint8_t *page_virtual;
		1739	unsigned int i;
		1740	page_virtual = AllocKernelSpace(obj->base.size);
		1741	if(page_virtual != NULL)
		1742	{
		1743	u32_t src, dst;
		1744	u32 count;
		1745
		1746	#define page_tabs 0xFDC00000 /* really dirty hack */
		1747
		1748	src = (u32_t*)obj->pages;
		1749	dst = &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12];
		1750	count = obj->base.size/4096;
		1751
		1752	while(count--)
		1753	{
		1754	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		1755	};
		1756
		1757	asm volatile("mfence");
		1758	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		1759	clflush(page_virtual + i);
		1760	asm volatile("mfence");
		1761	FreeKernelSpace(page_virtual);
		1762	}
		1763	else
		1764	{
		1765	asm volatile (
		1766	"mfence \n"
		1767	"wbinvd \n" /* this is really ugly */
		1768	"mfence");
		1769	}
		1770	}
2332	Serge	1771	}
		1772
		1773	/** Flushes any GPU write domain for the object if it's dirty. */
		1774	static int
		1775	i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
		1776	{
		1777	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
		1778	return 0;
		1779
		1780	/* Queue the GPU write cache flushing we need. */
		1781	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		1782	}
		1783
2344	Serge	1784	/** Flushes the GTT write domain for the object if it's dirty. */
		1785	static void
		1786	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		1787	{
		1788	uint32_t old_write_domain;
2332	Serge	1789
2344	Serge	1790	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		1791	return;
2332	Serge	1792
2344	Serge	1793	/* No actual flushing is required for the GTT write domain. Writes
		1794	* to it immediately go to main memory as far as we know, so there's
		1795	* no chipset flush. It also doesn't land in render cache.
		1796	*
		1797	* However, we do have to enforce the order so that all writes through
		1798	* the GTT land before any writes to the device, such as updates to
		1799	* the GATT itself.
		1800	*/
		1801	wmb();
2332	Serge	1802
2344	Serge	1803	old_write_domain = obj->base.write_domain;
		1804	obj->base.write_domain = 0;
2332	Serge	1805
2351	Serge	1806	trace_i915_gem_object_change_domain(obj,
		1807	obj->base.read_domains,
		1808	old_write_domain);
2344	Serge	1809	}
2332	Serge	1810
		1811	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	1812	static void
2332	Serge	1813	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		1814	{
		1815	uint32_t old_write_domain;
		1816
		1817	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		1818	return;
		1819
		1820	i915_gem_clflush_object(obj);
		1821	intel_gtt_chipset_flush();
		1822	old_write_domain = obj->base.write_domain;
		1823	obj->base.write_domain = 0;
		1824
2351	Serge	1825	trace_i915_gem_object_change_domain(obj,
		1826	obj->base.read_domains,
		1827	old_write_domain);
2332	Serge	1828	}
		1829
		1830	/**
		1831	* Moves a single object to the GTT read, and possibly write domain.
		1832	*
		1833	* This function returns when the move is complete, including waiting on
		1834	* flushes to occur.
		1835	*/
		1836	int
		1837	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		1838	{
		1839	uint32_t old_write_domain, old_read_domains;
		1840	int ret;
		1841
		1842	/* Not valid to be called on unbound objects. */
		1843	if (obj->gtt_space == NULL)
		1844	return -EINVAL;
		1845
		1846	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		1847	return 0;
		1848
		1849	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1850	if (ret)
		1851	return ret;
		1852
		1853	if (obj->pending_gpu_write \|\| write) {
		1854	ret = i915_gem_object_wait_rendering(obj);
		1855	if (ret)
		1856	return ret;
		1857	}
		1858
		1859	i915_gem_object_flush_cpu_write_domain(obj);
		1860
		1861	old_write_domain = obj->base.write_domain;
		1862	old_read_domains = obj->base.read_domains;
		1863
		1864	/* It should now be out of any other write domains, and we can update
		1865	* the domain values for our changes.
		1866	*/
		1867	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1868	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		1869	if (write) {
		1870	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		1871	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		1872	obj->dirty = 1;
		1873	}
		1874
2351	Serge	1875	trace_i915_gem_object_change_domain(obj,
		1876	old_read_domains,
		1877	old_write_domain);
		1878
2332	Serge	1879	return 0;
		1880	}
		1881
2335	Serge	1882	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		1883	enum i915_cache_level cache_level)
		1884	{
		1885	int ret;
2332	Serge	1886
2335	Serge	1887	if (obj->cache_level == cache_level)
		1888	return 0;
2332	Serge	1889
2335	Serge	1890	if (obj->pin_count) {
		1891	DRM_DEBUG("can not change the cache level of pinned objects\n");
		1892	return -EBUSY;
		1893	}
2332	Serge	1894
2335	Serge	1895	if (obj->gtt_space) {
		1896	ret = i915_gem_object_finish_gpu(obj);
		1897	if (ret)
		1898	return ret;
2332	Serge	1899
2335	Serge	1900	i915_gem_object_finish_gtt(obj);
2332	Serge	1901
2335	Serge	1902	/* Before SandyBridge, you could not use tiling or fence
		1903	* registers with snooped memory, so relinquish any fences
		1904	* currently pointing to our region in the aperture.
		1905	*/
		1906	if (INTEL_INFO(obj->base.dev)->gen < 6) {
		1907	ret = i915_gem_object_put_fence(obj);
		1908	if (ret)
		1909	return ret;
		1910	}
2332	Serge	1911
2335	Serge	1912	i915_gem_gtt_rebind_object(obj, cache_level);
		1913	}
2332	Serge	1914
2335	Serge	1915	if (cache_level == I915_CACHE_NONE) {
		1916	u32 old_read_domains, old_write_domain;
2332	Serge	1917
2335	Serge	1918	/* If we're coming from LLC cached, then we haven't
		1919	* actually been tracking whether the data is in the
		1920	* CPU cache or not, since we only allow one bit set
		1921	* in obj->write_domain and have been skipping the clflushes.
		1922	* Just set it to the CPU cache for now.
		1923	*/
		1924	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		1925	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	1926
2335	Serge	1927	old_read_domains = obj->base.read_domains;
		1928	old_write_domain = obj->base.write_domain;
2332	Serge	1929
2335	Serge	1930	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1931	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	1932
2351	Serge	1933	trace_i915_gem_object_change_domain(obj,
		1934	old_read_domains,
		1935	old_write_domain);
2344	Serge	1936	}
2332	Serge	1937
2335	Serge	1938	obj->cache_level = cache_level;
		1939	return 0;
		1940	}
2332	Serge	1941
2335	Serge	1942	/*
		1943	* Prepare buffer for display plane (scanout, cursors, etc).
		1944	* Can be called from an uninterruptible phase (modesetting) and allows
		1945	* any flushes to be pipelined (for pageflips).
		1946	*
		1947	* For the display plane, we want to be in the GTT but out of any write
		1948	* domains. So in many ways this looks like set_to_gtt_domain() apart from the
		1949	* ability to pipeline the waits, pinning and any additional subtleties
		1950	* that may differentiate the display plane from ordinary buffers.
		1951	*/
		1952	int
		1953	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		1954	u32 alignment,
		1955	struct intel_ring_buffer *pipelined)
		1956	{
		1957	u32 old_read_domains, old_write_domain;
		1958	int ret;
2332	Serge	1959
2335	Serge	1960	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1961	if (ret)
		1962	return ret;
2332	Serge	1963
2335	Serge	1964	if (pipelined != obj->ring) {
		1965	ret = i915_gem_object_wait_rendering(obj);
		1966	if (ret == -ERESTARTSYS)
		1967	return ret;
		1968	}
2332	Serge	1969
2335	Serge	1970	/* The display engine is not coherent with the LLC cache on gen6. As
		1971	* a result, we make sure that the pinning that is about to occur is
		1972	* done with uncached PTEs. This is lowest common denominator for all
		1973	* chipsets.
		1974	*
		1975	* However for gen6+, we could do better by using the GFDT bit instead
		1976	* of uncaching, which would allow us to flush all the LLC-cached data
		1977	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		1978	*/
2360	Serge	1979	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		1980	if (ret)
		1981	return ret;
2332	Serge	1982
2335	Serge	1983	/* As the user may map the buffer once pinned in the display plane
		1984	* (e.g. libkms for the bootup splash), we have to ensure that we
		1985	* always use map_and_fenceable for all scanout buffers.
		1986	*/
		1987	ret = i915_gem_object_pin(obj, alignment, true);
		1988	if (ret)
		1989	return ret;
2332	Serge	1990
2335	Serge	1991	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	1992
2335	Serge	1993	old_write_domain = obj->base.write_domain;
		1994	old_read_domains = obj->base.read_domains;
2332	Serge	1995
2335	Serge	1996	/* It should now be out of any other write domains, and we can update
		1997	* the domain values for our changes.
		1998	*/
		1999	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		2000	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	2001
2351	Serge	2002	trace_i915_gem_object_change_domain(obj,
		2003	old_read_domains,
		2004	old_write_domain);
2332	Serge	2005
2335	Serge	2006	return 0;
		2007	}
2332	Serge	2008
2344	Serge	2009	int
		2010	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		2011	{
		2012	int ret;
2332	Serge	2013
2344	Serge	2014	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		2015	return 0;
2332	Serge	2016
2344	Serge	2017	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
		2018	ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		2019	if (ret)
		2020	return ret;
		2021	}
2332	Serge	2022
2344	Serge	2023	/* Ensure that we invalidate the GPU's caches and TLBs. */
		2024	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2332	Serge	2025
2344	Serge	2026	return i915_gem_object_wait_rendering(obj);
		2027	}
2332	Serge	2028
2344	Serge	2029	/**
		2030	* Moves a single object to the CPU read, and possibly write domain.
		2031	*
		2032	* This function returns when the move is complete, including waiting on
		2033	* flushes to occur.
		2034	*/
		2035	static int
		2036	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		2037	{
		2038	uint32_t old_write_domain, old_read_domains;
		2039	int ret;
2332	Serge	2040
2344	Serge	2041	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		2042	return 0;
2332	Serge	2043
2344	Serge	2044	ret = i915_gem_object_flush_gpu_write_domain(obj);
		2045	if (ret)
		2046	return ret;
2332	Serge	2047
2344	Serge	2048	ret = i915_gem_object_wait_rendering(obj);
		2049	if (ret)
		2050	return ret;
2332	Serge	2051
2344	Serge	2052	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	2053
		2054
2344	Serge	2055	old_write_domain = obj->base.write_domain;
		2056	old_read_domains = obj->base.read_domains;
2332	Serge	2057
2344	Serge	2058	/* Flush the CPU cache if it's still invalid. */
		2059	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		2060	i915_gem_clflush_object(obj);
2332	Serge	2061
2344	Serge	2062	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		2063	}
2332	Serge	2064
2344	Serge	2065	/* It should now be out of any other write domains, and we can update
		2066	* the domain values for our changes.
		2067	*/
		2068	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	2069
2344	Serge	2070	/* If we're writing through the CPU, then the GPU read domains will
		2071	* need to be invalidated at next use.
		2072	*/
		2073	if (write) {
		2074	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2075	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2076	}
2332	Serge	2077
2351	Serge	2078	trace_i915_gem_object_change_domain(obj,
		2079	old_read_domains,
		2080	old_write_domain);
2332	Serge	2081
2344	Serge	2082	return 0;
		2083	}
2332	Serge	2084
2344	Serge	2085	/**
		2086	* Moves the object from a partially CPU read to a full one.
		2087	*
		2088	* Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
		2089	* and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
		2090	*/
		2091	static void
		2092	i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
		2093	{
		2094	if (!obj->page_cpu_valid)
		2095	return;
2332	Serge	2096
2344	Serge	2097	/* If we're partially in the CPU read domain, finish moving it in.
		2098	*/
		2099	if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
		2100	}
2332	Serge	2101
2344	Serge	2102	/* Free the page_cpu_valid mappings which are now stale, whether
		2103	* or not we've got I915_GEM_DOMAIN_CPU.
		2104	*/
		2105	kfree(obj->page_cpu_valid);
		2106	obj->page_cpu_valid = NULL;
		2107	}
2332	Serge	2108
		2109
		2110
		2111
2352	Serge	2112	int gem_object_lock(struct drm_i915_gem_object *obj)
		2113	{
		2114	return i915_gem_object_set_to_cpu_domain(obj, true);
		2115	}
2332	Serge	2116
		2117
		2118
		2119
		2120
		2121
		2122
		2123
		2124
		2125
		2126
		2127
		2128
		2129
		2130
		2131
		2132
		2133
		2134
		2135	int
		2136	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		2137	uint32_t alignment,
		2138	bool map_and_fenceable)
		2139	{
		2140	struct drm_device *dev = obj->base.dev;
		2141	struct drm_i915_private *dev_priv = dev->dev_private;
		2142	int ret;
		2143
		2144	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
2352	Serge	2145	WARN_ON(i915_verify_lists(dev));
2332	Serge	2146
		2147	#if 0
		2148	if (obj->gtt_space != NULL) {
		2149	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		2150	(map_and_fenceable && !obj->map_and_fenceable)) {
		2151	WARN(obj->pin_count,
		2152	"bo is already pinned with incorrect alignment:"
		2153	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		2154	" obj->map_and_fenceable=%d\n",
		2155	obj->gtt_offset, alignment,
		2156	map_and_fenceable,
		2157	obj->map_and_fenceable);
		2158	ret = i915_gem_object_unbind(obj);
		2159	if (ret)
		2160	return ret;
		2161	}
		2162	}
		2163	#endif
		2164
		2165	if (obj->gtt_space == NULL) {
		2166	ret = i915_gem_object_bind_to_gtt(obj, alignment,
		2167	map_and_fenceable);
		2168	if (ret)
		2169	return ret;
		2170	}
		2171
		2172	if (obj->pin_count++ == 0) {
		2173	if (!obj->active)
		2174	list_move_tail(&obj->mm_list,
		2175	&dev_priv->mm.pinned_list);
		2176	}
		2177	obj->pin_mappable \|= map_and_fenceable;
		2178
2360	Serge	2179	WARN_ON(i915_verify_lists(dev));
2332	Serge	2180	return 0;
		2181	}
		2182
2344	Serge	2183	void
		2184	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		2185	{
		2186	struct drm_device *dev = obj->base.dev;
		2187	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	2188
2360	Serge	2189	WARN_ON(i915_verify_lists(dev));
2344	Serge	2190	BUG_ON(obj->pin_count == 0);
		2191	BUG_ON(obj->gtt_space == NULL);
2332	Serge	2192
2344	Serge	2193	if (--obj->pin_count == 0) {
		2194	if (!obj->active)
		2195	list_move_tail(&obj->mm_list,
		2196	&dev_priv->mm.inactive_list);
		2197	obj->pin_mappable = false;
		2198	}
2360	Serge	2199	WARN_ON(i915_verify_lists(dev));
2344	Serge	2200	}
2332	Serge	2201
		2202
		2203
		2204
		2205
		2206
		2207
		2208
		2209
		2210
		2211
		2212
		2213
		2214
		2215
		2216
		2217
		2218
		2219
		2220
		2221
		2222
		2223
		2224
		2225
		2226
		2227
		2228
		2229
		2230
		2231	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		2232	size_t size)
		2233	{
		2234	struct drm_i915_private *dev_priv = dev->dev_private;
		2235	struct drm_i915_gem_object *obj;
2340	Serge	2236
2332	Serge	2237	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		2238	if (obj == NULL)
		2239	return NULL;
		2240
		2241	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		2242	kfree(obj);
		2243	return NULL;
		2244	}
		2245
		2246
		2247	i915_gem_info_add_obj(dev_priv, size);
		2248
		2249	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2250	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2251
2342	Serge	2252	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
2332	Serge	2253	/* On Gen6, we can have the GPU use the LLC (the CPU
		2254	* cache) for about a 10% performance improvement
		2255	* compared to uncached. Graphics requests other than
		2256	* display scanout are coherent with the CPU in
		2257	* accessing this cache. This means in this mode we
		2258	* don't need to clflush on the CPU side, and on the
		2259	* GPU side we only need to flush internal caches to
		2260	* get data visible to the CPU.
		2261	*
		2262	* However, we maintain the display planes as UC, and so
		2263	* need to rebind when first used as such.
		2264	*/
		2265	obj->cache_level = I915_CACHE_LLC;
		2266	} else
		2267	obj->cache_level = I915_CACHE_NONE;
		2268
		2269	obj->base.driver_private = NULL;
		2270	obj->fence_reg = I915_FENCE_REG_NONE;
		2271	INIT_LIST_HEAD(&obj->mm_list);
		2272	INIT_LIST_HEAD(&obj->gtt_list);
		2273	INIT_LIST_HEAD(&obj->ring_list);
		2274	INIT_LIST_HEAD(&obj->exec_list);
		2275	INIT_LIST_HEAD(&obj->gpu_write_list);
		2276	obj->madv = I915_MADV_WILLNEED;
		2277	/* Avoid an unnecessary call to unbind on the first bind. */
		2278	obj->map_and_fenceable = true;
2340	Serge	2279
2332	Serge	2280	return obj;
		2281	}
		2282
2344	Serge	2283	int i915_gem_init_object(struct drm_gem_object *obj)
		2284	{
		2285	BUG();
2332	Serge	2286
2344	Serge	2287	return 0;
		2288	}
2332	Serge	2289
2344	Serge	2290	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
		2291	{
		2292	struct drm_device *dev = obj->base.dev;
		2293	drm_i915_private_t *dev_priv = dev->dev_private;
		2294	int ret;
2332	Serge	2295
2344	Serge	2296	ret = i915_gem_object_unbind(obj);
		2297	if (ret == -ERESTARTSYS) {
		2298	list_move(&obj->mm_list,
		2299	&dev_priv->mm.deferred_free_list);
		2300	return;
		2301	}
2332	Serge	2302
2351	Serge	2303	trace_i915_gem_object_destroy(obj);
2332	Serge	2304
2344	Serge	2305	// if (obj->base.map_list.map)
		2306	// drm_gem_free_mmap_offset(&obj->base);
2332	Serge	2307
2344	Serge	2308	drm_gem_object_release(&obj->base);
		2309	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	2310
2344	Serge	2311	kfree(obj->page_cpu_valid);
		2312	kfree(obj->bit_17);
		2313	kfree(obj);
		2314	}
2332	Serge	2315
2344	Serge	2316	void i915_gem_free_object(struct drm_gem_object *gem_obj)
		2317	{
		2318	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
		2319	struct drm_device *dev = obj->base.dev;
2332	Serge	2320
2351	Serge	2321	while (obj->pin_count > 0)
2344	Serge	2322	i915_gem_object_unpin(obj);
2332	Serge	2323
2344	Serge	2324	// if (obj->phys_obj)
		2325	// i915_gem_detach_phys_object(dev, obj);
2332	Serge	2326
2344	Serge	2327	i915_gem_free_object_tail(obj);
		2328	}
2332	Serge	2329
		2330
		2331
		2332
		2333
		2334
		2335
		2336
		2337
		2338
		2339
2344	Serge	2340
2332	Serge	2341	int
		2342	i915_gem_init_ringbuffer(struct drm_device *dev)
		2343	{
		2344	drm_i915_private_t *dev_priv = dev->dev_private;
		2345	int ret;
2351	Serge	2346
2332	Serge	2347	ret = intel_init_render_ring_buffer(dev);
		2348	if (ret)
		2349	return ret;
		2350
		2351	if (HAS_BSD(dev)) {
		2352	ret = intel_init_bsd_ring_buffer(dev);
		2353	if (ret)
		2354	goto cleanup_render_ring;
		2355	}
		2356
		2357	if (HAS_BLT(dev)) {
		2358	ret = intel_init_blt_ring_buffer(dev);
		2359	if (ret)
		2360	goto cleanup_bsd_ring;
		2361	}
		2362
		2363	dev_priv->next_seqno = 1;
2351	Serge	2364
2332	Serge	2365	return 0;
		2366
		2367	cleanup_bsd_ring:
		2368	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		2369	cleanup_render_ring:
		2370	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		2371	return ret;
		2372	}
		2373
		2374	#if 0
		2375	void
		2376	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		2377	{
		2378	drm_i915_private_t *dev_priv = dev->dev_private;
		2379	int i;
		2380
		2381	for (i = 0; i < I915_NUM_RINGS; i++)
		2382	intel_cleanup_ring_buffer(&dev_priv->ring[i]);
		2383	}
		2384
		2385	int
		2386	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		2387	struct drm_file *file_priv)
		2388	{
		2389	drm_i915_private_t *dev_priv = dev->dev_private;
		2390	int ret, i;
		2391
		2392	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2393	return 0;
		2394
		2395	if (atomic_read(&dev_priv->mm.wedged)) {
		2396	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		2397	atomic_set(&dev_priv->mm.wedged, 0);
		2398	}
		2399
		2400	mutex_lock(&dev->struct_mutex);
		2401	dev_priv->mm.suspended = 0;
		2402
		2403	ret = i915_gem_init_ringbuffer(dev);
		2404	if (ret != 0) {
		2405	mutex_unlock(&dev->struct_mutex);
		2406	return ret;
		2407	}
		2408
		2409	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		2410	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
		2411	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
		2412	for (i = 0; i < I915_NUM_RINGS; i++) {
		2413	BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
		2414	BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
		2415	}
		2416	mutex_unlock(&dev->struct_mutex);
		2417
		2418	ret = drm_irq_install(dev);
		2419	if (ret)
		2420	goto cleanup_ringbuffer;
		2421
		2422	return 0;
		2423
		2424	cleanup_ringbuffer:
		2425	mutex_lock(&dev->struct_mutex);
		2426	i915_gem_cleanup_ringbuffer(dev);
		2427	dev_priv->mm.suspended = 1;
		2428	mutex_unlock(&dev->struct_mutex);
		2429
		2430	return ret;
		2431	}
		2432
		2433	int
		2434	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		2435	struct drm_file *file_priv)
		2436	{
		2437	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2438	return 0;
		2439
		2440	drm_irq_uninstall(dev);
		2441	return i915_gem_idle(dev);
		2442	}
		2443
		2444	void
		2445	i915_gem_lastclose(struct drm_device *dev)
		2446	{
		2447	int ret;
		2448
		2449	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2450	return;
		2451
		2452	ret = i915_gem_idle(dev);
		2453	if (ret)
		2454	DRM_ERROR("failed to idle hardware: %d\n", ret);
		2455	}
		2456	#endif
		2457
		2458	static void
2326	Serge	2459	init_ring_lists(struct intel_ring_buffer *ring)
		2460	{
		2461	INIT_LIST_HEAD(&ring->active_list);
		2462	INIT_LIST_HEAD(&ring->request_list);
		2463	INIT_LIST_HEAD(&ring->gpu_write_list);
		2464	}
		2465
		2466	void
		2467	i915_gem_load(struct drm_device *dev)
		2468	{
		2469	int i;
		2470	drm_i915_private_t *dev_priv = dev->dev_private;
		2471
		2472	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		2473	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
		2474	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
		2475	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
		2476	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		2477	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
		2478	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
		2479	for (i = 0; i < I915_NUM_RINGS; i++)
		2480	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	2481	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	2482	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2360	Serge	2483	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		2484	i915_gem_retire_work_handler);
2326	Serge	2485
		2486	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		2487	if (IS_GEN3(dev)) {
		2488	u32 tmp = I915_READ(MI_ARB_STATE);
		2489	if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
		2490	/* arb state is a masked write, so set bit + bit in mask */
		2491	tmp = MI_ARB_C3_LP_WRITE_ENABLE \| (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
		2492	I915_WRITE(MI_ARB_STATE, tmp);
		2493	}
		2494	}
		2495
		2496	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		2497
		2498	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		2499	dev_priv->num_fence_regs = 16;
		2500	else
		2501	dev_priv->num_fence_regs = 8;
		2502
		2503	/* Initialize fence registers to zero */
		2504	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2505	i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
		2506	}
		2507
		2508	i915_gem_detect_bit_6_swizzle(dev);
		2509
		2510	dev_priv->mm.interruptible = true;
		2511
		2512	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		2513	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		2514	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		2515	}
		2516
		2517

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 3482 – Rev 2360