WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
		28	#include "drmP.h"
		29	#include "drm.h"
2330	Serge	30	#include "i915_drm.h"
2326	Serge	31	#include "i915_drv.h"
2351	Serge	32	#include "i915_trace.h"
2326	Serge	33	#include "intel_drv.h"
		34	//#include
2330	Serge	35	#include
2326	Serge	36	//#include
		37	#include
		38
2344	Serge	39	extern int x86_clflush_size;
2332	Serge	40
2344	Serge	41	#undef mb
		42	#undef rmb
		43	#undef wmb
		44	#define mb() asm volatile("mfence")
		45	#define rmb() asm volatile ("lfence")
		46	#define wmb() asm volatile ("sfence")
		47
		48	static inline void clflush(volatile void *__p)
		49	{
		50	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		51	}
		52
2332	Serge	53	#define MAX_ERRNO 4095
		54
		55	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		56
		57	static inline long IS_ERR(const void *ptr)
		58	{
		59	return IS_ERR_VALUE((unsigned long)ptr);
		60	}
		61
		62	static inline void *ERR_PTR(long error)
		63	{
		64	return (void *) error;
		65	}
		66
		67	static inline long PTR_ERR(const void *ptr)
		68	{
		69	return (long) ptr;
		70	}
		71
2344	Serge	72	void
		73	drm_gem_object_free(struct kref *kref)
		74	{
		75	struct drm_gem_object obj = (struct drm_gem_object ) kref;
		76	struct drm_device *dev = obj->dev;
2332	Serge	77
2344	Serge	78	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		79
		80	i915_gem_free_object(obj);
		81	}
		82
2332	Serge	83	/**
		84	* Initialize an already allocated GEM object of the specified size with
		85	* shmfs backing store.
		86	*/
		87	int drm_gem_object_init(struct drm_device *dev,
		88	struct drm_gem_object *obj, size_t size)
		89	{
		90	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		91
		92	obj->dev = dev;
2344	Serge	93	kref_init(&obj->refcount);
2332	Serge	94	atomic_set(&obj->handle_count, 0);
		95	obj->size = size;
		96
		97	return 0;
		98	}
		99
2344	Serge	100	void
		101	drm_gem_object_release(struct drm_gem_object *obj)
		102	{ }
2332	Serge	103
		104
2326	Serge	105	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		106	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		107	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		108	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		109
2332	Serge	110	static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
		111	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		112	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		113	static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
		114	bool write);
		115	static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
		116	uint64_t offset,
		117	uint64_t size);
		118	static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
		119	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		120	unsigned alignment,
		121	bool map_and_fenceable);
		122	static void i915_gem_clear_fence_reg(struct drm_device *dev,
		123	struct drm_i915_fence_reg *reg);
		124	static int i915_gem_phys_pwrite(struct drm_device *dev,
		125	struct drm_i915_gem_object *obj,
		126	struct drm_i915_gem_pwrite *args,
		127	struct drm_file *file);
		128	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
2326	Serge	129
2352	Serge	130	//static int i915_gem_inactive_shrink(struct shrinker *shrinker,
		131	// struct shrink_control *sc);
2332	Serge	132
		133	/* some bookkeeping */
		134	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		135	size_t size)
		136	{
		137	dev_priv->mm.object_count++;
		138	dev_priv->mm.object_memory += size;
		139	}
		140
		141	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		142	size_t size)
		143	{
		144	dev_priv->mm.object_count--;
		145	dev_priv->mm.object_memory -= size;
		146	}
		147
		148	#if 0
		149
		150	static int
		151	i915_gem_wait_for_error(struct drm_device *dev)
		152	{
		153	struct drm_i915_private *dev_priv = dev->dev_private;
		154	struct completion *x = &dev_priv->error_completion;
		155	unsigned long flags;
		156	int ret;
		157
		158	if (!atomic_read(&dev_priv->mm.wedged))
		159	return 0;
		160
		161	ret = wait_for_completion_interruptible(x);
		162	if (ret)
		163	return ret;
		164
		165	if (atomic_read(&dev_priv->mm.wedged)) {
		166	/* GPU is hung, bump the completion count to account for
		167	* the token we just consumed so that we never hit zero and
		168	* end up waiting upon a subsequent completion event that
		169	* will never happen.
		170	*/
		171	spin_lock_irqsave(&x->wait.lock, flags);
		172	x->done++;
		173	spin_unlock_irqrestore(&x->wait.lock, flags);
		174	}
		175	return 0;
		176	}
		177
		178	int i915_mutex_lock_interruptible(struct drm_device *dev)
		179	{
		180	int ret;
		181
		182	ret = i915_gem_wait_for_error(dev);
		183	if (ret)
		184	return ret;
		185
		186	ret = mutex_lock_interruptible(&dev->struct_mutex);
		187	if (ret)
		188	return ret;
		189
		190	WARN_ON(i915_verify_lists(dev));
		191	return 0;
		192	}
2352	Serge	193	#endif
2332	Serge	194
		195	static inline bool
		196	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		197	{
		198	return obj->gtt_space && !obj->active && obj->pin_count == 0;
		199	}
		200
		201	void i915_gem_do_init(struct drm_device *dev,
		202	unsigned long start,
		203	unsigned long mappable_end,
		204	unsigned long end)
		205	{
		206	drm_i915_private_t *dev_priv = dev->dev_private;
		207
		208	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
		209
		210	dev_priv->mm.gtt_start = start;
		211	dev_priv->mm.gtt_mappable_end = mappable_end;
		212	dev_priv->mm.gtt_end = end;
		213	dev_priv->mm.gtt_total = end - start;
		214	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
		215
		216	/* Take over this portion of the GTT */
		217	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
		218	}
		219
		220	#if 0
		221
		222	int
		223	i915_gem_init_ioctl(struct drm_device dev, void data,
		224	struct drm_file *file)
		225	{
		226	struct drm_i915_gem_init *args = data;
		227
		228	if (args->gtt_start >= args->gtt_end \|\|
		229	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		230	return -EINVAL;
		231
		232	mutex_lock(&dev->struct_mutex);
		233	i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
		234	mutex_unlock(&dev->struct_mutex);
		235
		236	return 0;
		237	}
2351	Serge	238	#endif
2332	Serge	239
		240	int
		241	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		242	struct drm_file *file)
		243	{
		244	struct drm_i915_private *dev_priv = dev->dev_private;
		245	struct drm_i915_gem_get_aperture *args = data;
		246	struct drm_i915_gem_object *obj;
		247	size_t pinned;
		248
		249
		250	pinned = 0;
		251	mutex_lock(&dev->struct_mutex);
		252	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
		253	pinned += obj->gtt_space->size;
		254	mutex_unlock(&dev->struct_mutex);
		255
		256	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	257	args->aper_available_size = args->aper_size - pinned;
2332	Serge	258
		259	return 0;
		260	}
		261
2351	Serge	262	#if 0
		263
		264	int i915_gem_create(struct drm_file *file,
2332	Serge	265	struct drm_device *dev,
		266	uint64_t size,
		267	uint32_t *handle_p)
		268	{
		269	struct drm_i915_gem_object *obj;
		270	int ret;
		271	u32 handle;
		272
		273	size = roundup(size, PAGE_SIZE);
2342	Serge	274	if (size == 0)
		275	return -EINVAL;
2332	Serge	276
		277	/* Allocate the new object */
		278	obj = i915_gem_alloc_object(dev, size);
		279	if (obj == NULL)
		280	return -ENOMEM;
		281
		282	ret = drm_gem_handle_create(file, &obj->base, &handle);
		283	if (ret) {
		284	drm_gem_object_release(&obj->base);
		285	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		286	kfree(obj);
		287	return ret;
		288	}
		289
		290	/* drop reference from allocate - handle holds it now */
		291	drm_gem_object_unreference(&obj->base);
2351	Serge	292	trace_i915_gem_object_create(obj);
2332	Serge	293
		294	*handle_p = handle;
		295	return 0;
		296	}
		297
		298	int
		299	i915_gem_dumb_create(struct drm_file *file,
		300	struct drm_device *dev,
		301	struct drm_mode_create_dumb *args)
		302	{
		303	/* have to work out size/pitch and return them */
		304	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		305	args->size = args->pitch * args->height;
		306	return i915_gem_create(file, dev,
		307	args->size, &args->handle);
		308	}
		309
		310	int i915_gem_dumb_destroy(struct drm_file *file,
		311	struct drm_device *dev,
		312	uint32_t handle)
		313	{
		314	return drm_gem_handle_delete(file, handle);
		315	}
		316
2326	Serge	317	/**
2332	Serge	318	* Creates a new mm object and returns a handle to it.
		319	*/
		320	int
		321	i915_gem_create_ioctl(struct drm_device dev, void data,
		322	struct drm_file *file)
		323	{
		324	struct drm_i915_gem_create *args = data;
		325	return i915_gem_create(file, dev,
		326	args->size, &args->handle);
		327	}
		328
		329	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		330	{
		331	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		332
		333	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		334	obj->tiling_mode != I915_TILING_NONE;
		335	}
		336
		337	static inline void
		338	slow_shmem_copy(struct page *dst_page,
		339	int dst_offset,
		340	struct page *src_page,
		341	int src_offset,
		342	int length)
		343	{
		344	char dst_vaddr, src_vaddr;
		345
		346	dst_vaddr = kmap(dst_page);
		347	src_vaddr = kmap(src_page);
		348
		349	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
		350
		351	kunmap(src_page);
		352	kunmap(dst_page);
		353	}
		354
		355	static inline void
		356	slow_shmem_bit17_copy(struct page *gpu_page,
		357	int gpu_offset,
		358	struct page *cpu_page,
		359	int cpu_offset,
		360	int length,
		361	int is_read)
		362	{
		363	char gpu_vaddr, cpu_vaddr;
		364
		365	/* Use the unswizzled path if this page isn't affected. */
		366	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
		367	if (is_read)
		368	return slow_shmem_copy(cpu_page, cpu_offset,
		369	gpu_page, gpu_offset, length);
		370	else
		371	return slow_shmem_copy(gpu_page, gpu_offset,
		372	cpu_page, cpu_offset, length);
		373	}
		374
		375	gpu_vaddr = kmap(gpu_page);
		376	cpu_vaddr = kmap(cpu_page);
		377
		378	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
		379	* XORing with the other bits (A9 for Y, A9 and A10 for X)
		380	*/
		381	while (length > 0) {
		382	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		383	int this_length = min(cacheline_end - gpu_offset, length);
		384	int swizzled_gpu_offset = gpu_offset ^ 64;
		385
		386	if (is_read) {
		387	memcpy(cpu_vaddr + cpu_offset,
		388	gpu_vaddr + swizzled_gpu_offset,
		389	this_length);
		390	} else {
		391	memcpy(gpu_vaddr + swizzled_gpu_offset,
		392	cpu_vaddr + cpu_offset,
		393	this_length);
		394	}
		395	cpu_offset += this_length;
		396	gpu_offset += this_length;
		397	length -= this_length;
		398	}
		399
		400	kunmap(cpu_page);
		401	kunmap(gpu_page);
		402	}
		403
		404	/**
		405	* This is the fast shmem pread path, which attempts to copy_from_user directly
		406	* from the backing pages of the object to the user's address space. On a
		407	* fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
		408	*/
		409	static int
		410	i915_gem_shmem_pread_fast(struct drm_device *dev,
		411	struct drm_i915_gem_object *obj,
		412	struct drm_i915_gem_pread *args,
		413	struct drm_file *file)
		414	{
		415	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		416	ssize_t remain;
		417	loff_t offset;
		418	char __user *user_data;
		419	int page_offset, page_length;
		420
		421	user_data = (char __user *) (uintptr_t) args->data_ptr;
		422	remain = args->size;
		423
		424	offset = args->offset;
		425
		426	while (remain > 0) {
		427	struct page *page;
		428	char *vaddr;
		429	int ret;
		430
		431	/* Operation in this page
		432	*
		433	* page_offset = offset within page
		434	* page_length = bytes to copy for this page
		435	*/
		436	page_offset = offset_in_page(offset);
		437	page_length = remain;
		438	if ((page_offset + remain) > PAGE_SIZE)
		439	page_length = PAGE_SIZE - page_offset;
		440
		441	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		442	if (IS_ERR(page))
		443	return PTR_ERR(page);
		444
		445	vaddr = kmap_atomic(page);
		446	ret = __copy_to_user_inatomic(user_data,
		447	vaddr + page_offset,
		448	page_length);
		449	kunmap_atomic(vaddr);
		450
		451	mark_page_accessed(page);
		452	page_cache_release(page);
		453	if (ret)
		454	return -EFAULT;
		455
		456	remain -= page_length;
		457	user_data += page_length;
		458	offset += page_length;
		459	}
		460
		461	return 0;
		462	}
		463
		464	/**
		465	* This is the fallback shmem pread path, which allocates temporary storage
		466	* in kernel space to copy_to_user into outside of the struct_mutex, so we
		467	* can copy out of the object's backing pages while holding the struct mutex
		468	* and not take page faults.
		469	*/
		470	static int
		471	i915_gem_shmem_pread_slow(struct drm_device *dev,
		472	struct drm_i915_gem_object *obj,
		473	struct drm_i915_gem_pread *args,
		474	struct drm_file *file)
		475	{
		476	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		477	struct mm_struct *mm = current->mm;
		478	struct page **user_pages;
		479	ssize_t remain;
		480	loff_t offset, pinned_pages, i;
		481	loff_t first_data_page, last_data_page, num_pages;
		482	int shmem_page_offset;
		483	int data_page_index, data_page_offset;
		484	int page_length;
		485	int ret;
		486	uint64_t data_ptr = args->data_ptr;
		487	int do_bit17_swizzling;
		488
		489	remain = args->size;
		490
		491	/* Pin the user pages containing the data. We can't fault while
		492	* holding the struct mutex, yet we want to hold it while
		493	* dereferencing the user data.
		494	*/
		495	first_data_page = data_ptr / PAGE_SIZE;
		496	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
		497	num_pages = last_data_page - first_data_page + 1;
		498
		499	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
		500	if (user_pages == NULL)
		501	return -ENOMEM;
		502
		503	mutex_unlock(&dev->struct_mutex);
		504	down_read(&mm->mmap_sem);
		505	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
		506	num_pages, 1, 0, user_pages, NULL);
		507	up_read(&mm->mmap_sem);
		508	mutex_lock(&dev->struct_mutex);
		509	if (pinned_pages < num_pages) {
		510	ret = -EFAULT;
		511	goto out;
		512	}
		513
		514	ret = i915_gem_object_set_cpu_read_domain_range(obj,
		515	args->offset,
		516	args->size);
		517	if (ret)
		518	goto out;
		519
		520	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		521
		522	offset = args->offset;
		523
		524	while (remain > 0) {
		525	struct page *page;
		526
		527	/* Operation in this page
		528	*
		529	* shmem_page_offset = offset within page in shmem file
		530	* data_page_index = page number in get_user_pages return
		531	* data_page_offset = offset with data_page_index page.
		532	* page_length = bytes to copy for this page
		533	*/
		534	shmem_page_offset = offset_in_page(offset);
		535	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		536	data_page_offset = offset_in_page(data_ptr);
		537
		538	page_length = remain;
		539	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		540	page_length = PAGE_SIZE - shmem_page_offset;
		541	if ((data_page_offset + page_length) > PAGE_SIZE)
		542	page_length = PAGE_SIZE - data_page_offset;
		543
		544	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		545	if (IS_ERR(page)) {
		546	ret = PTR_ERR(page);
		547	goto out;
		548	}
		549
		550	if (do_bit17_swizzling) {
		551	slow_shmem_bit17_copy(page,
		552	shmem_page_offset,
		553	user_pages[data_page_index],
		554	data_page_offset,
		555	page_length,
		556	1);
		557	} else {
		558	slow_shmem_copy(user_pages[data_page_index],
		559	data_page_offset,
		560	page,
		561	shmem_page_offset,
		562	page_length);
		563	}
		564
		565	mark_page_accessed(page);
		566	page_cache_release(page);
		567
		568	remain -= page_length;
		569	data_ptr += page_length;
		570	offset += page_length;
		571	}
		572
		573	out:
		574	for (i = 0; i < pinned_pages; i++) {
		575	SetPageDirty(user_pages[i]);
		576	mark_page_accessed(user_pages[i]);
		577	page_cache_release(user_pages[i]);
		578	}
		579	drm_free_large(user_pages);
		580
		581	return ret;
		582	}
		583	#endif
		584
		585
		586
		587
		588
		589
		590
		591
		592
		593
		594
		595
		596
		597
		598
		599
		600
		601
		602
		603
		604
		605
		606
		607
		608
		609
		610
		611
		612
		613
		614
		615
		616
		617
		618
		619
		620
		621
		622
		623
		624
		625
		626
		627
		628
		629
		630
		631
		632
		633
		634
		635
		636
		637
		638
		639
		640
		641
		642
		643
		644
		645
		646
		647
		648
		649
		650
		651
		652
		653
		654
		655
		656
		657	static uint32_t
		658	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		659	{
		660	uint32_t gtt_size;
		661
		662	if (INTEL_INFO(dev)->gen >= 4 \|\|
		663	tiling_mode == I915_TILING_NONE)
		664	return size;
		665
		666	/* Previous chips need a power-of-two fence region when tiling */
		667	if (INTEL_INFO(dev)->gen == 3)
		668	gtt_size = 1024*1024;
		669	else
		670	gtt_size = 512*1024;
		671
		672	while (gtt_size < size)
		673	gtt_size <<= 1;
		674
		675	return gtt_size;
		676	}
		677
		678	/**
		679	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		680	* @obj: object to check
		681	*
		682	* Return the required GTT alignment for an object, taking into account
		683	* potential fence register mapping.
		684	*/
		685	static uint32_t
		686	i915_gem_get_gtt_alignment(struct drm_device *dev,
		687	uint32_t size,
		688	int tiling_mode)
		689	{
		690	/*
		691	* Minimum alignment is 4k (GTT page size), but might be greater
		692	* if a fence register is needed for the object.
		693	*/
		694	if (INTEL_INFO(dev)->gen >= 4 \|\|
		695	tiling_mode == I915_TILING_NONE)
		696	return 4096;
		697
		698	/*
		699	* Previous chips need to be aligned to the size of the smallest
		700	* fence register that can contain the object.
		701	*/
		702	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		703	}
		704
		705	/**
		706	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		707	* unfenced object
		708	* @dev: the device
		709	* @size: size of the object
		710	* @tiling_mode: tiling mode of the object
		711	*
		712	* Return the required GTT alignment for an object, only taking into account
		713	* unfenced tiled surface requirements.
		714	*/
		715	uint32_t
		716	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		717	uint32_t size,
		718	int tiling_mode)
		719	{
		720	/*
		721	* Minimum alignment is 4k (GTT page size) for sane hw.
		722	*/
		723	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		724	tiling_mode == I915_TILING_NONE)
		725	return 4096;
		726
		727	/* Previous hardware however needs to be aligned to a power-of-two
		728	* tile height. The simplest method for determining this is to reuse
		729	* the power-of-tile object size.
		730	*/
		731	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		732	}
		733
		734
		735
		736
		737
		738
		739
		740
		741
		742
		743
		744
		745
		746
		747
		748	static int
		749	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
		750	gfp_t gfpmask)
		751	{
		752	int page_count, i;
		753	struct page *page;
		754
		755	/* Get the list of pages out of our struct file. They'll be pinned
		756	* at this point until we release them.
		757	*/
		758	page_count = obj->base.size / PAGE_SIZE;
		759	BUG_ON(obj->pages != NULL);
		760	obj->pages = malloc(page_count * sizeof(struct page *));
		761	if (obj->pages == NULL)
		762	return -ENOMEM;
		763
		764
		765	for (i = 0; i < page_count; i++) {
		766	page = (struct page*)AllocPage(); // oh-oh
		767	if (IS_ERR(page))
		768	goto err_pages;
		769
		770	obj->pages[i] = page;
		771	}
		772
		773	// if (obj->tiling_mode != I915_TILING_NONE)
		774	// i915_gem_object_do_bit_17_swizzle(obj);
		775
		776
2340	Serge	777
2332	Serge	778	return 0;
		779
		780	err_pages:
2344	Serge	781	while (i--)
2352	Serge	782	FreePage((addr_t)obj->pages[i]);
2332	Serge	783
		784	free(obj->pages);
		785	obj->pages = NULL;
		786	return PTR_ERR(page);
		787	}
		788
		789	static void
		790	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		791	{
		792	int page_count = obj->base.size / PAGE_SIZE;
		793	int i;
		794
		795	BUG_ON(obj->madv == __I915_MADV_PURGED);
		796
		797	// if (obj->tiling_mode != I915_TILING_NONE)
		798	// i915_gem_object_save_bit_17_swizzle(obj);
		799
		800	if (obj->madv == I915_MADV_DONTNEED)
		801	obj->dirty = 0;
2344	Serge	802
2332	Serge	803	for (i = 0; i < page_count; i++) {
2352	Serge	804	FreePage((addr_t)obj->pages[i]);
2332	Serge	805	}
		806	obj->dirty = 0;
		807
		808	free(obj->pages);
		809	obj->pages = NULL;
		810	}
		811
		812	void
		813	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
		814	struct intel_ring_buffer *ring,
		815	u32 seqno)
		816	{
		817	struct drm_device *dev = obj->base.dev;
		818	struct drm_i915_private *dev_priv = dev->dev_private;
		819
		820	BUG_ON(ring == NULL);
		821	obj->ring = ring;
		822
		823	/* Add a reference if we're newly entering the active list. */
		824	if (!obj->active) {
2344	Serge	825	drm_gem_object_reference(&obj->base);
2332	Serge	826	obj->active = 1;
		827	}
		828
		829	/* Move from whatever list we were on to the tail of execution. */
		830	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		831	list_move_tail(&obj->ring_list, &ring->active_list);
		832
		833	obj->last_rendering_seqno = seqno;
		834	if (obj->fenced_gpu_access) {
		835	struct drm_i915_fence_reg *reg;
		836
		837	BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
		838
		839	obj->last_fenced_seqno = seqno;
		840	obj->last_fenced_ring = ring;
		841
		842	reg = &dev_priv->fence_regs[obj->fence_reg];
		843	list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
		844	}
		845	}
		846
2344	Serge	847	static void
		848	i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
		849	{
		850	list_del_init(&obj->ring_list);
		851	obj->last_rendering_seqno = 0;
		852	}
2332	Serge	853
2344	Serge	854	static void
		855	i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
		856	{
		857	struct drm_device *dev = obj->base.dev;
		858	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	859
2344	Serge	860	BUG_ON(!obj->active);
		861	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
2332	Serge	862
2344	Serge	863	i915_gem_object_move_off_active(obj);
		864	}
		865
2352	Serge	866	static void
		867	i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
		868	{
		869	struct drm_device *dev = obj->base.dev;
		870	struct drm_i915_private *dev_priv = dev->dev_private;
2344	Serge	871
2352	Serge	872	if (obj->pin_count != 0)
		873	list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
		874	else
		875	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2344	Serge	876
2352	Serge	877	BUG_ON(!list_empty(&obj->gpu_write_list));
		878	BUG_ON(!obj->active);
		879	obj->ring = NULL;
2344	Serge	880
2352	Serge	881	i915_gem_object_move_off_active(obj);
		882	obj->fenced_gpu_access = false;
2344	Serge	883
2352	Serge	884	obj->active = 0;
		885	obj->pending_gpu_write = false;
		886	drm_gem_object_unreference(&obj->base);
		887
		888	WARN_ON(i915_verify_lists(dev));
		889	}
		890
2344	Serge	891	/* Immediately discard the backing storage */
2332	Serge	892	static void
2344	Serge	893	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		894	{
		895	struct inode *inode;
		896
		897	/* Our goal here is to return as much of the memory as
		898	* is possible back to the system as we are called from OOM.
		899	* To do this we must instruct the shmfs to drop all of its
		900	* backing pages, now.
		901	*/
		902
		903	obj->madv = __I915_MADV_PURGED;
		904	}
		905
		906	static inline int
		907	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		908	{
		909	return obj->madv == I915_MADV_DONTNEED;
		910	}
		911
		912	static void
2332	Serge	913	i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
		914	uint32_t flush_domains)
		915	{
		916	struct drm_i915_gem_object obj, next;
		917
		918	list_for_each_entry_safe(obj, next,
		919	&ring->gpu_write_list,
		920	gpu_write_list) {
		921	if (obj->base.write_domain & flush_domains) {
		922	uint32_t old_write_domain = obj->base.write_domain;
		923
		924	obj->base.write_domain = 0;
		925	list_del_init(&obj->gpu_write_list);
		926	i915_gem_object_move_to_active(obj, ring,
		927	i915_gem_next_request_seqno(ring));
		928
2352	Serge	929	trace_i915_gem_object_change_domain(obj,
		930	obj->base.read_domains,
		931	old_write_domain);
2332	Serge	932	}
		933	}
		934	}
		935
2352	Serge	936	int
		937	i915_add_request(struct intel_ring_buffer *ring,
		938	struct drm_file *file,
		939	struct drm_i915_gem_request *request)
		940	{
		941	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		942	uint32_t seqno;
		943	int was_empty;
		944	int ret;
2332	Serge	945
2352	Serge	946	BUG_ON(request == NULL);
2332	Serge	947
2352	Serge	948	ret = ring->add_request(ring, &seqno);
		949	if (ret)
		950	return ret;
2332	Serge	951
2352	Serge	952	trace_i915_gem_request_add(ring, seqno);
2332	Serge	953
2352	Serge	954	request->seqno = seqno;
		955	request->ring = ring;
		956	request->emitted_jiffies = jiffies;
		957	was_empty = list_empty(&ring->request_list);
		958	list_add_tail(&request->list, &ring->request_list);
2332	Serge	959
		960
2352	Serge	961	ring->outstanding_lazy_request = false;
2332	Serge	962
2352	Serge	963	// if (!dev_priv->mm.suspended) {
		964	// if (i915_enable_hangcheck) {
		965	// mod_timer(&dev_priv->hangcheck_timer,
		966	// jiffies +
		967	// msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
		968	// }
		969	// if (was_empty)
		970	// queue_delayed_work(dev_priv->wq,
		971	// &dev_priv->mm.retire_work, HZ);
		972	// }
		973	return 0;
		974	}
2332	Serge	975
		976
		977
		978
		979
		980
		981
		982
		983
		984
		985
		986
2352	Serge	987	/**
		988	* This function clears the request list as sequence numbers are passed.
		989	*/
		990	static void
		991	i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
		992	{
		993	uint32_t seqno;
		994	int i;
2332	Serge	995
2352	Serge	996	if (list_empty(&ring->request_list))
		997	return;
2332	Serge	998
2352	Serge	999	WARN_ON(i915_verify_lists(ring->dev));
2332	Serge	1000
2352	Serge	1001	seqno = ring->get_seqno(ring);
2332	Serge	1002
2352	Serge	1003	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
		1004	if (seqno >= ring->sync_seqno[i])
		1005	ring->sync_seqno[i] = 0;
2332	Serge	1006
2352	Serge	1007	while (!list_empty(&ring->request_list)) {
		1008	struct drm_i915_gem_request *request;
2332	Serge	1009
2352	Serge	1010	request = list_first_entry(&ring->request_list,
		1011	struct drm_i915_gem_request,
		1012	list);
2332	Serge	1013
2352	Serge	1014	if (!i915_seqno_passed(seqno, request->seqno))
		1015	break;
2332	Serge	1016
2352	Serge	1017	trace_i915_gem_request_retire(ring, request->seqno);
2332	Serge	1018
2352	Serge	1019	list_del(&request->list);
		1020	kfree(request);
		1021	}
2332	Serge	1022
2352	Serge	1023	/* Move any buffers on the active list that are no longer referenced
		1024	* by the ringbuffer to the flushing/inactive lists as appropriate.
		1025	*/
		1026	while (!list_empty(&ring->active_list)) {
		1027	struct drm_i915_gem_object *obj;
2332	Serge	1028
2352	Serge	1029	obj = list_first_entry(&ring->active_list,
		1030	struct drm_i915_gem_object,
		1031	ring_list);
2332	Serge	1032
2352	Serge	1033	if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
		1034	break;
2332	Serge	1035
2352	Serge	1036	if (obj->base.write_domain != 0)
		1037	i915_gem_object_move_to_flushing(obj);
		1038	else
		1039	i915_gem_object_move_to_inactive(obj);
		1040	}
2332	Serge	1041
2352	Serge	1042	if (unlikely(ring->trace_irq_seqno &&
		1043	i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
		1044	ring->irq_put(ring);
		1045	ring->trace_irq_seqno = 0;
		1046	}
2332	Serge	1047
2352	Serge	1048	WARN_ON(i915_verify_lists(ring->dev));
		1049	}
2332	Serge	1050
2352	Serge	1051	void
		1052	i915_gem_retire_requests(struct drm_device *dev)
		1053	{
		1054	drm_i915_private_t *dev_priv = dev->dev_private;
		1055	int i;
2332	Serge	1056
2352	Serge	1057	if (!list_empty(&dev_priv->mm.deferred_free_list)) {
		1058	struct drm_i915_gem_object obj, next;
2332	Serge	1059
2352	Serge	1060	/* We must be careful that during unbind() we do not
		1061	* accidentally infinitely recurse into retire requests.
		1062	* Currently:
		1063	* retire -> free -> unbind -> wait -> retire_ring
		1064	*/
		1065	list_for_each_entry_safe(obj, next,
		1066	&dev_priv->mm.deferred_free_list,
		1067	mm_list)
		1068	i915_gem_free_object_tail(obj);
		1069	}
2332	Serge	1070
2352	Serge	1071	for (i = 0; i < I915_NUM_RINGS; i++)
		1072	i915_gem_retire_requests_ring(&dev_priv->ring[i]);
		1073	}
		1074
		1075
		1076
		1077
		1078
		1079
		1080
		1081
		1082
		1083
2344	Serge	1084	/**
2352	Serge	1085	* Waits for a sequence number to be signaled, and cleans up the
		1086	* request and object lists appropriately for that event.
		1087	*/
		1088	int
		1089	i915_wait_request(struct intel_ring_buffer *ring,
		1090	uint32_t seqno)
		1091	{
		1092	drm_i915_private_t *dev_priv = ring->dev->dev_private;
		1093	u32 ier;
		1094	int ret = 0;
		1095
		1096	BUG_ON(seqno == 0);
		1097
		1098	// if (atomic_read(&dev_priv->mm.wedged)) {
		1099	// struct completion *x = &dev_priv->error_completion;
		1100	// bool recovery_complete;
		1101	// unsigned long flags;
		1102
		1103	/* Give the error handler a chance to run. */
		1104	// spin_lock_irqsave(&x->wait.lock, flags);
		1105	// recovery_complete = x->done > 0;
		1106	// spin_unlock_irqrestore(&x->wait.lock, flags);
		1107	//
		1108	// return recovery_complete ? -EIO : -EAGAIN;
		1109	// }
		1110
		1111	if (seqno == ring->outstanding_lazy_request) {
		1112	struct drm_i915_gem_request *request;
		1113
		1114	request = kzalloc(sizeof(*request), GFP_KERNEL);
		1115	if (request == NULL)
		1116	return -ENOMEM;
		1117
		1118	ret = i915_add_request(ring, NULL, request);
		1119	if (ret) {
		1120	kfree(request);
		1121	return ret;
		1122	}
		1123
		1124	seqno = request->seqno;
		1125	}
		1126
		1127	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
		1128	if (HAS_PCH_SPLIT(ring->dev))
		1129	ier = I915_READ(DEIER) \| I915_READ(GTIER);
		1130	else
		1131	ier = I915_READ(IER);
		1132	if (!ier) {
		1133	DRM_ERROR("something (likely vbetool) disabled "
		1134	"interrupts, re-enabling\n");
		1135	// ring->dev->driver->irq_preinstall(ring->dev);
		1136	// ring->dev->driver->irq_postinstall(ring->dev);
		1137	}
		1138
		1139	trace_i915_gem_request_wait_begin(ring, seqno);
		1140
		1141	ring->waiting_seqno = seqno;
		1142	if (ring->irq_get(ring)) {
		1143	// printf("enter wait\n");
		1144	wait_event(ring->irq_queue,
		1145	i915_seqno_passed(ring->get_seqno(ring), seqno)
		1146	\|\| atomic_read(&dev_priv->mm.wedged));
		1147
		1148	ring->irq_put(ring);
		1149	} else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
		1150	seqno) \|\|
		1151	atomic_read(&dev_priv->mm.wedged), 3000))
		1152	ret = -EBUSY;
		1153	ring->waiting_seqno = 0;
		1154
		1155	trace_i915_gem_request_wait_end(ring, seqno);
		1156	}
		1157	if (atomic_read(&dev_priv->mm.wedged))
		1158	ret = -EAGAIN;
		1159
		1160	if (ret && ret != -ERESTARTSYS)
		1161	DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
		1162	__func__, ret, seqno, ring->get_seqno(ring),
		1163	dev_priv->next_seqno);
		1164
		1165	/* Directly dispatch request retiring. While we have the work queue
		1166	* to handle this, the waiter on a request often wants an associated
		1167	* buffer to have made it to the inactive list, and we would need
		1168	* a separate wait queue to handle that.
		1169	*/
		1170	if (ret == 0)
		1171	i915_gem_retire_requests_ring(ring);
		1172
		1173	return ret;
		1174	}
		1175
		1176	/**
2344	Serge	1177	* Ensures that all rendering to the object has completed and the object is
		1178	* safe to unbind from the GTT or access from the CPU.
		1179	*/
		1180	int
		1181	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
		1182	{
		1183	int ret;
2332	Serge	1184
2344	Serge	1185	/* This function only exists to support waiting for existing rendering,
		1186	* not for emitting required flushes.
		1187	*/
		1188	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2332	Serge	1189
2344	Serge	1190	/* If there is rendering queued on the buffer being evicted, wait for
		1191	* it.
		1192	*/
		1193	if (obj->active) {
2352	Serge	1194	ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
		1195	if (ret)
		1196	return ret;
2344	Serge	1197	}
2332	Serge	1198
2344	Serge	1199	return 0;
		1200	}
2332	Serge	1201
2344	Serge	1202	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		1203	{
		1204	u32 old_write_domain, old_read_domains;
2332	Serge	1205
2344	Serge	1206	/* Act a barrier for all accesses through the GTT */
		1207	mb();
2332	Serge	1208
2344	Serge	1209	/* Force a pagefault for domain tracking on next user access */
		1210	// i915_gem_release_mmap(obj);
2332	Serge	1211
2344	Serge	1212	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		1213	return;
2332	Serge	1214
2344	Serge	1215	old_read_domains = obj->base.read_domains;
		1216	old_write_domain = obj->base.write_domain;
2351	Serge	1217
2344	Serge	1218	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		1219	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	1220
2351	Serge	1221	trace_i915_gem_object_change_domain(obj,
		1222	old_read_domains,
		1223	old_write_domain);
2344	Serge	1224	}
2332	Serge	1225
2344	Serge	1226	/**
		1227	* Unbinds an object from the GTT aperture.
		1228	*/
		1229	int
		1230	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		1231	{
		1232	int ret = 0;
2332	Serge	1233
2344	Serge	1234	if (obj->gtt_space == NULL)
		1235	return 0;
2332	Serge	1236
2344	Serge	1237	if (obj->pin_count != 0) {
		1238	DRM_ERROR("Attempting to unbind pinned buffer\n");
		1239	return -EINVAL;
		1240	}
2332	Serge	1241
2344	Serge	1242	ret = i915_gem_object_finish_gpu(obj);
		1243	if (ret == -ERESTARTSYS)
		1244	return ret;
		1245	/* Continue on if we fail due to EIO, the GPU is hung so we
		1246	* should be safe and we need to cleanup or else we might
		1247	* cause memory corruption through use-after-free.
		1248	*/
2332	Serge	1249
2344	Serge	1250	i915_gem_object_finish_gtt(obj);
2332	Serge	1251
2344	Serge	1252	/* Move the object to the CPU domain to ensure that
		1253	* any possible CPU writes while it's not in the GTT
		1254	* are flushed when we go to remap it.
		1255	*/
		1256	if (ret == 0)
		1257	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
		1258	if (ret == -ERESTARTSYS)
		1259	return ret;
		1260	if (ret) {
		1261	/* In the event of a disaster, abandon all caches and
		1262	* hope for the best.
		1263	*/
		1264	i915_gem_clflush_object(obj);
		1265	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1266	}
2332	Serge	1267
2344	Serge	1268	/* release the fence reg _after_ flushing */
		1269	ret = i915_gem_object_put_fence(obj);
		1270	if (ret == -ERESTARTSYS)
		1271	return ret;
2332	Serge	1272
2351	Serge	1273	trace_i915_gem_object_unbind(obj);
2332	Serge	1274
2344	Serge	1275	i915_gem_gtt_unbind_object(obj);
		1276	i915_gem_object_put_pages_gtt(obj);
2332	Serge	1277
2344	Serge	1278	list_del_init(&obj->gtt_list);
		1279	list_del_init(&obj->mm_list);
		1280	/* Avoid an unnecessary call to unbind on rebind. */
		1281	obj->map_and_fenceable = true;
2332	Serge	1282
2344	Serge	1283	drm_mm_put_block(obj->gtt_space);
		1284	obj->gtt_space = NULL;
		1285	obj->gtt_offset = 0;
2332	Serge	1286
2344	Serge	1287	if (i915_gem_object_is_purgeable(obj))
		1288	i915_gem_object_truncate(obj);
2332	Serge	1289
2344	Serge	1290	return ret;
		1291	}
2332	Serge	1292
2344	Serge	1293	int
		1294	i915_gem_flush_ring(struct intel_ring_buffer *ring,
		1295	uint32_t invalidate_domains,
		1296	uint32_t flush_domains)
		1297	{
		1298	int ret;
2332	Serge	1299
2344	Serge	1300	if (((invalidate_domains \| flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
		1301	return 0;
2332	Serge	1302
2351	Serge	1303	trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2332	Serge	1304
2344	Serge	1305	ret = ring->flush(ring, invalidate_domains, flush_domains);
		1306	if (ret)
		1307	return ret;
2332	Serge	1308
2344	Serge	1309	if (flush_domains & I915_GEM_GPU_DOMAINS)
		1310	i915_gem_process_flushing_list(ring, flush_domains);
2332	Serge	1311
2344	Serge	1312	return 0;
		1313	}
2332	Serge	1314
2344	Serge	1315	static int i915_ring_idle(struct intel_ring_buffer *ring)
		1316	{
		1317	int ret;
2332	Serge	1318
2344	Serge	1319	if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
		1320	return 0;
2332	Serge	1321
2344	Serge	1322	if (!list_empty(&ring->gpu_write_list)) {
		1323	ret = i915_gem_flush_ring(ring,
		1324	I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
		1325	if (ret)
		1326	return ret;
		1327	}
2332	Serge	1328
2344	Serge	1329	return 0; //i915_wait_request(ring, i915_gem_next_request_seqno(ring));
		1330	}
2332	Serge	1331
2344	Serge	1332	int
		1333	i915_gpu_idle(struct drm_device *dev)
		1334	{
		1335	drm_i915_private_t *dev_priv = dev->dev_private;
		1336	int ret, i;
2332	Serge	1337
2344	Serge	1338	/* Flush everything onto the inactive list. */
		1339	for (i = 0; i < I915_NUM_RINGS; i++) {
		1340	ret = i915_ring_idle(&dev_priv->ring[i]);
		1341	if (ret)
		1342	return ret;
		1343	}
2332	Serge	1344
2344	Serge	1345	return 0;
		1346	}
2332	Serge	1347
		1348
		1349
		1350
		1351
		1352
		1353
		1354
		1355
		1356
		1357
		1358
		1359
		1360
		1361
		1362
		1363
		1364
2344	Serge	1365
		1366
		1367
		1368
		1369	static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332	Serge	1370	{
2344	Serge	1371	return i915_seqno_passed(ring->get_seqno(ring), seqno);
		1372	}
		1373
		1374	static int
		1375	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
		1376	struct intel_ring_buffer *pipelined)
		1377	{
2332	Serge	1378	int ret;
		1379
2344	Serge	1380	if (obj->fenced_gpu_access) {
		1381	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
		1382	ret = i915_gem_flush_ring(obj->last_fenced_ring,
		1383	0, obj->base.write_domain);
		1384	if (ret)
		1385	return ret;
		1386	}
2332	Serge	1387
2344	Serge	1388	obj->fenced_gpu_access = false;
		1389	}
		1390
		1391	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
		1392	if (!ring_passed_seqno(obj->last_fenced_ring,
		1393	obj->last_fenced_seqno)) {
2352	Serge	1394	ret = i915_wait_request(obj->last_fenced_ring,
		1395	obj->last_fenced_seqno);
		1396	if (ret)
		1397	return ret;
2344	Serge	1398	}
		1399
		1400	obj->last_fenced_seqno = 0;
		1401	obj->last_fenced_ring = NULL;
		1402	}
		1403
		1404	/* Ensure that all CPU reads are completed before installing a fence
		1405	* and all writes before removing the fence.
2332	Serge	1406	*/
2344	Serge	1407	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		1408	mb();
2332	Serge	1409
		1410	return 0;
		1411	}
		1412
		1413	int
2344	Serge	1414	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	1415	{
		1416	int ret;
		1417
2344	Serge	1418	// if (obj->tiling_mode)
		1419	// i915_gem_release_mmap(obj);
2332	Serge	1420
2344	Serge	1421	ret = i915_gem_object_flush_fence(obj, NULL);
2332	Serge	1422	if (ret)
		1423	return ret;
		1424
2344	Serge	1425	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		1426	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1427	i915_gem_clear_fence_reg(obj->base.dev,
		1428	&dev_priv->fence_regs[obj->fence_reg]);
2332	Serge	1429
2344	Serge	1430	obj->fence_reg = I915_FENCE_REG_NONE;
		1431	}
		1432
2332	Serge	1433	return 0;
		1434	}
		1435
		1436
		1437
		1438
		1439
		1440
		1441
		1442
		1443
		1444
		1445
		1446
		1447
		1448
		1449
		1450
		1451
		1452
		1453
		1454
2344	Serge	1455
		1456
		1457
		1458
		1459
		1460
		1461
		1462
		1463
		1464
		1465
2332	Serge	1466	/**
2326	Serge	1467	* i915_gem_clear_fence_reg - clear out fence register info
		1468	* @obj: object to clear
		1469	*
		1470	* Zeroes out the fence register itself and clears out the associated
		1471	* data structures in dev_priv and obj.
		1472	*/
		1473	static void
		1474	i915_gem_clear_fence_reg(struct drm_device *dev,
		1475	struct drm_i915_fence_reg *reg)
		1476	{
		1477	drm_i915_private_t *dev_priv = dev->dev_private;
		1478	uint32_t fence_reg = reg - dev_priv->fence_regs;
		1479
		1480	switch (INTEL_INFO(dev)->gen) {
		1481	case 7:
		1482	case 6:
		1483	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
		1484	break;
		1485	case 5:
		1486	case 4:
		1487	I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
		1488	break;
		1489	case 3:
		1490	if (fence_reg >= 8)
		1491	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
		1492	else
		1493	case 2:
		1494	fence_reg = FENCE_REG_830_0 + fence_reg * 4;
		1495
		1496	I915_WRITE(fence_reg, 0);
		1497	break;
		1498	}
		1499
		1500	list_del_init(®->lru_list);
		1501	reg->obj = NULL;
		1502	reg->setup_seqno = 0;
		1503	}
		1504
2332	Serge	1505	/**
		1506	* Finds free space in the GTT aperture and binds the object there.
		1507	*/
		1508	static int
		1509	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		1510	unsigned alignment,
		1511	bool map_and_fenceable)
		1512	{
		1513	struct drm_device *dev = obj->base.dev;
		1514	drm_i915_private_t *dev_priv = dev->dev_private;
		1515	struct drm_mm_node *free_space;
		1516	gfp_t gfpmask = 0; //__GFP_NORETRY \| __GFP_NOWARN;
		1517	u32 size, fence_size, fence_alignment, unfenced_alignment;
		1518	bool mappable, fenceable;
		1519	int ret;
2326	Serge	1520
2332	Serge	1521	if (obj->madv != I915_MADV_WILLNEED) {
		1522	DRM_ERROR("Attempting to bind a purgeable object\n");
		1523	return -EINVAL;
		1524	}
		1525
		1526	fence_size = i915_gem_get_gtt_size(dev,
		1527	obj->base.size,
		1528	obj->tiling_mode);
		1529	fence_alignment = i915_gem_get_gtt_alignment(dev,
		1530	obj->base.size,
		1531	obj->tiling_mode);
		1532	unfenced_alignment =
		1533	i915_gem_get_unfenced_gtt_alignment(dev,
		1534	obj->base.size,
		1535	obj->tiling_mode);
		1536
		1537	if (alignment == 0)
		1538	alignment = map_and_fenceable ? fence_alignment :
		1539	unfenced_alignment;
		1540	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		1541	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		1542	return -EINVAL;
		1543	}
		1544
		1545	size = map_and_fenceable ? fence_size : obj->base.size;
		1546
		1547	/* If the object is bigger than the entire aperture, reject it early
		1548	* before evicting everything in a vain attempt to find space.
		1549	*/
		1550	if (obj->base.size >
		1551	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		1552	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		1553	return -E2BIG;
		1554	}
		1555
		1556	search_free:
		1557	if (map_and_fenceable)
		1558	free_space =
		1559	drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
		1560	size, alignment, 0,
		1561	dev_priv->mm.gtt_mappable_end,
		1562	0);
		1563	else
		1564	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
		1565	size, alignment, 0);
		1566
		1567	if (free_space != NULL) {
		1568	if (map_and_fenceable)
		1569	obj->gtt_space =
		1570	drm_mm_get_block_range_generic(free_space,
		1571	size, alignment, 0,
		1572	dev_priv->mm.gtt_mappable_end,
		1573	0);
		1574	else
		1575	obj->gtt_space =
		1576	drm_mm_get_block(free_space, size, alignment);
		1577	}
		1578	if (obj->gtt_space == NULL) {
		1579	/* If the gtt is empty and we're still having trouble
		1580	* fitting our object in, we're out of memory.
		1581	*/
		1582	ret = 1; //i915_gem_evict_something(dev, size, alignment,
		1583	// map_and_fenceable);
		1584	if (ret)
		1585	return ret;
		1586
		1587	goto search_free;
		1588	}
		1589
		1590	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
		1591	if (ret) {
		1592	drm_mm_put_block(obj->gtt_space);
		1593	obj->gtt_space = NULL;
		1594	#if 0
		1595	if (ret == -ENOMEM) {
		1596	/* first try to reclaim some memory by clearing the GTT */
		1597	ret = i915_gem_evict_everything(dev, false);
		1598	if (ret) {
		1599	/* now try to shrink everyone else */
		1600	if (gfpmask) {
		1601	gfpmask = 0;
		1602	goto search_free;
		1603	}
		1604
		1605	return -ENOMEM;
		1606	}
		1607
		1608	goto search_free;
		1609	}
		1610	#endif
		1611	return ret;
		1612	}
		1613
		1614	ret = i915_gem_gtt_bind_object(obj);
		1615	if (ret) {
2344	Serge	1616	i915_gem_object_put_pages_gtt(obj);
2332	Serge	1617	drm_mm_put_block(obj->gtt_space);
		1618	obj->gtt_space = NULL;
		1619
		1620	// if (i915_gem_evict_everything(dev, false))
		1621	return ret;
		1622
		1623	// goto search_free;
		1624	}
		1625
		1626	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
		1627	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		1628
		1629	/* Assert that the object is not currently in any GPU domain. As it
		1630	* wasn't in the GTT, there shouldn't be any way it could have been in
		1631	* a GPU cache
		1632	*/
		1633	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1634	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1635
		1636	obj->gtt_offset = obj->gtt_space->start;
		1637
		1638	fenceable =
		1639	obj->gtt_space->size == fence_size &&
2342	Serge	1640	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2332	Serge	1641
		1642	mappable =
		1643	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		1644
		1645	obj->map_and_fenceable = mappable && fenceable;
		1646
2351	Serge	1647	trace_i915_gem_object_bind(obj, map_and_fenceable);
2332	Serge	1648	return 0;
		1649	}
		1650
		1651	void
		1652	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		1653	{
		1654	/* If we don't have a page list set up, then we're not pinned
		1655	* to GPU, and we can ignore the cache flush because it'll happen
		1656	* again at bind time.
		1657	*/
		1658	if (obj->pages == NULL)
		1659	return;
		1660
		1661	/* If the GPU is snooping the contents of the CPU cache,
		1662	* we do not need to manually clear the CPU cache lines. However,
		1663	* the caches are only snooped when the render cache is
		1664	* flushed/invalidated. As we always have to emit invalidations
		1665	* and flushes when moving into and out of the RENDER domain, correct
		1666	* snooping behaviour occurs naturally as the result of our domain
		1667	* tracking.
		1668	*/
		1669	if (obj->cache_level != I915_CACHE_NONE)
		1670	return;
		1671
2344	Serge	1672	if(obj->mapped != NULL)
		1673	{
		1674	uint8_t *page_virtual;
		1675	unsigned int i;
2332	Serge	1676
2344	Serge	1677	page_virtual = obj->mapped;
		1678	asm volatile("mfence");
		1679	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		1680	clflush(page_virtual + i);
		1681	asm volatile("mfence");
		1682	}
		1683	else
		1684	{
		1685	uint8_t *page_virtual;
		1686	unsigned int i;
		1687	page_virtual = AllocKernelSpace(obj->base.size);
		1688	if(page_virtual != NULL)
		1689	{
		1690	u32_t src, dst;
		1691	u32 count;
		1692
		1693	#define page_tabs 0xFDC00000 /* really dirty hack */
		1694
		1695	src = (u32_t*)obj->pages;
		1696	dst = &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12];
		1697	count = obj->base.size/4096;
		1698
		1699	while(count--)
		1700	{
		1701	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		1702	};
		1703
		1704	asm volatile("mfence");
		1705	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		1706	clflush(page_virtual + i);
		1707	asm volatile("mfence");
		1708	FreeKernelSpace(page_virtual);
		1709	}
		1710	else
		1711	{
		1712	asm volatile (
		1713	"mfence \n"
		1714	"wbinvd \n" /* this is really ugly */
		1715	"mfence");
		1716	}
		1717	}
2332	Serge	1718	}
		1719
		1720	/** Flushes any GPU write domain for the object if it's dirty. */
		1721	static int
		1722	i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
		1723	{
		1724	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
		1725	return 0;
		1726
		1727	/* Queue the GPU write cache flushing we need. */
		1728	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		1729	}
		1730
2344	Serge	1731	/** Flushes the GTT write domain for the object if it's dirty. */
		1732	static void
		1733	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		1734	{
		1735	uint32_t old_write_domain;
2332	Serge	1736
2344	Serge	1737	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		1738	return;
2332	Serge	1739
2344	Serge	1740	/* No actual flushing is required for the GTT write domain. Writes
		1741	* to it immediately go to main memory as far as we know, so there's
		1742	* no chipset flush. It also doesn't land in render cache.
		1743	*
		1744	* However, we do have to enforce the order so that all writes through
		1745	* the GTT land before any writes to the device, such as updates to
		1746	* the GATT itself.
		1747	*/
		1748	wmb();
2332	Serge	1749
2344	Serge	1750	old_write_domain = obj->base.write_domain;
		1751	obj->base.write_domain = 0;
2332	Serge	1752
2351	Serge	1753	trace_i915_gem_object_change_domain(obj,
		1754	obj->base.read_domains,
		1755	old_write_domain);
2344	Serge	1756	}
2332	Serge	1757
		1758	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	1759	static void
2332	Serge	1760	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		1761	{
		1762	uint32_t old_write_domain;
		1763
		1764	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		1765	return;
		1766
		1767	i915_gem_clflush_object(obj);
		1768	intel_gtt_chipset_flush();
		1769	old_write_domain = obj->base.write_domain;
		1770	obj->base.write_domain = 0;
		1771
2351	Serge	1772	trace_i915_gem_object_change_domain(obj,
		1773	obj->base.read_domains,
		1774	old_write_domain);
2332	Serge	1775	}
		1776
		1777	/**
		1778	* Moves a single object to the GTT read, and possibly write domain.
		1779	*
		1780	* This function returns when the move is complete, including waiting on
		1781	* flushes to occur.
		1782	*/
		1783	int
		1784	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		1785	{
		1786	uint32_t old_write_domain, old_read_domains;
		1787	int ret;
		1788
		1789	/* Not valid to be called on unbound objects. */
		1790	if (obj->gtt_space == NULL)
		1791	return -EINVAL;
		1792
		1793	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		1794	return 0;
		1795
		1796	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1797	if (ret)
		1798	return ret;
		1799
		1800	if (obj->pending_gpu_write \|\| write) {
		1801	ret = i915_gem_object_wait_rendering(obj);
		1802	if (ret)
		1803	return ret;
		1804	}
		1805
		1806	i915_gem_object_flush_cpu_write_domain(obj);
		1807
		1808	old_write_domain = obj->base.write_domain;
		1809	old_read_domains = obj->base.read_domains;
		1810
		1811	/* It should now be out of any other write domains, and we can update
		1812	* the domain values for our changes.
		1813	*/
		1814	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1815	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		1816	if (write) {
		1817	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		1818	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		1819	obj->dirty = 1;
		1820	}
		1821
2351	Serge	1822	trace_i915_gem_object_change_domain(obj,
		1823	old_read_domains,
		1824	old_write_domain);
		1825
2332	Serge	1826	return 0;
		1827	}
		1828
2335	Serge	1829	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		1830	enum i915_cache_level cache_level)
		1831	{
		1832	int ret;
2332	Serge	1833
2335	Serge	1834	if (obj->cache_level == cache_level)
		1835	return 0;
2332	Serge	1836
2335	Serge	1837	if (obj->pin_count) {
		1838	DRM_DEBUG("can not change the cache level of pinned objects\n");
		1839	return -EBUSY;
		1840	}
2332	Serge	1841
2335	Serge	1842	if (obj->gtt_space) {
		1843	ret = i915_gem_object_finish_gpu(obj);
		1844	if (ret)
		1845	return ret;
2332	Serge	1846
2335	Serge	1847	i915_gem_object_finish_gtt(obj);
2332	Serge	1848
2335	Serge	1849	/* Before SandyBridge, you could not use tiling or fence
		1850	* registers with snooped memory, so relinquish any fences
		1851	* currently pointing to our region in the aperture.
		1852	*/
		1853	if (INTEL_INFO(obj->base.dev)->gen < 6) {
		1854	ret = i915_gem_object_put_fence(obj);
		1855	if (ret)
		1856	return ret;
		1857	}
2332	Serge	1858
2335	Serge	1859	i915_gem_gtt_rebind_object(obj, cache_level);
		1860	}
2332	Serge	1861
2335	Serge	1862	if (cache_level == I915_CACHE_NONE) {
		1863	u32 old_read_domains, old_write_domain;
2332	Serge	1864
2335	Serge	1865	/* If we're coming from LLC cached, then we haven't
		1866	* actually been tracking whether the data is in the
		1867	* CPU cache or not, since we only allow one bit set
		1868	* in obj->write_domain and have been skipping the clflushes.
		1869	* Just set it to the CPU cache for now.
		1870	*/
		1871	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		1872	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	1873
2335	Serge	1874	old_read_domains = obj->base.read_domains;
		1875	old_write_domain = obj->base.write_domain;
2332	Serge	1876
2335	Serge	1877	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1878	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	1879
2351	Serge	1880	trace_i915_gem_object_change_domain(obj,
		1881	old_read_domains,
		1882	old_write_domain);
2344	Serge	1883	}
2332	Serge	1884
2335	Serge	1885	obj->cache_level = cache_level;
		1886	return 0;
		1887	}
2332	Serge	1888
2335	Serge	1889	/*
		1890	* Prepare buffer for display plane (scanout, cursors, etc).
		1891	* Can be called from an uninterruptible phase (modesetting) and allows
		1892	* any flushes to be pipelined (for pageflips).
		1893	*
		1894	* For the display plane, we want to be in the GTT but out of any write
		1895	* domains. So in many ways this looks like set_to_gtt_domain() apart from the
		1896	* ability to pipeline the waits, pinning and any additional subtleties
		1897	* that may differentiate the display plane from ordinary buffers.
		1898	*/
		1899	int
		1900	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		1901	u32 alignment,
		1902	struct intel_ring_buffer *pipelined)
		1903	{
		1904	u32 old_read_domains, old_write_domain;
		1905	int ret;
2332	Serge	1906
2335	Serge	1907	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1908	if (ret)
		1909	return ret;
2332	Serge	1910
2335	Serge	1911	if (pipelined != obj->ring) {
		1912	ret = i915_gem_object_wait_rendering(obj);
		1913	if (ret == -ERESTARTSYS)
		1914	return ret;
		1915	}
2332	Serge	1916
2335	Serge	1917	/* The display engine is not coherent with the LLC cache on gen6. As
		1918	* a result, we make sure that the pinning that is about to occur is
		1919	* done with uncached PTEs. This is lowest common denominator for all
		1920	* chipsets.
		1921	*
		1922	* However for gen6+, we could do better by using the GFDT bit instead
		1923	* of uncaching, which would allow us to flush all the LLC-cached data
		1924	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		1925	*/
		1926	// ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		1927	// if (ret)
		1928	// return ret;
2332	Serge	1929
2335	Serge	1930	/* As the user may map the buffer once pinned in the display plane
		1931	* (e.g. libkms for the bootup splash), we have to ensure that we
		1932	* always use map_and_fenceable for all scanout buffers.
		1933	*/
		1934	ret = i915_gem_object_pin(obj, alignment, true);
		1935	if (ret)
		1936	return ret;
2332	Serge	1937
2335	Serge	1938	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	1939
2335	Serge	1940	old_write_domain = obj->base.write_domain;
		1941	old_read_domains = obj->base.read_domains;
2332	Serge	1942
2335	Serge	1943	/* It should now be out of any other write domains, and we can update
		1944	* the domain values for our changes.
		1945	*/
		1946	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1947	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	1948
2351	Serge	1949	trace_i915_gem_object_change_domain(obj,
		1950	old_read_domains,
		1951	old_write_domain);
2332	Serge	1952
2335	Serge	1953	return 0;
		1954	}
2332	Serge	1955
2344	Serge	1956	int
		1957	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		1958	{
		1959	int ret;
2332	Serge	1960
2344	Serge	1961	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		1962	return 0;
2332	Serge	1963
2344	Serge	1964	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
		1965	ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		1966	if (ret)
		1967	return ret;
		1968	}
2332	Serge	1969
2344	Serge	1970	/* Ensure that we invalidate the GPU's caches and TLBs. */
		1971	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2332	Serge	1972
2344	Serge	1973	return i915_gem_object_wait_rendering(obj);
		1974	}
2332	Serge	1975
2344	Serge	1976	/**
		1977	* Moves a single object to the CPU read, and possibly write domain.
		1978	*
		1979	* This function returns when the move is complete, including waiting on
		1980	* flushes to occur.
		1981	*/
		1982	static int
		1983	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		1984	{
		1985	uint32_t old_write_domain, old_read_domains;
		1986	int ret;
2332	Serge	1987
2344	Serge	1988	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		1989	return 0;
2332	Serge	1990
2344	Serge	1991	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1992	if (ret)
		1993	return ret;
2332	Serge	1994
2344	Serge	1995	ret = i915_gem_object_wait_rendering(obj);
		1996	if (ret)
		1997	return ret;
2332	Serge	1998
2344	Serge	1999	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	2000
		2001
2344	Serge	2002	old_write_domain = obj->base.write_domain;
		2003	old_read_domains = obj->base.read_domains;
2332	Serge	2004
2344	Serge	2005	/* Flush the CPU cache if it's still invalid. */
		2006	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		2007	i915_gem_clflush_object(obj);
2332	Serge	2008
2344	Serge	2009	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		2010	}
2332	Serge	2011
2344	Serge	2012	/* It should now be out of any other write domains, and we can update
		2013	* the domain values for our changes.
		2014	*/
		2015	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	2016
2344	Serge	2017	/* If we're writing through the CPU, then the GPU read domains will
		2018	* need to be invalidated at next use.
		2019	*/
		2020	if (write) {
		2021	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2022	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2023	}
2332	Serge	2024
2351	Serge	2025	trace_i915_gem_object_change_domain(obj,
		2026	old_read_domains,
		2027	old_write_domain);
2332	Serge	2028
2344	Serge	2029	return 0;
		2030	}
2332	Serge	2031
2344	Serge	2032	/**
		2033	* Moves the object from a partially CPU read to a full one.
		2034	*
		2035	* Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
		2036	* and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
		2037	*/
		2038	static void
		2039	i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
		2040	{
		2041	if (!obj->page_cpu_valid)
		2042	return;
2332	Serge	2043
2344	Serge	2044	/* If we're partially in the CPU read domain, finish moving it in.
		2045	*/
		2046	if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
		2047	}
2332	Serge	2048
2344	Serge	2049	/* Free the page_cpu_valid mappings which are now stale, whether
		2050	* or not we've got I915_GEM_DOMAIN_CPU.
		2051	*/
		2052	kfree(obj->page_cpu_valid);
		2053	obj->page_cpu_valid = NULL;
		2054	}
2332	Serge	2055
		2056
		2057
		2058
2352	Serge	2059	int gem_object_lock(struct drm_i915_gem_object *obj)
		2060	{
		2061	return i915_gem_object_set_to_cpu_domain(obj, true);
		2062	}
2332	Serge	2063
		2064
		2065
		2066
		2067
		2068
		2069
		2070
		2071
		2072
		2073
		2074
		2075
		2076
		2077
		2078
		2079
		2080
		2081
		2082	int
		2083	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		2084	uint32_t alignment,
		2085	bool map_and_fenceable)
		2086	{
		2087	struct drm_device *dev = obj->base.dev;
		2088	struct drm_i915_private *dev_priv = dev->dev_private;
		2089	int ret;
		2090
		2091	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
2352	Serge	2092	WARN_ON(i915_verify_lists(dev));
2332	Serge	2093
		2094	#if 0
		2095	if (obj->gtt_space != NULL) {
		2096	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		2097	(map_and_fenceable && !obj->map_and_fenceable)) {
		2098	WARN(obj->pin_count,
		2099	"bo is already pinned with incorrect alignment:"
		2100	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		2101	" obj->map_and_fenceable=%d\n",
		2102	obj->gtt_offset, alignment,
		2103	map_and_fenceable,
		2104	obj->map_and_fenceable);
		2105	ret = i915_gem_object_unbind(obj);
		2106	if (ret)
		2107	return ret;
		2108	}
		2109	}
		2110	#endif
		2111
		2112	if (obj->gtt_space == NULL) {
		2113	ret = i915_gem_object_bind_to_gtt(obj, alignment,
		2114	map_and_fenceable);
		2115	if (ret)
		2116	return ret;
		2117	}
		2118
		2119	if (obj->pin_count++ == 0) {
		2120	if (!obj->active)
		2121	list_move_tail(&obj->mm_list,
		2122	&dev_priv->mm.pinned_list);
		2123	}
		2124	obj->pin_mappable \|= map_and_fenceable;
		2125
		2126	return 0;
		2127	}
		2128
2344	Serge	2129	void
		2130	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		2131	{
		2132	struct drm_device *dev = obj->base.dev;
		2133	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	2134
2344	Serge	2135	BUG_ON(obj->pin_count == 0);
		2136	BUG_ON(obj->gtt_space == NULL);
2332	Serge	2137
2344	Serge	2138	if (--obj->pin_count == 0) {
		2139	if (!obj->active)
		2140	list_move_tail(&obj->mm_list,
		2141	&dev_priv->mm.inactive_list);
		2142	obj->pin_mappable = false;
		2143	}
		2144	}
2332	Serge	2145
		2146
		2147
		2148
		2149
		2150
		2151
		2152
		2153
		2154
		2155
		2156
		2157
		2158
		2159
		2160
		2161
		2162
		2163
		2164
		2165
		2166
		2167
		2168
		2169
		2170
		2171
		2172
		2173
		2174
		2175	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		2176	size_t size)
		2177	{
		2178	struct drm_i915_private *dev_priv = dev->dev_private;
		2179	struct drm_i915_gem_object *obj;
2340	Serge	2180
2332	Serge	2181	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		2182	if (obj == NULL)
		2183	return NULL;
		2184
		2185	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		2186	kfree(obj);
		2187	return NULL;
		2188	}
		2189
		2190
		2191	i915_gem_info_add_obj(dev_priv, size);
		2192
		2193	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		2194	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		2195
2342	Serge	2196	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
2332	Serge	2197	/* On Gen6, we can have the GPU use the LLC (the CPU
		2198	* cache) for about a 10% performance improvement
		2199	* compared to uncached. Graphics requests other than
		2200	* display scanout are coherent with the CPU in
		2201	* accessing this cache. This means in this mode we
		2202	* don't need to clflush on the CPU side, and on the
		2203	* GPU side we only need to flush internal caches to
		2204	* get data visible to the CPU.
		2205	*
		2206	* However, we maintain the display planes as UC, and so
		2207	* need to rebind when first used as such.
		2208	*/
		2209	obj->cache_level = I915_CACHE_LLC;
		2210	} else
		2211	obj->cache_level = I915_CACHE_NONE;
		2212
		2213	obj->base.driver_private = NULL;
		2214	obj->fence_reg = I915_FENCE_REG_NONE;
		2215	INIT_LIST_HEAD(&obj->mm_list);
		2216	INIT_LIST_HEAD(&obj->gtt_list);
		2217	INIT_LIST_HEAD(&obj->ring_list);
		2218	INIT_LIST_HEAD(&obj->exec_list);
		2219	INIT_LIST_HEAD(&obj->gpu_write_list);
		2220	obj->madv = I915_MADV_WILLNEED;
		2221	/* Avoid an unnecessary call to unbind on the first bind. */
		2222	obj->map_and_fenceable = true;
2340	Serge	2223
2332	Serge	2224	return obj;
		2225	}
		2226
2344	Serge	2227	int i915_gem_init_object(struct drm_gem_object *obj)
		2228	{
		2229	BUG();
2332	Serge	2230
2344	Serge	2231	return 0;
		2232	}
2332	Serge	2233
2344	Serge	2234	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
		2235	{
		2236	struct drm_device *dev = obj->base.dev;
		2237	drm_i915_private_t *dev_priv = dev->dev_private;
		2238	int ret;
2332	Serge	2239
2344	Serge	2240	ret = i915_gem_object_unbind(obj);
		2241	if (ret == -ERESTARTSYS) {
		2242	list_move(&obj->mm_list,
		2243	&dev_priv->mm.deferred_free_list);
		2244	return;
		2245	}
2332	Serge	2246
2351	Serge	2247	trace_i915_gem_object_destroy(obj);
2332	Serge	2248
2344	Serge	2249	// if (obj->base.map_list.map)
		2250	// drm_gem_free_mmap_offset(&obj->base);
2332	Serge	2251
2344	Serge	2252	drm_gem_object_release(&obj->base);
		2253	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	2254
2344	Serge	2255	kfree(obj->page_cpu_valid);
		2256	kfree(obj->bit_17);
		2257	kfree(obj);
		2258	}
2332	Serge	2259
2344	Serge	2260	void i915_gem_free_object(struct drm_gem_object *gem_obj)
		2261	{
		2262	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
		2263	struct drm_device *dev = obj->base.dev;
2332	Serge	2264
2351	Serge	2265	while (obj->pin_count > 0)
2344	Serge	2266	i915_gem_object_unpin(obj);
2332	Serge	2267
2344	Serge	2268	// if (obj->phys_obj)
		2269	// i915_gem_detach_phys_object(dev, obj);
2332	Serge	2270
2344	Serge	2271	i915_gem_free_object_tail(obj);
		2272	}
2332	Serge	2273
		2274
		2275
		2276
		2277
		2278
		2279
		2280
		2281
		2282
		2283
2344	Serge	2284
2332	Serge	2285	int
		2286	i915_gem_init_ringbuffer(struct drm_device *dev)
		2287	{
		2288	drm_i915_private_t *dev_priv = dev->dev_private;
		2289	int ret;
2351	Serge	2290
2332	Serge	2291	ret = intel_init_render_ring_buffer(dev);
		2292	if (ret)
		2293	return ret;
		2294
		2295	if (HAS_BSD(dev)) {
		2296	ret = intel_init_bsd_ring_buffer(dev);
		2297	if (ret)
		2298	goto cleanup_render_ring;
		2299	}
		2300
		2301	if (HAS_BLT(dev)) {
		2302	ret = intel_init_blt_ring_buffer(dev);
		2303	if (ret)
		2304	goto cleanup_bsd_ring;
		2305	}
		2306
		2307	dev_priv->next_seqno = 1;
2351	Serge	2308
2332	Serge	2309	return 0;
		2310
		2311	cleanup_bsd_ring:
		2312	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		2313	cleanup_render_ring:
		2314	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		2315	return ret;
		2316	}
		2317
		2318	#if 0
		2319	void
		2320	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		2321	{
		2322	drm_i915_private_t *dev_priv = dev->dev_private;
		2323	int i;
		2324
		2325	for (i = 0; i < I915_NUM_RINGS; i++)
		2326	intel_cleanup_ring_buffer(&dev_priv->ring[i]);
		2327	}
		2328
		2329	int
		2330	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		2331	struct drm_file *file_priv)
		2332	{
		2333	drm_i915_private_t *dev_priv = dev->dev_private;
		2334	int ret, i;
		2335
		2336	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2337	return 0;
		2338
		2339	if (atomic_read(&dev_priv->mm.wedged)) {
		2340	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		2341	atomic_set(&dev_priv->mm.wedged, 0);
		2342	}
		2343
		2344	mutex_lock(&dev->struct_mutex);
		2345	dev_priv->mm.suspended = 0;
		2346
		2347	ret = i915_gem_init_ringbuffer(dev);
		2348	if (ret != 0) {
		2349	mutex_unlock(&dev->struct_mutex);
		2350	return ret;
		2351	}
		2352
		2353	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		2354	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
		2355	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
		2356	for (i = 0; i < I915_NUM_RINGS; i++) {
		2357	BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
		2358	BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
		2359	}
		2360	mutex_unlock(&dev->struct_mutex);
		2361
		2362	ret = drm_irq_install(dev);
		2363	if (ret)
		2364	goto cleanup_ringbuffer;
		2365
		2366	return 0;
		2367
		2368	cleanup_ringbuffer:
		2369	mutex_lock(&dev->struct_mutex);
		2370	i915_gem_cleanup_ringbuffer(dev);
		2371	dev_priv->mm.suspended = 1;
		2372	mutex_unlock(&dev->struct_mutex);
		2373
		2374	return ret;
		2375	}
		2376
		2377	int
		2378	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		2379	struct drm_file *file_priv)
		2380	{
		2381	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2382	return 0;
		2383
		2384	drm_irq_uninstall(dev);
		2385	return i915_gem_idle(dev);
		2386	}
		2387
		2388	void
		2389	i915_gem_lastclose(struct drm_device *dev)
		2390	{
		2391	int ret;
		2392
		2393	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2394	return;
		2395
		2396	ret = i915_gem_idle(dev);
		2397	if (ret)
		2398	DRM_ERROR("failed to idle hardware: %d\n", ret);
		2399	}
		2400	#endif
		2401
		2402	static void
2326	Serge	2403	init_ring_lists(struct intel_ring_buffer *ring)
		2404	{
		2405	INIT_LIST_HEAD(&ring->active_list);
		2406	INIT_LIST_HEAD(&ring->request_list);
		2407	INIT_LIST_HEAD(&ring->gpu_write_list);
		2408	}
		2409
		2410	void
		2411	i915_gem_load(struct drm_device *dev)
		2412	{
		2413	int i;
		2414	drm_i915_private_t *dev_priv = dev->dev_private;
		2415
		2416	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		2417	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
		2418	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
		2419	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
		2420	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		2421	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
		2422	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
		2423	for (i = 0; i < I915_NUM_RINGS; i++)
		2424	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	2425	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	2426	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
		2427
		2428	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		2429	if (IS_GEN3(dev)) {
		2430	u32 tmp = I915_READ(MI_ARB_STATE);
		2431	if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
		2432	/* arb state is a masked write, so set bit + bit in mask */
		2433	tmp = MI_ARB_C3_LP_WRITE_ENABLE \| (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
		2434	I915_WRITE(MI_ARB_STATE, tmp);
		2435	}
		2436	}
		2437
		2438	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		2439
		2440	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		2441	dev_priv->num_fence_regs = 16;
		2442	else
		2443	dev_priv->num_fence_regs = 8;
		2444
		2445	/* Initialize fence registers to zero */
		2446	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2447	i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
		2448	}
		2449
		2450	i915_gem_detect_bit_6_swizzle(dev);
		2451
		2452	dev_priv->mm.interruptible = true;
		2453
		2454	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		2455	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		2456	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		2457	}
		2458
		2459
		2460

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 2352