WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
		28	#include "drmP.h"
		29	#include "drm.h"
2330	Serge	30	#include "i915_drm.h"
2326	Serge	31	#include "i915_drv.h"
		32	//#include "i915_trace.h"
		33	#include "intel_drv.h"
		34	//#include
2330	Serge	35	#include
2326	Serge	36	//#include
		37	#include
		38
2344	Serge	39	extern int x86_clflush_size;
2332	Serge	40
2344	Serge	41	#undef mb
		42	#undef rmb
		43	#undef wmb
		44	#define mb() asm volatile("mfence")
		45	#define rmb() asm volatile ("lfence")
		46	#define wmb() asm volatile ("sfence")
		47
		48	static inline void clflush(volatile void *__p)
		49	{
		50	asm volatile("clflush %0" : "+m" ((volatile char)__p));
		51	}
		52
2332	Serge	53	#define MAX_ERRNO 4095
		54
		55	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		56
		57	static inline long IS_ERR(const void *ptr)
		58	{
		59	return IS_ERR_VALUE((unsigned long)ptr);
		60	}
		61
		62	static inline void *ERR_PTR(long error)
		63	{
		64	return (void *) error;
		65	}
		66
		67	static inline long PTR_ERR(const void *ptr)
		68	{
		69	return (long) ptr;
		70	}
		71
2344	Serge	72	void
		73	drm_gem_object_free(struct kref *kref)
		74	{
		75	struct drm_gem_object obj = (struct drm_gem_object ) kref;
		76	struct drm_device *dev = obj->dev;
2332	Serge	77
2344	Serge	78	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		79
		80	i915_gem_free_object(obj);
		81	}
		82
2332	Serge	83	/**
		84	* Initialize an already allocated GEM object of the specified size with
		85	* shmfs backing store.
		86	*/
		87	int drm_gem_object_init(struct drm_device *dev,
		88	struct drm_gem_object *obj, size_t size)
		89	{
		90	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		91
		92	obj->dev = dev;
2344	Serge	93	kref_init(&obj->refcount);
2332	Serge	94	atomic_set(&obj->handle_count, 0);
		95	obj->size = size;
		96
		97	return 0;
		98	}
		99
2344	Serge	100	void
		101	drm_gem_object_release(struct drm_gem_object *obj)
		102	{ }
2332	Serge	103
		104
2326	Serge	105	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		106	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		107	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		108	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		109
2332	Serge	110	static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
		111	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		112	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		113	static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
		114	bool write);
		115	static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
		116	uint64_t offset,
		117	uint64_t size);
		118	static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
		119	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		120	unsigned alignment,
		121	bool map_and_fenceable);
		122	static void i915_gem_clear_fence_reg(struct drm_device *dev,
		123	struct drm_i915_fence_reg *reg);
		124	static int i915_gem_phys_pwrite(struct drm_device *dev,
		125	struct drm_i915_gem_object *obj,
		126	struct drm_i915_gem_pwrite *args,
		127	struct drm_file *file);
		128	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
2326	Serge	129
2332	Serge	130	static int i915_gem_inactive_shrink(struct shrinker *shrinker,
		131	struct shrink_control *sc);
		132
		133	/* some bookkeeping */
		134	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		135	size_t size)
		136	{
		137	dev_priv->mm.object_count++;
		138	dev_priv->mm.object_memory += size;
		139	}
		140
		141	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		142	size_t size)
		143	{
		144	dev_priv->mm.object_count--;
		145	dev_priv->mm.object_memory -= size;
		146	}
		147
		148	#if 0
		149
		150	static int
		151	i915_gem_wait_for_error(struct drm_device *dev)
		152	{
		153	struct drm_i915_private *dev_priv = dev->dev_private;
		154	struct completion *x = &dev_priv->error_completion;
		155	unsigned long flags;
		156	int ret;
		157
		158	if (!atomic_read(&dev_priv->mm.wedged))
		159	return 0;
		160
		161	ret = wait_for_completion_interruptible(x);
		162	if (ret)
		163	return ret;
		164
		165	if (atomic_read(&dev_priv->mm.wedged)) {
		166	/* GPU is hung, bump the completion count to account for
		167	* the token we just consumed so that we never hit zero and
		168	* end up waiting upon a subsequent completion event that
		169	* will never happen.
		170	*/
		171	spin_lock_irqsave(&x->wait.lock, flags);
		172	x->done++;
		173	spin_unlock_irqrestore(&x->wait.lock, flags);
		174	}
		175	return 0;
		176	}
		177
		178	int i915_mutex_lock_interruptible(struct drm_device *dev)
		179	{
		180	int ret;
		181
		182	ret = i915_gem_wait_for_error(dev);
		183	if (ret)
		184	return ret;
		185
		186	ret = mutex_lock_interruptible(&dev->struct_mutex);
		187	if (ret)
		188	return ret;
		189
		190	WARN_ON(i915_verify_lists(dev));
		191	return 0;
		192	}
		193
		194	static inline bool
		195	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		196	{
		197	return obj->gtt_space && !obj->active && obj->pin_count == 0;
		198	}
		199
		200	#endif
		201
		202	void i915_gem_do_init(struct drm_device *dev,
		203	unsigned long start,
		204	unsigned long mappable_end,
		205	unsigned long end)
		206	{
		207	drm_i915_private_t *dev_priv = dev->dev_private;
		208
		209	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
		210
		211	dev_priv->mm.gtt_start = start;
		212	dev_priv->mm.gtt_mappable_end = mappable_end;
		213	dev_priv->mm.gtt_end = end;
		214	dev_priv->mm.gtt_total = end - start;
		215	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
		216
		217	/* Take over this portion of the GTT */
		218	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
		219	}
		220
		221	#if 0
		222
		223	int
		224	i915_gem_init_ioctl(struct drm_device dev, void data,
		225	struct drm_file *file)
		226	{
		227	struct drm_i915_gem_init *args = data;
		228
		229	if (args->gtt_start >= args->gtt_end \|\|
		230	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		231	return -EINVAL;
		232
		233	mutex_lock(&dev->struct_mutex);
		234	i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
		235	mutex_unlock(&dev->struct_mutex);
		236
		237	return 0;
		238	}
		239
		240	int
		241	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		242	struct drm_file *file)
		243	{
		244	struct drm_i915_private *dev_priv = dev->dev_private;
		245	struct drm_i915_gem_get_aperture *args = data;
		246	struct drm_i915_gem_object *obj;
		247	size_t pinned;
		248
		249	if (!(dev->driver->driver_features & DRIVER_GEM))
		250	return -ENODEV;
		251
		252	pinned = 0;
		253	mutex_lock(&dev->struct_mutex);
		254	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
		255	pinned += obj->gtt_space->size;
		256	mutex_unlock(&dev->struct_mutex);
		257
		258	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	259	args->aper_available_size = args->aper_size - pinned;
2332	Serge	260
		261	return 0;
		262	}
		263
		264	static int
		265	i915_gem_create(struct drm_file *file,
		266	struct drm_device *dev,
		267	uint64_t size,
		268	uint32_t *handle_p)
		269	{
		270	struct drm_i915_gem_object *obj;
		271	int ret;
		272	u32 handle;
		273
		274	size = roundup(size, PAGE_SIZE);
2342	Serge	275	if (size == 0)
		276	return -EINVAL;
2332	Serge	277
		278	/* Allocate the new object */
		279	obj = i915_gem_alloc_object(dev, size);
		280	if (obj == NULL)
		281	return -ENOMEM;
		282
		283	ret = drm_gem_handle_create(file, &obj->base, &handle);
		284	if (ret) {
		285	drm_gem_object_release(&obj->base);
		286	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		287	kfree(obj);
		288	return ret;
		289	}
		290
		291	/* drop reference from allocate - handle holds it now */
		292	drm_gem_object_unreference(&obj->base);
		293
		294	*handle_p = handle;
		295	return 0;
		296	}
		297
		298	int
		299	i915_gem_dumb_create(struct drm_file *file,
		300	struct drm_device *dev,
		301	struct drm_mode_create_dumb *args)
		302	{
		303	/* have to work out size/pitch and return them */
		304	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		305	args->size = args->pitch * args->height;
		306	return i915_gem_create(file, dev,
		307	args->size, &args->handle);
		308	}
		309
		310	int i915_gem_dumb_destroy(struct drm_file *file,
		311	struct drm_device *dev,
		312	uint32_t handle)
		313	{
		314	return drm_gem_handle_delete(file, handle);
		315	}
		316
2326	Serge	317	/**
2332	Serge	318	* Creates a new mm object and returns a handle to it.
		319	*/
		320	int
		321	i915_gem_create_ioctl(struct drm_device dev, void data,
		322	struct drm_file *file)
		323	{
		324	struct drm_i915_gem_create *args = data;
		325	return i915_gem_create(file, dev,
		326	args->size, &args->handle);
		327	}
		328
		329	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		330	{
		331	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		332
		333	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		334	obj->tiling_mode != I915_TILING_NONE;
		335	}
		336
		337	static inline void
		338	slow_shmem_copy(struct page *dst_page,
		339	int dst_offset,
		340	struct page *src_page,
		341	int src_offset,
		342	int length)
		343	{
		344	char dst_vaddr, src_vaddr;
		345
		346	dst_vaddr = kmap(dst_page);
		347	src_vaddr = kmap(src_page);
		348
		349	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
		350
		351	kunmap(src_page);
		352	kunmap(dst_page);
		353	}
		354
		355	static inline void
		356	slow_shmem_bit17_copy(struct page *gpu_page,
		357	int gpu_offset,
		358	struct page *cpu_page,
		359	int cpu_offset,
		360	int length,
		361	int is_read)
		362	{
		363	char gpu_vaddr, cpu_vaddr;
		364
		365	/* Use the unswizzled path if this page isn't affected. */
		366	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
		367	if (is_read)
		368	return slow_shmem_copy(cpu_page, cpu_offset,
		369	gpu_page, gpu_offset, length);
		370	else
		371	return slow_shmem_copy(gpu_page, gpu_offset,
		372	cpu_page, cpu_offset, length);
		373	}
		374
		375	gpu_vaddr = kmap(gpu_page);
		376	cpu_vaddr = kmap(cpu_page);
		377
		378	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
		379	* XORing with the other bits (A9 for Y, A9 and A10 for X)
		380	*/
		381	while (length > 0) {
		382	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		383	int this_length = min(cacheline_end - gpu_offset, length);
		384	int swizzled_gpu_offset = gpu_offset ^ 64;
		385
		386	if (is_read) {
		387	memcpy(cpu_vaddr + cpu_offset,
		388	gpu_vaddr + swizzled_gpu_offset,
		389	this_length);
		390	} else {
		391	memcpy(gpu_vaddr + swizzled_gpu_offset,
		392	cpu_vaddr + cpu_offset,
		393	this_length);
		394	}
		395	cpu_offset += this_length;
		396	gpu_offset += this_length;
		397	length -= this_length;
		398	}
		399
		400	kunmap(cpu_page);
		401	kunmap(gpu_page);
		402	}
		403
		404	/**
		405	* This is the fast shmem pread path, which attempts to copy_from_user directly
		406	* from the backing pages of the object to the user's address space. On a
		407	* fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
		408	*/
		409	static int
		410	i915_gem_shmem_pread_fast(struct drm_device *dev,
		411	struct drm_i915_gem_object *obj,
		412	struct drm_i915_gem_pread *args,
		413	struct drm_file *file)
		414	{
		415	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		416	ssize_t remain;
		417	loff_t offset;
		418	char __user *user_data;
		419	int page_offset, page_length;
		420
		421	user_data = (char __user *) (uintptr_t) args->data_ptr;
		422	remain = args->size;
		423
		424	offset = args->offset;
		425
		426	while (remain > 0) {
		427	struct page *page;
		428	char *vaddr;
		429	int ret;
		430
		431	/* Operation in this page
		432	*
		433	* page_offset = offset within page
		434	* page_length = bytes to copy for this page
		435	*/
		436	page_offset = offset_in_page(offset);
		437	page_length = remain;
		438	if ((page_offset + remain) > PAGE_SIZE)
		439	page_length = PAGE_SIZE - page_offset;
		440
		441	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		442	if (IS_ERR(page))
		443	return PTR_ERR(page);
		444
		445	vaddr = kmap_atomic(page);
		446	ret = __copy_to_user_inatomic(user_data,
		447	vaddr + page_offset,
		448	page_length);
		449	kunmap_atomic(vaddr);
		450
		451	mark_page_accessed(page);
		452	page_cache_release(page);
		453	if (ret)
		454	return -EFAULT;
		455
		456	remain -= page_length;
		457	user_data += page_length;
		458	offset += page_length;
		459	}
		460
		461	return 0;
		462	}
		463
		464	/**
		465	* This is the fallback shmem pread path, which allocates temporary storage
		466	* in kernel space to copy_to_user into outside of the struct_mutex, so we
		467	* can copy out of the object's backing pages while holding the struct mutex
		468	* and not take page faults.
		469	*/
		470	static int
		471	i915_gem_shmem_pread_slow(struct drm_device *dev,
		472	struct drm_i915_gem_object *obj,
		473	struct drm_i915_gem_pread *args,
		474	struct drm_file *file)
		475	{
		476	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		477	struct mm_struct *mm = current->mm;
		478	struct page **user_pages;
		479	ssize_t remain;
		480	loff_t offset, pinned_pages, i;
		481	loff_t first_data_page, last_data_page, num_pages;
		482	int shmem_page_offset;
		483	int data_page_index, data_page_offset;
		484	int page_length;
		485	int ret;
		486	uint64_t data_ptr = args->data_ptr;
		487	int do_bit17_swizzling;
		488
		489	remain = args->size;
		490
		491	/* Pin the user pages containing the data. We can't fault while
		492	* holding the struct mutex, yet we want to hold it while
		493	* dereferencing the user data.
		494	*/
		495	first_data_page = data_ptr / PAGE_SIZE;
		496	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
		497	num_pages = last_data_page - first_data_page + 1;
		498
		499	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
		500	if (user_pages == NULL)
		501	return -ENOMEM;
		502
		503	mutex_unlock(&dev->struct_mutex);
		504	down_read(&mm->mmap_sem);
		505	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
		506	num_pages, 1, 0, user_pages, NULL);
		507	up_read(&mm->mmap_sem);
		508	mutex_lock(&dev->struct_mutex);
		509	if (pinned_pages < num_pages) {
		510	ret = -EFAULT;
		511	goto out;
		512	}
		513
		514	ret = i915_gem_object_set_cpu_read_domain_range(obj,
		515	args->offset,
		516	args->size);
		517	if (ret)
		518	goto out;
		519
		520	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		521
		522	offset = args->offset;
		523
		524	while (remain > 0) {
		525	struct page *page;
		526
		527	/* Operation in this page
		528	*
		529	* shmem_page_offset = offset within page in shmem file
		530	* data_page_index = page number in get_user_pages return
		531	* data_page_offset = offset with data_page_index page.
		532	* page_length = bytes to copy for this page
		533	*/
		534	shmem_page_offset = offset_in_page(offset);
		535	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		536	data_page_offset = offset_in_page(data_ptr);
		537
		538	page_length = remain;
		539	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		540	page_length = PAGE_SIZE - shmem_page_offset;
		541	if ((data_page_offset + page_length) > PAGE_SIZE)
		542	page_length = PAGE_SIZE - data_page_offset;
		543
		544	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		545	if (IS_ERR(page)) {
		546	ret = PTR_ERR(page);
		547	goto out;
		548	}
		549
		550	if (do_bit17_swizzling) {
		551	slow_shmem_bit17_copy(page,
		552	shmem_page_offset,
		553	user_pages[data_page_index],
		554	data_page_offset,
		555	page_length,
		556	1);
		557	} else {
		558	slow_shmem_copy(user_pages[data_page_index],
		559	data_page_offset,
		560	page,
		561	shmem_page_offset,
		562	page_length);
		563	}
		564
		565	mark_page_accessed(page);
		566	page_cache_release(page);
		567
		568	remain -= page_length;
		569	data_ptr += page_length;
		570	offset += page_length;
		571	}
		572
		573	out:
		574	for (i = 0; i < pinned_pages; i++) {
		575	SetPageDirty(user_pages[i]);
		576	mark_page_accessed(user_pages[i]);
		577	page_cache_release(user_pages[i]);
		578	}
		579	drm_free_large(user_pages);
		580
		581	return ret;
		582	}
		583	#endif
		584
		585
		586
		587
		588
		589
		590
		591
		592
		593
		594
		595
		596
		597
		598
		599
		600
		601
		602
		603
		604
		605
		606
		607
		608
		609
		610
		611
		612
		613
		614
		615
		616
		617
		618
		619
		620
		621
		622
		623
		624
		625
		626
		627
		628
		629
		630
		631
		632
		633
		634
		635
		636
		637
		638
		639
		640
		641
		642
		643
		644
		645
		646
		647
		648
		649
		650
		651
		652
		653
		654
		655
		656
		657	static uint32_t
		658	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		659	{
		660	uint32_t gtt_size;
		661
		662	if (INTEL_INFO(dev)->gen >= 4 \|\|
		663	tiling_mode == I915_TILING_NONE)
		664	return size;
		665
		666	/* Previous chips need a power-of-two fence region when tiling */
		667	if (INTEL_INFO(dev)->gen == 3)
		668	gtt_size = 1024*1024;
		669	else
		670	gtt_size = 512*1024;
		671
		672	while (gtt_size < size)
		673	gtt_size <<= 1;
		674
		675	return gtt_size;
		676	}
		677
		678	/**
		679	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		680	* @obj: object to check
		681	*
		682	* Return the required GTT alignment for an object, taking into account
		683	* potential fence register mapping.
		684	*/
		685	static uint32_t
		686	i915_gem_get_gtt_alignment(struct drm_device *dev,
		687	uint32_t size,
		688	int tiling_mode)
		689	{
		690	/*
		691	* Minimum alignment is 4k (GTT page size), but might be greater
		692	* if a fence register is needed for the object.
		693	*/
		694	if (INTEL_INFO(dev)->gen >= 4 \|\|
		695	tiling_mode == I915_TILING_NONE)
		696	return 4096;
		697
		698	/*
		699	* Previous chips need to be aligned to the size of the smallest
		700	* fence register that can contain the object.
		701	*/
		702	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		703	}
		704
		705	/**
		706	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		707	* unfenced object
		708	* @dev: the device
		709	* @size: size of the object
		710	* @tiling_mode: tiling mode of the object
		711	*
		712	* Return the required GTT alignment for an object, only taking into account
		713	* unfenced tiled surface requirements.
		714	*/
		715	uint32_t
		716	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		717	uint32_t size,
		718	int tiling_mode)
		719	{
		720	/*
		721	* Minimum alignment is 4k (GTT page size) for sane hw.
		722	*/
		723	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		724	tiling_mode == I915_TILING_NONE)
		725	return 4096;
		726
		727	/* Previous hardware however needs to be aligned to a power-of-two
		728	* tile height. The simplest method for determining this is to reuse
		729	* the power-of-tile object size.
		730	*/
		731	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		732	}
		733
		734
		735
		736
		737
		738
		739
		740
		741
		742
		743
		744
		745
		746
		747
		748
		749
		750	static int
		751	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
		752	gfp_t gfpmask)
		753	{
		754	int page_count, i;
		755	struct page *page;
		756
		757	/* Get the list of pages out of our struct file. They'll be pinned
		758	* at this point until we release them.
		759	*/
		760	page_count = obj->base.size / PAGE_SIZE;
		761	BUG_ON(obj->pages != NULL);
		762	obj->pages = malloc(page_count * sizeof(struct page *));
		763	if (obj->pages == NULL)
		764	return -ENOMEM;
		765
		766
		767	for (i = 0; i < page_count; i++) {
		768	page = (struct page*)AllocPage(); // oh-oh
		769	if (IS_ERR(page))
		770	goto err_pages;
		771
		772	obj->pages[i] = page;
		773	}
		774
		775	// if (obj->tiling_mode != I915_TILING_NONE)
		776	// i915_gem_object_do_bit_17_swizzle(obj);
		777
		778
2340	Serge	779
2332	Serge	780	return 0;
		781
		782	err_pages:
2344	Serge	783	while (i--)
		784	FreePage(obj->pages[i]);
2332	Serge	785
		786	free(obj->pages);
		787	obj->pages = NULL;
		788	return PTR_ERR(page);
		789	}
		790
		791	static void
		792	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		793	{
		794	int page_count = obj->base.size / PAGE_SIZE;
		795	int i;
		796
2344	Serge	797	ENTER();
		798
2332	Serge	799	BUG_ON(obj->madv == __I915_MADV_PURGED);
		800
		801	// if (obj->tiling_mode != I915_TILING_NONE)
		802	// i915_gem_object_save_bit_17_swizzle(obj);
		803
		804	if (obj->madv == I915_MADV_DONTNEED)
		805	obj->dirty = 0;
2344	Serge	806
2332	Serge	807	for (i = 0; i < page_count; i++) {
2344	Serge	808	FreePage(obj->pages[i]);
2332	Serge	809	}
		810	obj->dirty = 0;
		811
		812	free(obj->pages);
		813	obj->pages = NULL;
2344	Serge	814
		815	LEAVE();
2332	Serge	816	}
		817
		818	void
		819	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
		820	struct intel_ring_buffer *ring,
		821	u32 seqno)
		822	{
		823	struct drm_device *dev = obj->base.dev;
		824	struct drm_i915_private *dev_priv = dev->dev_private;
		825
		826	BUG_ON(ring == NULL);
		827	obj->ring = ring;
		828
		829	/* Add a reference if we're newly entering the active list. */
		830	if (!obj->active) {
2344	Serge	831	drm_gem_object_reference(&obj->base);
2332	Serge	832	obj->active = 1;
		833	}
		834
		835	/* Move from whatever list we were on to the tail of execution. */
		836	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		837	list_move_tail(&obj->ring_list, &ring->active_list);
		838
		839	obj->last_rendering_seqno = seqno;
		840	if (obj->fenced_gpu_access) {
		841	struct drm_i915_fence_reg *reg;
		842
		843	BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
		844
		845	obj->last_fenced_seqno = seqno;
		846	obj->last_fenced_ring = ring;
		847
		848	reg = &dev_priv->fence_regs[obj->fence_reg];
		849	list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
		850	}
		851	}
		852
2344	Serge	853	static void
		854	i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
		855	{
		856	list_del_init(&obj->ring_list);
		857	obj->last_rendering_seqno = 0;
		858	}
2332	Serge	859
2344	Serge	860	static void
		861	i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
		862	{
		863	struct drm_device *dev = obj->base.dev;
		864	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	865
2344	Serge	866	BUG_ON(!obj->active);
		867	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
2332	Serge	868
2344	Serge	869	i915_gem_object_move_off_active(obj);
		870	}
		871
		872
		873
		874
		875
		876	/* Immediately discard the backing storage */
2332	Serge	877	static void
2344	Serge	878	i915_gem_object_truncate(struct drm_i915_gem_object *obj)
		879	{
		880	struct inode *inode;
		881
		882	/* Our goal here is to return as much of the memory as
		883	* is possible back to the system as we are called from OOM.
		884	* To do this we must instruct the shmfs to drop all of its
		885	* backing pages, now.
		886	*/
		887
		888	obj->madv = __I915_MADV_PURGED;
		889	}
		890
		891	static inline int
		892	i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
		893	{
		894	return obj->madv == I915_MADV_DONTNEED;
		895	}
		896
		897	static void
2332	Serge	898	i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
		899	uint32_t flush_domains)
		900	{
		901	struct drm_i915_gem_object obj, next;
		902
		903	list_for_each_entry_safe(obj, next,
		904	&ring->gpu_write_list,
		905	gpu_write_list) {
		906	if (obj->base.write_domain & flush_domains) {
		907	uint32_t old_write_domain = obj->base.write_domain;
		908
		909	obj->base.write_domain = 0;
		910	list_del_init(&obj->gpu_write_list);
		911	i915_gem_object_move_to_active(obj, ring,
		912	i915_gem_next_request_seqno(ring));
		913
		914	}
		915	}
		916	}
		917
		918
		919
		920
		921
		922
		923
		924
		925
		926
		927
		928
		929
		930
		931
		932
		933
		934
		935
		936
		937
		938
		939
		940
		941
		942
		943
		944
		945
		946
		947
		948
		949
		950
		951
		952
		953
		954
		955
2344	Serge	956	/**
		957	* Ensures that all rendering to the object has completed and the object is
		958	* safe to unbind from the GTT or access from the CPU.
		959	*/
		960	int
		961	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
		962	{
		963	int ret;
2332	Serge	964
2344	Serge	965	/* This function only exists to support waiting for existing rendering,
		966	* not for emitting required flushes.
		967	*/
		968	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2332	Serge	969
2344	Serge	970	/* If there is rendering queued on the buffer being evicted, wait for
		971	* it.
		972	*/
		973	if (obj->active) {
		974	// ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
		975	// if (ret)
		976	// return ret;
		977	}
2332	Serge	978
2344	Serge	979	return 0;
		980	}
2332	Serge	981
2344	Serge	982	static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
		983	{
		984	u32 old_write_domain, old_read_domains;
2332	Serge	985
2344	Serge	986	/* Act a barrier for all accesses through the GTT */
		987	mb();
2332	Serge	988
2344	Serge	989	/* Force a pagefault for domain tracking on next user access */
		990	// i915_gem_release_mmap(obj);
2332	Serge	991
2344	Serge	992	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		993	return;
2332	Serge	994
2344	Serge	995	old_read_domains = obj->base.read_domains;
		996	old_write_domain = obj->base.write_domain;
		997	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
		998	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2332	Serge	999
2344	Serge	1000	}
2332	Serge	1001
2344	Serge	1002	/**
		1003	* Unbinds an object from the GTT aperture.
		1004	*/
		1005	int
		1006	i915_gem_object_unbind(struct drm_i915_gem_object *obj)
		1007	{
		1008	int ret = 0;
2332	Serge	1009
2344	Serge	1010	ENTER();
		1011	if (obj->gtt_space == NULL)
		1012	return 0;
2332	Serge	1013
2344	Serge	1014	if (obj->pin_count != 0) {
		1015	DRM_ERROR("Attempting to unbind pinned buffer\n");
		1016	return -EINVAL;
		1017	}
2332	Serge	1018
2344	Serge	1019	ret = i915_gem_object_finish_gpu(obj);
		1020	if (ret == -ERESTARTSYS)
		1021	return ret;
		1022	/* Continue on if we fail due to EIO, the GPU is hung so we
		1023	* should be safe and we need to cleanup or else we might
		1024	* cause memory corruption through use-after-free.
		1025	*/
2332	Serge	1026
2344	Serge	1027	i915_gem_object_finish_gtt(obj);
2332	Serge	1028
2344	Serge	1029	/* Move the object to the CPU domain to ensure that
		1030	* any possible CPU writes while it's not in the GTT
		1031	* are flushed when we go to remap it.
		1032	*/
		1033	if (ret == 0)
		1034	ret = i915_gem_object_set_to_cpu_domain(obj, 1);
		1035	if (ret == -ERESTARTSYS)
		1036	return ret;
		1037	if (ret) {
		1038	/* In the event of a disaster, abandon all caches and
		1039	* hope for the best.
		1040	*/
		1041	i915_gem_clflush_object(obj);
		1042	obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1043	}
2332	Serge	1044
2344	Serge	1045	/* release the fence reg _after_ flushing */
		1046	ret = i915_gem_object_put_fence(obj);
		1047	if (ret == -ERESTARTSYS)
		1048	return ret;
2332	Serge	1049
		1050
2344	Serge	1051	i915_gem_gtt_unbind_object(obj);
		1052	i915_gem_object_put_pages_gtt(obj);
2332	Serge	1053
2344	Serge	1054	list_del_init(&obj->gtt_list);
		1055	list_del_init(&obj->mm_list);
		1056	/* Avoid an unnecessary call to unbind on rebind. */
		1057	obj->map_and_fenceable = true;
2332	Serge	1058
2344	Serge	1059	drm_mm_put_block(obj->gtt_space);
		1060	obj->gtt_space = NULL;
		1061	obj->gtt_offset = 0;
2332	Serge	1062
2344	Serge	1063	if (i915_gem_object_is_purgeable(obj))
		1064	i915_gem_object_truncate(obj);
2332	Serge	1065
2344	Serge	1066	LEAVE();
		1067	return ret;
		1068	}
2332	Serge	1069
2344	Serge	1070	int
		1071	i915_gem_flush_ring(struct intel_ring_buffer *ring,
		1072	uint32_t invalidate_domains,
		1073	uint32_t flush_domains)
		1074	{
		1075	int ret;
2332	Serge	1076
2344	Serge	1077	if (((invalidate_domains \| flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
		1078	return 0;
2332	Serge	1079
		1080
2344	Serge	1081	ret = ring->flush(ring, invalidate_domains, flush_domains);
		1082	if (ret)
		1083	return ret;
2332	Serge	1084
2344	Serge	1085	if (flush_domains & I915_GEM_GPU_DOMAINS)
		1086	i915_gem_process_flushing_list(ring, flush_domains);
2332	Serge	1087
2344	Serge	1088	return 0;
		1089	}
2332	Serge	1090
2344	Serge	1091	static int i915_ring_idle(struct intel_ring_buffer *ring)
		1092	{
		1093	int ret;
2332	Serge	1094
2344	Serge	1095	if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
		1096	return 0;
2332	Serge	1097
2344	Serge	1098	if (!list_empty(&ring->gpu_write_list)) {
		1099	ret = i915_gem_flush_ring(ring,
		1100	I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
		1101	if (ret)
		1102	return ret;
		1103	}
2332	Serge	1104
2344	Serge	1105	return 0; //i915_wait_request(ring, i915_gem_next_request_seqno(ring));
		1106	}
2332	Serge	1107
2344	Serge	1108	int
		1109	i915_gpu_idle(struct drm_device *dev)
		1110	{
		1111	drm_i915_private_t *dev_priv = dev->dev_private;
		1112	int ret, i;
2332	Serge	1113
2344	Serge	1114	/* Flush everything onto the inactive list. */
		1115	for (i = 0; i < I915_NUM_RINGS; i++) {
		1116	ret = i915_ring_idle(&dev_priv->ring[i]);
		1117	if (ret)
		1118	return ret;
		1119	}
2332	Serge	1120
2344	Serge	1121	return 0;
		1122	}
2332	Serge	1123
		1124
		1125
		1126
		1127
		1128
		1129
		1130
		1131
		1132
		1133
		1134
		1135
		1136
		1137
		1138
		1139
		1140
2344	Serge	1141
		1142
		1143
		1144
		1145	static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2332	Serge	1146	{
2344	Serge	1147	return i915_seqno_passed(ring->get_seqno(ring), seqno);
		1148	}
		1149
		1150	static int
		1151	i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
		1152	struct intel_ring_buffer *pipelined)
		1153	{
2332	Serge	1154	int ret;
		1155
2344	Serge	1156	if (obj->fenced_gpu_access) {
		1157	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
		1158	ret = i915_gem_flush_ring(obj->last_fenced_ring,
		1159	0, obj->base.write_domain);
		1160	if (ret)
		1161	return ret;
		1162	}
2332	Serge	1163
2344	Serge	1164	obj->fenced_gpu_access = false;
		1165	}
		1166
		1167	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
		1168	if (!ring_passed_seqno(obj->last_fenced_ring,
		1169	obj->last_fenced_seqno)) {
		1170	// ret = i915_wait_request(obj->last_fenced_ring,
		1171	// obj->last_fenced_seqno);
		1172	// if (ret)
		1173	// return ret;
		1174	}
		1175
		1176	obj->last_fenced_seqno = 0;
		1177	obj->last_fenced_ring = NULL;
		1178	}
		1179
		1180	/* Ensure that all CPU reads are completed before installing a fence
		1181	* and all writes before removing the fence.
2332	Serge	1182	*/
2344	Serge	1183	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
		1184	mb();
2332	Serge	1185
		1186	return 0;
		1187	}
		1188
		1189	int
2344	Serge	1190	i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2332	Serge	1191	{
		1192	int ret;
		1193
2344	Serge	1194	// if (obj->tiling_mode)
		1195	// i915_gem_release_mmap(obj);
2332	Serge	1196
2344	Serge	1197	ret = i915_gem_object_flush_fence(obj, NULL);
2332	Serge	1198	if (ret)
		1199	return ret;
		1200
2344	Serge	1201	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		1202	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
		1203	i915_gem_clear_fence_reg(obj->base.dev,
		1204	&dev_priv->fence_regs[obj->fence_reg]);
2332	Serge	1205
2344	Serge	1206	obj->fence_reg = I915_FENCE_REG_NONE;
		1207	}
		1208
2332	Serge	1209	return 0;
		1210	}
		1211
		1212
		1213
		1214
		1215
		1216
		1217
		1218
		1219
		1220
		1221
		1222
		1223
		1224
		1225
		1226
		1227
		1228
		1229
		1230
2344	Serge	1231
		1232
		1233
		1234
		1235
		1236
		1237
		1238
		1239
		1240
		1241
		1242
		1243
		1244
2332	Serge	1245	/**
2326	Serge	1246	* i915_gem_clear_fence_reg - clear out fence register info
		1247	* @obj: object to clear
		1248	*
		1249	* Zeroes out the fence register itself and clears out the associated
		1250	* data structures in dev_priv and obj.
		1251	*/
		1252	static void
		1253	i915_gem_clear_fence_reg(struct drm_device *dev,
		1254	struct drm_i915_fence_reg *reg)
		1255	{
		1256	drm_i915_private_t *dev_priv = dev->dev_private;
		1257	uint32_t fence_reg = reg - dev_priv->fence_regs;
		1258
		1259	switch (INTEL_INFO(dev)->gen) {
		1260	case 7:
		1261	case 6:
		1262	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
		1263	break;
		1264	case 5:
		1265	case 4:
		1266	I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
		1267	break;
		1268	case 3:
		1269	if (fence_reg >= 8)
		1270	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
		1271	else
		1272	case 2:
		1273	fence_reg = FENCE_REG_830_0 + fence_reg * 4;
		1274
		1275	I915_WRITE(fence_reg, 0);
		1276	break;
		1277	}
		1278
		1279	list_del_init(®->lru_list);
		1280	reg->obj = NULL;
		1281	reg->setup_seqno = 0;
		1282	}
		1283
2332	Serge	1284	/**
		1285	* Finds free space in the GTT aperture and binds the object there.
		1286	*/
		1287	static int
		1288	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		1289	unsigned alignment,
		1290	bool map_and_fenceable)
		1291	{
		1292	struct drm_device *dev = obj->base.dev;
		1293	drm_i915_private_t *dev_priv = dev->dev_private;
		1294	struct drm_mm_node *free_space;
		1295	gfp_t gfpmask = 0; //__GFP_NORETRY \| __GFP_NOWARN;
		1296	u32 size, fence_size, fence_alignment, unfenced_alignment;
		1297	bool mappable, fenceable;
		1298	int ret;
2326	Serge	1299
2332	Serge	1300	if (obj->madv != I915_MADV_WILLNEED) {
		1301	DRM_ERROR("Attempting to bind a purgeable object\n");
		1302	return -EINVAL;
		1303	}
		1304
		1305	fence_size = i915_gem_get_gtt_size(dev,
		1306	obj->base.size,
		1307	obj->tiling_mode);
		1308	fence_alignment = i915_gem_get_gtt_alignment(dev,
		1309	obj->base.size,
		1310	obj->tiling_mode);
		1311	unfenced_alignment =
		1312	i915_gem_get_unfenced_gtt_alignment(dev,
		1313	obj->base.size,
		1314	obj->tiling_mode);
		1315
		1316	if (alignment == 0)
		1317	alignment = map_and_fenceable ? fence_alignment :
		1318	unfenced_alignment;
		1319	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		1320	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		1321	return -EINVAL;
		1322	}
		1323
		1324	size = map_and_fenceable ? fence_size : obj->base.size;
		1325
		1326	/* If the object is bigger than the entire aperture, reject it early
		1327	* before evicting everything in a vain attempt to find space.
		1328	*/
		1329	if (obj->base.size >
		1330	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		1331	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		1332	return -E2BIG;
		1333	}
		1334
		1335	search_free:
		1336	if (map_and_fenceable)
		1337	free_space =
		1338	drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
		1339	size, alignment, 0,
		1340	dev_priv->mm.gtt_mappable_end,
		1341	0);
		1342	else
		1343	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
		1344	size, alignment, 0);
		1345
		1346	if (free_space != NULL) {
		1347	if (map_and_fenceable)
		1348	obj->gtt_space =
		1349	drm_mm_get_block_range_generic(free_space,
		1350	size, alignment, 0,
		1351	dev_priv->mm.gtt_mappable_end,
		1352	0);
		1353	else
		1354	obj->gtt_space =
		1355	drm_mm_get_block(free_space, size, alignment);
		1356	}
		1357	if (obj->gtt_space == NULL) {
		1358	/* If the gtt is empty and we're still having trouble
		1359	* fitting our object in, we're out of memory.
		1360	*/
		1361	ret = 1; //i915_gem_evict_something(dev, size, alignment,
		1362	// map_and_fenceable);
		1363	if (ret)
		1364	return ret;
		1365
		1366	goto search_free;
		1367	}
		1368
		1369	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
		1370	if (ret) {
		1371	drm_mm_put_block(obj->gtt_space);
		1372	obj->gtt_space = NULL;
		1373	#if 0
		1374	if (ret == -ENOMEM) {
		1375	/* first try to reclaim some memory by clearing the GTT */
		1376	ret = i915_gem_evict_everything(dev, false);
		1377	if (ret) {
		1378	/* now try to shrink everyone else */
		1379	if (gfpmask) {
		1380	gfpmask = 0;
		1381	goto search_free;
		1382	}
		1383
		1384	return -ENOMEM;
		1385	}
		1386
		1387	goto search_free;
		1388	}
		1389	#endif
		1390	return ret;
		1391	}
		1392
		1393	ret = i915_gem_gtt_bind_object(obj);
		1394	if (ret) {
2344	Serge	1395	i915_gem_object_put_pages_gtt(obj);
2332	Serge	1396	drm_mm_put_block(obj->gtt_space);
		1397	obj->gtt_space = NULL;
		1398
		1399	// if (i915_gem_evict_everything(dev, false))
		1400	return ret;
		1401
		1402	// goto search_free;
		1403	}
		1404
		1405	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
		1406	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		1407
		1408	/* Assert that the object is not currently in any GPU domain. As it
		1409	* wasn't in the GTT, there shouldn't be any way it could have been in
		1410	* a GPU cache
		1411	*/
		1412	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1413	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1414
		1415	obj->gtt_offset = obj->gtt_space->start;
		1416
		1417	fenceable =
		1418	obj->gtt_space->size == fence_size &&
2342	Serge	1419	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2332	Serge	1420
		1421	mappable =
		1422	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		1423
		1424	obj->map_and_fenceable = mappable && fenceable;
		1425
		1426	return 0;
		1427	}
		1428
		1429	void
		1430	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		1431	{
		1432	/* If we don't have a page list set up, then we're not pinned
		1433	* to GPU, and we can ignore the cache flush because it'll happen
		1434	* again at bind time.
		1435	*/
		1436	if (obj->pages == NULL)
		1437	return;
		1438
		1439	/* If the GPU is snooping the contents of the CPU cache,
		1440	* we do not need to manually clear the CPU cache lines. However,
		1441	* the caches are only snooped when the render cache is
		1442	* flushed/invalidated. As we always have to emit invalidations
		1443	* and flushes when moving into and out of the RENDER domain, correct
		1444	* snooping behaviour occurs naturally as the result of our domain
		1445	* tracking.
		1446	*/
		1447	if (obj->cache_level != I915_CACHE_NONE)
		1448	return;
		1449
2344	Serge	1450	if(obj->mapped != NULL)
		1451	{
		1452	uint8_t *page_virtual;
		1453	unsigned int i;
2332	Serge	1454
2344	Serge	1455	page_virtual = obj->mapped;
		1456	asm volatile("mfence");
		1457	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		1458	clflush(page_virtual + i);
		1459	asm volatile("mfence");
		1460	}
		1461	else
		1462	{
		1463	uint8_t *page_virtual;
		1464	unsigned int i;
		1465	page_virtual = AllocKernelSpace(obj->base.size);
		1466	if(page_virtual != NULL)
		1467	{
		1468	u32_t src, dst;
		1469	u32 count;
		1470
		1471	#define page_tabs 0xFDC00000 /* really dirty hack */
		1472
		1473	src = (u32_t*)obj->pages;
		1474	dst = &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12];
		1475	count = obj->base.size/4096;
		1476
		1477	while(count--)
		1478	{
		1479	dst++ = (0xFFFFF000 & src++) \| 0x001 ;
		1480	};
		1481
		1482	asm volatile("mfence");
		1483	for (i = 0; i < obj->base.size; i += x86_clflush_size)
		1484	clflush(page_virtual + i);
		1485	asm volatile("mfence");
		1486	FreeKernelSpace(page_virtual);
		1487	}
		1488	else
		1489	{
		1490	asm volatile (
		1491	"mfence \n"
		1492	"wbinvd \n" /* this is really ugly */
		1493	"mfence");
		1494	}
		1495	}
2332	Serge	1496	}
		1497
		1498	/** Flushes any GPU write domain for the object if it's dirty. */
		1499	static int
		1500	i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
		1501	{
		1502	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
		1503	return 0;
		1504
		1505	/* Queue the GPU write cache flushing we need. */
		1506	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		1507	}
		1508
2344	Serge	1509	/** Flushes the GTT write domain for the object if it's dirty. */
		1510	static void
		1511	i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
		1512	{
		1513	uint32_t old_write_domain;
2332	Serge	1514
2344	Serge	1515	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
		1516	return;
2332	Serge	1517
2344	Serge	1518	/* No actual flushing is required for the GTT write domain. Writes
		1519	* to it immediately go to main memory as far as we know, so there's
		1520	* no chipset flush. It also doesn't land in render cache.
		1521	*
		1522	* However, we do have to enforce the order so that all writes through
		1523	* the GTT land before any writes to the device, such as updates to
		1524	* the GATT itself.
		1525	*/
		1526	wmb();
2332	Serge	1527
2344	Serge	1528	old_write_domain = obj->base.write_domain;
		1529	obj->base.write_domain = 0;
2332	Serge	1530
2344	Serge	1531	}
2332	Serge	1532
		1533	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	1534	static void
2332	Serge	1535	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		1536	{
		1537	uint32_t old_write_domain;
		1538
		1539	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		1540	return;
		1541
		1542	i915_gem_clflush_object(obj);
		1543	intel_gtt_chipset_flush();
		1544	old_write_domain = obj->base.write_domain;
		1545	obj->base.write_domain = 0;
		1546
		1547	}
		1548
		1549	/**
		1550	* Moves a single object to the GTT read, and possibly write domain.
		1551	*
		1552	* This function returns when the move is complete, including waiting on
		1553	* flushes to occur.
		1554	*/
		1555	int
		1556	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		1557	{
		1558	uint32_t old_write_domain, old_read_domains;
		1559	int ret;
		1560
		1561	/* Not valid to be called on unbound objects. */
		1562	if (obj->gtt_space == NULL)
		1563	return -EINVAL;
		1564
		1565	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		1566	return 0;
		1567
		1568	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1569	if (ret)
		1570	return ret;
		1571
		1572	if (obj->pending_gpu_write \|\| write) {
		1573	ret = i915_gem_object_wait_rendering(obj);
		1574	if (ret)
		1575	return ret;
		1576	}
		1577
		1578	i915_gem_object_flush_cpu_write_domain(obj);
		1579
		1580	old_write_domain = obj->base.write_domain;
		1581	old_read_domains = obj->base.read_domains;
		1582
		1583	/* It should now be out of any other write domains, and we can update
		1584	* the domain values for our changes.
		1585	*/
		1586	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1587	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		1588	if (write) {
		1589	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		1590	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		1591	obj->dirty = 1;
		1592	}
		1593
		1594	return 0;
		1595	}
		1596
2335	Serge	1597	#if 0
		1598	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		1599	enum i915_cache_level cache_level)
		1600	{
		1601	int ret;
2332	Serge	1602
2335	Serge	1603	if (obj->cache_level == cache_level)
		1604	return 0;
2332	Serge	1605
2335	Serge	1606	if (obj->pin_count) {
		1607	DRM_DEBUG("can not change the cache level of pinned objects\n");
		1608	return -EBUSY;
		1609	}
2332	Serge	1610
2335	Serge	1611	if (obj->gtt_space) {
		1612	ret = i915_gem_object_finish_gpu(obj);
		1613	if (ret)
		1614	return ret;
2332	Serge	1615
2335	Serge	1616	i915_gem_object_finish_gtt(obj);
2332	Serge	1617
2335	Serge	1618	/* Before SandyBridge, you could not use tiling or fence
		1619	* registers with snooped memory, so relinquish any fences
		1620	* currently pointing to our region in the aperture.
		1621	*/
		1622	if (INTEL_INFO(obj->base.dev)->gen < 6) {
		1623	ret = i915_gem_object_put_fence(obj);
		1624	if (ret)
		1625	return ret;
		1626	}
2332	Serge	1627
2335	Serge	1628	i915_gem_gtt_rebind_object(obj, cache_level);
		1629	}
2332	Serge	1630
2335	Serge	1631	if (cache_level == I915_CACHE_NONE) {
		1632	u32 old_read_domains, old_write_domain;
2332	Serge	1633
2335	Serge	1634	/* If we're coming from LLC cached, then we haven't
		1635	* actually been tracking whether the data is in the
		1636	* CPU cache or not, since we only allow one bit set
		1637	* in obj->write_domain and have been skipping the clflushes.
		1638	* Just set it to the CPU cache for now.
		1639	*/
		1640	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		1641	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	1642
2335	Serge	1643	old_read_domains = obj->base.read_domains;
		1644	old_write_domain = obj->base.write_domain;
2332	Serge	1645
2335	Serge	1646	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1647	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	1648
2344	Serge	1649	}
2332	Serge	1650
2335	Serge	1651	obj->cache_level = cache_level;
		1652	return 0;
		1653	}
		1654	#endif
2332	Serge	1655
2335	Serge	1656	/*
		1657	* Prepare buffer for display plane (scanout, cursors, etc).
		1658	* Can be called from an uninterruptible phase (modesetting) and allows
		1659	* any flushes to be pipelined (for pageflips).
		1660	*
		1661	* For the display plane, we want to be in the GTT but out of any write
		1662	* domains. So in many ways this looks like set_to_gtt_domain() apart from the
		1663	* ability to pipeline the waits, pinning and any additional subtleties
		1664	* that may differentiate the display plane from ordinary buffers.
		1665	*/
		1666	int
		1667	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		1668	u32 alignment,
		1669	struct intel_ring_buffer *pipelined)
		1670	{
		1671	u32 old_read_domains, old_write_domain;
		1672	int ret;
2332	Serge	1673
2335	Serge	1674	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1675	if (ret)
		1676	return ret;
2332	Serge	1677
2335	Serge	1678	if (pipelined != obj->ring) {
		1679	ret = i915_gem_object_wait_rendering(obj);
		1680	if (ret == -ERESTARTSYS)
		1681	return ret;
		1682	}
2332	Serge	1683
2335	Serge	1684	/* The display engine is not coherent with the LLC cache on gen6. As
		1685	* a result, we make sure that the pinning that is about to occur is
		1686	* done with uncached PTEs. This is lowest common denominator for all
		1687	* chipsets.
		1688	*
		1689	* However for gen6+, we could do better by using the GFDT bit instead
		1690	* of uncaching, which would allow us to flush all the LLC-cached data
		1691	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		1692	*/
		1693	// ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		1694	// if (ret)
		1695	// return ret;
2332	Serge	1696
2335	Serge	1697	/* As the user may map the buffer once pinned in the display plane
		1698	* (e.g. libkms for the bootup splash), we have to ensure that we
		1699	* always use map_and_fenceable for all scanout buffers.
		1700	*/
		1701	ret = i915_gem_object_pin(obj, alignment, true);
		1702	if (ret)
		1703	return ret;
2332	Serge	1704
2335	Serge	1705	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	1706
2335	Serge	1707	old_write_domain = obj->base.write_domain;
		1708	old_read_domains = obj->base.read_domains;
2332	Serge	1709
2335	Serge	1710	/* It should now be out of any other write domains, and we can update
		1711	* the domain values for our changes.
		1712	*/
		1713	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1714	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	1715
		1716
2335	Serge	1717	return 0;
		1718	}
2332	Serge	1719
2344	Serge	1720	int
		1721	i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
		1722	{
		1723	int ret;
2332	Serge	1724
2344	Serge	1725	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
		1726	return 0;
2332	Serge	1727
2344	Serge	1728	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
		1729	ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		1730	if (ret)
		1731	return ret;
		1732	}
2332	Serge	1733
2344	Serge	1734	/* Ensure that we invalidate the GPU's caches and TLBs. */
		1735	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2332	Serge	1736
2344	Serge	1737	return i915_gem_object_wait_rendering(obj);
		1738	}
2332	Serge	1739
2344	Serge	1740	/**
		1741	* Moves a single object to the CPU read, and possibly write domain.
		1742	*
		1743	* This function returns when the move is complete, including waiting on
		1744	* flushes to occur.
		1745	*/
		1746	static int
		1747	i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
		1748	{
		1749	uint32_t old_write_domain, old_read_domains;
		1750	int ret;
2332	Serge	1751
2344	Serge	1752	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		1753	return 0;
2332	Serge	1754
2344	Serge	1755	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1756	if (ret)
		1757	return ret;
2332	Serge	1758
2344	Serge	1759	ret = i915_gem_object_wait_rendering(obj);
		1760	if (ret)
		1761	return ret;
2332	Serge	1762
2344	Serge	1763	i915_gem_object_flush_gtt_write_domain(obj);
2332	Serge	1764
2344	Serge	1765	/* If we have a partially-valid cache of the object in the CPU,
		1766	* finish invalidating it and free the per-page flags.
		1767	*/
		1768	i915_gem_object_set_to_full_cpu_read_domain(obj);
2332	Serge	1769
2344	Serge	1770	old_write_domain = obj->base.write_domain;
		1771	old_read_domains = obj->base.read_domains;
2332	Serge	1772
2344	Serge	1773	/* Flush the CPU cache if it's still invalid. */
		1774	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		1775	i915_gem_clflush_object(obj);
2332	Serge	1776
2344	Serge	1777	obj->base.read_domains \|= I915_GEM_DOMAIN_CPU;
		1778	}
2332	Serge	1779
2344	Serge	1780	/* It should now be out of any other write domains, and we can update
		1781	* the domain values for our changes.
		1782	*/
		1783	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2332	Serge	1784
2344	Serge	1785	/* If we're writing through the CPU, then the GPU read domains will
		1786	* need to be invalidated at next use.
		1787	*/
		1788	if (write) {
		1789	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1790	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1791	}
2332	Serge	1792
		1793
2344	Serge	1794	return 0;
		1795	}
2332	Serge	1796
2344	Serge	1797	/**
		1798	* Moves the object from a partially CPU read to a full one.
		1799	*
		1800	* Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
		1801	* and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
		1802	*/
		1803	static void
		1804	i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
		1805	{
		1806	if (!obj->page_cpu_valid)
		1807	return;
2332	Serge	1808
2344	Serge	1809	/* If we're partially in the CPU read domain, finish moving it in.
		1810	*/
		1811	if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
		1812	}
2332	Serge	1813
2344	Serge	1814	/* Free the page_cpu_valid mappings which are now stale, whether
		1815	* or not we've got I915_GEM_DOMAIN_CPU.
		1816	*/
		1817	kfree(obj->page_cpu_valid);
		1818	obj->page_cpu_valid = NULL;
		1819	}
2332	Serge	1820
		1821
		1822
		1823
		1824
		1825
		1826
		1827
		1828
		1829
		1830
		1831
		1832
		1833
		1834
		1835
		1836
		1837
		1838
		1839
		1840
		1841
		1842
2335	Serge	1843
2332	Serge	1844	int
		1845	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		1846	uint32_t alignment,
		1847	bool map_and_fenceable)
		1848	{
		1849	struct drm_device *dev = obj->base.dev;
		1850	struct drm_i915_private *dev_priv = dev->dev_private;
		1851	int ret;
		1852
		1853	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
		1854
		1855	#if 0
		1856	if (obj->gtt_space != NULL) {
		1857	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		1858	(map_and_fenceable && !obj->map_and_fenceable)) {
		1859	WARN(obj->pin_count,
		1860	"bo is already pinned with incorrect alignment:"
		1861	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		1862	" obj->map_and_fenceable=%d\n",
		1863	obj->gtt_offset, alignment,
		1864	map_and_fenceable,
		1865	obj->map_and_fenceable);
		1866	ret = i915_gem_object_unbind(obj);
		1867	if (ret)
		1868	return ret;
		1869	}
		1870	}
		1871	#endif
		1872
		1873	if (obj->gtt_space == NULL) {
		1874	ret = i915_gem_object_bind_to_gtt(obj, alignment,
		1875	map_and_fenceable);
		1876	if (ret)
		1877	return ret;
		1878	}
		1879
		1880	if (obj->pin_count++ == 0) {
		1881	if (!obj->active)
		1882	list_move_tail(&obj->mm_list,
		1883	&dev_priv->mm.pinned_list);
		1884	}
		1885	obj->pin_mappable \|= map_and_fenceable;
		1886
		1887	return 0;
		1888	}
		1889
2344	Serge	1890	void
		1891	i915_gem_object_unpin(struct drm_i915_gem_object *obj)
		1892	{
		1893	struct drm_device *dev = obj->base.dev;
		1894	drm_i915_private_t *dev_priv = dev->dev_private;
2332	Serge	1895
2344	Serge	1896	BUG_ON(obj->pin_count == 0);
		1897	BUG_ON(obj->gtt_space == NULL);
2332	Serge	1898
2344	Serge	1899	if (--obj->pin_count == 0) {
		1900	if (!obj->active)
		1901	list_move_tail(&obj->mm_list,
		1902	&dev_priv->mm.inactive_list);
		1903	obj->pin_mappable = false;
		1904	}
		1905	}
2332	Serge	1906
		1907
		1908
		1909
		1910
		1911
		1912
		1913
		1914
		1915
		1916
		1917
		1918
		1919
		1920
		1921
		1922
		1923
		1924
		1925
		1926
		1927
		1928
		1929
		1930
		1931
		1932
		1933
		1934
		1935
		1936
		1937
		1938	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		1939	size_t size)
		1940	{
		1941	struct drm_i915_private *dev_priv = dev->dev_private;
		1942	struct drm_i915_gem_object *obj;
2340	Serge	1943
2332	Serge	1944	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		1945	if (obj == NULL)
		1946	return NULL;
		1947
		1948	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		1949	kfree(obj);
		1950	return NULL;
		1951	}
		1952
		1953
		1954	i915_gem_info_add_obj(dev_priv, size);
		1955
		1956	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1957	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1958
2342	Serge	1959	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
2332	Serge	1960	/* On Gen6, we can have the GPU use the LLC (the CPU
		1961	* cache) for about a 10% performance improvement
		1962	* compared to uncached. Graphics requests other than
		1963	* display scanout are coherent with the CPU in
		1964	* accessing this cache. This means in this mode we
		1965	* don't need to clflush on the CPU side, and on the
		1966	* GPU side we only need to flush internal caches to
		1967	* get data visible to the CPU.
		1968	*
		1969	* However, we maintain the display planes as UC, and so
		1970	* need to rebind when first used as such.
		1971	*/
		1972	obj->cache_level = I915_CACHE_LLC;
		1973	} else
		1974	obj->cache_level = I915_CACHE_NONE;
		1975
		1976	obj->base.driver_private = NULL;
		1977	obj->fence_reg = I915_FENCE_REG_NONE;
		1978	INIT_LIST_HEAD(&obj->mm_list);
		1979	INIT_LIST_HEAD(&obj->gtt_list);
		1980	INIT_LIST_HEAD(&obj->ring_list);
		1981	INIT_LIST_HEAD(&obj->exec_list);
		1982	INIT_LIST_HEAD(&obj->gpu_write_list);
		1983	obj->madv = I915_MADV_WILLNEED;
		1984	/* Avoid an unnecessary call to unbind on the first bind. */
		1985	obj->map_and_fenceable = true;
2340	Serge	1986
2332	Serge	1987	return obj;
		1988	}
		1989
2344	Serge	1990	int i915_gem_init_object(struct drm_gem_object *obj)
		1991	{
		1992	BUG();
2332	Serge	1993
2344	Serge	1994	return 0;
		1995	}
2332	Serge	1996
2344	Serge	1997	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
		1998	{
		1999	struct drm_device *dev = obj->base.dev;
		2000	drm_i915_private_t *dev_priv = dev->dev_private;
		2001	int ret;
2332	Serge	2002
2344	Serge	2003	ENTER();
2332	Serge	2004
2344	Serge	2005	ret = i915_gem_object_unbind(obj);
		2006	if (ret == -ERESTARTSYS) {
		2007	list_move(&obj->mm_list,
		2008	&dev_priv->mm.deferred_free_list);
		2009	return;
		2010	}
2332	Serge	2011
		2012
2344	Serge	2013	// if (obj->base.map_list.map)
		2014	// drm_gem_free_mmap_offset(&obj->base);
2332	Serge	2015
2344	Serge	2016	drm_gem_object_release(&obj->base);
		2017	i915_gem_info_remove_obj(dev_priv, obj->base.size);
2332	Serge	2018
2344	Serge	2019	kfree(obj->page_cpu_valid);
		2020	kfree(obj->bit_17);
		2021	kfree(obj);
		2022	LEAVE();
		2023	}
2332	Serge	2024
2344	Serge	2025	void i915_gem_free_object(struct drm_gem_object *gem_obj)
		2026	{
		2027	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
		2028	struct drm_device *dev = obj->base.dev;
2332	Serge	2029
2344	Serge	2030	ENTER();
		2031	while (obj->pin_count > 0)
		2032	i915_gem_object_unpin(obj);
2332	Serge	2033
2344	Serge	2034	// if (obj->phys_obj)
		2035	// i915_gem_detach_phys_object(dev, obj);
2332	Serge	2036
2344	Serge	2037	i915_gem_free_object_tail(obj);
		2038	LEAVE();
		2039	}
2332	Serge	2040
		2041
		2042
		2043
		2044
		2045
		2046
		2047
		2048
		2049
		2050
2344	Serge	2051
2332	Serge	2052	int
		2053	i915_gem_init_ringbuffer(struct drm_device *dev)
		2054	{
		2055	drm_i915_private_t *dev_priv = dev->dev_private;
		2056	int ret;
		2057	ENTER();
		2058	ret = intel_init_render_ring_buffer(dev);
		2059	if (ret)
		2060	return ret;
		2061
		2062	if (HAS_BSD(dev)) {
		2063	ret = intel_init_bsd_ring_buffer(dev);
		2064	if (ret)
		2065	goto cleanup_render_ring;
		2066	}
		2067
		2068	if (HAS_BLT(dev)) {
		2069	ret = intel_init_blt_ring_buffer(dev);
		2070	if (ret)
		2071	goto cleanup_bsd_ring;
		2072	}
		2073
		2074	dev_priv->next_seqno = 1;
		2075	LEAVE();
		2076	return 0;
		2077
		2078	cleanup_bsd_ring:
		2079	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		2080	cleanup_render_ring:
		2081	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		2082	return ret;
		2083	}
		2084
		2085	#if 0
		2086	void
		2087	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		2088	{
		2089	drm_i915_private_t *dev_priv = dev->dev_private;
		2090	int i;
		2091
		2092	for (i = 0; i < I915_NUM_RINGS; i++)
		2093	intel_cleanup_ring_buffer(&dev_priv->ring[i]);
		2094	}
		2095
		2096	int
		2097	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		2098	struct drm_file *file_priv)
		2099	{
		2100	drm_i915_private_t *dev_priv = dev->dev_private;
		2101	int ret, i;
		2102
		2103	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2104	return 0;
		2105
		2106	if (atomic_read(&dev_priv->mm.wedged)) {
		2107	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		2108	atomic_set(&dev_priv->mm.wedged, 0);
		2109	}
		2110
		2111	mutex_lock(&dev->struct_mutex);
		2112	dev_priv->mm.suspended = 0;
		2113
		2114	ret = i915_gem_init_ringbuffer(dev);
		2115	if (ret != 0) {
		2116	mutex_unlock(&dev->struct_mutex);
		2117	return ret;
		2118	}
		2119
		2120	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		2121	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
		2122	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
		2123	for (i = 0; i < I915_NUM_RINGS; i++) {
		2124	BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
		2125	BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
		2126	}
		2127	mutex_unlock(&dev->struct_mutex);
		2128
		2129	ret = drm_irq_install(dev);
		2130	if (ret)
		2131	goto cleanup_ringbuffer;
		2132
		2133	return 0;
		2134
		2135	cleanup_ringbuffer:
		2136	mutex_lock(&dev->struct_mutex);
		2137	i915_gem_cleanup_ringbuffer(dev);
		2138	dev_priv->mm.suspended = 1;
		2139	mutex_unlock(&dev->struct_mutex);
		2140
		2141	return ret;
		2142	}
		2143
		2144	int
		2145	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		2146	struct drm_file *file_priv)
		2147	{
		2148	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2149	return 0;
		2150
		2151	drm_irq_uninstall(dev);
		2152	return i915_gem_idle(dev);
		2153	}
		2154
		2155	void
		2156	i915_gem_lastclose(struct drm_device *dev)
		2157	{
		2158	int ret;
		2159
		2160	if (drm_core_check_feature(dev, DRIVER_MODESET))
		2161	return;
		2162
		2163	ret = i915_gem_idle(dev);
		2164	if (ret)
		2165	DRM_ERROR("failed to idle hardware: %d\n", ret);
		2166	}
		2167	#endif
		2168
		2169	static void
2326	Serge	2170	init_ring_lists(struct intel_ring_buffer *ring)
		2171	{
		2172	INIT_LIST_HEAD(&ring->active_list);
		2173	INIT_LIST_HEAD(&ring->request_list);
		2174	INIT_LIST_HEAD(&ring->gpu_write_list);
		2175	}
		2176
		2177	void
		2178	i915_gem_load(struct drm_device *dev)
		2179	{
		2180	int i;
		2181	drm_i915_private_t *dev_priv = dev->dev_private;
		2182
		2183	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		2184	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
		2185	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
		2186	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
		2187	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		2188	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
		2189	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
		2190	for (i = 0; i < I915_NUM_RINGS; i++)
		2191	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	2192	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	2193	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
		2194
		2195	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		2196	if (IS_GEN3(dev)) {
		2197	u32 tmp = I915_READ(MI_ARB_STATE);
		2198	if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
		2199	/* arb state is a masked write, so set bit + bit in mask */
		2200	tmp = MI_ARB_C3_LP_WRITE_ENABLE \| (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
		2201	I915_WRITE(MI_ARB_STATE, tmp);
		2202	}
		2203	}
		2204
		2205	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		2206
		2207	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		2208	dev_priv->num_fence_regs = 16;
		2209	else
		2210	dev_priv->num_fence_regs = 8;
		2211
		2212	/* Initialize fence registers to zero */
		2213	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		2214	i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
		2215	}
		2216
		2217	i915_gem_detect_bit_6_swizzle(dev);
		2218
		2219	dev_priv->mm.interruptible = true;
		2220
		2221	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		2222	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		2223	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		2224	}
		2225
		2226
		2227

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c – Rev 2344