WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_gem.c

Rev	Author	Line No.	Line
2326	Serge	1	/*
		2	* Copyright © 2008 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eric Anholt
		25	*
		26	*/
		27
		28	#include "drmP.h"
		29	#include "drm.h"
2330	Serge	30	#include "i915_drm.h"
2326	Serge	31	#include "i915_drv.h"
		32	//#include "i915_trace.h"
		33	#include "intel_drv.h"
		34	//#include
2330	Serge	35	#include
2326	Serge	36	//#include
		37	#include
		38
2332	Serge	39
		40	#define MAX_ERRNO 4095
		41
		42	#define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO)
		43
		44	static inline long IS_ERR(const void *ptr)
		45	{
		46	return IS_ERR_VALUE((unsigned long)ptr);
		47	}
		48
		49	static inline void *ERR_PTR(long error)
		50	{
		51	return (void *) error;
		52	}
		53
		54	static inline long PTR_ERR(const void *ptr)
		55	{
		56	return (long) ptr;
		57	}
		58
		59
		60	/**
		61	* Initialize an already allocated GEM object of the specified size with
		62	* shmfs backing store.
		63	*/
		64	int drm_gem_object_init(struct drm_device *dev,
		65	struct drm_gem_object *obj, size_t size)
		66	{
		67	BUG_ON((size & (PAGE_SIZE - 1)) != 0);
		68
		69	obj->dev = dev;
		70	atomic_set(&obj->handle_count, 0);
		71	obj->size = size;
		72
		73	return 0;
		74	}
		75
		76
		77
		78
2326	Serge	79	#define I915_EXEC_CONSTANTS_MASK (3<<6)
		80	#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
		81	#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
		82	#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
		83
2332	Serge	84	static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
		85	static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
		86	static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
		87	static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
		88	bool write);
		89	static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
		90	uint64_t offset,
		91	uint64_t size);
		92	static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
		93	static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		94	unsigned alignment,
		95	bool map_and_fenceable);
		96	static void i915_gem_clear_fence_reg(struct drm_device *dev,
		97	struct drm_i915_fence_reg *reg);
		98	static int i915_gem_phys_pwrite(struct drm_device *dev,
		99	struct drm_i915_gem_object *obj,
		100	struct drm_i915_gem_pwrite *args,
		101	struct drm_file *file);
		102	static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
2326	Serge	103
2332	Serge	104	static int i915_gem_inactive_shrink(struct shrinker *shrinker,
		105	struct shrink_control *sc);
		106
		107	/* some bookkeeping */
		108	static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
		109	size_t size)
		110	{
		111	dev_priv->mm.object_count++;
		112	dev_priv->mm.object_memory += size;
		113	}
		114
		115	static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
		116	size_t size)
		117	{
		118	dev_priv->mm.object_count--;
		119	dev_priv->mm.object_memory -= size;
		120	}
		121
		122	#if 0
		123
		124	static int
		125	i915_gem_wait_for_error(struct drm_device *dev)
		126	{
		127	struct drm_i915_private *dev_priv = dev->dev_private;
		128	struct completion *x = &dev_priv->error_completion;
		129	unsigned long flags;
		130	int ret;
		131
		132	if (!atomic_read(&dev_priv->mm.wedged))
		133	return 0;
		134
		135	ret = wait_for_completion_interruptible(x);
		136	if (ret)
		137	return ret;
		138
		139	if (atomic_read(&dev_priv->mm.wedged)) {
		140	/* GPU is hung, bump the completion count to account for
		141	* the token we just consumed so that we never hit zero and
		142	* end up waiting upon a subsequent completion event that
		143	* will never happen.
		144	*/
		145	spin_lock_irqsave(&x->wait.lock, flags);
		146	x->done++;
		147	spin_unlock_irqrestore(&x->wait.lock, flags);
		148	}
		149	return 0;
		150	}
		151
		152	int i915_mutex_lock_interruptible(struct drm_device *dev)
		153	{
		154	int ret;
		155
		156	ret = i915_gem_wait_for_error(dev);
		157	if (ret)
		158	return ret;
		159
		160	ret = mutex_lock_interruptible(&dev->struct_mutex);
		161	if (ret)
		162	return ret;
		163
		164	WARN_ON(i915_verify_lists(dev));
		165	return 0;
		166	}
		167
		168	static inline bool
		169	i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
		170	{
		171	return obj->gtt_space && !obj->active && obj->pin_count == 0;
		172	}
		173
		174	#endif
		175
		176	void i915_gem_do_init(struct drm_device *dev,
		177	unsigned long start,
		178	unsigned long mappable_end,
		179	unsigned long end)
		180	{
		181	drm_i915_private_t *dev_priv = dev->dev_private;
		182
		183	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
		184
		185	dev_priv->mm.gtt_start = start;
		186	dev_priv->mm.gtt_mappable_end = mappable_end;
		187	dev_priv->mm.gtt_end = end;
		188	dev_priv->mm.gtt_total = end - start;
		189	dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
		190
		191	/* Take over this portion of the GTT */
		192	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
		193	}
		194
		195	#if 0
		196
		197	int
		198	i915_gem_init_ioctl(struct drm_device dev, void data,
		199	struct drm_file *file)
		200	{
		201	struct drm_i915_gem_init *args = data;
		202
		203	if (args->gtt_start >= args->gtt_end \|\|
		204	(args->gtt_end \| args->gtt_start) & (PAGE_SIZE - 1))
		205	return -EINVAL;
		206
		207	mutex_lock(&dev->struct_mutex);
		208	i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
		209	mutex_unlock(&dev->struct_mutex);
		210
		211	return 0;
		212	}
		213
		214	int
		215	i915_gem_get_aperture_ioctl(struct drm_device dev, void data,
		216	struct drm_file *file)
		217	{
		218	struct drm_i915_private *dev_priv = dev->dev_private;
		219	struct drm_i915_gem_get_aperture *args = data;
		220	struct drm_i915_gem_object *obj;
		221	size_t pinned;
		222
		223	if (!(dev->driver->driver_features & DRIVER_GEM))
		224	return -ENODEV;
		225
		226	pinned = 0;
		227	mutex_lock(&dev->struct_mutex);
		228	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
		229	pinned += obj->gtt_space->size;
		230	mutex_unlock(&dev->struct_mutex);
		231
		232	args->aper_size = dev_priv->mm.gtt_total;
2342	Serge	233	args->aper_available_size = args->aper_size - pinned;
2332	Serge	234
		235	return 0;
		236	}
		237
		238	static int
		239	i915_gem_create(struct drm_file *file,
		240	struct drm_device *dev,
		241	uint64_t size,
		242	uint32_t *handle_p)
		243	{
		244	struct drm_i915_gem_object *obj;
		245	int ret;
		246	u32 handle;
		247
		248	size = roundup(size, PAGE_SIZE);
2342	Serge	249	if (size == 0)
		250	return -EINVAL;
2332	Serge	251
		252	/* Allocate the new object */
		253	obj = i915_gem_alloc_object(dev, size);
		254	if (obj == NULL)
		255	return -ENOMEM;
		256
		257	ret = drm_gem_handle_create(file, &obj->base, &handle);
		258	if (ret) {
		259	drm_gem_object_release(&obj->base);
		260	i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
		261	kfree(obj);
		262	return ret;
		263	}
		264
		265	/* drop reference from allocate - handle holds it now */
		266	drm_gem_object_unreference(&obj->base);
		267	// trace_i915_gem_object_create(obj);
		268
		269	*handle_p = handle;
		270	return 0;
		271	}
		272
		273	int
		274	i915_gem_dumb_create(struct drm_file *file,
		275	struct drm_device *dev,
		276	struct drm_mode_create_dumb *args)
		277	{
		278	/* have to work out size/pitch and return them */
		279	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
		280	args->size = args->pitch * args->height;
		281	return i915_gem_create(file, dev,
		282	args->size, &args->handle);
		283	}
		284
		285	int i915_gem_dumb_destroy(struct drm_file *file,
		286	struct drm_device *dev,
		287	uint32_t handle)
		288	{
		289	return drm_gem_handle_delete(file, handle);
		290	}
		291
2326	Serge	292	/**
2332	Serge	293	* Creates a new mm object and returns a handle to it.
		294	*/
		295	int
		296	i915_gem_create_ioctl(struct drm_device dev, void data,
		297	struct drm_file *file)
		298	{
		299	struct drm_i915_gem_create *args = data;
		300	return i915_gem_create(file, dev,
		301	args->size, &args->handle);
		302	}
		303
		304	static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
		305	{
		306	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
		307
		308	return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
		309	obj->tiling_mode != I915_TILING_NONE;
		310	}
		311
		312	static inline void
		313	slow_shmem_copy(struct page *dst_page,
		314	int dst_offset,
		315	struct page *src_page,
		316	int src_offset,
		317	int length)
		318	{
		319	char dst_vaddr, src_vaddr;
		320
		321	dst_vaddr = kmap(dst_page);
		322	src_vaddr = kmap(src_page);
		323
		324	memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
		325
		326	kunmap(src_page);
		327	kunmap(dst_page);
		328	}
		329
		330	static inline void
		331	slow_shmem_bit17_copy(struct page *gpu_page,
		332	int gpu_offset,
		333	struct page *cpu_page,
		334	int cpu_offset,
		335	int length,
		336	int is_read)
		337	{
		338	char gpu_vaddr, cpu_vaddr;
		339
		340	/* Use the unswizzled path if this page isn't affected. */
		341	if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
		342	if (is_read)
		343	return slow_shmem_copy(cpu_page, cpu_offset,
		344	gpu_page, gpu_offset, length);
		345	else
		346	return slow_shmem_copy(gpu_page, gpu_offset,
		347	cpu_page, cpu_offset, length);
		348	}
		349
		350	gpu_vaddr = kmap(gpu_page);
		351	cpu_vaddr = kmap(cpu_page);
		352
		353	/* Copy the data, XORing A6 with A17 (1). The user already knows he's
		354	* XORing with the other bits (A9 for Y, A9 and A10 for X)
		355	*/
		356	while (length > 0) {
		357	int cacheline_end = ALIGN(gpu_offset + 1, 64);
		358	int this_length = min(cacheline_end - gpu_offset, length);
		359	int swizzled_gpu_offset = gpu_offset ^ 64;
		360
		361	if (is_read) {
		362	memcpy(cpu_vaddr + cpu_offset,
		363	gpu_vaddr + swizzled_gpu_offset,
		364	this_length);
		365	} else {
		366	memcpy(gpu_vaddr + swizzled_gpu_offset,
		367	cpu_vaddr + cpu_offset,
		368	this_length);
		369	}
		370	cpu_offset += this_length;
		371	gpu_offset += this_length;
		372	length -= this_length;
		373	}
		374
		375	kunmap(cpu_page);
		376	kunmap(gpu_page);
		377	}
		378
		379	/**
		380	* This is the fast shmem pread path, which attempts to copy_from_user directly
		381	* from the backing pages of the object to the user's address space. On a
		382	* fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
		383	*/
		384	static int
		385	i915_gem_shmem_pread_fast(struct drm_device *dev,
		386	struct drm_i915_gem_object *obj,
		387	struct drm_i915_gem_pread *args,
		388	struct drm_file *file)
		389	{
		390	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		391	ssize_t remain;
		392	loff_t offset;
		393	char __user *user_data;
		394	int page_offset, page_length;
		395
		396	user_data = (char __user *) (uintptr_t) args->data_ptr;
		397	remain = args->size;
		398
		399	offset = args->offset;
		400
		401	while (remain > 0) {
		402	struct page *page;
		403	char *vaddr;
		404	int ret;
		405
		406	/* Operation in this page
		407	*
		408	* page_offset = offset within page
		409	* page_length = bytes to copy for this page
		410	*/
		411	page_offset = offset_in_page(offset);
		412	page_length = remain;
		413	if ((page_offset + remain) > PAGE_SIZE)
		414	page_length = PAGE_SIZE - page_offset;
		415
		416	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		417	if (IS_ERR(page))
		418	return PTR_ERR(page);
		419
		420	vaddr = kmap_atomic(page);
		421	ret = __copy_to_user_inatomic(user_data,
		422	vaddr + page_offset,
		423	page_length);
		424	kunmap_atomic(vaddr);
		425
		426	mark_page_accessed(page);
		427	page_cache_release(page);
		428	if (ret)
		429	return -EFAULT;
		430
		431	remain -= page_length;
		432	user_data += page_length;
		433	offset += page_length;
		434	}
		435
		436	return 0;
		437	}
		438
		439	/**
		440	* This is the fallback shmem pread path, which allocates temporary storage
		441	* in kernel space to copy_to_user into outside of the struct_mutex, so we
		442	* can copy out of the object's backing pages while holding the struct mutex
		443	* and not take page faults.
		444	*/
		445	static int
		446	i915_gem_shmem_pread_slow(struct drm_device *dev,
		447	struct drm_i915_gem_object *obj,
		448	struct drm_i915_gem_pread *args,
		449	struct drm_file *file)
		450	{
		451	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
		452	struct mm_struct *mm = current->mm;
		453	struct page **user_pages;
		454	ssize_t remain;
		455	loff_t offset, pinned_pages, i;
		456	loff_t first_data_page, last_data_page, num_pages;
		457	int shmem_page_offset;
		458	int data_page_index, data_page_offset;
		459	int page_length;
		460	int ret;
		461	uint64_t data_ptr = args->data_ptr;
		462	int do_bit17_swizzling;
		463
		464	remain = args->size;
		465
		466	/* Pin the user pages containing the data. We can't fault while
		467	* holding the struct mutex, yet we want to hold it while
		468	* dereferencing the user data.
		469	*/
		470	first_data_page = data_ptr / PAGE_SIZE;
		471	last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
		472	num_pages = last_data_page - first_data_page + 1;
		473
		474	user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
		475	if (user_pages == NULL)
		476	return -ENOMEM;
		477
		478	mutex_unlock(&dev->struct_mutex);
		479	down_read(&mm->mmap_sem);
		480	pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
		481	num_pages, 1, 0, user_pages, NULL);
		482	up_read(&mm->mmap_sem);
		483	mutex_lock(&dev->struct_mutex);
		484	if (pinned_pages < num_pages) {
		485	ret = -EFAULT;
		486	goto out;
		487	}
		488
		489	ret = i915_gem_object_set_cpu_read_domain_range(obj,
		490	args->offset,
		491	args->size);
		492	if (ret)
		493	goto out;
		494
		495	do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
		496
		497	offset = args->offset;
		498
		499	while (remain > 0) {
		500	struct page *page;
		501
		502	/* Operation in this page
		503	*
		504	* shmem_page_offset = offset within page in shmem file
		505	* data_page_index = page number in get_user_pages return
		506	* data_page_offset = offset with data_page_index page.
		507	* page_length = bytes to copy for this page
		508	*/
		509	shmem_page_offset = offset_in_page(offset);
		510	data_page_index = data_ptr / PAGE_SIZE - first_data_page;
		511	data_page_offset = offset_in_page(data_ptr);
		512
		513	page_length = remain;
		514	if ((shmem_page_offset + page_length) > PAGE_SIZE)
		515	page_length = PAGE_SIZE - shmem_page_offset;
		516	if ((data_page_offset + page_length) > PAGE_SIZE)
		517	page_length = PAGE_SIZE - data_page_offset;
		518
		519	page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
		520	if (IS_ERR(page)) {
		521	ret = PTR_ERR(page);
		522	goto out;
		523	}
		524
		525	if (do_bit17_swizzling) {
		526	slow_shmem_bit17_copy(page,
		527	shmem_page_offset,
		528	user_pages[data_page_index],
		529	data_page_offset,
		530	page_length,
		531	1);
		532	} else {
		533	slow_shmem_copy(user_pages[data_page_index],
		534	data_page_offset,
		535	page,
		536	shmem_page_offset,
		537	page_length);
		538	}
		539
		540	mark_page_accessed(page);
		541	page_cache_release(page);
		542
		543	remain -= page_length;
		544	data_ptr += page_length;
		545	offset += page_length;
		546	}
		547
		548	out:
		549	for (i = 0; i < pinned_pages; i++) {
		550	SetPageDirty(user_pages[i]);
		551	mark_page_accessed(user_pages[i]);
		552	page_cache_release(user_pages[i]);
		553	}
		554	drm_free_large(user_pages);
		555
		556	return ret;
		557	}
		558	#endif
		559
		560
		561
		562
		563
		564
		565
		566
		567
		568
		569
		570
		571
		572
		573
		574
		575
		576
		577
		578
		579
		580
		581
		582
		583
		584
		585
		586
		587
		588
		589
		590
		591
		592
		593
		594
		595
		596
		597
		598
		599
		600
		601
		602
		603
		604
		605
		606
		607
		608
		609
		610
		611
		612
		613
		614
		615
		616
		617
		618
		619
		620
		621
		622
		623
		624
		625
		626
		627
		628
		629
		630
		631
		632	static uint32_t
		633	i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
		634	{
		635	uint32_t gtt_size;
		636
		637	if (INTEL_INFO(dev)->gen >= 4 \|\|
		638	tiling_mode == I915_TILING_NONE)
		639	return size;
		640
		641	/* Previous chips need a power-of-two fence region when tiling */
		642	if (INTEL_INFO(dev)->gen == 3)
		643	gtt_size = 1024*1024;
		644	else
		645	gtt_size = 512*1024;
		646
		647	while (gtt_size < size)
		648	gtt_size <<= 1;
		649
		650	return gtt_size;
		651	}
		652
		653	/**
		654	* i915_gem_get_gtt_alignment - return required GTT alignment for an object
		655	* @obj: object to check
		656	*
		657	* Return the required GTT alignment for an object, taking into account
		658	* potential fence register mapping.
		659	*/
		660	static uint32_t
		661	i915_gem_get_gtt_alignment(struct drm_device *dev,
		662	uint32_t size,
		663	int tiling_mode)
		664	{
		665	/*
		666	* Minimum alignment is 4k (GTT page size), but might be greater
		667	* if a fence register is needed for the object.
		668	*/
		669	if (INTEL_INFO(dev)->gen >= 4 \|\|
		670	tiling_mode == I915_TILING_NONE)
		671	return 4096;
		672
		673	/*
		674	* Previous chips need to be aligned to the size of the smallest
		675	* fence register that can contain the object.
		676	*/
		677	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		678	}
		679
		680	/**
		681	* i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
		682	* unfenced object
		683	* @dev: the device
		684	* @size: size of the object
		685	* @tiling_mode: tiling mode of the object
		686	*
		687	* Return the required GTT alignment for an object, only taking into account
		688	* unfenced tiled surface requirements.
		689	*/
		690	uint32_t
		691	i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
		692	uint32_t size,
		693	int tiling_mode)
		694	{
		695	/*
		696	* Minimum alignment is 4k (GTT page size) for sane hw.
		697	*/
		698	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_G33(dev) \|\|
		699	tiling_mode == I915_TILING_NONE)
		700	return 4096;
		701
		702	/* Previous hardware however needs to be aligned to a power-of-two
		703	* tile height. The simplest method for determining this is to reuse
		704	* the power-of-tile object size.
		705	*/
		706	return i915_gem_get_gtt_size(dev, size, tiling_mode);
		707	}
		708
		709
		710
		711
		712
		713
		714
		715
		716
		717
		718
		719
		720
		721
		722
		723
		724
		725	static int
		726	i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
		727	gfp_t gfpmask)
		728	{
		729	int page_count, i;
		730	struct page *page;
		731
		732	/* Get the list of pages out of our struct file. They'll be pinned
		733	* at this point until we release them.
		734	*/
		735	page_count = obj->base.size / PAGE_SIZE;
		736	BUG_ON(obj->pages != NULL);
		737	obj->pages = malloc(page_count * sizeof(struct page *));
		738	if (obj->pages == NULL)
		739	return -ENOMEM;
		740
		741
		742	for (i = 0; i < page_count; i++) {
		743	page = (struct page*)AllocPage(); // oh-oh
		744	if (IS_ERR(page))
		745	goto err_pages;
		746
		747	obj->pages[i] = page;
		748	}
		749
		750	// if (obj->tiling_mode != I915_TILING_NONE)
		751	// i915_gem_object_do_bit_17_swizzle(obj);
		752
		753
2340	Serge	754
2332	Serge	755	return 0;
		756
		757	err_pages:
		758	// while (i--)
		759	// page_cache_release(obj->pages[i]);
		760
		761	free(obj->pages);
		762	obj->pages = NULL;
		763	return PTR_ERR(page);
		764	}
		765
		766	static void
		767	i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
		768	{
		769	int page_count = obj->base.size / PAGE_SIZE;
		770	int i;
		771
		772	BUG_ON(obj->madv == __I915_MADV_PURGED);
		773
		774	// if (obj->tiling_mode != I915_TILING_NONE)
		775	// i915_gem_object_save_bit_17_swizzle(obj);
		776
		777	if (obj->madv == I915_MADV_DONTNEED)
		778	obj->dirty = 0;
		779	/* It's a swap!!!
		780	for (i = 0; i < page_count; i++) {
		781	if (obj->dirty)
		782	set_page_dirty(obj->pages[i]);
		783
		784	if (obj->madv == I915_MADV_WILLNEED)
		785	mark_page_accessed(obj->pages[i]);
		786
		787	//page_cache_release(obj->pages[i]);
		788	}
		789	obj->dirty = 0;
		790	*/
		791
		792	free(obj->pages);
		793	obj->pages = NULL;
		794	}
		795
		796	void
		797	i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
		798	struct intel_ring_buffer *ring,
		799	u32 seqno)
		800	{
		801	struct drm_device *dev = obj->base.dev;
		802	struct drm_i915_private *dev_priv = dev->dev_private;
		803
		804	BUG_ON(ring == NULL);
		805	obj->ring = ring;
		806
		807	/* Add a reference if we're newly entering the active list. */
		808	if (!obj->active) {
		809	// drm_gem_object_reference(&obj->base);
		810	obj->active = 1;
		811	}
		812
		813	/* Move from whatever list we were on to the tail of execution. */
		814	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
		815	list_move_tail(&obj->ring_list, &ring->active_list);
		816
		817	obj->last_rendering_seqno = seqno;
		818	if (obj->fenced_gpu_access) {
		819	struct drm_i915_fence_reg *reg;
		820
		821	BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
		822
		823	obj->last_fenced_seqno = seqno;
		824	obj->last_fenced_ring = ring;
		825
		826	reg = &dev_priv->fence_regs[obj->fence_reg];
		827	list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
		828	}
		829	}
		830
		831
		832
		833
		834	static void
		835	i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
		836	uint32_t flush_domains)
		837	{
		838	struct drm_i915_gem_object obj, next;
		839
		840	list_for_each_entry_safe(obj, next,
		841	&ring->gpu_write_list,
		842	gpu_write_list) {
		843	if (obj->base.write_domain & flush_domains) {
		844	uint32_t old_write_domain = obj->base.write_domain;
		845
		846	obj->base.write_domain = 0;
		847	list_del_init(&obj->gpu_write_list);
		848	i915_gem_object_move_to_active(obj, ring,
		849	i915_gem_next_request_seqno(ring));
		850
		851	// trace_i915_gem_object_change_domain(obj,
		852	// obj->base.read_domains,
		853	// old_write_domain);
		854	}
		855	}
		856	}
		857
		858
		859
		860
		861
		862
		863
		864
		865
		866
		867
		868
		869
		870
		871
		872
		873
		874
		875
		876
		877
		878
		879
		880
		881
		882
		883
		884
		885
		886
		887
		888
		889
		890
		891
		892
		893
		894
		895
		896
		897
		898
		899
		900
		901
		902
		903
		904
		905
		906
		907
		908
		909
		910
		911
		912
		913
		914
		915
		916
		917
		918
		919
		920
		921
		922
		923
		924
		925
		926
		927
		928
		929
		930
		931
		932
		933
		934
		935
		936
		937
		938
		939
		940
		941
		942
		943
		944
		945
		946
		947
		948
		949	/**
		950	* Ensures that all rendering to the object has completed and the object is
		951	* safe to unbind from the GTT or access from the CPU.
		952	*/
		953	int
		954	i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
		955	{
		956	int ret;
		957
		958	/* This function only exists to support waiting for existing rendering,
		959	* not for emitting required flushes.
		960	*/
		961	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
		962
		963	/* If there is rendering queued on the buffer being evicted, wait for
		964	* it.
		965	*/
		966	// if (obj->active) {
		967	// ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
		968	// if (ret)
		969	// return ret;
		970	// }
		971
		972	return 0;
		973	}
		974
		975
		976	int
		977	i915_gem_flush_ring(struct intel_ring_buffer *ring,
		978	uint32_t invalidate_domains,
		979	uint32_t flush_domains)
		980	{
		981	int ret;
		982
		983	if (((invalidate_domains \| flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
		984	return 0;
		985
		986	// trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
		987
		988	ret = ring->flush(ring, invalidate_domains, flush_domains);
		989	if (ret)
		990	return ret;
		991
		992	if (flush_domains & I915_GEM_GPU_DOMAINS)
		993	i915_gem_process_flushing_list(ring, flush_domains);
		994
		995	return 0;
		996	}
		997
		998
		999
		1000
		1001
		1002
		1003
		1004
		1005
		1006
		1007
		1008
		1009
		1010
		1011
		1012
		1013
		1014
		1015
		1016
		1017	/**
2326	Serge	1018	* i915_gem_clear_fence_reg - clear out fence register info
		1019	* @obj: object to clear
		1020	*
		1021	* Zeroes out the fence register itself and clears out the associated
		1022	* data structures in dev_priv and obj.
		1023	*/
		1024	static void
		1025	i915_gem_clear_fence_reg(struct drm_device *dev,
		1026	struct drm_i915_fence_reg *reg)
		1027	{
		1028	drm_i915_private_t *dev_priv = dev->dev_private;
		1029	uint32_t fence_reg = reg - dev_priv->fence_regs;
		1030
		1031	switch (INTEL_INFO(dev)->gen) {
		1032	case 7:
		1033	case 6:
		1034	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
		1035	break;
		1036	case 5:
		1037	case 4:
		1038	I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
		1039	break;
		1040	case 3:
		1041	if (fence_reg >= 8)
		1042	fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
		1043	else
		1044	case 2:
		1045	fence_reg = FENCE_REG_830_0 + fence_reg * 4;
		1046
		1047	I915_WRITE(fence_reg, 0);
		1048	break;
		1049	}
		1050
		1051	list_del_init(®->lru_list);
		1052	reg->obj = NULL;
		1053	reg->setup_seqno = 0;
		1054	}
		1055
2332	Serge	1056	/**
		1057	* Finds free space in the GTT aperture and binds the object there.
		1058	*/
		1059	static int
		1060	i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
		1061	unsigned alignment,
		1062	bool map_and_fenceable)
		1063	{
		1064	struct drm_device *dev = obj->base.dev;
		1065	drm_i915_private_t *dev_priv = dev->dev_private;
		1066	struct drm_mm_node *free_space;
		1067	gfp_t gfpmask = 0; //__GFP_NORETRY \| __GFP_NOWARN;
		1068	u32 size, fence_size, fence_alignment, unfenced_alignment;
		1069	bool mappable, fenceable;
		1070	int ret;
2326	Serge	1071
2332	Serge	1072	if (obj->madv != I915_MADV_WILLNEED) {
		1073	DRM_ERROR("Attempting to bind a purgeable object\n");
		1074	return -EINVAL;
		1075	}
		1076
		1077	fence_size = i915_gem_get_gtt_size(dev,
		1078	obj->base.size,
		1079	obj->tiling_mode);
		1080	fence_alignment = i915_gem_get_gtt_alignment(dev,
		1081	obj->base.size,
		1082	obj->tiling_mode);
		1083	unfenced_alignment =
		1084	i915_gem_get_unfenced_gtt_alignment(dev,
		1085	obj->base.size,
		1086	obj->tiling_mode);
		1087
		1088	if (alignment == 0)
		1089	alignment = map_and_fenceable ? fence_alignment :
		1090	unfenced_alignment;
		1091	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
		1092	DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		1093	return -EINVAL;
		1094	}
		1095
		1096	size = map_and_fenceable ? fence_size : obj->base.size;
		1097
		1098	/* If the object is bigger than the entire aperture, reject it early
		1099	* before evicting everything in a vain attempt to find space.
		1100	*/
		1101	if (obj->base.size >
		1102	(map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
		1103	DRM_ERROR("Attempting to bind an object larger than the aperture\n");
		1104	return -E2BIG;
		1105	}
		1106
		1107	search_free:
		1108	if (map_and_fenceable)
		1109	free_space =
		1110	drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
		1111	size, alignment, 0,
		1112	dev_priv->mm.gtt_mappable_end,
		1113	0);
		1114	else
		1115	free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
		1116	size, alignment, 0);
		1117
		1118	if (free_space != NULL) {
		1119	if (map_and_fenceable)
		1120	obj->gtt_space =
		1121	drm_mm_get_block_range_generic(free_space,
		1122	size, alignment, 0,
		1123	dev_priv->mm.gtt_mappable_end,
		1124	0);
		1125	else
		1126	obj->gtt_space =
		1127	drm_mm_get_block(free_space, size, alignment);
		1128	}
		1129	if (obj->gtt_space == NULL) {
		1130	/* If the gtt is empty and we're still having trouble
		1131	* fitting our object in, we're out of memory.
		1132	*/
		1133	ret = 1; //i915_gem_evict_something(dev, size, alignment,
		1134	// map_and_fenceable);
		1135	if (ret)
		1136	return ret;
		1137
		1138	goto search_free;
		1139	}
		1140
		1141	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
		1142	if (ret) {
		1143	drm_mm_put_block(obj->gtt_space);
		1144	obj->gtt_space = NULL;
		1145	#if 0
		1146	if (ret == -ENOMEM) {
		1147	/* first try to reclaim some memory by clearing the GTT */
		1148	ret = i915_gem_evict_everything(dev, false);
		1149	if (ret) {
		1150	/* now try to shrink everyone else */
		1151	if (gfpmask) {
		1152	gfpmask = 0;
		1153	goto search_free;
		1154	}
		1155
		1156	return -ENOMEM;
		1157	}
		1158
		1159	goto search_free;
		1160	}
		1161	#endif
		1162	return ret;
		1163	}
		1164
		1165	ret = i915_gem_gtt_bind_object(obj);
		1166	if (ret) {
		1167	// i915_gem_object_put_pages_gtt(obj);
		1168	drm_mm_put_block(obj->gtt_space);
		1169	obj->gtt_space = NULL;
		1170
		1171	// if (i915_gem_evict_everything(dev, false))
		1172	return ret;
		1173
		1174	// goto search_free;
		1175	}
		1176
		1177	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
		1178	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
		1179
		1180	/* Assert that the object is not currently in any GPU domain. As it
		1181	* wasn't in the GTT, there shouldn't be any way it could have been in
		1182	* a GPU cache
		1183	*/
		1184	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
		1185	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
		1186
		1187	obj->gtt_offset = obj->gtt_space->start;
		1188
		1189	fenceable =
		1190	obj->gtt_space->size == fence_size &&
2342	Serge	1191	(obj->gtt_space->start & (fence_alignment - 1)) == 0;
2332	Serge	1192
		1193	mappable =
		1194	obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
		1195
		1196	obj->map_and_fenceable = mappable && fenceable;
		1197
		1198	// trace_i915_gem_object_bind(obj, map_and_fenceable);
		1199	return 0;
		1200	}
		1201
		1202	void
		1203	i915_gem_clflush_object(struct drm_i915_gem_object *obj)
		1204	{
		1205	/* If we don't have a page list set up, then we're not pinned
		1206	* to GPU, and we can ignore the cache flush because it'll happen
		1207	* again at bind time.
		1208	*/
		1209	if (obj->pages == NULL)
		1210	return;
		1211
		1212	/* If the GPU is snooping the contents of the CPU cache,
		1213	* we do not need to manually clear the CPU cache lines. However,
		1214	* the caches are only snooped when the render cache is
		1215	* flushed/invalidated. As we always have to emit invalidations
		1216	* and flushes when moving into and out of the RENDER domain, correct
		1217	* snooping behaviour occurs naturally as the result of our domain
		1218	* tracking.
		1219	*/
		1220	if (obj->cache_level != I915_CACHE_NONE)
		1221	return;
		1222
		1223	// trace_i915_gem_object_clflush(obj);
		1224
		1225	// drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
		1226	mb();
		1227	__asm__ ("wbinvd"); // this is really ugly
		1228	mb();
		1229	}
		1230
		1231	/** Flushes any GPU write domain for the object if it's dirty. */
		1232	static int
		1233	i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
		1234	{
		1235	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
		1236	return 0;
		1237
		1238	/* Queue the GPU write cache flushing we need. */
		1239	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
		1240	}
		1241
		1242
		1243
		1244
		1245
		1246
		1247	/** Flushes the CPU write domain for the object if it's dirty. */
2326	Serge	1248	static void
2332	Serge	1249	i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
		1250	{
		1251	uint32_t old_write_domain;
		1252
		1253	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
		1254	return;
		1255
		1256	i915_gem_clflush_object(obj);
		1257	intel_gtt_chipset_flush();
		1258	old_write_domain = obj->base.write_domain;
		1259	obj->base.write_domain = 0;
		1260
		1261	// trace_i915_gem_object_change_domain(obj,
		1262	// obj->base.read_domains,
		1263	// old_write_domain);
		1264	}
		1265
		1266	/**
		1267	* Moves a single object to the GTT read, and possibly write domain.
		1268	*
		1269	* This function returns when the move is complete, including waiting on
		1270	* flushes to occur.
		1271	*/
		1272	int
		1273	i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
		1274	{
		1275	uint32_t old_write_domain, old_read_domains;
		1276	int ret;
		1277
		1278	/* Not valid to be called on unbound objects. */
		1279	if (obj->gtt_space == NULL)
		1280	return -EINVAL;
		1281
		1282	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		1283	return 0;
		1284
		1285	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1286	if (ret)
		1287	return ret;
		1288
		1289	if (obj->pending_gpu_write \|\| write) {
		1290	ret = i915_gem_object_wait_rendering(obj);
		1291	if (ret)
		1292	return ret;
		1293	}
		1294
		1295	i915_gem_object_flush_cpu_write_domain(obj);
		1296
		1297	old_write_domain = obj->base.write_domain;
		1298	old_read_domains = obj->base.read_domains;
		1299
		1300	/* It should now be out of any other write domains, and we can update
		1301	* the domain values for our changes.
		1302	*/
		1303	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1304	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
		1305	if (write) {
		1306	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		1307	obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		1308	obj->dirty = 1;
		1309	}
		1310
		1311	return 0;
		1312	}
		1313
2335	Serge	1314	#if 0
		1315	int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
		1316	enum i915_cache_level cache_level)
		1317	{
		1318	int ret;
2332	Serge	1319
2335	Serge	1320	if (obj->cache_level == cache_level)
		1321	return 0;
2332	Serge	1322
2335	Serge	1323	if (obj->pin_count) {
		1324	DRM_DEBUG("can not change the cache level of pinned objects\n");
		1325	return -EBUSY;
		1326	}
2332	Serge	1327
2335	Serge	1328	if (obj->gtt_space) {
		1329	ret = i915_gem_object_finish_gpu(obj);
		1330	if (ret)
		1331	return ret;
2332	Serge	1332
2335	Serge	1333	i915_gem_object_finish_gtt(obj);
2332	Serge	1334
2335	Serge	1335	/* Before SandyBridge, you could not use tiling or fence
		1336	* registers with snooped memory, so relinquish any fences
		1337	* currently pointing to our region in the aperture.
		1338	*/
		1339	if (INTEL_INFO(obj->base.dev)->gen < 6) {
		1340	ret = i915_gem_object_put_fence(obj);
		1341	if (ret)
		1342	return ret;
		1343	}
2332	Serge	1344
2335	Serge	1345	i915_gem_gtt_rebind_object(obj, cache_level);
		1346	}
2332	Serge	1347
2335	Serge	1348	if (cache_level == I915_CACHE_NONE) {
		1349	u32 old_read_domains, old_write_domain;
2332	Serge	1350
2335	Serge	1351	/* If we're coming from LLC cached, then we haven't
		1352	* actually been tracking whether the data is in the
		1353	* CPU cache or not, since we only allow one bit set
		1354	* in obj->write_domain and have been skipping the clflushes.
		1355	* Just set it to the CPU cache for now.
		1356	*/
		1357	WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		1358	WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2332	Serge	1359
2335	Serge	1360	old_read_domains = obj->base.read_domains;
		1361	old_write_domain = obj->base.write_domain;
2332	Serge	1362
2335	Serge	1363	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1364	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2332	Serge	1365
2335	Serge	1366	trace_i915_gem_object_change_domain(obj,
		1367	old_read_domains,
		1368	old_write_domain);
		1369	}
2332	Serge	1370
2335	Serge	1371	obj->cache_level = cache_level;
		1372	return 0;
		1373	}
		1374	#endif
2332	Serge	1375
2335	Serge	1376	/*
		1377	* Prepare buffer for display plane (scanout, cursors, etc).
		1378	* Can be called from an uninterruptible phase (modesetting) and allows
		1379	* any flushes to be pipelined (for pageflips).
		1380	*
		1381	* For the display plane, we want to be in the GTT but out of any write
		1382	* domains. So in many ways this looks like set_to_gtt_domain() apart from the
		1383	* ability to pipeline the waits, pinning and any additional subtleties
		1384	* that may differentiate the display plane from ordinary buffers.
		1385	*/
		1386	int
		1387	i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
		1388	u32 alignment,
		1389	struct intel_ring_buffer *pipelined)
		1390	{
		1391	u32 old_read_domains, old_write_domain;
		1392	int ret;
2332	Serge	1393
2335	Serge	1394	ret = i915_gem_object_flush_gpu_write_domain(obj);
		1395	if (ret)
		1396	return ret;
2332	Serge	1397
2335	Serge	1398	if (pipelined != obj->ring) {
		1399	ret = i915_gem_object_wait_rendering(obj);
		1400	if (ret == -ERESTARTSYS)
		1401	return ret;
		1402	}
2332	Serge	1403
2335	Serge	1404	/* The display engine is not coherent with the LLC cache on gen6. As
		1405	* a result, we make sure that the pinning that is about to occur is
		1406	* done with uncached PTEs. This is lowest common denominator for all
		1407	* chipsets.
		1408	*
		1409	* However for gen6+, we could do better by using the GFDT bit instead
		1410	* of uncaching, which would allow us to flush all the LLC-cached data
		1411	* with that bit in the PTE to main memory with just one PIPE_CONTROL.
		1412	*/
		1413	// ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
		1414	// if (ret)
		1415	// return ret;
2332	Serge	1416
2335	Serge	1417	/* As the user may map the buffer once pinned in the display plane
		1418	* (e.g. libkms for the bootup splash), we have to ensure that we
		1419	* always use map_and_fenceable for all scanout buffers.
		1420	*/
		1421	ret = i915_gem_object_pin(obj, alignment, true);
		1422	if (ret)
		1423	return ret;
2332	Serge	1424
2335	Serge	1425	i915_gem_object_flush_cpu_write_domain(obj);
2332	Serge	1426
2335	Serge	1427	old_write_domain = obj->base.write_domain;
		1428	old_read_domains = obj->base.read_domains;
2332	Serge	1429
2335	Serge	1430	/* It should now be out of any other write domains, and we can update
		1431	* the domain values for our changes.
		1432	*/
		1433	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
		1434	obj->base.read_domains \|= I915_GEM_DOMAIN_GTT;
2332	Serge	1435
2335	Serge	1436	// trace_i915_gem_object_change_domain(obj,
		1437	// old_read_domains,
		1438	// old_write_domain);
2332	Serge	1439
2335	Serge	1440	return 0;
		1441	}
2332	Serge	1442
		1443
		1444
		1445
		1446
		1447
		1448
		1449
		1450
		1451
		1452
		1453
		1454
		1455
		1456
		1457
		1458
		1459
		1460
		1461
		1462
		1463
		1464
		1465
		1466
		1467
		1468
		1469
		1470
		1471
		1472
		1473
		1474
		1475
		1476
		1477
		1478
		1479
		1480
		1481
		1482
		1483
		1484
		1485
2335	Serge	1486
		1487
2332	Serge	1488	int
		1489	i915_gem_object_pin(struct drm_i915_gem_object *obj,
		1490	uint32_t alignment,
		1491	bool map_and_fenceable)
		1492	{
		1493	struct drm_device *dev = obj->base.dev;
		1494	struct drm_i915_private *dev_priv = dev->dev_private;
		1495	int ret;
		1496
		1497	BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
		1498	// WARN_ON(i915_verify_lists(dev));
		1499
		1500	#if 0
		1501	if (obj->gtt_space != NULL) {
		1502	if ((alignment && obj->gtt_offset & (alignment - 1)) \|\|
		1503	(map_and_fenceable && !obj->map_and_fenceable)) {
		1504	WARN(obj->pin_count,
		1505	"bo is already pinned with incorrect alignment:"
		1506	" offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
		1507	" obj->map_and_fenceable=%d\n",
		1508	obj->gtt_offset, alignment,
		1509	map_and_fenceable,
		1510	obj->map_and_fenceable);
		1511	ret = i915_gem_object_unbind(obj);
		1512	if (ret)
		1513	return ret;
		1514	}
		1515	}
		1516	#endif
		1517
		1518	if (obj->gtt_space == NULL) {
		1519	ret = i915_gem_object_bind_to_gtt(obj, alignment,
		1520	map_and_fenceable);
		1521	if (ret)
		1522	return ret;
		1523	}
		1524
		1525	if (obj->pin_count++ == 0) {
		1526	if (!obj->active)
		1527	list_move_tail(&obj->mm_list,
		1528	&dev_priv->mm.pinned_list);
		1529	}
		1530	obj->pin_mappable \|= map_and_fenceable;
		1531
		1532	// WARN_ON(i915_verify_lists(dev));
		1533	return 0;
		1534	}
		1535
		1536
		1537
		1538
		1539
		1540
		1541
		1542
		1543
		1544
		1545
		1546
		1547
		1548
		1549
		1550
		1551
		1552
		1553
		1554
		1555
		1556
		1557
		1558
		1559
		1560
		1561
		1562
		1563
		1564
		1565
		1566
		1567
		1568
		1569
		1570	struct drm_i915_gem_object i915_gem_alloc_object(struct drm_device dev,
		1571	size_t size)
		1572	{
		1573	struct drm_i915_private *dev_priv = dev->dev_private;
		1574	struct drm_i915_gem_object *obj;
2340	Serge	1575
2332	Serge	1576	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
		1577	if (obj == NULL)
		1578	return NULL;
		1579
		1580	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
		1581	kfree(obj);
		1582	return NULL;
		1583	}
		1584
		1585
		1586	i915_gem_info_add_obj(dev_priv, size);
		1587
		1588	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
		1589	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		1590
2342	Serge	1591	if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
2332	Serge	1592	/* On Gen6, we can have the GPU use the LLC (the CPU
		1593	* cache) for about a 10% performance improvement
		1594	* compared to uncached. Graphics requests other than
		1595	* display scanout are coherent with the CPU in
		1596	* accessing this cache. This means in this mode we
		1597	* don't need to clflush on the CPU side, and on the
		1598	* GPU side we only need to flush internal caches to
		1599	* get data visible to the CPU.
		1600	*
		1601	* However, we maintain the display planes as UC, and so
		1602	* need to rebind when first used as such.
		1603	*/
		1604	obj->cache_level = I915_CACHE_LLC;
		1605	} else
		1606	obj->cache_level = I915_CACHE_NONE;
		1607
		1608	obj->base.driver_private = NULL;
		1609	obj->fence_reg = I915_FENCE_REG_NONE;
		1610	INIT_LIST_HEAD(&obj->mm_list);
		1611	INIT_LIST_HEAD(&obj->gtt_list);
		1612	INIT_LIST_HEAD(&obj->ring_list);
		1613	INIT_LIST_HEAD(&obj->exec_list);
		1614	INIT_LIST_HEAD(&obj->gpu_write_list);
		1615	obj->madv = I915_MADV_WILLNEED;
		1616	/* Avoid an unnecessary call to unbind on the first bind. */
		1617	obj->map_and_fenceable = true;
2340	Serge	1618
2332	Serge	1619	return obj;
		1620	}
		1621
		1622
		1623
		1624
		1625
		1626
		1627
		1628
		1629
		1630
		1631
		1632
		1633
		1634
		1635
		1636
		1637
		1638
		1639
		1640
		1641
		1642
		1643
		1644
		1645	int
		1646	i915_gem_init_ringbuffer(struct drm_device *dev)
		1647	{
		1648	drm_i915_private_t *dev_priv = dev->dev_private;
		1649	int ret;
		1650	ENTER();
		1651	ret = intel_init_render_ring_buffer(dev);
		1652	if (ret)
		1653	return ret;
		1654
		1655	if (HAS_BSD(dev)) {
		1656	ret = intel_init_bsd_ring_buffer(dev);
		1657	if (ret)
		1658	goto cleanup_render_ring;
		1659	}
		1660
		1661	if (HAS_BLT(dev)) {
		1662	ret = intel_init_blt_ring_buffer(dev);
		1663	if (ret)
		1664	goto cleanup_bsd_ring;
		1665	}
		1666
		1667	dev_priv->next_seqno = 1;
		1668	LEAVE();
		1669	return 0;
		1670
		1671	cleanup_bsd_ring:
		1672	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
		1673	cleanup_render_ring:
		1674	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
		1675	return ret;
		1676	}
		1677
		1678	#if 0
		1679	void
		1680	i915_gem_cleanup_ringbuffer(struct drm_device *dev)
		1681	{
		1682	drm_i915_private_t *dev_priv = dev->dev_private;
		1683	int i;
		1684
		1685	for (i = 0; i < I915_NUM_RINGS; i++)
		1686	intel_cleanup_ring_buffer(&dev_priv->ring[i]);
		1687	}
		1688
		1689	int
		1690	i915_gem_entervt_ioctl(struct drm_device dev, void data,
		1691	struct drm_file *file_priv)
		1692	{
		1693	drm_i915_private_t *dev_priv = dev->dev_private;
		1694	int ret, i;
		1695
		1696	if (drm_core_check_feature(dev, DRIVER_MODESET))
		1697	return 0;
		1698
		1699	if (atomic_read(&dev_priv->mm.wedged)) {
		1700	DRM_ERROR("Reenabling wedged hardware, good luck\n");
		1701	atomic_set(&dev_priv->mm.wedged, 0);
		1702	}
		1703
		1704	mutex_lock(&dev->struct_mutex);
		1705	dev_priv->mm.suspended = 0;
		1706
		1707	ret = i915_gem_init_ringbuffer(dev);
		1708	if (ret != 0) {
		1709	mutex_unlock(&dev->struct_mutex);
		1710	return ret;
		1711	}
		1712
		1713	BUG_ON(!list_empty(&dev_priv->mm.active_list));
		1714	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
		1715	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
		1716	for (i = 0; i < I915_NUM_RINGS; i++) {
		1717	BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
		1718	BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
		1719	}
		1720	mutex_unlock(&dev->struct_mutex);
		1721
		1722	ret = drm_irq_install(dev);
		1723	if (ret)
		1724	goto cleanup_ringbuffer;
		1725
		1726	return 0;
		1727
		1728	cleanup_ringbuffer:
		1729	mutex_lock(&dev->struct_mutex);
		1730	i915_gem_cleanup_ringbuffer(dev);
		1731	dev_priv->mm.suspended = 1;
		1732	mutex_unlock(&dev->struct_mutex);
		1733
		1734	return ret;
		1735	}
		1736
		1737	int
		1738	i915_gem_leavevt_ioctl(struct drm_device dev, void data,
		1739	struct drm_file *file_priv)
		1740	{
		1741	if (drm_core_check_feature(dev, DRIVER_MODESET))
		1742	return 0;
		1743
		1744	drm_irq_uninstall(dev);
		1745	return i915_gem_idle(dev);
		1746	}
		1747
		1748	void
		1749	i915_gem_lastclose(struct drm_device *dev)
		1750	{
		1751	int ret;
		1752
		1753	if (drm_core_check_feature(dev, DRIVER_MODESET))
		1754	return;
		1755
		1756	ret = i915_gem_idle(dev);
		1757	if (ret)
		1758	DRM_ERROR("failed to idle hardware: %d\n", ret);
		1759	}
		1760	#endif
		1761
		1762	static void
2326	Serge	1763	init_ring_lists(struct intel_ring_buffer *ring)
		1764	{
		1765	INIT_LIST_HEAD(&ring->active_list);
		1766	INIT_LIST_HEAD(&ring->request_list);
		1767	INIT_LIST_HEAD(&ring->gpu_write_list);
		1768	}
		1769
		1770	void
		1771	i915_gem_load(struct drm_device *dev)
		1772	{
		1773	int i;
		1774	drm_i915_private_t *dev_priv = dev->dev_private;
		1775
		1776	INIT_LIST_HEAD(&dev_priv->mm.active_list);
		1777	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
		1778	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
		1779	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
		1780	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
		1781	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
		1782	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
		1783	for (i = 0; i < I915_NUM_RINGS; i++)
		1784	init_ring_lists(&dev_priv->ring[i]);
2342	Serge	1785	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2326	Serge	1786	INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
		1787	// INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
		1788	// i915_gem_retire_work_handler);
		1789	// init_completion(&dev_priv->error_completion);
		1790
		1791	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		1792	if (IS_GEN3(dev)) {
		1793	u32 tmp = I915_READ(MI_ARB_STATE);
		1794	if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
		1795	/* arb state is a masked write, so set bit + bit in mask */
		1796	tmp = MI_ARB_C3_LP_WRITE_ENABLE \| (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
		1797	I915_WRITE(MI_ARB_STATE, tmp);
		1798	}
		1799	}
		1800
		1801	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
		1802
		1803	if (INTEL_INFO(dev)->gen >= 4 \|\| IS_I945G(dev) \|\| IS_I945GM(dev) \|\| IS_G33(dev))
		1804	dev_priv->num_fence_regs = 16;
		1805	else
		1806	dev_priv->num_fence_regs = 8;
		1807
		1808	/* Initialize fence registers to zero */
		1809	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		1810	i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
		1811	}
		1812
		1813	i915_gem_detect_bit_6_swizzle(dev);
		1814	// init_waitqueue_head(&dev_priv->pending_flip_queue);
		1815
		1816	dev_priv->mm.interruptible = true;
		1817
		1818	// dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
		1819	// dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
		1820	// register_shrinker(&dev_priv->mm.inactive_shrinker);
		1821	}
		1822
		1823
		1824

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_gem.c @ 3482 – Rev 2342