WebSVN – Kolibri OS – Blame – /contrib/sdk/sources/libdrm/intel/intel_bufmgr_gem.c

Rev	Author	Line No.	Line
4363	Serge	1	/**************************************************************************
		2	*
		3	* Copyright � 2007 Red Hat Inc.
		4	* Copyright � 2007-2012 Intel Corporation
		5	* Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
		6	* All Rights Reserved.
		7	*
		8	* Permission is hereby granted, free of charge, to any person obtaining a
		9	* copy of this software and associated documentation files (the
		10	* "Software"), to deal in the Software without restriction, including
		11	* without limitation the rights to use, copy, modify, merge, publish,
		12	* distribute, sub license, and/or sell copies of the Software, and to
		13	* permit persons to whom the Software is furnished to do so, subject to
		14	* the following conditions:
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
		20	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
		21	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
		22	* USE OR OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* The above copyright notice and this permission notice (including the
		25	* next paragraph) shall be included in all copies or substantial portions
		26	* of the Software.
		27	*
		28	*
		29	**************************************************************************/
		30	/*
		31	* Authors: Thomas Hellstr�m
		32	* Keith Whitwell
		33	* Eric Anholt
		34	* Dave Airlie
		35	*/
		36
		37	#ifdef HAVE_CONFIG_H
		38	#include "config.h"
		39	#endif
		40
		41	#include
		42	#include
		43	#include
		44	#include
		45	#include
		46	#include
		47	#include
		48	#include
6667	serge	49	#include
6110	serge	50	#include
		51	#include
4363	Serge	52	#include
		53
		54	#include "errno.h"
		55	#ifndef ETIME
		56	#define ETIME ETIMEDOUT
		57	#endif
6110	serge	58	#include "libdrm_macros.h"
4363	Serge	59	#include "libdrm_lists.h"
		60	#include "intel_bufmgr.h"
		61	#include "intel_bufmgr_priv.h"
		62	#include "intel_chipset.h"
		63	#include "string.h"
		64
		65	#include "i915_drm.h"
		66
		67	#ifdef HAVE_VALGRIND
		68	#include
		69	#include
		70	#define VG(x) x
		71	#else
		72	#define VG(x)
		73	#endif
		74
6110	serge	75	#define memclear(s) memset(&s, 0, sizeof(s))
4363	Serge	76
6667	serge	77
		78	#define pthread_mutex_lock __gthread_mutex_lock
		79	#define pthread_mutex_unlock __gthread_mutex_unlock
		80	#define pthread_mutex_destroy __gthread_mutex_destroy
		81
4363	Serge	82	#if 0
		83	#define DBG(...) do { \
6110	serge	84	if (bufmgr_gem->bufmgr.debug) \
		85	fprintf(stderr, __VA_ARGS__); \
4363	Serge	86	} while (0)
5368	serge	87	#else
		88	#define DBG(...)
4363	Serge	89	#endif
		90
		91	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
6110	serge	92	#define MAX2(A, B) ((A) > (B) ? (A) : (B))
4363	Serge	93
6110	serge	94	/**
		95	* upper_32_bits - return bits 32-63 of a number
		96	* @n: the number we're accessing
		97	*
		98	* A basic shift-right of a 64- or 32-bit quantity. Use this to suppress
		99	* the "right shift count >= width of type" warning when that quantity is
		100	* 32-bits.
		101	*/
		102	#define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))
		103
		104	/**
		105	* lower_32_bits - return bits 0-31 of a number
		106	* @n: the number we're accessing
		107	*/
		108	#define lower_32_bits(n) ((__u32)(n))
		109
4363	Serge	110	typedef struct _drm_intel_bo_gem drm_intel_bo_gem;
		111
		112	struct drm_intel_gem_bo_bucket {
		113	drmMMListHead head;
		114	unsigned long size;
		115	};
		116
		117	typedef struct _drm_intel_bufmgr_gem {
		118	drm_intel_bufmgr bufmgr;
		119
6110	serge	120	atomic_t refcount;
		121
4363	Serge	122	int fd;
		123
		124	int max_relocs;
		125
6667	serge	126	__gthread_mutex_t lock;
4363	Serge	127
		128	struct drm_i915_gem_exec_object *exec_objects;
		129	struct drm_i915_gem_exec_object2 *exec2_objects;
		130	drm_intel_bo **exec_bos;
		131	int exec_size;
		132	int exec_count;
		133
		134	/** Array of lists of cached gem objects of power-of-two sizes */
		135	struct drm_intel_gem_bo_bucket cache_bucket[14 * 4];
		136	int num_buckets;
		137	time_t time;
		138
6110	serge	139	drmMMListHead managers;
		140
4363	Serge	141	drmMMListHead named;
		142	drmMMListHead vma_cache;
		143	int vma_count, vma_open, vma_max;
		144
		145	uint64_t gtt_size;
		146	int available_fences;
		147	int pci_device;
		148	int gen;
		149	unsigned int has_bsd : 1;
		150	unsigned int has_blt : 1;
		151	unsigned int has_relaxed_fencing : 1;
		152	unsigned int has_llc : 1;
		153	unsigned int has_wait_timeout : 1;
		154	unsigned int bo_reuse : 1;
		155	unsigned int no_exec : 1;
		156	unsigned int has_vebox : 1;
		157	bool fenced_relocs;
		158
6110	serge	159	struct {
		160	void *ptr;
		161	uint32_t handle;
		162	} userptr_active;
		163
4363	Serge	164	} drm_intel_bufmgr_gem;
		165
		166	#define DRM_INTEL_RELOC_FENCE (1<<0)
		167
		168	typedef struct _drm_intel_reloc_target_info {
		169	drm_intel_bo *bo;
		170	int flags;
		171	} drm_intel_reloc_target;
		172
		173	struct _drm_intel_bo_gem {
		174	drm_intel_bo bo;
		175
		176	atomic_t refcount;
		177	uint32_t gem_handle;
		178	const char *name;
		179
		180	/**
		181	* Kenel-assigned global name for this object
		182	*
		183	* List contains both flink named and prime fd'd objects
		184	*/
		185	unsigned int global_name;
		186	drmMMListHead name_list;
		187
		188	/**
		189	* Index of the buffer within the validation list while preparing a
		190	* batchbuffer execution.
		191	*/
		192	int validate_index;
		193
		194	/**
		195	* Current tiling mode
		196	*/
		197	uint32_t tiling_mode;
		198	uint32_t swizzle_mode;
		199	unsigned long stride;
		200
		201	time_t free_time;
		202
		203	/** Array passed to the DRM containing relocation information. */
		204	struct drm_i915_gem_relocation_entry *relocs;
		205	/**
		206	* Array of info structs corresponding to relocs[i].target_handle etc
		207	*/
		208	drm_intel_reloc_target *reloc_target_info;
		209	/** Number of entries in relocs */
		210	int reloc_count;
6110	serge	211	/** Array of BOs that are referenced by this buffer and will be softpinned */
		212	drm_intel_bo **softpin_target;
		213	/** Number softpinned BOs that are referenced by this buffer */
		214	int softpin_target_count;
		215	/** Maximum amount of softpinned BOs that are referenced by this buffer */
		216	int softpin_target_size;
		217
4363	Serge	218	/** Mapped address for the buffer, saved across map/unmap cycles */
		219	void *mem_virtual;
		220	/** GTT virtual address for the buffer, saved across map/unmap cycles */
		221	void *gtt_virtual;
6110	serge	222	/**
		223	* Virtual address of the buffer allocated by user, used for userptr
		224	* objects only.
		225	*/
		226	void *user_virtual;
4363	Serge	227	int map_count;
		228	drmMMListHead vma_list;
		229
		230	/** BO cache list */
		231	drmMMListHead head;
		232
		233	/**
		234	* Boolean of whether this BO and its children have been included in
		235	* the current drm_intel_bufmgr_check_aperture_space() total.
		236	*/
		237	bool included_in_check_aperture;
		238
		239	/**
		240	* Boolean of whether this buffer has been used as a relocation
		241	* target and had its size accounted for, and thus can't have any
		242	* further relocations added to it.
		243	*/
		244	bool used_as_reloc_target;
		245
		246	/**
		247	* Boolean of whether we have encountered an error whilst building the relocation tree.
		248	*/
		249	bool has_error;
		250
		251	/**
		252	* Boolean of whether this buffer can be re-used
		253	*/
		254	bool reusable;
		255
		256	/**
5068	serge	257	* Boolean of whether the GPU is definitely not accessing the buffer.
		258	*
		259	* This is only valid when reusable, since non-reusable
		260	* buffers are those that have been shared wth other
		261	* processes, so we don't know their state.
		262	*/
		263	bool idle;
		264
		265	/**
6110	serge	266	* Boolean of whether this buffer was allocated with userptr
		267	*/
		268	bool is_userptr;
		269
		270	/**
		271	* Boolean of whether this buffer can be placed in the full 48-bit
		272	* address range on gen8+.
		273	*
		274	* By default, buffers will be keep in a 32-bit range, unless this
		275	* flag is explicitly set.
		276	*/
		277	bool use_48b_address_range;
		278
		279	/**
		280	* Whether this buffer is softpinned at offset specified by the user
		281	*/
		282	bool is_softpin;
		283
		284	/**
4363	Serge	285	* Size in bytes of this buffer and its relocation descendents.
		286	*
		287	* Used to avoid costly tree walking in
		288	* drm_intel_bufmgr_check_aperture in the common case.
		289	*/
		290	int reloc_tree_size;
		291
		292	/**
		293	* Number of potential fence registers required by this buffer and its
		294	* relocations.
		295	*/
		296	int reloc_tree_fences;
		297
		298	/** Flags that we may need to do the SW_FINSIH ioctl on unmap. */
		299	bool mapped_cpu_write;
		300	};
		301
		302	static unsigned int
		303	drm_intel_gem_estimate_batch_space(drm_intel_bo ** bo_array, int count);
		304
		305	static unsigned int
		306	drm_intel_gem_compute_batch_space(drm_intel_bo ** bo_array, int count);
		307
		308	static int
		309	drm_intel_gem_bo_get_tiling(drm_intel_bo bo, uint32_t tiling_mode,
		310	uint32_t * swizzle_mode);
		311
		312	static int
		313	drm_intel_gem_bo_set_tiling_internal(drm_intel_bo *bo,
		314	uint32_t tiling_mode,
		315	uint32_t stride);
		316
		317	static void drm_intel_gem_bo_unreference_locked_timed(drm_intel_bo *bo,
		318	time_t time);
		319
		320	static void drm_intel_gem_bo_unreference(drm_intel_bo *bo);
		321
		322	static void drm_intel_gem_bo_free(drm_intel_bo *bo);
		323
6110	serge	324	static inline drm_intel_bo_gem to_bo_gem(drm_intel_bo bo)
		325	{
		326	return (drm_intel_bo_gem *)bo;
		327	}
		328
4363	Serge	329	static unsigned long
		330	drm_intel_gem_bo_tile_size(drm_intel_bufmgr_gem *bufmgr_gem, unsigned long size,
		331	uint32_t *tiling_mode)
		332	{
		333	unsigned long min_size, max_size;
		334	unsigned long i;
		335
		336	if (*tiling_mode == I915_TILING_NONE)
		337	return size;
		338
		339	/* 965+ just need multiples of page size for tiling */
		340	if (bufmgr_gem->gen >= 4)
		341	return ROUND_UP_TO(size, 4096);
		342
		343	/* Older chips need powers of two, of at least 512k or 1M */
		344	if (bufmgr_gem->gen == 3) {
		345	min_size = 1024*1024;
		346	max_size = 12810241024;
		347	} else {
		348	min_size = 512*1024;
		349	max_size = 6410241024;
		350	}
		351
		352	if (size > max_size) {
		353	*tiling_mode = I915_TILING_NONE;
		354	return size;
		355	}
		356
		357	/* Do we need to allocate every page for the fence? */
		358	if (bufmgr_gem->has_relaxed_fencing)
		359	return ROUND_UP_TO(size, 4096);
		360
		361	for (i = min_size; i < size; i <<= 1)
		362	;
		363
		364	return i;
		365	}
		366
		367	/*
		368	* Round a given pitch up to the minimum required for X tiling on a
		369	* given chip. We use 512 as the minimum to allow for a later tiling
		370	* change.
		371	*/
		372	static unsigned long
		373	drm_intel_gem_bo_tile_pitch(drm_intel_bufmgr_gem *bufmgr_gem,
		374	unsigned long pitch, uint32_t *tiling_mode)
		375	{
		376	unsigned long tile_width;
		377	unsigned long i;
		378
		379	/* If untiled, then just align it so that we can do rendering
		380	* to it with the 3D engine.
		381	*/
		382	if (*tiling_mode == I915_TILING_NONE)
		383	return ALIGN(pitch, 64);
		384
		385	if (*tiling_mode == I915_TILING_X
		386	\|\| (IS_915(bufmgr_gem->pci_device)
		387	&& *tiling_mode == I915_TILING_Y))
		388	tile_width = 512;
		389	else
		390	tile_width = 128;
		391
		392	/* 965 is flexible */
		393	if (bufmgr_gem->gen >= 4)
		394	return ROUND_UP_TO(pitch, tile_width);
		395
		396	/* The older hardware has a maximum pitch of 8192 with tiled
		397	* surfaces, so fallback to untiled if it's too large.
		398	*/
		399	if (pitch > 8192) {
		400	*tiling_mode = I915_TILING_NONE;
		401	return ALIGN(pitch, 64);
		402	}
		403
		404	/* Pre-965 needs power of two tile width */
		405	for (i = tile_width; i < pitch; i <<= 1)
		406	;
		407
		408	return i;
		409	}
		410
		411	static struct drm_intel_gem_bo_bucket *
		412	drm_intel_gem_bo_bucket_for_size(drm_intel_bufmgr_gem *bufmgr_gem,
		413	unsigned long size)
		414	{
		415	int i;
		416
		417	for (i = 0; i < bufmgr_gem->num_buckets; i++) {
		418	struct drm_intel_gem_bo_bucket *bucket =
		419	&bufmgr_gem->cache_bucket[i];
		420	if (bucket->size >= size) {
		421	return bucket;
		422	}
		423	}
		424
		425	return NULL;
		426	}
		427
		428	static void
		429	drm_intel_gem_dump_validation_list(drm_intel_bufmgr_gem *bufmgr_gem)
		430	{
		431	int i, j;
		432
		433	for (i = 0; i < bufmgr_gem->exec_count; i++) {
		434	drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
		435	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		436
6110	serge	437	if (bo_gem->relocs == NULL && bo_gem->softpin_target == NULL) {
		438	DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,
		439	bo_gem->is_softpin ? "*" : "",
4363	Serge	440	bo_gem->name);
		441	continue;
		442	}
		443
		444	for (j = 0; j < bo_gem->reloc_count; j++) {
		445	drm_intel_bo *target_bo = bo_gem->reloc_target_info[j].bo;
		446	drm_intel_bo_gem *target_gem =
		447	(drm_intel_bo_gem *) target_bo;
		448
6110	serge	449	DBG("%2d: %d %s(%s)@0x%08x %08x -> "
		450	"%d (%s)@0x%08x %08x + 0x%08x\n",
4363	Serge	451	i,
6110	serge	452	bo_gem->gem_handle,
		453	bo_gem->is_softpin ? "*" : "",
		454	bo_gem->name,
		455	upper_32_bits(bo_gem->relocs[j].offset),
		456	lower_32_bits(bo_gem->relocs[j].offset),
4363	Serge	457	target_gem->gem_handle,
		458	target_gem->name,
6110	serge	459	upper_32_bits(target_bo->offset64),
		460	lower_32_bits(target_bo->offset64),
4363	Serge	461	bo_gem->relocs[j].delta);
		462	}
6110	serge	463
		464	for (j = 0; j < bo_gem->softpin_target_count; j++) {
		465	drm_intel_bo *target_bo = bo_gem->softpin_target[j];
		466	drm_intel_bo_gem *target_gem =
		467	(drm_intel_bo_gem *) target_bo;
		468	DBG("%2d: %d %s(%s) -> "
		469	"%d *(%s)@0x%08x %08x\n",
		470	i,
		471	bo_gem->gem_handle,
		472	bo_gem->is_softpin ? "*" : "",
		473	bo_gem->name,
		474	target_gem->gem_handle,
		475	target_gem->name,
		476	upper_32_bits(target_bo->offset64),
		477	lower_32_bits(target_bo->offset64));
		478	}
4363	Serge	479	}
		480	}
		481
		482	static inline void
		483	drm_intel_gem_bo_reference(drm_intel_bo *bo)
		484	{
		485	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		486
		487	atomic_inc(&bo_gem->refcount);
		488	}
		489
		490	/**
		491	* Adds the given buffer to the list of buffers to be validated (moved into the
		492	* appropriate memory type) with the next batch submission.
		493	*
		494	* If a buffer is validated multiple times in a batch submission, it ends up
		495	* with the intersection of the memory type flags and the union of the
		496	* access flags.
		497	*/
		498	static void
		499	drm_intel_add_validate_buffer(drm_intel_bo *bo)
		500	{
		501	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		502	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		503	int index;
		504
		505	if (bo_gem->validate_index != -1)
		506	return;
		507
		508	/* Extend the array of validation entries as necessary. */
		509	if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
		510	int new_size = bufmgr_gem->exec_size * 2;
		511
		512	if (new_size == 0)
		513	new_size = 5;
		514
		515	bufmgr_gem->exec_objects =
		516	realloc(bufmgr_gem->exec_objects,
		517	sizeof(bufmgr_gem->exec_objects) new_size);
		518	bufmgr_gem->exec_bos =
		519	realloc(bufmgr_gem->exec_bos,
		520	sizeof(bufmgr_gem->exec_bos) new_size);
		521	bufmgr_gem->exec_size = new_size;
		522	}
		523
		524	index = bufmgr_gem->exec_count;
		525	bo_gem->validate_index = index;
		526	/* Fill in array entry */
		527	bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
		528	bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
		529	bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;
6110	serge	530	bufmgr_gem->exec_objects[index].alignment = bo->align;
4363	Serge	531	bufmgr_gem->exec_objects[index].offset = 0;
		532	bufmgr_gem->exec_bos[index] = bo;
		533	bufmgr_gem->exec_count++;
		534	}
		535
		536	static void
		537	drm_intel_add_validate_buffer2(drm_intel_bo *bo, int need_fence)
		538	{
		539	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bo->bufmgr;
		540	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem )bo;
		541	int index;
6110	serge	542	int flags = 0;
4363	Serge	543
6110	serge	544	if (need_fence)
		545	flags \|= EXEC_OBJECT_NEEDS_FENCE;
		546	if (bo_gem->use_48b_address_range)
		547	flags \|= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
		548	if (bo_gem->is_softpin)
		549	flags \|= EXEC_OBJECT_PINNED;
		550
4363	Serge	551	if (bo_gem->validate_index != -1) {
6110	serge	552	bufmgr_gem->exec2_objects[bo_gem->validate_index].flags \|= flags;
4363	Serge	553	return;
		554	}
		555
		556	/* Extend the array of validation entries as necessary. */
		557	if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
		558	int new_size = bufmgr_gem->exec_size * 2;
		559
		560	if (new_size == 0)
		561	new_size = 5;
		562
		563	bufmgr_gem->exec2_objects =
		564	realloc(bufmgr_gem->exec2_objects,
		565	sizeof(bufmgr_gem->exec2_objects) new_size);
		566	bufmgr_gem->exec_bos =
		567	realloc(bufmgr_gem->exec_bos,
		568	sizeof(bufmgr_gem->exec_bos) new_size);
		569	bufmgr_gem->exec_size = new_size;
		570	}
		571
		572	index = bufmgr_gem->exec_count;
		573	bo_gem->validate_index = index;
		574	/* Fill in array entry */
		575	bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
		576	bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
		577	bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
6110	serge	578	bufmgr_gem->exec2_objects[index].alignment = bo->align;
		579	bufmgr_gem->exec2_objects[index].offset = bo_gem->is_softpin ?
		580	bo->offset64 : 0;
4363	Serge	581	bufmgr_gem->exec_bos[index] = bo;
6110	serge	582	bufmgr_gem->exec2_objects[index].flags = flags;
4363	Serge	583	bufmgr_gem->exec2_objects[index].rsvd1 = 0;
		584	bufmgr_gem->exec2_objects[index].rsvd2 = 0;
		585	bufmgr_gem->exec_count++;
		586	}
		587
		588	#define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
		589	sizeof(uint32_t))
		590
		591	static void
		592	drm_intel_bo_gem_set_in_aperture_size(drm_intel_bufmgr_gem *bufmgr_gem,
6110	serge	593	drm_intel_bo_gem *bo_gem,
		594	unsigned int alignment)
4363	Serge	595	{
6110	serge	596	unsigned int size;
4363	Serge	597
		598	assert(!bo_gem->used_as_reloc_target);
		599
		600	/* The older chipsets are far-less flexible in terms of tiling,
		601	* and require tiled buffer to be size aligned in the aperture.
		602	* This means that in the worst possible case we will need a hole
		603	* twice as large as the object in order for it to fit into the
		604	* aperture. Optimal packing is for wimps.
		605	*/
		606	size = bo_gem->bo.size;
		607	if (bufmgr_gem->gen < 4 && bo_gem->tiling_mode != I915_TILING_NONE) {
6110	serge	608	unsigned int min_size;
4363	Serge	609
		610	if (bufmgr_gem->has_relaxed_fencing) {
		611	if (bufmgr_gem->gen == 3)
		612	min_size = 1024*1024;
		613	else
		614	min_size = 512*1024;
		615
		616	while (min_size < size)
		617	min_size *= 2;
		618	} else
		619	min_size = size;
		620
		621	/* Account for worst-case alignment. */
6110	serge	622	alignment = MAX2(alignment, min_size);
4363	Serge	623	}
		624
6110	serge	625	bo_gem->reloc_tree_size = size + alignment;
4363	Serge	626	}
		627
		628	static int
		629	drm_intel_setup_reloc_list(drm_intel_bo *bo)
		630	{
		631	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		632	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		633	unsigned int max_relocs = bufmgr_gem->max_relocs;
		634
		635	if (bo->size / 4 < max_relocs)
		636	max_relocs = bo->size / 4;
		637
		638	bo_gem->relocs = malloc(max_relocs *
		639	sizeof(struct drm_i915_gem_relocation_entry));
		640	bo_gem->reloc_target_info = malloc(max_relocs *
		641	sizeof(drm_intel_reloc_target));
		642	if (bo_gem->relocs == NULL \|\| bo_gem->reloc_target_info == NULL) {
		643	bo_gem->has_error = true;
		644
		645	free (bo_gem->relocs);
		646	bo_gem->relocs = NULL;
		647
		648	free (bo_gem->reloc_target_info);
		649	bo_gem->reloc_target_info = NULL;
		650
		651	return 1;
		652	}
		653
		654	return 0;
		655	}
		656
		657	static int
		658	drm_intel_gem_bo_busy(drm_intel_bo *bo)
		659	{
		660	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		661	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		662	struct drm_i915_gem_busy busy;
		663	int ret;
		664
5068	serge	665	if (bo_gem->reusable && bo_gem->idle)
		666	return false;
		667
6110	serge	668	memclear(busy);
4363	Serge	669	busy.handle = bo_gem->gem_handle;
		670
		671	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
5068	serge	672	if (ret == 0) {
		673	bo_gem->idle = !busy.busy;
		674	return busy.busy;
		675	} else {
		676	return false;
		677	}
4363	Serge	678	return (ret == 0 && busy.busy);
		679	}
		680
		681	static int
		682	drm_intel_gem_bo_madvise_internal(drm_intel_bufmgr_gem *bufmgr_gem,
		683	drm_intel_bo_gem *bo_gem, int state)
		684	{
		685	struct drm_i915_gem_madvise madv;
		686
6110	serge	687	memclear(madv);
4363	Serge	688	madv.handle = bo_gem->gem_handle;
		689	madv.madv = state;
		690	madv.retained = 1;
		691	// drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
		692
		693	return madv.retained;
		694	}
		695
		696	static int
		697	drm_intel_gem_bo_madvise(drm_intel_bo *bo, int madv)
		698	{
		699	return drm_intel_gem_bo_madvise_internal
		700	((drm_intel_bufmgr_gem *) bo->bufmgr,
		701	(drm_intel_bo_gem *) bo,
		702	madv);
		703	}
		704
		705	/* drop the oldest entries that have been purged by the kernel */
		706	static void
		707	drm_intel_gem_bo_cache_purge_bucket(drm_intel_bufmgr_gem *bufmgr_gem,
		708	struct drm_intel_gem_bo_bucket *bucket)
		709	{
		710	while (!DRMLISTEMPTY(&bucket->head)) {
		711	drm_intel_bo_gem *bo_gem;
		712
		713	bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
		714	bucket->head.next, head);
		715	if (drm_intel_gem_bo_madvise_internal
		716	(bufmgr_gem, bo_gem, I915_MADV_DONTNEED))
		717	break;
		718
		719	DRMLISTDEL(&bo_gem->head);
		720	drm_intel_gem_bo_free(&bo_gem->bo);
		721	}
		722	}
		723
		724	static drm_intel_bo *
		725	drm_intel_gem_bo_alloc_internal(drm_intel_bufmgr *bufmgr,
		726	const char *name,
		727	unsigned long size,
		728	unsigned long flags,
		729	uint32_t tiling_mode,
6110	serge	730	unsigned long stride,
		731	unsigned int alignment)
4363	Serge	732	{
		733	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		734	drm_intel_bo_gem *bo_gem;
		735	unsigned int page_size = 4096;
		736	int ret;
		737	struct drm_intel_gem_bo_bucket *bucket;
		738	bool alloc_from_cache;
		739	unsigned long bo_size;
		740	bool for_render = false;
		741
		742	if (flags & BO_ALLOC_FOR_RENDER)
		743	for_render = true;
		744
		745	/* Round the allocated size up to a power of two number of pages. */
		746	bucket = drm_intel_gem_bo_bucket_for_size(bufmgr_gem, size);
		747
		748	/* If we don't have caching at this size, don't actually round the
		749	* allocation up.
		750	*/
		751	if (bucket == NULL) {
		752	bo_size = size;
		753	if (bo_size < page_size)
		754	bo_size = page_size;
		755	} else {
		756	bo_size = bucket->size;
		757	}
		758
6667	serge	759	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	760	/* Get a buffer out of the cache if available */
		761	retry:
		762	alloc_from_cache = false;
		763	if (bucket != NULL && !DRMLISTEMPTY(&bucket->head)) {
		764	if (for_render) {
		765	/* Allocate new render-target BOs from the tail (MRU)
		766	* of the list, as it will likely be hot in the GPU
		767	* cache and in the aperture for us.
		768	*/
		769	bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
		770	bucket->head.prev, head);
		771	DRMLISTDEL(&bo_gem->head);
		772	alloc_from_cache = true;
6110	serge	773	bo_gem->bo.align = alignment;
4363	Serge	774	} else {
6110	serge	775	assert(alignment == 0);
4363	Serge	776	/* For non-render-target BOs (where we're probably
		777	* going to map it first thing in order to fill it
		778	* with data), check if the last BO in the cache is
		779	* unbusy, and only reuse in that case. Otherwise,
		780	* allocating a new buffer is probably faster than
		781	* waiting for the GPU to finish.
		782	*/
		783	bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
		784	bucket->head.next, head);
		785	if (!drm_intel_gem_bo_busy(&bo_gem->bo)) {
		786	alloc_from_cache = true;
		787	DRMLISTDEL(&bo_gem->head);
		788	}
		789	}
		790
		791	if (alloc_from_cache) {
		792	if (!drm_intel_gem_bo_madvise_internal
		793	(bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {
		794	drm_intel_gem_bo_free(&bo_gem->bo);
		795	drm_intel_gem_bo_cache_purge_bucket(bufmgr_gem,
		796	bucket);
		797	goto retry;
		798	}
		799
		800	if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,
		801	tiling_mode,
		802	stride)) {
		803	drm_intel_gem_bo_free(&bo_gem->bo);
		804	goto retry;
		805	}
		806	}
		807	}
6667	serge	808	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	809
		810	if (!alloc_from_cache) {
		811	struct drm_i915_gem_create create;
		812
		813	bo_gem = calloc(1, sizeof(*bo_gem));
		814	if (!bo_gem)
		815	return NULL;
		816
		817	bo_gem->bo.size = bo_size;
		818
6110	serge	819	memclear(create);
4363	Serge	820	create.size = bo_size;
		821
		822	ret = drmIoctl(bufmgr_gem->fd,
		823	DRM_IOCTL_I915_GEM_CREATE,
		824	&create);
		825	bo_gem->gem_handle = create.handle;
		826	bo_gem->bo.handle = bo_gem->gem_handle;
		827	if (ret != 0) {
		828	free(bo_gem);
		829	return NULL;
		830	}
		831	bo_gem->bo.bufmgr = bufmgr;
6110	serge	832	bo_gem->bo.align = alignment;
4363	Serge	833
		834	bo_gem->tiling_mode = I915_TILING_NONE;
		835	bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		836	bo_gem->stride = 0;
		837
6110	serge	838	/* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
		839	list (vma_list), so better set the list head here */
		840	DRMINITLISTHEAD(&bo_gem->name_list);
		841	DRMINITLISTHEAD(&bo_gem->vma_list);
4363	Serge	842	if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,
		843	tiling_mode,
		844	stride)) {
		845	drm_intel_gem_bo_free(&bo_gem->bo);
		846	return NULL;
		847	}
		848	}
		849
		850	bo_gem->name = name;
		851	atomic_set(&bo_gem->refcount, 1);
		852	bo_gem->validate_index = -1;
		853	bo_gem->reloc_tree_fences = 0;
		854	bo_gem->used_as_reloc_target = false;
		855	bo_gem->has_error = false;
		856	bo_gem->reusable = true;
6110	serge	857	bo_gem->use_48b_address_range = false;
4363	Serge	858
6110	serge	859	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alignment);
4363	Serge	860
		861	DBG("bo_create: buf %d (%s) %ldb\n",
		862	bo_gem->gem_handle, bo_gem->name, size);
		863
		864	return &bo_gem->bo;
		865	}
		866
		867	static drm_intel_bo *
		868	drm_intel_gem_bo_alloc_for_render(drm_intel_bufmgr *bufmgr,
		869	const char *name,
		870	unsigned long size,
		871	unsigned int alignment)
		872	{
		873	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size,
		874	BO_ALLOC_FOR_RENDER,
6110	serge	875	I915_TILING_NONE, 0,
		876	alignment);
4363	Serge	877	}
		878
		879	static drm_intel_bo *
		880	drm_intel_gem_bo_alloc(drm_intel_bufmgr *bufmgr,
		881	const char *name,
		882	unsigned long size,
		883	unsigned int alignment)
		884	{
		885	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, 0,
6110	serge	886	I915_TILING_NONE, 0, 0);
4363	Serge	887	}
		888
		889	static drm_intel_bo *
		890	drm_intel_gem_bo_alloc_tiled(drm_intel_bufmgr bufmgr, const char name,
		891	int x, int y, int cpp, uint32_t *tiling_mode,
		892	unsigned long *pitch, unsigned long flags)
		893	{
		894	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		895	unsigned long size, stride;
		896	uint32_t tiling;
		897
		898	do {
		899	unsigned long aligned_y, height_alignment;
		900
		901	tiling = *tiling_mode;
		902
		903	/* If we're tiled, our allocations are in 8 or 32-row blocks,
		904	* so failure to align our height means that we won't allocate
		905	* enough pages.
		906	*
		907	* If we're untiled, we still have to align to 2 rows high
		908	* because the data port accesses 2x2 blocks even if the
		909	* bottom row isn't to be rendered, so failure to align means
		910	* we could walk off the end of the GTT and fault. This is
		911	* documented on 965, and may be the case on older chipsets
		912	* too so we try to be careful.
		913	*/
		914	aligned_y = y;
		915	height_alignment = 2;
		916
		917	if ((bufmgr_gem->gen == 2) && tiling != I915_TILING_NONE)
		918	height_alignment = 16;
		919	else if (tiling == I915_TILING_X
		920	\|\| (IS_915(bufmgr_gem->pci_device)
		921	&& tiling == I915_TILING_Y))
		922	height_alignment = 8;
		923	else if (tiling == I915_TILING_Y)
		924	height_alignment = 32;
		925	aligned_y = ALIGN(y, height_alignment);
		926
		927	stride = x * cpp;
		928	stride = drm_intel_gem_bo_tile_pitch(bufmgr_gem, stride, tiling_mode);
		929	size = stride * aligned_y;
		930	size = drm_intel_gem_bo_tile_size(bufmgr_gem, size, tiling_mode);
		931	} while (*tiling_mode != tiling);
		932	*pitch = stride;
		933
		934	if (tiling == I915_TILING_NONE)
		935	stride = 0;
		936
		937	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, flags,
6110	serge	938	tiling, stride, 0);
4363	Serge	939	}
		940
6110	serge	941	#if 0
		942	static drm_intel_bo *
		943	drm_intel_gem_bo_alloc_userptr(drm_intel_bufmgr *bufmgr,
		944	const char *name,
		945	void *addr,
		946	uint32_t tiling_mode,
		947	uint32_t stride,
		948	unsigned long size,
		949	unsigned long flags)
		950	{
		951	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		952	drm_intel_bo_gem *bo_gem;
		953	int ret;
		954	struct drm_i915_gem_userptr userptr;
		955
		956	/* Tiling with userptr surfaces is not supported
		957	* on all hardware so refuse it for time being.
		958	*/
		959	if (tiling_mode != I915_TILING_NONE)
		960	return NULL;
		961
		962	bo_gem = calloc(1, sizeof(*bo_gem));
		963	if (!bo_gem)
		964	return NULL;
		965
		966	bo_gem->bo.size = size;
		967
		968	memclear(userptr);
		969	userptr.user_ptr = (__u64)((unsigned long)addr);
		970	userptr.user_size = size;
		971	userptr.flags = flags;
		972
		973	ret = drmIoctl(bufmgr_gem->fd,
		974	DRM_IOCTL_I915_GEM_USERPTR,
		975	&userptr);
		976	if (ret != 0) {
		977	DBG("bo_create_userptr: "
		978	"ioctl failed with user ptr %p size 0x%lx, "
		979	"user flags 0x%lx\n", addr, size, flags);
		980	free(bo_gem);
		981	return NULL;
		982	}
		983
		984	bo_gem->gem_handle = userptr.handle;
		985	bo_gem->bo.handle = bo_gem->gem_handle;
		986	bo_gem->bo.bufmgr = bufmgr;
		987	bo_gem->is_userptr = true;
		988	bo_gem->bo.virtual = addr;
		989	/* Save the address provided by user */
		990	bo_gem->user_virtual = addr;
		991	bo_gem->tiling_mode = I915_TILING_NONE;
		992	bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		993	bo_gem->stride = 0;
		994
		995	DRMINITLISTHEAD(&bo_gem->name_list);
		996	DRMINITLISTHEAD(&bo_gem->vma_list);
		997
		998	bo_gem->name = name;
		999	atomic_set(&bo_gem->refcount, 1);
		1000	bo_gem->validate_index = -1;
		1001	bo_gem->reloc_tree_fences = 0;
		1002	bo_gem->used_as_reloc_target = false;
		1003	bo_gem->has_error = false;
		1004	bo_gem->reusable = false;
		1005	bo_gem->use_48b_address_range = false;
		1006
		1007	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
		1008
		1009	DBG("bo_create_userptr: "
		1010	"ptr %p buf %d (%s) size %ldb, stride 0x%x, tile mode %d\n",
		1011	addr, bo_gem->gem_handle, bo_gem->name,
		1012	size, stride, tiling_mode);
		1013
		1014	return &bo_gem->bo;
		1015	}
		1016
		1017	static bool
		1018	has_userptr(drm_intel_bufmgr_gem *bufmgr_gem)
		1019	{
		1020	int ret;
		1021	void *ptr;
		1022	long pgsz;
		1023	struct drm_i915_gem_userptr userptr;
		1024
		1025	pgsz = sysconf(_SC_PAGESIZE);
		1026	assert(pgsz > 0);
		1027
		1028	ret = posix_memalign(&ptr, pgsz, pgsz);
		1029	if (ret) {
		1030	DBG("Failed to get a page (%ld) for userptr detection!\n",
		1031	pgsz);
		1032	return false;
		1033	}
		1034
		1035	memclear(userptr);
		1036	userptr.user_ptr = (__u64)(unsigned long)ptr;
		1037	userptr.user_size = pgsz;
		1038
		1039	retry:
		1040	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_USERPTR, &userptr);
		1041	if (ret) {
		1042	if (errno == ENODEV && userptr.flags == 0) {
		1043	userptr.flags = I915_USERPTR_UNSYNCHRONIZED;
		1044	goto retry;
		1045	}
		1046	free(ptr);
		1047	return false;
		1048	}
		1049
		1050	/* We don't release the userptr bo here as we want to keep the
		1051	* kernel mm tracking alive for our lifetime. The first time we
		1052	* create a userptr object the kernel has to install a mmu_notifer
		1053	* which is a heavyweight operation (e.g. it requires taking all
		1054	* mm_locks and stop_machine()).
		1055	*/
		1056
		1057	bufmgr_gem->userptr_active.ptr = ptr;
		1058	bufmgr_gem->userptr_active.handle = userptr.handle;
		1059
		1060	return true;
		1061	}
		1062
		1063	#endif
		1064
		1065	static drm_intel_bo *
		1066	check_bo_alloc_userptr(drm_intel_bufmgr *bufmgr,
		1067	const char *name,
		1068	void *addr,
		1069	uint32_t tiling_mode,
		1070	uint32_t stride,
		1071	unsigned long size,
		1072	unsigned long flags)
		1073	{
		1074	bufmgr->bo_alloc_userptr = NULL;
		1075
		1076	return drm_intel_bo_alloc_userptr(bufmgr, name, addr,
		1077	tiling_mode, stride, size, flags);
		1078	}
		1079
4363	Serge	1080	/**
		1081	* Returns a drm_intel_bo wrapping the given buffer object handle.
		1082	*
		1083	* This can be used when one application needs to pass a buffer object
		1084	* to another.
		1085	*/
		1086	drm_intel_bo *
		1087	drm_intel_bo_gem_create_from_name(drm_intel_bufmgr *bufmgr,
		1088	const char *name,
		1089	unsigned int handle)
		1090	{
		1091	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		1092	drm_intel_bo_gem *bo_gem;
		1093	int ret;
		1094	struct drm_gem_open open_arg;
		1095	struct drm_i915_gem_get_tiling get_tiling;
		1096	drmMMListHead *list;
		1097
		1098	/* At the moment most applications only have a few named bo.
		1099	* For instance, in a DRI client only the render buffers passed
		1100	* between X and the client are named. And since X returns the
		1101	* alternating names for the front/back buffer a linear search
		1102	* provides a sufficiently fast match.
		1103	*/
6667	serge	1104	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	1105	for (list = bufmgr_gem->named.next;
		1106	list != &bufmgr_gem->named;
		1107	list = list->next) {
		1108	bo_gem = DRMLISTENTRY(drm_intel_bo_gem, list, name_list);
		1109	if (bo_gem->global_name == handle) {
		1110	drm_intel_gem_bo_reference(&bo_gem->bo);
6667	serge	1111	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1112	return &bo_gem->bo;
		1113	}
		1114	}
		1115
6110	serge	1116	memclear(open_arg);
4363	Serge	1117	open_arg.name = handle;
		1118	ret = drmIoctl(bufmgr_gem->fd,
		1119	DRM_IOCTL_GEM_OPEN,
		1120	&open_arg);
		1121	if (ret != 0) {
		1122	DBG("Couldn't reference %s handle 0x%08x: %s\n",
		1123	name, handle, strerror(errno));
6667	serge	1124	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1125	return NULL;
		1126	}
5068	serge	1127	/* Now see if someone has used a prime handle to get this
		1128	* object from the kernel before by looking through the list
		1129	* again for a matching gem_handle
		1130	*/
		1131	for (list = bufmgr_gem->named.next;
		1132	list != &bufmgr_gem->named;
		1133	list = list->next) {
		1134	bo_gem = DRMLISTENTRY(drm_intel_bo_gem, list, name_list);
		1135	if (bo_gem->gem_handle == open_arg.handle) {
		1136	drm_intel_gem_bo_reference(&bo_gem->bo);
6667	serge	1137	pthread_mutex_unlock(&bufmgr_gem->lock);
5068	serge	1138	return &bo_gem->bo;
		1139	}
		1140	}
		1141
		1142	bo_gem = calloc(1, sizeof(*bo_gem));
6667	serge	1143	if (!bo_gem) {
		1144	pthread_mutex_unlock(&bufmgr_gem->lock);
5068	serge	1145	return NULL;
6667	serge	1146	}
5068	serge	1147
4363	Serge	1148	bo_gem->bo.size = open_arg.size;
		1149	bo_gem->bo.offset = 0;
5068	serge	1150	bo_gem->bo.offset64 = 0;
4363	Serge	1151	bo_gem->bo.virtual = NULL;
		1152	bo_gem->bo.bufmgr = bufmgr;
		1153	bo_gem->name = name;
		1154	atomic_set(&bo_gem->refcount, 1);
		1155	bo_gem->validate_index = -1;
		1156	bo_gem->gem_handle = open_arg.handle;
		1157	bo_gem->bo.handle = open_arg.handle;
		1158	bo_gem->global_name = handle;
		1159	bo_gem->reusable = false;
6110	serge	1160	bo_gem->use_48b_address_range = false;
4363	Serge	1161
6110	serge	1162	memclear(get_tiling);
4363	Serge	1163	get_tiling.handle = bo_gem->gem_handle;
		1164	ret = drmIoctl(bufmgr_gem->fd,
		1165	DRM_IOCTL_I915_GEM_GET_TILING,
		1166	&get_tiling);
		1167	if (ret != 0) {
		1168	drm_intel_gem_bo_unreference(&bo_gem->bo);
6667	serge	1169	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1170	return NULL;
		1171	}
		1172	bo_gem->tiling_mode = get_tiling.tiling_mode;
		1173	bo_gem->swizzle_mode = get_tiling.swizzle_mode;
		1174	/* XXX stride is unknown */
6110	serge	1175	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
4363	Serge	1176
		1177	DRMINITLISTHEAD(&bo_gem->vma_list);
		1178	DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
6667	serge	1179	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1180	DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
		1181
		1182	return &bo_gem->bo;
		1183	}
		1184
		1185	static void
		1186	drm_intel_gem_bo_free(drm_intel_bo *bo)
		1187	{
		1188	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1189	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1190	struct drm_gem_close close;
		1191	int ret;
		1192
		1193	DRMLISTDEL(&bo_gem->vma_list);
		1194	if (bo_gem->mem_virtual) {
		1195	VG(VALGRIND_FREELIKE_BLOCK(bo_gem->mem_virtual, 0));
		1196	bufmgr_gem->vma_count--;
		1197	}
		1198	if (bo_gem->gtt_virtual) {
		1199	bufmgr_gem->vma_count--;
		1200	}
		1201
		1202	/* Close this object */
6110	serge	1203	memclear(close);
4363	Serge	1204	close.handle = bo_gem->gem_handle;
		1205	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
		1206	if (ret != 0) {
		1207	DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
		1208	bo_gem->gem_handle, bo_gem->name, strerror(errno));
		1209	}
		1210	free(bo);
		1211	}
		1212
		1213	static void
		1214	drm_intel_gem_bo_mark_mmaps_incoherent(drm_intel_bo *bo)
		1215	{
		1216	#if HAVE_VALGRIND
		1217	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1218
		1219	if (bo_gem->mem_virtual)
		1220	VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
		1221
		1222	if (bo_gem->gtt_virtual)
		1223	VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, bo->size);
		1224	#endif
		1225	}
		1226
		1227	/** Frees all cached buffers significantly older than @time. */
		1228	static void
		1229	drm_intel_gem_cleanup_bo_cache(drm_intel_bufmgr_gem *bufmgr_gem, time_t time)
		1230	{
		1231	int i;
		1232
		1233	if (bufmgr_gem->time == time)
		1234	return;
		1235
		1236	for (i = 0; i < bufmgr_gem->num_buckets; i++) {
		1237	struct drm_intel_gem_bo_bucket *bucket =
		1238	&bufmgr_gem->cache_bucket[i];
		1239
		1240	while (!DRMLISTEMPTY(&bucket->head)) {
		1241	drm_intel_bo_gem *bo_gem;
		1242
		1243	bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
		1244	bucket->head.next, head);
		1245	if (time - bo_gem->free_time <= 1)
		1246	break;
		1247
		1248	DRMLISTDEL(&bo_gem->head);
		1249
		1250	drm_intel_gem_bo_free(&bo_gem->bo);
		1251	}
		1252	}
		1253
		1254	bufmgr_gem->time = time;
		1255	}
		1256
		1257	static void drm_intel_gem_bo_purge_vma_cache(drm_intel_bufmgr_gem *bufmgr_gem)
		1258	{
		1259	int limit;
		1260
		1261	DBG("%s: cached=%d, open=%d, limit=%d\n", __FUNCTION__,
		1262	bufmgr_gem->vma_count, bufmgr_gem->vma_open, bufmgr_gem->vma_max);
		1263
		1264	if (bufmgr_gem->vma_max < 0)
		1265	return;
		1266
		1267	/* We may need to evict a few entries in order to create new mmaps */
		1268	limit = bufmgr_gem->vma_max - 2*bufmgr_gem->vma_open;
		1269	if (limit < 0)
		1270	limit = 0;
		1271
		1272	while (bufmgr_gem->vma_count > limit) {
		1273	drm_intel_bo_gem *bo_gem;
		1274
		1275	bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
		1276	bufmgr_gem->vma_cache.next,
		1277	vma_list);
		1278	assert(bo_gem->map_count == 0);
		1279	DRMLISTDELINIT(&bo_gem->vma_list);
		1280
		1281	if (bo_gem->mem_virtual) {
		1282	// munmap(bo_gem->mem_virtual, bo_gem->bo.size);
		1283	bo_gem->mem_virtual = NULL;
		1284	bufmgr_gem->vma_count--;
		1285	}
		1286	if (bo_gem->gtt_virtual) {
		1287	// munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
		1288	bo_gem->gtt_virtual = NULL;
		1289	bufmgr_gem->vma_count--;
		1290	}
		1291	}
		1292	}
		1293
		1294	static void drm_intel_gem_bo_close_vma(drm_intel_bufmgr_gem *bufmgr_gem,
		1295	drm_intel_bo_gem *bo_gem)
		1296	{
		1297	bufmgr_gem->vma_open--;
		1298	DRMLISTADDTAIL(&bo_gem->vma_list, &bufmgr_gem->vma_cache);
		1299	if (bo_gem->mem_virtual)
		1300	bufmgr_gem->vma_count++;
		1301	if (bo_gem->gtt_virtual)
		1302	bufmgr_gem->vma_count++;
		1303	drm_intel_gem_bo_purge_vma_cache(bufmgr_gem);
		1304	}
		1305
		1306	static void drm_intel_gem_bo_open_vma(drm_intel_bufmgr_gem *bufmgr_gem,
		1307	drm_intel_bo_gem *bo_gem)
		1308	{
		1309	bufmgr_gem->vma_open++;
		1310	DRMLISTDEL(&bo_gem->vma_list);
		1311	if (bo_gem->mem_virtual)
		1312	bufmgr_gem->vma_count--;
		1313	if (bo_gem->gtt_virtual)
		1314	bufmgr_gem->vma_count--;
		1315	drm_intel_gem_bo_purge_vma_cache(bufmgr_gem);
		1316	}
		1317
		1318	static void
		1319	drm_intel_gem_bo_unreference_final(drm_intel_bo *bo, time_t time)
		1320	{
		1321	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1322	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1323	struct drm_intel_gem_bo_bucket *bucket;
		1324	int i;
		1325
		1326	/* Unreference all the target buffers */
		1327	for (i = 0; i < bo_gem->reloc_count; i++) {
		1328	if (bo_gem->reloc_target_info[i].bo != bo) {
		1329	drm_intel_gem_bo_unreference_locked_timed(bo_gem->
		1330	reloc_target_info[i].bo,
		1331	time);
		1332	}
		1333	}
6110	serge	1334	for (i = 0; i < bo_gem->softpin_target_count; i++)
		1335	drm_intel_gem_bo_unreference_locked_timed(bo_gem->softpin_target[i],
		1336	time);
4363	Serge	1337	bo_gem->reloc_count = 0;
		1338	bo_gem->used_as_reloc_target = false;
6110	serge	1339	bo_gem->softpin_target_count = 0;
4363	Serge	1340
		1341	DBG("bo_unreference final: %d (%s)\n",
		1342	bo_gem->gem_handle, bo_gem->name);
		1343
		1344	/* release memory associated with this object */
		1345	if (bo_gem->reloc_target_info) {
		1346	free(bo_gem->reloc_target_info);
		1347	bo_gem->reloc_target_info = NULL;
		1348	}
		1349	if (bo_gem->relocs) {
		1350	free(bo_gem->relocs);
		1351	bo_gem->relocs = NULL;
		1352	}
6110	serge	1353	if (bo_gem->softpin_target) {
		1354	free(bo_gem->softpin_target);
		1355	bo_gem->softpin_target = NULL;
		1356	bo_gem->softpin_target_size = 0;
		1357	}
4363	Serge	1358
		1359	/* Clear any left-over mappings */
		1360	if (bo_gem->map_count) {
		1361	DBG("bo freed with non-zero map-count %d\n", bo_gem->map_count);
		1362	bo_gem->map_count = 0;
		1363	drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
		1364	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		1365	}
		1366
		1367	DRMLISTDEL(&bo_gem->name_list);
		1368
		1369	bucket = drm_intel_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
		1370	/* Put the buffer into our internal cache for reuse if we can. */
		1371	if (bufmgr_gem->bo_reuse && bo_gem->reusable && bucket != NULL &&
		1372	drm_intel_gem_bo_madvise_internal(bufmgr_gem, bo_gem,
		1373	I915_MADV_DONTNEED)) {
		1374	bo_gem->free_time = time;
		1375
		1376	bo_gem->name = NULL;
		1377	bo_gem->validate_index = -1;
		1378
		1379	DRMLISTADDTAIL(&bo_gem->head, &bucket->head);
		1380	} else {
		1381	drm_intel_gem_bo_free(bo);
		1382	}
		1383	}
		1384
		1385	static void drm_intel_gem_bo_unreference_locked_timed(drm_intel_bo *bo,
		1386	time_t time)
		1387	{
		1388	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1389
		1390	assert(atomic_read(&bo_gem->refcount) > 0);
		1391	if (atomic_dec_and_test(&bo_gem->refcount))
		1392	drm_intel_gem_bo_unreference_final(bo, time);
		1393	}
		1394
		1395	static void drm_intel_gem_bo_unreference(drm_intel_bo *bo)
		1396	{
		1397	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1398
		1399	assert(atomic_read(&bo_gem->refcount) > 0);
6110	serge	1400
		1401	if (atomic_add_unless(&bo_gem->refcount, -1, 1)) {
4363	Serge	1402	drm_intel_bufmgr_gem *bufmgr_gem =
		1403	(drm_intel_bufmgr_gem *) bo->bufmgr;
6110	serge	1404	struct timespec time;
4363	Serge	1405
6110	serge	1406	clock_gettime(CLOCK_MONOTONIC, &time);
4363	Serge	1407
6667	serge	1408	pthread_mutex_lock(&bufmgr_gem->lock);
6110	serge	1409
		1410	if (atomic_dec_and_test(&bo_gem->refcount)) {
		1411	drm_intel_gem_bo_unreference_final(bo, time.tv_sec);
		1412	drm_intel_gem_cleanup_bo_cache(bufmgr_gem, time.tv_sec);
		1413	}
		1414
6667	serge	1415	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1416	}
		1417	}
		1418
		1419	static int drm_intel_gem_bo_map(drm_intel_bo *bo, int write_enable)
		1420	{
		1421	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1422	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1423	struct drm_i915_gem_set_domain set_domain;
		1424	int ret;
		1425
6110	serge	1426	if (bo_gem->is_userptr) {
		1427	/* Return the same user ptr */
		1428	bo->virtual = bo_gem->user_virtual;
		1429	return 0;
		1430	}
		1431
6667	serge	1432	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	1433
		1434	if (bo_gem->map_count++ == 0)
		1435	drm_intel_gem_bo_open_vma(bufmgr_gem, bo_gem);
		1436
		1437	if (!bo_gem->mem_virtual) {
		1438	struct drm_i915_gem_mmap mmap_arg;
		1439
		1440	DBG("bo_map: %d (%s), map_count=%d\n",
		1441	bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
		1442
6110	serge	1443	memclear(mmap_arg);
4363	Serge	1444	mmap_arg.handle = bo_gem->gem_handle;
		1445	mmap_arg.size = bo->size;
		1446	ret = drmIoctl(bufmgr_gem->fd,
		1447	DRM_IOCTL_I915_GEM_MMAP,
		1448	&mmap_arg);
		1449	if (ret != 0) {
		1450	ret = -errno;
		1451	DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
		1452	__FILE__, __LINE__, bo_gem->gem_handle,
		1453	bo_gem->name, strerror(errno));
		1454	if (--bo_gem->map_count == 0)
		1455	drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
6667	serge	1456	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1457	return ret;
		1458	}
		1459	VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
		1460	bo_gem->mem_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
		1461	}
		1462	DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
		1463	bo_gem->mem_virtual);
		1464	bo->virtual = bo_gem->mem_virtual;
		1465
6110	serge	1466	memclear(set_domain);
4363	Serge	1467	set_domain.handle = bo_gem->gem_handle;
		1468	set_domain.read_domains = I915_GEM_DOMAIN_CPU;
		1469	if (write_enable)
		1470	set_domain.write_domain = I915_GEM_DOMAIN_CPU;
		1471	else
		1472	set_domain.write_domain = 0;
		1473	ret = drmIoctl(bufmgr_gem->fd,
		1474	DRM_IOCTL_I915_GEM_SET_DOMAIN,
		1475	&set_domain);
		1476	if (ret != 0) {
		1477	DBG("%s:%d: Error setting to CPU domain %d: %s\n",
		1478	__FILE__, __LINE__, bo_gem->gem_handle,
		1479	strerror(errno));
		1480	}
		1481
		1482	if (write_enable)
		1483	bo_gem->mapped_cpu_write = true;
		1484
		1485	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		1486	VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
6667	serge	1487	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1488
		1489	return 0;
		1490	}
		1491
		1492	static int
		1493	map_gtt(drm_intel_bo *bo)
		1494	{
		1495	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1496	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1497	int ret;
		1498
6110	serge	1499	if (bo_gem->is_userptr)
		1500	return -EINVAL;
		1501
4363	Serge	1502	if (bo_gem->map_count++ == 0)
		1503	drm_intel_gem_bo_open_vma(bufmgr_gem, bo_gem);
		1504
		1505	/* Get a mapping of the buffer if we haven't before. */
		1506	if (bo_gem->gtt_virtual == NULL) {
		1507	struct drm_i915_gem_mmap_gtt mmap_arg;
		1508
		1509	DBG("bo_map_gtt: mmap %d (%s), map_count=%d\n",
		1510	bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
		1511
6110	serge	1512	memclear(mmap_arg);
4363	Serge	1513	mmap_arg.handle = bo_gem->gem_handle;
		1514
		1515	/* Get the fake offset back... */
		1516	ret = drmIoctl(bufmgr_gem->fd,
		1517	DRM_IOCTL_I915_GEM_MMAP_GTT,
		1518	&mmap_arg);
		1519	if (ret != 0) {
		1520	ret = -errno;
		1521	DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
		1522	__FILE__, __LINE__,
		1523	bo_gem->gem_handle, bo_gem->name,
		1524	strerror(errno));
		1525	if (--bo_gem->map_count == 0)
		1526	drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
		1527	return ret;
		1528	}
		1529
		1530	/* and mmap it */
6110	serge	1531	bo_gem->gtt_virtual = (void*)(__u32)mmap_arg.offset;
4363	Serge	1532	if (bo_gem->gtt_virtual == 0) {
		1533	bo_gem->gtt_virtual = NULL;
		1534	ret = -errno;
		1535	DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
		1536	__FILE__, __LINE__,
		1537	bo_gem->gem_handle, bo_gem->name,
		1538	strerror(errno));
		1539	if (--bo_gem->map_count == 0)
		1540	drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
		1541	return ret;
		1542	}
		1543	}
		1544
		1545	bo->virtual = bo_gem->gtt_virtual;
		1546
		1547	DBG("bo_map_gtt: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
		1548	bo_gem->gtt_virtual);
		1549
		1550	return 0;
		1551	}
		1552
6110	serge	1553	int
		1554	drm_intel_gem_bo_map_gtt(drm_intel_bo *bo)
4363	Serge	1555	{
		1556	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1557	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1558	struct drm_i915_gem_set_domain set_domain;
		1559	int ret;
		1560
6667	serge	1561	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	1562
		1563	ret = map_gtt(bo);
		1564	if (ret) {
6667	serge	1565	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1566	return ret;
		1567	}
		1568
		1569	/* Now move it to the GTT domain so that the GPU and CPU
		1570	* caches are flushed and the GPU isn't actively using the
		1571	* buffer.
		1572	*
		1573	* The pagefault handler does this domain change for us when
		1574	* it has unbound the BO from the GTT, but it's up to us to
		1575	* tell it when we're about to use things if we had done
		1576	* rendering and it still happens to be bound to the GTT.
		1577	*/
6110	serge	1578	memclear(set_domain);
4363	Serge	1579	set_domain.handle = bo_gem->gem_handle;
		1580	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		1581	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		1582	ret = drmIoctl(bufmgr_gem->fd,
		1583	DRM_IOCTL_I915_GEM_SET_DOMAIN,
		1584	&set_domain);
		1585	if (ret != 0) {
		1586	DBG("%s:%d: Error setting domain %d: %s\n",
		1587	__FILE__, __LINE__, bo_gem->gem_handle,
		1588	strerror(errno));
		1589	}
		1590
		1591	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		1592	VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
6667	serge	1593	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1594
		1595	return 0;
		1596	}
		1597
		1598	/**
		1599	* Performs a mapping of the buffer object like the normal GTT
		1600	* mapping, but avoids waiting for the GPU to be done reading from or
		1601	* rendering to the buffer.
		1602	*
		1603	* This is used in the implementation of GL_ARB_map_buffer_range: The
		1604	* user asks to create a buffer, then does a mapping, fills some
		1605	* space, runs a drawing command, then asks to map it again without
		1606	* synchronizing because it guarantees that it won't write over the
		1607	* data that the GPU is busy using (or, more specifically, that if it
		1608	* does write over the data, it acknowledges that rendering is
		1609	* undefined).
		1610	*/
		1611
6110	serge	1612	int
		1613	drm_intel_gem_bo_map_unsynchronized(drm_intel_bo *bo)
4363	Serge	1614	{
		1615	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
5068	serge	1616	#ifdef HAVE_VALGRIND
		1617	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1618	#endif
4363	Serge	1619	int ret;
		1620
		1621	/* If the CPU cache isn't coherent with the GTT, then use a
		1622	* regular synchronized mapping. The problem is that we don't
		1623	* track where the buffer was last used on the CPU side in
		1624	* terms of drm_intel_bo_map vs drm_intel_gem_bo_map_gtt, so
		1625	* we would potentially corrupt the buffer even when the user
		1626	* does reasonable things.
		1627	*/
		1628	if (!bufmgr_gem->has_llc)
		1629	return drm_intel_gem_bo_map_gtt(bo);
		1630
6667	serge	1631	pthread_mutex_lock(&bufmgr_gem->lock);
		1632
4363	Serge	1633	ret = map_gtt(bo);
6667	serge	1634	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1635
		1636	return ret;
		1637	}
		1638
		1639	static int drm_intel_gem_bo_unmap(drm_intel_bo *bo)
		1640	{
6110	serge	1641	drm_intel_bufmgr_gem *bufmgr_gem;
4363	Serge	1642	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1643	int ret = 0;
		1644
		1645	if (bo == NULL)
		1646	return 0;
		1647
6110	serge	1648	if (bo_gem->is_userptr)
		1649	return 0;
		1650
		1651	bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;
4363	Serge	1652
6667	serge	1653	pthread_mutex_lock(&bufmgr_gem->lock);
		1654
4363	Serge	1655	if (bo_gem->map_count <= 0) {
		1656	DBG("attempted to unmap an unmapped bo\n");
6667	serge	1657	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1658	/* Preserve the old behaviour of just treating this as a
		1659	* no-op rather than reporting the error.
		1660	*/
		1661	return 0;
		1662	}
		1663
		1664	if (bo_gem->mapped_cpu_write) {
		1665	struct drm_i915_gem_sw_finish sw_finish;
		1666
		1667	/* Cause a flush to happen if the buffer's pinned for
		1668	* scanout, so the results show up in a timely manner.
		1669	* Unlike GTT set domains, this only does work if the
		1670	* buffer should be scanout-related.
		1671	*/
		1672
		1673	bo_gem->mapped_cpu_write = false;
		1674	}
		1675
		1676	/* We need to unmap after every innovation as we cannot track
		1677	* an open vma for every bo as that will exhaasut the system
		1678	* limits and cause later failures.
		1679	*/
		1680	if (--bo_gem->map_count == 0) {
		1681	drm_intel_gem_bo_close_vma(bufmgr_gem, bo_gem);
		1682	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		1683	bo->virtual = NULL;
		1684	}
6667	serge	1685	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	1686
		1687	return ret;
		1688	}
		1689
6110	serge	1690	int
		1691	drm_intel_gem_bo_unmap_gtt(drm_intel_bo *bo)
4363	Serge	1692	{
		1693	return drm_intel_gem_bo_unmap(bo);
		1694	}
		1695
		1696	static int
		1697	drm_intel_gem_bo_subdata(drm_intel_bo *bo, unsigned long offset,
		1698	unsigned long size, const void *data)
		1699	{
		1700	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1701	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1702	struct drm_i915_gem_pwrite pwrite;
		1703	int ret;
		1704
6110	serge	1705	if (bo_gem->is_userptr)
		1706	return -EINVAL;
		1707
		1708	memclear(pwrite);
4363	Serge	1709	pwrite.handle = bo_gem->gem_handle;
		1710	pwrite.offset = offset;
		1711	pwrite.size = size;
		1712	pwrite.data_ptr = (uint64_t) (uintptr_t) data;
		1713	ret = drmIoctl(bufmgr_gem->fd,
		1714	DRM_IOCTL_I915_GEM_PWRITE,
		1715	&pwrite);
		1716	if (ret != 0) {
		1717	ret = -errno;
		1718	DBG("%s:%d: Error writing data to buffer %d: (%d %d) %s .\n",
		1719	__FILE__, __LINE__, bo_gem->gem_handle, (int)offset,
		1720	(int)size, strerror(errno));
		1721	}
		1722
		1723	return ret;
		1724	}
		1725
		1726	#if 0
		1727	static int
		1728	drm_intel_gem_get_pipe_from_crtc_id(drm_intel_bufmgr *bufmgr, int crtc_id)
		1729	{
		1730	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		1731	struct drm_i915_get_pipe_from_crtc_id get_pipe_from_crtc_id;
		1732	int ret;
		1733
6110	serge	1734	memclear(get_pipe_from_crtc_id);
4363	Serge	1735	get_pipe_from_crtc_id.crtc_id = crtc_id;
		1736	ret = drmIoctl(bufmgr_gem->fd,
		1737	DRM_IOCTL_I915_GET_PIPE_FROM_CRTC_ID,
		1738	&get_pipe_from_crtc_id);
		1739	if (ret != 0) {
		1740	/* We return -1 here to signal that we don't
		1741	* know which pipe is associated with this crtc.
		1742	* This lets the caller know that this information
		1743	* isn't available; using the wrong pipe for
		1744	* vblank waiting can cause the chipset to lock up
		1745	*/
		1746	return -1;
		1747	}
		1748
		1749	return get_pipe_from_crtc_id.pipe;
		1750	}
6110	serge	1751	#endif
4363	Serge	1752
		1753	static int
		1754	drm_intel_gem_bo_get_subdata(drm_intel_bo *bo, unsigned long offset,
		1755	unsigned long size, void *data)
		1756	{
		1757	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1758	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1759	struct drm_i915_gem_pread pread;
		1760	int ret;
		1761
6110	serge	1762	if (bo_gem->is_userptr)
		1763	return -EINVAL;
		1764
		1765	memclear(pread);
4363	Serge	1766	pread.handle = bo_gem->gem_handle;
		1767	pread.offset = offset;
		1768	pread.size = size;
		1769	pread.data_ptr = (uint64_t) (uintptr_t) data;
		1770	ret = drmIoctl(bufmgr_gem->fd,
		1771	DRM_IOCTL_I915_GEM_PREAD,
		1772	&pread);
		1773	if (ret != 0) {
		1774	ret = -errno;
		1775	DBG("%s:%d: Error reading data from buffer %d: (%d %d) %s .\n",
		1776	__FILE__, __LINE__, bo_gem->gem_handle, (int)offset,
		1777	(int)size, strerror(errno));
		1778	}
		1779
		1780	return ret;
		1781	}
		1782
		1783	/** Waits for all GPU rendering with the object to have completed. */
		1784	static void
		1785	drm_intel_gem_bo_wait_rendering(drm_intel_bo *bo)
		1786	{
		1787	drm_intel_gem_bo_start_gtt_access(bo, 1);
		1788	}
		1789
		1790	/**
		1791	* Waits on a BO for the given amount of time.
		1792	*
		1793	* @bo: buffer object to wait for
		1794	* @timeout_ns: amount of time to wait in nanoseconds.
		1795	* If value is less than 0, an infinite wait will occur.
		1796	*
		1797	* Returns 0 if the wait was successful ie. the last batch referencing the
		1798	* object has completed within the allotted time. Otherwise some negative return
		1799	* value describes the error. Of particular interest is -ETIME when the wait has
		1800	* failed to yield the desired result.
		1801	*
		1802	* Similar to drm_intel_gem_bo_wait_rendering except a timeout parameter allows
		1803	* the operation to give up after a certain amount of time. Another subtle
		1804	* difference is the internal locking semantics are different (this variant does
		1805	* not hold the lock for the duration of the wait). This makes the wait subject
		1806	* to a larger userspace race window.
		1807	*
		1808	* The implementation shall wait until the object is no longer actively
		1809	* referenced within a batch buffer at the time of the call. The wait will
		1810	* not guarantee that the buffer is re-issued via another thread, or an flinked
		1811	* handle. Userspace must make sure this race does not occur if such precision
		1812	* is important.
6110	serge	1813	*
		1814	* Note that some kernels have broken the inifite wait for negative values
		1815	* promise, upgrade to latest stable kernels if this is the case.
4363	Serge	1816	*/
6110	serge	1817	int
		1818	drm_intel_gem_bo_wait(drm_intel_bo *bo, int64_t timeout_ns)
4363	Serge	1819	{
		1820	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1821	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1822	struct drm_i915_gem_wait wait;
		1823	int ret;
		1824
		1825	if (!bufmgr_gem->has_wait_timeout) {
		1826	DBG("%s:%d: Timed wait is not supported. Falling back to "
		1827	"infinite wait\n", __FILE__, __LINE__);
		1828	if (timeout_ns) {
		1829	drm_intel_gem_bo_wait_rendering(bo);
		1830	return 0;
		1831	} else {
6110	serge	1832	return drm_intel_gem_bo_busy(bo) ? -ETIME : 0;
4363	Serge	1833	}
		1834	}
		1835
6110	serge	1836	memclear(wait);
4363	Serge	1837	wait.bo_handle = bo_gem->gem_handle;
		1838	wait.timeout_ns = timeout_ns;
		1839	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
		1840	if (ret == -1)
		1841	return -errno;
		1842
		1843	return ret;
		1844	}
		1845
		1846	/**
		1847	* Sets the object to the GTT read and possibly write domain, used by the X
		1848	* 2D driver in the absence of kernel support to do drm_intel_gem_bo_map_gtt().
		1849	*
		1850	* In combination with drm_intel_gem_bo_pin() and manual fence management, we
		1851	* can do tiled pixmaps this way.
		1852	*/
		1853	void
		1854	drm_intel_gem_bo_start_gtt_access(drm_intel_bo *bo, int write_enable)
		1855	{
		1856	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1857	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1858	struct drm_i915_gem_set_domain set_domain;
		1859	int ret;
		1860
6110	serge	1861	memclear(set_domain);
4363	Serge	1862	set_domain.handle = bo_gem->gem_handle;
		1863	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		1864	set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_GTT : 0;
		1865	ret = drmIoctl(bufmgr_gem->fd,
		1866	DRM_IOCTL_I915_GEM_SET_DOMAIN,
		1867	&set_domain);
		1868	if (ret != 0) {
		1869	DBG("%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
		1870	__FILE__, __LINE__, bo_gem->gem_handle,
		1871	set_domain.read_domains, set_domain.write_domain,
		1872	strerror(errno));
		1873	}
		1874	}
		1875
		1876	static void
		1877	drm_intel_bufmgr_gem_destroy(drm_intel_bufmgr *bufmgr)
		1878	{
		1879	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
6110	serge	1880	struct drm_gem_close close_bo;
		1881	int i, ret;
4363	Serge	1882
6667	serge	1883	printf("\nENTER %s\n", __FUNCTION__);
		1884
4363	Serge	1885	free(bufmgr_gem->exec2_objects);
		1886	free(bufmgr_gem->exec_objects);
		1887	free(bufmgr_gem->exec_bos);
		1888
6667	serge	1889	pthread_mutex_destroy(&bufmgr_gem->lock);
4363	Serge	1890
		1891	/* Free any cached buffer objects we were going to reuse */
		1892	for (i = 0; i < bufmgr_gem->num_buckets; i++) {
		1893	struct drm_intel_gem_bo_bucket *bucket =
		1894	&bufmgr_gem->cache_bucket[i];
		1895	drm_intel_bo_gem *bo_gem;
		1896
		1897	while (!DRMLISTEMPTY(&bucket->head)) {
		1898	bo_gem = DRMLISTENTRY(drm_intel_bo_gem,
		1899	bucket->head.next, head);
		1900	DRMLISTDEL(&bo_gem->head);
		1901
		1902	drm_intel_gem_bo_free(&bo_gem->bo);
		1903	}
		1904	}
		1905
		1906	free(bufmgr);
6667	serge	1907	printf("\nLEAVE %s\n", __FUNCTION__);
		1908
4363	Serge	1909	}
		1910
		1911	/**
		1912	* Adds the target buffer to the validation list and adds the relocation
		1913	* to the reloc_buffer's relocation list.
		1914	*
		1915	* The relocation entry at the given offset must already contain the
		1916	* precomputed relocation value, because the kernel will optimize out
		1917	* the relocation entry write when the buffer hasn't moved from the
		1918	* last known offset in target_bo.
		1919	*/
		1920	static int
		1921	do_bo_emit_reloc(drm_intel_bo *bo, uint32_t offset,
		1922	drm_intel_bo *target_bo, uint32_t target_offset,
		1923	uint32_t read_domains, uint32_t write_domain,
		1924	bool need_fence)
		1925	{
		1926	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		1927	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		1928	drm_intel_bo_gem target_bo_gem = (drm_intel_bo_gem ) target_bo;
		1929	bool fenced_command;
		1930
		1931	if (bo_gem->has_error)
		1932	return -ENOMEM;
		1933
		1934	if (target_bo_gem->has_error) {
		1935	bo_gem->has_error = true;
		1936	return -ENOMEM;
		1937	}
		1938
		1939	/* We never use HW fences for rendering on 965+ */
		1940	if (bufmgr_gem->gen >= 4)
		1941	need_fence = false;
		1942
		1943	fenced_command = need_fence;
		1944	if (target_bo_gem->tiling_mode == I915_TILING_NONE)
		1945	need_fence = false;
		1946
		1947	/* Create a new relocation list if needed */
		1948	if (bo_gem->relocs == NULL && drm_intel_setup_reloc_list(bo))
		1949	return -ENOMEM;
		1950
		1951	/* Check overflow */
		1952	assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
		1953
		1954	/* Check args */
		1955	assert(offset <= bo->size - 4);
		1956	assert((write_domain & (write_domain - 1)) == 0);
		1957
6110	serge	1958	/* An object needing a fence is a tiled buffer, so it won't have
		1959	* relocs to other buffers.
		1960	*/
		1961	if (need_fence) {
		1962	assert(target_bo_gem->reloc_count == 0);
		1963	target_bo_gem->reloc_tree_fences = 1;
		1964	}
		1965
4363	Serge	1966	/* Make sure that we're not adding a reloc to something whose size has
		1967	* already been accounted for.
		1968	*/
		1969	assert(!bo_gem->used_as_reloc_target);
		1970	if (target_bo_gem != bo_gem) {
		1971	target_bo_gem->used_as_reloc_target = true;
		1972	bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
6110	serge	1973	bo_gem->reloc_tree_fences += target_bo_gem->reloc_tree_fences;
4363	Serge	1974	}
		1975
		1976	bo_gem->reloc_target_info[bo_gem->reloc_count].bo = target_bo;
		1977	if (target_bo != bo)
		1978	drm_intel_gem_bo_reference(target_bo);
		1979	if (fenced_command)
		1980	bo_gem->reloc_target_info[bo_gem->reloc_count].flags =
		1981	DRM_INTEL_RELOC_FENCE;
		1982	else
		1983	bo_gem->reloc_target_info[bo_gem->reloc_count].flags = 0;
		1984
6110	serge	1985	bo_gem->relocs[bo_gem->reloc_count].offset = offset;
		1986	bo_gem->relocs[bo_gem->reloc_count].delta = target_offset;
		1987	bo_gem->relocs[bo_gem->reloc_count].target_handle =
		1988	target_bo_gem->gem_handle;
		1989	bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
		1990	bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
		1991	bo_gem->relocs[bo_gem->reloc_count].presumed_offset = target_bo->offset64;
4363	Serge	1992	bo_gem->reloc_count++;
		1993
		1994	return 0;
		1995	}
		1996
6110	serge	1997	static void
		1998	drm_intel_gem_bo_use_48b_address_range(drm_intel_bo *bo, uint32_t enable)
		1999	{
		2000	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2001	bo_gem->use_48b_address_range = enable;
		2002	}
		2003
4363	Serge	2004	static int
6110	serge	2005	drm_intel_gem_bo_add_softpin_target(drm_intel_bo bo, drm_intel_bo target_bo)
		2006	{
		2007	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2008	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2009	drm_intel_bo_gem target_bo_gem = (drm_intel_bo_gem ) target_bo;
		2010	if (bo_gem->has_error)
		2011	return -ENOMEM;
		2012
		2013	if (target_bo_gem->has_error) {
		2014	bo_gem->has_error = true;
		2015	return -ENOMEM;
		2016	}
		2017
		2018	if (!target_bo_gem->is_softpin)
		2019	return -EINVAL;
		2020	if (target_bo_gem == bo_gem)
		2021	return -EINVAL;
		2022
		2023	if (bo_gem->softpin_target_count == bo_gem->softpin_target_size) {
		2024	int new_size = bo_gem->softpin_target_size * 2;
		2025	if (new_size == 0)
		2026	new_size = bufmgr_gem->max_relocs;
		2027
		2028	bo_gem->softpin_target = realloc(bo_gem->softpin_target, new_size *
		2029	sizeof(drm_intel_bo *));
		2030	if (!bo_gem->softpin_target)
		2031	return -ENOMEM;
		2032
		2033	bo_gem->softpin_target_size = new_size;
		2034	}
		2035	bo_gem->softpin_target[bo_gem->softpin_target_count] = target_bo;
		2036	drm_intel_gem_bo_reference(target_bo);
		2037	bo_gem->softpin_target_count++;
		2038
		2039	return 0;
		2040	}
		2041
		2042	static int
4363	Serge	2043	drm_intel_gem_bo_emit_reloc(drm_intel_bo *bo, uint32_t offset,
		2044	drm_intel_bo *target_bo, uint32_t target_offset,
		2045	uint32_t read_domains, uint32_t write_domain)
		2046	{
		2047	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bo->bufmgr;
6110	serge	2048	drm_intel_bo_gem target_bo_gem = (drm_intel_bo_gem )target_bo;
4363	Serge	2049
6110	serge	2050	if (target_bo_gem->is_softpin)
		2051	return drm_intel_gem_bo_add_softpin_target(bo, target_bo);
		2052	else
		2053	return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
		2054	read_domains, write_domain,
		2055	!bufmgr_gem->fenced_relocs);
4363	Serge	2056	}
		2057
		2058	static int
		2059	drm_intel_gem_bo_emit_reloc_fence(drm_intel_bo *bo, uint32_t offset,
		2060	drm_intel_bo *target_bo,
		2061	uint32_t target_offset,
		2062	uint32_t read_domains, uint32_t write_domain)
		2063	{
		2064	return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
		2065	read_domains, write_domain, true);
		2066	}
		2067
		2068	int
		2069	drm_intel_gem_bo_get_reloc_count(drm_intel_bo *bo)
		2070	{
		2071	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2072
		2073	return bo_gem->reloc_count;
		2074	}
		2075
		2076	/**
		2077	* Removes existing relocation entries in the BO after "start".
		2078	*
		2079	* This allows a user to avoid a two-step process for state setup with
		2080	* counting up all the buffer objects and doing a
		2081	* drm_intel_bufmgr_check_aperture_space() before emitting any of the
		2082	* relocations for the state setup. Instead, save the state of the
		2083	* batchbuffer including drm_intel_gem_get_reloc_count(), emit all the
		2084	* state, and then check if it still fits in the aperture.
		2085	*
		2086	* Any further drm_intel_bufmgr_check_aperture_space() queries
		2087	* involving this buffer in the tree are undefined after this call.
6110	serge	2088	*
		2089	* This also removes all softpinned targets being referenced by the BO.
4363	Serge	2090	*/
		2091	void
		2092	drm_intel_gem_bo_clear_relocs(drm_intel_bo *bo, int start)
		2093	{
6110	serge	2094	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
4363	Serge	2095	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2096	int i;
6110	serge	2097	struct timespec time;
4363	Serge	2098
6110	serge	2099	clock_gettime(CLOCK_MONOTONIC, &time);
4363	Serge	2100
		2101	assert(bo_gem->reloc_count >= start);
6110	serge	2102
4363	Serge	2103	/* Unreference the cleared target buffers */
6667	serge	2104	pthread_mutex_lock(&bufmgr_gem->lock);
		2105
4363	Serge	2106	for (i = start; i < bo_gem->reloc_count; i++) {
		2107	drm_intel_bo_gem target_bo_gem = (drm_intel_bo_gem ) bo_gem->reloc_target_info[i].bo;
		2108	if (&target_bo_gem->bo != bo) {
		2109	bo_gem->reloc_tree_fences -= target_bo_gem->reloc_tree_fences;
		2110	drm_intel_gem_bo_unreference_locked_timed(&target_bo_gem->bo,
6110	serge	2111	time.tv_sec);
4363	Serge	2112	}
		2113	}
		2114	bo_gem->reloc_count = start;
6110	serge	2115
		2116	for (i = 0; i < bo_gem->softpin_target_count; i++) {
		2117	drm_intel_bo_gem target_bo_gem = (drm_intel_bo_gem ) bo_gem->softpin_target[i];
		2118	drm_intel_gem_bo_unreference_locked_timed(&target_bo_gem->bo, time.tv_sec);
		2119	}
		2120	bo_gem->softpin_target_count = 0;
6667	serge	2121
		2122	pthread_mutex_unlock(&bufmgr_gem->lock);
		2123
4363	Serge	2124	}
		2125
		2126	/**
		2127	* Walk the tree of relocations rooted at BO and accumulate the list of
		2128	* validations to be performed and update the relocation buffers with
		2129	* index values into the validation list.
		2130	*/
		2131	static void
		2132	drm_intel_gem_bo_process_reloc(drm_intel_bo *bo)
		2133	{
		2134	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2135	int i;
		2136
		2137	if (bo_gem->relocs == NULL)
		2138	return;
		2139
		2140	for (i = 0; i < bo_gem->reloc_count; i++) {
		2141	drm_intel_bo *target_bo = bo_gem->reloc_target_info[i].bo;
		2142
		2143	if (target_bo == bo)
		2144	continue;
		2145
		2146	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		2147
		2148	/* Continue walking the tree depth-first. */
		2149	drm_intel_gem_bo_process_reloc(target_bo);
		2150
		2151	/* Add the target to the validate list */
		2152	drm_intel_add_validate_buffer(target_bo);
		2153	}
		2154	}
		2155
		2156	static void
		2157	drm_intel_gem_bo_process_reloc2(drm_intel_bo *bo)
		2158	{
		2159	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem )bo;
		2160	int i;
		2161
6110	serge	2162	if (bo_gem->relocs == NULL && bo_gem->softpin_target == NULL)
4363	Serge	2163	return;
		2164
		2165	for (i = 0; i < bo_gem->reloc_count; i++) {
		2166	drm_intel_bo *target_bo = bo_gem->reloc_target_info[i].bo;
		2167	int need_fence;
		2168
		2169	if (target_bo == bo)
		2170	continue;
		2171
		2172	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		2173
		2174	/* Continue walking the tree depth-first. */
		2175	drm_intel_gem_bo_process_reloc2(target_bo);
		2176
		2177	need_fence = (bo_gem->reloc_target_info[i].flags &
		2178	DRM_INTEL_RELOC_FENCE);
		2179
		2180	/* Add the target to the validate list */
		2181	drm_intel_add_validate_buffer2(target_bo, need_fence);
		2182	}
6110	serge	2183
		2184	for (i = 0; i < bo_gem->softpin_target_count; i++) {
		2185	drm_intel_bo *target_bo = bo_gem->softpin_target[i];
		2186
		2187	if (target_bo == bo)
		2188	continue;
		2189
		2190	drm_intel_gem_bo_mark_mmaps_incoherent(bo);
		2191	drm_intel_gem_bo_process_reloc2(target_bo);
		2192	drm_intel_add_validate_buffer2(target_bo, false);
		2193	}
4363	Serge	2194	}
		2195
		2196
		2197	static void
		2198	drm_intel_update_buffer_offsets(drm_intel_bufmgr_gem *bufmgr_gem)
		2199	{
		2200	int i;
		2201
		2202	for (i = 0; i < bufmgr_gem->exec_count; i++) {
		2203	drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
		2204	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2205
		2206	/* Update the buffer offset */
5068	serge	2207	if (bufmgr_gem->exec_objects[i].offset != bo->offset64) {
6110	serge	2208	DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
		2209	bo_gem->gem_handle, bo_gem->name,
		2210	upper_32_bits(bo->offset64),
		2211	lower_32_bits(bo->offset64),
		2212	upper_32_bits(bufmgr_gem->exec_objects[i].offset),
		2213	lower_32_bits(bufmgr_gem->exec_objects[i].offset));
5068	serge	2214	bo->offset64 = bufmgr_gem->exec_objects[i].offset;
4363	Serge	2215	bo->offset = bufmgr_gem->exec_objects[i].offset;
		2216	}
		2217	}
		2218	}
		2219
		2220	static void
		2221	drm_intel_update_buffer_offsets2 (drm_intel_bufmgr_gem *bufmgr_gem)
		2222	{
		2223	int i;
		2224
		2225	for (i = 0; i < bufmgr_gem->exec_count; i++) {
		2226	drm_intel_bo *bo = bufmgr_gem->exec_bos[i];
		2227	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem )bo;
		2228
		2229	/* Update the buffer offset */
5068	serge	2230	if (bufmgr_gem->exec2_objects[i].offset != bo->offset64) {
6110	serge	2231	/* If we're seeing softpinned object here it means that the kernel
		2232	* has relocated our object... Indicating a programming error
		2233	*/
		2234	assert(!bo_gem->is_softpin);
		2235	DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
		2236	bo_gem->gem_handle, bo_gem->name,
		2237	upper_32_bits(bo->offset64),
		2238	lower_32_bits(bo->offset64),
		2239	upper_32_bits(bufmgr_gem->exec2_objects[i].offset),
		2240	lower_32_bits(bufmgr_gem->exec2_objects[i].offset));
5068	serge	2241	bo->offset64 = bufmgr_gem->exec2_objects[i].offset;
4363	Serge	2242	bo->offset = bufmgr_gem->exec2_objects[i].offset;
		2243	}
		2244	}
		2245	}
		2246
6110	serge	2247	void
		2248	drm_intel_gem_bo_aub_dump_bmp(drm_intel_bo *bo,
		2249	int x1, int y1, int width, int height,
		2250	enum aub_dump_bmp_format format,
		2251	int pitch, int offset)
4363	Serge	2252	{
		2253	}
		2254
6110	serge	2255	static int
		2256	drm_intel_gem_bo_exec(drm_intel_bo *bo, int used,
		2257	drm_clip_rect_t * cliprects, int num_cliprects, int DR4)
4363	Serge	2258	{
		2259	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
6110	serge	2260	struct drm_i915_gem_execbuffer execbuf;
		2261	int ret, i;
4363	Serge	2262
6110	serge	2263	if (to_bo_gem(bo)->has_error)
		2264	return -ENOMEM;
4363	Serge	2265
6667	serge	2266	pthread_mutex_lock(&bufmgr_gem->lock);
6110	serge	2267	/* Update indices and set up the validate list. */
		2268	drm_intel_gem_bo_process_reloc(bo);
4363	Serge	2269
6110	serge	2270	/* Add the batch buffer to the validation list. There are no
		2271	* relocations pointing to it.
4363	Serge	2272	*/
6110	serge	2273	drm_intel_add_validate_buffer(bo);
4363	Serge	2274
6110	serge	2275	memclear(execbuf);
		2276	execbuf.buffers_ptr = (uintptr_t) bufmgr_gem->exec_objects;
		2277	execbuf.buffer_count = bufmgr_gem->exec_count;
		2278	execbuf.batch_start_offset = 0;
		2279	execbuf.batch_len = used;
		2280	execbuf.cliprects_ptr = (uintptr_t) cliprects;
		2281	execbuf.num_cliprects = num_cliprects;
		2282	execbuf.DR1 = 0;
		2283	execbuf.DR4 = DR4;
4363	Serge	2284
6110	serge	2285	ret = drmIoctl(bufmgr_gem->fd,
		2286	DRM_IOCTL_I915_GEM_EXECBUFFER,
		2287	&execbuf);
		2288	if (ret != 0) {
		2289	ret = -errno;
		2290	if (errno == ENOSPC) {
		2291	DBG("Execbuffer fails to pin. "
		2292	"Estimate: %u. Actual: %u. Available: %u\n",
		2293	drm_intel_gem_estimate_batch_space(bufmgr_gem->exec_bos,
		2294	bufmgr_gem->
		2295	exec_count),
		2296	drm_intel_gem_compute_batch_space(bufmgr_gem->exec_bos,
		2297	bufmgr_gem->
		2298	exec_count),
		2299	(unsigned int)bufmgr_gem->gtt_size);
4363	Serge	2300	}
		2301	}
6110	serge	2302	drm_intel_update_buffer_offsets(bufmgr_gem);
4363	Serge	2303
6110	serge	2304	if (bufmgr_gem->bufmgr.debug)
		2305	drm_intel_gem_dump_validation_list(bufmgr_gem);
4363	Serge	2306
6110	serge	2307	for (i = 0; i < bufmgr_gem->exec_count; i++) {
		2308	drm_intel_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[i]);
4363	Serge	2309
6110	serge	2310	bo_gem->idle = false;
4363	Serge	2311
6110	serge	2312	/* Disconnect the buffer from the validate list */
		2313	bo_gem->validate_index = -1;
		2314	bufmgr_gem->exec_bos[i] = NULL;
4363	Serge	2315	}
6110	serge	2316	bufmgr_gem->exec_count = 0;
6667	serge	2317	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2318
6110	serge	2319	return ret;
4363	Serge	2320	}
		2321
		2322	static int
		2323	do_exec2(drm_intel_bo bo, int used, drm_intel_context ctx,
		2324	drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
		2325	unsigned int flags)
		2326	{
		2327	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bo->bufmgr;
		2328	struct drm_i915_gem_execbuffer2 execbuf;
		2329	int ret = 0;
		2330	int i;
		2331
6110	serge	2332	if (to_bo_gem(bo)->has_error)
		2333	return -ENOMEM;
		2334
4363	Serge	2335	switch (flags & 0x7) {
		2336	default:
		2337	return -EINVAL;
		2338	case I915_EXEC_BLT:
		2339	if (!bufmgr_gem->has_blt)
		2340	return -EINVAL;
		2341	break;
		2342	case I915_EXEC_BSD:
		2343	if (!bufmgr_gem->has_bsd)
		2344	return -EINVAL;
		2345	break;
		2346	case I915_EXEC_VEBOX:
		2347	if (!bufmgr_gem->has_vebox)
		2348	return -EINVAL;
		2349	break;
		2350	case I915_EXEC_RENDER:
		2351	case I915_EXEC_DEFAULT:
		2352	break;
		2353	}
		2354
6667	serge	2355	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	2356	/* Update indices and set up the validate list. */
		2357	drm_intel_gem_bo_process_reloc2(bo);
		2358
		2359	/* Add the batch buffer to the validation list. There are no relocations
		2360	* pointing to it.
		2361	*/
		2362	drm_intel_add_validate_buffer2(bo, 0);
		2363
6110	serge	2364	memclear(execbuf);
4363	Serge	2365	execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
		2366	execbuf.buffer_count = bufmgr_gem->exec_count;
		2367	execbuf.batch_start_offset = 0;
		2368	execbuf.batch_len = used;
		2369	execbuf.cliprects_ptr = (uintptr_t)cliprects;
		2370	execbuf.num_cliprects = num_cliprects;
		2371	execbuf.DR1 = 0;
		2372	execbuf.DR4 = DR4;
		2373	execbuf.flags = flags;
		2374	if (ctx == NULL)
		2375	i915_execbuffer2_set_context_id(execbuf, 0);
		2376	else
		2377	i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
		2378	execbuf.rsvd2 = 0;
		2379
		2380	if (bufmgr_gem->no_exec)
		2381	goto skip_execution;
		2382
		2383	ret = drmIoctl(bufmgr_gem->fd,
		2384	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2385	&execbuf);
		2386	if (ret != 0) {
		2387	ret = -errno;
		2388	if (ret == -ENOSPC) {
		2389	DBG("Execbuffer fails to pin. "
		2390	"Estimate: %u. Actual: %u. Available: %u\n",
		2391	drm_intel_gem_estimate_batch_space(bufmgr_gem->exec_bos,
		2392	bufmgr_gem->exec_count),
		2393	drm_intel_gem_compute_batch_space(bufmgr_gem->exec_bos,
		2394	bufmgr_gem->exec_count),
		2395	(unsigned int) bufmgr_gem->gtt_size);
		2396	}
		2397	}
		2398	drm_intel_update_buffer_offsets2(bufmgr_gem);
		2399
		2400	skip_execution:
		2401	if (bufmgr_gem->bufmgr.debug)
		2402	drm_intel_gem_dump_validation_list(bufmgr_gem);
		2403
		2404	for (i = 0; i < bufmgr_gem->exec_count; i++) {
6110	serge	2405	drm_intel_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[i]);
4363	Serge	2406
5068	serge	2407	bo_gem->idle = false;
		2408
4363	Serge	2409	/* Disconnect the buffer from the validate list */
		2410	bo_gem->validate_index = -1;
		2411	bufmgr_gem->exec_bos[i] = NULL;
		2412	}
		2413	bufmgr_gem->exec_count = 0;
6667	serge	2414	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2415
		2416	return ret;
		2417	}
		2418
		2419	static int
		2420	drm_intel_gem_bo_exec2(drm_intel_bo *bo, int used,
		2421	drm_clip_rect_t *cliprects, int num_cliprects,
		2422	int DR4)
		2423	{
		2424	return do_exec2(bo, used, NULL, cliprects, num_cliprects, DR4,
		2425	I915_EXEC_RENDER);
		2426	}
		2427
		2428	static int
		2429	drm_intel_gem_bo_mrb_exec2(drm_intel_bo *bo, int used,
		2430	drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
		2431	unsigned int flags)
		2432	{
		2433	return do_exec2(bo, used, NULL, cliprects, num_cliprects, DR4,
		2434	flags);
		2435	}
		2436
		2437	int
		2438	drm_intel_gem_bo_context_exec(drm_intel_bo bo, drm_intel_context ctx,
		2439	int used, unsigned int flags)
		2440	{
		2441	return do_exec2(bo, used, ctx, NULL, 0, 0, flags);
		2442	}
		2443
		2444	static int
		2445	drm_intel_gem_bo_pin(drm_intel_bo *bo, uint32_t alignment)
		2446	{
		2447	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2448	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2449	struct drm_i915_gem_pin pin;
		2450	int ret;
		2451
6110	serge	2452	memclear(pin);
4363	Serge	2453	pin.handle = bo_gem->gem_handle;
		2454	pin.alignment = alignment;
		2455
		2456	ret = drmIoctl(bufmgr_gem->fd,
		2457	DRM_IOCTL_I915_GEM_PIN,
		2458	&pin);
		2459	if (ret != 0)
		2460	return -errno;
		2461
5068	serge	2462	bo->offset64 = pin.offset;
4363	Serge	2463	bo->offset = pin.offset;
		2464	return 0;
		2465	}
		2466
		2467	static int
		2468	drm_intel_gem_bo_unpin(drm_intel_bo *bo)
		2469	{
		2470	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2471	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2472	struct drm_i915_gem_unpin unpin;
		2473	int ret;
		2474
6110	serge	2475	memclear(unpin);
4363	Serge	2476	unpin.handle = bo_gem->gem_handle;
		2477
		2478	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_UNPIN, &unpin);
		2479	if (ret != 0)
		2480	return -errno;
		2481
		2482	return 0;
		2483	}
		2484
		2485	static int
		2486	drm_intel_gem_bo_set_tiling_internal(drm_intel_bo *bo,
		2487	uint32_t tiling_mode,
		2488	uint32_t stride)
		2489	{
		2490	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2491	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2492	struct drm_i915_gem_set_tiling set_tiling;
		2493	int ret;
		2494
		2495	if (bo_gem->global_name == 0 &&
		2496	tiling_mode == bo_gem->tiling_mode &&
		2497	stride == bo_gem->stride)
		2498	return 0;
		2499
		2500	memset(&set_tiling, 0, sizeof(set_tiling));
		2501	// do {
		2502	/* set_tiling is slightly broken and overwrites the
		2503	* input on the error path, so we have to open code
		2504	* rmIoctl.
		2505	*/
		2506	set_tiling.handle = bo_gem->gem_handle;
		2507	set_tiling.tiling_mode = tiling_mode;
		2508	set_tiling.stride = stride;
		2509
		2510	ret = drmIoctl(bufmgr_gem->fd,
		2511	DRM_IOCTL_I915_GEM_SET_TILING,
		2512	&set_tiling);
		2513	// } while (ret == -1 && (errno == EINTR \|\| errno == EAGAIN));
		2514	if (ret == -1)
		2515	return -errno;
		2516
		2517	bo_gem->tiling_mode = set_tiling.tiling_mode;
		2518	bo_gem->swizzle_mode = set_tiling.swizzle_mode;
		2519	bo_gem->stride = set_tiling.stride;
		2520	return 0;
		2521	}
		2522
		2523	static int
		2524	drm_intel_gem_bo_set_tiling(drm_intel_bo bo, uint32_t tiling_mode,
		2525	uint32_t stride)
		2526	{
		2527	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2528	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2529	int ret;
		2530
6110	serge	2531	/* Tiling with userptr surfaces is not supported
		2532	* on all hardware so refuse it for time being.
		2533	*/
		2534	if (bo_gem->is_userptr)
		2535	return -EINVAL;
		2536
4363	Serge	2537	/* Linear buffers have no stride. By ensuring that we only ever use
		2538	* stride 0 with linear buffers, we simplify our code.
		2539	*/
		2540	if (*tiling_mode == I915_TILING_NONE)
		2541	stride = 0;
		2542
		2543	ret = drm_intel_gem_bo_set_tiling_internal(bo, *tiling_mode, stride);
		2544	if (ret == 0)
6110	serge	2545	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
4363	Serge	2546
		2547	*tiling_mode = bo_gem->tiling_mode;
		2548	return ret;
		2549	}
		2550
		2551	static int
		2552	drm_intel_gem_bo_get_tiling(drm_intel_bo bo, uint32_t tiling_mode,
		2553	uint32_t * swizzle_mode)
		2554	{
		2555	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2556
		2557	*tiling_mode = bo_gem->tiling_mode;
		2558	*swizzle_mode = bo_gem->swizzle_mode;
		2559	return 0;
		2560	}
		2561
6110	serge	2562	static int
		2563	drm_intel_gem_bo_set_softpin_offset(drm_intel_bo *bo, uint64_t offset)
		2564	{
		2565	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2566
		2567	bo_gem->is_softpin = true;
		2568	bo->offset64 = offset;
		2569	bo->offset = offset;
		2570	return 0;
		2571	}
4363	Serge	2572	#if 0
		2573	drm_intel_bo *
		2574	drm_intel_bo_gem_create_from_prime(drm_intel_bufmgr *bufmgr, int prime_fd, int size)
		2575	{
		2576	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		2577	int ret;
		2578	uint32_t handle;
		2579	drm_intel_bo_gem *bo_gem;
		2580	struct drm_i915_gem_get_tiling get_tiling;
		2581	drmMMListHead *list;
		2582
6667	serge	2583	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	2584	ret = drmPrimeFDToHandle(bufmgr_gem->fd, prime_fd, &handle);
6110	serge	2585	if (ret) {
		2586	DBG("create_from_prime: failed to obtain handle from fd: %s\n", strerror(errno));
		2587	pthread_mutex_unlock(&bufmgr_gem->lock);
		2588	return NULL;
		2589	}
4363	Serge	2590
		2591	/*
		2592	* See if the kernel has already returned this buffer to us. Just as
		2593	* for named buffers, we must not create two bo's pointing at the same
		2594	* kernel object
		2595	*/
		2596	for (list = bufmgr_gem->named.next;
		2597	list != &bufmgr_gem->named;
		2598	list = list->next) {
		2599	bo_gem = DRMLISTENTRY(drm_intel_bo_gem, list, name_list);
		2600	if (bo_gem->gem_handle == handle) {
		2601	drm_intel_gem_bo_reference(&bo_gem->bo);
6667	serge	2602	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2603	return &bo_gem->bo;
		2604	}
		2605	}
		2606
6110	serge	2607	bo_gem = calloc(1, sizeof(*bo_gem));
		2608	if (!bo_gem) {
6667	serge	2609	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2610	return NULL;
		2611	}
		2612	/* Determine size of bo. The fd-to-handle ioctl really should
		2613	* return the size, but it doesn't. If we have kernel 3.12 or
		2614	* later, we can lseek on the prime fd to get the size. Older
		2615	* kernels will just fail, in which case we fall back to the
		2616	* provided (estimated or guess size). */
		2617	ret = lseek(prime_fd, 0, SEEK_END);
		2618	if (ret != -1)
		2619	bo_gem->bo.size = ret;
		2620	else
6110	serge	2621	bo_gem->bo.size = size;
4363	Serge	2622
		2623	bo_gem->bo.handle = handle;
		2624	bo_gem->bo.bufmgr = bufmgr;
		2625
		2626	bo_gem->gem_handle = handle;
		2627
		2628	atomic_set(&bo_gem->refcount, 1);
		2629
		2630	bo_gem->name = "prime";
		2631	bo_gem->validate_index = -1;
		2632	bo_gem->reloc_tree_fences = 0;
		2633	bo_gem->used_as_reloc_target = false;
		2634	bo_gem->has_error = false;
		2635	bo_gem->reusable = false;
6110	serge	2636	bo_gem->use_48b_address_range = false;
4363	Serge	2637
		2638	DRMINITLISTHEAD(&bo_gem->vma_list);
		2639	DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
6667	serge	2640	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2641
6110	serge	2642	memclear(get_tiling);
4363	Serge	2643	get_tiling.handle = bo_gem->gem_handle;
		2644	ret = drmIoctl(bufmgr_gem->fd,
		2645	DRM_IOCTL_I915_GEM_GET_TILING,
		2646	&get_tiling);
		2647	if (ret != 0) {
6110	serge	2648	DBG("create_from_prime: failed to get tiling: %s\n", strerror(errno));
4363	Serge	2649	drm_intel_gem_bo_unreference(&bo_gem->bo);
		2650	return NULL;
		2651	}
		2652	bo_gem->tiling_mode = get_tiling.tiling_mode;
		2653	bo_gem->swizzle_mode = get_tiling.swizzle_mode;
		2654	/* XXX stride is unknown */
6110	serge	2655	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
4363	Serge	2656
		2657	return &bo_gem->bo;
		2658	}
		2659
		2660	int
		2661	drm_intel_bo_gem_export_to_prime(drm_intel_bo bo, int prime_fd)
		2662	{
		2663	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2664	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2665
6667	serge	2666	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	2667	if (DRMLISTEMPTY(&bo_gem->name_list))
		2668	DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
6667	serge	2669	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2670
		2671	if (drmPrimeHandleToFD(bufmgr_gem->fd, bo_gem->gem_handle,
		2672	DRM_CLOEXEC, prime_fd) != 0)
		2673	return -errno;
		2674
		2675	bo_gem->reusable = false;
		2676
		2677	return 0;
		2678	}
		2679	#endif
		2680
		2681	static int
		2682	drm_intel_gem_bo_flink(drm_intel_bo bo, uint32_t name)
		2683	{
		2684	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bo->bufmgr;
		2685	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2686	int ret;
		2687
		2688	if (!bo_gem->global_name) {
		2689	struct drm_gem_flink flink;
		2690
6110	serge	2691	memclear(flink);
4363	Serge	2692	flink.handle = bo_gem->gem_handle;
		2693
6667	serge	2694	pthread_mutex_lock(&bufmgr_gem->lock);
		2695
4363	Serge	2696	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
6667	serge	2697	if (ret != 0) {
		2698	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2699	return -errno;
6667	serge	2700	}
4363	Serge	2701
		2702	bo_gem->global_name = flink.name;
		2703	bo_gem->reusable = false;
		2704
6110	serge	2705	if (DRMLISTEMPTY(&bo_gem->name_list))
		2706	DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
6667	serge	2707	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	2708	}
		2709
		2710	*name = bo_gem->global_name;
		2711	return 0;
		2712	}
		2713
		2714	/**
		2715	* Enables unlimited caching of buffer objects for reuse.
		2716	*
		2717	* This is potentially very memory expensive, as the cache at each bucket
		2718	* size is only bounded by how many buffers of that size we've managed to have
		2719	* in flight at once.
		2720	*/
		2721	void
		2722	drm_intel_bufmgr_gem_enable_reuse(drm_intel_bufmgr *bufmgr)
		2723	{
		2724	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		2725
		2726	bufmgr_gem->bo_reuse = true;
		2727	}
		2728
		2729	/**
		2730	* Enable use of fenced reloc type.
		2731	*
		2732	* New code should enable this to avoid unnecessary fence register
		2733	* allocation. If this option is not enabled, all relocs will have fence
		2734	* register allocated.
		2735	*/
		2736	void
		2737	drm_intel_bufmgr_gem_enable_fenced_relocs(drm_intel_bufmgr *bufmgr)
		2738	{
		2739	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		2740
		2741	if (bufmgr_gem->bufmgr.bo_exec == drm_intel_gem_bo_exec2)
		2742	bufmgr_gem->fenced_relocs = true;
		2743	}
		2744
		2745	/**
		2746	* Return the additional aperture space required by the tree of buffer objects
		2747	* rooted at bo.
		2748	*/
		2749	static int
		2750	drm_intel_gem_bo_get_aperture_space(drm_intel_bo *bo)
		2751	{
		2752	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2753	int i;
		2754	int total = 0;
		2755
		2756	if (bo == NULL \|\| bo_gem->included_in_check_aperture)
		2757	return 0;
		2758
		2759	total += bo->size;
		2760	bo_gem->included_in_check_aperture = true;
		2761
		2762	for (i = 0; i < bo_gem->reloc_count; i++)
		2763	total +=
		2764	drm_intel_gem_bo_get_aperture_space(bo_gem->
		2765	reloc_target_info[i].bo);
		2766
		2767	return total;
		2768	}
		2769
		2770	/**
		2771	* Count the number of buffers in this list that need a fence reg
		2772	*
		2773	* If the count is greater than the number of available regs, we'll have
		2774	* to ask the caller to resubmit a batch with fewer tiled buffers.
		2775	*
		2776	* This function over-counts if the same buffer is used multiple times.
		2777	*/
		2778	static unsigned int
		2779	drm_intel_gem_total_fences(drm_intel_bo ** bo_array, int count)
		2780	{
		2781	int i;
		2782	unsigned int total = 0;
		2783
		2784	for (i = 0; i < count; i++) {
		2785	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo_array[i];
		2786
		2787	if (bo_gem == NULL)
		2788	continue;
		2789
		2790	total += bo_gem->reloc_tree_fences;
		2791	}
		2792	return total;
		2793	}
		2794
		2795	/**
		2796	* Clear the flag set by drm_intel_gem_bo_get_aperture_space() so we're ready
		2797	* for the next drm_intel_bufmgr_check_aperture_space() call.
		2798	*/
		2799	static void
		2800	drm_intel_gem_bo_clear_aperture_space_flag(drm_intel_bo *bo)
		2801	{
		2802	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2803	int i;
		2804
		2805	if (bo == NULL \|\| !bo_gem->included_in_check_aperture)
		2806	return;
		2807
		2808	bo_gem->included_in_check_aperture = false;
		2809
		2810	for (i = 0; i < bo_gem->reloc_count; i++)
		2811	drm_intel_gem_bo_clear_aperture_space_flag(bo_gem->
		2812	reloc_target_info[i].bo);
		2813	}
		2814
		2815	/**
		2816	* Return a conservative estimate for the amount of aperture required
		2817	* for a collection of buffers. This may double-count some buffers.
		2818	*/
		2819	static unsigned int
		2820	drm_intel_gem_estimate_batch_space(drm_intel_bo **bo_array, int count)
		2821	{
		2822	int i;
		2823	unsigned int total = 0;
		2824
		2825	for (i = 0; i < count; i++) {
		2826	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo_array[i];
		2827	if (bo_gem != NULL)
		2828	total += bo_gem->reloc_tree_size;
		2829	}
		2830	return total;
		2831	}
		2832
		2833	/**
		2834	* Return the amount of aperture needed for a collection of buffers.
		2835	* This avoids double counting any buffers, at the cost of looking
		2836	* at every buffer in the set.
		2837	*/
		2838	static unsigned int
		2839	drm_intel_gem_compute_batch_space(drm_intel_bo **bo_array, int count)
		2840	{
		2841	int i;
		2842	unsigned int total = 0;
		2843
		2844	for (i = 0; i < count; i++) {
		2845	total += drm_intel_gem_bo_get_aperture_space(bo_array[i]);
		2846	/* For the first buffer object in the array, we get an
		2847	* accurate count back for its reloc_tree size (since nothing
		2848	* had been flagged as being counted yet). We can save that
		2849	* value out as a more conservative reloc_tree_size that
		2850	* avoids double-counting target buffers. Since the first
		2851	* buffer happens to usually be the batch buffer in our
		2852	* callers, this can pull us back from doing the tree
		2853	* walk on every new batch emit.
		2854	*/
		2855	if (i == 0) {
		2856	drm_intel_bo_gem *bo_gem =
		2857	(drm_intel_bo_gem *) bo_array[i];
		2858	bo_gem->reloc_tree_size = total;
		2859	}
		2860	}
		2861
		2862	for (i = 0; i < count; i++)
		2863	drm_intel_gem_bo_clear_aperture_space_flag(bo_array[i]);
		2864	return total;
		2865	}
		2866
		2867	/**
		2868	* Return -1 if the batchbuffer should be flushed before attempting to
		2869	* emit rendering referencing the buffers pointed to by bo_array.
		2870	*
		2871	* This is required because if we try to emit a batchbuffer with relocations
		2872	* to a tree of buffers that won't simultaneously fit in the aperture,
		2873	* the rendering will return an error at a point where the software is not
		2874	* prepared to recover from it.
		2875	*
		2876	* However, we also want to emit the batchbuffer significantly before we reach
		2877	* the limit, as a series of batchbuffers each of which references buffers
		2878	* covering almost all of the aperture means that at each emit we end up
		2879	* waiting to evict a buffer from the last rendering, and we get synchronous
		2880	* performance. By emitting smaller batchbuffers, we eat some CPU overhead to
		2881	* get better parallelism.
		2882	*/
		2883	static int
		2884	drm_intel_gem_check_aperture_space(drm_intel_bo **bo_array, int count)
		2885	{
		2886	drm_intel_bufmgr_gem *bufmgr_gem =
		2887	(drm_intel_bufmgr_gem *) bo_array[0]->bufmgr;
		2888	unsigned int total = 0;
		2889	unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
		2890	int total_fences;
		2891
		2892	/* Check for fence reg constraints if necessary */
		2893	if (bufmgr_gem->available_fences) {
		2894	total_fences = drm_intel_gem_total_fences(bo_array, count);
		2895	if (total_fences > bufmgr_gem->available_fences)
		2896	return -ENOSPC;
		2897	}
		2898
		2899	total = drm_intel_gem_estimate_batch_space(bo_array, count);
		2900
		2901	if (total > threshold)
		2902	total = drm_intel_gem_compute_batch_space(bo_array, count);
		2903
		2904	if (total > threshold) {
		2905	DBG("check_space: overflowed available aperture, "
		2906	"%dkb vs %dkb\n",
		2907	total / 1024, (int)bufmgr_gem->gtt_size / 1024);
		2908	return -ENOSPC;
		2909	} else {
		2910	DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024,
		2911	(int)bufmgr_gem->gtt_size / 1024);
		2912	return 0;
		2913	}
		2914	}
		2915
		2916	/*
		2917	* Disable buffer reuse for objects which are shared with the kernel
		2918	* as scanout buffers
		2919	*/
		2920	static int
		2921	drm_intel_gem_bo_disable_reuse(drm_intel_bo *bo)
		2922	{
		2923	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2924
		2925	bo_gem->reusable = false;
		2926	return 0;
		2927	}
		2928
		2929	static int
		2930	drm_intel_gem_bo_is_reusable(drm_intel_bo *bo)
		2931	{
		2932	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2933
		2934	return bo_gem->reusable;
		2935	}
		2936
		2937	static int
		2938	_drm_intel_gem_bo_references(drm_intel_bo bo, drm_intel_bo target_bo)
		2939	{
		2940	drm_intel_bo_gem bo_gem = (drm_intel_bo_gem ) bo;
		2941	int i;
		2942
		2943	for (i = 0; i < bo_gem->reloc_count; i++) {
		2944	if (bo_gem->reloc_target_info[i].bo == target_bo)
		2945	return 1;
		2946	if (bo == bo_gem->reloc_target_info[i].bo)
		2947	continue;
		2948	if (_drm_intel_gem_bo_references(bo_gem->reloc_target_info[i].bo,
		2949	target_bo))
		2950	return 1;
		2951	}
		2952
6110	serge	2953	for (i = 0; i< bo_gem->softpin_target_count; i++) {
		2954	if (bo_gem->softpin_target[i] == target_bo)
		2955	return 1;
		2956	if (_drm_intel_gem_bo_references(bo_gem->softpin_target[i], target_bo))
		2957	return 1;
		2958	}
		2959
4363	Serge	2960	return 0;
		2961	}
		2962
		2963	/** Return true if target_bo is referenced by bo's relocation tree. */
		2964	static int
		2965	drm_intel_gem_bo_references(drm_intel_bo bo, drm_intel_bo target_bo)
		2966	{
		2967	drm_intel_bo_gem target_bo_gem = (drm_intel_bo_gem ) target_bo;
		2968
		2969	if (bo == NULL \|\| target_bo == NULL)
		2970	return 0;
		2971	if (target_bo_gem->used_as_reloc_target)
		2972	return _drm_intel_gem_bo_references(bo, target_bo);
		2973	return 0;
		2974	}
		2975
		2976	static void
		2977	add_bucket(drm_intel_bufmgr_gem *bufmgr_gem, int size)
		2978	{
		2979	unsigned int i = bufmgr_gem->num_buckets;
		2980
		2981	assert(i < ARRAY_SIZE(bufmgr_gem->cache_bucket));
		2982
		2983	DRMINITLISTHEAD(&bufmgr_gem->cache_bucket[i].head);
		2984	bufmgr_gem->cache_bucket[i].size = size;
		2985	bufmgr_gem->num_buckets++;
		2986	}
		2987
		2988	static void
		2989	init_cache_buckets(drm_intel_bufmgr_gem *bufmgr_gem)
		2990	{
		2991	unsigned long size, cache_max_size = 64 * 1024 * 1024;
		2992
		2993	/* OK, so power of two buckets was too wasteful of memory.
		2994	* Give 3 other sizes between each power of two, to hopefully
		2995	* cover things accurately enough. (The alternative is
		2996	* probably to just go for exact matching of sizes, and assume
		2997	* that for things like composited window resize the tiled
		2998	* width/height alignment and rounding of sizes to pages will
		2999	* get us useful cache hit rates anyway)
		3000	*/
		3001	add_bucket(bufmgr_gem, 4096);
		3002	add_bucket(bufmgr_gem, 4096 * 2);
		3003	add_bucket(bufmgr_gem, 4096 * 3);
		3004
		3005	/* Initialize the linked lists for BO reuse cache. */
		3006	for (size = 4 * 4096; size <= cache_max_size; size *= 2) {
		3007	add_bucket(bufmgr_gem, size);
		3008
		3009	add_bucket(bufmgr_gem, size + size * 1 / 4);
		3010	add_bucket(bufmgr_gem, size + size * 2 / 4);
		3011	add_bucket(bufmgr_gem, size + size * 3 / 4);
		3012	}
		3013	}
		3014
		3015	void
		3016	drm_intel_bufmgr_gem_set_vma_cache_size(drm_intel_bufmgr *bufmgr, int limit)
		3017	{
		3018	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		3019
		3020	bufmgr_gem->vma_max = limit;
		3021
		3022	drm_intel_gem_bo_purge_vma_cache(bufmgr_gem);
		3023	}
		3024
		3025	/**
		3026	* Get the PCI ID for the device. This can be overridden by setting the
		3027	* INTEL_DEVID_OVERRIDE environment variable to the desired ID.
		3028	*/
		3029	static int
		3030	get_pci_device_id(drm_intel_bufmgr_gem *bufmgr_gem)
		3031	{
		3032	char *devid_override;
6110	serge	3033	int devid = 0;
4363	Serge	3034	int ret;
		3035	drm_i915_getparam_t gp;
		3036
6110	serge	3037	memclear(gp);
4363	Serge	3038	gp.param = I915_PARAM_CHIPSET_ID;
		3039	gp.value = &devid;
		3040	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3041	if (ret) {
		3042	fprintf(stderr, "get chip id failed: %d [%d]\n", ret, errno);
		3043	fprintf(stderr, "param: %d, val: %d\n", gp.param, *gp.value);
		3044	}
		3045	return devid;
		3046	}
		3047
		3048	int
		3049	drm_intel_bufmgr_gem_get_devid(drm_intel_bufmgr *bufmgr)
		3050	{
		3051	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		3052
		3053	return bufmgr_gem->pci_device;
		3054	}
		3055
		3056	/**
6110	serge	3057	* Sets the AUB filename.
		3058	*
		3059	* This function has to be called before drm_intel_bufmgr_gem_set_aub_dump()
		3060	* for it to have any effect.
		3061	*/
		3062	void
		3063	drm_intel_bufmgr_gem_set_aub_filename(drm_intel_bufmgr *bufmgr,
		3064	const char *filename)
		3065	{
		3066	}
		3067
		3068	/**
4363	Serge	3069	* Sets up AUB dumping.
		3070	*
		3071	* This is a trace file format that can be used with the simulator.
		3072	* Packets are emitted in a format somewhat like GPU command packets.
		3073	* You can set up a GTT and upload your objects into the referenced
		3074	* space, then send off batchbuffers and get BMPs out the other end.
		3075	*/
		3076	void
		3077	drm_intel_bufmgr_gem_set_aub_dump(drm_intel_bufmgr *bufmgr, int enable)
		3078	{
6110	serge	3079	fprintf(stderr, "libdrm aub dumping is deprecated.\n\n"
		3080	"Use intel_aubdump from intel-gpu-tools instead. Install intel-gpu-tools,\n"
		3081	"then run (for example)\n\n"
		3082	"\t$ intel_aubdump --output=trace.aub glxgears -geometry 500x500\n\n"
		3083	"See the intel_aubdump man page for more details.\n");
4363	Serge	3084	}
		3085
		3086	drm_intel_context *
		3087	drm_intel_gem_context_create(drm_intel_bufmgr *bufmgr)
		3088	{
		3089	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		3090	struct drm_i915_gem_context_create create;
		3091	drm_intel_context *context = NULL;
		3092	int ret;
		3093
5068	serge	3094	context = calloc(1, sizeof(*context));
		3095	if (!context)
		3096	return NULL;
		3097
6110	serge	3098	memclear(create);
4363	Serge	3099	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
		3100	if (ret != 0) {
		3101	DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
		3102	strerror(errno));
5068	serge	3103	free(context);
4363	Serge	3104	return NULL;
		3105	}
		3106
		3107	context->ctx_id = create.ctx_id;
		3108	context->bufmgr = bufmgr;
		3109
		3110	return context;
		3111	}
		3112
		3113	void
		3114	drm_intel_gem_context_destroy(drm_intel_context *ctx)
		3115	{
		3116	drm_intel_bufmgr_gem *bufmgr_gem;
		3117	struct drm_i915_gem_context_destroy destroy;
		3118	int ret;
		3119
		3120	if (ctx == NULL)
		3121	return;
		3122
6110	serge	3123	memclear(destroy);
4363	Serge	3124
		3125	bufmgr_gem = (drm_intel_bufmgr_gem *)ctx->bufmgr;
		3126	destroy.ctx_id = ctx->ctx_id;
		3127	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY,
		3128	&destroy);
		3129	if (ret != 0)
		3130	fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
		3131	strerror(errno));
		3132
		3133	free(ctx);
		3134	}
		3135
		3136	int
		3137	drm_intel_reg_read(drm_intel_bufmgr *bufmgr,
		3138	uint32_t offset,
		3139	uint64_t *result)
		3140	{
		3141	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		3142	struct drm_i915_reg_read reg_read;
		3143	int ret;
		3144
6110	serge	3145	memclear(reg_read);
4363	Serge	3146	reg_read.offset = offset;
		3147
		3148	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_REG_READ, ®_read);
		3149
		3150	*result = reg_read.val;
		3151	return ret;
		3152	}
		3153
6110	serge	3154	int
		3155	drm_intel_get_subslice_total(int fd, unsigned int *subslice_total)
		3156	{
		3157	drm_i915_getparam_t gp;
		3158	int ret;
4363	Serge	3159
6110	serge	3160	memclear(gp);
		3161	gp.value = (int*)subslice_total;
		3162	gp.param = I915_PARAM_SUBSLICE_TOTAL;
		3163	ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3164	if (ret)
		3165	return -errno;
		3166
		3167	return 0;
		3168	}
		3169
		3170	int
		3171	drm_intel_get_eu_total(int fd, unsigned int *eu_total)
		3172	{
		3173	drm_i915_getparam_t gp;
		3174	int ret;
		3175
		3176	memclear(gp);
		3177	gp.value = (int*)eu_total;
		3178	gp.param = I915_PARAM_EU_TOTAL;
		3179	ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3180	if (ret)
		3181	return -errno;
		3182
		3183	return 0;
		3184	}
		3185
4363	Serge	3186	/**
		3187	* Annotate the given bo for use in aub dumping.
		3188	*
		3189	* \param annotations is an array of drm_intel_aub_annotation objects
		3190	* describing the type of data in various sections of the bo. Each
		3191	* element of the array specifies the type and subtype of a section of
		3192	* the bo, and the past-the-end offset of that section. The elements
		3193	* of \c annotations must be sorted so that ending_offset is
		3194	* increasing.
		3195	*
		3196	* \param count is the number of elements in the \c annotations array.
		3197	* If \c count is zero, then \c annotations will not be dereferenced.
		3198	*
		3199	* Annotations are copied into a private data structure, so caller may
		3200	* re-use the memory pointed to by \c annotations after the call
		3201	* returns.
		3202	*
		3203	* Annotations are stored for the lifetime of the bo; to reset to the
		3204	* default state (no annotations), call this function with a \c count
		3205	* of zero.
		3206	*/
		3207	void
		3208	drm_intel_bufmgr_gem_set_aub_annotations(drm_intel_bo *bo,
		3209	drm_intel_aub_annotation *annotations,
		3210	unsigned count)
		3211	{
6110	serge	3212	}
		3213
6667	serge	3214	static __gthread_mutex_t bufmgr_list_mutex = {0, -1};
6110	serge	3215	static drmMMListHead bufmgr_list = { &bufmgr_list, &bufmgr_list };
		3216
		3217	static drm_intel_bufmgr_gem *
		3218	drm_intel_bufmgr_gem_find(int fd)
		3219	{
		3220	drm_intel_bufmgr_gem *bufmgr_gem;
		3221
		3222	DRMLISTFOREACHENTRY(bufmgr_gem, &bufmgr_list, managers) {
		3223	if (bufmgr_gem->fd == fd) {
		3224	atomic_inc(&bufmgr_gem->refcount);
		3225	return bufmgr_gem;
		3226	}
4363	Serge	3227	}
6110	serge	3228
		3229	return NULL;
4363	Serge	3230	}
		3231
6110	serge	3232	static void
		3233	drm_intel_bufmgr_gem_unref(drm_intel_bufmgr *bufmgr)
		3234	{
		3235	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem )bufmgr;
		3236
		3237	if (atomic_add_unless(&bufmgr_gem->refcount, -1, 1)) {
6667	serge	3238	pthread_mutex_lock(&bufmgr_list_mutex);
6110	serge	3239
		3240	if (atomic_dec_and_test(&bufmgr_gem->refcount)) {
		3241	DRMLISTDEL(&bufmgr_gem->managers);
		3242	drm_intel_bufmgr_gem_destroy(bufmgr);
		3243	}
		3244
6667	serge	3245	pthread_mutex_unlock(&bufmgr_list_mutex);
6110	serge	3246	}
		3247	}
		3248
4363	Serge	3249	/**
		3250	* Initializes the GEM buffer manager, which uses the kernel to allocate, map,
		3251	* and manage map buffer objections.
		3252	*
		3253	* \param fd File descriptor of the opened DRM device.
		3254	*/
		3255	drm_intel_bufmgr *
		3256	drm_intel_bufmgr_gem_init(int fd, int batch_size)
		3257	{
		3258	drm_intel_bufmgr_gem *bufmgr_gem;
		3259	struct drm_i915_gem_get_aperture aperture;
		3260	drm_i915_getparam_t gp;
		3261	int ret, tmp;
		3262	bool exec2 = false;
		3263
6667	serge	3264	if(bufmgr_list_mutex.handle == -1)
		3265	__gthread_mutex_init_function(&bufmgr_gem->lock);
6110	serge	3266
6667	serge	3267	pthread_mutex_lock(&bufmgr_list_mutex);
		3268
6110	serge	3269	bufmgr_gem = drm_intel_bufmgr_gem_find(fd);
		3270	if (bufmgr_gem)
		3271	goto exit;
		3272
4363	Serge	3273	bufmgr_gem = calloc(1, sizeof(*bufmgr_gem));
		3274	if (bufmgr_gem == NULL)
6110	serge	3275	goto exit;
4363	Serge	3276
		3277	bufmgr_gem->fd = fd;
6110	serge	3278	atomic_set(&bufmgr_gem->refcount, 1);
4363	Serge	3279
6667	serge	3280	__gthread_mutex_init_function(&bufmgr_gem->lock);
4363	Serge	3281
6110	serge	3282	memclear(aperture);
4363	Serge	3283	ret = drmIoctl(bufmgr_gem->fd,
		3284	DRM_IOCTL_I915_GEM_GET_APERTURE,
		3285	&aperture);
		3286
		3287	if (ret == 0)
		3288	bufmgr_gem->gtt_size = aperture.aper_available_size;
		3289	else {
6110	serge	3290	fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
4363	Serge	3291	strerror(errno));
		3292	bufmgr_gem->gtt_size = 128 * 1024 * 1024;
6110	serge	3293	fprintf(stderr, "Assuming %dkB available aperture size.\n"
4363	Serge	3294	"May lead to reduced performance or incorrect "
		3295	"rendering.\n",
		3296	(int)bufmgr_gem->gtt_size / 1024);
		3297	}
		3298
		3299	bufmgr_gem->pci_device = get_pci_device_id(bufmgr_gem);
		3300
		3301	if (IS_GEN2(bufmgr_gem->pci_device))
		3302	bufmgr_gem->gen = 2;
		3303	else if (IS_GEN3(bufmgr_gem->pci_device))
		3304	bufmgr_gem->gen = 3;
		3305	else if (IS_GEN4(bufmgr_gem->pci_device))
		3306	bufmgr_gem->gen = 4;
		3307	else if (IS_GEN5(bufmgr_gem->pci_device))
		3308	bufmgr_gem->gen = 5;
		3309	else if (IS_GEN6(bufmgr_gem->pci_device))
		3310	bufmgr_gem->gen = 6;
		3311	else if (IS_GEN7(bufmgr_gem->pci_device))
		3312	bufmgr_gem->gen = 7;
6110	serge	3313	else if (IS_GEN8(bufmgr_gem->pci_device))
		3314	bufmgr_gem->gen = 8;
		3315	else if (IS_GEN9(bufmgr_gem->pci_device))
		3316	bufmgr_gem->gen = 9;
4363	Serge	3317	else {
		3318	free(bufmgr_gem);
6110	serge	3319	bufmgr_gem = NULL;
		3320	goto exit;
4363	Serge	3321	}
		3322
		3323	if (IS_GEN3(bufmgr_gem->pci_device) &&
		3324	bufmgr_gem->gtt_size > 25610241024) {
		3325	/* The unmappable part of gtt on gen 3 (i.e. above 256MB) can't
		3326	* be used for tiled blits. To simplify the accounting, just
		3327	* substract the unmappable part (fixed to 256MB on all known
		3328	* gen3 devices) if the kernel advertises it. */
		3329	bufmgr_gem->gtt_size -= 25610241024;
		3330	}
		3331
6110	serge	3332	memclear(gp);
4363	Serge	3333	gp.value = &tmp;
		3334
		3335	gp.param = I915_PARAM_HAS_EXECBUF2;
		3336	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3337	if (!ret)
		3338	exec2 = true;
		3339
		3340	gp.param = I915_PARAM_HAS_BSD;
		3341	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3342	bufmgr_gem->has_bsd = ret == 0;
		3343
		3344	gp.param = I915_PARAM_HAS_BLT;
		3345	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3346	bufmgr_gem->has_blt = ret == 0;
		3347
		3348	gp.param = I915_PARAM_HAS_RELAXED_FENCING;
		3349	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3350	bufmgr_gem->has_relaxed_fencing = ret == 0;
		3351
6110	serge	3352	bufmgr_gem->bufmgr.bo_alloc_userptr = NULL;
		3353
4363	Serge	3354	gp.param = I915_PARAM_HAS_WAIT_TIMEOUT;
		3355	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3356	bufmgr_gem->has_wait_timeout = ret == 0;
		3357
		3358	gp.param = I915_PARAM_HAS_LLC;
		3359	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3360	if (ret != 0) {
		3361	/* Kernel does not supports HAS_LLC query, fallback to GPU
		3362	* generation detection and assume that we have LLC on GEN6/7
		3363	*/
		3364	bufmgr_gem->has_llc = (IS_GEN6(bufmgr_gem->pci_device) \|
		3365	IS_GEN7(bufmgr_gem->pci_device));
		3366	} else
		3367	bufmgr_gem->has_llc = *gp.value;
		3368
		3369	gp.param = I915_PARAM_HAS_VEBOX;
		3370	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3371	bufmgr_gem->has_vebox = (ret == 0) & (*gp.value > 0);
		3372
6110	serge	3373	gp.param = I915_PARAM_HAS_EXEC_SOFTPIN;
		3374	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3375	if (ret == 0 && *gp.value > 0)
		3376	bufmgr_gem->bufmgr.bo_set_softpin_offset = drm_intel_gem_bo_set_softpin_offset;
		3377
4363	Serge	3378	if (bufmgr_gem->gen < 4) {
		3379	gp.param = I915_PARAM_NUM_FENCES_AVAIL;
		3380	gp.value = &bufmgr_gem->available_fences;
		3381	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
		3382	if (ret) {
		3383	fprintf(stderr, "get fences failed: %d [%d]\n", ret,
		3384	errno);
		3385	fprintf(stderr, "param: %d, val: %d\n", gp.param,
		3386	*gp.value);
		3387	bufmgr_gem->available_fences = 0;
		3388	} else {
		3389	/* XXX The kernel reports the total number of fences,
		3390	* including any that may be pinned.
		3391	*
		3392	* We presume that there will be at least one pinned
		3393	* fence for the scanout buffer, but there may be more
		3394	* than one scanout and the user may be manually
		3395	* pinning buffers. Let's move to execbuffer2 and
		3396	* thereby forget the insanity of using fences...
		3397	*/
		3398	bufmgr_gem->available_fences -= 2;
		3399	if (bufmgr_gem->available_fences < 0)
		3400	bufmgr_gem->available_fences = 0;
		3401	}
		3402	}
		3403
		3404	/* Let's go with one relocation per every 2 dwords (but round down a bit
		3405	* since a power of two will mean an extra page allocation for the reloc
		3406	* buffer).
		3407	*
		3408	* Every 4 was too few for the blender benchmark.
		3409	*/
		3410	bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
		3411
		3412	bufmgr_gem->bufmgr.bo_alloc = drm_intel_gem_bo_alloc;
5068	serge	3413	bufmgr_gem->bufmgr.bo_alloc_for_render =
		3414	drm_intel_gem_bo_alloc_for_render;
4363	Serge	3415	bufmgr_gem->bufmgr.bo_alloc_tiled = drm_intel_gem_bo_alloc_tiled;
		3416	bufmgr_gem->bufmgr.bo_reference = drm_intel_gem_bo_reference;
		3417	bufmgr_gem->bufmgr.bo_unreference = drm_intel_gem_bo_unreference;
		3418	bufmgr_gem->bufmgr.bo_map = drm_intel_gem_bo_map;
		3419	bufmgr_gem->bufmgr.bo_unmap = drm_intel_gem_bo_unmap;
		3420	bufmgr_gem->bufmgr.bo_subdata = drm_intel_gem_bo_subdata;
6110	serge	3421	bufmgr_gem->bufmgr.bo_get_subdata = drm_intel_gem_bo_get_subdata;
4363	Serge	3422	bufmgr_gem->bufmgr.bo_wait_rendering = drm_intel_gem_bo_wait_rendering;
		3423	bufmgr_gem->bufmgr.bo_emit_reloc = drm_intel_gem_bo_emit_reloc;
		3424	bufmgr_gem->bufmgr.bo_emit_reloc_fence = drm_intel_gem_bo_emit_reloc_fence;
		3425	bufmgr_gem->bufmgr.bo_pin = drm_intel_gem_bo_pin;
		3426	bufmgr_gem->bufmgr.bo_unpin = drm_intel_gem_bo_unpin;
		3427	bufmgr_gem->bufmgr.bo_get_tiling = drm_intel_gem_bo_get_tiling;
		3428	bufmgr_gem->bufmgr.bo_set_tiling = drm_intel_gem_bo_set_tiling;
		3429	bufmgr_gem->bufmgr.bo_flink = drm_intel_gem_bo_flink;
		3430	/* Use the new one if available */
6110	serge	3431	if (exec2) {
4363	Serge	3432	bufmgr_gem->bufmgr.bo_exec = drm_intel_gem_bo_exec2;
		3433	bufmgr_gem->bufmgr.bo_mrb_exec = drm_intel_gem_bo_mrb_exec2;
6110	serge	3434	} else
		3435	bufmgr_gem->bufmgr.bo_exec = drm_intel_gem_bo_exec;
		3436	bufmgr_gem->bufmgr.bo_busy = drm_intel_gem_bo_busy;
4363	Serge	3437	bufmgr_gem->bufmgr.bo_madvise = drm_intel_gem_bo_madvise;
6110	serge	3438	bufmgr_gem->bufmgr.destroy = drm_intel_bufmgr_gem_unref;
4363	Serge	3439	bufmgr_gem->bufmgr.debug = 0;
		3440	bufmgr_gem->bufmgr.check_aperture_space =
		3441	drm_intel_gem_check_aperture_space;
		3442	bufmgr_gem->bufmgr.bo_disable_reuse = drm_intel_gem_bo_disable_reuse;
		3443	bufmgr_gem->bufmgr.bo_is_reusable = drm_intel_gem_bo_is_reusable;
		3444	// bufmgr_gem->bufmgr.get_pipe_from_crtc_id =
		3445	// drm_intel_gem_get_pipe_from_crtc_id;
		3446	bufmgr_gem->bufmgr.bo_references = drm_intel_gem_bo_references;
		3447
		3448	DRMINITLISTHEAD(&bufmgr_gem->named);
		3449	init_cache_buckets(bufmgr_gem);
		3450
		3451	DRMINITLISTHEAD(&bufmgr_gem->vma_cache);
		3452	bufmgr_gem->vma_max = -1; /* unlimited by default */
		3453
6110	serge	3454	DRMLISTADD(&bufmgr_gem->managers, &bufmgr_list);
		3455
		3456	exit:
6667	serge	3457	pthread_mutex_unlock(&bufmgr_list_mutex);
6110	serge	3458
		3459	return bufmgr_gem != NULL ? &bufmgr_gem->bufmgr : NULL;
4363	Serge	3460	}
		3461
		3462
		3463	drm_intel_bo *
		3464	bo_create_from_gem_handle(drm_intel_bufmgr *bufmgr,
		3465	unsigned int size, unsigned int handle)
		3466	{
		3467	drm_intel_bufmgr_gem bufmgr_gem = (drm_intel_bufmgr_gem ) bufmgr;
		3468	drm_intel_bo_gem *bo_gem;
		3469	int ret;
		3470	struct drm_i915_gem_get_tiling get_tiling;
		3471	drmMMListHead *list;
		3472
		3473	/* At the moment most applications only have a few named bo.
		3474	* For instance, in a DRI client only the render buffers passed
		3475	* between X and the client are named. And since X returns the
		3476	* alternating names for the front/back buffer a linear search
		3477	* provides a sufficiently fast match.
		3478	*/
6667	serge	3479	pthread_mutex_lock(&bufmgr_gem->lock);
4363	Serge	3480	for (list = bufmgr_gem->named.next;
		3481	list != &bufmgr_gem->named;
		3482	list = list->next) {
		3483	bo_gem = DRMLISTENTRY(drm_intel_bo_gem, list, name_list);
		3484	if (bo_gem->gem_handle == handle) {
6667	serge	3485	drm_intel_gem_bo_reference(&bo_gem->bo);
		3486	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	3487	return &bo_gem->bo;
		3488	}
		3489	}
		3490
		3491	bo_gem = calloc(1, sizeof(*bo_gem));
		3492	if (!bo_gem)
		3493	return NULL;
		3494
		3495	bo_gem->bo.size = size;
		3496	bo_gem->bo.offset = 0;
		3497	bo_gem->bo.virtual = NULL;
		3498	bo_gem->bo.bufmgr = bufmgr;
		3499	bo_gem->name = NULL;
		3500	atomic_set(&bo_gem->refcount, 1);
		3501	bo_gem->validate_index = -1;
		3502	bo_gem->gem_handle = handle;
		3503	bo_gem->bo.handle = handle;
		3504	bo_gem->global_name = 0;
		3505	bo_gem->reusable = false;
		3506
6110	serge	3507	memclear(get_tiling);
4363	Serge	3508	get_tiling.handle = bo_gem->gem_handle;
		3509	ret = drmIoctl(bufmgr_gem->fd,
		3510	DRM_IOCTL_I915_GEM_GET_TILING,
		3511	&get_tiling);
		3512	if (ret != 0) {
		3513	drm_intel_gem_bo_unreference(&bo_gem->bo);
6667	serge	3514	pthread_mutex_unlock(&bufmgr_gem->lock);
4363	Serge	3515	return NULL;
		3516	}
		3517	bo_gem->tiling_mode = get_tiling.tiling_mode;
		3518	bo_gem->swizzle_mode = get_tiling.swizzle_mode;
		3519	/* XXX stride is unknown */
6110	serge	3520	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
4363	Serge	3521
		3522	DRMINITLISTHEAD(&bo_gem->vma_list);
		3523	DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
6667	serge	3524	pthread_mutex_unlock(&bufmgr_gem->lock);
		3525	printf("bo_create_from_gem_handle: %d\n", handle);
4363	Serge	3526
		3527	return &bo_gem->bo;
		3528	}

Subversion Repositories Kolibri OS

(root)/contrib/sdk/sources/libdrm/intel/intel_bufmgr_gem.c @ 5368 – Rev 6667