WebSVN – Kolibri OS – Blame – /contrib/sdk/sources/Intel-2D/sna/kgem.c

Rev	Author	Line No.	Line
4304	Serge	1	/*
		2	* Copyright (c) 2011 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
		20	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		21	* SOFTWARE.
		22	*
		23	* Authors:
		24	* Chris Wilson
		25	*
		26	*/
		27
		28	#ifdef HAVE_CONFIG_H
		29	#include "config.h"
		30	#endif
		31
		32	#include "sna.h"
		33	#include "sna_reg.h"
		34
		35	#include
		36	#include
		37	#include
		38
		39	#ifdef HAVE_VALGRIND
		40	#include
		41	#include
		42	#endif
		43
		44	#ifdef HAVE_STRUCT_SYSINFO_TOTALRAM
		45	#include
		46	#endif
		47
		48	#include "sna_cpuid.h"
		49
4368	Serge	50
4304	Serge	51	static struct kgem_bo *
		52	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
		53
		54	static struct kgem_bo *
		55	search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
		56
		57	#define DBG_NO_HW 0
		58	#define DBG_NO_TILING 0
		59	#define DBG_NO_CACHE 0
		60	#define DBG_NO_CACHE_LEVEL 0
		61	#define DBG_NO_CPU 0
		62	#define DBG_NO_CREATE2 1
		63	#define DBG_NO_USERPTR 0
		64	#define DBG_NO_UNSYNCHRONIZED_USERPTR 0
		65	#define DBG_NO_LLC 0
		66	#define DBG_NO_SEMAPHORES 0
		67	#define DBG_NO_MADV 1
		68	#define DBG_NO_UPLOAD_CACHE 0
		69	#define DBG_NO_UPLOAD_ACTIVE 0
		70	#define DBG_NO_MAP_UPLOAD 0
		71	#define DBG_NO_RELAXED_FENCING 0
		72	#define DBG_NO_SECURE_BATCHES 0
		73	#define DBG_NO_PINNED_BATCHES 0
		74	#define DBG_NO_FAST_RELOC 0
		75	#define DBG_NO_HANDLE_LUT 1
		76	#define DBG_NO_WT 0
		77	#define DBG_DUMP 0
		78
		79	#define FORCE_MMAP_SYNC 0 /* ((1 << DOMAIN_CPU) \| (1 << DOMAIN_GTT)) */
		80
		81	#ifndef DEBUG_SYNC
		82	#define DEBUG_SYNC 0
		83	#endif
		84
		85
		86	#if 0
		87	#define ASSERT_IDLE(kgem__, handle__) assert(!__kgem_busy(kgem__, handle__))
		88	#define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__) assert(!(expect__) \|\| !__kgem_busy(kgem__, handle__))
		89	#else
		90	#define ASSERT_IDLE(kgem__, handle__)
		91	#define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__)
		92	#endif
		93
		94	/* Worst case seems to be 965gm where we cannot write within a cacheline that
		95	* is being simultaneously being read by the GPU, or within the sampler
		96	* prefetch. In general, the chipsets seem to have a requirement that sampler
		97	* offsets be aligned to a cacheline (64 bytes).
		98	*/
		99	#define UPLOAD_ALIGNMENT 128
		100
		101	#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
		102	#define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE)
		103
		104	#define MAX_GTT_VMA_CACHE 512
		105	#define MAX_CPU_VMA_CACHE INT16_MAX
		106	#define MAP_PRESERVE_TIME 10
		107
		108	#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 1))
		109	#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 3))
		110	#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 2)
		111	#define __MAP_TYPE(ptr) ((uintptr_t)(ptr) & 3)
		112
		113	#define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) \| (ring)))
		114
		115	#define LOCAL_I915_PARAM_HAS_BLT 11
		116	#define LOCAL_I915_PARAM_HAS_RELAXED_FENCING 12
		117	#define LOCAL_I915_PARAM_HAS_RELAXED_DELTA 15
		118	#define LOCAL_I915_PARAM_HAS_SEMAPHORES 20
		119	#define LOCAL_I915_PARAM_HAS_SECURE_BATCHES 23
		120	#define LOCAL_I915_PARAM_HAS_PINNED_BATCHES 24
		121	#define LOCAL_I915_PARAM_HAS_NO_RELOC 25
		122	#define LOCAL_I915_PARAM_HAS_HANDLE_LUT 26
		123	#define LOCAL_I915_PARAM_HAS_WT 27
		124
		125	#define LOCAL_I915_EXEC_IS_PINNED (1<<10)
		126	#define LOCAL_I915_EXEC_NO_RELOC (1<<11)
		127	#define LOCAL_I915_EXEC_HANDLE_LUT (1<<12)
		128	struct local_i915_gem_userptr {
		129	uint64_t user_ptr;
		130	uint64_t user_size;
		131	uint32_t flags;
		132	#define I915_USERPTR_READ_ONLY (1<<0)
		133	#define I915_USERPTR_UNSYNCHRONIZED (1<<31)
		134	uint32_t handle;
		135	};
		136
		137	#define UNCACHED 0
		138	#define SNOOPED 1
		139	#define DISPLAY 2
		140
		141	struct local_i915_gem_caching {
		142	uint32_t handle;
		143	uint32_t caching;
		144	};
		145
		146	#define LOCAL_IOCTL_I915_GEM_SET_CACHING SRV_I915_GEM_SET_CACHING
		147
		148	struct local_fbinfo {
		149	int width;
		150	int height;
		151	int pitch;
		152	int tiling;
		153	};
		154
		155	struct kgem_buffer {
		156	struct kgem_bo base;
		157	void *mem;
		158	uint32_t used;
		159	uint32_t need_io : 1;
		160	uint32_t write : 2;
		161	uint32_t mmapped : 1;
		162	};
		163
		164	static struct kgem_bo *__kgem_freed_bo;
		165	static struct kgem_request *__kgem_freed_request;
		166	static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;
		167
		168	static inline int bytes(struct kgem_bo *bo)
		169	{
		170	return __kgem_bo_size(bo);
		171	}
		172
		173	#define bucket(B) (B)->size.pages.bucket
		174	#define num_pages(B) (B)->size.pages.count
		175
		176	#ifdef DEBUG_MEMORY
		177	static void debug_alloc(struct kgem *kgem, size_t size)
		178	{
		179	kgem->debug_memory.bo_allocs++;
		180	kgem->debug_memory.bo_bytes += size;
		181	}
		182	static void debug_alloc__bo(struct kgem kgem, struct kgem_bo bo)
		183	{
		184	debug_alloc(kgem, bytes(bo));
		185	}
		186	#else
		187	#define debug_alloc(k, b)
		188	#define debug_alloc__bo(k, b)
		189	#endif
		190
		191	#ifndef NDEBUG
		192	static void assert_tiling(struct kgem kgem, struct kgem_bo bo)
		193	{
		194	struct drm_i915_gem_get_tiling tiling;
		195
		196	assert(bo);
		197
		198	VG_CLEAR(tiling);
		199	tiling.handle = bo->handle;
		200	tiling.tiling_mode = -1;
		201	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling);
		202	assert(tiling.tiling_mode == bo->tiling);
		203	}
		204	#else
		205	#define assert_tiling(kgem, bo)
		206	#endif
		207
		208	static void kgem_sna_reset(struct kgem *kgem)
		209	{
		210	struct sna *sna = container_of(kgem, struct sna, kgem);
		211
		212	sna->render.reset(sna);
		213	sna->blt_state.fill_bo = 0;
		214	}
		215
		216	static void kgem_sna_flush(struct kgem *kgem)
		217	{
		218	struct sna *sna = container_of(kgem, struct sna, kgem);
		219
		220	sna->render.flush(sna);
		221
		222	// if (sna->render.solid_cache.dirty)
		223	// sna_render_flush_solid(sna);
		224	}
		225
		226	static bool gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
		227	{
		228	struct drm_i915_gem_set_tiling set_tiling;
		229	int ret;
		230
		231	if (DBG_NO_TILING)
		232	return false;
		233
		234	VG_CLEAR(set_tiling);
		235	do {
		236	set_tiling.handle = handle;
		237	set_tiling.tiling_mode = tiling;
		238	set_tiling.stride = stride;
		239
		240	ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling);
		241	} while (ret != 0);
		242	return ret == 0;
		243	}
		244
		245	static bool gem_set_caching(int fd, uint32_t handle, int caching)
		246	{
		247	struct local_i915_gem_caching arg;
		248
		249	VG_CLEAR(arg);
		250	arg.handle = handle;
		251	arg.caching = caching;
		252	return drmIoctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0;
		253	}
		254
		255
		256
		257
		258
		259	static bool __kgem_throttle_retire(struct kgem *kgem, unsigned flags)
		260	{
		261	if (flags & CREATE_NO_RETIRE) {
		262	DBG(("%s: not retiring per-request\n", __FUNCTION__));
		263	return false;
		264	}
		265
		266	if (!kgem->need_retire) {
		267	DBG(("%s: nothing to retire\n", __FUNCTION__));
		268	return false;
		269	}
		270
		271	if (kgem_retire(kgem))
		272	return true;
		273
		274	if (flags & CREATE_NO_THROTTLE \|\| !kgem->need_throttle) {
		275	DBG(("%s: not throttling\n", __FUNCTION__));
		276	return false;
		277	}
		278
		279	kgem_throttle(kgem);
		280	return kgem_retire(kgem);
		281	}
		282
		283	static void __kgem_bo_map__gtt(struct kgem kgem, struct kgem_bo *bo)
		284	{
		285	struct drm_i915_gem_mmap_gtt mmap_arg;
		286	void *ptr;
		287
		288	DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__,
		289	bo->handle, bytes(bo)));
		290	assert(bo->proxy == NULL);
		291	assert(!bo->snoop);
		292	assert(kgem_bo_can_map(kgem, bo));
		293
		294	retry_gtt:
		295	VG_CLEAR(mmap_arg);
		296	mmap_arg.handle = bo->handle;
		297	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg)) {
		298
		299	(void)__kgem_throttle_retire(kgem, 0);
		300	if (kgem_expire_cache(kgem))
		301	goto retry_gtt;
		302
		303	if (kgem->need_expire) {
		304	kgem_cleanup_cache(kgem);
		305	goto retry_gtt;
		306	}
		307
		308	printf("%s: failed to retrieve GTT offset for handle=%d\n",
		309	__FUNCTION__, bo->handle);
		310	return NULL;
		311	}
		312
		313	retry_mmap:
		314	ptr = (void*)(int)mmap_arg.offset;
		315	if (ptr == NULL) {
		316	ErrorF("%s: failed to mmap handle=%d, %d bytes, into GTT domain\n",
		317	__FUNCTION__, bo->handle, bytes(bo));
		318	ptr = NULL;
		319	}
		320
		321	return ptr;
		322	}
		323
		324	static int __gem_write(int fd, uint32_t handle,
		325	int offset, int length,
		326	const void *src)
		327	{
		328	struct drm_i915_gem_pwrite pwrite;
		329
		330	DBG(("%s(handle=%d, offset=%d, len=%d)\n", __FUNCTION__,
		331	handle, offset, length));
		332
		333	VG_CLEAR(pwrite);
		334	pwrite.handle = handle;
		335	pwrite.offset = offset;
		336	pwrite.size = length;
		337	pwrite.data_ptr = (uintptr_t)src;
		338	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
		339	}
		340
		341	static int gem_write(int fd, uint32_t handle,
		342	int offset, int length,
		343	const void *src)
		344	{
		345	struct drm_i915_gem_pwrite pwrite;
		346
		347	DBG(("%s(handle=%d, offset=%d, len=%d)\n", __FUNCTION__,
		348	handle, offset, length));
		349
		350	VG_CLEAR(pwrite);
		351	pwrite.handle = handle;
		352	/* align the transfer to cachelines; fortuitously this is safe! */
		353	if ((offset \| length) & 63) {
		354	pwrite.offset = offset & ~63;
		355	pwrite.size = ALIGN(offset+length, 64) - pwrite.offset;
		356	pwrite.data_ptr = (uintptr_t)src + pwrite.offset - offset;
		357	} else {
		358	pwrite.offset = offset;
		359	pwrite.size = length;
		360	pwrite.data_ptr = (uintptr_t)src;
		361	}
		362	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
		363	}
		364
		365
		366	bool __kgem_busy(struct kgem *kgem, int handle)
		367	{
		368	struct drm_i915_gem_busy busy;
		369
		370	VG_CLEAR(busy);
		371	busy.handle = handle;
		372	busy.busy = !kgem->wedged;
		373	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
		374	DBG(("%s: handle=%d, busy=%d, wedged=%d\n",
		375	__FUNCTION__, handle, busy.busy, kgem->wedged));
		376
		377	return busy.busy;
		378	}
		379
		380	static void kgem_bo_retire(struct kgem kgem, struct kgem_bo bo)
		381	{
		382	DBG(("%s: retiring bo handle=%d (needed flush? %d), rq? %d [busy?=%d]\n",
		383	__FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL,
		384	__kgem_busy(kgem, bo->handle)));
		385	assert(bo->exec == NULL);
		386	assert(list_is_empty(&bo->vma));
		387
		388	if (bo->rq) {
		389	if (!__kgem_busy(kgem, bo->handle)) {
		390	__kgem_bo_clear_busy(bo);
		391	kgem_retire(kgem);
		392	}
		393	} else {
		394	assert(!bo->needs_flush);
		395	ASSERT_IDLE(kgem, bo->handle);
		396	}
		397	}
		398
		399	bool kgem_bo_write(struct kgem kgem, struct kgem_bo bo,
		400	const void *data, int length)
		401	{
		402	assert(bo->refcnt);
		403	assert(!bo->purged);
		404	assert(bo->proxy == NULL);
		405	ASSERT_IDLE(kgem, bo->handle);
		406
		407	assert(length <= bytes(bo));
		408	if (gem_write(kgem->fd, bo->handle, 0, length, data))
		409	return false;
		410
		411	DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain));
		412	if (bo->exec == NULL) {
		413	kgem_bo_retire(kgem, bo);
		414	bo->domain = DOMAIN_NONE;
		415	}
		416	bo->gtt_dirty = true;
		417	return true;
		418	}
		419
		420	static uint32_t gem_create(int fd, int num_pages)
		421	{
		422	struct drm_i915_gem_create create;
		423
		424	VG_CLEAR(create);
		425	create.handle = 0;
		426	create.size = PAGE_SIZE * num_pages;
		427	(void)drmIoctl(fd, DRM_IOCTL_I915_GEM_CREATE, &create);
		428
		429	return create.handle;
		430	}
		431
		432	static bool
		433	kgem_bo_set_purgeable(struct kgem kgem, struct kgem_bo bo)
		434	{
		435	#if DBG_NO_MADV
		436	return true;
		437	#else
		438	struct drm_i915_gem_madvise madv;
		439
		440	assert(bo->exec == NULL);
		441	assert(!bo->purged);
		442
		443	VG_CLEAR(madv);
		444	madv.handle = bo->handle;
		445	madv.madv = I915_MADV_DONTNEED;
		446	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
		447	bo->purged = 1;
		448	kgem->need_purge \|= !madv.retained && bo->domain == DOMAIN_GPU;
		449	return madv.retained;
		450	}
		451
		452	return true;
		453	#endif
		454	}
		455
		456	static bool
		457	kgem_bo_is_retained(struct kgem kgem, struct kgem_bo bo)
		458	{
		459	#if DBG_NO_MADV
		460	return true;
		461	#else
		462	struct drm_i915_gem_madvise madv;
		463
		464	if (!bo->purged)
		465	return true;
		466
		467	VG_CLEAR(madv);
		468	madv.handle = bo->handle;
		469	madv.madv = I915_MADV_DONTNEED;
		470	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0)
		471	return madv.retained;
		472
		473	return false;
		474	#endif
		475	}
		476
		477	static bool
		478	kgem_bo_clear_purgeable(struct kgem kgem, struct kgem_bo bo)
		479	{
		480	#if DBG_NO_MADV
		481	return true;
		482	#else
		483	struct drm_i915_gem_madvise madv;
		484
		485	assert(bo->purged);
		486
		487	VG_CLEAR(madv);
		488	madv.handle = bo->handle;
		489	madv.madv = I915_MADV_WILLNEED;
		490	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
		491	bo->purged = !madv.retained;
		492	kgem->need_purge \|= !madv.retained && bo->domain == DOMAIN_GPU;
		493	return madv.retained;
		494	}
		495
		496	return false;
		497	#endif
		498	}
		499
		500	static void gem_close(int fd, uint32_t handle)
		501	{
		502	struct drm_gem_close close;
		503
		504	VG_CLEAR(close);
		505	close.handle = handle;
		506	(void)drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, &close);
		507	}
		508
		509	constant inline static unsigned long __fls(unsigned long word)
		510	{
		511	#if defined(__GNUC__) && (defined(__i386__) \|\| defined(__x86__) \|\| defined(__x86_64__))
		512	asm("bsr %1,%0"
		513	: "=r" (word)
		514	: "rm" (word));
		515	return word;
		516	#else
		517	unsigned int v = 0;
		518
		519	while (word >>= 1)
		520	v++;
		521
		522	return v;
		523	#endif
		524	}
		525
		526	constant inline static int cache_bucket(int num_pages)
		527	{
		528	return __fls(num_pages);
		529	}
		530
		531	static struct kgem_bo __kgem_bo_init(struct kgem_bo bo,
		532	int handle, int num_pages)
		533	{
		534	assert(num_pages);
		535	memset(bo, 0, sizeof(*bo));
		536
		537	bo->refcnt = 1;
		538	bo->handle = handle;
		539	bo->target_handle = -1;
		540	num_pages(bo) = num_pages;
		541	bucket(bo) = cache_bucket(num_pages);
		542	bo->reusable = true;
		543	bo->domain = DOMAIN_CPU;
		544	list_init(&bo->request);
		545	list_init(&bo->list);
		546	list_init(&bo->vma);
		547
		548	return bo;
		549	}
		550
		551	static struct kgem_bo *__kgem_bo_alloc(int handle, int num_pages)
		552	{
		553	struct kgem_bo *bo;
		554
		555	if (__kgem_freed_bo) {
		556	bo = __kgem_freed_bo;
		557	__kgem_freed_bo = (struct kgem_bo *)bo;
		558	} else {
		559	bo = malloc(sizeof(*bo));
		560	if (bo == NULL)
		561	return NULL;
		562	}
		563
		564	return __kgem_bo_init(bo, handle, num_pages);
		565	}
		566
		567	static struct kgem_request __kgem_request_alloc(struct kgem kgem)
		568	{
		569	struct kgem_request *rq;
		570
		571	rq = __kgem_freed_request;
		572	if (rq) {
		573	__kgem_freed_request = (struct kgem_request *)rq;
		574	} else {
		575	rq = malloc(sizeof(*rq));
		576	if (rq == NULL)
		577	rq = &kgem->static_request;
		578	}
		579
		580	list_init(&rq->buffers);
		581	rq->bo = NULL;
		582	rq->ring = 0;
		583
		584	return rq;
		585	}
		586
		587	static void __kgem_request_free(struct kgem_request *rq)
		588	{
		589	_list_del(&rq->list);
		590	(struct kgem_request *)rq = __kgem_freed_request;
		591	__kgem_freed_request = rq;
		592	}
		593
		594	static struct list inactive(struct kgem kgem, int num_pages)
		595	{
		596	assert(num_pages < MAX_CACHE_SIZE / PAGE_SIZE);
		597	assert(cache_bucket(num_pages) < NUM_CACHE_BUCKETS);
		598	return &kgem->inactive[cache_bucket(num_pages)];
		599	}
		600
		601	static struct list active(struct kgem kgem, int num_pages, int tiling)
		602	{
		603	assert(num_pages < MAX_CACHE_SIZE / PAGE_SIZE);
		604	assert(cache_bucket(num_pages) < NUM_CACHE_BUCKETS);
		605	return &kgem->active[cache_bucket(num_pages)][tiling];
		606	}
		607
		608	static size_t
		609	agp_aperture_size(struct pci_device *dev, unsigned gen)
		610	{
		611	/* XXX assume that only future chipsets are unknown and follow
		612	* the post gen2 PCI layout.
		613	*/
		614	return 0;
		615	}
		616
		617	static size_t
		618	total_ram_size(void)
		619	{
		620	uint32_t data[9];
		621	size_t size = 0;
		622
		623	asm volatile("int $0x40"
		624	: "=a" (size)
		625	: "a" (18),"b"(20), "c" (data)
		626	: "memory");
		627
		628	return size != -1 ? size : 0;
		629	}
		630
		631	static unsigned
		632	cpu_cache_size__cpuid4(void)
		633	{
		634	/* Deterministic Cache Parmaeters (Function 04h)":
		635	* When EAX is initialized to a value of 4, the CPUID instruction
		636	* returns deterministic cache information in the EAX, EBX, ECX
		637	* and EDX registers. This function requires ECX be initialized
		638	* with an index which indicates which cache to return information
		639	* about. The OS is expected to call this function (CPUID.4) with
		640	* ECX = 0, 1, 2, until EAX[4:0] == 0, indicating no more caches.
		641	* The order in which the caches are returned is not specified
		642	* and may change at Intel's discretion.
		643	*
		644	* Calculating the Cache Size in bytes:
		645	* = (Ways +1) * (Partitions +1) * (Line Size +1) * (Sets +1)
		646	*/
		647
		648	unsigned int eax, ebx, ecx, edx;
		649	unsigned int llc_size = 0;
		650	int cnt = 0;
		651
		652	if (__get_cpuid_max(BASIC_CPUID, NULL) < 4)
		653	return 0;
		654
		655	do {
		656	unsigned associativity, line_partitions, line_size, sets;
		657
		658	__cpuid_count(4, cnt++, eax, ebx, ecx, edx);
		659
		660	if ((eax & 0x1f) == 0)
		661	break;
		662
		663	associativity = ((ebx >> 22) & 0x3ff) + 1;
		664	line_partitions = ((ebx >> 12) & 0x3ff) + 1;
		665	line_size = (ebx & 0xfff) + 1;
		666	sets = ecx + 1;
		667
		668	llc_size = associativity * line_partitions * line_size * sets;
		669	} while (1);
		670
		671	return llc_size;
		672	}
		673
		674	static int gem_param(struct kgem *kgem, int name)
		675	{
		676	drm_i915_getparam_t gp;
		677	int v = -1; /* No param uses the sign bit, reserve it for errors */
		678
		679	VG_CLEAR(gp);
		680	gp.param = name;
		681	gp.value = &v;
		682	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GETPARAM, &gp))
		683	return -1;
		684
		685	VG(VALGRIND_MAKE_MEM_DEFINED(&v, sizeof(v)));
		686	return v;
		687	}
		688
		689	static bool test_has_execbuffer2(struct kgem *kgem)
		690	{
		691	return 1;
		692	}
		693
		694	static bool test_has_no_reloc(struct kgem *kgem)
		695	{
		696	if (DBG_NO_FAST_RELOC)
		697	return false;
		698
		699	return gem_param(kgem, LOCAL_I915_PARAM_HAS_NO_RELOC) > 0;
		700	}
		701
		702	static bool test_has_handle_lut(struct kgem *kgem)
		703	{
		704	if (DBG_NO_HANDLE_LUT)
		705	return false;
		706
		707	return gem_param(kgem, LOCAL_I915_PARAM_HAS_HANDLE_LUT) > 0;
		708	}
		709
		710	static bool test_has_wt(struct kgem *kgem)
		711	{
		712	if (DBG_NO_WT)
		713	return false;
		714
		715	return gem_param(kgem, LOCAL_I915_PARAM_HAS_WT) > 0;
		716	}
		717
		718	static bool test_has_semaphores_enabled(struct kgem *kgem)
		719	{
		720	bool detected = false;
		721	int ret;
		722
		723	if (DBG_NO_SEMAPHORES)
		724	return false;
		725
		726	ret = gem_param(kgem, LOCAL_I915_PARAM_HAS_SEMAPHORES);
		727	if (ret != -1)
		728	return ret > 0;
		729
		730	return detected;
		731	}
		732
		733	static bool __kgem_throttle(struct kgem *kgem)
		734	{
		735	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_THROTTLE, NULL) == 0)
		736	return false;
		737
		738	return errno == EIO;
		739	}
		740
		741	static bool is_hw_supported(struct kgem *kgem,
		742	struct pci_device *dev)
		743	{
		744	if (DBG_NO_HW)
		745	return false;
		746
		747	if (!test_has_execbuffer2(kgem))
		748	return false;
		749
		750	if (kgem->gen == (unsigned)-1) /* unknown chipset, assume future gen */
		751	return kgem->has_blt;
		752
		753	/* Although pre-855gm the GMCH is fubar, it works mostly. So
		754	* let the user decide through "NoAccel" whether or not to risk
		755	* hw acceleration.
		756	*/
		757
		758	if (kgem->gen == 060 && dev->revision < 8) {
		759	/* pre-production SNB with dysfunctional BLT */
		760	return false;
		761	}
		762
		763	if (kgem->gen >= 060) /* Only if the kernel supports the BLT ring */
		764	return kgem->has_blt;
		765
		766	return true;
		767	}
		768
		769	static bool test_has_relaxed_fencing(struct kgem *kgem)
		770	{
		771	if (kgem->gen < 040) {
		772	if (DBG_NO_RELAXED_FENCING)
		773	return false;
		774
		775	return gem_param(kgem, LOCAL_I915_PARAM_HAS_RELAXED_FENCING) > 0;
		776	} else
		777	return true;
		778	}
		779
		780	static bool test_has_llc(struct kgem *kgem)
		781	{
		782	int has_llc = -1;
		783
		784	if (DBG_NO_LLC)
		785	return false;
		786
		787	#if defined(I915_PARAM_HAS_LLC) /* Expected in libdrm-2.4.31 */
		788	has_llc = gem_param(kgem, I915_PARAM_HAS_LLC);
		789	#endif
		790	if (has_llc == -1) {
		791	DBG(("%s: no kernel/drm support for HAS_LLC, assuming support for LLC based on GPU generation\n", __FUNCTION__));
		792	has_llc = kgem->gen >= 060;
		793	}
		794
		795	return has_llc;
		796	}
		797
		798	static bool test_has_caching(struct kgem *kgem)
		799	{
		800	uint32_t handle;
		801	bool ret;
		802
		803	if (DBG_NO_CACHE_LEVEL)
		804	return false;
		805
		806	/* Incoherent blt and sampler hangs the GPU */
		807	if (kgem->gen == 040)
		808	return false;
		809
		810	handle = gem_create(kgem->fd, 1);
		811	if (handle == 0)
		812	return false;
		813
		814	ret = gem_set_caching(kgem->fd, handle, UNCACHED);
		815	gem_close(kgem->fd, handle);
		816	return ret;
		817	}
		818
		819	static bool test_has_userptr(struct kgem *kgem)
		820	{
		821	#if defined(USE_USERPTR)
		822	uint32_t handle;
		823	void *ptr;
		824
		825	if (DBG_NO_USERPTR)
		826	return false;
		827
		828	/* Incoherent blt and sampler hangs the GPU */
		829	if (kgem->gen == 040)
		830	return false;
		831
		832	if (posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE))
		833	return false;
		834
		835	handle = gem_userptr(kgem->fd, ptr, PAGE_SIZE, false);
		836	gem_close(kgem->fd, handle);
		837	free(ptr);
		838
		839	return handle != 0;
		840	#else
		841	return false;
		842	#endif
		843	}
		844
		845	static bool test_has_create2(struct kgem *kgem)
		846	{
		847	#if defined(USE_CREATE2)
		848	struct local_i915_gem_create2 args;
		849
		850	if (DBG_NO_CREATE2)
		851	return false;
		852
		853	memset(&args, 0, sizeof(args));
		854	args.size = PAGE_SIZE;
		855	args.caching = DISPLAY;
		856	if (drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args) == 0)
		857	gem_close(kgem->fd, args.handle);
		858
		859	return args.handle != 0;
		860	#else
		861	return false;
		862	#endif
		863	}
		864
		865	static bool test_has_secure_batches(struct kgem *kgem)
		866	{
		867	if (DBG_NO_SECURE_BATCHES)
		868	return false;
		869
		870	return gem_param(kgem, LOCAL_I915_PARAM_HAS_SECURE_BATCHES) > 0;
		871	}
		872
		873	static bool test_has_pinned_batches(struct kgem *kgem)
		874	{
		875	if (DBG_NO_PINNED_BATCHES)
		876	return false;
		877
		878	return gem_param(kgem, LOCAL_I915_PARAM_HAS_PINNED_BATCHES) > 0;
		879	}
		880
		881
		882	static bool kgem_init_pinned_batches(struct kgem *kgem)
		883	{
4315	Serge	884	int count[2] = { 2, 2 };
4304	Serge	885	int size[2] = { 1, 2 };
		886	int n, i;
		887
		888	if (kgem->wedged)
		889	return true;
		890
		891	for (n = 0; n < ARRAY_SIZE(count); n++) {
		892	for (i = 0; i < count[n]; i++) {
		893	struct drm_i915_gem_pin pin;
		894	struct kgem_bo *bo;
		895
		896	VG_CLEAR(pin);
		897
		898	pin.handle = gem_create(kgem->fd, size[n]);
		899	if (pin.handle == 0)
		900	goto err;
		901
		902	DBG(("%s: new handle=%d, num_pages=%d\n",
		903	__FUNCTION__, pin.handle, size[n]));
		904
		905	bo = __kgem_bo_alloc(pin.handle, size[n]);
		906	if (bo == NULL) {
		907	gem_close(kgem->fd, pin.handle);
		908	goto err;
		909	}
		910
		911	pin.alignment = 0;
		912	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) {
		913	gem_close(kgem->fd, pin.handle);
		914	goto err;
		915	}
		916	bo->presumed_offset = pin.offset;
		917	debug_alloc__bo(kgem, bo);
		918	list_add(&bo->list, &kgem->pinned_batches[n]);
		919	}
		920	}
		921
		922	return true;
		923
		924	err:
		925	for (n = 0; n < ARRAY_SIZE(kgem->pinned_batches); n++) {
		926	while (!list_is_empty(&kgem->pinned_batches[n])) {
		927	kgem_bo_destroy(kgem,
		928	list_first_entry(&kgem->pinned_batches[n],
		929	struct kgem_bo, list));
		930	}
		931	}
		932
		933	/* For simplicity populate the lists with a single unpinned bo */
		934	for (n = 0; n < ARRAY_SIZE(count); n++) {
		935	struct kgem_bo *bo;
		936	uint32_t handle;
		937
		938	handle = gem_create(kgem->fd, size[n]);
		939	if (handle == 0)
		940	break;
		941
		942	bo = __kgem_bo_alloc(handle, size[n]);
		943	if (bo == NULL) {
		944	gem_close(kgem->fd, handle);
		945	break;
		946	}
		947
		948	debug_alloc__bo(kgem, bo);
		949	list_add(&bo->list, &kgem->pinned_batches[n]);
		950	}
		951	return false;
		952	}
		953
		954	void kgem_init(struct kgem kgem, int fd, struct pci_device dev, unsigned gen)
		955	{
		956	struct drm_i915_gem_get_aperture aperture;
		957	size_t totalram;
		958	unsigned half_gpu_max;
		959	unsigned int i, j;
		960
		961	DBG(("%s: fd=%d, gen=%d\n", __FUNCTION__, fd, gen));
		962
		963	memset(kgem, 0, sizeof(*kgem));
		964
		965	kgem->fd = fd;
		966	kgem->gen = gen;
		967
		968	list_init(&kgem->requests[0]);
		969	list_init(&kgem->requests[1]);
		970	list_init(&kgem->batch_buffers);
		971	list_init(&kgem->active_buffers);
		972	list_init(&kgem->flushing);
		973	list_init(&kgem->large);
		974	list_init(&kgem->large_inactive);
		975	list_init(&kgem->snoop);
		976	list_init(&kgem->scanout);
		977	for (i = 0; i < ARRAY_SIZE(kgem->pinned_batches); i++)
		978	list_init(&kgem->pinned_batches[i]);
		979	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		980	list_init(&kgem->inactive[i]);
		981	for (i = 0; i < ARRAY_SIZE(kgem->active); i++) {
		982	for (j = 0; j < ARRAY_SIZE(kgem->active[i]); j++)
		983	list_init(&kgem->active[i][j]);
		984	}
		985	for (i = 0; i < ARRAY_SIZE(kgem->vma); i++) {
		986	for (j = 0; j < ARRAY_SIZE(kgem->vma[i].inactive); j++)
		987	list_init(&kgem->vma[i].inactive[j]);
		988	}
		989	kgem->vma[MAP_GTT].count = -MAX_GTT_VMA_CACHE;
		990	kgem->vma[MAP_CPU].count = -MAX_CPU_VMA_CACHE;
		991
		992	kgem->has_blt = gem_param(kgem, LOCAL_I915_PARAM_HAS_BLT) > 0;
		993	DBG(("%s: has BLT ring? %d\n", __FUNCTION__,
		994	kgem->has_blt));
		995
		996	kgem->has_relaxed_delta =
		997	gem_param(kgem, LOCAL_I915_PARAM_HAS_RELAXED_DELTA) > 0;
		998	DBG(("%s: has relaxed delta? %d\n", __FUNCTION__,
		999	kgem->has_relaxed_delta));
		1000
		1001	kgem->has_relaxed_fencing = test_has_relaxed_fencing(kgem);
		1002	DBG(("%s: has relaxed fencing? %d\n", __FUNCTION__,
		1003	kgem->has_relaxed_fencing));
		1004
		1005	kgem->has_llc = test_has_llc(kgem);
		1006	DBG(("%s: has shared last-level-cache? %d\n", __FUNCTION__,
		1007	kgem->has_llc));
		1008
		1009	kgem->has_wt = test_has_wt(kgem);
		1010	DBG(("%s: has write-through caching for scanouts? %d\n", __FUNCTION__,
		1011	kgem->has_wt));
		1012
		1013	kgem->has_caching = test_has_caching(kgem);
		1014	DBG(("%s: has set-cache-level? %d\n", __FUNCTION__,
		1015	kgem->has_caching));
		1016
		1017	kgem->has_userptr = test_has_userptr(kgem);
		1018	DBG(("%s: has userptr? %d\n", __FUNCTION__,
		1019	kgem->has_userptr));
		1020
		1021	kgem->has_create2 = test_has_create2(kgem);
		1022	kgem->has_create2 = 0;
		1023	DBG(("%s: has create2? %d\n", __FUNCTION__,
		1024	kgem->has_create2));
		1025
		1026	kgem->has_no_reloc = test_has_no_reloc(kgem);
		1027	DBG(("%s: has no-reloc? %d\n", __FUNCTION__,
		1028	kgem->has_no_reloc));
		1029
		1030	kgem->has_handle_lut = test_has_handle_lut(kgem);
		1031	kgem->has_handle_lut = 0;
		1032	DBG(("%s: has handle-lut? %d\n", __FUNCTION__,
		1033	kgem->has_handle_lut));
		1034
		1035	kgem->has_semaphores = false;
		1036	if (kgem->has_blt && test_has_semaphores_enabled(kgem))
		1037	kgem->has_semaphores = true;
		1038	DBG(("%s: semaphores enabled? %d\n", __FUNCTION__,
		1039	kgem->has_semaphores));
		1040
		1041	kgem->can_blt_cpu = gen >= 030;
		1042	DBG(("%s: can blt to cpu? %d\n", __FUNCTION__,
		1043	kgem->can_blt_cpu));
		1044
		1045	kgem->has_secure_batches = test_has_secure_batches(kgem);
		1046	DBG(("%s: can use privileged batchbuffers? %d\n", __FUNCTION__,
		1047	kgem->has_secure_batches));
		1048
		1049	kgem->has_pinned_batches = test_has_pinned_batches(kgem);
		1050	DBG(("%s: can use pinned batchbuffers (to avoid CS w/a)? %d\n", __FUNCTION__,
		1051	kgem->has_pinned_batches));
		1052
		1053	if (!is_hw_supported(kgem, dev)) {
		1054	printf("Detected unsupported/dysfunctional hardware, disabling acceleration.\n");
		1055	kgem->wedged = 1;
		1056	} else if (__kgem_throttle(kgem)) {
		1057	printf("Detected a hung GPU, disabling acceleration.\n");
		1058	kgem->wedged = 1;
		1059	}
		1060
		1061	kgem->batch_size = ARRAY_SIZE(kgem->batch);
		1062	if (gen == 020 && !kgem->has_pinned_batches)
		1063	/* Limited to what we can pin */
		1064	kgem->batch_size = 4*1024;
		1065	if (gen == 022)
		1066	/* 865g cannot handle a batch spanning multiple pages */
		1067	kgem->batch_size = PAGE_SIZE / sizeof(uint32_t);
		1068	if ((gen >> 3) == 7)
		1069	kgem->batch_size = 16*1024;
		1070	if (!kgem->has_relaxed_delta && kgem->batch_size > 4*1024)
		1071	kgem->batch_size = 4*1024;
		1072
		1073	if (!kgem_init_pinned_batches(kgem) && gen == 020) {
		1074	printf("Unable to reserve memory for GPU, disabling acceleration.\n");
		1075	kgem->wedged = 1;
		1076	}
		1077
		1078	DBG(("%s: maximum batch size? %d\n", __FUNCTION__,
		1079	kgem->batch_size));
		1080
4403	Serge	1081	kgem->min_alignment = 16;
4304	Serge	1082	if (gen < 040)
		1083	kgem->min_alignment = 64;
		1084
		1085	kgem->half_cpu_cache_pages = cpu_cache_size() >> 13;
		1086	DBG(("%s: last-level cache size: %d bytes, threshold in pages: %d\n",
		1087	__FUNCTION__, cpu_cache_size(), kgem->half_cpu_cache_pages));
		1088
		1089	kgem->next_request = __kgem_request_alloc(kgem);
		1090
		1091	DBG(("%s: cpu bo enabled %d: llc? %d, set-cache-level? %d, userptr? %d\n", __FUNCTION__,
		1092	!DBG_NO_CPU && (kgem->has_llc \| kgem->has_userptr \| kgem->has_caching),
		1093	kgem->has_llc, kgem->has_caching, kgem->has_userptr));
		1094
		1095	VG_CLEAR(aperture);
		1096	aperture.aper_size = 0;
		1097	(void)drmIoctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
		1098	if (aperture.aper_size == 0)
		1099	aperture.aper_size = 6410241024;
		1100
		1101	DBG(("%s: aperture size %lld, available now %lld\n",
		1102	__FUNCTION__,
		1103	(long long)aperture.aper_size,
		1104	(long long)aperture.aper_available_size));
		1105
		1106	kgem->aperture_total = aperture.aper_size;
		1107	kgem->aperture_high = aperture.aper_size * 3/4;
		1108	kgem->aperture_low = aperture.aper_size * 1/3;
		1109	if (gen < 033) {
		1110	/* Severe alignment penalties */
		1111	kgem->aperture_high /= 2;
		1112	kgem->aperture_low /= 2;
		1113	}
		1114	DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
		1115	kgem->aperture_low, kgem->aperture_low / (1024*1024),
		1116	kgem->aperture_high, kgem->aperture_high / (1024*1024)));
		1117
		1118	kgem->aperture_mappable = agp_aperture_size(dev, gen);
		1119	if (kgem->aperture_mappable == 0 \|\|
		1120	kgem->aperture_mappable > aperture.aper_size)
		1121	kgem->aperture_mappable = aperture.aper_size;
		1122	DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
		1123	kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));
		1124
		1125	kgem->buffer_size = 64 * 1024;
		1126	while (kgem->buffer_size < kgem->aperture_mappable >> 10)
		1127	kgem->buffer_size *= 2;
		1128	if (kgem->buffer_size >> 12 > kgem->half_cpu_cache_pages)
		1129	kgem->buffer_size = kgem->half_cpu_cache_pages << 12;
		1130	kgem->buffer_size = 1 << __fls(kgem->buffer_size);
		1131	DBG(("%s: buffer size=%d [%d KiB]\n", __FUNCTION__,
		1132	kgem->buffer_size, kgem->buffer_size / 1024));
		1133	assert(kgem->buffer_size);
		1134
		1135	kgem->max_object_size = 3 * (kgem->aperture_high >> 12) << 10;
		1136	kgem->max_gpu_size = kgem->max_object_size;
		1137	if (!kgem->has_llc && kgem->max_gpu_size > MAX_CACHE_SIZE)
		1138	kgem->max_gpu_size = MAX_CACHE_SIZE;
		1139
		1140	totalram = total_ram_size();
		1141	if (totalram == 0) {
		1142	DBG(("%s: total ram size unknown, assuming maximum of total aperture\n",
		1143	__FUNCTION__));
		1144	totalram = kgem->aperture_total;
		1145	}
		1146	DBG(("%s: total ram=%ld\n", __FUNCTION__, (long)totalram));
		1147	if (kgem->max_object_size > totalram / 2)
		1148	kgem->max_object_size = totalram / 2;
		1149	if (kgem->max_gpu_size > totalram / 4)
		1150	kgem->max_gpu_size = totalram / 4;
		1151
		1152	kgem->max_cpu_size = kgem->max_object_size;
		1153
		1154	half_gpu_max = kgem->max_gpu_size / 2;
		1155	kgem->max_copy_tile_size = (MAX_CACHE_SIZE + 1)/2;
		1156	if (kgem->max_copy_tile_size > half_gpu_max)
		1157	kgem->max_copy_tile_size = half_gpu_max;
		1158
		1159	if (kgem->has_llc)
		1160	kgem->max_upload_tile_size = kgem->max_copy_tile_size;
		1161	else
		1162	kgem->max_upload_tile_size = kgem->aperture_mappable / 4;
		1163	if (kgem->max_upload_tile_size > half_gpu_max)
		1164	kgem->max_upload_tile_size = half_gpu_max;
		1165	if (kgem->max_upload_tile_size > kgem->aperture_high/2)
		1166	kgem->max_upload_tile_size = kgem->aperture_high/2;
		1167	if (kgem->max_upload_tile_size > kgem->aperture_low)
		1168	kgem->max_upload_tile_size = kgem->aperture_low;
		1169	if (kgem->max_upload_tile_size < 16*PAGE_SIZE)
		1170	kgem->max_upload_tile_size = 16*PAGE_SIZE;
		1171
		1172	kgem->large_object_size = MAX_CACHE_SIZE;
		1173	if (kgem->large_object_size > half_gpu_max)
		1174	kgem->large_object_size = half_gpu_max;
		1175	if (kgem->max_copy_tile_size > kgem->aperture_high/2)
		1176	kgem->max_copy_tile_size = kgem->aperture_high/2;
		1177	if (kgem->max_copy_tile_size > kgem->aperture_low)
		1178	kgem->max_copy_tile_size = kgem->aperture_low;
		1179	if (kgem->max_copy_tile_size < 16*PAGE_SIZE)
		1180	kgem->max_copy_tile_size = 16*PAGE_SIZE;
		1181
		1182	if (kgem->has_llc \| kgem->has_caching \| kgem->has_userptr) {
		1183	if (kgem->large_object_size > kgem->max_cpu_size)
		1184	kgem->large_object_size = kgem->max_cpu_size;
		1185	} else
		1186	kgem->max_cpu_size = 0;
		1187	if (DBG_NO_CPU)
		1188	kgem->max_cpu_size = 0;
		1189
		1190	DBG(("%s: maximum object size=%d\n",
		1191	__FUNCTION__, kgem->max_object_size));
		1192	DBG(("%s: large object thresold=%d\n",
		1193	__FUNCTION__, kgem->large_object_size));
		1194	DBG(("%s: max object sizes (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
		1195	__FUNCTION__,
		1196	kgem->max_gpu_size, kgem->max_cpu_size,
		1197	kgem->max_upload_tile_size, kgem->max_copy_tile_size));
		1198
		1199	/* Convert the aperture thresholds to pages */
		1200	kgem->aperture_low /= PAGE_SIZE;
		1201	kgem->aperture_high /= PAGE_SIZE;
		1202
		1203	kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
		1204	if ((int)kgem->fence_max < 0)
		1205	kgem->fence_max = 5; /* minimum safe value for all hw */
		1206	DBG(("%s: max fences=%d\n", __FUNCTION__, kgem->fence_max));
		1207
		1208	kgem->batch_flags_base = 0;
		1209	if (kgem->has_no_reloc)
		1210	kgem->batch_flags_base \|= LOCAL_I915_EXEC_NO_RELOC;
		1211	if (kgem->has_handle_lut)
		1212	kgem->batch_flags_base \|= LOCAL_I915_EXEC_HANDLE_LUT;
		1213	if (kgem->has_pinned_batches)
		1214	kgem->batch_flags_base \|= LOCAL_I915_EXEC_IS_PINNED;
		1215	}
		1216
		1217	/* XXX hopefully a good approximation */
		1218	uint32_t kgem_get_unique_id(struct kgem *kgem)
		1219	{
		1220	uint32_t id;
		1221	id = ++kgem->unique_id;
		1222	if (id == 0)
		1223	id = ++kgem->unique_id;
		1224	return id;
		1225	}
		1226
		1227	inline static uint32_t kgem_pitch_alignment(struct kgem *kgem, unsigned flags)
		1228	{
		1229	if (flags & CREATE_PRIME)
		1230	return 256;
		1231	if (flags & CREATE_SCANOUT)
		1232	return 64;
		1233	return kgem->min_alignment;
		1234	}
		1235
		1236	void kgem_get_tile_size(struct kgem *kgem, int tiling,
		1237	int tile_width, int tile_height, int *tile_size)
		1238	{
		1239	if (kgem->gen <= 030) {
		1240	if (tiling) {
		1241	if (kgem->gen < 030) {
		1242	*tile_width = 128;
		1243	*tile_height = 16;
		1244	*tile_size = 2048;
		1245	} else {
		1246	*tile_width = 512;
		1247	*tile_height = 8;
		1248	*tile_size = 4096;
		1249	}
		1250	} else {
		1251	*tile_width = 1;
		1252	*tile_height = 1;
		1253	*tile_size = 1;
		1254	}
		1255	} else switch (tiling) {
		1256	default:
		1257	case I915_TILING_NONE:
		1258	*tile_width = 1;
		1259	*tile_height = 1;
		1260	*tile_size = 1;
		1261	break;
		1262	case I915_TILING_X:
		1263	*tile_width = 512;
		1264	*tile_height = 8;
		1265	*tile_size = 4096;
		1266	break;
		1267	case I915_TILING_Y:
		1268	*tile_width = 128;
		1269	*tile_height = 32;
		1270	*tile_size = 4096;
		1271	break;
		1272	}
		1273	}
		1274
		1275	uint32_t kgem_surface_size(struct kgem *kgem,
		1276	bool relaxed_fencing,
		1277	unsigned flags,
		1278	uint32_t width,
		1279	uint32_t height,
		1280	uint32_t bpp,
		1281	uint32_t tiling,
		1282	uint32_t *pitch)
		1283	{
		1284	uint32_t tile_width, tile_height;
		1285	uint32_t size;
		1286
		1287	assert(width <= MAXSHORT);
		1288	assert(height <= MAXSHORT);
		1289	assert(bpp >= 8);
		1290
		1291	if (kgem->gen <= 030) {
		1292	if (tiling) {
		1293	if (kgem->gen < 030) {
		1294	tile_width = 128;
		1295	tile_height = 32;
		1296	} else {
		1297	tile_width = 512;
		1298	tile_height = 16;
		1299	}
		1300	} else {
		1301	tile_width = 2 * bpp >> 3;
		1302	tile_width = ALIGN(tile_width,
		1303	kgem_pitch_alignment(kgem, flags));
		1304	tile_height = 2;
		1305	}
		1306	} else switch (tiling) {
		1307	default:
		1308	case I915_TILING_NONE:
		1309	tile_width = 2 * bpp >> 3;
		1310	tile_width = ALIGN(tile_width,
		1311	kgem_pitch_alignment(kgem, flags));
		1312	tile_height = 2;
		1313	break;
		1314
		1315	/* XXX align to an even tile row */
		1316	case I915_TILING_X:
		1317	tile_width = 512;
		1318	tile_height = 16;
		1319	break;
		1320	case I915_TILING_Y:
		1321	tile_width = 128;
		1322	tile_height = 64;
		1323	break;
		1324	}
		1325
		1326	pitch = ALIGN(width bpp / 8, tile_width);
		1327	height = ALIGN(height, tile_height);
		1328	if (kgem->gen >= 040)
		1329	return PAGE_ALIGN(pitch height);
		1330
		1331	/* If it is too wide for the blitter, don't even bother. */
		1332	if (tiling != I915_TILING_NONE) {
		1333	if (*pitch > 8192)
		1334	return 0;
		1335
		1336	for (size = tile_width; size < *pitch; size <<= 1)
		1337	;
		1338	*pitch = size;
		1339	} else {
		1340	if (*pitch >= 32768)
		1341	return 0;
		1342	}
		1343
		1344	size = pitch height;
		1345	if (relaxed_fencing \|\| tiling == I915_TILING_NONE)
		1346	return PAGE_ALIGN(size);
		1347
		1348	/* We need to allocate a pot fence region for a tiled buffer. */
		1349	if (kgem->gen < 030)
		1350	tile_width = 512 * 1024;
		1351	else
		1352	tile_width = 1024 * 1024;
		1353	while (tile_width < size)
		1354	tile_width *= 2;
		1355	return tile_width;
		1356	}
		1357
		1358	static uint32_t kgem_aligned_height(struct kgem *kgem,
		1359	uint32_t height, uint32_t tiling)
		1360	{
		1361	uint32_t tile_height;
		1362
		1363	if (kgem->gen <= 030) {
		1364	tile_height = tiling ? kgem->gen < 030 ? 32 : 16 : 1;
		1365	} else switch (tiling) {
		1366	/* XXX align to an even tile row */
		1367	default:
		1368	case I915_TILING_NONE:
		1369	tile_height = 1;
		1370	break;
		1371	case I915_TILING_X:
		1372	tile_height = 16;
		1373	break;
		1374	case I915_TILING_Y:
		1375	tile_height = 64;
		1376	break;
		1377	}
		1378
		1379	return ALIGN(height, tile_height);
		1380	}
		1381
		1382	static struct drm_i915_gem_exec_object2 *
		1383	kgem_add_handle(struct kgem kgem, struct kgem_bo bo)
		1384	{
		1385	struct drm_i915_gem_exec_object2 *exec;
		1386
		1387	DBG(("%s: handle=%d, index=%d\n",
		1388	__FUNCTION__, bo->handle, kgem->nexec));
		1389
		1390	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		1391	bo->target_handle = kgem->has_handle_lut ? kgem->nexec : bo->handle;
		1392	exec = memset(&kgem->exec[kgem->nexec++], 0, sizeof(*exec));
		1393	exec->handle = bo->handle;
		1394	exec->offset = bo->presumed_offset;
		1395
		1396	kgem->aperture += num_pages(bo);
		1397
		1398	return exec;
		1399	}
		1400
		1401	static void kgem_add_bo(struct kgem kgem, struct kgem_bo bo)
		1402	{
		1403	bo->exec = kgem_add_handle(kgem, bo);
		1404	bo->rq = MAKE_REQUEST(kgem->next_request, kgem->ring);
		1405
		1406	list_move_tail(&bo->request, &kgem->next_request->buffers);
		1407
		1408	/* XXX is it worth working around gcc here? */
		1409	kgem->flush \|= bo->flush;
		1410	}
		1411
		1412	static uint32_t kgem_end_batch(struct kgem *kgem)
		1413	{
		1414	kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
		1415	if (kgem->nbatch & 1)
		1416	kgem->batch[kgem->nbatch++] = MI_NOOP;
		1417
		1418	return kgem->nbatch;
		1419	}
		1420
		1421	static void kgem_fixup_self_relocs(struct kgem kgem, struct kgem_bo bo)
		1422	{
		1423	int n;
		1424
		1425	assert(kgem->nreloc__self <= 256);
		1426	if (kgem->nreloc__self == 0)
		1427	return;
		1428
		1429	for (n = 0; n < kgem->nreloc__self; n++) {
		1430	int i = kgem->reloc__self[n];
		1431	assert(kgem->reloc[i].target_handle == ~0U);
		1432	kgem->reloc[i].target_handle = bo->target_handle;
		1433	kgem->reloc[i].presumed_offset = bo->presumed_offset;
		1434	kgem->batch[kgem->reloc[i].offset/sizeof(kgem->batch[0])] =
		1435	kgem->reloc[i].delta + bo->presumed_offset;
		1436	}
		1437
		1438	if (n == 256) {
		1439	for (n = kgem->reloc__self[255]; n < kgem->nreloc; n++) {
		1440	if (kgem->reloc[n].target_handle == ~0U) {
		1441	kgem->reloc[n].target_handle = bo->target_handle;
		1442	kgem->reloc[n].presumed_offset = bo->presumed_offset;
		1443	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		1444	kgem->reloc[n].delta + bo->presumed_offset;
		1445	}
		1446	}
		1447
		1448	}
		1449
		1450	}
		1451
		1452	static void kgem_bo_binding_free(struct kgem kgem, struct kgem_bo bo)
		1453	{
		1454	struct kgem_bo_binding *b;
		1455
		1456	b = bo->binding.next;
		1457	while (b) {
		1458	struct kgem_bo_binding *next = b->next;
		1459	free (b);
		1460	b = next;
		1461	}
		1462	}
		1463
		1464	static void kgem_bo_release_map(struct kgem kgem, struct kgem_bo bo)
		1465	{
		1466	int type = IS_CPU_MAP(bo->map);
		1467
		1468	assert(!IS_USER_MAP(bo->map));
		1469
		1470	DBG(("%s: releasing %s vma for handle=%d, count=%d\n",
		1471	__FUNCTION__, type ? "CPU" : "GTT",
		1472	bo->handle, kgem->vma[type].count));
		1473
		1474	VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo)));
		1475	user_free(MAP(bo->map));
		1476	bo->map = NULL;
		1477
		1478	if (!list_is_empty(&bo->vma)) {
		1479	list_del(&bo->vma);
		1480	kgem->vma[type].count--;
		1481	}
		1482	}
		1483
		1484	static void kgem_bo_free(struct kgem kgem, struct kgem_bo bo)
		1485	{
		1486	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		1487	assert(bo->refcnt == 0);
		1488	assert(bo->proxy == NULL);
		1489	assert(bo->exec == NULL);
		1490	assert(!bo->snoop \|\| bo->rq == NULL);
		1491
		1492	#ifdef DEBUG_MEMORY
		1493	kgem->debug_memory.bo_allocs--;
		1494	kgem->debug_memory.bo_bytes -= bytes(bo);
		1495	#endif
		1496
		1497	kgem_bo_binding_free(kgem, bo);
		1498
		1499	if (IS_USER_MAP(bo->map)) {
		1500	assert(bo->rq == NULL);
		1501	assert(!__kgem_busy(kgem, bo->handle));
		1502	assert(MAP(bo->map) != bo \|\| bo->io \|\| bo->flush);
		1503	if (!(bo->io \|\| bo->flush)) {
		1504	DBG(("%s: freeing snooped base\n", __FUNCTION__));
		1505	assert(bo != MAP(bo->map));
		1506	free(MAP(bo->map));
		1507	}
		1508	bo->map = NULL;
		1509	}
		1510	if (bo->map)
		1511	kgem_bo_release_map(kgem, bo);
		1512	assert(list_is_empty(&bo->vma));
		1513	assert(bo->map == NULL);
		1514
		1515	_list_del(&bo->list);
		1516	_list_del(&bo->request);
		1517	gem_close(kgem->fd, bo->handle);
		1518
		1519	if (!bo->io) {
		1520	(struct kgem_bo *)bo = __kgem_freed_bo;
		1521	__kgem_freed_bo = bo;
		1522	} else
		1523	free(bo);
		1524	}
		1525
		1526	inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
		1527	struct kgem_bo *bo)
		1528	{
		1529	DBG(("%s: moving handle=%d to inactive\n", __FUNCTION__, bo->handle));
		1530
		1531	assert(bo->refcnt == 0);
		1532	assert(bo->reusable);
		1533	assert(bo->rq == NULL);
		1534	assert(bo->exec == NULL);
		1535	assert(bo->domain != DOMAIN_GPU);
		1536	assert(!bo->proxy);
		1537	assert(!bo->io);
		1538	assert(!bo->scanout);
		1539	assert(!bo->snoop);
		1540	assert(!bo->flush);
		1541	assert(!bo->needs_flush);
		1542	assert(list_is_empty(&bo->vma));
		1543	assert_tiling(kgem, bo);
		1544	ASSERT_IDLE(kgem, bo->handle);
		1545
		1546	kgem->need_expire = true;
		1547
		1548	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
		1549	list_move(&bo->list, &kgem->large_inactive);
		1550	return;
		1551	}
		1552
		1553	assert(bo->flush == false);
		1554	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
		1555	if (bo->map) {
		1556	int type = IS_CPU_MAP(bo->map);
		1557	if (bucket(bo) >= NUM_CACHE_BUCKETS \|\|
		1558	(!type && !__kgem_bo_is_mappable(kgem, bo))) {
		1559	// munmap(MAP(bo->map), bytes(bo));
		1560	bo->map = NULL;
		1561	}
		1562	if (bo->map) {
		1563	list_add(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]);
		1564	kgem->vma[type].count++;
		1565	}
		1566	}
		1567	}
		1568
		1569	static struct kgem_bo kgem_bo_replace_io(struct kgem_bo bo)
		1570	{
		1571	struct kgem_bo *base;
		1572
		1573	if (!bo->io)
		1574	return bo;
		1575
		1576	assert(!bo->snoop);
		1577	base = malloc(sizeof(*base));
		1578	if (base) {
		1579	DBG(("%s: transferring io handle=%d to bo\n",
		1580	__FUNCTION__, bo->handle));
		1581	/* transfer the handle to a minimum bo */
		1582	memcpy(base, bo, sizeof(*base));
		1583	base->io = false;
		1584	list_init(&base->list);
		1585	list_replace(&bo->request, &base->request);
		1586	list_replace(&bo->vma, &base->vma);
		1587	free(bo);
		1588	bo = base;
		1589	} else
		1590	bo->reusable = false;
		1591
		1592	return bo;
		1593	}
		1594
		1595	inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
		1596	struct kgem_bo *bo)
		1597	{
		1598	DBG(("%s: removing handle=%d from inactive\n", __FUNCTION__, bo->handle));
		1599
		1600	list_del(&bo->list);
		1601	assert(bo->rq == NULL);
		1602	assert(bo->exec == NULL);
		1603	if (bo->map) {
		1604	assert(!list_is_empty(&bo->vma));
		1605	list_del(&bo->vma);
		1606	kgem->vma[IS_CPU_MAP(bo->map)].count--;
		1607	}
		1608	}
		1609
		1610	inline static void kgem_bo_remove_from_active(struct kgem *kgem,
		1611	struct kgem_bo *bo)
		1612	{
		1613	DBG(("%s: removing handle=%d from active\n", __FUNCTION__, bo->handle));
		1614
		1615	list_del(&bo->list);
		1616	assert(bo->rq != NULL);
		1617	if (bo->rq == (void *)kgem)
		1618	list_del(&bo->request);
		1619	assert(list_is_empty(&bo->vma));
		1620	}
		1621
		1622	static void _kgem_bo_delete_buffer(struct kgem kgem, struct kgem_bo bo)
		1623	{
		1624	struct kgem_buffer io = (struct kgem_buffer )bo->proxy;
		1625
		1626	DBG(("%s: size=%d, offset=%d, parent used=%d\n",
		1627	__FUNCTION__, bo->size.bytes, bo->delta, io->used));
		1628
		1629	if (ALIGN(bo->delta + bo->size.bytes, UPLOAD_ALIGNMENT) == io->used)
		1630	io->used = bo->delta;
		1631	}
		1632
		1633	static void kgem_bo_move_to_scanout(struct kgem kgem, struct kgem_bo bo)
		1634	{
		1635	assert(bo->refcnt == 0);
		1636	assert(bo->scanout);
		1637	assert(bo->delta);
		1638	assert(!bo->flush);
		1639	assert(!bo->snoop);
		1640	assert(!bo->io);
		1641
		1642	if (bo->purged) {
		1643	DBG(("%s: discarding purged scanout - external name?\n",
		1644	__FUNCTION__));
		1645	kgem_bo_free(kgem, bo);
		1646	return;
		1647	}
		1648
		1649	DBG(("%s: moving %d [fb %d] to scanout cache, active? %d\n",
		1650	__FUNCTION__, bo->handle, bo->delta, bo->rq != NULL));
		1651	if (bo->rq)
		1652	list_move_tail(&bo->list, &kgem->scanout);
		1653	else
		1654	list_move(&bo->list, &kgem->scanout);
		1655	}
		1656
		1657	static void kgem_bo_move_to_snoop(struct kgem kgem, struct kgem_bo bo)
		1658	{
		1659	assert(bo->reusable);
		1660	assert(!bo->flush);
		1661	assert(!bo->needs_flush);
		1662	assert(bo->refcnt == 0);
		1663	assert(bo->exec == NULL);
		1664
		1665	if (num_pages(bo) > kgem->max_cpu_size >> 13) {
		1666	DBG(("%s handle=%d discarding large CPU buffer (%d >%d pages)\n",
		1667	__FUNCTION__, bo->handle, num_pages(bo), kgem->max_cpu_size >> 13));
		1668	kgem_bo_free(kgem, bo);
		1669	return;
		1670	}
		1671
		1672	assert(bo->tiling == I915_TILING_NONE);
		1673	assert(bo->rq == NULL);
		1674
		1675	DBG(("%s: moving %d to snoop cachee\n", __FUNCTION__, bo->handle));
		1676	list_add(&bo->list, &kgem->snoop);
		1677	}
		1678
		1679	static struct kgem_bo *
		1680	search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		1681	{
		1682	struct kgem_bo bo, first = NULL;
		1683
		1684	DBG(("%s: num_pages=%d, flags=%x\n", __FUNCTION__, num_pages, flags));
		1685
		1686	if ((kgem->has_caching \| kgem->has_userptr) == 0)
		1687	return NULL;
		1688
		1689	if (list_is_empty(&kgem->snoop)) {
		1690	DBG(("%s: inactive and cache empty\n", __FUNCTION__));
		1691	if (!__kgem_throttle_retire(kgem, flags)) {
		1692	DBG(("%s: nothing retired\n", __FUNCTION__));
		1693	return NULL;
		1694	}
		1695	}
		1696
		1697	list_for_each_entry(bo, &kgem->snoop, list) {
		1698	assert(bo->refcnt == 0);
		1699	assert(bo->snoop);
		1700	assert(!bo->scanout);
		1701	assert(!bo->purged);
		1702	assert(bo->proxy == NULL);
		1703	assert(bo->tiling == I915_TILING_NONE);
		1704	assert(bo->rq == NULL);
		1705	assert(bo->exec == NULL);
		1706
		1707	if (num_pages > num_pages(bo))
		1708	continue;
		1709
		1710	if (num_pages(bo) > 2*num_pages) {
		1711	if (first == NULL)
		1712	first = bo;
		1713	continue;
		1714	}
		1715
		1716	list_del(&bo->list);
		1717	bo->pitch = 0;
		1718	bo->delta = 0;
		1719
		1720	DBG((" %s: found handle=%d (num_pages=%d) in snoop cache\n",
		1721	__FUNCTION__, bo->handle, num_pages(bo)));
		1722	return bo;
		1723	}
		1724
		1725	if (first) {
		1726	list_del(&first->list);
		1727	first->pitch = 0;
		1728	first->delta = 0;
		1729
		1730	DBG((" %s: found handle=%d (num_pages=%d) in snoop cache\n",
		1731	__FUNCTION__, first->handle, num_pages(first)));
		1732	return first;
		1733	}
		1734
		1735	return NULL;
		1736	}
		1737
		1738	void kgem_bo_undo(struct kgem kgem, struct kgem_bo bo)
		1739	{
		1740	if (kgem->nexec != 1 \|\| bo->exec == NULL)
		1741	return;
		1742
		1743	DBG(("%s: only handle in batch, discarding last operations for handle=%d\n",
		1744	__FUNCTION__, bo->handle));
		1745
		1746	assert(bo->exec == &kgem->exec[0]);
		1747	assert(kgem->exec[0].handle == bo->handle);
		1748	assert(RQ(bo->rq) == kgem->next_request);
		1749
		1750	bo->refcnt++;
		1751	kgem_reset(kgem);
		1752	bo->refcnt--;
		1753	}
		1754
		1755	static void __kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		1756	{
		1757	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		1758
		1759	assert(list_is_empty(&bo->list));
		1760	assert(bo->refcnt == 0);
		1761	assert(!bo->purged \|\| !bo->reusable);
		1762	assert(bo->proxy == NULL);
		1763	assert_tiling(kgem, bo);
		1764
		1765	bo->binding.offset = 0;
		1766
		1767	if (DBG_NO_CACHE)
		1768	goto destroy;
		1769
		1770	if (bo->snoop && !bo->flush) {
		1771	DBG(("%s: handle=%d is snooped\n", __FUNCTION__, bo->handle));
		1772	assert(bo->reusable);
		1773	assert(list_is_empty(&bo->list));
		1774	if (bo->exec == NULL && bo->rq && !__kgem_busy(kgem, bo->handle))
		1775	__kgem_bo_clear_busy(bo);
		1776	if (bo->rq == NULL)
		1777	kgem_bo_move_to_snoop(kgem, bo);
		1778	return;
		1779	}
		1780	if (!IS_USER_MAP(bo->map))
		1781	bo->flush = false;
		1782
		1783	if (bo->scanout) {
		1784	kgem_bo_move_to_scanout(kgem, bo);
		1785	return;
		1786	}
		1787
		1788	if (bo->io)
		1789	bo = kgem_bo_replace_io(bo);
		1790	if (!bo->reusable) {
		1791	DBG(("%s: handle=%d, not reusable\n",
		1792	__FUNCTION__, bo->handle));
		1793	goto destroy;
		1794	}
		1795
		1796	if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU)
		1797	kgem_bo_release_map(kgem, bo);
		1798
		1799	assert(list_is_empty(&bo->vma));
		1800	assert(list_is_empty(&bo->list));
		1801	assert(bo->flush == false);
		1802	assert(bo->snoop == false);
		1803	assert(bo->io == false);
		1804	assert(bo->scanout == false);
		1805
		1806	kgem_bo_undo(kgem, bo);
		1807	assert(bo->refcnt == 0);
		1808
		1809	if (bo->rq && bo->exec == NULL && !__kgem_busy(kgem, bo->handle))
		1810	__kgem_bo_clear_busy(bo);
		1811
		1812	if (bo->rq) {
		1813	struct list *cache;
		1814
		1815	DBG(("%s: handle=%d -> active\n", __FUNCTION__, bo->handle));
		1816	if (bucket(bo) < NUM_CACHE_BUCKETS)
		1817	cache = &kgem->active[bucket(bo)][bo->tiling];
		1818	else
		1819	cache = &kgem->large;
		1820	list_add(&bo->list, cache);
		1821	return;
		1822	}
		1823
		1824	assert(bo->exec == NULL);
		1825	assert(list_is_empty(&bo->request));
		1826
		1827	if (!IS_CPU_MAP(bo->map)) {
		1828	if (!kgem_bo_set_purgeable(kgem, bo))
		1829	goto destroy;
		1830
		1831	if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
		1832	goto destroy;
		1833
		1834	DBG(("%s: handle=%d, purged\n",
		1835	__FUNCTION__, bo->handle));
		1836	}
		1837
		1838	kgem_bo_move_to_inactive(kgem, bo);
		1839	return;
		1840
		1841	destroy:
		1842	if (!bo->exec)
		1843	kgem_bo_free(kgem, bo);
		1844	}
		1845
		1846	static void kgem_bo_unref(struct kgem kgem, struct kgem_bo bo)
		1847	{
		1848	assert(bo->refcnt);
		1849	if (--bo->refcnt == 0)
		1850	__kgem_bo_destroy(kgem, bo);
		1851	}
		1852
		1853	static void kgem_buffer_release(struct kgem kgem, struct kgem_buffer bo)
		1854	{
		1855	while (!list_is_empty(&bo->base.vma)) {
		1856	struct kgem_bo *cached;
		1857
		1858	cached = list_first_entry(&bo->base.vma, struct kgem_bo, vma);
		1859	assert(cached->proxy == &bo->base);
		1860	list_del(&cached->vma);
		1861
		1862	assert((struct kgem_bo *)cached->map == cached);
		1863	(struct kgem_bo *)cached->map = NULL;
		1864	cached->map = NULL;
		1865
		1866	kgem_bo_destroy(kgem, cached);
		1867	}
		1868	}
		1869
		1870	static bool kgem_retire__buffers(struct kgem *kgem)
		1871	{
		1872	bool retired = false;
		1873
		1874	while (!list_is_empty(&kgem->active_buffers)) {
		1875	struct kgem_buffer *bo =
		1876	list_last_entry(&kgem->active_buffers,
		1877	struct kgem_buffer,
		1878	base.list);
		1879
		1880	if (bo->base.rq)
		1881	break;
		1882
		1883	DBG(("%s: releasing upload cache for handle=%d? %d\n",
		1884	__FUNCTION__, bo->base.handle, !list_is_empty(&bo->base.vma)));
		1885	list_del(&bo->base.list);
		1886	kgem_buffer_release(kgem, bo);
		1887	kgem_bo_unref(kgem, &bo->base);
		1888	retired = true;
		1889	}
		1890
		1891	return retired;
		1892	}
		1893
		1894	static bool kgem_retire__flushing(struct kgem *kgem)
		1895	{
		1896	struct kgem_bo bo, next;
		1897	bool retired = false;
		1898
		1899	list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
		1900	assert(bo->rq == (void *)kgem);
		1901	assert(bo->exec == NULL);
		1902
		1903	if (__kgem_busy(kgem, bo->handle))
		1904	break;
		1905
		1906	__kgem_bo_clear_busy(bo);
		1907
		1908	if (bo->refcnt)
		1909	continue;
		1910
		1911	if (bo->snoop) {
		1912	kgem_bo_move_to_snoop(kgem, bo);
		1913	} else if (bo->scanout) {
		1914	kgem_bo_move_to_scanout(kgem, bo);
		1915	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		1916	kgem_bo_set_purgeable(kgem, bo)) {
		1917	kgem_bo_move_to_inactive(kgem, bo);
		1918	retired = true;
		1919	} else
		1920	kgem_bo_free(kgem, bo);
		1921	}
		1922	#if HAS_DEBUG_FULL
		1923	{
		1924	int count = 0;
		1925	list_for_each_entry(bo, &kgem->flushing, request)
		1926	count++;
		1927	ErrorF("%s: %d bo on flushing list\n", __FUNCTION__, count);
		1928	}
		1929	#endif
		1930
		1931	kgem->need_retire \|= !list_is_empty(&kgem->flushing);
		1932
		1933	return retired;
		1934	}
		1935
		1936
		1937	static bool __kgem_retire_rq(struct kgem kgem, struct kgem_request rq)
		1938	{
		1939	bool retired = false;
		1940
		1941	DBG(("%s: request %d complete\n",
		1942	__FUNCTION__, rq->bo->handle));
		1943
		1944	while (!list_is_empty(&rq->buffers)) {
		1945	struct kgem_bo *bo;
		1946
		1947	bo = list_first_entry(&rq->buffers,
		1948	struct kgem_bo,
		1949	request);
		1950
		1951	assert(RQ(bo->rq) == rq);
		1952	assert(bo->exec == NULL);
		1953	assert(bo->domain == DOMAIN_GPU \|\| bo->domain == DOMAIN_NONE);
		1954
		1955	list_del(&bo->request);
		1956
		1957	if (bo->needs_flush)
		1958	bo->needs_flush = __kgem_busy(kgem, bo->handle);
		1959	if (bo->needs_flush) {
		1960	DBG(("%s: moving %d to flushing\n",
		1961	__FUNCTION__, bo->handle));
		1962	list_add(&bo->request, &kgem->flushing);
		1963	bo->rq = (void *)kgem;
		1964	continue;
		1965	}
		1966
		1967	bo->domain = DOMAIN_NONE;
		1968	bo->rq = NULL;
		1969	if (bo->refcnt)
		1970	continue;
		1971
		1972	if (bo->snoop) {
		1973	kgem_bo_move_to_snoop(kgem, bo);
		1974	} else if (bo->scanout) {
		1975	kgem_bo_move_to_scanout(kgem, bo);
		1976	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		1977	kgem_bo_set_purgeable(kgem, bo)) {
		1978	kgem_bo_move_to_inactive(kgem, bo);
		1979	retired = true;
		1980	} else {
		1981	DBG(("%s: closing %d\n",
		1982	__FUNCTION__, bo->handle));
		1983	kgem_bo_free(kgem, bo);
		1984	}
		1985	}
		1986
		1987	assert(rq->bo->rq == NULL);
		1988	assert(list_is_empty(&rq->bo->request));
		1989
		1990	if (--rq->bo->refcnt == 0) {
		1991	if (kgem_bo_set_purgeable(kgem, rq->bo)) {
		1992	kgem_bo_move_to_inactive(kgem, rq->bo);
		1993	retired = true;
		1994	} else {
		1995	DBG(("%s: closing %d\n",
		1996	__FUNCTION__, rq->bo->handle));
		1997	kgem_bo_free(kgem, rq->bo);
		1998	}
		1999	}
		2000
		2001	__kgem_request_free(rq);
		2002	return retired;
		2003	}
		2004
		2005	static bool kgem_retire__requests_ring(struct kgem *kgem, int ring)
		2006	{
		2007	bool retired = false;
		2008
		2009	while (!list_is_empty(&kgem->requests[ring])) {
		2010	struct kgem_request *rq;
		2011
		2012	rq = list_first_entry(&kgem->requests[ring],
		2013	struct kgem_request,
		2014	list);
		2015	if (__kgem_busy(kgem, rq->bo->handle))
		2016	break;
		2017
		2018	retired \|= __kgem_retire_rq(kgem, rq);
		2019	}
		2020
		2021	#if HAS_DEBUG_FULL
		2022	{
		2023	struct kgem_bo *bo;
		2024	int count = 0;
		2025
		2026	list_for_each_entry(bo, &kgem->requests[ring], request)
		2027	count++;
		2028
		2029	bo = NULL;
		2030	if (!list_is_empty(&kgem->requests[ring]))
		2031	bo = list_first_entry(&kgem->requests[ring],
		2032	struct kgem_request,
		2033	list)->bo;
		2034
		2035	ErrorF("%s: ring=%d, %d outstanding requests, oldest=%d\n",
		2036	__FUNCTION__, ring, count, bo ? bo->handle : 0);
		2037	}
		2038	#endif
		2039
		2040	return retired;
		2041	}
		2042
		2043	static bool kgem_retire__requests(struct kgem *kgem)
		2044	{
		2045	bool retired = false;
		2046	int n;
		2047
		2048	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2049	retired \|= kgem_retire__requests_ring(kgem, n);
		2050	kgem->need_retire \|= !list_is_empty(&kgem->requests[n]);
		2051	}
		2052
		2053	return retired;
		2054	}
		2055
		2056	bool kgem_retire(struct kgem *kgem)
		2057	{
		2058	bool retired = false;
		2059
		2060	DBG(("%s\n", __FUNCTION__));
		2061
		2062	kgem->need_retire = false;
		2063
		2064	retired \|= kgem_retire__flushing(kgem);
		2065	retired \|= kgem_retire__requests(kgem);
		2066	retired \|= kgem_retire__buffers(kgem);
		2067
		2068	DBG(("%s -- retired=%d, need_retire=%d\n",
		2069	__FUNCTION__, retired, kgem->need_retire));
		2070
		2071	kgem->retire(kgem);
		2072
		2073	return retired;
		2074	}
		2075
		2076	bool __kgem_ring_is_idle(struct kgem *kgem, int ring)
		2077	{
		2078	struct kgem_request *rq;
		2079
		2080	assert(!list_is_empty(&kgem->requests[ring]));
		2081
		2082	rq = list_last_entry(&kgem->requests[ring],
		2083	struct kgem_request, list);
		2084	if (__kgem_busy(kgem, rq->bo->handle)) {
		2085	DBG(("%s: last requests handle=%d still busy\n",
		2086	__FUNCTION__, rq->bo->handle));
		2087	return false;
		2088	}
		2089
		2090	DBG(("%s: ring=%d idle (handle=%d)\n",
		2091	__FUNCTION__, ring, rq->bo->handle));
		2092
		2093	kgem_retire__requests_ring(kgem, ring);
		2094	assert(list_is_empty(&kgem->requests[ring]));
		2095	return true;
		2096	}
		2097
		2098	static void kgem_commit(struct kgem *kgem)
		2099	{
		2100	struct kgem_request *rq = kgem->next_request;
		2101	struct kgem_bo bo, next;
		2102
		2103	list_for_each_entry_safe(bo, next, &rq->buffers, request) {
		2104	assert(next->request.prev == &bo->request);
		2105
		2106	DBG(("%s: release handle=%d (proxy? %d), dirty? %d flush? %d, snoop? %d -> offset=%x\n",
		2107	__FUNCTION__, bo->handle, bo->proxy != NULL,
		2108	bo->gpu_dirty, bo->needs_flush, bo->snoop,
		2109	(unsigned)bo->exec->offset));
		2110
		2111	assert(bo->exec);
		2112	assert(bo->proxy == NULL \|\| bo->exec == &_kgem_dummy_exec);
		2113	assert(RQ(bo->rq) == rq \|\| (RQ(bo->proxy->rq) == rq));
		2114
		2115	bo->presumed_offset = bo->exec->offset;
		2116	bo->exec = NULL;
		2117	bo->target_handle = -1;
		2118
		2119	if (!bo->refcnt && !bo->reusable) {
		2120	assert(!bo->snoop);
		2121	kgem_bo_free(kgem, bo);
		2122	continue;
		2123	}
		2124
		2125	bo->binding.offset = 0;
		2126	bo->domain = DOMAIN_GPU;
		2127	bo->gpu_dirty = false;
		2128
		2129	if (bo->proxy) {
		2130	/* proxies are not used for domain tracking */
		2131	bo->exec = NULL;
		2132	__kgem_bo_clear_busy(bo);
		2133	}
		2134
		2135	kgem->scanout_busy \|= bo->scanout;
		2136	}
		2137
		2138	if (rq == &kgem->static_request) {
		2139	struct drm_i915_gem_set_domain set_domain;
		2140
		2141	DBG(("%s: syncing due to allocation failure\n", __FUNCTION__));
		2142
		2143	VG_CLEAR(set_domain);
		2144	set_domain.handle = rq->bo->handle;
		2145	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2146	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2147	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
		2148	DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
		2149	kgem_throttle(kgem);
		2150	}
		2151
		2152	kgem_retire(kgem);
		2153	assert(list_is_empty(&rq->buffers));
		2154
		2155	assert(rq->bo->map == NULL);
		2156	gem_close(kgem->fd, rq->bo->handle);
		2157	kgem_cleanup_cache(kgem);
		2158	} else {
		2159	list_add_tail(&rq->list, &kgem->requests[rq->ring]);
		2160	kgem->need_throttle = kgem->need_retire = 1;
		2161	}
		2162
		2163	kgem->next_request = NULL;
		2164	}
		2165
		2166	static void kgem_close_list(struct kgem kgem, struct list head)
		2167	{
		2168	while (!list_is_empty(head))
		2169	kgem_bo_free(kgem, list_first_entry(head, struct kgem_bo, list));
		2170	}
		2171
		2172	static void kgem_close_inactive(struct kgem *kgem)
		2173	{
		2174	unsigned int i;
		2175
		2176	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		2177	kgem_close_list(kgem, &kgem->inactive[i]);
		2178	}
		2179
		2180	static void kgem_finish_buffers(struct kgem *kgem)
		2181	{
		2182	struct kgem_buffer bo, next;
		2183
		2184	list_for_each_entry_safe(bo, next, &kgem->batch_buffers, base.list) {
		2185	DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s\n",
		2186	__FUNCTION__, bo->base.handle, bo->used, bo->base.exec!=NULL,
		2187	bo->write, bo->mmapped ? IS_CPU_MAP(bo->base.map) ? "cpu" : "gtt" : "no"));
		2188
		2189	assert(next->base.list.prev == &bo->base.list);
		2190	assert(bo->base.io);
		2191	assert(bo->base.refcnt >= 1);
		2192
		2193	if (!bo->base.exec) {
		2194	DBG(("%s: skipping unattached handle=%d, used=%d\n",
		2195	__FUNCTION__, bo->base.handle, bo->used));
		2196	continue;
		2197	}
		2198
		2199	if (!bo->write) {
		2200	assert(bo->base.exec \|\| bo->base.refcnt > 1);
		2201	goto decouple;
		2202	}
		2203
		2204	if (bo->mmapped) {
		2205	int used;
		2206
		2207	assert(!bo->need_io);
		2208
		2209	used = ALIGN(bo->used, PAGE_SIZE);
		2210	if (!DBG_NO_UPLOAD_ACTIVE &&
		2211	used + PAGE_SIZE <= bytes(&bo->base) &&
		2212	(kgem->has_llc \|\| !IS_CPU_MAP(bo->base.map) \|\| bo->base.snoop)) {
		2213	DBG(("%s: retaining upload buffer (%d/%d)\n",
		2214	__FUNCTION__, bo->used, bytes(&bo->base)));
		2215	bo->used = used;
		2216	list_move(&bo->base.list,
		2217	&kgem->active_buffers);
		2218	continue;
		2219	}
		2220	DBG(("%s: discarding mmapped buffer, used=%d, map type=%d\n",
		2221	__FUNCTION__, bo->used, (int)__MAP_TYPE(bo->base.map)));
		2222	goto decouple;
		2223	}
		2224
		2225	if (!bo->used) {
		2226	/* Unless we replace the handle in the execbuffer,
		2227	* then this bo will become active. So decouple it
		2228	* from the buffer list and track it in the normal
		2229	* manner.
		2230	*/
		2231	goto decouple;
		2232	}
		2233
		2234	assert(bo->need_io);
		2235	assert(bo->base.rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		2236	assert(bo->base.domain != DOMAIN_GPU);
		2237
		2238	if (bo->base.refcnt == 1 &&
		2239	bo->base.size.pages.count > 1 &&
		2240	bo->used < bytes(&bo->base) / 2) {
		2241	struct kgem_bo *shrink;
		2242	unsigned alloc = NUM_PAGES(bo->used);
		2243
		2244	shrink = search_snoop_cache(kgem, alloc,
		2245	CREATE_INACTIVE \| CREATE_NO_RETIRE);
		2246	if (shrink) {
		2247	void *map;
		2248	int n;
		2249
		2250	DBG(("%s: used=%d, shrinking %d to %d, handle %d to %d\n",
		2251	__FUNCTION__,
		2252	bo->used, bytes(&bo->base), bytes(shrink),
		2253	bo->base.handle, shrink->handle));
		2254
		2255	assert(bo->used <= bytes(shrink));
		2256	map = kgem_bo_map__cpu(kgem, shrink);
		2257	if (map) {
		2258	kgem_bo_sync__cpu(kgem, shrink);
		2259	memcpy(map, bo->mem, bo->used);
		2260
		2261	shrink->target_handle =
		2262	kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
		2263	for (n = 0; n < kgem->nreloc; n++) {
		2264	if (kgem->reloc[n].target_handle == bo->base.target_handle) {
		2265	kgem->reloc[n].target_handle = shrink->target_handle;
		2266	kgem->reloc[n].presumed_offset = shrink->presumed_offset;
		2267	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		2268	kgem->reloc[n].delta + shrink->presumed_offset;
		2269	}
		2270	}
		2271
		2272	bo->base.exec->handle = shrink->handle;
		2273	bo->base.exec->offset = shrink->presumed_offset;
		2274	shrink->exec = bo->base.exec;
		2275	shrink->rq = bo->base.rq;
		2276	list_replace(&bo->base.request,
		2277	&shrink->request);
		2278	list_init(&bo->base.request);
		2279	shrink->needs_flush = bo->base.gpu_dirty;
		2280
		2281	bo->base.exec = NULL;
		2282	bo->base.rq = NULL;
		2283	bo->base.gpu_dirty = false;
		2284	bo->base.needs_flush = false;
		2285	bo->used = 0;
		2286
		2287	goto decouple;
		2288	}
		2289
		2290	__kgem_bo_destroy(kgem, shrink);
		2291	}
		2292
		2293	shrink = search_linear_cache(kgem, alloc,
		2294	CREATE_INACTIVE \| CREATE_NO_RETIRE);
		2295	if (shrink) {
		2296	int n;
		2297
		2298	DBG(("%s: used=%d, shrinking %d to %d, handle %d to %d\n",
		2299	__FUNCTION__,
		2300	bo->used, bytes(&bo->base), bytes(shrink),
		2301	bo->base.handle, shrink->handle));
		2302
		2303	assert(bo->used <= bytes(shrink));
		2304	if (gem_write(kgem->fd, shrink->handle,
		2305	0, bo->used, bo->mem) == 0) {
		2306	shrink->target_handle =
		2307	kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
		2308	for (n = 0; n < kgem->nreloc; n++) {
		2309	if (kgem->reloc[n].target_handle == bo->base.target_handle) {
		2310	kgem->reloc[n].target_handle = shrink->target_handle;
		2311	kgem->reloc[n].presumed_offset = shrink->presumed_offset;
		2312	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		2313	kgem->reloc[n].delta + shrink->presumed_offset;
		2314	}
		2315	}
		2316
		2317	bo->base.exec->handle = shrink->handle;
		2318	bo->base.exec->offset = shrink->presumed_offset;
		2319	shrink->exec = bo->base.exec;
		2320	shrink->rq = bo->base.rq;
		2321	list_replace(&bo->base.request,
		2322	&shrink->request);
		2323	list_init(&bo->base.request);
		2324	shrink->needs_flush = bo->base.gpu_dirty;
		2325
		2326	bo->base.exec = NULL;
		2327	bo->base.rq = NULL;
		2328	bo->base.gpu_dirty = false;
		2329	bo->base.needs_flush = false;
		2330	bo->used = 0;
		2331
		2332	goto decouple;
		2333	}
		2334
		2335	__kgem_bo_destroy(kgem, shrink);
		2336	}
		2337	}
		2338
		2339	DBG(("%s: handle=%d, uploading %d/%d\n",
		2340	__FUNCTION__, bo->base.handle, bo->used, bytes(&bo->base)));
		2341	ASSERT_IDLE(kgem, bo->base.handle);
		2342	assert(bo->used <= bytes(&bo->base));
		2343	gem_write(kgem->fd, bo->base.handle,
		2344	0, bo->used, bo->mem);
		2345	bo->need_io = 0;
		2346
		2347	decouple:
		2348	DBG(("%s: releasing handle=%d\n",
		2349	__FUNCTION__, bo->base.handle));
		2350	list_del(&bo->base.list);
		2351	kgem_bo_unref(kgem, &bo->base);
		2352	}
		2353	}
		2354
		2355	static void kgem_cleanup(struct kgem *kgem)
		2356	{
		2357	int n;
		2358
		2359	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2360	while (!list_is_empty(&kgem->requests[n])) {
		2361	struct kgem_request *rq;
		2362
		2363	rq = list_first_entry(&kgem->requests[n],
		2364	struct kgem_request,
		2365	list);
		2366	while (!list_is_empty(&rq->buffers)) {
		2367	struct kgem_bo *bo;
		2368
		2369	bo = list_first_entry(&rq->buffers,
		2370	struct kgem_bo,
		2371	request);
		2372
		2373	bo->exec = NULL;
		2374	bo->gpu_dirty = false;
		2375	__kgem_bo_clear_busy(bo);
		2376	if (bo->refcnt == 0)
		2377	kgem_bo_free(kgem, bo);
		2378	}
		2379
		2380	__kgem_request_free(rq);
		2381	}
		2382	}
		2383
		2384	kgem_close_inactive(kgem);
		2385	}
		2386
		2387	static int kgem_batch_write(struct kgem *kgem, uint32_t handle, uint32_t size)
		2388	{
		2389	int ret;
		2390
		2391	ASSERT_IDLE(kgem, handle);
		2392
		2393	/* If there is no surface data, just upload the batch */
		2394	if (kgem->surface == kgem->batch_size)
		2395	return gem_write(kgem->fd, handle,
		2396	0, sizeof(uint32_t)*kgem->nbatch,
		2397	kgem->batch);
		2398
		2399	/* Are the batch pages conjoint with the surface pages? */
		2400	if (kgem->surface < kgem->nbatch + PAGE_SIZE/sizeof(uint32_t)) {
		2401	assert(size == PAGE_ALIGN(kgem->batch_size*sizeof(uint32_t)));
		2402	return gem_write(kgem->fd, handle,
		2403	0, kgem->batch_size*sizeof(uint32_t),
		2404	kgem->batch);
		2405	}
		2406
		2407	/* Disjoint surface/batch, upload separately */
		2408	ret = gem_write(kgem->fd, handle,
		2409	0, sizeof(uint32_t)*kgem->nbatch,
		2410	kgem->batch);
		2411	if (ret)
		2412	return ret;
		2413
		2414	ret = PAGE_ALIGN(sizeof(uint32_t) * kgem->batch_size);
		2415	ret -= sizeof(uint32_t) * kgem->surface;
		2416	assert(size-ret >= kgem->nbatch*sizeof(uint32_t));
		2417	return __gem_write(kgem->fd, handle,
		2418	size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t),
		2419	kgem->batch + kgem->surface);
		2420	}
		2421
		2422	void kgem_reset(struct kgem *kgem)
		2423	{
		2424	if (kgem->next_request) {
		2425	struct kgem_request *rq = kgem->next_request;
		2426
		2427	while (!list_is_empty(&rq->buffers)) {
		2428	struct kgem_bo *bo =
		2429	list_first_entry(&rq->buffers,
		2430	struct kgem_bo,
		2431	request);
		2432	list_del(&bo->request);
		2433
		2434	assert(RQ(bo->rq) == rq);
		2435
		2436	bo->binding.offset = 0;
		2437	bo->exec = NULL;
		2438	bo->target_handle = -1;
		2439	bo->gpu_dirty = false;
		2440
		2441	if (bo->needs_flush && __kgem_busy(kgem, bo->handle)) {
		2442	assert(bo->domain == DOMAIN_GPU \|\| bo->domain == DOMAIN_NONE);
		2443	list_add(&bo->request, &kgem->flushing);
		2444	bo->rq = (void *)kgem;
		2445	} else
		2446	__kgem_bo_clear_busy(bo);
		2447
		2448	if (bo->refcnt \|\| bo->rq)
		2449	continue;
		2450
		2451	if (bo->snoop) {
		2452	kgem_bo_move_to_snoop(kgem, bo);
		2453	} else if (bo->scanout) {
		2454	kgem_bo_move_to_scanout(kgem, bo);
		2455	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		2456	kgem_bo_set_purgeable(kgem, bo)) {
		2457	kgem_bo_move_to_inactive(kgem, bo);
		2458	} else {
		2459	DBG(("%s: closing %d\n",
		2460	__FUNCTION__, bo->handle));
		2461	kgem_bo_free(kgem, bo);
		2462	}
		2463	}
		2464
		2465	if (rq != &kgem->static_request) {
		2466	list_init(&rq->list);
		2467	__kgem_request_free(rq);
		2468	}
		2469	}
		2470
		2471	kgem->nfence = 0;
		2472	kgem->nexec = 0;
		2473	kgem->nreloc = 0;
		2474	kgem->nreloc__self = 0;
		2475	kgem->aperture = 0;
		2476	kgem->aperture_fenced = 0;
		2477	kgem->nbatch = 0;
		2478	kgem->surface = kgem->batch_size;
		2479	kgem->mode = KGEM_NONE;
		2480	kgem->flush = 0;
		2481	kgem->batch_flags = kgem->batch_flags_base;
		2482
		2483	kgem->next_request = __kgem_request_alloc(kgem);
		2484
		2485	kgem_sna_reset(kgem);
		2486	}
		2487
		2488	static int compact_batch_surface(struct kgem *kgem)
		2489	{
		2490	int size, shrink, n;
		2491
		2492	if (!kgem->has_relaxed_delta)
		2493	return kgem->batch_size;
		2494
		2495	/* See if we can pack the contents into one or two pages */
		2496	n = ALIGN(kgem->batch_size, 1024);
		2497	size = n - kgem->surface + kgem->nbatch;
		2498	size = ALIGN(size, 1024);
		2499
		2500	shrink = n - size;
		2501	if (shrink) {
		2502	DBG(("shrinking from %d to %d\n", kgem->batch_size, size));
		2503
		2504	shrink *= sizeof(uint32_t);
		2505	for (n = 0; n < kgem->nreloc; n++) {
		2506	if (kgem->reloc[n].read_domains == I915_GEM_DOMAIN_INSTRUCTION &&
		2507	kgem->reloc[n].target_handle == ~0U)
		2508	kgem->reloc[n].delta -= shrink;
		2509
		2510	if (kgem->reloc[n].offset >= sizeof(uint32_t)*kgem->nbatch)
		2511	kgem->reloc[n].offset -= shrink;
		2512	}
		2513	}
		2514
		2515	return size * sizeof(uint32_t);
		2516	}
		2517
		2518	static struct kgem_bo *
		2519	kgem_create_batch(struct kgem *kgem, int size)
		2520	{
		2521	struct drm_i915_gem_set_domain set_domain;
		2522	struct kgem_bo *bo;
		2523
		2524	if (size <= 4096) {
		2525	bo = list_first_entry(&kgem->pinned_batches[0],
		2526	struct kgem_bo,
		2527	list);
		2528	if (!bo->rq) {
		2529	out_4096:
		2530	list_move_tail(&bo->list, &kgem->pinned_batches[0]);
		2531	return kgem_bo_reference(bo);
		2532	}
		2533
		2534	if (!__kgem_busy(kgem, bo->handle)) {
		2535	assert(RQ(bo->rq)->bo == bo);
		2536	__kgem_retire_rq(kgem, RQ(bo->rq));
		2537	goto out_4096;
		2538	}
		2539	}
		2540
		2541	if (size <= 16384) {
		2542	bo = list_first_entry(&kgem->pinned_batches[1],
		2543	struct kgem_bo,
		2544	list);
		2545	if (!bo->rq) {
		2546	out_16384:
		2547	list_move_tail(&bo->list, &kgem->pinned_batches[1]);
		2548	return kgem_bo_reference(bo);
		2549	}
		2550
		2551	if (!__kgem_busy(kgem, bo->handle)) {
		2552	assert(RQ(bo->rq)->bo == bo);
		2553	__kgem_retire_rq(kgem, RQ(bo->rq));
		2554	goto out_16384;
		2555	}
		2556	}
		2557
		2558	if (kgem->gen == 020 && !kgem->has_pinned_batches) {
		2559	assert(size <= 16384);
		2560
		2561	bo = list_first_entry(&kgem->pinned_batches[size > 4096],
		2562	struct kgem_bo,
		2563	list);
		2564	list_move_tail(&bo->list, &kgem->pinned_batches[size > 4096]);
		2565
		2566	DBG(("%s: syncing due to busy batches\n", __FUNCTION__));
		2567
		2568	VG_CLEAR(set_domain);
		2569	set_domain.handle = bo->handle;
		2570	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2571	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2572	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
		2573	DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
		2574	kgem_throttle(kgem);
		2575	return NULL;
		2576	}
		2577
		2578	kgem_retire(kgem);
		2579	assert(bo->rq == NULL);
		2580	return kgem_bo_reference(bo);
		2581	}
		2582
		2583	return kgem_create_linear(kgem, size, CREATE_NO_THROTTLE);
		2584	}
		2585
		2586	void _kgem_submit(struct kgem *kgem)
		2587	{
		2588	struct kgem_request *rq;
		2589	uint32_t batch_end;
		2590	int size;
		2591
		2592	assert(!DBG_NO_HW);
		2593	assert(!kgem->wedged);
		2594
		2595	assert(kgem->nbatch);
		2596	assert(kgem->nbatch <= KGEM_BATCH_SIZE(kgem));
		2597	assert(kgem->nbatch <= kgem->surface);
		2598
		2599	batch_end = kgem_end_batch(kgem);
		2600	kgem_sna_flush(kgem);
		2601
		2602	DBG(("batch[%d/%d, flags=%x]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n",
		2603	kgem->mode, kgem->ring, kgem->batch_flags,
		2604	batch_end, kgem->nbatch, kgem->surface, kgem->batch_size,
		2605	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture));
		2606
		2607	assert(kgem->nbatch <= kgem->batch_size);
		2608	assert(kgem->nbatch <= kgem->surface);
		2609	assert(kgem->nreloc <= ARRAY_SIZE(kgem->reloc));
		2610	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		2611	assert(kgem->nfence <= kgem->fence_max);
		2612
		2613	kgem_finish_buffers(kgem);
		2614
		2615	#if SHOW_BATCH
		2616	__kgem_batch_debug(kgem, batch_end);
		2617	#endif
		2618
		2619	rq = kgem->next_request;
		2620	if (kgem->surface != kgem->batch_size)
		2621	size = compact_batch_surface(kgem);
		2622	else
		2623	size = kgem->nbatch * sizeof(kgem->batch[0]);
		2624	rq->bo = kgem_create_batch(kgem, size);
		2625	if (rq->bo) {
		2626	uint32_t handle = rq->bo->handle;
		2627	int i;
		2628
		2629	assert(!rq->bo->needs_flush);
		2630
		2631	i = kgem->nexec++;
		2632	kgem->exec[i].handle = handle;
		2633	kgem->exec[i].relocation_count = kgem->nreloc;
		2634	kgem->exec[i].relocs_ptr = (uintptr_t)kgem->reloc;
		2635	kgem->exec[i].alignment = 0;
		2636	kgem->exec[i].offset = rq->bo->presumed_offset;
		2637	kgem->exec[i].flags = 0;
		2638	kgem->exec[i].rsvd1 = 0;
		2639	kgem->exec[i].rsvd2 = 0;
		2640
		2641	rq->bo->target_handle = kgem->has_handle_lut ? i : handle;
		2642	rq->bo->exec = &kgem->exec[i];
		2643	rq->bo->rq = MAKE_REQUEST(rq, kgem->ring); /* useful sanity check */
		2644	list_add(&rq->bo->request, &rq->buffers);
		2645	rq->ring = kgem->ring == KGEM_BLT;
		2646
		2647	kgem_fixup_self_relocs(kgem, rq->bo);
		2648
		2649	if (kgem_batch_write(kgem, handle, size) == 0) {
		2650	struct drm_i915_gem_execbuffer2 execbuf;
		2651	int ret, retry = 3;
		2652
		2653	memset(&execbuf, 0, sizeof(execbuf));
		2654	execbuf.buffers_ptr = (uintptr_t)kgem->exec;
		2655	execbuf.buffer_count = kgem->nexec;
		2656	execbuf.batch_len = batch_end*sizeof(uint32_t);
		2657	execbuf.flags = kgem->ring \| kgem->batch_flags;
		2658
		2659	if (DEBUG_DUMP)
		2660	{
		2661	int fd = open("/tmp1/1/batchbuffer.bin", O_CREAT\|O_WRONLY\|O_BINARY);
		2662	if (fd != -1) {
		2663	write(fd, kgem->batch, size);
		2664	close(fd);
		2665	}
		2666	else printf("SNA: failed to write batchbuffer\n");
		2667	asm volatile("int3");
		2668	}
		2669
		2670	ret = drmIoctl(kgem->fd,
		2671	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2672	&execbuf);
		2673	while (ret == -1 && errno == EBUSY && retry--) {
		2674	__kgem_throttle(kgem);
		2675	ret = drmIoctl(kgem->fd,
		2676	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2677	&execbuf);
		2678	}
		2679	if (DEBUG_SYNC && ret == 0) {
		2680	struct drm_i915_gem_set_domain set_domain;
		2681
		2682	VG_CLEAR(set_domain);
		2683	set_domain.handle = handle;
		2684	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2685	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2686
		2687	ret = drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
		2688	}
		2689	if (ret == -1) {
		2690	DBG(("%s: GPU hang detected [%d]\n",
		2691	__FUNCTION__, errno));
		2692	kgem_throttle(kgem);
		2693	kgem->wedged = true;
		2694
		2695	#if 0
		2696	ret = errno;
		2697	ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d: errno=%d\n",
		2698	kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
		2699	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, errno);
		2700
		2701	for (i = 0; i < kgem->nexec; i++) {
		2702	struct kgem_bo bo, found = NULL;
		2703
		2704	list_for_each_entry(bo, &kgem->next_request->buffers, request) {
		2705	if (bo->handle == kgem->exec[i].handle) {
		2706	found = bo;
		2707	break;
		2708	}
		2709	}
		2710	ErrorF("exec[%d] = handle:%d, presumed offset: %x, size: %d, tiling %d, fenced %d, snooped %d, deleted %d\n",
		2711	i,
		2712	kgem->exec[i].handle,
		2713	(int)kgem->exec[i].offset,
		2714	found ? kgem_bo_size(found) : -1,
		2715	found ? found->tiling : -1,
		2716	(int)(kgem->exec[i].flags & EXEC_OBJECT_NEEDS_FENCE),
		2717	found ? found->snoop : -1,
		2718	found ? found->purged : -1);
		2719	}
		2720	for (i = 0; i < kgem->nreloc; i++) {
		2721	ErrorF("reloc[%d] = pos:%d, target:%d, delta:%d, read:%x, write:%x, offset:%x\n",
		2722	i,
		2723	(int)kgem->reloc[i].offset,
		2724	kgem->reloc[i].target_handle,
		2725	kgem->reloc[i].delta,
		2726	kgem->reloc[i].read_domains,
		2727	kgem->reloc[i].write_domain,
		2728	(int)kgem->reloc[i].presumed_offset);
		2729	}
		2730
		2731	if (DEBUG_SYNC) {
		2732	int fd = open("/tmp/batchbuffer", O_WRONLY \| O_CREAT \| O_APPEND, 0666);
		2733	if (fd != -1) {
		2734	write(fd, kgem->batch, batch_end*sizeof(uint32_t));
		2735	close(fd);
		2736	}
		2737
		2738	FatalError("SNA: failed to submit batchbuffer, errno=%d\n", ret);
		2739	}
		2740	#endif
		2741	}
		2742	}
		2743
		2744	kgem_commit(kgem);
		2745	}
		2746	if (kgem->wedged)
		2747	kgem_cleanup(kgem);
		2748
		2749	kgem_reset(kgem);
		2750
		2751	assert(kgem->next_request != NULL);
		2752	}
		2753
		2754	void kgem_throttle(struct kgem *kgem)
		2755	{
		2756	kgem->need_throttle = 0;
		2757	if (kgem->wedged)
		2758	return;
		2759
		2760	kgem->wedged = __kgem_throttle(kgem);
		2761	if (kgem->wedged) {
		2762	printf("Detected a hung GPU, disabling acceleration.\n");
		2763	printf("When reporting this, please include i915_error_state from debugfs and the full dmesg.\n");
		2764	}
		2765	}
		2766
		2767	void kgem_purge_cache(struct kgem *kgem)
		2768	{
		2769	struct kgem_bo bo, next;
		2770	int i;
		2771
		2772	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2773	list_for_each_entry_safe(bo, next, &kgem->inactive[i], list) {
		2774	if (!kgem_bo_is_retained(kgem, bo)) {
		2775	DBG(("%s: purging %d\n",
		2776	__FUNCTION__, bo->handle));
		2777	kgem_bo_free(kgem, bo);
		2778	}
		2779	}
		2780	}
		2781
		2782	kgem->need_purge = false;
		2783	}
		2784
		2785
		2786	void kgem_clean_large_cache(struct kgem *kgem)
		2787	{
		2788	while (!list_is_empty(&kgem->large_inactive)) {
		2789	kgem_bo_free(kgem,
		2790	list_first_entry(&kgem->large_inactive,
		2791	struct kgem_bo, list));
		2792
		2793	}
		2794	}
		2795
		2796	bool kgem_expire_cache(struct kgem *kgem)
		2797	{
		2798	time_t now, expire;
		2799	struct kgem_bo *bo;
		2800	unsigned int size = 0, count = 0;
		2801	bool idle;
		2802	unsigned int i;
		2803
		2804	time(&now);
		2805
		2806	while (__kgem_freed_bo) {
		2807	bo = __kgem_freed_bo;
		2808	__kgem_freed_bo = (struct kgem_bo *)bo;
		2809	free(bo);
		2810	}
		2811
		2812	while (__kgem_freed_request) {
		2813	struct kgem_request *rq = __kgem_freed_request;
		2814	__kgem_freed_request = (struct kgem_request *)rq;
		2815	free(rq);
		2816	}
		2817
		2818	kgem_clean_large_cache(kgem);
		2819
		2820	expire = 0;
		2821	list_for_each_entry(bo, &kgem->snoop, list) {
		2822	if (bo->delta) {
		2823	expire = now - MAX_INACTIVE_TIME/2;
		2824	break;
		2825	}
		2826
		2827	bo->delta = now;
		2828	}
		2829	if (expire) {
		2830	while (!list_is_empty(&kgem->snoop)) {
		2831	bo = list_last_entry(&kgem->snoop, struct kgem_bo, list);
		2832
		2833	if (bo->delta > expire)
		2834	break;
		2835
		2836	kgem_bo_free(kgem, bo);
		2837	}
		2838	}
		2839	#ifdef DEBUG_MEMORY
		2840	{
		2841	long snoop_size = 0;
		2842	int snoop_count = 0;
		2843	list_for_each_entry(bo, &kgem->snoop, list)
		2844	snoop_count++, snoop_size += bytes(bo);
		2845	ErrorF("%s: still allocated %d bo, %ld bytes, in snoop cache\n",
		2846	__FUNCTION__, snoop_count, snoop_size);
		2847	}
		2848	#endif
		2849
		2850	kgem_retire(kgem);
		2851	if (kgem->wedged)
		2852	kgem_cleanup(kgem);
		2853
		2854	kgem->expire(kgem);
		2855
		2856	if (kgem->need_purge)
		2857	kgem_purge_cache(kgem);
		2858
		2859	expire = 0;
		2860
		2861	idle = !kgem->need_retire;
		2862	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2863	idle &= list_is_empty(&kgem->inactive[i]);
		2864	list_for_each_entry(bo, &kgem->inactive[i], list) {
		2865	if (bo->delta) {
		2866	expire = now - MAX_INACTIVE_TIME;
		2867	break;
		2868	}
		2869
		2870	bo->delta = now;
		2871	}
		2872	}
		2873	if (idle) {
		2874	DBG(("%s: idle\n", __FUNCTION__));
		2875	kgem->need_expire = false;
		2876	return false;
		2877	}
		2878	if (expire == 0)
		2879	return true;
		2880
		2881	idle = !kgem->need_retire;
		2882	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2883	struct list preserve;
		2884
		2885	list_init(&preserve);
		2886	while (!list_is_empty(&kgem->inactive[i])) {
		2887	bo = list_last_entry(&kgem->inactive[i],
		2888	struct kgem_bo, list);
		2889
		2890	if (bo->delta > expire) {
		2891	idle = false;
		2892	break;
		2893	}
		2894
		2895	if (bo->map && bo->delta + MAP_PRESERVE_TIME > expire) {
		2896	idle = false;
		2897	list_move_tail(&bo->list, &preserve);
		2898	} else {
		2899	count++;
		2900	size += bytes(bo);
		2901	kgem_bo_free(kgem, bo);
		2902	DBG(("%s: expiring %d\n",
		2903	__FUNCTION__, bo->handle));
		2904	}
		2905	}
		2906	if (!list_is_empty(&preserve)) {
		2907	preserve.prev->next = kgem->inactive[i].next;
		2908	kgem->inactive[i].next->prev = preserve.prev;
		2909	kgem->inactive[i].next = preserve.next;
		2910	preserve.next->prev = &kgem->inactive[i];
		2911	}
		2912	}
		2913
		2914	#ifdef DEBUG_MEMORY
		2915	{
		2916	long inactive_size = 0;
		2917	int inactive_count = 0;
		2918	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		2919	list_for_each_entry(bo, &kgem->inactive[i], list)
		2920	inactive_count++, inactive_size += bytes(bo);
		2921	ErrorF("%s: still allocated %d bo, %ld bytes, in inactive cache\n",
		2922	__FUNCTION__, inactive_count, inactive_size);
		2923	}
		2924	#endif
		2925
		2926	DBG(("%s: expired %d objects, %d bytes, idle? %d\n",
		2927	__FUNCTION__, count, size, idle));
		2928
		2929	kgem->need_expire = !idle;
		2930	return !idle;
		2931	(void)count;
		2932	(void)size;
		2933	}
		2934
		2935	void kgem_cleanup_cache(struct kgem *kgem)
		2936	{
		2937	unsigned int i;
		2938	int n;
		2939
		2940	/* sync to the most recent request */
		2941	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2942	if (!list_is_empty(&kgem->requests[n])) {
		2943	struct kgem_request *rq;
		2944	struct drm_i915_gem_set_domain set_domain;
		2945
		2946	rq = list_first_entry(&kgem->requests[n],
		2947	struct kgem_request,
		2948	list);
		2949
		2950	DBG(("%s: sync on cleanup\n", __FUNCTION__));
		2951
		2952	VG_CLEAR(set_domain);
		2953	set_domain.handle = rq->bo->handle;
		2954	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2955	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2956	(void)drmIoctl(kgem->fd,
		2957	DRM_IOCTL_I915_GEM_SET_DOMAIN,
		2958	&set_domain);
		2959	}
		2960	}
		2961
		2962	kgem_retire(kgem);
		2963	kgem_cleanup(kgem);
		2964
		2965	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2966	while (!list_is_empty(&kgem->inactive[i]))
		2967	kgem_bo_free(kgem,
		2968	list_last_entry(&kgem->inactive[i],
		2969	struct kgem_bo, list));
		2970	}
		2971
		2972	kgem_clean_large_cache(kgem);
		2973
		2974	while (!list_is_empty(&kgem->snoop))
		2975	kgem_bo_free(kgem,
		2976	list_last_entry(&kgem->snoop,
		2977	struct kgem_bo, list));
		2978
		2979	while (__kgem_freed_bo) {
		2980	struct kgem_bo *bo = __kgem_freed_bo;
		2981	__kgem_freed_bo = (struct kgem_bo *)bo;
		2982	free(bo);
		2983	}
		2984
		2985	kgem->need_purge = false;
		2986	kgem->need_expire = false;
		2987	}
		2988
		2989	static struct kgem_bo *
		2990	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		2991	{
		2992	struct kgem_bo bo, first = NULL;
		2993	bool use_active = (flags & CREATE_INACTIVE) == 0;
		2994	struct list *cache;
		2995
		2996	DBG(("%s: num_pages=%d, flags=%x, use_active? %d, use_large=%d [max=%d]\n",
		2997	__FUNCTION__, num_pages, flags, use_active,
		2998	num_pages >= MAX_CACHE_SIZE / PAGE_SIZE,
		2999	MAX_CACHE_SIZE / PAGE_SIZE));
		3000
		3001	assert(num_pages);
		3002
		3003	if (num_pages >= MAX_CACHE_SIZE / PAGE_SIZE) {
		3004	DBG(("%s: searching large buffers\n", __FUNCTION__));
		3005	retry_large:
		3006	cache = use_active ? &kgem->large : &kgem->large_inactive;
		3007	list_for_each_entry_safe(bo, first, cache, list) {
		3008	assert(bo->refcnt == 0);
		3009	assert(bo->reusable);
		3010	assert(!bo->scanout);
		3011
		3012	if (num_pages > num_pages(bo))
		3013	goto discard;
		3014
		3015	if (bo->tiling != I915_TILING_NONE) {
		3016	if (use_active)
		3017	goto discard;
		3018
		3019	if (!gem_set_tiling(kgem->fd, bo->handle,
		3020	I915_TILING_NONE, 0))
		3021	goto discard;
		3022
		3023	bo->tiling = I915_TILING_NONE;
		3024	bo->pitch = 0;
		3025	}
		3026
		3027	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo))
		3028	goto discard;
		3029
		3030	list_del(&bo->list);
		3031	if (bo->rq == (void *)kgem)
		3032	list_del(&bo->request);
		3033
		3034	bo->delta = 0;
		3035	assert_tiling(kgem, bo);
		3036	return bo;
		3037
		3038	discard:
		3039	if (!use_active)
		3040	kgem_bo_free(kgem, bo);
		3041	}
		3042
		3043	if (use_active) {
		3044	use_active = false;
		3045	goto retry_large;
		3046	}
		3047
		3048	if (__kgem_throttle_retire(kgem, flags))
		3049	goto retry_large;
		3050
		3051	return NULL;
		3052	}
		3053
		3054	if (!use_active && list_is_empty(inactive(kgem, num_pages))) {
		3055	DBG(("%s: inactive and cache bucket empty\n",
		3056	__FUNCTION__));
		3057
		3058	if (flags & CREATE_NO_RETIRE) {
		3059	DBG(("%s: can not retire\n", __FUNCTION__));
		3060	return NULL;
		3061	}
		3062
		3063	if (list_is_empty(active(kgem, num_pages, I915_TILING_NONE))) {
		3064	DBG(("%s: active cache bucket empty\n", __FUNCTION__));
		3065	return NULL;
		3066	}
		3067
		3068	if (!__kgem_throttle_retire(kgem, flags)) {
		3069	DBG(("%s: nothing retired\n", __FUNCTION__));
		3070	return NULL;
		3071	}
		3072
		3073	if (list_is_empty(inactive(kgem, num_pages))) {
		3074	DBG(("%s: active cache bucket still empty after retire\n",
		3075	__FUNCTION__));
		3076	return NULL;
		3077	}
		3078	}
		3079
		3080	if (!use_active && flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3081	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3082	DBG(("%s: searching for inactive %s map\n",
		3083	__FUNCTION__, for_cpu ? "cpu" : "gtt"));
		3084	cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
		3085	list_for_each_entry(bo, cache, vma) {
		3086	assert(IS_CPU_MAP(bo->map) == for_cpu);
		3087	assert(bucket(bo) == cache_bucket(num_pages));
		3088	assert(bo->proxy == NULL);
		3089	assert(bo->rq == NULL);
		3090	assert(bo->exec == NULL);
		3091	assert(!bo->scanout);
		3092
		3093	if (num_pages > num_pages(bo)) {
		3094	DBG(("inactive too small: %d < %d\n",
		3095	num_pages(bo), num_pages));
		3096	continue;
		3097	}
		3098
		3099	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3100	kgem_bo_free(kgem, bo);
		3101	break;
		3102	}
		3103
		3104	if (I915_TILING_NONE != bo->tiling &&
		3105	!gem_set_tiling(kgem->fd, bo->handle,
		3106	I915_TILING_NONE, 0))
		3107	continue;
		3108
		3109	kgem_bo_remove_from_inactive(kgem, bo);
		3110
		3111	bo->tiling = I915_TILING_NONE;
		3112	bo->pitch = 0;
		3113	bo->delta = 0;
		3114	DBG((" %s: found handle=%d (num_pages=%d) in linear vma cache\n",
		3115	__FUNCTION__, bo->handle, num_pages(bo)));
		3116	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		3117	assert(!bo->needs_flush);
		3118	assert_tiling(kgem, bo);
		3119	ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
		3120	return bo;
		3121	}
		3122
		3123	if (flags & CREATE_EXACT)
		3124	return NULL;
		3125
		3126	if (flags & CREATE_CPU_MAP && !kgem->has_llc)
		3127	return NULL;
		3128	}
		3129
		3130	cache = use_active ? active(kgem, num_pages, I915_TILING_NONE) : inactive(kgem, num_pages);
		3131	list_for_each_entry(bo, cache, list) {
		3132	assert(bo->refcnt == 0);
		3133	assert(bo->reusable);
		3134	assert(!!bo->rq == !!use_active);
		3135	assert(bo->proxy == NULL);
		3136	assert(!bo->scanout);
		3137
		3138	if (num_pages > num_pages(bo))
		3139	continue;
		3140
		3141	if (use_active &&
		3142	kgem->gen <= 040 &&
		3143	bo->tiling != I915_TILING_NONE)
		3144	continue;
		3145
		3146	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3147	kgem_bo_free(kgem, bo);
		3148	break;
		3149	}
		3150
		3151	if (I915_TILING_NONE != bo->tiling) {
		3152	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP))
		3153	continue;
		3154
		3155	if (first)
		3156	continue;
		3157
		3158	if (!gem_set_tiling(kgem->fd, bo->handle,
		3159	I915_TILING_NONE, 0))
		3160	continue;
		3161
		3162	bo->tiling = I915_TILING_NONE;
		3163	bo->pitch = 0;
		3164	}
		3165
		3166	if (bo->map) {
		3167	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3168	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3169	if (IS_CPU_MAP(bo->map) != for_cpu) {
		3170	if (first != NULL)
		3171	break;
		3172
		3173	first = bo;
		3174	continue;
		3175	}
		3176	} else {
		3177	if (first != NULL)
		3178	break;
		3179
		3180	first = bo;
		3181	continue;
		3182	}
		3183	} else {
		3184	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3185	if (first != NULL)
		3186	break;
		3187
		3188	first = bo;
		3189	continue;
		3190	}
		3191	}
		3192
		3193	if (use_active)
		3194	kgem_bo_remove_from_active(kgem, bo);
		3195	else
		3196	kgem_bo_remove_from_inactive(kgem, bo);
		3197
		3198	assert(bo->tiling == I915_TILING_NONE);
		3199	bo->pitch = 0;
		3200	bo->delta = 0;
		3201	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		3202	__FUNCTION__, bo->handle, num_pages(bo),
		3203	use_active ? "active" : "inactive"));
		3204	assert(list_is_empty(&bo->list));
		3205	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		3206	assert(!bo->needs_flush \|\| use_active);
		3207	assert_tiling(kgem, bo);
		3208	ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
		3209	return bo;
		3210	}
		3211
		3212	if (first) {
		3213	assert(first->tiling == I915_TILING_NONE);
		3214
		3215	if (use_active)
		3216	kgem_bo_remove_from_active(kgem, first);
		3217	else
		3218	kgem_bo_remove_from_inactive(kgem, first);
		3219
		3220	first->pitch = 0;
		3221	first->delta = 0;
		3222	DBG((" %s: found handle=%d (near-miss) (num_pages=%d) in linear %s cache\n",
		3223	__FUNCTION__, first->handle, num_pages(first),
		3224	use_active ? "active" : "inactive"));
		3225	assert(list_is_empty(&first->list));
		3226	assert(use_active \|\| first->domain != DOMAIN_GPU);
		3227	assert(!first->needs_flush \|\| use_active);
		3228	ASSERT_MAYBE_IDLE(kgem, first->handle, !use_active);
		3229	return first;
		3230	}
		3231
		3232	return NULL;
		3233	}
		3234
		3235
		3236	struct kgem_bo kgem_create_linear(struct kgem kgem, int size, unsigned flags)
		3237	{
		3238	struct kgem_bo *bo;
		3239	uint32_t handle;
		3240
		3241	DBG(("%s(%d)\n", __FUNCTION__, size));
		3242	assert(size);
		3243
		3244	if (flags & CREATE_GTT_MAP && kgem->has_llc) {
		3245	flags &= ~CREATE_GTT_MAP;
		3246	flags \|= CREATE_CPU_MAP;
		3247	}
		3248
		3249	size = NUM_PAGES(size);
		3250	bo = search_linear_cache(kgem, size, CREATE_INACTIVE \| flags);
		3251	if (bo) {
		3252	assert(bo->domain != DOMAIN_GPU);
		3253	ASSERT_IDLE(kgem, bo->handle);
		3254	bo->refcnt = 1;
		3255	return bo;
		3256	}
		3257
		3258	if (flags & CREATE_CACHED)
		3259	return NULL;
		3260
		3261	handle = gem_create(kgem->fd, size);
		3262	if (handle == 0)
		3263	return NULL;
		3264
		3265	DBG(("%s: new handle=%d, num_pages=%d\n", __FUNCTION__, handle, size));
		3266	bo = __kgem_bo_alloc(handle, size);
		3267	if (bo == NULL) {
		3268	gem_close(kgem->fd, handle);
		3269	return NULL;
		3270	}
		3271
		3272	debug_alloc__bo(kgem, bo);
		3273	return bo;
		3274	}
		3275
		3276	inline int kgem_bo_fenced_size(struct kgem kgem, struct kgem_bo bo)
		3277	{
		3278	unsigned int size;
		3279
		3280	assert(bo->tiling);
		3281	assert_tiling(kgem, bo);
		3282	assert(kgem->gen < 040);
		3283
		3284	if (kgem->gen < 030)
		3285	size = 512 * 1024;
		3286	else
		3287	size = 1024 * 1024;
		3288	while (size < bytes(bo))
		3289	size *= 2;
		3290
		3291	return size;
		3292	}
		3293
		3294	struct kgem_bo kgem_create_2d(struct kgem kgem,
		3295	int width,
		3296	int height,
		3297	int bpp,
		3298	int tiling,
		3299	uint32_t flags)
		3300	{
		3301	struct list *cache;
		3302	struct kgem_bo *bo;
		3303	uint32_t pitch, tiled_height, size;
		3304	uint32_t handle;
		3305	int i, bucket, retry;
		3306	bool exact = flags & (CREATE_EXACT \| CREATE_SCANOUT);
		3307
		3308	if (tiling < 0)
		3309	exact = true, tiling = -tiling;
		3310
		3311
		3312	DBG(("%s(%dx%d, bpp=%d, tiling=%d, exact=%d, inactive=%d, cpu-mapping=%d, gtt-mapping=%d, scanout?=%d, prime?=%d, temp?=%d)\n", __FUNCTION__,
		3313	width, height, bpp, tiling, exact,
		3314	!!(flags & CREATE_INACTIVE),
		3315	!!(flags & CREATE_CPU_MAP),
		3316	!!(flags & CREATE_GTT_MAP),
		3317	!!(flags & CREATE_SCANOUT),
		3318	!!(flags & CREATE_PRIME),
		3319	!!(flags & CREATE_TEMPORARY)));
		3320
		3321	size = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags,
		3322	width, height, bpp, tiling, &pitch);
		3323	assert(size && size <= kgem->max_object_size);
		3324	size /= PAGE_SIZE;
		3325	bucket = cache_bucket(size);
		3326
		3327	if (flags & CREATE_SCANOUT) {
		3328	struct kgem_bo *last = NULL;
		3329
		3330	list_for_each_entry_reverse(bo, &kgem->scanout, list) {
		3331	assert(bo->scanout);
		3332	assert(bo->delta);
		3333	assert(!bo->flush);
		3334	assert_tiling(kgem, bo);
		3335
		3336	if (size > num_pages(bo) \|\| num_pages(bo) > 2*size)
		3337	continue;
		3338
		3339	if (bo->tiling != tiling \|\|
		3340	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3341	if (!gem_set_tiling(kgem->fd, bo->handle,
		3342	tiling, pitch))
		3343	continue;
		3344
		3345	bo->tiling = tiling;
		3346	bo->pitch = pitch;
		3347	}
		3348
		3349	if (flags & CREATE_INACTIVE && bo->rq) {
		3350	last = bo;
		3351	continue;
		3352	}
		3353
		3354	list_del(&bo->list);
		3355
		3356	bo->unique_id = kgem_get_unique_id(kgem);
		3357	DBG((" 1:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3358	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3359	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3360	assert_tiling(kgem, bo);
		3361	bo->refcnt = 1;
		3362	return bo;
		3363	}
		3364
		3365	if (last) {
		3366	list_del(&last->list);
		3367
		3368	last->unique_id = kgem_get_unique_id(kgem);
		3369	DBG((" 1:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3370	last->pitch, last->tiling, last->handle, last->unique_id));
		3371	assert(last->pitch*kgem_aligned_height(kgem, height, last->tiling) <= kgem_bo_size(last));
		3372	assert_tiling(kgem, last);
		3373	last->refcnt = 1;
		3374	return last;
		3375	}
		3376
		3377	bo = NULL; //__kgem_bo_create_as_display(kgem, size, tiling, pitch);
		3378	if (bo)
		3379	return bo;
		3380	}
		3381
		3382	if (bucket >= NUM_CACHE_BUCKETS) {
		3383	DBG(("%s: large bo num pages=%d, bucket=%d\n",
		3384	__FUNCTION__, size, bucket));
		3385
		3386	if (flags & CREATE_INACTIVE)
		3387	goto large_inactive;
		3388
		3389	tiled_height = kgem_aligned_height(kgem, height, tiling);
		3390
		3391	list_for_each_entry(bo, &kgem->large, list) {
		3392	assert(!bo->purged);
		3393	assert(!bo->scanout);
		3394	assert(bo->refcnt == 0);
		3395	assert(bo->reusable);
		3396	assert_tiling(kgem, bo);
		3397
		3398	if (kgem->gen < 040) {
		3399	if (bo->pitch < pitch) {
		3400	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3401	bo->tiling, tiling,
		3402	bo->pitch, pitch));
		3403	continue;
		3404	}
		3405
		3406	if (bo->pitch * tiled_height > bytes(bo))
		3407	continue;
		3408	} else {
		3409	if (num_pages(bo) < size)
		3410	continue;
		3411
		3412	if (bo->pitch != pitch \|\| bo->tiling != tiling) {
		3413	if (!gem_set_tiling(kgem->fd, bo->handle,
		3414	tiling, pitch))
		3415	continue;
		3416
		3417	bo->pitch = pitch;
		3418	bo->tiling = tiling;
		3419	}
		3420	}
		3421
		3422	kgem_bo_remove_from_active(kgem, bo);
		3423
		3424	bo->unique_id = kgem_get_unique_id(kgem);
		3425	bo->delta = 0;
		3426	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3427	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3428	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3429	assert_tiling(kgem, bo);
		3430	bo->refcnt = 1;
		3431	bo->flush = true;
		3432	return bo;
		3433	}
		3434
		3435	large_inactive:
		3436	__kgem_throttle_retire(kgem, flags);
		3437	list_for_each_entry(bo, &kgem->large_inactive, list) {
		3438	assert(bo->refcnt == 0);
		3439	assert(bo->reusable);
		3440	assert(!bo->scanout);
		3441	assert_tiling(kgem, bo);
		3442
		3443	if (size > num_pages(bo))
		3444	continue;
		3445
		3446	if (bo->tiling != tiling \|\|
		3447	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3448	if (!gem_set_tiling(kgem->fd, bo->handle,
		3449	tiling, pitch))
		3450	continue;
		3451
		3452	bo->tiling = tiling;
		3453	bo->pitch = pitch;
		3454	}
		3455
		3456	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3457	kgem_bo_free(kgem, bo);
		3458	break;
		3459	}
		3460
		3461	list_del(&bo->list);
		3462
		3463	assert(bo->domain != DOMAIN_GPU);
		3464	bo->unique_id = kgem_get_unique_id(kgem);
		3465	bo->pitch = pitch;
		3466	bo->delta = 0;
		3467	DBG((" 1:from large inactive: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3468	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3469	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3470	assert_tiling(kgem, bo);
		3471	bo->refcnt = 1;
		3472	return bo;
		3473	}
		3474
		3475	goto create;
		3476	}
		3477
		3478	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3479	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3480	if (kgem->has_llc && tiling == I915_TILING_NONE)
		3481	for_cpu = 1;
		3482	/* We presume that we will need to upload to this bo,
		3483	* and so would prefer to have an active VMA.
		3484	*/
		3485	cache = &kgem->vma[for_cpu].inactive[bucket];
		3486	do {
		3487	list_for_each_entry(bo, cache, vma) {
		3488	assert(bucket(bo) == bucket);
		3489	assert(bo->refcnt == 0);
		3490	assert(!bo->scanout);
		3491	assert(bo->map);
		3492	assert(IS_CPU_MAP(bo->map) == for_cpu);
		3493	assert(bo->rq == NULL);
		3494	assert(list_is_empty(&bo->request));
		3495	assert(bo->flush == false);
		3496	assert_tiling(kgem, bo);
		3497
		3498	if (size > num_pages(bo)) {
		3499	DBG(("inactive too small: %d < %d\n",
		3500	num_pages(bo), size));
		3501	continue;
		3502	}
		3503
		3504	if (bo->tiling != tiling \|\|
		3505	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3506	DBG(("inactive vma with wrong tiling: %d < %d\n",
		3507	bo->tiling, tiling));
		3508	continue;
		3509	}
		3510
		3511	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3512	kgem_bo_free(kgem, bo);
		3513	break;
		3514	}
		3515
		3516	assert(bo->tiling == tiling);
		3517	bo->pitch = pitch;
		3518	bo->delta = 0;
		3519	bo->unique_id = kgem_get_unique_id(kgem);
		3520	bo->domain = DOMAIN_NONE;
		3521
		3522	kgem_bo_remove_from_inactive(kgem, bo);
		3523
		3524	DBG((" from inactive vma: pitch=%d, tiling=%d: handle=%d, id=%d\n",
		3525	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3526	assert(bo->reusable);
		3527	assert(bo->domain != DOMAIN_GPU);
		3528	ASSERT_IDLE(kgem, bo->handle);
		3529	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3530	assert_tiling(kgem, bo);
		3531	bo->refcnt = 1;
		3532	return bo;
		3533	}
		3534	} while (!list_is_empty(cache) &&
		3535	__kgem_throttle_retire(kgem, flags));
		3536
		3537	if (flags & CREATE_CPU_MAP && !kgem->has_llc) {
		3538	if (list_is_empty(&kgem->active[bucket][tiling]) &&
		3539	list_is_empty(&kgem->inactive[bucket]))
		3540	flags &= ~CREATE_CACHED;
		3541
		3542	goto create;
		3543	}
		3544	}
		3545
		3546	if (flags & CREATE_INACTIVE)
		3547	goto skip_active_search;
		3548
		3549	/* Best active match */
		3550	retry = NUM_CACHE_BUCKETS - bucket;
		3551	if (retry > 3 && (flags & CREATE_TEMPORARY) == 0)
		3552	retry = 3;
		3553	search_again:
		3554	assert(bucket < NUM_CACHE_BUCKETS);
		3555	cache = &kgem->active[bucket][tiling];
		3556	if (tiling) {
		3557	tiled_height = kgem_aligned_height(kgem, height, tiling);
		3558	list_for_each_entry(bo, cache, list) {
		3559	assert(!bo->purged);
		3560	assert(bo->refcnt == 0);
		3561	assert(bucket(bo) == bucket);
		3562	assert(bo->reusable);
		3563	assert(bo->tiling == tiling);
		3564	assert(bo->flush == false);
		3565	assert(!bo->scanout);
		3566	assert_tiling(kgem, bo);
		3567
		3568	if (kgem->gen < 040) {
		3569	if (bo->pitch < pitch) {
		3570	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3571	bo->tiling, tiling,
		3572	bo->pitch, pitch));
		3573	continue;
		3574	}
		3575
		3576	if (bo->pitch * tiled_height > bytes(bo))
		3577	continue;
		3578	} else {
		3579	if (num_pages(bo) < size)
		3580	continue;
		3581
		3582	if (bo->pitch != pitch) {
		3583	if (!gem_set_tiling(kgem->fd,
		3584	bo->handle,
		3585	tiling, pitch))
		3586	continue;
		3587
		3588	bo->pitch = pitch;
		3589	}
		3590	}
		3591
		3592	kgem_bo_remove_from_active(kgem, bo);
		3593
		3594	bo->unique_id = kgem_get_unique_id(kgem);
		3595	bo->delta = 0;
		3596	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3597	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3598	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3599	assert_tiling(kgem, bo);
		3600	bo->refcnt = 1;
		3601	return bo;
		3602	}
		3603	} else {
		3604	list_for_each_entry(bo, cache, list) {
		3605	assert(bucket(bo) == bucket);
		3606	assert(!bo->purged);
		3607	assert(bo->refcnt == 0);
		3608	assert(bo->reusable);
		3609	assert(!bo->scanout);
		3610	assert(bo->tiling == tiling);
		3611	assert(bo->flush == false);
		3612	assert_tiling(kgem, bo);
		3613
		3614	if (num_pages(bo) < size)
		3615	continue;
		3616
		3617	kgem_bo_remove_from_active(kgem, bo);
		3618
		3619	bo->pitch = pitch;
		3620	bo->unique_id = kgem_get_unique_id(kgem);
		3621	bo->delta = 0;
		3622	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3623	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3624	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3625	assert_tiling(kgem, bo);
		3626	bo->refcnt = 1;
		3627	return bo;
		3628	}
		3629	}
		3630
		3631	if (--retry && exact) {
		3632	if (kgem->gen >= 040) {
		3633	for (i = I915_TILING_NONE; i <= I915_TILING_Y; i++) {
		3634	if (i == tiling)
		3635	continue;
		3636
		3637	cache = &kgem->active[bucket][i];
		3638	list_for_each_entry(bo, cache, list) {
		3639	assert(!bo->purged);
		3640	assert(bo->refcnt == 0);
		3641	assert(bo->reusable);
		3642	assert(!bo->scanout);
		3643	assert(bo->flush == false);
		3644	assert_tiling(kgem, bo);
		3645
		3646	if (num_pages(bo) < size)
		3647	continue;
		3648
		3649	if (!gem_set_tiling(kgem->fd,
		3650	bo->handle,
		3651	tiling, pitch))
		3652	continue;
		3653
		3654	kgem_bo_remove_from_active(kgem, bo);
		3655
		3656	bo->unique_id = kgem_get_unique_id(kgem);
		3657	bo->pitch = pitch;
		3658	bo->tiling = tiling;
		3659	bo->delta = 0;
		3660	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3661	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3662	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3663	assert_tiling(kgem, bo);
		3664	bo->refcnt = 1;
		3665	return bo;
		3666	}
		3667	}
		3668	}
		3669
		3670	bucket++;
		3671	goto search_again;
		3672	}
		3673
		3674	if (!exact) { /* allow an active near-miss? */
		3675	i = tiling;
		3676	while (--i >= 0) {
		3677	tiled_height = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags,
		3678	width, height, bpp, tiling, &pitch);
		3679	cache = active(kgem, tiled_height / PAGE_SIZE, i);
		3680	tiled_height = kgem_aligned_height(kgem, height, i);
		3681	list_for_each_entry(bo, cache, list) {
		3682	assert(!bo->purged);
		3683	assert(bo->refcnt == 0);
		3684	assert(bo->reusable);
		3685	assert(!bo->scanout);
		3686	assert(bo->flush == false);
		3687	assert_tiling(kgem, bo);
		3688
		3689	if (bo->tiling) {
		3690	if (bo->pitch < pitch) {
		3691	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3692	bo->tiling, tiling,
		3693	bo->pitch, pitch));
		3694	continue;
		3695	}
		3696	} else
		3697	bo->pitch = pitch;
		3698
		3699	if (bo->pitch * tiled_height > bytes(bo))
		3700	continue;
		3701
		3702	kgem_bo_remove_from_active(kgem, bo);
		3703
		3704	bo->unique_id = kgem_get_unique_id(kgem);
		3705	bo->delta = 0;
		3706	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3707	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3708	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3709	assert_tiling(kgem, bo);
		3710	bo->refcnt = 1;
		3711	return bo;
		3712	}
		3713	}
		3714	}
		3715
		3716	skip_active_search:
		3717	bucket = cache_bucket(size);
		3718	retry = NUM_CACHE_BUCKETS - bucket;
		3719	if (retry > 3)
		3720	retry = 3;
		3721	search_inactive:
		3722	/* Now just look for a close match and prefer any currently active */
		3723	assert(bucket < NUM_CACHE_BUCKETS);
		3724	cache = &kgem->inactive[bucket];
		3725	list_for_each_entry(bo, cache, list) {
		3726	assert(bucket(bo) == bucket);
		3727	assert(bo->reusable);
		3728	assert(!bo->scanout);
		3729	assert(bo->flush == false);
		3730	assert_tiling(kgem, bo);
		3731
		3732	if (size > num_pages(bo)) {
		3733	DBG(("inactive too small: %d < %d\n",
		3734	num_pages(bo), size));
		3735	continue;
		3736	}
		3737
		3738	if (bo->tiling != tiling \|\|
		3739	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3740	if (!gem_set_tiling(kgem->fd, bo->handle,
		3741	tiling, pitch))
		3742	continue;
		3743
		3744	if (bo->map)
		3745	kgem_bo_release_map(kgem, bo);
		3746	}
		3747
		3748	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3749	kgem_bo_free(kgem, bo);
		3750	break;
		3751	}
		3752
		3753	kgem_bo_remove_from_inactive(kgem, bo);
		3754
		3755	bo->pitch = pitch;
		3756	bo->tiling = tiling;
		3757
		3758	bo->delta = 0;
		3759	bo->unique_id = kgem_get_unique_id(kgem);
		3760	assert(bo->pitch);
		3761	DBG((" from inactive: pitch=%d, tiling=%d: handle=%d, id=%d\n",
		3762	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3763	assert(bo->refcnt == 0);
		3764	assert(bo->reusable);
		3765	assert((flags & CREATE_INACTIVE) == 0 \|\| bo->domain != DOMAIN_GPU);
		3766	ASSERT_MAYBE_IDLE(kgem, bo->handle, flags & CREATE_INACTIVE);
		3767	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3768	assert_tiling(kgem, bo);
		3769	bo->refcnt = 1;
		3770	return bo;
		3771	}
		3772
		3773	if (flags & CREATE_INACTIVE &&
		3774	!list_is_empty(&kgem->active[bucket][tiling]) &&
		3775	__kgem_throttle_retire(kgem, flags)) {
		3776	flags &= ~CREATE_INACTIVE;
		3777	goto search_inactive;
		3778	}
		3779
		3780	if (--retry) {
		3781	bucket++;
		3782	flags &= ~CREATE_INACTIVE;
		3783	goto search_inactive;
		3784	}
		3785
		3786	create:
		3787	if (flags & CREATE_CACHED)
		3788	return NULL;
		3789
		3790	if (bucket >= NUM_CACHE_BUCKETS)
		3791	size = ALIGN(size, 1024);
		3792	handle = gem_create(kgem->fd, size);
		3793	if (handle == 0)
		3794	return NULL;
		3795
		3796	bo = __kgem_bo_alloc(handle, size);
		3797	if (!bo) {
		3798	gem_close(kgem->fd, handle);
		3799	return NULL;
		3800	}
		3801
		3802	if (bucket >= NUM_CACHE_BUCKETS) {
		3803	DBG(("%s: marking large bo for automatic flushing\n",
		3804	__FUNCTION__));
		3805	bo->flush = true;
		3806	}
		3807
		3808	bo->unique_id = kgem_get_unique_id(kgem);
		3809	if (tiling == I915_TILING_NONE \|\|
		3810	gem_set_tiling(kgem->fd, handle, tiling, pitch)) {
		3811	bo->tiling = tiling;
		3812	bo->pitch = pitch;
		3813	} else {
		3814	if (flags & CREATE_EXACT) {
		3815	if (bo->pitch != pitch \|\| bo->tiling != tiling) {
		3816	kgem_bo_free(kgem, bo);
		3817	return NULL;
		3818	}
		3819	}
		3820	}
		3821
		3822	assert(bytes(bo) >= bo->pitch * kgem_aligned_height(kgem, height, bo->tiling));
		3823	assert_tiling(kgem, bo);
		3824
		3825	debug_alloc__bo(kgem, bo);
		3826
		3827	DBG((" new pitch=%d, tiling=%d, handle=%d, id=%d, num_pages=%d [%d], bucket=%d\n",
		3828	bo->pitch, bo->tiling, bo->handle, bo->unique_id,
		3829	size, num_pages(bo), bucket(bo)));
		3830	return bo;
		3831	}
		3832
		3833	#if 0
		3834	struct kgem_bo kgem_create_cpu_2d(struct kgem kgem,
		3835	int width,
		3836	int height,
		3837	int bpp,
		3838	uint32_t flags)
		3839	{
		3840	struct kgem_bo *bo;
		3841	int stride, size;
		3842
		3843	if (DBG_NO_CPU)
		3844	return NULL;
		3845
		3846	DBG(("%s(%dx%d, bpp=%d)\n", __FUNCTION__, width, height, bpp));
		3847
		3848	if (kgem->has_llc) {
		3849	bo = kgem_create_2d(kgem, width, height, bpp,
		3850	I915_TILING_NONE, flags);
		3851	if (bo == NULL)
		3852	return bo;
		3853
		3854	assert(bo->tiling == I915_TILING_NONE);
		3855	assert_tiling(kgem, bo);
		3856
		3857	if (kgem_bo_map__cpu(kgem, bo) == NULL) {
		3858	kgem_bo_destroy(kgem, bo);
		3859	return NULL;
		3860	}
		3861
		3862	return bo;
		3863	}
		3864
		3865	assert(width > 0 && height > 0);
		3866	stride = ALIGN(width, 2) * bpp >> 3;
		3867	stride = ALIGN(stride, 4);
		3868	size = stride * ALIGN(height, 2);
		3869	assert(size >= PAGE_SIZE);
		3870
		3871	DBG(("%s: %dx%d, %d bpp, stride=%d\n",
		3872	__FUNCTION__, width, height, bpp, stride));
		3873
		3874	bo = search_snoop_cache(kgem, NUM_PAGES(size), 0);
		3875	if (bo) {
		3876	assert(bo->tiling == I915_TILING_NONE);
		3877	assert_tiling(kgem, bo);
		3878	assert(bo->snoop);
		3879	bo->refcnt = 1;
		3880	bo->pitch = stride;
		3881	bo->unique_id = kgem_get_unique_id(kgem);
		3882	return bo;
		3883	}
		3884
		3885	if (kgem->has_caching) {
		3886	bo = kgem_create_linear(kgem, size, flags);
		3887	if (bo == NULL)
		3888	return NULL;
		3889
		3890	assert(bo->tiling == I915_TILING_NONE);
		3891	assert_tiling(kgem, bo);
		3892
		3893	if (!gem_set_caching(kgem->fd, bo->handle, SNOOPED)) {
		3894	kgem_bo_destroy(kgem, bo);
		3895	return NULL;
		3896	}
		3897	bo->snoop = true;
		3898
		3899	if (kgem_bo_map__cpu(kgem, bo) == NULL) {
		3900	kgem_bo_destroy(kgem, bo);
		3901	return NULL;
		3902	}
		3903
		3904	bo->pitch = stride;
		3905	bo->unique_id = kgem_get_unique_id(kgem);
		3906	return bo;
		3907	}
		3908
		3909	if (kgem->has_userptr) {
		3910	void *ptr;
		3911
		3912	/* XXX */
		3913	//if (posix_memalign(&ptr, 64, ALIGN(size, 64)))
		3914	if (posix_memalign(&ptr, PAGE_SIZE, ALIGN(size, PAGE_SIZE)))
		3915	return NULL;
		3916
		3917	bo = kgem_create_map(kgem, ptr, size, false);
		3918	if (bo == NULL) {
		3919	free(ptr);
		3920	return NULL;
		3921	}
		3922
		3923	bo->pitch = stride;
		3924	bo->unique_id = kgem_get_unique_id(kgem);
		3925	return bo;
		3926	}
		3927
		3928	return NULL;
		3929	}
		3930	#endif
		3931
		3932	void _kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		3933	{
		3934	DBG(("%s: handle=%d, proxy? %d\n",
		3935	__FUNCTION__, bo->handle, bo->proxy != NULL));
		3936
		3937	if (bo->proxy) {
		3938	_list_del(&bo->vma);
		3939	_list_del(&bo->request);
		3940	if (bo->io && bo->exec == NULL)
		3941	_kgem_bo_delete_buffer(kgem, bo);
		3942	kgem_bo_unref(kgem, bo->proxy);
		3943	kgem_bo_binding_free(kgem, bo);
		3944	free(bo);
		3945	return;
		3946	}
		3947
		3948	__kgem_bo_destroy(kgem, bo);
		3949	}
		3950
		3951	static void __kgem_flush(struct kgem kgem, struct kgem_bo bo)
		3952	{
		3953	assert(bo->rq);
		3954	assert(bo->exec == NULL);
		3955	assert(bo->needs_flush);
		3956
		3957	/* The kernel will emit a flush and update its own flushing lists. */
		3958	if (!__kgem_busy(kgem, bo->handle))
		3959	__kgem_bo_clear_busy(bo);
		3960
		3961	DBG(("%s: handle=%d, busy?=%d\n",
		3962	__FUNCTION__, bo->handle, bo->rq != NULL));
		3963	}
		3964
		3965	void kgem_scanout_flush(struct kgem kgem, struct kgem_bo bo)
		3966	{
		3967	kgem_bo_submit(kgem, bo);
		3968	if (!bo->needs_flush)
		3969	return;
		3970
		3971	/* If the kernel fails to emit the flush, then it will be forced when
		3972	* we assume direct access. And as the usual failure is EIO, we do
		3973	* not actually care.
		3974	*/
		3975	assert(bo->exec == NULL);
		3976	if (bo->rq)
		3977	__kgem_flush(kgem, bo);
		3978
		3979	/* Whatever actually happens, we can regard the GTT write domain
		3980	* as being flushed.
		3981	*/
		3982	bo->gtt_dirty = false;
		3983	bo->needs_flush = false;
		3984	bo->domain = DOMAIN_NONE;
		3985	}
		3986
		3987	inline static bool needs_semaphore(struct kgem kgem, struct kgem_bo bo)
		3988	{
		3989	return kgem->nreloc && bo->rq && RQ_RING(bo->rq) != kgem->ring;
		3990	}
		3991
		3992	bool kgem_check_bo(struct kgem *kgem, ...)
		3993	{
		3994	va_list ap;
		3995	struct kgem_bo *bo;
		3996	int num_exec = 0;
		3997	int num_pages = 0;
		3998	bool flush = false;
		3999
		4000	va_start(ap, kgem);
		4001	while ((bo = va_arg(ap, struct kgem_bo *))) {
		4002	while (bo->proxy)
		4003	bo = bo->proxy;
		4004	if (bo->exec)
		4005	continue;
		4006
		4007	if (needs_semaphore(kgem, bo))
		4008	return false;
		4009
		4010	num_pages += num_pages(bo);
		4011	num_exec++;
		4012
		4013	flush \|= bo->flush;
		4014	}
		4015	va_end(ap);
		4016
		4017	DBG(("%s: num_pages=+%d, num_exec=+%d\n",
		4018	__FUNCTION__, num_pages, num_exec));
		4019
		4020	if (!num_pages)
		4021	return true;
		4022
		4023	if (kgem_flush(kgem, flush))
		4024	return false;
		4025
		4026	if (kgem->aperture > kgem->aperture_low &&
		4027	kgem_ring_is_idle(kgem, kgem->ring)) {
		4028	DBG(("%s: current aperture usage (%d) is greater than low water mark (%d)\n",
		4029	__FUNCTION__, kgem->aperture, kgem->aperture_low));
		4030	return false;
		4031	}
		4032
		4033	if (num_pages + kgem->aperture > kgem->aperture_high) {
		4034	DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n",
		4035	__FUNCTION__, num_pages + kgem->aperture, kgem->aperture_high));
		4036	return false;
		4037	}
		4038
		4039	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) {
		4040	DBG(("%s: out of exec slots (%d + %d / %d)\n", __FUNCTION__,
		4041	kgem->nexec, num_exec, KGEM_EXEC_SIZE(kgem)));
		4042	return false;
		4043	}
		4044
		4045	return true;
		4046	}
		4047
		4048
		4049
		4050
		4051
		4052
		4053
		4054
		4055
		4056
		4057
		4058
		4059
		4060
		4061
		4062
		4063
		4064
		4065
		4066
		4067
		4068
		4069
		4070
		4071
		4072
		4073
		4074
		4075
		4076
		4077	uint32_t kgem_add_reloc(struct kgem *kgem,
		4078	uint32_t pos,
		4079	struct kgem_bo *bo,
		4080	uint32_t read_write_domain,
		4081	uint32_t delta)
		4082	{
		4083	int index;
		4084
		4085	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
		4086	__FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
		4087
		4088	assert((read_write_domain & 0x7fff) == 0 \|\| bo != NULL);
		4089
		4090	if( bo != NULL && bo->handle == -2)
		4091	{
		4092	if (bo->exec == NULL)
		4093	kgem_add_bo(kgem, bo);
		4094
		4095	if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
		4096	__kgem_bo_mark_dirty(bo);
		4097	}
		4098	return 0;
		4099	};
		4100
		4101	index = kgem->nreloc++;
		4102	assert(index < ARRAY_SIZE(kgem->reloc));
		4103	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
		4104	if (bo) {
		4105	assert(bo->refcnt);
		4106	while (bo->proxy) {
		4107	DBG(("%s: adding proxy [delta=%d] for handle=%d\n",
		4108	__FUNCTION__, bo->delta, bo->handle));
		4109	delta += bo->delta;
		4110	assert(bo->handle == bo->proxy->handle);
		4111	/* need to release the cache upon batch submit */
		4112	if (bo->exec == NULL) {
		4113	list_move_tail(&bo->request,
		4114	&kgem->next_request->buffers);
		4115	bo->rq = MAKE_REQUEST(kgem->next_request,
		4116	kgem->ring);
		4117	bo->exec = &_kgem_dummy_exec;
		4118	}
		4119
		4120	if (read_write_domain & 0x7fff && !bo->gpu_dirty)
		4121	__kgem_bo_mark_dirty(bo);
		4122
		4123	bo = bo->proxy;
		4124	assert(bo->refcnt);
		4125	}
		4126	assert(bo->refcnt);
		4127
		4128	if (bo->exec == NULL)
		4129	kgem_add_bo(kgem, bo);
		4130	assert(bo->rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		4131	assert(RQ_RING(bo->rq) == kgem->ring);
		4132
		4133	if (kgem->gen < 040 && read_write_domain & KGEM_RELOC_FENCED) {
		4134	if (bo->tiling &&
		4135	(bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		4136	assert(kgem->nfence < kgem->fence_max);
		4137	kgem->aperture_fenced +=
		4138	kgem_bo_fenced_size(kgem, bo);
		4139	kgem->nfence++;
		4140	}
		4141	bo->exec->flags \|= EXEC_OBJECT_NEEDS_FENCE;
		4142	}
		4143
		4144	kgem->reloc[index].delta = delta;
		4145	kgem->reloc[index].target_handle = bo->target_handle;
		4146	kgem->reloc[index].presumed_offset = bo->presumed_offset;
		4147
		4148	if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
		4149	assert(!bo->snoop \|\| kgem->can_blt_cpu);
		4150	__kgem_bo_mark_dirty(bo);
		4151	}
		4152
		4153	delta += bo->presumed_offset;
		4154	} else {
		4155	kgem->reloc[index].delta = delta;
		4156	kgem->reloc[index].target_handle = ~0U;
		4157	kgem->reloc[index].presumed_offset = 0;
		4158	if (kgem->nreloc__self < 256)
		4159	kgem->reloc__self[kgem->nreloc__self++] = index;
		4160	}
		4161	kgem->reloc[index].read_domains = read_write_domain >> 16;
		4162	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
		4163
		4164	return delta;
		4165	}
		4166
		4167	static void kgem_trim_vma_cache(struct kgem *kgem, int type, int bucket)
		4168	{
		4169	int i, j;
		4170
		4171	DBG(("%s: type=%d, count=%d (bucket: %d)\n",
		4172	__FUNCTION__, type, kgem->vma[type].count, bucket));
		4173	if (kgem->vma[type].count <= 0)
		4174	return;
		4175
		4176	if (kgem->need_purge)
		4177	kgem_purge_cache(kgem);
		4178
		4179	/* vma are limited on a per-process basis to around 64k.
		4180	* This includes all malloc arenas as well as other file
		4181	* mappings. In order to be fair and not hog the cache,
		4182	* and more importantly not to exhaust that limit and to
		4183	* start failing mappings, we keep our own number of open
		4184	* vma to within a conservative value.
		4185	*/
		4186	i = 0;
		4187	while (kgem->vma[type].count > 0) {
		4188	struct kgem_bo *bo = NULL;
		4189
		4190	for (j = 0;
		4191	bo == NULL && j < ARRAY_SIZE(kgem->vma[type].inactive);
		4192	j++) {
		4193	struct list *head = &kgem->vma[type].inactive[i++%ARRAY_SIZE(kgem->vma[type].inactive)];
		4194	if (!list_is_empty(head))
		4195	bo = list_last_entry(head, struct kgem_bo, vma);
		4196	}
		4197	if (bo == NULL)
		4198	break;
		4199
		4200	DBG(("%s: discarding inactive %s vma cache for %d\n",
		4201	__FUNCTION__,
		4202	IS_CPU_MAP(bo->map) ? "CPU" : "GTT", bo->handle));
		4203	assert(IS_CPU_MAP(bo->map) == type);
		4204	assert(bo->map);
		4205	assert(bo->rq == NULL);
		4206
		4207	VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo)));
		4208	// munmap(MAP(bo->map), bytes(bo));
		4209	bo->map = NULL;
		4210	list_del(&bo->vma);
		4211	kgem->vma[type].count--;
		4212
		4213	if (!bo->purged && !kgem_bo_set_purgeable(kgem, bo)) {
		4214	DBG(("%s: freeing unpurgeable old mapping\n",
		4215	__FUNCTION__));
		4216	kgem_bo_free(kgem, bo);
		4217	}
		4218	}
		4219	}
		4220
		4221	void kgem_bo_map__async(struct kgem kgem, struct kgem_bo *bo)
		4222	{
		4223	void *ptr;
		4224
		4225	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		4226	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		4227
		4228	assert(bo->proxy == NULL);
		4229	assert(list_is_empty(&bo->list));
		4230	assert(!IS_USER_MAP(bo->map));
		4231	assert_tiling(kgem, bo);
		4232
		4233	if (bo->tiling == I915_TILING_NONE && !bo->scanout && kgem->has_llc) {
		4234	DBG(("%s: converting request for GTT map into CPU map\n",
		4235	__FUNCTION__));
		4236	return kgem_bo_map__cpu(kgem, bo);
		4237	}
		4238
		4239	if (IS_CPU_MAP(bo->map))
		4240	kgem_bo_release_map(kgem, bo);
		4241
		4242	ptr = bo->map;
		4243	if (ptr == NULL) {
		4244	assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2);
		4245
		4246	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4247
		4248	ptr = __kgem_bo_map__gtt(kgem, bo);
		4249	if (ptr == NULL)
		4250	return NULL;
		4251
		4252	/* Cache this mapping to avoid the overhead of an
		4253	* excruciatingly slow GTT pagefault. This is more an
		4254	* issue with compositing managers which need to frequently
		4255	* flush CPU damage to their GPU bo.
		4256	*/
		4257	bo->map = ptr;
		4258	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4259	}
		4260
		4261	return ptr;
		4262	}
		4263
		4264	void kgem_bo_map(struct kgem kgem, struct kgem_bo *bo)
		4265	{
		4266	void *ptr;
		4267
		4268	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		4269	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		4270
		4271	assert(bo->proxy == NULL);
		4272	assert(list_is_empty(&bo->list));
		4273	assert(!IS_USER_MAP(bo->map));
		4274	assert(bo->exec == NULL);
		4275	assert_tiling(kgem, bo);
		4276
		4277	if (bo->tiling == I915_TILING_NONE && !bo->scanout &&
		4278	(kgem->has_llc \|\| bo->domain == DOMAIN_CPU)) {
		4279	DBG(("%s: converting request for GTT map into CPU map\n",
		4280	__FUNCTION__));
		4281	ptr = kgem_bo_map__cpu(kgem, bo);
		4282	if (ptr)
		4283	kgem_bo_sync__cpu(kgem, bo);
		4284	return ptr;
		4285	}
		4286
		4287	if (IS_CPU_MAP(bo->map))
		4288	kgem_bo_release_map(kgem, bo);
		4289
		4290	ptr = bo->map;
		4291	if (ptr == NULL) {
		4292	assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2);
		4293	assert(kgem->gen != 021 \|\| bo->tiling != I915_TILING_Y);
		4294
		4295	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4296
		4297	ptr = __kgem_bo_map__gtt(kgem, bo);
		4298	if (ptr == NULL)
		4299	return NULL;
		4300
		4301	/* Cache this mapping to avoid the overhead of an
		4302	* excruciatingly slow GTT pagefault. This is more an
		4303	* issue with compositing managers which need to frequently
		4304	* flush CPU damage to their GPU bo.
		4305	*/
		4306	bo->map = ptr;
		4307	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4308	}
		4309
		4310	if (bo->domain != DOMAIN_GTT \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_GTT)) {
		4311	struct drm_i915_gem_set_domain set_domain;
		4312
		4313	DBG(("%s: sync: needs_flush? %d, domain? %d, busy? %d\n", __FUNCTION__,
		4314	bo->needs_flush, bo->domain, __kgem_busy(kgem, bo->handle)));
		4315
		4316	/* XXX use PROT_READ to avoid the write flush? */
		4317
		4318	VG_CLEAR(set_domain);
		4319	set_domain.handle = bo->handle;
		4320	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		4321	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		4322	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4323	kgem_bo_retire(kgem, bo);
		4324	bo->domain = DOMAIN_GTT;
		4325	bo->gtt_dirty = true;
		4326	}
		4327	}
		4328
		4329	return ptr;
		4330	}
		4331
		4332	void kgem_bo_map__gtt(struct kgem kgem, struct kgem_bo *bo)
		4333	{
		4334	void *ptr;
		4335
		4336	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		4337	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		4338
		4339	assert(bo->exec == NULL);
		4340	assert(list_is_empty(&bo->list));
		4341	assert(!IS_USER_MAP(bo->map));
		4342	assert_tiling(kgem, bo);
		4343
		4344	if (IS_CPU_MAP(bo->map))
		4345	kgem_bo_release_map(kgem, bo);
		4346
		4347	ptr = bo->map;
		4348	if (ptr == NULL) {
		4349	assert(bytes(bo) <= kgem->aperture_mappable / 4);
		4350
		4351	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4352
		4353	ptr = __kgem_bo_map__gtt(kgem, bo);
		4354	if (ptr == NULL)
		4355	return NULL;
		4356
		4357	/* Cache this mapping to avoid the overhead of an
		4358	* excruciatingly slow GTT pagefault. This is more an
		4359	* issue with compositing managers which need to frequently
		4360	* flush CPU damage to their GPU bo.
		4361	*/
		4362	bo->map = ptr;
		4363	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4364	}
		4365
		4366	return ptr;
		4367	}
		4368
		4369	void kgem_bo_map__debug(struct kgem kgem, struct kgem_bo *bo)
		4370	{
		4371	if (bo->map)
		4372	return MAP(bo->map);
		4373
		4374	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4375	return bo->map = __kgem_bo_map__gtt(kgem, bo);
		4376	}
		4377
		4378	void kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		4379	{
		4380	struct drm_i915_gem_mmap mmap_arg;
		4381
		4382	DBG(("%s(handle=%d, size=%d, mapped? %d)\n",
		4383	__FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map)));
		4384	assert(!bo->purged);
		4385	assert(list_is_empty(&bo->list));
		4386	assert(bo->proxy == NULL);
		4387
		4388	if (IS_CPU_MAP(bo->map))
		4389	return MAP(bo->map);
		4390
		4391	if (bo->map)
		4392	kgem_bo_release_map(kgem, bo);
		4393
		4394	kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));
		4395
		4396	retry:
		4397	VG_CLEAR(mmap_arg);
		4398	mmap_arg.handle = bo->handle;
		4399	mmap_arg.offset = 0;
		4400	mmap_arg.size = bytes(bo);
		4401	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
		4402
		4403	if (__kgem_throttle_retire(kgem, 0))
		4404	goto retry;
		4405
		4406	if (kgem->need_expire) {
		4407	kgem_cleanup_cache(kgem);
		4408	goto retry;
		4409	}
		4410
		4411	ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain\n",
		4412	__FUNCTION__, bo->handle, bytes(bo));
		4413	return NULL;
		4414	}
		4415
		4416	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
		4417
		4418	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
		4419	bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr);
		4420	return (void *)(uintptr_t)mmap_arg.addr_ptr;
		4421	}
		4422
		4423	void __kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		4424	{
		4425	struct drm_i915_gem_mmap mmap_arg;
		4426
		4427	DBG(("%s(handle=%d, size=%d, mapped? %d)\n",
		4428	__FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map)));
		4429	assert(bo->refcnt);
		4430	assert(!bo->purged);
		4431	assert(list_is_empty(&bo->list));
		4432	assert(bo->proxy == NULL);
		4433
		4434	if (IS_CPU_MAP(bo->map))
		4435	return MAP(bo->map);
		4436
		4437	retry:
		4438	VG_CLEAR(mmap_arg);
		4439	mmap_arg.handle = bo->handle;
		4440	mmap_arg.offset = 0;
		4441	mmap_arg.size = bytes(bo);
		4442	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
		4443	int err = errno;
		4444
		4445	assert(err != EINVAL);
		4446
		4447	if (__kgem_throttle_retire(kgem, 0))
		4448	goto retry;
		4449
		4450	if (kgem->need_expire) {
		4451	kgem_cleanup_cache(kgem);
		4452	goto retry;
		4453	}
		4454
		4455	ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n",
		4456	__FUNCTION__, bo->handle, bytes(bo), err);
		4457	return NULL;
		4458	}
		4459
		4460	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
		4461	if (bo->map && bo->domain == DOMAIN_CPU) {
		4462	DBG(("%s: discarding GTT vma for %d\n", __FUNCTION__, bo->handle));
		4463	kgem_bo_release_map(kgem, bo);
		4464	}
		4465	if (bo->map == NULL) {
		4466	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
		4467	bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr);
		4468	}
		4469	return (void *)(uintptr_t)mmap_arg.addr_ptr;
		4470	}
		4471	void kgem_bo_sync__cpu(struct kgem kgem, struct kgem_bo bo)
		4472	{
		4473	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		4474	assert(!bo->scanout);
		4475	kgem_bo_submit(kgem, bo);
		4476
		4477	/* SHM pixmaps use proxies for subpage offsets */
		4478	assert(!bo->purged);
		4479	while (bo->proxy)
		4480	bo = bo->proxy;
		4481	assert(!bo->purged);
		4482
		4483	if (bo->domain != DOMAIN_CPU \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_CPU)) {
		4484	struct drm_i915_gem_set_domain set_domain;
		4485
		4486	DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
		4487	__FUNCTION__, bo->handle,
		4488	bo->needs_flush, bo->domain,
		4489	__kgem_busy(kgem, bo->handle)));
		4490
		4491	VG_CLEAR(set_domain);
		4492	set_domain.handle = bo->handle;
		4493	set_domain.read_domains = I915_GEM_DOMAIN_CPU;
		4494	set_domain.write_domain = I915_GEM_DOMAIN_CPU;
		4495
		4496	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4497	kgem_bo_retire(kgem, bo);
		4498	bo->domain = DOMAIN_CPU;
		4499	}
		4500	}
		4501	}
		4502
		4503	void kgem_clear_dirty(struct kgem *kgem)
		4504	{
		4505	struct list * const buffers = &kgem->next_request->buffers;
		4506	struct kgem_bo *bo;
		4507
		4508	list_for_each_entry(bo, buffers, request) {
		4509	if (!bo->gpu_dirty)
		4510	break;
		4511
		4512	bo->gpu_dirty = false;
		4513	}
		4514	}
		4515
		4516	struct kgem_bo kgem_create_proxy(struct kgem kgem,
		4517	struct kgem_bo *target,
		4518	int offset, int length)
		4519	{
		4520	struct kgem_bo *bo;
		4521
		4522	DBG(("%s: target handle=%d [proxy? %d], offset=%d, length=%d, io=%d\n",
		4523	__FUNCTION__, target->handle, target->proxy ? target->proxy->delta : -1,
		4524	offset, length, target->io));
		4525
		4526	bo = __kgem_bo_alloc(target->handle, length);
		4527	if (bo == NULL)
		4528	return NULL;
		4529
		4530	bo->unique_id = kgem_get_unique_id(kgem);
		4531	bo->reusable = false;
		4532	bo->size.bytes = length;
		4533
		4534	bo->io = target->io && target->proxy == NULL;
		4535	bo->gpu_dirty = target->gpu_dirty;
		4536	bo->tiling = target->tiling;
		4537	bo->pitch = target->pitch;
		4538	bo->flush = target->flush;
		4539	bo->snoop = target->snoop;
		4540
		4541	assert(!bo->scanout);
		4542	bo->proxy = kgem_bo_reference(target);
		4543	bo->delta = offset;
		4544
		4545	if (target->exec) {
		4546	list_move_tail(&bo->request, &kgem->next_request->buffers);
		4547	bo->exec = &_kgem_dummy_exec;
		4548	}
		4549	bo->rq = target->rq;
		4550
		4551	return bo;
		4552	}
		4553
		4554	#if 0
		4555	static struct kgem_buffer *
		4556	buffer_alloc(void)
		4557	{
		4558	struct kgem_buffer *bo;
		4559
		4560	bo = malloc(sizeof(*bo));
		4561	if (bo == NULL)
		4562	return NULL;
		4563
		4564	bo->mem = NULL;
		4565	bo->need_io = false;
		4566	bo->mmapped = true;
		4567
		4568	return bo;
		4569	}
		4570
		4571	static struct kgem_buffer *
		4572	buffer_alloc_with_data(int num_pages)
		4573	{
		4574	struct kgem_buffer *bo;
		4575
		4576	bo = malloc(sizeof(bo) + 2UPLOAD_ALIGNMENT + num_pages * PAGE_SIZE);
		4577	if (bo == NULL)
		4578	return NULL;
		4579
		4580	bo->mem = (void )ALIGN((uintptr_t)bo + sizeof(bo), UPLOAD_ALIGNMENT);
		4581	bo->mmapped = false;
		4582	return bo;
		4583	}
		4584
		4585	static inline bool
		4586	use_snoopable_buffer(struct kgem *kgem, uint32_t flags)
		4587	{
		4588	if ((flags & KGEM_BUFFER_WRITE) == 0)
		4589	return kgem->gen >= 030;
		4590
		4591	return true;
		4592	}
		4593
		4594	static void
		4595	init_buffer_from_bo(struct kgem_buffer bo, struct kgem_bo old)
		4596	{
		4597	DBG(("%s: reusing handle=%d for buffer\n",
		4598	__FUNCTION__, old->handle));
		4599
		4600	assert(old->proxy == NULL);
		4601
		4602	memcpy(&bo->base, old, sizeof(*old));
		4603	if (old->rq)
		4604	list_replace(&old->request, &bo->base.request);
		4605	else
		4606	list_init(&bo->base.request);
		4607	list_replace(&old->vma, &bo->base.vma);
		4608	list_init(&bo->base.list);
		4609	free(old);
		4610
		4611	assert(bo->base.tiling == I915_TILING_NONE);
		4612
		4613	bo->base.refcnt = 1;
		4614	}
		4615
		4616	static struct kgem_buffer *
		4617	search_snoopable_buffer(struct kgem *kgem, unsigned alloc)
		4618	{
		4619	struct kgem_buffer *bo;
		4620	struct kgem_bo *old;
		4621
		4622	old = search_snoop_cache(kgem, alloc, 0);
		4623	if (old) {
		4624	if (!old->io) {
		4625	bo = buffer_alloc();
		4626	if (bo == NULL)
		4627	return NULL;
		4628
		4629	init_buffer_from_bo(bo, old);
		4630	} else {
		4631	bo = (struct kgem_buffer *)old;
		4632	bo->base.refcnt = 1;
		4633	}
		4634
		4635	DBG(("%s: created CPU handle=%d for buffer, size %d\n",
		4636	__FUNCTION__, bo->base.handle, num_pages(&bo->base)));
		4637
		4638	assert(bo->base.snoop);
		4639	assert(bo->base.tiling == I915_TILING_NONE);
		4640	assert(num_pages(&bo->base) >= alloc);
		4641	assert(bo->mmapped == true);
		4642	assert(bo->need_io == false);
		4643
		4644	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4645	if (bo->mem == NULL) {
		4646	bo->base.refcnt = 0;
		4647	kgem_bo_free(kgem, &bo->base);
		4648	bo = NULL;
		4649	}
		4650
		4651	return bo;
		4652	}
		4653
		4654	return NULL;
		4655	}
		4656
		4657	static struct kgem_buffer *
		4658	create_snoopable_buffer(struct kgem *kgem, unsigned alloc)
		4659	{
		4660	struct kgem_buffer *bo;
		4661	uint32_t handle;
		4662
		4663	if (kgem->has_llc) {
		4664	struct kgem_bo *old;
		4665
		4666	bo = buffer_alloc();
		4667	if (bo == NULL)
		4668	return NULL;
		4669
		4670	old = search_linear_cache(kgem, alloc,
		4671	CREATE_INACTIVE \| CREATE_CPU_MAP \| CREATE_EXACT);
		4672	if (old) {
		4673	init_buffer_from_bo(bo, old);
		4674	} else {
		4675	handle = gem_create(kgem->fd, alloc);
		4676	if (handle == 0) {
		4677	free(bo);
		4678	return NULL;
		4679	}
		4680
		4681	debug_alloc(kgem, alloc);
		4682	__kgem_bo_init(&bo->base, handle, alloc);
		4683	DBG(("%s: created CPU (LLC) handle=%d for buffer, size %d\n",
		4684	__FUNCTION__, bo->base.handle, alloc));
		4685	}
		4686
		4687	assert(bo->base.refcnt == 1);
		4688	assert(bo->mmapped == true);
		4689	assert(bo->need_io == false);
		4690
		4691	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4692	if (bo->mem != NULL)
		4693	return bo;
		4694
		4695	bo->base.refcnt = 0; /* for valgrind */
		4696	kgem_bo_free(kgem, &bo->base);
		4697	}
		4698
		4699	if (kgem->has_caching) {
		4700	struct kgem_bo *old;
		4701
		4702	bo = buffer_alloc();
		4703	if (bo == NULL)
		4704	return NULL;
		4705
		4706	old = search_linear_cache(kgem, alloc,
		4707	CREATE_INACTIVE \| CREATE_CPU_MAP \| CREATE_EXACT);
		4708	if (old) {
		4709	init_buffer_from_bo(bo, old);
		4710	} else {
		4711	handle = gem_create(kgem->fd, alloc);
		4712	if (handle == 0) {
		4713	free(bo);
		4714	return NULL;
		4715	}
		4716
		4717	debug_alloc(kgem, alloc);
		4718	__kgem_bo_init(&bo->base, handle, alloc);
		4719	DBG(("%s: created CPU handle=%d for buffer, size %d\n",
		4720	__FUNCTION__, bo->base.handle, alloc));
		4721	}
		4722
		4723	assert(bo->base.refcnt == 1);
		4724	assert(bo->mmapped == true);
		4725	assert(bo->need_io == false);
		4726
		4727	if (!gem_set_caching(kgem->fd, bo->base.handle, SNOOPED))
		4728	goto free_caching;
		4729
		4730	bo->base.snoop = true;
		4731
		4732	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4733	if (bo->mem == NULL)
		4734	goto free_caching;
		4735
		4736	return bo;
		4737
		4738	free_caching:
		4739	bo->base.refcnt = 0; /* for valgrind */
		4740	kgem_bo_free(kgem, &bo->base);
		4741	}
		4742
		4743	if (kgem->has_userptr) {
		4744	bo = buffer_alloc();
		4745	if (bo == NULL)
		4746	return NULL;
		4747
		4748	//if (posix_memalign(&ptr, 64, ALIGN(size, 64)))
		4749	if (posix_memalign(&bo->mem, PAGE_SIZE, alloc * PAGE_SIZE)) {
		4750	free(bo);
		4751	return NULL;
		4752	}
		4753
		4754	handle = gem_userptr(kgem->fd, bo->mem, alloc * PAGE_SIZE, false);
		4755	if (handle == 0) {
		4756	free(bo->mem);
		4757	free(bo);
		4758	return NULL;
		4759	}
		4760
		4761	debug_alloc(kgem, alloc);
		4762	__kgem_bo_init(&bo->base, handle, alloc);
		4763	DBG(("%s: created snoop handle=%d for buffer\n",
		4764	__FUNCTION__, bo->base.handle));
		4765
		4766	assert(bo->mmapped == true);
		4767	assert(bo->need_io == false);
		4768
		4769	bo->base.refcnt = 1;
		4770	bo->base.snoop = true;
		4771	bo->base.map = MAKE_USER_MAP(bo->mem);
		4772
		4773	return bo;
		4774	}
		4775
		4776	return NULL;
		4777	}
		4778
		4779	struct kgem_bo kgem_create_buffer(struct kgem kgem,
		4780	uint32_t size, uint32_t flags,
		4781	void **ret)
		4782	{
		4783	struct kgem_buffer *bo;
		4784	unsigned offset, alloc;
		4785	struct kgem_bo *old;
		4786
		4787	DBG(("%s: size=%d, flags=%x [write?=%d, inplace?=%d, last?=%d]\n",
		4788	__FUNCTION__, size, flags,
		4789	!!(flags & KGEM_BUFFER_WRITE),
		4790	!!(flags & KGEM_BUFFER_INPLACE),
		4791	!!(flags & KGEM_BUFFER_LAST)));
		4792	assert(size);
		4793	/* we should never be asked to create anything TOO large */
		4794	assert(size <= kgem->max_object_size);
		4795
		4796	#if !DBG_NO_UPLOAD_CACHE
		4797	list_for_each_entry(bo, &kgem->batch_buffers, base.list) {
		4798	assert(bo->base.io);
		4799	assert(bo->base.refcnt >= 1);
		4800
		4801	/* We can reuse any write buffer which we can fit */
		4802	if (flags == KGEM_BUFFER_LAST &&
		4803	bo->write == KGEM_BUFFER_WRITE &&
		4804	bo->base.refcnt == 1 && !bo->mmapped &&
		4805	size <= bytes(&bo->base)) {
		4806	DBG(("%s: reusing write buffer for read of %d bytes? used=%d, total=%d\n",
		4807	__FUNCTION__, size, bo->used, bytes(&bo->base)));
		4808	gem_write(kgem->fd, bo->base.handle,
		4809	0, bo->used, bo->mem);
		4810	kgem_buffer_release(kgem, bo);
		4811	bo->need_io = 0;
		4812	bo->write = 0;
		4813	offset = 0;
		4814	bo->used = size;
		4815	goto done;
		4816	}
		4817
		4818	if (flags & KGEM_BUFFER_WRITE) {
		4819	if ((bo->write & KGEM_BUFFER_WRITE) == 0 \|\|
		4820	(((bo->write & ~flags) & KGEM_BUFFER_INPLACE) &&
		4821	!bo->base.snoop)) {
		4822	DBG(("%s: skip write %x buffer, need %x\n",
		4823	__FUNCTION__, bo->write, flags));
		4824	continue;
		4825	}
		4826	assert(bo->mmapped \|\| bo->need_io);
		4827	} else {
		4828	if (bo->write & KGEM_BUFFER_WRITE) {
		4829	DBG(("%s: skip write %x buffer, need %x\n",
		4830	__FUNCTION__, bo->write, flags));
		4831	continue;
		4832	}
		4833	}
		4834
		4835	if (bo->used + size <= bytes(&bo->base)) {
		4836	DBG(("%s: reusing buffer? used=%d + size=%d, total=%d\n",
		4837	__FUNCTION__, bo->used, size, bytes(&bo->base)));
		4838	offset = bo->used;
		4839	bo->used += size;
		4840	goto done;
		4841	}
		4842	}
		4843
		4844	if (flags & KGEM_BUFFER_WRITE) {
		4845	list_for_each_entry(bo, &kgem->active_buffers, base.list) {
		4846	assert(bo->base.io);
		4847	assert(bo->base.refcnt >= 1);
		4848	assert(bo->mmapped);
		4849	assert(!IS_CPU_MAP(bo->base.map) \|\| kgem->has_llc \|\| bo->base.snoop);
		4850
		4851	if (!kgem->has_llc && (bo->write & ~flags) & KGEM_BUFFER_INPLACE) {
		4852	DBG(("%s: skip write %x buffer, need %x\n",
		4853	__FUNCTION__, bo->write, flags));
		4854	continue;
		4855	}
		4856
		4857	if (bo->used + size <= bytes(&bo->base)) {
		4858	DBG(("%s: reusing buffer? used=%d + size=%d, total=%d\n",
		4859	__FUNCTION__, bo->used, size, bytes(&bo->base)));
		4860	offset = bo->used;
		4861	bo->used += size;
		4862	list_move(&bo->base.list, &kgem->batch_buffers);
		4863	goto done;
		4864	}
		4865	}
		4866	}
		4867	#endif
		4868
		4869	#if !DBG_NO_MAP_UPLOAD
		4870	/* Be a little more generous and hope to hold fewer mmappings */
		4871	alloc = ALIGN(2*size, kgem->buffer_size);
		4872	if (alloc > MAX_CACHE_SIZE)
		4873	alloc = ALIGN(size, kgem->buffer_size);
		4874	if (alloc > MAX_CACHE_SIZE)
		4875	alloc = PAGE_ALIGN(size);
		4876	assert(alloc);
		4877
		4878	if (alloc > kgem->aperture_mappable / 4)
		4879	flags &= ~KGEM_BUFFER_INPLACE;
		4880	alloc /= PAGE_SIZE;
		4881
		4882	if (kgem->has_llc &&
		4883	(flags & KGEM_BUFFER_WRITE_INPLACE) != KGEM_BUFFER_WRITE_INPLACE) {
		4884	bo = buffer_alloc();
		4885	if (bo == NULL)
		4886	goto skip_llc;
		4887
		4888	old = NULL;
		4889	if ((flags & KGEM_BUFFER_WRITE) == 0)
		4890	old = search_linear_cache(kgem, alloc, CREATE_CPU_MAP);
		4891	if (old == NULL)
		4892	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE \| CREATE_CPU_MAP);
		4893	if (old == NULL)
		4894	old = search_linear_cache(kgem, NUM_PAGES(size), CREATE_INACTIVE \| CREATE_CPU_MAP);
		4895	if (old) {
		4896	DBG(("%s: found LLC handle=%d for buffer\n",
		4897	__FUNCTION__, old->handle));
		4898
		4899	init_buffer_from_bo(bo, old);
		4900	} else {
		4901	uint32_t handle = gem_create(kgem->fd, alloc);
		4902	if (handle == 0) {
		4903	free(bo);
		4904	goto skip_llc;
		4905	}
		4906	__kgem_bo_init(&bo->base, handle, alloc);
		4907	DBG(("%s: created LLC handle=%d for buffer\n",
		4908	__FUNCTION__, bo->base.handle));
		4909
		4910	debug_alloc(kgem, alloc);
		4911	}
		4912
		4913	assert(bo->mmapped);
		4914	assert(!bo->need_io);
		4915
		4916	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4917	if (bo->mem) {
		4918	if (flags & KGEM_BUFFER_WRITE)
		4919	kgem_bo_sync__cpu(kgem, &bo->base);
		4920	flags &= ~KGEM_BUFFER_INPLACE;
		4921	goto init;
		4922	} else {
		4923	bo->base.refcnt = 0; /* for valgrind */
		4924	kgem_bo_free(kgem, &bo->base);
		4925	}
		4926	}
		4927	skip_llc:
		4928
		4929	if ((flags & KGEM_BUFFER_WRITE_INPLACE) == KGEM_BUFFER_WRITE_INPLACE) {
		4930	/* The issue with using a GTT upload buffer is that we may
		4931	* cause eviction-stalls in order to free up some GTT space.
		4932	* An is-mappable? ioctl could help us detect when we are
		4933	* about to block, or some per-page magic in the kernel.
		4934	*
		4935	* XXX This is especially noticeable on memory constrained
		4936	* devices like gen2 or with relatively slow gpu like i3.
		4937	*/
		4938	DBG(("%s: searching for an inactive GTT map for upload\n",
		4939	__FUNCTION__));
		4940	old = search_linear_cache(kgem, alloc,
		4941	CREATE_EXACT \| CREATE_INACTIVE \| CREATE_GTT_MAP);
		4942	#if HAVE_I915_GEM_BUFFER_INFO
		4943	if (old) {
		4944	struct drm_i915_gem_buffer_info info;
		4945
		4946	/* An example of such a non-blocking ioctl might work */
		4947
		4948	VG_CLEAR(info);
		4949	info.handle = handle;
		4950	if (drmIoctl(kgem->fd,
		4951	DRM_IOCTL_I915_GEM_BUFFER_INFO,
		4952	&fino) == 0) {
		4953	old->presumed_offset = info.addr;
		4954	if ((info.flags & I915_GEM_MAPPABLE) == 0) {
		4955	kgem_bo_move_to_inactive(kgem, old);
		4956	old = NULL;
		4957	}
		4958	}
		4959	}
		4960	#endif
		4961	if (old == NULL)
		4962	old = search_linear_cache(kgem, NUM_PAGES(size),
		4963	CREATE_EXACT \| CREATE_INACTIVE \| CREATE_GTT_MAP);
		4964	if (old == NULL) {
		4965	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
		4966	if (old && !__kgem_bo_is_mappable(kgem, old)) {
		4967	_kgem_bo_destroy(kgem, old);
		4968	old = NULL;
		4969	}
		4970	}
		4971	if (old) {
		4972	DBG(("%s: reusing handle=%d for buffer\n",
		4973	__FUNCTION__, old->handle));
		4974	assert(__kgem_bo_is_mappable(kgem, old));
		4975	assert(!old->snoop);
		4976	assert(old->rq == NULL);
		4977
		4978	bo = buffer_alloc();
		4979	if (bo == NULL)
		4980	return NULL;
		4981
		4982	init_buffer_from_bo(bo, old);
		4983	assert(num_pages(&bo->base) >= NUM_PAGES(size));
		4984
		4985	assert(bo->mmapped);
		4986	assert(bo->base.refcnt == 1);
		4987
		4988	bo->mem = kgem_bo_map(kgem, &bo->base);
		4989	if (bo->mem) {
		4990	if (IS_CPU_MAP(bo->base.map))
		4991	flags &= ~KGEM_BUFFER_INPLACE;
		4992	goto init;
		4993	} else {
		4994	bo->base.refcnt = 0;
		4995	kgem_bo_free(kgem, &bo->base);
		4996	}
		4997	}
		4998	}
		4999	#else
		5000	flags &= ~KGEM_BUFFER_INPLACE;
		5001	#endif
		5002	/* Be more parsimonious with pwrite/pread/cacheable buffers */
		5003	if ((flags & KGEM_BUFFER_INPLACE) == 0)
		5004	alloc = NUM_PAGES(size);
		5005
		5006	if (use_snoopable_buffer(kgem, flags)) {
		5007	bo = search_snoopable_buffer(kgem, alloc);
		5008	if (bo) {
		5009	if (flags & KGEM_BUFFER_WRITE)
		5010	kgem_bo_sync__cpu(kgem, &bo->base);
		5011	flags &= ~KGEM_BUFFER_INPLACE;
		5012	goto init;
		5013	}
		5014
		5015	if ((flags & KGEM_BUFFER_INPLACE) == 0) {
		5016	bo = create_snoopable_buffer(kgem, alloc);
		5017	if (bo)
		5018	goto init;
		5019	}
		5020	}
		5021
		5022	flags &= ~KGEM_BUFFER_INPLACE;
		5023
		5024	old = NULL;
		5025	if ((flags & KGEM_BUFFER_WRITE) == 0)
		5026	old = search_linear_cache(kgem, alloc, 0);
		5027	if (old == NULL)
		5028	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
		5029	if (old) {
		5030	DBG(("%s: reusing ordinary handle %d for io\n",
		5031	__FUNCTION__, old->handle));
		5032	bo = buffer_alloc_with_data(num_pages(old));
		5033	if (bo == NULL)
		5034	return NULL;
		5035
		5036	init_buffer_from_bo(bo, old);
		5037	bo->need_io = flags & KGEM_BUFFER_WRITE;
		5038	} else {
		5039	unsigned hint;
		5040
		5041	if (use_snoopable_buffer(kgem, flags)) {
		5042	bo = create_snoopable_buffer(kgem, alloc);
		5043	if (bo)
		5044	goto init;
		5045	}
		5046
		5047	bo = buffer_alloc();
		5048	if (bo == NULL)
		5049	return NULL;
		5050
		5051	hint = CREATE_INACTIVE;
		5052	if (flags & KGEM_BUFFER_WRITE)
		5053	hint \|= CREATE_CPU_MAP;
		5054	old = search_linear_cache(kgem, alloc, hint);
		5055	if (old) {
		5056	DBG(("%s: reusing handle=%d for buffer\n",
		5057	__FUNCTION__, old->handle));
		5058
		5059	init_buffer_from_bo(bo, old);
		5060	} else {
		5061	uint32_t handle = gem_create(kgem->fd, alloc);
		5062	if (handle == 0) {
		5063	free(bo);
		5064	return NULL;
		5065	}
		5066
		5067	DBG(("%s: created handle=%d for buffer\n",
		5068	__FUNCTION__, handle));
		5069
		5070	__kgem_bo_init(&bo->base, handle, alloc);
		5071	debug_alloc(kgem, alloc * PAGE_SIZE);
		5072	}
		5073
		5074	assert(bo->mmapped);
		5075	assert(!bo->need_io);
		5076	assert(bo->base.refcnt == 1);
		5077
		5078	if (flags & KGEM_BUFFER_WRITE) {
		5079	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		5080	if (bo->mem != NULL) {
		5081	kgem_bo_sync__cpu(kgem, &bo->base);
		5082	goto init;
		5083	}
		5084	}
		5085
		5086	DBG(("%s: failing back to new pwrite buffer\n", __FUNCTION__));
		5087	old = &bo->base;
		5088	bo = buffer_alloc_with_data(num_pages(old));
		5089	if (bo == NULL) {
		5090	old->refcnt= 0;
		5091	kgem_bo_free(kgem, old);
		5092	return NULL;
		5093	}
		5094
		5095	init_buffer_from_bo(bo, old);
		5096
		5097	assert(bo->mem);
		5098	assert(!bo->mmapped);
		5099	assert(bo->base.refcnt == 1);
		5100
		5101	bo->need_io = flags & KGEM_BUFFER_WRITE;
		5102	}
		5103	init:
		5104	bo->base.io = true;
		5105	assert(bo->base.refcnt == 1);
		5106	assert(num_pages(&bo->base) >= NUM_PAGES(size));
		5107	assert(!bo->need_io \|\| !bo->base.needs_flush);
		5108	assert(!bo->need_io \|\| bo->base.domain != DOMAIN_GPU);
		5109	assert(bo->mem);
		5110	assert(!bo->mmapped \|\| bo->base.map != NULL);
		5111
		5112	bo->used = size;
		5113	bo->write = flags & KGEM_BUFFER_WRITE_INPLACE;
		5114	offset = 0;
		5115
		5116	assert(list_is_empty(&bo->base.list));
		5117	list_add(&bo->base.list, &kgem->batch_buffers);
		5118
		5119	DBG(("%s(pages=%d [%d]) new handle=%d, used=%d, write=%d\n",
		5120	__FUNCTION__, num_pages(&bo->base), alloc, bo->base.handle, bo->used, bo->write));
		5121
		5122	done:
		5123	bo->used = ALIGN(bo->used, UPLOAD_ALIGNMENT);
		5124	assert(bo->mem);
		5125	ret = (char )bo->mem + offset;
		5126	return kgem_create_proxy(kgem, &bo->base, offset, size);
		5127	}
		5128
		5129	bool kgem_buffer_is_inplace(struct kgem_bo *_bo)
		5130	{
		5131	struct kgem_buffer bo = (struct kgem_buffer )_bo->proxy;
		5132	return bo->write & KGEM_BUFFER_WRITE_INPLACE;
		5133	}
		5134
		5135	struct kgem_bo kgem_create_buffer_2d(struct kgem kgem,
		5136	int width, int height, int bpp,
		5137	uint32_t flags,
		5138	void **ret)
		5139	{
		5140	struct kgem_bo *bo;
		5141	int stride;
		5142
		5143	assert(width > 0 && height > 0);
		5144	assert(ret != NULL);
		5145	stride = ALIGN(width, 2) * bpp >> 3;
		5146	stride = ALIGN(stride, 4);
		5147
		5148	DBG(("%s: %dx%d, %d bpp, stride=%d\n",
		5149	__FUNCTION__, width, height, bpp, stride));
		5150
		5151	bo = kgem_create_buffer(kgem, stride * ALIGN(height, 2), flags, ret);
		5152	if (bo == NULL) {
		5153	DBG(("%s: allocation failure for upload buffer\n",
		5154	__FUNCTION__));
		5155	return NULL;
		5156	}
		5157	assert(*ret != NULL);
		5158	assert(bo->proxy != NULL);
		5159
		5160	if (height & 1) {
		5161	struct kgem_buffer io = (struct kgem_buffer )bo->proxy;
		5162	int min;
		5163
		5164	assert(io->used);
		5165
		5166	/* Having padded this surface to ensure that accesses to
		5167	* the last pair of rows is valid, remove the padding so
		5168	* that it can be allocated to other pixmaps.
		5169	*/
		5170	min = bo->delta + height * stride;
		5171	min = ALIGN(min, UPLOAD_ALIGNMENT);
		5172	if (io->used != min) {
		5173	DBG(("%s: trimming buffer from %d to %d\n",
		5174	__FUNCTION__, io->used, min));
		5175	io->used = min;
		5176	}
		5177	bo->size.bytes -= stride;
		5178	}
		5179
		5180	bo->map = MAKE_CPU_MAP(*ret);
		5181	bo->pitch = stride;
		5182	bo->unique_id = kgem_get_unique_id(kgem);
		5183	return bo;
		5184	}
		5185
		5186	struct kgem_bo kgem_upload_source_image(struct kgem kgem,
		5187	const void *data,
		5188	const BoxRec *box,
		5189	int stride, int bpp)
		5190	{
		5191	int width = box->x2 - box->x1;
		5192	int height = box->y2 - box->y1;
		5193	struct kgem_bo *bo;
		5194	void *dst;
		5195
		5196	if (!kgem_can_create_2d(kgem, width, height, bpp))
		5197	return NULL;
		5198
		5199	DBG(("%s : (%d, %d), (%d, %d), stride=%d, bpp=%d\n",
		5200	__FUNCTION__, box->x1, box->y1, box->x2, box->y2, stride, bpp));
		5201
		5202	assert(data);
		5203	assert(width > 0);
		5204	assert(height > 0);
		5205	assert(stride);
		5206	assert(bpp);
		5207
		5208	bo = kgem_create_buffer_2d(kgem,
		5209	width, height, bpp,
		5210	KGEM_BUFFER_WRITE_INPLACE, &dst);
		5211	if (bo)
		5212	memcpy_blt(data, dst, bpp,
		5213	stride, bo->pitch,
		5214	box->x1, box->y1,
		5215	0, 0,
		5216	width, height);
		5217
		5218	return bo;
		5219	}
		5220
		5221	void kgem_proxy_bo_attach(struct kgem_bo *bo,
		5222	struct kgem_bo **ptr)
		5223	{
		5224	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		5225	assert(bo->map == NULL \|\| IS_CPU_MAP(bo->map));
		5226	assert(bo->proxy);
		5227	list_add(&bo->vma, &bo->proxy->vma);
		5228	bo->map = ptr;
		5229	*ptr = kgem_bo_reference(bo);
		5230	}
		5231
		5232	void kgem_buffer_read_sync(struct kgem kgem, struct kgem_bo _bo)
		5233	{
		5234	struct kgem_buffer *bo;
		5235	uint32_t offset = _bo->delta, length = _bo->size.bytes;
		5236
		5237	/* We expect the caller to have already submitted the batch */
		5238	assert(_bo->io);
		5239	assert(_bo->exec == NULL);
		5240	assert(_bo->rq == NULL);
		5241	assert(_bo->proxy);
		5242
		5243	_bo = _bo->proxy;
		5244	assert(_bo->proxy == NULL);
		5245	assert(_bo->exec == NULL);
		5246
		5247	bo = (struct kgem_buffer *)_bo;
		5248
		5249	DBG(("%s(offset=%d, length=%d, snooped=%d)\n", __FUNCTION__,
		5250	offset, length, bo->base.snoop));
		5251
		5252	if (bo->mmapped) {
		5253	struct drm_i915_gem_set_domain set_domain;
		5254
		5255	DBG(("%s: sync: needs_flush? %d, domain? %d, busy? %d\n",
		5256	__FUNCTION__,
		5257	bo->base.needs_flush,
		5258	bo->base.domain,
		5259	__kgem_busy(kgem, bo->base.handle)));
		5260
		5261	assert(!IS_CPU_MAP(bo->base.map) \|\| bo->base.snoop \|\| kgem->has_llc);
		5262
		5263	VG_CLEAR(set_domain);
		5264	set_domain.handle = bo->base.handle;
		5265	set_domain.write_domain = 0;
		5266	set_domain.read_domains =
		5267	IS_CPU_MAP(bo->base.map) ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT;
		5268
		5269	if (drmIoctl(kgem->fd,
		5270	DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain))
		5271	return;
		5272	} else {
		5273	if (gem_read(kgem->fd,
		5274	bo->base.handle, (char *)bo->mem+offset,
		5275	offset, length))
		5276	return;
		5277	}
		5278	kgem_bo_retire(kgem, &bo->base);
		5279	bo->base.domain = DOMAIN_NONE;
		5280	}
		5281	#endif
		5282
		5283	uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format)
		5284	{
		5285	struct kgem_bo_binding *b;
		5286
		5287	for (b = &bo->binding; b && b->offset; b = b->next)
		5288	if (format == b->format)
		5289	return b->offset;
		5290
		5291	return 0;
		5292	}
		5293
		5294	void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset)
		5295	{
		5296	struct kgem_bo_binding *b;
		5297
		5298	for (b = &bo->binding; b; b = b->next) {
		5299	if (b->offset)
		5300	continue;
		5301
		5302	b->offset = offset;
		5303	b->format = format;
		5304
		5305	if (b->next)
		5306	b->next->offset = 0;
		5307
		5308	return;
		5309	}
		5310
		5311	b = malloc(sizeof(*b));
		5312	if (b) {
		5313	b->next = bo->binding.next;
		5314	b->format = format;
		5315	b->offset = offset;
		5316	bo->binding.next = b;
		5317	}
		5318	}
		5319
		5320	int kgem_init_fb(struct kgem kgem, struct sna_fb fb)
		5321	{
		5322	struct kgem_bo *bo;
4397	Serge	5323	struct drm_gem_open open_arg;
		5324	struct drm_i915_gem_get_tiling get_tiling;
		5325
4304	Serge	5326	size_t size;
		5327	int ret;
		5328
		5329	ret = drmIoctl(kgem->fd, SRV_FBINFO, fb);
		5330	if( ret != 0 )
		5331	return 0;
		5332
4397	Serge	5333	open_arg.name = fb->name;
		5334	ret = drmIoctl(kgem->fd, DRM_IOCTL_GEM_OPEN, &open_arg);
		5335	if (ret != 0) {
		5336	printf("Couldn't reference %s handle 0x%08x\n",
		5337	fb->name, fb->name);
		5338	return NULL;
		5339	}
		5340	size = open_arg.size / PAGE_SIZE;
4304	Serge	5341
4397	Serge	5342	bo = __kgem_bo_alloc(open_arg.handle, size);
4304	Serge	5343	if (!bo) {
		5344	return 0;
		5345	}
		5346
4397	Serge	5347	get_tiling.handle = bo->handle;
		5348	ret = drmIoctl(kgem->fd,DRM_IOCTL_I915_GEM_GET_TILING,&get_tiling);
		5349	if (ret != 0) {
		5350	printf("%s: couldn't get tiling for handle %d\n", __FUNCTION__, bo->handle);
		5351	// drm_intel_gem_bo_unreference(&bo_gem->bo);
		5352	return 0;
		5353	}
		5354
4304	Serge	5355	bo->domain = DOMAIN_GTT;
		5356	bo->unique_id = kgem_get_unique_id(kgem);
		5357	bo->pitch = fb->pitch;
4397	Serge	5358	bo->tiling = get_tiling.tiling_mode;
4304	Serge	5359	bo->scanout = 1;
		5360	fb->fb_bo = bo;
		5361
4397	Serge	5362	printf("fb handle %d w: %d h: %d pitch %d tilng %d bo %p\n",
		5363	bo->handle, fb->width, fb->height, fb->pitch, fb->tiling, fb->fb_bo);
4304	Serge	5364
		5365	return 1;
		5366	};
		5367
		5368
		5369	int kgem_update_fb(struct kgem kgem, struct sna_fb fb)
		5370	{
		5371	struct kgem_bo *bo;
		5372	size_t size;
		5373	int ret;
		5374
		5375	bo = fb->fb_bo;
		5376
		5377	ret = drmIoctl(kgem->fd, SRV_FBINFO, fb);
		5378	if( ret != 0 )
		5379	return 0;
		5380
		5381	fb->fb_bo = bo;
		5382
		5383	size = fb->pitch * fb->height / PAGE_SIZE;
		5384
		5385	if((size != bo->size.pages.count) \|\|
		5386	(fb->pitch != bo->pitch))
		5387	{
		5388	bo->size.pages.count = size;
		5389	bo->pitch = fb->pitch;
		5390
		5391	printf("fb width %d height %d pitch %d bo %p\n",
		5392	fb->width, fb->height, fb->pitch, fb->fb_bo);
		5393
		5394	return 1;
		5395	}
		5396
		5397	return 0;
		5398	};
		5399
		5400	void sna_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		5401	{
		5402	kgem_bo_destroy(kgem, bo);
		5403	kgem_bo_free(kgem, bo);
		5404	}
		5405
		5406
		5407	void kgem_close_batches(struct kgem *kgem)
		5408	{
		5409	int n;
		5410	for (n = 0; n < ARRAY_SIZE(kgem->pinned_batches); n++) {
4368	Serge	5411	while (!list_is_empty(&kgem->pinned_batches[n]))
		5412	{
		5413	struct kgem_bo *bo =
4304	Serge	5414	list_first_entry(&kgem->pinned_batches[n],
4368	Serge	5415	struct kgem_bo, list);
		5416	list_del(&bo->list);
		5417	kgem_bo_destroy(kgem,bo);
4304	Serge	5418	}
		5419	}
		5420	};
		5421
		5422	struct kgem_bo kgem_bo_from_handle(struct kgem kgem, int handle,
		5423	int pitch, int height)
		5424	{
		5425	struct kgem_bo *bo;
		5426	int size;
		5427
		5428	size = pitch * height / PAGE_SIZE;
		5429
		5430	bo = __kgem_bo_alloc(handle, size);
		5431	if(bo == NULL)
		5432	return NULL;
		5433
		5434	bo->domain = DOMAIN_GTT;
		5435	bo->unique_id = kgem_get_unique_id(kgem);
		5436	bo->pitch = pitch;
		5437	bo->tiling = I915_TILING_X;
		5438	bo->scanout = 0;
		5439
		5440	return bo;
		5441	}

Subversion Repositories Kolibri OS

(root)/contrib/sdk/sources/Intel-2D/sna/kgem.c – Rev 4411