WebSVN – Kolibri OS – Blame – /contrib/sdk/sources/Intel-2D/sna/kgem.c

Rev	Author	Line No.	Line
4304	Serge	1	/*
		2	* Copyright (c) 2011 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
		20	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		21	* SOFTWARE.
		22	*
		23	* Authors:
		24	* Chris Wilson
		25	*
		26	*/
		27
		28	#ifdef HAVE_CONFIG_H
		29	#include "config.h"
		30	#endif
		31
		32	#include "sna.h"
		33	#include "sna_reg.h"
		34
		35	#include
		36	#include
		37	#include
		38
		39	#ifdef HAVE_VALGRIND
		40	#include
		41	#include
		42	#endif
		43
		44	#ifdef HAVE_STRUCT_SYSINFO_TOTALRAM
		45	#include
		46	#endif
		47
		48	#include "sna_cpuid.h"
		49
4368	Serge	50
4304	Serge	51	static struct kgem_bo *
		52	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
		53
		54	static struct kgem_bo *
		55	search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
		56
		57	#define DBG_NO_HW 0
		58	#define DBG_NO_TILING 0
		59	#define DBG_NO_CACHE 0
		60	#define DBG_NO_CACHE_LEVEL 0
		61	#define DBG_NO_CPU 0
		62	#define DBG_NO_CREATE2 1
		63	#define DBG_NO_USERPTR 0
		64	#define DBG_NO_UNSYNCHRONIZED_USERPTR 0
		65	#define DBG_NO_LLC 0
		66	#define DBG_NO_SEMAPHORES 0
		67	#define DBG_NO_MADV 1
		68	#define DBG_NO_UPLOAD_CACHE 0
		69	#define DBG_NO_UPLOAD_ACTIVE 0
		70	#define DBG_NO_MAP_UPLOAD 0
		71	#define DBG_NO_RELAXED_FENCING 0
		72	#define DBG_NO_SECURE_BATCHES 0
		73	#define DBG_NO_PINNED_BATCHES 0
		74	#define DBG_NO_FAST_RELOC 0
		75	#define DBG_NO_HANDLE_LUT 1
		76	#define DBG_NO_WT 0
		77	#define DBG_DUMP 0
		78
		79	#define FORCE_MMAP_SYNC 0 /* ((1 << DOMAIN_CPU) \| (1 << DOMAIN_GTT)) */
		80
		81	#ifndef DEBUG_SYNC
		82	#define DEBUG_SYNC 0
		83	#endif
		84
		85
		86	#if 0
		87	#define ASSERT_IDLE(kgem__, handle__) assert(!__kgem_busy(kgem__, handle__))
		88	#define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__) assert(!(expect__) \|\| !__kgem_busy(kgem__, handle__))
		89	#else
		90	#define ASSERT_IDLE(kgem__, handle__)
		91	#define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__)
		92	#endif
		93
		94	/* Worst case seems to be 965gm where we cannot write within a cacheline that
		95	* is being simultaneously being read by the GPU, or within the sampler
		96	* prefetch. In general, the chipsets seem to have a requirement that sampler
		97	* offsets be aligned to a cacheline (64 bytes).
		98	*/
		99	#define UPLOAD_ALIGNMENT 128
		100
		101	#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
		102	#define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE)
		103
		104	#define MAX_GTT_VMA_CACHE 512
		105	#define MAX_CPU_VMA_CACHE INT16_MAX
		106	#define MAP_PRESERVE_TIME 10
		107
		108	#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 1))
		109	#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 3))
		110	#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 2)
		111	#define __MAP_TYPE(ptr) ((uintptr_t)(ptr) & 3)
		112
		113	#define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) \| (ring)))
		114
		115	#define LOCAL_I915_PARAM_HAS_BLT 11
		116	#define LOCAL_I915_PARAM_HAS_RELAXED_FENCING 12
		117	#define LOCAL_I915_PARAM_HAS_RELAXED_DELTA 15
		118	#define LOCAL_I915_PARAM_HAS_SEMAPHORES 20
		119	#define LOCAL_I915_PARAM_HAS_SECURE_BATCHES 23
		120	#define LOCAL_I915_PARAM_HAS_PINNED_BATCHES 24
		121	#define LOCAL_I915_PARAM_HAS_NO_RELOC 25
		122	#define LOCAL_I915_PARAM_HAS_HANDLE_LUT 26
		123	#define LOCAL_I915_PARAM_HAS_WT 27
		124
		125	#define LOCAL_I915_EXEC_IS_PINNED (1<<10)
		126	#define LOCAL_I915_EXEC_NO_RELOC (1<<11)
		127	#define LOCAL_I915_EXEC_HANDLE_LUT (1<<12)
		128	struct local_i915_gem_userptr {
		129	uint64_t user_ptr;
		130	uint64_t user_size;
		131	uint32_t flags;
		132	#define I915_USERPTR_READ_ONLY (1<<0)
		133	#define I915_USERPTR_UNSYNCHRONIZED (1<<31)
		134	uint32_t handle;
		135	};
		136
		137	#define UNCACHED 0
		138	#define SNOOPED 1
		139	#define DISPLAY 2
		140
		141	struct local_i915_gem_caching {
		142	uint32_t handle;
		143	uint32_t caching;
		144	};
		145
		146	#define LOCAL_IOCTL_I915_GEM_SET_CACHING SRV_I915_GEM_SET_CACHING
		147
		148	struct local_fbinfo {
		149	int width;
		150	int height;
		151	int pitch;
		152	int tiling;
		153	};
		154
		155	struct kgem_buffer {
		156	struct kgem_bo base;
		157	void *mem;
		158	uint32_t used;
		159	uint32_t need_io : 1;
		160	uint32_t write : 2;
		161	uint32_t mmapped : 1;
		162	};
		163
		164	static struct kgem_bo *__kgem_freed_bo;
		165	static struct kgem_request *__kgem_freed_request;
		166	static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;
		167
		168	static inline int bytes(struct kgem_bo *bo)
		169	{
		170	return __kgem_bo_size(bo);
		171	}
		172
		173	#define bucket(B) (B)->size.pages.bucket
		174	#define num_pages(B) (B)->size.pages.count
		175
		176	#ifdef DEBUG_MEMORY
		177	static void debug_alloc(struct kgem *kgem, size_t size)
		178	{
		179	kgem->debug_memory.bo_allocs++;
		180	kgem->debug_memory.bo_bytes += size;
		181	}
		182	static void debug_alloc__bo(struct kgem kgem, struct kgem_bo bo)
		183	{
		184	debug_alloc(kgem, bytes(bo));
		185	}
		186	#else
		187	#define debug_alloc(k, b)
		188	#define debug_alloc__bo(k, b)
		189	#endif
		190
		191	#ifndef NDEBUG
		192	static void assert_tiling(struct kgem kgem, struct kgem_bo bo)
		193	{
		194	struct drm_i915_gem_get_tiling tiling;
		195
		196	assert(bo);
		197
		198	VG_CLEAR(tiling);
		199	tiling.handle = bo->handle;
		200	tiling.tiling_mode = -1;
		201	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling);
		202	assert(tiling.tiling_mode == bo->tiling);
		203	}
		204	#else
		205	#define assert_tiling(kgem, bo)
		206	#endif
		207
		208	static void kgem_sna_reset(struct kgem *kgem)
		209	{
		210	struct sna *sna = container_of(kgem, struct sna, kgem);
		211
		212	sna->render.reset(sna);
		213	sna->blt_state.fill_bo = 0;
		214	}
		215
		216	static void kgem_sna_flush(struct kgem *kgem)
		217	{
		218	struct sna *sna = container_of(kgem, struct sna, kgem);
		219
		220	sna->render.flush(sna);
		221
		222	// if (sna->render.solid_cache.dirty)
		223	// sna_render_flush_solid(sna);
		224	}
		225
		226	static bool gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
		227	{
		228	struct drm_i915_gem_set_tiling set_tiling;
		229	int ret;
		230
		231	if (DBG_NO_TILING)
		232	return false;
		233
		234	VG_CLEAR(set_tiling);
		235	do {
		236	set_tiling.handle = handle;
		237	set_tiling.tiling_mode = tiling;
		238	set_tiling.stride = stride;
		239
		240	ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling);
		241	} while (ret != 0);
		242	return ret == 0;
		243	}
		244
		245	static bool gem_set_caching(int fd, uint32_t handle, int caching)
		246	{
		247	struct local_i915_gem_caching arg;
		248
		249	VG_CLEAR(arg);
		250	arg.handle = handle;
		251	arg.caching = caching;
		252	return drmIoctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0;
		253	}
		254
		255
		256
		257
		258
		259	static bool __kgem_throttle_retire(struct kgem *kgem, unsigned flags)
		260	{
		261	if (flags & CREATE_NO_RETIRE) {
		262	DBG(("%s: not retiring per-request\n", __FUNCTION__));
		263	return false;
		264	}
		265
		266	if (!kgem->need_retire) {
		267	DBG(("%s: nothing to retire\n", __FUNCTION__));
		268	return false;
		269	}
		270
		271	if (kgem_retire(kgem))
		272	return true;
		273
		274	if (flags & CREATE_NO_THROTTLE \|\| !kgem->need_throttle) {
		275	DBG(("%s: not throttling\n", __FUNCTION__));
		276	return false;
		277	}
		278
		279	kgem_throttle(kgem);
		280	return kgem_retire(kgem);
		281	}
		282
		283	static void __kgem_bo_map__gtt(struct kgem kgem, struct kgem_bo *bo)
		284	{
		285	struct drm_i915_gem_mmap_gtt mmap_arg;
		286	void *ptr;
		287
		288	DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__,
		289	bo->handle, bytes(bo)));
		290	assert(bo->proxy == NULL);
		291	assert(!bo->snoop);
		292	assert(kgem_bo_can_map(kgem, bo));
		293
		294	retry_gtt:
		295	VG_CLEAR(mmap_arg);
		296	mmap_arg.handle = bo->handle;
		297	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg)) {
		298
		299	(void)__kgem_throttle_retire(kgem, 0);
		300	if (kgem_expire_cache(kgem))
		301	goto retry_gtt;
		302
		303	if (kgem->need_expire) {
		304	kgem_cleanup_cache(kgem);
		305	goto retry_gtt;
		306	}
		307
		308	printf("%s: failed to retrieve GTT offset for handle=%d\n",
		309	__FUNCTION__, bo->handle);
		310	return NULL;
		311	}
		312
		313	retry_mmap:
		314	ptr = (void*)(int)mmap_arg.offset;
		315	if (ptr == NULL) {
		316	ErrorF("%s: failed to mmap handle=%d, %d bytes, into GTT domain\n",
		317	__FUNCTION__, bo->handle, bytes(bo));
		318	ptr = NULL;
		319	}
		320
		321	return ptr;
		322	}
		323
		324	static int __gem_write(int fd, uint32_t handle,
		325	int offset, int length,
		326	const void *src)
		327	{
		328	struct drm_i915_gem_pwrite pwrite;
		329
		330	DBG(("%s(handle=%d, offset=%d, len=%d)\n", __FUNCTION__,
		331	handle, offset, length));
		332
		333	VG_CLEAR(pwrite);
		334	pwrite.handle = handle;
		335	pwrite.offset = offset;
		336	pwrite.size = length;
		337	pwrite.data_ptr = (uintptr_t)src;
		338	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
		339	}
		340
		341	static int gem_write(int fd, uint32_t handle,
		342	int offset, int length,
		343	const void *src)
		344	{
		345	struct drm_i915_gem_pwrite pwrite;
		346
		347	DBG(("%s(handle=%d, offset=%d, len=%d)\n", __FUNCTION__,
		348	handle, offset, length));
		349
		350	VG_CLEAR(pwrite);
		351	pwrite.handle = handle;
		352	/* align the transfer to cachelines; fortuitously this is safe! */
		353	if ((offset \| length) & 63) {
		354	pwrite.offset = offset & ~63;
		355	pwrite.size = ALIGN(offset+length, 64) - pwrite.offset;
		356	pwrite.data_ptr = (uintptr_t)src + pwrite.offset - offset;
		357	} else {
		358	pwrite.offset = offset;
		359	pwrite.size = length;
		360	pwrite.data_ptr = (uintptr_t)src;
		361	}
		362	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
		363	}
		364
		365
		366	bool __kgem_busy(struct kgem *kgem, int handle)
		367	{
		368	struct drm_i915_gem_busy busy;
		369
		370	VG_CLEAR(busy);
		371	busy.handle = handle;
		372	busy.busy = !kgem->wedged;
		373	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
		374	DBG(("%s: handle=%d, busy=%d, wedged=%d\n",
		375	__FUNCTION__, handle, busy.busy, kgem->wedged));
		376
		377	return busy.busy;
		378	}
		379
		380	static void kgem_bo_retire(struct kgem kgem, struct kgem_bo bo)
		381	{
		382	DBG(("%s: retiring bo handle=%d (needed flush? %d), rq? %d [busy?=%d]\n",
		383	__FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL,
		384	__kgem_busy(kgem, bo->handle)));
		385	assert(bo->exec == NULL);
		386	assert(list_is_empty(&bo->vma));
		387
		388	if (bo->rq) {
		389	if (!__kgem_busy(kgem, bo->handle)) {
		390	__kgem_bo_clear_busy(bo);
		391	kgem_retire(kgem);
		392	}
		393	} else {
		394	assert(!bo->needs_flush);
		395	ASSERT_IDLE(kgem, bo->handle);
		396	}
		397	}
		398
		399	bool kgem_bo_write(struct kgem kgem, struct kgem_bo bo,
		400	const void *data, int length)
		401	{
		402	assert(bo->refcnt);
		403	assert(!bo->purged);
		404	assert(bo->proxy == NULL);
		405	ASSERT_IDLE(kgem, bo->handle);
		406
		407	assert(length <= bytes(bo));
		408	if (gem_write(kgem->fd, bo->handle, 0, length, data))
		409	return false;
		410
		411	DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain));
		412	if (bo->exec == NULL) {
		413	kgem_bo_retire(kgem, bo);
		414	bo->domain = DOMAIN_NONE;
		415	}
		416	bo->gtt_dirty = true;
		417	return true;
		418	}
		419
		420	static uint32_t gem_create(int fd, int num_pages)
		421	{
		422	struct drm_i915_gem_create create;
		423
		424	VG_CLEAR(create);
		425	create.handle = 0;
		426	create.size = PAGE_SIZE * num_pages;
		427	(void)drmIoctl(fd, DRM_IOCTL_I915_GEM_CREATE, &create);
		428
		429	return create.handle;
		430	}
		431
		432	static bool
		433	kgem_bo_set_purgeable(struct kgem kgem, struct kgem_bo bo)
		434	{
		435	#if DBG_NO_MADV
		436	return true;
		437	#else
		438	struct drm_i915_gem_madvise madv;
		439
		440	assert(bo->exec == NULL);
		441	assert(!bo->purged);
		442
		443	VG_CLEAR(madv);
		444	madv.handle = bo->handle;
		445	madv.madv = I915_MADV_DONTNEED;
		446	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
		447	bo->purged = 1;
		448	kgem->need_purge \|= !madv.retained && bo->domain == DOMAIN_GPU;
		449	return madv.retained;
		450	}
		451
		452	return true;
		453	#endif
		454	}
		455
		456	static bool
		457	kgem_bo_is_retained(struct kgem kgem, struct kgem_bo bo)
		458	{
		459	#if DBG_NO_MADV
		460	return true;
		461	#else
		462	struct drm_i915_gem_madvise madv;
		463
		464	if (!bo->purged)
		465	return true;
		466
		467	VG_CLEAR(madv);
		468	madv.handle = bo->handle;
		469	madv.madv = I915_MADV_DONTNEED;
		470	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0)
		471	return madv.retained;
		472
		473	return false;
		474	#endif
		475	}
		476
		477	static bool
		478	kgem_bo_clear_purgeable(struct kgem kgem, struct kgem_bo bo)
		479	{
		480	#if DBG_NO_MADV
		481	return true;
		482	#else
		483	struct drm_i915_gem_madvise madv;
		484
		485	assert(bo->purged);
		486
		487	VG_CLEAR(madv);
		488	madv.handle = bo->handle;
		489	madv.madv = I915_MADV_WILLNEED;
		490	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
		491	bo->purged = !madv.retained;
		492	kgem->need_purge \|= !madv.retained && bo->domain == DOMAIN_GPU;
		493	return madv.retained;
		494	}
		495
		496	return false;
		497	#endif
		498	}
		499
		500	static void gem_close(int fd, uint32_t handle)
		501	{
		502	struct drm_gem_close close;
		503
		504	VG_CLEAR(close);
		505	close.handle = handle;
		506	(void)drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, &close);
		507	}
		508
		509	constant inline static unsigned long __fls(unsigned long word)
		510	{
		511	#if defined(__GNUC__) && (defined(__i386__) \|\| defined(__x86__) \|\| defined(__x86_64__))
		512	asm("bsr %1,%0"
		513	: "=r" (word)
		514	: "rm" (word));
		515	return word;
		516	#else
		517	unsigned int v = 0;
		518
		519	while (word >>= 1)
		520	v++;
		521
		522	return v;
		523	#endif
		524	}
		525
		526	constant inline static int cache_bucket(int num_pages)
		527	{
		528	return __fls(num_pages);
		529	}
		530
		531	static struct kgem_bo __kgem_bo_init(struct kgem_bo bo,
		532	int handle, int num_pages)
		533	{
		534	assert(num_pages);
		535	memset(bo, 0, sizeof(*bo));
		536
		537	bo->refcnt = 1;
		538	bo->handle = handle;
		539	bo->target_handle = -1;
		540	num_pages(bo) = num_pages;
		541	bucket(bo) = cache_bucket(num_pages);
		542	bo->reusable = true;
		543	bo->domain = DOMAIN_CPU;
		544	list_init(&bo->request);
		545	list_init(&bo->list);
		546	list_init(&bo->vma);
		547
		548	return bo;
		549	}
		550
		551	static struct kgem_bo *__kgem_bo_alloc(int handle, int num_pages)
		552	{
		553	struct kgem_bo *bo;
		554
		555	if (__kgem_freed_bo) {
		556	bo = __kgem_freed_bo;
		557	__kgem_freed_bo = (struct kgem_bo *)bo;
		558	} else {
		559	bo = malloc(sizeof(*bo));
		560	if (bo == NULL)
		561	return NULL;
		562	}
		563
		564	return __kgem_bo_init(bo, handle, num_pages);
		565	}
		566
		567	static struct kgem_request __kgem_request_alloc(struct kgem kgem)
		568	{
		569	struct kgem_request *rq;
		570
		571	rq = __kgem_freed_request;
		572	if (rq) {
		573	__kgem_freed_request = (struct kgem_request *)rq;
		574	} else {
		575	rq = malloc(sizeof(*rq));
		576	if (rq == NULL)
		577	rq = &kgem->static_request;
		578	}
		579
		580	list_init(&rq->buffers);
		581	rq->bo = NULL;
		582	rq->ring = 0;
		583
		584	return rq;
		585	}
		586
		587	static void __kgem_request_free(struct kgem_request *rq)
		588	{
		589	_list_del(&rq->list);
		590	(struct kgem_request *)rq = __kgem_freed_request;
		591	__kgem_freed_request = rq;
		592	}
		593
		594	static struct list inactive(struct kgem kgem, int num_pages)
		595	{
		596	assert(num_pages < MAX_CACHE_SIZE / PAGE_SIZE);
		597	assert(cache_bucket(num_pages) < NUM_CACHE_BUCKETS);
		598	return &kgem->inactive[cache_bucket(num_pages)];
		599	}
		600
		601	static struct list active(struct kgem kgem, int num_pages, int tiling)
		602	{
		603	assert(num_pages < MAX_CACHE_SIZE / PAGE_SIZE);
		604	assert(cache_bucket(num_pages) < NUM_CACHE_BUCKETS);
		605	return &kgem->active[cache_bucket(num_pages)][tiling];
		606	}
		607
		608	static size_t
		609	agp_aperture_size(struct pci_device *dev, unsigned gen)
		610	{
		611	/* XXX assume that only future chipsets are unknown and follow
		612	* the post gen2 PCI layout.
		613	*/
		614	return 0;
		615	}
		616
		617	static size_t
		618	total_ram_size(void)
		619	{
		620	uint32_t data[9];
		621	size_t size = 0;
		622
		623	asm volatile("int $0x40"
		624	: "=a" (size)
		625	: "a" (18),"b"(20), "c" (data)
		626	: "memory");
		627
		628	return size != -1 ? size : 0;
		629	}
		630
		631	static unsigned
		632	cpu_cache_size__cpuid4(void)
		633	{
		634	/* Deterministic Cache Parmaeters (Function 04h)":
		635	* When EAX is initialized to a value of 4, the CPUID instruction
		636	* returns deterministic cache information in the EAX, EBX, ECX
		637	* and EDX registers. This function requires ECX be initialized
		638	* with an index which indicates which cache to return information
		639	* about. The OS is expected to call this function (CPUID.4) with
		640	* ECX = 0, 1, 2, until EAX[4:0] == 0, indicating no more caches.
		641	* The order in which the caches are returned is not specified
		642	* and may change at Intel's discretion.
		643	*
		644	* Calculating the Cache Size in bytes:
		645	* = (Ways +1) * (Partitions +1) * (Line Size +1) * (Sets +1)
		646	*/
		647
		648	unsigned int eax, ebx, ecx, edx;
		649	unsigned int llc_size = 0;
		650	int cnt = 0;
		651
		652	if (__get_cpuid_max(BASIC_CPUID, NULL) < 4)
		653	return 0;
		654
		655	do {
		656	unsigned associativity, line_partitions, line_size, sets;
		657
		658	__cpuid_count(4, cnt++, eax, ebx, ecx, edx);
		659
		660	if ((eax & 0x1f) == 0)
		661	break;
		662
		663	associativity = ((ebx >> 22) & 0x3ff) + 1;
		664	line_partitions = ((ebx >> 12) & 0x3ff) + 1;
		665	line_size = (ebx & 0xfff) + 1;
		666	sets = ecx + 1;
		667
		668	llc_size = associativity * line_partitions * line_size * sets;
		669	} while (1);
		670
		671	return llc_size;
		672	}
		673
		674	static int gem_param(struct kgem *kgem, int name)
		675	{
		676	drm_i915_getparam_t gp;
		677	int v = -1; /* No param uses the sign bit, reserve it for errors */
		678
		679	VG_CLEAR(gp);
		680	gp.param = name;
		681	gp.value = &v;
		682	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GETPARAM, &gp))
		683	return -1;
		684
		685	VG(VALGRIND_MAKE_MEM_DEFINED(&v, sizeof(v)));
		686	return v;
		687	}
		688
		689	static bool test_has_execbuffer2(struct kgem *kgem)
		690	{
		691	return 1;
		692	}
		693
		694	static bool test_has_no_reloc(struct kgem *kgem)
		695	{
		696	if (DBG_NO_FAST_RELOC)
		697	return false;
		698
		699	return gem_param(kgem, LOCAL_I915_PARAM_HAS_NO_RELOC) > 0;
		700	}
		701
		702	static bool test_has_handle_lut(struct kgem *kgem)
		703	{
		704	if (DBG_NO_HANDLE_LUT)
		705	return false;
		706
		707	return gem_param(kgem, LOCAL_I915_PARAM_HAS_HANDLE_LUT) > 0;
		708	}
		709
		710	static bool test_has_wt(struct kgem *kgem)
		711	{
		712	if (DBG_NO_WT)
		713	return false;
		714
		715	return gem_param(kgem, LOCAL_I915_PARAM_HAS_WT) > 0;
		716	}
		717
		718	static bool test_has_semaphores_enabled(struct kgem *kgem)
		719	{
		720	bool detected = false;
		721	int ret;
		722
		723	if (DBG_NO_SEMAPHORES)
		724	return false;
		725
		726	ret = gem_param(kgem, LOCAL_I915_PARAM_HAS_SEMAPHORES);
		727	if (ret != -1)
		728	return ret > 0;
		729
		730	return detected;
		731	}
		732
		733	static bool __kgem_throttle(struct kgem *kgem)
		734	{
		735	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_THROTTLE, NULL) == 0)
		736	return false;
		737
		738	return errno == EIO;
		739	}
		740
		741	static bool is_hw_supported(struct kgem *kgem,
		742	struct pci_device *dev)
		743	{
		744	if (DBG_NO_HW)
		745	return false;
		746
		747	if (!test_has_execbuffer2(kgem))
		748	return false;
		749
		750	if (kgem->gen == (unsigned)-1) /* unknown chipset, assume future gen */
		751	return kgem->has_blt;
		752
		753	/* Although pre-855gm the GMCH is fubar, it works mostly. So
		754	* let the user decide through "NoAccel" whether or not to risk
		755	* hw acceleration.
		756	*/
		757
		758	if (kgem->gen == 060 && dev->revision < 8) {
		759	/* pre-production SNB with dysfunctional BLT */
		760	return false;
		761	}
		762
		763	if (kgem->gen >= 060) /* Only if the kernel supports the BLT ring */
		764	return kgem->has_blt;
		765
		766	return true;
		767	}
		768
		769	static bool test_has_relaxed_fencing(struct kgem *kgem)
		770	{
		771	if (kgem->gen < 040) {
		772	if (DBG_NO_RELAXED_FENCING)
		773	return false;
		774
		775	return gem_param(kgem, LOCAL_I915_PARAM_HAS_RELAXED_FENCING) > 0;
		776	} else
		777	return true;
		778	}
		779
		780	static bool test_has_llc(struct kgem *kgem)
		781	{
		782	int has_llc = -1;
		783
		784	if (DBG_NO_LLC)
		785	return false;
		786
		787	#if defined(I915_PARAM_HAS_LLC) /* Expected in libdrm-2.4.31 */
		788	has_llc = gem_param(kgem, I915_PARAM_HAS_LLC);
		789	#endif
		790	if (has_llc == -1) {
		791	DBG(("%s: no kernel/drm support for HAS_LLC, assuming support for LLC based on GPU generation\n", __FUNCTION__));
		792	has_llc = kgem->gen >= 060;
		793	}
		794
		795	return has_llc;
		796	}
		797
		798	static bool test_has_caching(struct kgem *kgem)
		799	{
		800	uint32_t handle;
		801	bool ret;
		802
		803	if (DBG_NO_CACHE_LEVEL)
		804	return false;
		805
		806	/* Incoherent blt and sampler hangs the GPU */
		807	if (kgem->gen == 040)
		808	return false;
		809
		810	handle = gem_create(kgem->fd, 1);
		811	if (handle == 0)
		812	return false;
		813
		814	ret = gem_set_caching(kgem->fd, handle, UNCACHED);
		815	gem_close(kgem->fd, handle);
		816	return ret;
		817	}
		818
		819	static bool test_has_userptr(struct kgem *kgem)
		820	{
		821	#if defined(USE_USERPTR)
		822	uint32_t handle;
		823	void *ptr;
		824
		825	if (DBG_NO_USERPTR)
		826	return false;
		827
		828	/* Incoherent blt and sampler hangs the GPU */
		829	if (kgem->gen == 040)
		830	return false;
		831
		832	if (posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE))
		833	return false;
		834
		835	handle = gem_userptr(kgem->fd, ptr, PAGE_SIZE, false);
		836	gem_close(kgem->fd, handle);
		837	free(ptr);
		838
		839	return handle != 0;
		840	#else
		841	return false;
		842	#endif
		843	}
		844
		845	static bool test_has_create2(struct kgem *kgem)
		846	{
		847	#if defined(USE_CREATE2)
		848	struct local_i915_gem_create2 args;
		849
		850	if (DBG_NO_CREATE2)
		851	return false;
		852
		853	memset(&args, 0, sizeof(args));
		854	args.size = PAGE_SIZE;
		855	args.caching = DISPLAY;
		856	if (drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args) == 0)
		857	gem_close(kgem->fd, args.handle);
		858
		859	return args.handle != 0;
		860	#else
		861	return false;
		862	#endif
		863	}
		864
		865	static bool test_has_secure_batches(struct kgem *kgem)
		866	{
		867	if (DBG_NO_SECURE_BATCHES)
		868	return false;
		869
		870	return gem_param(kgem, LOCAL_I915_PARAM_HAS_SECURE_BATCHES) > 0;
		871	}
		872
		873	static bool test_has_pinned_batches(struct kgem *kgem)
		874	{
		875	if (DBG_NO_PINNED_BATCHES)
		876	return false;
		877
		878	return gem_param(kgem, LOCAL_I915_PARAM_HAS_PINNED_BATCHES) > 0;
		879	}
		880
		881
		882	static bool kgem_init_pinned_batches(struct kgem *kgem)
		883	{
4315	Serge	884	int count[2] = { 2, 2 };
4304	Serge	885	int size[2] = { 1, 2 };
		886	int n, i;
		887
		888	if (kgem->wedged)
		889	return true;
		890
		891	for (n = 0; n < ARRAY_SIZE(count); n++) {
		892	for (i = 0; i < count[n]; i++) {
		893	struct drm_i915_gem_pin pin;
		894	struct kgem_bo *bo;
		895
		896	VG_CLEAR(pin);
		897
		898	pin.handle = gem_create(kgem->fd, size[n]);
		899	if (pin.handle == 0)
		900	goto err;
		901
		902	DBG(("%s: new handle=%d, num_pages=%d\n",
		903	__FUNCTION__, pin.handle, size[n]));
		904
		905	bo = __kgem_bo_alloc(pin.handle, size[n]);
		906	if (bo == NULL) {
		907	gem_close(kgem->fd, pin.handle);
		908	goto err;
		909	}
		910
		911	pin.alignment = 0;
		912	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) {
		913	gem_close(kgem->fd, pin.handle);
		914	goto err;
		915	}
		916	bo->presumed_offset = pin.offset;
		917	debug_alloc__bo(kgem, bo);
		918	list_add(&bo->list, &kgem->pinned_batches[n]);
		919	}
		920	}
		921
		922	return true;
		923
		924	err:
		925	for (n = 0; n < ARRAY_SIZE(kgem->pinned_batches); n++) {
		926	while (!list_is_empty(&kgem->pinned_batches[n])) {
		927	kgem_bo_destroy(kgem,
		928	list_first_entry(&kgem->pinned_batches[n],
		929	struct kgem_bo, list));
		930	}
		931	}
		932
		933	/* For simplicity populate the lists with a single unpinned bo */
		934	for (n = 0; n < ARRAY_SIZE(count); n++) {
		935	struct kgem_bo *bo;
		936	uint32_t handle;
		937
		938	handle = gem_create(kgem->fd, size[n]);
		939	if (handle == 0)
		940	break;
		941
		942	bo = __kgem_bo_alloc(handle, size[n]);
		943	if (bo == NULL) {
		944	gem_close(kgem->fd, handle);
		945	break;
		946	}
		947
		948	debug_alloc__bo(kgem, bo);
		949	list_add(&bo->list, &kgem->pinned_batches[n]);
		950	}
		951	return false;
		952	}
		953
		954	void kgem_init(struct kgem kgem, int fd, struct pci_device dev, unsigned gen)
		955	{
		956	struct drm_i915_gem_get_aperture aperture;
		957	size_t totalram;
		958	unsigned half_gpu_max;
		959	unsigned int i, j;
		960
		961	DBG(("%s: fd=%d, gen=%d\n", __FUNCTION__, fd, gen));
		962
		963	memset(kgem, 0, sizeof(*kgem));
		964
		965	kgem->fd = fd;
		966	kgem->gen = gen;
		967
		968	list_init(&kgem->requests[0]);
		969	list_init(&kgem->requests[1]);
		970	list_init(&kgem->batch_buffers);
		971	list_init(&kgem->active_buffers);
		972	list_init(&kgem->flushing);
		973	list_init(&kgem->large);
		974	list_init(&kgem->large_inactive);
		975	list_init(&kgem->snoop);
		976	list_init(&kgem->scanout);
		977	for (i = 0; i < ARRAY_SIZE(kgem->pinned_batches); i++)
		978	list_init(&kgem->pinned_batches[i]);
		979	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		980	list_init(&kgem->inactive[i]);
		981	for (i = 0; i < ARRAY_SIZE(kgem->active); i++) {
		982	for (j = 0; j < ARRAY_SIZE(kgem->active[i]); j++)
		983	list_init(&kgem->active[i][j]);
		984	}
		985	for (i = 0; i < ARRAY_SIZE(kgem->vma); i++) {
		986	for (j = 0; j < ARRAY_SIZE(kgem->vma[i].inactive); j++)
		987	list_init(&kgem->vma[i].inactive[j]);
		988	}
		989	kgem->vma[MAP_GTT].count = -MAX_GTT_VMA_CACHE;
		990	kgem->vma[MAP_CPU].count = -MAX_CPU_VMA_CACHE;
		991
		992	kgem->has_blt = gem_param(kgem, LOCAL_I915_PARAM_HAS_BLT) > 0;
		993	DBG(("%s: has BLT ring? %d\n", __FUNCTION__,
		994	kgem->has_blt));
		995
		996	kgem->has_relaxed_delta =
		997	gem_param(kgem, LOCAL_I915_PARAM_HAS_RELAXED_DELTA) > 0;
		998	DBG(("%s: has relaxed delta? %d\n", __FUNCTION__,
		999	kgem->has_relaxed_delta));
		1000
		1001	kgem->has_relaxed_fencing = test_has_relaxed_fencing(kgem);
		1002	DBG(("%s: has relaxed fencing? %d\n", __FUNCTION__,
		1003	kgem->has_relaxed_fencing));
		1004
		1005	kgem->has_llc = test_has_llc(kgem);
		1006	DBG(("%s: has shared last-level-cache? %d\n", __FUNCTION__,
		1007	kgem->has_llc));
		1008
		1009	kgem->has_wt = test_has_wt(kgem);
		1010	DBG(("%s: has write-through caching for scanouts? %d\n", __FUNCTION__,
		1011	kgem->has_wt));
		1012
		1013	kgem->has_caching = test_has_caching(kgem);
		1014	DBG(("%s: has set-cache-level? %d\n", __FUNCTION__,
		1015	kgem->has_caching));
		1016
		1017	kgem->has_userptr = test_has_userptr(kgem);
		1018	DBG(("%s: has userptr? %d\n", __FUNCTION__,
		1019	kgem->has_userptr));
		1020
		1021	kgem->has_create2 = test_has_create2(kgem);
		1022	kgem->has_create2 = 0;
		1023	DBG(("%s: has create2? %d\n", __FUNCTION__,
		1024	kgem->has_create2));
		1025
		1026	kgem->has_no_reloc = test_has_no_reloc(kgem);
		1027	DBG(("%s: has no-reloc? %d\n", __FUNCTION__,
		1028	kgem->has_no_reloc));
		1029
		1030	kgem->has_handle_lut = test_has_handle_lut(kgem);
		1031	kgem->has_handle_lut = 0;
		1032	DBG(("%s: has handle-lut? %d\n", __FUNCTION__,
		1033	kgem->has_handle_lut));
		1034
		1035	kgem->has_semaphores = false;
		1036	if (kgem->has_blt && test_has_semaphores_enabled(kgem))
		1037	kgem->has_semaphores = true;
		1038	DBG(("%s: semaphores enabled? %d\n", __FUNCTION__,
		1039	kgem->has_semaphores));
		1040
		1041	kgem->can_blt_cpu = gen >= 030;
		1042	DBG(("%s: can blt to cpu? %d\n", __FUNCTION__,
		1043	kgem->can_blt_cpu));
		1044
		1045	kgem->has_secure_batches = test_has_secure_batches(kgem);
		1046	DBG(("%s: can use privileged batchbuffers? %d\n", __FUNCTION__,
		1047	kgem->has_secure_batches));
		1048
		1049	kgem->has_pinned_batches = test_has_pinned_batches(kgem);
		1050	DBG(("%s: can use pinned batchbuffers (to avoid CS w/a)? %d\n", __FUNCTION__,
		1051	kgem->has_pinned_batches));
		1052
		1053	if (!is_hw_supported(kgem, dev)) {
		1054	printf("Detected unsupported/dysfunctional hardware, disabling acceleration.\n");
		1055	kgem->wedged = 1;
		1056	} else if (__kgem_throttle(kgem)) {
		1057	printf("Detected a hung GPU, disabling acceleration.\n");
		1058	kgem->wedged = 1;
		1059	}
		1060
		1061	kgem->batch_size = ARRAY_SIZE(kgem->batch);
		1062	if (gen == 020 && !kgem->has_pinned_batches)
		1063	/* Limited to what we can pin */
		1064	kgem->batch_size = 4*1024;
		1065	if (gen == 022)
		1066	/* 865g cannot handle a batch spanning multiple pages */
		1067	kgem->batch_size = PAGE_SIZE / sizeof(uint32_t);
		1068	if ((gen >> 3) == 7)
		1069	kgem->batch_size = 16*1024;
		1070	if (!kgem->has_relaxed_delta && kgem->batch_size > 4*1024)
		1071	kgem->batch_size = 4*1024;
		1072
		1073	if (!kgem_init_pinned_batches(kgem) && gen == 020) {
		1074	printf("Unable to reserve memory for GPU, disabling acceleration.\n");
		1075	kgem->wedged = 1;
		1076	}
		1077
		1078	DBG(("%s: maximum batch size? %d\n", __FUNCTION__,
		1079	kgem->batch_size));
		1080
		1081	kgem->min_alignment = 4;
		1082	if (gen < 040)
		1083	kgem->min_alignment = 64;
		1084
		1085	kgem->half_cpu_cache_pages = cpu_cache_size() >> 13;
		1086	DBG(("%s: last-level cache size: %d bytes, threshold in pages: %d\n",
		1087	__FUNCTION__, cpu_cache_size(), kgem->half_cpu_cache_pages));
		1088
		1089	kgem->next_request = __kgem_request_alloc(kgem);
		1090
		1091	DBG(("%s: cpu bo enabled %d: llc? %d, set-cache-level? %d, userptr? %d\n", __FUNCTION__,
		1092	!DBG_NO_CPU && (kgem->has_llc \| kgem->has_userptr \| kgem->has_caching),
		1093	kgem->has_llc, kgem->has_caching, kgem->has_userptr));
		1094
		1095	VG_CLEAR(aperture);
		1096	aperture.aper_size = 0;
		1097	(void)drmIoctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
		1098	if (aperture.aper_size == 0)
		1099	aperture.aper_size = 6410241024;
		1100
		1101	DBG(("%s: aperture size %lld, available now %lld\n",
		1102	__FUNCTION__,
		1103	(long long)aperture.aper_size,
		1104	(long long)aperture.aper_available_size));
		1105
		1106	kgem->aperture_total = aperture.aper_size;
		1107	kgem->aperture_high = aperture.aper_size * 3/4;
		1108	kgem->aperture_low = aperture.aper_size * 1/3;
		1109	if (gen < 033) {
		1110	/* Severe alignment penalties */
		1111	kgem->aperture_high /= 2;
		1112	kgem->aperture_low /= 2;
		1113	}
		1114	DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
		1115	kgem->aperture_low, kgem->aperture_low / (1024*1024),
		1116	kgem->aperture_high, kgem->aperture_high / (1024*1024)));
		1117
		1118	kgem->aperture_mappable = agp_aperture_size(dev, gen);
		1119	if (kgem->aperture_mappable == 0 \|\|
		1120	kgem->aperture_mappable > aperture.aper_size)
		1121	kgem->aperture_mappable = aperture.aper_size;
		1122	DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
		1123	kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));
		1124
		1125	kgem->buffer_size = 64 * 1024;
		1126	while (kgem->buffer_size < kgem->aperture_mappable >> 10)
		1127	kgem->buffer_size *= 2;
		1128	if (kgem->buffer_size >> 12 > kgem->half_cpu_cache_pages)
		1129	kgem->buffer_size = kgem->half_cpu_cache_pages << 12;
		1130	kgem->buffer_size = 1 << __fls(kgem->buffer_size);
		1131	DBG(("%s: buffer size=%d [%d KiB]\n", __FUNCTION__,
		1132	kgem->buffer_size, kgem->buffer_size / 1024));
		1133	assert(kgem->buffer_size);
		1134
		1135	kgem->max_object_size = 3 * (kgem->aperture_high >> 12) << 10;
		1136	kgem->max_gpu_size = kgem->max_object_size;
		1137	if (!kgem->has_llc && kgem->max_gpu_size > MAX_CACHE_SIZE)
		1138	kgem->max_gpu_size = MAX_CACHE_SIZE;
		1139
		1140	totalram = total_ram_size();
		1141	if (totalram == 0) {
		1142	DBG(("%s: total ram size unknown, assuming maximum of total aperture\n",
		1143	__FUNCTION__));
		1144	totalram = kgem->aperture_total;
		1145	}
		1146	DBG(("%s: total ram=%ld\n", __FUNCTION__, (long)totalram));
		1147	if (kgem->max_object_size > totalram / 2)
		1148	kgem->max_object_size = totalram / 2;
		1149	if (kgem->max_gpu_size > totalram / 4)
		1150	kgem->max_gpu_size = totalram / 4;
		1151
		1152	kgem->max_cpu_size = kgem->max_object_size;
		1153
		1154	half_gpu_max = kgem->max_gpu_size / 2;
		1155	kgem->max_copy_tile_size = (MAX_CACHE_SIZE + 1)/2;
		1156	if (kgem->max_copy_tile_size > half_gpu_max)
		1157	kgem->max_copy_tile_size = half_gpu_max;
		1158
		1159	if (kgem->has_llc)
		1160	kgem->max_upload_tile_size = kgem->max_copy_tile_size;
		1161	else
		1162	kgem->max_upload_tile_size = kgem->aperture_mappable / 4;
		1163	if (kgem->max_upload_tile_size > half_gpu_max)
		1164	kgem->max_upload_tile_size = half_gpu_max;
		1165	if (kgem->max_upload_tile_size > kgem->aperture_high/2)
		1166	kgem->max_upload_tile_size = kgem->aperture_high/2;
		1167	if (kgem->max_upload_tile_size > kgem->aperture_low)
		1168	kgem->max_upload_tile_size = kgem->aperture_low;
		1169	if (kgem->max_upload_tile_size < 16*PAGE_SIZE)
		1170	kgem->max_upload_tile_size = 16*PAGE_SIZE;
		1171
		1172	kgem->large_object_size = MAX_CACHE_SIZE;
		1173	if (kgem->large_object_size > half_gpu_max)
		1174	kgem->large_object_size = half_gpu_max;
		1175	if (kgem->max_copy_tile_size > kgem->aperture_high/2)
		1176	kgem->max_copy_tile_size = kgem->aperture_high/2;
		1177	if (kgem->max_copy_tile_size > kgem->aperture_low)
		1178	kgem->max_copy_tile_size = kgem->aperture_low;
		1179	if (kgem->max_copy_tile_size < 16*PAGE_SIZE)
		1180	kgem->max_copy_tile_size = 16*PAGE_SIZE;
		1181
		1182	if (kgem->has_llc \| kgem->has_caching \| kgem->has_userptr) {
		1183	if (kgem->large_object_size > kgem->max_cpu_size)
		1184	kgem->large_object_size = kgem->max_cpu_size;
		1185	} else
		1186	kgem->max_cpu_size = 0;
		1187	if (DBG_NO_CPU)
		1188	kgem->max_cpu_size = 0;
		1189
		1190	DBG(("%s: maximum object size=%d\n",
		1191	__FUNCTION__, kgem->max_object_size));
		1192	DBG(("%s: large object thresold=%d\n",
		1193	__FUNCTION__, kgem->large_object_size));
		1194	DBG(("%s: max object sizes (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
		1195	__FUNCTION__,
		1196	kgem->max_gpu_size, kgem->max_cpu_size,
		1197	kgem->max_upload_tile_size, kgem->max_copy_tile_size));
		1198
		1199	/* Convert the aperture thresholds to pages */
		1200	kgem->aperture_low /= PAGE_SIZE;
		1201	kgem->aperture_high /= PAGE_SIZE;
		1202
		1203	kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
		1204	if ((int)kgem->fence_max < 0)
		1205	kgem->fence_max = 5; /* minimum safe value for all hw */
		1206	DBG(("%s: max fences=%d\n", __FUNCTION__, kgem->fence_max));
		1207
		1208	kgem->batch_flags_base = 0;
		1209	if (kgem->has_no_reloc)
		1210	kgem->batch_flags_base \|= LOCAL_I915_EXEC_NO_RELOC;
		1211	if (kgem->has_handle_lut)
		1212	kgem->batch_flags_base \|= LOCAL_I915_EXEC_HANDLE_LUT;
		1213	if (kgem->has_pinned_batches)
		1214	kgem->batch_flags_base \|= LOCAL_I915_EXEC_IS_PINNED;
		1215	}
		1216
		1217	/* XXX hopefully a good approximation */
		1218	uint32_t kgem_get_unique_id(struct kgem *kgem)
		1219	{
		1220	uint32_t id;
		1221	id = ++kgem->unique_id;
		1222	if (id == 0)
		1223	id = ++kgem->unique_id;
		1224	return id;
		1225	}
		1226
		1227	inline static uint32_t kgem_pitch_alignment(struct kgem *kgem, unsigned flags)
		1228	{
		1229	if (flags & CREATE_PRIME)
		1230	return 256;
		1231	if (flags & CREATE_SCANOUT)
		1232	return 64;
		1233	return kgem->min_alignment;
		1234	}
		1235
		1236	void kgem_get_tile_size(struct kgem *kgem, int tiling,
		1237	int tile_width, int tile_height, int *tile_size)
		1238	{
		1239	if (kgem->gen <= 030) {
		1240	if (tiling) {
		1241	if (kgem->gen < 030) {
		1242	*tile_width = 128;
		1243	*tile_height = 16;
		1244	*tile_size = 2048;
		1245	} else {
		1246	*tile_width = 512;
		1247	*tile_height = 8;
		1248	*tile_size = 4096;
		1249	}
		1250	} else {
		1251	*tile_width = 1;
		1252	*tile_height = 1;
		1253	*tile_size = 1;
		1254	}
		1255	} else switch (tiling) {
		1256	default:
		1257	case I915_TILING_NONE:
		1258	*tile_width = 1;
		1259	*tile_height = 1;
		1260	*tile_size = 1;
		1261	break;
		1262	case I915_TILING_X:
		1263	*tile_width = 512;
		1264	*tile_height = 8;
		1265	*tile_size = 4096;
		1266	break;
		1267	case I915_TILING_Y:
		1268	*tile_width = 128;
		1269	*tile_height = 32;
		1270	*tile_size = 4096;
		1271	break;
		1272	}
		1273	}
		1274
		1275	uint32_t kgem_surface_size(struct kgem *kgem,
		1276	bool relaxed_fencing,
		1277	unsigned flags,
		1278	uint32_t width,
		1279	uint32_t height,
		1280	uint32_t bpp,
		1281	uint32_t tiling,
		1282	uint32_t *pitch)
		1283	{
		1284	uint32_t tile_width, tile_height;
		1285	uint32_t size;
		1286
		1287	assert(width <= MAXSHORT);
		1288	assert(height <= MAXSHORT);
		1289	assert(bpp >= 8);
		1290
		1291	if (kgem->gen <= 030) {
		1292	if (tiling) {
		1293	if (kgem->gen < 030) {
		1294	tile_width = 128;
		1295	tile_height = 32;
		1296	} else {
		1297	tile_width = 512;
		1298	tile_height = 16;
		1299	}
		1300	} else {
		1301	tile_width = 2 * bpp >> 3;
		1302	tile_width = ALIGN(tile_width,
		1303	kgem_pitch_alignment(kgem, flags));
		1304	tile_height = 2;
		1305	}
		1306	} else switch (tiling) {
		1307	default:
		1308	case I915_TILING_NONE:
		1309	tile_width = 2 * bpp >> 3;
		1310	tile_width = ALIGN(tile_width,
		1311	kgem_pitch_alignment(kgem, flags));
		1312	tile_height = 2;
		1313	break;
		1314
		1315	/* XXX align to an even tile row */
		1316	case I915_TILING_X:
		1317	tile_width = 512;
		1318	tile_height = 16;
		1319	break;
		1320	case I915_TILING_Y:
		1321	tile_width = 128;
		1322	tile_height = 64;
		1323	break;
		1324	}
		1325
		1326	pitch = ALIGN(width bpp / 8, tile_width);
		1327	height = ALIGN(height, tile_height);
		1328	if (kgem->gen >= 040)
		1329	return PAGE_ALIGN(pitch height);
		1330
		1331	/* If it is too wide for the blitter, don't even bother. */
		1332	if (tiling != I915_TILING_NONE) {
		1333	if (*pitch > 8192)
		1334	return 0;
		1335
		1336	for (size = tile_width; size < *pitch; size <<= 1)
		1337	;
		1338	*pitch = size;
		1339	} else {
		1340	if (*pitch >= 32768)
		1341	return 0;
		1342	}
		1343
		1344	size = pitch height;
		1345	if (relaxed_fencing \|\| tiling == I915_TILING_NONE)
		1346	return PAGE_ALIGN(size);
		1347
		1348	/* We need to allocate a pot fence region for a tiled buffer. */
		1349	if (kgem->gen < 030)
		1350	tile_width = 512 * 1024;
		1351	else
		1352	tile_width = 1024 * 1024;
		1353	while (tile_width < size)
		1354	tile_width *= 2;
		1355	return tile_width;
		1356	}
		1357
		1358	static uint32_t kgem_aligned_height(struct kgem *kgem,
		1359	uint32_t height, uint32_t tiling)
		1360	{
		1361	uint32_t tile_height;
		1362
		1363	if (kgem->gen <= 030) {
		1364	tile_height = tiling ? kgem->gen < 030 ? 32 : 16 : 1;
		1365	} else switch (tiling) {
		1366	/* XXX align to an even tile row */
		1367	default:
		1368	case I915_TILING_NONE:
		1369	tile_height = 1;
		1370	break;
		1371	case I915_TILING_X:
		1372	tile_height = 16;
		1373	break;
		1374	case I915_TILING_Y:
		1375	tile_height = 64;
		1376	break;
		1377	}
		1378
		1379	return ALIGN(height, tile_height);
		1380	}
		1381
		1382	static struct drm_i915_gem_exec_object2 *
		1383	kgem_add_handle(struct kgem kgem, struct kgem_bo bo)
		1384	{
		1385	struct drm_i915_gem_exec_object2 *exec;
		1386
		1387	DBG(("%s: handle=%d, index=%d\n",
		1388	__FUNCTION__, bo->handle, kgem->nexec));
		1389
		1390	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		1391	bo->target_handle = kgem->has_handle_lut ? kgem->nexec : bo->handle;
		1392	exec = memset(&kgem->exec[kgem->nexec++], 0, sizeof(*exec));
		1393	exec->handle = bo->handle;
		1394	exec->offset = bo->presumed_offset;
		1395
		1396	kgem->aperture += num_pages(bo);
		1397
		1398	return exec;
		1399	}
		1400
		1401	static void kgem_add_bo(struct kgem kgem, struct kgem_bo bo)
		1402	{
		1403	bo->exec = kgem_add_handle(kgem, bo);
		1404	bo->rq = MAKE_REQUEST(kgem->next_request, kgem->ring);
		1405
		1406	list_move_tail(&bo->request, &kgem->next_request->buffers);
		1407
		1408	/* XXX is it worth working around gcc here? */
		1409	kgem->flush \|= bo->flush;
		1410	}
		1411
		1412	static uint32_t kgem_end_batch(struct kgem *kgem)
		1413	{
		1414	kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
		1415	if (kgem->nbatch & 1)
		1416	kgem->batch[kgem->nbatch++] = MI_NOOP;
		1417
		1418	return kgem->nbatch;
		1419	}
		1420
		1421	static void kgem_fixup_self_relocs(struct kgem kgem, struct kgem_bo bo)
		1422	{
		1423	int n;
		1424
		1425	assert(kgem->nreloc__self <= 256);
		1426	if (kgem->nreloc__self == 0)
		1427	return;
		1428
		1429	for (n = 0; n < kgem->nreloc__self; n++) {
		1430	int i = kgem->reloc__self[n];
		1431	assert(kgem->reloc[i].target_handle == ~0U);
		1432	kgem->reloc[i].target_handle = bo->target_handle;
		1433	kgem->reloc[i].presumed_offset = bo->presumed_offset;
		1434	kgem->batch[kgem->reloc[i].offset/sizeof(kgem->batch[0])] =
		1435	kgem->reloc[i].delta + bo->presumed_offset;
		1436	}
		1437
		1438	if (n == 256) {
		1439	for (n = kgem->reloc__self[255]; n < kgem->nreloc; n++) {
		1440	if (kgem->reloc[n].target_handle == ~0U) {
		1441	kgem->reloc[n].target_handle = bo->target_handle;
		1442	kgem->reloc[n].presumed_offset = bo->presumed_offset;
		1443	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		1444	kgem->reloc[n].delta + bo->presumed_offset;
		1445	}
		1446	}
		1447
		1448	}
		1449
		1450	}
		1451
		1452	static void kgem_bo_binding_free(struct kgem kgem, struct kgem_bo bo)
		1453	{
		1454	struct kgem_bo_binding *b;
		1455
		1456	b = bo->binding.next;
		1457	while (b) {
		1458	struct kgem_bo_binding *next = b->next;
		1459	free (b);
		1460	b = next;
		1461	}
		1462	}
		1463
		1464	static void kgem_bo_release_map(struct kgem kgem, struct kgem_bo bo)
		1465	{
		1466	int type = IS_CPU_MAP(bo->map);
		1467
		1468	assert(!IS_USER_MAP(bo->map));
		1469
		1470	DBG(("%s: releasing %s vma for handle=%d, count=%d\n",
		1471	__FUNCTION__, type ? "CPU" : "GTT",
		1472	bo->handle, kgem->vma[type].count));
		1473
		1474	VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo)));
		1475	user_free(MAP(bo->map));
		1476	bo->map = NULL;
		1477
		1478	if (!list_is_empty(&bo->vma)) {
		1479	list_del(&bo->vma);
		1480	kgem->vma[type].count--;
		1481	}
		1482	}
		1483
		1484	static void kgem_bo_free(struct kgem kgem, struct kgem_bo bo)
		1485	{
		1486	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		1487	assert(bo->refcnt == 0);
		1488	assert(bo->proxy == NULL);
		1489	assert(bo->exec == NULL);
		1490	assert(!bo->snoop \|\| bo->rq == NULL);
		1491
		1492	#ifdef DEBUG_MEMORY
		1493	kgem->debug_memory.bo_allocs--;
		1494	kgem->debug_memory.bo_bytes -= bytes(bo);
		1495	#endif
		1496
		1497	kgem_bo_binding_free(kgem, bo);
		1498
		1499	if (IS_USER_MAP(bo->map)) {
		1500	assert(bo->rq == NULL);
		1501	assert(!__kgem_busy(kgem, bo->handle));
		1502	assert(MAP(bo->map) != bo \|\| bo->io \|\| bo->flush);
		1503	if (!(bo->io \|\| bo->flush)) {
		1504	DBG(("%s: freeing snooped base\n", __FUNCTION__));
		1505	assert(bo != MAP(bo->map));
		1506	free(MAP(bo->map));
		1507	}
		1508	bo->map = NULL;
		1509	}
		1510	if (bo->map)
		1511	kgem_bo_release_map(kgem, bo);
		1512	assert(list_is_empty(&bo->vma));
		1513	assert(bo->map == NULL);
		1514
		1515	_list_del(&bo->list);
		1516	_list_del(&bo->request);
		1517	gem_close(kgem->fd, bo->handle);
		1518
		1519	if (!bo->io) {
		1520	(struct kgem_bo *)bo = __kgem_freed_bo;
		1521	__kgem_freed_bo = bo;
		1522	} else
		1523	free(bo);
		1524	}
		1525
		1526	inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
		1527	struct kgem_bo *bo)
		1528	{
		1529	DBG(("%s: moving handle=%d to inactive\n", __FUNCTION__, bo->handle));
		1530
		1531	assert(bo->refcnt == 0);
		1532	assert(bo->reusable);
		1533	assert(bo->rq == NULL);
		1534	assert(bo->exec == NULL);
		1535	assert(bo->domain != DOMAIN_GPU);
		1536	assert(!bo->proxy);
		1537	assert(!bo->io);
		1538	assert(!bo->scanout);
		1539	assert(!bo->snoop);
		1540	assert(!bo->flush);
		1541	assert(!bo->needs_flush);
		1542	assert(list_is_empty(&bo->vma));
		1543	assert_tiling(kgem, bo);
		1544	ASSERT_IDLE(kgem, bo->handle);
		1545
		1546	kgem->need_expire = true;
		1547
		1548	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
		1549	list_move(&bo->list, &kgem->large_inactive);
		1550	return;
		1551	}
		1552
		1553	assert(bo->flush == false);
		1554	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
		1555	if (bo->map) {
		1556	int type = IS_CPU_MAP(bo->map);
		1557	if (bucket(bo) >= NUM_CACHE_BUCKETS \|\|
		1558	(!type && !__kgem_bo_is_mappable(kgem, bo))) {
		1559	// munmap(MAP(bo->map), bytes(bo));
		1560	bo->map = NULL;
		1561	}
		1562	if (bo->map) {
		1563	list_add(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]);
		1564	kgem->vma[type].count++;
		1565	}
		1566	}
		1567	}
		1568
		1569	static struct kgem_bo kgem_bo_replace_io(struct kgem_bo bo)
		1570	{
		1571	struct kgem_bo *base;
		1572
		1573	if (!bo->io)
		1574	return bo;
		1575
		1576	assert(!bo->snoop);
		1577	base = malloc(sizeof(*base));
		1578	if (base) {
		1579	DBG(("%s: transferring io handle=%d to bo\n",
		1580	__FUNCTION__, bo->handle));
		1581	/* transfer the handle to a minimum bo */
		1582	memcpy(base, bo, sizeof(*base));
		1583	base->io = false;
		1584	list_init(&base->list);
		1585	list_replace(&bo->request, &base->request);
		1586	list_replace(&bo->vma, &base->vma);
		1587	free(bo);
		1588	bo = base;
		1589	} else
		1590	bo->reusable = false;
		1591
		1592	return bo;
		1593	}
		1594
		1595	inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
		1596	struct kgem_bo *bo)
		1597	{
		1598	DBG(("%s: removing handle=%d from inactive\n", __FUNCTION__, bo->handle));
		1599
		1600	list_del(&bo->list);
		1601	assert(bo->rq == NULL);
		1602	assert(bo->exec == NULL);
		1603	if (bo->map) {
		1604	assert(!list_is_empty(&bo->vma));
		1605	list_del(&bo->vma);
		1606	kgem->vma[IS_CPU_MAP(bo->map)].count--;
		1607	}
		1608	}
		1609
		1610	inline static void kgem_bo_remove_from_active(struct kgem *kgem,
		1611	struct kgem_bo *bo)
		1612	{
		1613	DBG(("%s: removing handle=%d from active\n", __FUNCTION__, bo->handle));
		1614
		1615	list_del(&bo->list);
		1616	assert(bo->rq != NULL);
		1617	if (bo->rq == (void *)kgem)
		1618	list_del(&bo->request);
		1619	assert(list_is_empty(&bo->vma));
		1620	}
		1621
		1622	static void _kgem_bo_delete_buffer(struct kgem kgem, struct kgem_bo bo)
		1623	{
		1624	struct kgem_buffer io = (struct kgem_buffer )bo->proxy;
		1625
		1626	DBG(("%s: size=%d, offset=%d, parent used=%d\n",
		1627	__FUNCTION__, bo->size.bytes, bo->delta, io->used));
		1628
		1629	if (ALIGN(bo->delta + bo->size.bytes, UPLOAD_ALIGNMENT) == io->used)
		1630	io->used = bo->delta;
		1631	}
		1632
		1633	static void kgem_bo_move_to_scanout(struct kgem kgem, struct kgem_bo bo)
		1634	{
		1635	assert(bo->refcnt == 0);
		1636	assert(bo->scanout);
		1637	assert(bo->delta);
		1638	assert(!bo->flush);
		1639	assert(!bo->snoop);
		1640	assert(!bo->io);
		1641
		1642	if (bo->purged) {
		1643	DBG(("%s: discarding purged scanout - external name?\n",
		1644	__FUNCTION__));
		1645	kgem_bo_free(kgem, bo);
		1646	return;
		1647	}
		1648
		1649	DBG(("%s: moving %d [fb %d] to scanout cache, active? %d\n",
		1650	__FUNCTION__, bo->handle, bo->delta, bo->rq != NULL));
		1651	if (bo->rq)
		1652	list_move_tail(&bo->list, &kgem->scanout);
		1653	else
		1654	list_move(&bo->list, &kgem->scanout);
		1655	}
		1656
		1657	static void kgem_bo_move_to_snoop(struct kgem kgem, struct kgem_bo bo)
		1658	{
		1659	assert(bo->reusable);
		1660	assert(!bo->flush);
		1661	assert(!bo->needs_flush);
		1662	assert(bo->refcnt == 0);
		1663	assert(bo->exec == NULL);
		1664
		1665	if (num_pages(bo) > kgem->max_cpu_size >> 13) {
		1666	DBG(("%s handle=%d discarding large CPU buffer (%d >%d pages)\n",
		1667	__FUNCTION__, bo->handle, num_pages(bo), kgem->max_cpu_size >> 13));
		1668	kgem_bo_free(kgem, bo);
		1669	return;
		1670	}
		1671
		1672	assert(bo->tiling == I915_TILING_NONE);
		1673	assert(bo->rq == NULL);
		1674
		1675	DBG(("%s: moving %d to snoop cachee\n", __FUNCTION__, bo->handle));
		1676	list_add(&bo->list, &kgem->snoop);
		1677	}
		1678
		1679	static struct kgem_bo *
		1680	search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		1681	{
		1682	struct kgem_bo bo, first = NULL;
		1683
		1684	DBG(("%s: num_pages=%d, flags=%x\n", __FUNCTION__, num_pages, flags));
		1685
		1686	if ((kgem->has_caching \| kgem->has_userptr) == 0)
		1687	return NULL;
		1688
		1689	if (list_is_empty(&kgem->snoop)) {
		1690	DBG(("%s: inactive and cache empty\n", __FUNCTION__));
		1691	if (!__kgem_throttle_retire(kgem, flags)) {
		1692	DBG(("%s: nothing retired\n", __FUNCTION__));
		1693	return NULL;
		1694	}
		1695	}
		1696
		1697	list_for_each_entry(bo, &kgem->snoop, list) {
		1698	assert(bo->refcnt == 0);
		1699	assert(bo->snoop);
		1700	assert(!bo->scanout);
		1701	assert(!bo->purged);
		1702	assert(bo->proxy == NULL);
		1703	assert(bo->tiling == I915_TILING_NONE);
		1704	assert(bo->rq == NULL);
		1705	assert(bo->exec == NULL);
		1706
		1707	if (num_pages > num_pages(bo))
		1708	continue;
		1709
		1710	if (num_pages(bo) > 2*num_pages) {
		1711	if (first == NULL)
		1712	first = bo;
		1713	continue;
		1714	}
		1715
		1716	list_del(&bo->list);
		1717	bo->pitch = 0;
		1718	bo->delta = 0;
		1719
		1720	DBG((" %s: found handle=%d (num_pages=%d) in snoop cache\n",
		1721	__FUNCTION__, bo->handle, num_pages(bo)));
		1722	return bo;
		1723	}
		1724
		1725	if (first) {
		1726	list_del(&first->list);
		1727	first->pitch = 0;
		1728	first->delta = 0;
		1729
		1730	DBG((" %s: found handle=%d (num_pages=%d) in snoop cache\n",
		1731	__FUNCTION__, first->handle, num_pages(first)));
		1732	return first;
		1733	}
		1734
		1735	return NULL;
		1736	}
		1737
		1738	void kgem_bo_undo(struct kgem kgem, struct kgem_bo bo)
		1739	{
		1740	if (kgem->nexec != 1 \|\| bo->exec == NULL)
		1741	return;
		1742
		1743	DBG(("%s: only handle in batch, discarding last operations for handle=%d\n",
		1744	__FUNCTION__, bo->handle));
		1745
		1746	assert(bo->exec == &kgem->exec[0]);
		1747	assert(kgem->exec[0].handle == bo->handle);
		1748	assert(RQ(bo->rq) == kgem->next_request);
		1749
		1750	bo->refcnt++;
		1751	kgem_reset(kgem);
		1752	bo->refcnt--;
		1753	}
		1754
		1755	static void __kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		1756	{
		1757	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		1758
		1759	assert(list_is_empty(&bo->list));
		1760	assert(bo->refcnt == 0);
		1761	assert(!bo->purged \|\| !bo->reusable);
		1762	assert(bo->proxy == NULL);
		1763	assert_tiling(kgem, bo);
		1764
		1765	bo->binding.offset = 0;
		1766
		1767	if (DBG_NO_CACHE)
		1768	goto destroy;
		1769
		1770	if (bo->snoop && !bo->flush) {
		1771	DBG(("%s: handle=%d is snooped\n", __FUNCTION__, bo->handle));
		1772	assert(bo->reusable);
		1773	assert(list_is_empty(&bo->list));
		1774	if (bo->exec == NULL && bo->rq && !__kgem_busy(kgem, bo->handle))
		1775	__kgem_bo_clear_busy(bo);
		1776	if (bo->rq == NULL)
		1777	kgem_bo_move_to_snoop(kgem, bo);
		1778	return;
		1779	}
		1780	if (!IS_USER_MAP(bo->map))
		1781	bo->flush = false;
		1782
		1783	if (bo->scanout) {
		1784	kgem_bo_move_to_scanout(kgem, bo);
		1785	return;
		1786	}
		1787
		1788	if (bo->io)
		1789	bo = kgem_bo_replace_io(bo);
		1790	if (!bo->reusable) {
		1791	DBG(("%s: handle=%d, not reusable\n",
		1792	__FUNCTION__, bo->handle));
		1793	goto destroy;
		1794	}
		1795
		1796	if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU)
		1797	kgem_bo_release_map(kgem, bo);
		1798
		1799	assert(list_is_empty(&bo->vma));
		1800	assert(list_is_empty(&bo->list));
		1801	assert(bo->flush == false);
		1802	assert(bo->snoop == false);
		1803	assert(bo->io == false);
		1804	assert(bo->scanout == false);
		1805
		1806	kgem_bo_undo(kgem, bo);
		1807	assert(bo->refcnt == 0);
		1808
		1809	if (bo->rq && bo->exec == NULL && !__kgem_busy(kgem, bo->handle))
		1810	__kgem_bo_clear_busy(bo);
		1811
		1812	if (bo->rq) {
		1813	struct list *cache;
		1814
		1815	DBG(("%s: handle=%d -> active\n", __FUNCTION__, bo->handle));
		1816	if (bucket(bo) < NUM_CACHE_BUCKETS)
		1817	cache = &kgem->active[bucket(bo)][bo->tiling];
		1818	else
		1819	cache = &kgem->large;
		1820	list_add(&bo->list, cache);
		1821	return;
		1822	}
		1823
		1824	assert(bo->exec == NULL);
		1825	assert(list_is_empty(&bo->request));
		1826
		1827	if (!IS_CPU_MAP(bo->map)) {
		1828	if (!kgem_bo_set_purgeable(kgem, bo))
		1829	goto destroy;
		1830
		1831	if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
		1832	goto destroy;
		1833
		1834	DBG(("%s: handle=%d, purged\n",
		1835	__FUNCTION__, bo->handle));
		1836	}
		1837
		1838	kgem_bo_move_to_inactive(kgem, bo);
		1839	return;
		1840
		1841	destroy:
		1842	if (!bo->exec)
		1843	kgem_bo_free(kgem, bo);
		1844	}
		1845
		1846	static void kgem_bo_unref(struct kgem kgem, struct kgem_bo bo)
		1847	{
		1848	assert(bo->refcnt);
		1849	if (--bo->refcnt == 0)
		1850	__kgem_bo_destroy(kgem, bo);
		1851	}
		1852
		1853	static void kgem_buffer_release(struct kgem kgem, struct kgem_buffer bo)
		1854	{
		1855	while (!list_is_empty(&bo->base.vma)) {
		1856	struct kgem_bo *cached;
		1857
		1858	cached = list_first_entry(&bo->base.vma, struct kgem_bo, vma);
		1859	assert(cached->proxy == &bo->base);
		1860	list_del(&cached->vma);
		1861
		1862	assert((struct kgem_bo *)cached->map == cached);
		1863	(struct kgem_bo *)cached->map = NULL;
		1864	cached->map = NULL;
		1865
		1866	kgem_bo_destroy(kgem, cached);
		1867	}
		1868	}
		1869
		1870	static bool kgem_retire__buffers(struct kgem *kgem)
		1871	{
		1872	bool retired = false;
		1873
		1874	while (!list_is_empty(&kgem->active_buffers)) {
		1875	struct kgem_buffer *bo =
		1876	list_last_entry(&kgem->active_buffers,
		1877	struct kgem_buffer,
		1878	base.list);
		1879
		1880	if (bo->base.rq)
		1881	break;
		1882
		1883	DBG(("%s: releasing upload cache for handle=%d? %d\n",
		1884	__FUNCTION__, bo->base.handle, !list_is_empty(&bo->base.vma)));
		1885	list_del(&bo->base.list);
		1886	kgem_buffer_release(kgem, bo);
		1887	kgem_bo_unref(kgem, &bo->base);
		1888	retired = true;
		1889	}
		1890
		1891	return retired;
		1892	}
		1893
		1894	static bool kgem_retire__flushing(struct kgem *kgem)
		1895	{
		1896	struct kgem_bo bo, next;
		1897	bool retired = false;
		1898
		1899	list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
		1900	assert(bo->rq == (void *)kgem);
		1901	assert(bo->exec == NULL);
		1902
		1903	if (__kgem_busy(kgem, bo->handle))
		1904	break;
		1905
		1906	__kgem_bo_clear_busy(bo);
		1907
		1908	if (bo->refcnt)
		1909	continue;
		1910
		1911	if (bo->snoop) {
		1912	kgem_bo_move_to_snoop(kgem, bo);
		1913	} else if (bo->scanout) {
		1914	kgem_bo_move_to_scanout(kgem, bo);
		1915	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		1916	kgem_bo_set_purgeable(kgem, bo)) {
		1917	kgem_bo_move_to_inactive(kgem, bo);
		1918	retired = true;
		1919	} else
		1920	kgem_bo_free(kgem, bo);
		1921	}
		1922	#if HAS_DEBUG_FULL
		1923	{
		1924	int count = 0;
		1925	list_for_each_entry(bo, &kgem->flushing, request)
		1926	count++;
		1927	ErrorF("%s: %d bo on flushing list\n", __FUNCTION__, count);
		1928	}
		1929	#endif
		1930
		1931	kgem->need_retire \|= !list_is_empty(&kgem->flushing);
		1932
		1933	return retired;
		1934	}
		1935
		1936
		1937	static bool __kgem_retire_rq(struct kgem kgem, struct kgem_request rq)
		1938	{
		1939	bool retired = false;
		1940
		1941	DBG(("%s: request %d complete\n",
		1942	__FUNCTION__, rq->bo->handle));
		1943
		1944	while (!list_is_empty(&rq->buffers)) {
		1945	struct kgem_bo *bo;
		1946
		1947	bo = list_first_entry(&rq->buffers,
		1948	struct kgem_bo,
		1949	request);
		1950
		1951	assert(RQ(bo->rq) == rq);
		1952	assert(bo->exec == NULL);
		1953	assert(bo->domain == DOMAIN_GPU \|\| bo->domain == DOMAIN_NONE);
		1954
		1955	list_del(&bo->request);
		1956
		1957	if (bo->needs_flush)
		1958	bo->needs_flush = __kgem_busy(kgem, bo->handle);
		1959	if (bo->needs_flush) {
		1960	DBG(("%s: moving %d to flushing\n",
		1961	__FUNCTION__, bo->handle));
		1962	list_add(&bo->request, &kgem->flushing);
		1963	bo->rq = (void *)kgem;
		1964	continue;
		1965	}
		1966
		1967	bo->domain = DOMAIN_NONE;
		1968	bo->rq = NULL;
		1969	if (bo->refcnt)
		1970	continue;
		1971
		1972	if (bo->snoop) {
		1973	kgem_bo_move_to_snoop(kgem, bo);
		1974	} else if (bo->scanout) {
		1975	kgem_bo_move_to_scanout(kgem, bo);
		1976	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		1977	kgem_bo_set_purgeable(kgem, bo)) {
		1978	kgem_bo_move_to_inactive(kgem, bo);
		1979	retired = true;
		1980	} else {
		1981	DBG(("%s: closing %d\n",
		1982	__FUNCTION__, bo->handle));
		1983	kgem_bo_free(kgem, bo);
		1984	}
		1985	}
		1986
		1987	assert(rq->bo->rq == NULL);
		1988	assert(list_is_empty(&rq->bo->request));
		1989
		1990	if (--rq->bo->refcnt == 0) {
		1991	if (kgem_bo_set_purgeable(kgem, rq->bo)) {
		1992	kgem_bo_move_to_inactive(kgem, rq->bo);
		1993	retired = true;
		1994	} else {
		1995	DBG(("%s: closing %d\n",
		1996	__FUNCTION__, rq->bo->handle));
		1997	kgem_bo_free(kgem, rq->bo);
		1998	}
		1999	}
		2000
		2001	__kgem_request_free(rq);
		2002	return retired;
		2003	}
		2004
		2005	static bool kgem_retire__requests_ring(struct kgem *kgem, int ring)
		2006	{
		2007	bool retired = false;
		2008
		2009	while (!list_is_empty(&kgem->requests[ring])) {
		2010	struct kgem_request *rq;
		2011
		2012	rq = list_first_entry(&kgem->requests[ring],
		2013	struct kgem_request,
		2014	list);
		2015	if (__kgem_busy(kgem, rq->bo->handle))
		2016	break;
		2017
		2018	retired \|= __kgem_retire_rq(kgem, rq);
		2019	}
		2020
		2021	#if HAS_DEBUG_FULL
		2022	{
		2023	struct kgem_bo *bo;
		2024	int count = 0;
		2025
		2026	list_for_each_entry(bo, &kgem->requests[ring], request)
		2027	count++;
		2028
		2029	bo = NULL;
		2030	if (!list_is_empty(&kgem->requests[ring]))
		2031	bo = list_first_entry(&kgem->requests[ring],
		2032	struct kgem_request,
		2033	list)->bo;
		2034
		2035	ErrorF("%s: ring=%d, %d outstanding requests, oldest=%d\n",
		2036	__FUNCTION__, ring, count, bo ? bo->handle : 0);
		2037	}
		2038	#endif
		2039
		2040	return retired;
		2041	}
		2042
		2043	static bool kgem_retire__requests(struct kgem *kgem)
		2044	{
		2045	bool retired = false;
		2046	int n;
		2047
		2048	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2049	retired \|= kgem_retire__requests_ring(kgem, n);
		2050	kgem->need_retire \|= !list_is_empty(&kgem->requests[n]);
		2051	}
		2052
		2053	return retired;
		2054	}
		2055
		2056	bool kgem_retire(struct kgem *kgem)
		2057	{
		2058	bool retired = false;
		2059
		2060	DBG(("%s\n", __FUNCTION__));
		2061
		2062	kgem->need_retire = false;
		2063
		2064	retired \|= kgem_retire__flushing(kgem);
		2065	retired \|= kgem_retire__requests(kgem);
		2066	retired \|= kgem_retire__buffers(kgem);
		2067
		2068	DBG(("%s -- retired=%d, need_retire=%d\n",
		2069	__FUNCTION__, retired, kgem->need_retire));
		2070
		2071	kgem->retire(kgem);
		2072
		2073	return retired;
		2074	}
		2075
		2076	bool __kgem_ring_is_idle(struct kgem *kgem, int ring)
		2077	{
		2078	struct kgem_request *rq;
		2079
		2080	assert(!list_is_empty(&kgem->requests[ring]));
		2081
		2082	rq = list_last_entry(&kgem->requests[ring],
		2083	struct kgem_request, list);
		2084	if (__kgem_busy(kgem, rq->bo->handle)) {
		2085	DBG(("%s: last requests handle=%d still busy\n",
		2086	__FUNCTION__, rq->bo->handle));
		2087	return false;
		2088	}
		2089
		2090	DBG(("%s: ring=%d idle (handle=%d)\n",
		2091	__FUNCTION__, ring, rq->bo->handle));
		2092
		2093	kgem_retire__requests_ring(kgem, ring);
		2094	assert(list_is_empty(&kgem->requests[ring]));
		2095	return true;
		2096	}
		2097
		2098	static void kgem_commit(struct kgem *kgem)
		2099	{
		2100	struct kgem_request *rq = kgem->next_request;
		2101	struct kgem_bo bo, next;
		2102
		2103	list_for_each_entry_safe(bo, next, &rq->buffers, request) {
		2104	assert(next->request.prev == &bo->request);
		2105
		2106	DBG(("%s: release handle=%d (proxy? %d), dirty? %d flush? %d, snoop? %d -> offset=%x\n",
		2107	__FUNCTION__, bo->handle, bo->proxy != NULL,
		2108	bo->gpu_dirty, bo->needs_flush, bo->snoop,
		2109	(unsigned)bo->exec->offset));
		2110
		2111	assert(bo->exec);
		2112	assert(bo->proxy == NULL \|\| bo->exec == &_kgem_dummy_exec);
		2113	assert(RQ(bo->rq) == rq \|\| (RQ(bo->proxy->rq) == rq));
		2114
		2115	bo->presumed_offset = bo->exec->offset;
		2116	bo->exec = NULL;
		2117	bo->target_handle = -1;
		2118
		2119	if (!bo->refcnt && !bo->reusable) {
		2120	assert(!bo->snoop);
		2121	kgem_bo_free(kgem, bo);
		2122	continue;
		2123	}
		2124
		2125	bo->binding.offset = 0;
		2126	bo->domain = DOMAIN_GPU;
		2127	bo->gpu_dirty = false;
		2128
		2129	if (bo->proxy) {
		2130	/* proxies are not used for domain tracking */
		2131	bo->exec = NULL;
		2132	__kgem_bo_clear_busy(bo);
		2133	}
		2134
		2135	kgem->scanout_busy \|= bo->scanout;
		2136	}
		2137
		2138	if (rq == &kgem->static_request) {
		2139	struct drm_i915_gem_set_domain set_domain;
		2140
		2141	DBG(("%s: syncing due to allocation failure\n", __FUNCTION__));
		2142
		2143	VG_CLEAR(set_domain);
		2144	set_domain.handle = rq->bo->handle;
		2145	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2146	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2147	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
		2148	DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
		2149	kgem_throttle(kgem);
		2150	}
		2151
		2152	kgem_retire(kgem);
		2153	assert(list_is_empty(&rq->buffers));
		2154
		2155	assert(rq->bo->map == NULL);
		2156	gem_close(kgem->fd, rq->bo->handle);
		2157	kgem_cleanup_cache(kgem);
		2158	} else {
		2159	list_add_tail(&rq->list, &kgem->requests[rq->ring]);
		2160	kgem->need_throttle = kgem->need_retire = 1;
		2161	}
		2162
		2163	kgem->next_request = NULL;
		2164	}
		2165
		2166	static void kgem_close_list(struct kgem kgem, struct list head)
		2167	{
		2168	while (!list_is_empty(head))
		2169	kgem_bo_free(kgem, list_first_entry(head, struct kgem_bo, list));
		2170	}
		2171
		2172	static void kgem_close_inactive(struct kgem *kgem)
		2173	{
		2174	unsigned int i;
		2175
		2176	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		2177	kgem_close_list(kgem, &kgem->inactive[i]);
		2178	}
		2179
		2180	static void kgem_finish_buffers(struct kgem *kgem)
		2181	{
		2182	struct kgem_buffer bo, next;
		2183
		2184	list_for_each_entry_safe(bo, next, &kgem->batch_buffers, base.list) {
		2185	DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s\n",
		2186	__FUNCTION__, bo->base.handle, bo->used, bo->base.exec!=NULL,
		2187	bo->write, bo->mmapped ? IS_CPU_MAP(bo->base.map) ? "cpu" : "gtt" : "no"));
		2188
		2189	assert(next->base.list.prev == &bo->base.list);
		2190	assert(bo->base.io);
		2191	assert(bo->base.refcnt >= 1);
		2192
		2193	if (!bo->base.exec) {
		2194	DBG(("%s: skipping unattached handle=%d, used=%d\n",
		2195	__FUNCTION__, bo->base.handle, bo->used));
		2196	continue;
		2197	}
		2198
		2199	if (!bo->write) {
		2200	assert(bo->base.exec \|\| bo->base.refcnt > 1);
		2201	goto decouple;
		2202	}
		2203
		2204	if (bo->mmapped) {
		2205	int used;
		2206
		2207	assert(!bo->need_io);
		2208
		2209	used = ALIGN(bo->used, PAGE_SIZE);
		2210	if (!DBG_NO_UPLOAD_ACTIVE &&
		2211	used + PAGE_SIZE <= bytes(&bo->base) &&
		2212	(kgem->has_llc \|\| !IS_CPU_MAP(bo->base.map) \|\| bo->base.snoop)) {
		2213	DBG(("%s: retaining upload buffer (%d/%d)\n",
		2214	__FUNCTION__, bo->used, bytes(&bo->base)));
		2215	bo->used = used;
		2216	list_move(&bo->base.list,
		2217	&kgem->active_buffers);
		2218	continue;
		2219	}
		2220	DBG(("%s: discarding mmapped buffer, used=%d, map type=%d\n",
		2221	__FUNCTION__, bo->used, (int)__MAP_TYPE(bo->base.map)));
		2222	goto decouple;
		2223	}
		2224
		2225	if (!bo->used) {
		2226	/* Unless we replace the handle in the execbuffer,
		2227	* then this bo will become active. So decouple it
		2228	* from the buffer list and track it in the normal
		2229	* manner.
		2230	*/
		2231	goto decouple;
		2232	}
		2233
		2234	assert(bo->need_io);
		2235	assert(bo->base.rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		2236	assert(bo->base.domain != DOMAIN_GPU);
		2237
		2238	if (bo->base.refcnt == 1 &&
		2239	bo->base.size.pages.count > 1 &&
		2240	bo->used < bytes(&bo->base) / 2) {
		2241	struct kgem_bo *shrink;
		2242	unsigned alloc = NUM_PAGES(bo->used);
		2243
		2244	shrink = search_snoop_cache(kgem, alloc,
		2245	CREATE_INACTIVE \| CREATE_NO_RETIRE);
		2246	if (shrink) {
		2247	void *map;
		2248	int n;
		2249
		2250	DBG(("%s: used=%d, shrinking %d to %d, handle %d to %d\n",
		2251	__FUNCTION__,
		2252	bo->used, bytes(&bo->base), bytes(shrink),
		2253	bo->base.handle, shrink->handle));
		2254
		2255	assert(bo->used <= bytes(shrink));
		2256	map = kgem_bo_map__cpu(kgem, shrink);
		2257	if (map) {
		2258	kgem_bo_sync__cpu(kgem, shrink);
		2259	memcpy(map, bo->mem, bo->used);
		2260
		2261	shrink->target_handle =
		2262	kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
		2263	for (n = 0; n < kgem->nreloc; n++) {
		2264	if (kgem->reloc[n].target_handle == bo->base.target_handle) {
		2265	kgem->reloc[n].target_handle = shrink->target_handle;
		2266	kgem->reloc[n].presumed_offset = shrink->presumed_offset;
		2267	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		2268	kgem->reloc[n].delta + shrink->presumed_offset;
		2269	}
		2270	}
		2271
		2272	bo->base.exec->handle = shrink->handle;
		2273	bo->base.exec->offset = shrink->presumed_offset;
		2274	shrink->exec = bo->base.exec;
		2275	shrink->rq = bo->base.rq;
		2276	list_replace(&bo->base.request,
		2277	&shrink->request);
		2278	list_init(&bo->base.request);
		2279	shrink->needs_flush = bo->base.gpu_dirty;
		2280
		2281	bo->base.exec = NULL;
		2282	bo->base.rq = NULL;
		2283	bo->base.gpu_dirty = false;
		2284	bo->base.needs_flush = false;
		2285	bo->used = 0;
		2286
		2287	goto decouple;
		2288	}
		2289
		2290	__kgem_bo_destroy(kgem, shrink);
		2291	}
		2292
		2293	shrink = search_linear_cache(kgem, alloc,
		2294	CREATE_INACTIVE \| CREATE_NO_RETIRE);
		2295	if (shrink) {
		2296	int n;
		2297
		2298	DBG(("%s: used=%d, shrinking %d to %d, handle %d to %d\n",
		2299	__FUNCTION__,
		2300	bo->used, bytes(&bo->base), bytes(shrink),
		2301	bo->base.handle, shrink->handle));
		2302
		2303	assert(bo->used <= bytes(shrink));
		2304	if (gem_write(kgem->fd, shrink->handle,
		2305	0, bo->used, bo->mem) == 0) {
		2306	shrink->target_handle =
		2307	kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
		2308	for (n = 0; n < kgem->nreloc; n++) {
		2309	if (kgem->reloc[n].target_handle == bo->base.target_handle) {
		2310	kgem->reloc[n].target_handle = shrink->target_handle;
		2311	kgem->reloc[n].presumed_offset = shrink->presumed_offset;
		2312	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		2313	kgem->reloc[n].delta + shrink->presumed_offset;
		2314	}
		2315	}
		2316
		2317	bo->base.exec->handle = shrink->handle;
		2318	bo->base.exec->offset = shrink->presumed_offset;
		2319	shrink->exec = bo->base.exec;
		2320	shrink->rq = bo->base.rq;
		2321	list_replace(&bo->base.request,
		2322	&shrink->request);
		2323	list_init(&bo->base.request);
		2324	shrink->needs_flush = bo->base.gpu_dirty;
		2325
		2326	bo->base.exec = NULL;
		2327	bo->base.rq = NULL;
		2328	bo->base.gpu_dirty = false;
		2329	bo->base.needs_flush = false;
		2330	bo->used = 0;
		2331
		2332	goto decouple;
		2333	}
		2334
		2335	__kgem_bo_destroy(kgem, shrink);
		2336	}
		2337	}
		2338
		2339	DBG(("%s: handle=%d, uploading %d/%d\n",
		2340	__FUNCTION__, bo->base.handle, bo->used, bytes(&bo->base)));
		2341	ASSERT_IDLE(kgem, bo->base.handle);
		2342	assert(bo->used <= bytes(&bo->base));
		2343	gem_write(kgem->fd, bo->base.handle,
		2344	0, bo->used, bo->mem);
		2345	bo->need_io = 0;
		2346
		2347	decouple:
		2348	DBG(("%s: releasing handle=%d\n",
		2349	__FUNCTION__, bo->base.handle));
		2350	list_del(&bo->base.list);
		2351	kgem_bo_unref(kgem, &bo->base);
		2352	}
		2353	}
		2354
		2355	static void kgem_cleanup(struct kgem *kgem)
		2356	{
		2357	int n;
		2358
		2359	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2360	while (!list_is_empty(&kgem->requests[n])) {
		2361	struct kgem_request *rq;
		2362
		2363	rq = list_first_entry(&kgem->requests[n],
		2364	struct kgem_request,
		2365	list);
		2366	while (!list_is_empty(&rq->buffers)) {
		2367	struct kgem_bo *bo;
		2368
		2369	bo = list_first_entry(&rq->buffers,
		2370	struct kgem_bo,
		2371	request);
		2372
		2373	bo->exec = NULL;
		2374	bo->gpu_dirty = false;
		2375	__kgem_bo_clear_busy(bo);
		2376	if (bo->refcnt == 0)
		2377	kgem_bo_free(kgem, bo);
		2378	}
		2379
		2380	__kgem_request_free(rq);
		2381	}
		2382	}
		2383
		2384	kgem_close_inactive(kgem);
		2385	}
		2386
		2387	static int kgem_batch_write(struct kgem *kgem, uint32_t handle, uint32_t size)
		2388	{
		2389	int ret;
		2390
		2391	ASSERT_IDLE(kgem, handle);
		2392
		2393	/* If there is no surface data, just upload the batch */
		2394	if (kgem->surface == kgem->batch_size)
		2395	return gem_write(kgem->fd, handle,
		2396	0, sizeof(uint32_t)*kgem->nbatch,
		2397	kgem->batch);
		2398
		2399	/* Are the batch pages conjoint with the surface pages? */
		2400	if (kgem->surface < kgem->nbatch + PAGE_SIZE/sizeof(uint32_t)) {
		2401	assert(size == PAGE_ALIGN(kgem->batch_size*sizeof(uint32_t)));
		2402	return gem_write(kgem->fd, handle,
		2403	0, kgem->batch_size*sizeof(uint32_t),
		2404	kgem->batch);
		2405	}
		2406
		2407	/* Disjoint surface/batch, upload separately */
		2408	ret = gem_write(kgem->fd, handle,
		2409	0, sizeof(uint32_t)*kgem->nbatch,
		2410	kgem->batch);
		2411	if (ret)
		2412	return ret;
		2413
		2414	ret = PAGE_ALIGN(sizeof(uint32_t) * kgem->batch_size);
		2415	ret -= sizeof(uint32_t) * kgem->surface;
		2416	assert(size-ret >= kgem->nbatch*sizeof(uint32_t));
		2417	return __gem_write(kgem->fd, handle,
		2418	size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t),
		2419	kgem->batch + kgem->surface);
		2420	}
		2421
		2422	void kgem_reset(struct kgem *kgem)
		2423	{
		2424	if (kgem->next_request) {
		2425	struct kgem_request *rq = kgem->next_request;
		2426
		2427	while (!list_is_empty(&rq->buffers)) {
		2428	struct kgem_bo *bo =
		2429	list_first_entry(&rq->buffers,
		2430	struct kgem_bo,
		2431	request);
		2432	list_del(&bo->request);
		2433
		2434	assert(RQ(bo->rq) == rq);
		2435
		2436	bo->binding.offset = 0;
		2437	bo->exec = NULL;
		2438	bo->target_handle = -1;
		2439	bo->gpu_dirty = false;
		2440
		2441	if (bo->needs_flush && __kgem_busy(kgem, bo->handle)) {
		2442	assert(bo->domain == DOMAIN_GPU \|\| bo->domain == DOMAIN_NONE);
		2443	list_add(&bo->request, &kgem->flushing);
		2444	bo->rq = (void *)kgem;
		2445	} else
		2446	__kgem_bo_clear_busy(bo);
		2447
		2448	if (bo->refcnt \|\| bo->rq)
		2449	continue;
		2450
		2451	if (bo->snoop) {
		2452	kgem_bo_move_to_snoop(kgem, bo);
		2453	} else if (bo->scanout) {
		2454	kgem_bo_move_to_scanout(kgem, bo);
		2455	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		2456	kgem_bo_set_purgeable(kgem, bo)) {
		2457	kgem_bo_move_to_inactive(kgem, bo);
		2458	} else {
		2459	DBG(("%s: closing %d\n",
		2460	__FUNCTION__, bo->handle));
		2461	kgem_bo_free(kgem, bo);
		2462	}
		2463	}
		2464
		2465	if (rq != &kgem->static_request) {
		2466	list_init(&rq->list);
		2467	__kgem_request_free(rq);
		2468	}
		2469	}
		2470
		2471	kgem->nfence = 0;
		2472	kgem->nexec = 0;
		2473	kgem->nreloc = 0;
		2474	kgem->nreloc__self = 0;
		2475	kgem->aperture = 0;
		2476	kgem->aperture_fenced = 0;
		2477	kgem->nbatch = 0;
		2478	kgem->surface = kgem->batch_size;
		2479	kgem->mode = KGEM_NONE;
		2480	kgem->flush = 0;
		2481	kgem->batch_flags = kgem->batch_flags_base;
		2482
		2483	kgem->next_request = __kgem_request_alloc(kgem);
		2484
		2485	kgem_sna_reset(kgem);
		2486	}
		2487
		2488	static int compact_batch_surface(struct kgem *kgem)
		2489	{
		2490	int size, shrink, n;
		2491
		2492	if (!kgem->has_relaxed_delta)
		2493	return kgem->batch_size;
		2494
		2495	/* See if we can pack the contents into one or two pages */
		2496	n = ALIGN(kgem->batch_size, 1024);
		2497	size = n - kgem->surface + kgem->nbatch;
		2498	size = ALIGN(size, 1024);
		2499
		2500	shrink = n - size;
		2501	if (shrink) {
		2502	DBG(("shrinking from %d to %d\n", kgem->batch_size, size));
		2503
		2504	shrink *= sizeof(uint32_t);
		2505	for (n = 0; n < kgem->nreloc; n++) {
		2506	if (kgem->reloc[n].read_domains == I915_GEM_DOMAIN_INSTRUCTION &&
		2507	kgem->reloc[n].target_handle == ~0U)
		2508	kgem->reloc[n].delta -= shrink;
		2509
		2510	if (kgem->reloc[n].offset >= sizeof(uint32_t)*kgem->nbatch)
		2511	kgem->reloc[n].offset -= shrink;
		2512	}
		2513	}
		2514
		2515	return size * sizeof(uint32_t);
		2516	}
		2517
		2518	static struct kgem_bo *
		2519	kgem_create_batch(struct kgem *kgem, int size)
		2520	{
		2521	struct drm_i915_gem_set_domain set_domain;
		2522	struct kgem_bo *bo;
		2523
		2524	if (size <= 4096) {
		2525	bo = list_first_entry(&kgem->pinned_batches[0],
		2526	struct kgem_bo,
		2527	list);
		2528	if (!bo->rq) {
		2529	out_4096:
		2530	list_move_tail(&bo->list, &kgem->pinned_batches[0]);
		2531	return kgem_bo_reference(bo);
		2532	}
		2533
		2534	if (!__kgem_busy(kgem, bo->handle)) {
		2535	assert(RQ(bo->rq)->bo == bo);
		2536	__kgem_retire_rq(kgem, RQ(bo->rq));
		2537	goto out_4096;
		2538	}
		2539	}
		2540
		2541	if (size <= 16384) {
		2542	bo = list_first_entry(&kgem->pinned_batches[1],
		2543	struct kgem_bo,
		2544	list);
		2545	if (!bo->rq) {
		2546	out_16384:
		2547	list_move_tail(&bo->list, &kgem->pinned_batches[1]);
		2548	return kgem_bo_reference(bo);
		2549	}
		2550
		2551	if (!__kgem_busy(kgem, bo->handle)) {
		2552	assert(RQ(bo->rq)->bo == bo);
		2553	__kgem_retire_rq(kgem, RQ(bo->rq));
		2554	goto out_16384;
		2555	}
		2556	}
		2557
		2558	if (kgem->gen == 020 && !kgem->has_pinned_batches) {
		2559	assert(size <= 16384);
		2560
		2561	bo = list_first_entry(&kgem->pinned_batches[size > 4096],
		2562	struct kgem_bo,
		2563	list);
		2564	list_move_tail(&bo->list, &kgem->pinned_batches[size > 4096]);
		2565
		2566	DBG(("%s: syncing due to busy batches\n", __FUNCTION__));
		2567
		2568	VG_CLEAR(set_domain);
		2569	set_domain.handle = bo->handle;
		2570	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2571	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2572	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
		2573	DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
		2574	kgem_throttle(kgem);
		2575	return NULL;
		2576	}
		2577
		2578	kgem_retire(kgem);
		2579	assert(bo->rq == NULL);
		2580	return kgem_bo_reference(bo);
		2581	}
		2582
		2583	return kgem_create_linear(kgem, size, CREATE_NO_THROTTLE);
		2584	}
		2585
		2586	void _kgem_submit(struct kgem *kgem)
		2587	{
		2588	struct kgem_request *rq;
		2589	uint32_t batch_end;
		2590	int size;
		2591
		2592	assert(!DBG_NO_HW);
		2593	assert(!kgem->wedged);
		2594
		2595	assert(kgem->nbatch);
		2596	assert(kgem->nbatch <= KGEM_BATCH_SIZE(kgem));
		2597	assert(kgem->nbatch <= kgem->surface);
		2598
		2599	batch_end = kgem_end_batch(kgem);
		2600	kgem_sna_flush(kgem);
		2601
		2602	DBG(("batch[%d/%d, flags=%x]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n",
		2603	kgem->mode, kgem->ring, kgem->batch_flags,
		2604	batch_end, kgem->nbatch, kgem->surface, kgem->batch_size,
		2605	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture));
		2606
		2607	assert(kgem->nbatch <= kgem->batch_size);
		2608	assert(kgem->nbatch <= kgem->surface);
		2609	assert(kgem->nreloc <= ARRAY_SIZE(kgem->reloc));
		2610	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		2611	assert(kgem->nfence <= kgem->fence_max);
		2612
		2613	kgem_finish_buffers(kgem);
		2614
		2615	#if SHOW_BATCH
		2616	__kgem_batch_debug(kgem, batch_end);
		2617	#endif
		2618
		2619	rq = kgem->next_request;
		2620	if (kgem->surface != kgem->batch_size)
		2621	size = compact_batch_surface(kgem);
		2622	else
		2623	size = kgem->nbatch * sizeof(kgem->batch[0]);
		2624	rq->bo = kgem_create_batch(kgem, size);
		2625	if (rq->bo) {
		2626	uint32_t handle = rq->bo->handle;
		2627	int i;
		2628
		2629	assert(!rq->bo->needs_flush);
		2630
		2631	i = kgem->nexec++;
		2632	kgem->exec[i].handle = handle;
		2633	kgem->exec[i].relocation_count = kgem->nreloc;
		2634	kgem->exec[i].relocs_ptr = (uintptr_t)kgem->reloc;
		2635	kgem->exec[i].alignment = 0;
		2636	kgem->exec[i].offset = rq->bo->presumed_offset;
		2637	kgem->exec[i].flags = 0;
		2638	kgem->exec[i].rsvd1 = 0;
		2639	kgem->exec[i].rsvd2 = 0;
		2640
		2641	rq->bo->target_handle = kgem->has_handle_lut ? i : handle;
		2642	rq->bo->exec = &kgem->exec[i];
		2643	rq->bo->rq = MAKE_REQUEST(rq, kgem->ring); /* useful sanity check */
		2644	list_add(&rq->bo->request, &rq->buffers);
		2645	rq->ring = kgem->ring == KGEM_BLT;
		2646
		2647	kgem_fixup_self_relocs(kgem, rq->bo);
		2648
		2649	if (kgem_batch_write(kgem, handle, size) == 0) {
		2650	struct drm_i915_gem_execbuffer2 execbuf;
		2651	int ret, retry = 3;
		2652
		2653	memset(&execbuf, 0, sizeof(execbuf));
		2654	execbuf.buffers_ptr = (uintptr_t)kgem->exec;
		2655	execbuf.buffer_count = kgem->nexec;
		2656	execbuf.batch_len = batch_end*sizeof(uint32_t);
		2657	execbuf.flags = kgem->ring \| kgem->batch_flags;
		2658
		2659	if (DEBUG_DUMP)
		2660	{
		2661	int fd = open("/tmp1/1/batchbuffer.bin", O_CREAT\|O_WRONLY\|O_BINARY);
		2662	if (fd != -1) {
		2663	write(fd, kgem->batch, size);
		2664	close(fd);
		2665	}
		2666	else printf("SNA: failed to write batchbuffer\n");
		2667	asm volatile("int3");
		2668	}
		2669
		2670	ret = drmIoctl(kgem->fd,
		2671	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2672	&execbuf);
		2673	while (ret == -1 && errno == EBUSY && retry--) {
		2674	__kgem_throttle(kgem);
		2675	ret = drmIoctl(kgem->fd,
		2676	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2677	&execbuf);
		2678	}
		2679	if (DEBUG_SYNC && ret == 0) {
		2680	struct drm_i915_gem_set_domain set_domain;
		2681
		2682	VG_CLEAR(set_domain);
		2683	set_domain.handle = handle;
		2684	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2685	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2686
		2687	ret = drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
		2688	}
		2689	if (ret == -1) {
		2690	DBG(("%s: GPU hang detected [%d]\n",
		2691	__FUNCTION__, errno));
		2692	kgem_throttle(kgem);
		2693	kgem->wedged = true;
		2694
		2695	#if 0
		2696	ret = errno;
		2697	ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d: errno=%d\n",
		2698	kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
		2699	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, errno);
		2700
		2701	for (i = 0; i < kgem->nexec; i++) {
		2702	struct kgem_bo bo, found = NULL;
		2703
		2704	list_for_each_entry(bo, &kgem->next_request->buffers, request) {
		2705	if (bo->handle == kgem->exec[i].handle) {
		2706	found = bo;
		2707	break;
		2708	}
		2709	}
		2710	ErrorF("exec[%d] = handle:%d, presumed offset: %x, size: %d, tiling %d, fenced %d, snooped %d, deleted %d\n",
		2711	i,
		2712	kgem->exec[i].handle,
		2713	(int)kgem->exec[i].offset,
		2714	found ? kgem_bo_size(found) : -1,
		2715	found ? found->tiling : -1,
		2716	(int)(kgem->exec[i].flags & EXEC_OBJECT_NEEDS_FENCE),
		2717	found ? found->snoop : -1,
		2718	found ? found->purged : -1);
		2719	}
		2720	for (i = 0; i < kgem->nreloc; i++) {
		2721	ErrorF("reloc[%d] = pos:%d, target:%d, delta:%d, read:%x, write:%x, offset:%x\n",
		2722	i,
		2723	(int)kgem->reloc[i].offset,
		2724	kgem->reloc[i].target_handle,
		2725	kgem->reloc[i].delta,
		2726	kgem->reloc[i].read_domains,
		2727	kgem->reloc[i].write_domain,
		2728	(int)kgem->reloc[i].presumed_offset);
		2729	}
		2730
		2731	if (DEBUG_SYNC) {
		2732	int fd = open("/tmp/batchbuffer", O_WRONLY \| O_CREAT \| O_APPEND, 0666);
		2733	if (fd != -1) {
		2734	write(fd, kgem->batch, batch_end*sizeof(uint32_t));
		2735	close(fd);
		2736	}
		2737
		2738	FatalError("SNA: failed to submit batchbuffer, errno=%d\n", ret);
		2739	}
		2740	#endif
		2741	}
		2742	}
		2743
		2744	kgem_commit(kgem);
		2745	}
		2746	if (kgem->wedged)
		2747	kgem_cleanup(kgem);
		2748
		2749	kgem_reset(kgem);
		2750
		2751	assert(kgem->next_request != NULL);
		2752	}
		2753
		2754	void kgem_throttle(struct kgem *kgem)
		2755	{
		2756	kgem->need_throttle = 0;
		2757	if (kgem->wedged)
		2758	return;
		2759
		2760	kgem->wedged = __kgem_throttle(kgem);
		2761	if (kgem->wedged) {
		2762	printf("Detected a hung GPU, disabling acceleration.\n");
		2763	printf("When reporting this, please include i915_error_state from debugfs and the full dmesg.\n");
		2764	}
		2765	}
		2766
		2767	void kgem_purge_cache(struct kgem *kgem)
		2768	{
		2769	struct kgem_bo bo, next;
		2770	int i;
		2771
		2772	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2773	list_for_each_entry_safe(bo, next, &kgem->inactive[i], list) {
		2774	if (!kgem_bo_is_retained(kgem, bo)) {
		2775	DBG(("%s: purging %d\n",
		2776	__FUNCTION__, bo->handle));
		2777	kgem_bo_free(kgem, bo);
		2778	}
		2779	}
		2780	}
		2781
		2782	kgem->need_purge = false;
		2783	}
		2784
		2785
		2786	void kgem_clean_large_cache(struct kgem *kgem)
		2787	{
		2788	while (!list_is_empty(&kgem->large_inactive)) {
		2789	kgem_bo_free(kgem,
		2790	list_first_entry(&kgem->large_inactive,
		2791	struct kgem_bo, list));
		2792
		2793	}
		2794	}
		2795
		2796	bool kgem_expire_cache(struct kgem *kgem)
		2797	{
		2798	time_t now, expire;
		2799	struct kgem_bo *bo;
		2800	unsigned int size = 0, count = 0;
		2801	bool idle;
		2802	unsigned int i;
		2803
		2804	time(&now);
		2805
		2806	while (__kgem_freed_bo) {
		2807	bo = __kgem_freed_bo;
		2808	__kgem_freed_bo = (struct kgem_bo *)bo;
		2809	free(bo);
		2810	}
		2811
		2812	while (__kgem_freed_request) {
		2813	struct kgem_request *rq = __kgem_freed_request;
		2814	__kgem_freed_request = (struct kgem_request *)rq;
		2815	free(rq);
		2816	}
		2817
		2818	kgem_clean_large_cache(kgem);
		2819
		2820	expire = 0;
		2821	list_for_each_entry(bo, &kgem->snoop, list) {
		2822	if (bo->delta) {
		2823	expire = now - MAX_INACTIVE_TIME/2;
		2824	break;
		2825	}
		2826
		2827	bo->delta = now;
		2828	}
		2829	if (expire) {
		2830	while (!list_is_empty(&kgem->snoop)) {
		2831	bo = list_last_entry(&kgem->snoop, struct kgem_bo, list);
		2832
		2833	if (bo->delta > expire)
		2834	break;
		2835
		2836	kgem_bo_free(kgem, bo);
		2837	}
		2838	}
		2839	#ifdef DEBUG_MEMORY
		2840	{
		2841	long snoop_size = 0;
		2842	int snoop_count = 0;
		2843	list_for_each_entry(bo, &kgem->snoop, list)
		2844	snoop_count++, snoop_size += bytes(bo);
		2845	ErrorF("%s: still allocated %d bo, %ld bytes, in snoop cache\n",
		2846	__FUNCTION__, snoop_count, snoop_size);
		2847	}
		2848	#endif
		2849
		2850	kgem_retire(kgem);
		2851	if (kgem->wedged)
		2852	kgem_cleanup(kgem);
		2853
		2854	kgem->expire(kgem);
		2855
		2856	if (kgem->need_purge)
		2857	kgem_purge_cache(kgem);
		2858
		2859	expire = 0;
		2860
		2861	idle = !kgem->need_retire;
		2862	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2863	idle &= list_is_empty(&kgem->inactive[i]);
		2864	list_for_each_entry(bo, &kgem->inactive[i], list) {
		2865	if (bo->delta) {
		2866	expire = now - MAX_INACTIVE_TIME;
		2867	break;
		2868	}
		2869
		2870	bo->delta = now;
		2871	}
		2872	}
		2873	if (idle) {
		2874	DBG(("%s: idle\n", __FUNCTION__));
		2875	kgem->need_expire = false;
		2876	return false;
		2877	}
		2878	if (expire == 0)
		2879	return true;
		2880
		2881	idle = !kgem->need_retire;
		2882	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2883	struct list preserve;
		2884
		2885	list_init(&preserve);
		2886	while (!list_is_empty(&kgem->inactive[i])) {
		2887	bo = list_last_entry(&kgem->inactive[i],
		2888	struct kgem_bo, list);
		2889
		2890	if (bo->delta > expire) {
		2891	idle = false;
		2892	break;
		2893	}
		2894
		2895	if (bo->map && bo->delta + MAP_PRESERVE_TIME > expire) {
		2896	idle = false;
		2897	list_move_tail(&bo->list, &preserve);
		2898	} else {
		2899	count++;
		2900	size += bytes(bo);
		2901	kgem_bo_free(kgem, bo);
		2902	DBG(("%s: expiring %d\n",
		2903	__FUNCTION__, bo->handle));
		2904	}
		2905	}
		2906	if (!list_is_empty(&preserve)) {
		2907	preserve.prev->next = kgem->inactive[i].next;
		2908	kgem->inactive[i].next->prev = preserve.prev;
		2909	kgem->inactive[i].next = preserve.next;
		2910	preserve.next->prev = &kgem->inactive[i];
		2911	}
		2912	}
		2913
		2914	#ifdef DEBUG_MEMORY
		2915	{
		2916	long inactive_size = 0;
		2917	int inactive_count = 0;
		2918	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		2919	list_for_each_entry(bo, &kgem->inactive[i], list)
		2920	inactive_count++, inactive_size += bytes(bo);
		2921	ErrorF("%s: still allocated %d bo, %ld bytes, in inactive cache\n",
		2922	__FUNCTION__, inactive_count, inactive_size);
		2923	}
		2924	#endif
		2925
		2926	DBG(("%s: expired %d objects, %d bytes, idle? %d\n",
		2927	__FUNCTION__, count, size, idle));
		2928
		2929	kgem->need_expire = !idle;
		2930	return !idle;
		2931	(void)count;
		2932	(void)size;
		2933	}
		2934
		2935	void kgem_cleanup_cache(struct kgem *kgem)
		2936	{
		2937	unsigned int i;
		2938	int n;
		2939
4368	Serge	2940	ENTER();
		2941
4304	Serge	2942	/* sync to the most recent request */
		2943	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2944	if (!list_is_empty(&kgem->requests[n])) {
		2945	struct kgem_request *rq;
		2946	struct drm_i915_gem_set_domain set_domain;
		2947
		2948	rq = list_first_entry(&kgem->requests[n],
		2949	struct kgem_request,
		2950	list);
		2951
		2952	DBG(("%s: sync on cleanup\n", __FUNCTION__));
		2953
		2954	VG_CLEAR(set_domain);
		2955	set_domain.handle = rq->bo->handle;
		2956	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2957	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2958	(void)drmIoctl(kgem->fd,
		2959	DRM_IOCTL_I915_GEM_SET_DOMAIN,
		2960	&set_domain);
		2961	}
		2962	}
		2963
		2964	kgem_retire(kgem);
		2965	kgem_cleanup(kgem);
		2966
		2967	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2968	while (!list_is_empty(&kgem->inactive[i]))
		2969	kgem_bo_free(kgem,
		2970	list_last_entry(&kgem->inactive[i],
		2971	struct kgem_bo, list));
		2972	}
		2973
		2974	kgem_clean_large_cache(kgem);
		2975
		2976	while (!list_is_empty(&kgem->snoop))
		2977	kgem_bo_free(kgem,
		2978	list_last_entry(&kgem->snoop,
		2979	struct kgem_bo, list));
		2980
		2981	while (__kgem_freed_bo) {
		2982	struct kgem_bo *bo = __kgem_freed_bo;
		2983	__kgem_freed_bo = (struct kgem_bo *)bo;
		2984	free(bo);
		2985	}
		2986
		2987	kgem->need_purge = false;
		2988	kgem->need_expire = false;
4368	Serge	2989
		2990	LEAVE();
4304	Serge	2991	}
		2992
		2993	static struct kgem_bo *
		2994	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		2995	{
		2996	struct kgem_bo bo, first = NULL;
		2997	bool use_active = (flags & CREATE_INACTIVE) == 0;
		2998	struct list *cache;
		2999
		3000	DBG(("%s: num_pages=%d, flags=%x, use_active? %d, use_large=%d [max=%d]\n",
		3001	__FUNCTION__, num_pages, flags, use_active,
		3002	num_pages >= MAX_CACHE_SIZE / PAGE_SIZE,
		3003	MAX_CACHE_SIZE / PAGE_SIZE));
		3004
		3005	assert(num_pages);
		3006
		3007	if (num_pages >= MAX_CACHE_SIZE / PAGE_SIZE) {
		3008	DBG(("%s: searching large buffers\n", __FUNCTION__));
		3009	retry_large:
		3010	cache = use_active ? &kgem->large : &kgem->large_inactive;
		3011	list_for_each_entry_safe(bo, first, cache, list) {
		3012	assert(bo->refcnt == 0);
		3013	assert(bo->reusable);
		3014	assert(!bo->scanout);
		3015
		3016	if (num_pages > num_pages(bo))
		3017	goto discard;
		3018
		3019	if (bo->tiling != I915_TILING_NONE) {
		3020	if (use_active)
		3021	goto discard;
		3022
		3023	if (!gem_set_tiling(kgem->fd, bo->handle,
		3024	I915_TILING_NONE, 0))
		3025	goto discard;
		3026
		3027	bo->tiling = I915_TILING_NONE;
		3028	bo->pitch = 0;
		3029	}
		3030
		3031	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo))
		3032	goto discard;
		3033
		3034	list_del(&bo->list);
		3035	if (bo->rq == (void *)kgem)
		3036	list_del(&bo->request);
		3037
		3038	bo->delta = 0;
		3039	assert_tiling(kgem, bo);
		3040	return bo;
		3041
		3042	discard:
		3043	if (!use_active)
		3044	kgem_bo_free(kgem, bo);
		3045	}
		3046
		3047	if (use_active) {
		3048	use_active = false;
		3049	goto retry_large;
		3050	}
		3051
		3052	if (__kgem_throttle_retire(kgem, flags))
		3053	goto retry_large;
		3054
		3055	return NULL;
		3056	}
		3057
		3058	if (!use_active && list_is_empty(inactive(kgem, num_pages))) {
		3059	DBG(("%s: inactive and cache bucket empty\n",
		3060	__FUNCTION__));
		3061
		3062	if (flags & CREATE_NO_RETIRE) {
		3063	DBG(("%s: can not retire\n", __FUNCTION__));
		3064	return NULL;
		3065	}
		3066
		3067	if (list_is_empty(active(kgem, num_pages, I915_TILING_NONE))) {
		3068	DBG(("%s: active cache bucket empty\n", __FUNCTION__));
		3069	return NULL;
		3070	}
		3071
		3072	if (!__kgem_throttle_retire(kgem, flags)) {
		3073	DBG(("%s: nothing retired\n", __FUNCTION__));
		3074	return NULL;
		3075	}
		3076
		3077	if (list_is_empty(inactive(kgem, num_pages))) {
		3078	DBG(("%s: active cache bucket still empty after retire\n",
		3079	__FUNCTION__));
		3080	return NULL;
		3081	}
		3082	}
		3083
		3084	if (!use_active && flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3085	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3086	DBG(("%s: searching for inactive %s map\n",
		3087	__FUNCTION__, for_cpu ? "cpu" : "gtt"));
		3088	cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
		3089	list_for_each_entry(bo, cache, vma) {
		3090	assert(IS_CPU_MAP(bo->map) == for_cpu);
		3091	assert(bucket(bo) == cache_bucket(num_pages));
		3092	assert(bo->proxy == NULL);
		3093	assert(bo->rq == NULL);
		3094	assert(bo->exec == NULL);
		3095	assert(!bo->scanout);
		3096
		3097	if (num_pages > num_pages(bo)) {
		3098	DBG(("inactive too small: %d < %d\n",
		3099	num_pages(bo), num_pages));
		3100	continue;
		3101	}
		3102
		3103	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3104	kgem_bo_free(kgem, bo);
		3105	break;
		3106	}
		3107
		3108	if (I915_TILING_NONE != bo->tiling &&
		3109	!gem_set_tiling(kgem->fd, bo->handle,
		3110	I915_TILING_NONE, 0))
		3111	continue;
		3112
		3113	kgem_bo_remove_from_inactive(kgem, bo);
		3114
		3115	bo->tiling = I915_TILING_NONE;
		3116	bo->pitch = 0;
		3117	bo->delta = 0;
		3118	DBG((" %s: found handle=%d (num_pages=%d) in linear vma cache\n",
		3119	__FUNCTION__, bo->handle, num_pages(bo)));
		3120	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		3121	assert(!bo->needs_flush);
		3122	assert_tiling(kgem, bo);
		3123	ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
		3124	return bo;
		3125	}
		3126
		3127	if (flags & CREATE_EXACT)
		3128	return NULL;
		3129
		3130	if (flags & CREATE_CPU_MAP && !kgem->has_llc)
		3131	return NULL;
		3132	}
		3133
		3134	cache = use_active ? active(kgem, num_pages, I915_TILING_NONE) : inactive(kgem, num_pages);
		3135	list_for_each_entry(bo, cache, list) {
		3136	assert(bo->refcnt == 0);
		3137	assert(bo->reusable);
		3138	assert(!!bo->rq == !!use_active);
		3139	assert(bo->proxy == NULL);
		3140	assert(!bo->scanout);
		3141
		3142	if (num_pages > num_pages(bo))
		3143	continue;
		3144
		3145	if (use_active &&
		3146	kgem->gen <= 040 &&
		3147	bo->tiling != I915_TILING_NONE)
		3148	continue;
		3149
		3150	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3151	kgem_bo_free(kgem, bo);
		3152	break;
		3153	}
		3154
		3155	if (I915_TILING_NONE != bo->tiling) {
		3156	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP))
		3157	continue;
		3158
		3159	if (first)
		3160	continue;
		3161
		3162	if (!gem_set_tiling(kgem->fd, bo->handle,
		3163	I915_TILING_NONE, 0))
		3164	continue;
		3165
		3166	bo->tiling = I915_TILING_NONE;
		3167	bo->pitch = 0;
		3168	}
		3169
		3170	if (bo->map) {
		3171	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3172	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3173	if (IS_CPU_MAP(bo->map) != for_cpu) {
		3174	if (first != NULL)
		3175	break;
		3176
		3177	first = bo;
		3178	continue;
		3179	}
		3180	} else {
		3181	if (first != NULL)
		3182	break;
		3183
		3184	first = bo;
		3185	continue;
		3186	}
		3187	} else {
		3188	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3189	if (first != NULL)
		3190	break;
		3191
		3192	first = bo;
		3193	continue;
		3194	}
		3195	}
		3196
		3197	if (use_active)
		3198	kgem_bo_remove_from_active(kgem, bo);
		3199	else
		3200	kgem_bo_remove_from_inactive(kgem, bo);
		3201
		3202	assert(bo->tiling == I915_TILING_NONE);
		3203	bo->pitch = 0;
		3204	bo->delta = 0;
		3205	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		3206	__FUNCTION__, bo->handle, num_pages(bo),
		3207	use_active ? "active" : "inactive"));
		3208	assert(list_is_empty(&bo->list));
		3209	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		3210	assert(!bo->needs_flush \|\| use_active);
		3211	assert_tiling(kgem, bo);
		3212	ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
		3213	return bo;
		3214	}
		3215
		3216	if (first) {
		3217	assert(first->tiling == I915_TILING_NONE);
		3218
		3219	if (use_active)
		3220	kgem_bo_remove_from_active(kgem, first);
		3221	else
		3222	kgem_bo_remove_from_inactive(kgem, first);
		3223
		3224	first->pitch = 0;
		3225	first->delta = 0;
		3226	DBG((" %s: found handle=%d (near-miss) (num_pages=%d) in linear %s cache\n",
		3227	__FUNCTION__, first->handle, num_pages(first),
		3228	use_active ? "active" : "inactive"));
		3229	assert(list_is_empty(&first->list));
		3230	assert(use_active \|\| first->domain != DOMAIN_GPU);
		3231	assert(!first->needs_flush \|\| use_active);
		3232	ASSERT_MAYBE_IDLE(kgem, first->handle, !use_active);
		3233	return first;
		3234	}
		3235
		3236	return NULL;
		3237	}
		3238
		3239
		3240	struct kgem_bo kgem_create_linear(struct kgem kgem, int size, unsigned flags)
		3241	{
		3242	struct kgem_bo *bo;
		3243	uint32_t handle;
		3244
		3245	DBG(("%s(%d)\n", __FUNCTION__, size));
		3246	assert(size);
		3247
		3248	if (flags & CREATE_GTT_MAP && kgem->has_llc) {
		3249	flags &= ~CREATE_GTT_MAP;
		3250	flags \|= CREATE_CPU_MAP;
		3251	}
		3252
		3253	size = NUM_PAGES(size);
		3254	bo = search_linear_cache(kgem, size, CREATE_INACTIVE \| flags);
		3255	if (bo) {
		3256	assert(bo->domain != DOMAIN_GPU);
		3257	ASSERT_IDLE(kgem, bo->handle);
		3258	bo->refcnt = 1;
		3259	return bo;
		3260	}
		3261
		3262	if (flags & CREATE_CACHED)
		3263	return NULL;
		3264
		3265	handle = gem_create(kgem->fd, size);
		3266	if (handle == 0)
		3267	return NULL;
		3268
		3269	DBG(("%s: new handle=%d, num_pages=%d\n", __FUNCTION__, handle, size));
		3270	bo = __kgem_bo_alloc(handle, size);
		3271	if (bo == NULL) {
		3272	gem_close(kgem->fd, handle);
		3273	return NULL;
		3274	}
		3275
		3276	debug_alloc__bo(kgem, bo);
		3277	return bo;
		3278	}
		3279
		3280	inline int kgem_bo_fenced_size(struct kgem kgem, struct kgem_bo bo)
		3281	{
		3282	unsigned int size;
		3283
		3284	assert(bo->tiling);
		3285	assert_tiling(kgem, bo);
		3286	assert(kgem->gen < 040);
		3287
		3288	if (kgem->gen < 030)
		3289	size = 512 * 1024;
		3290	else
		3291	size = 1024 * 1024;
		3292	while (size < bytes(bo))
		3293	size *= 2;
		3294
		3295	return size;
		3296	}
		3297
		3298	struct kgem_bo kgem_create_2d(struct kgem kgem,
		3299	int width,
		3300	int height,
		3301	int bpp,
		3302	int tiling,
		3303	uint32_t flags)
		3304	{
		3305	struct list *cache;
		3306	struct kgem_bo *bo;
		3307	uint32_t pitch, tiled_height, size;
		3308	uint32_t handle;
		3309	int i, bucket, retry;
		3310	bool exact = flags & (CREATE_EXACT \| CREATE_SCANOUT);
		3311
		3312	if (tiling < 0)
		3313	exact = true, tiling = -tiling;
		3314
		3315
		3316	DBG(("%s(%dx%d, bpp=%d, tiling=%d, exact=%d, inactive=%d, cpu-mapping=%d, gtt-mapping=%d, scanout?=%d, prime?=%d, temp?=%d)\n", __FUNCTION__,
		3317	width, height, bpp, tiling, exact,
		3318	!!(flags & CREATE_INACTIVE),
		3319	!!(flags & CREATE_CPU_MAP),
		3320	!!(flags & CREATE_GTT_MAP),
		3321	!!(flags & CREATE_SCANOUT),
		3322	!!(flags & CREATE_PRIME),
		3323	!!(flags & CREATE_TEMPORARY)));
		3324
		3325	size = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags,
		3326	width, height, bpp, tiling, &pitch);
		3327	assert(size && size <= kgem->max_object_size);
		3328	size /= PAGE_SIZE;
		3329	bucket = cache_bucket(size);
		3330
		3331	if (flags & CREATE_SCANOUT) {
		3332	struct kgem_bo *last = NULL;
		3333
		3334	list_for_each_entry_reverse(bo, &kgem->scanout, list) {
		3335	assert(bo->scanout);
		3336	assert(bo->delta);
		3337	assert(!bo->flush);
		3338	assert_tiling(kgem, bo);
		3339
		3340	if (size > num_pages(bo) \|\| num_pages(bo) > 2*size)
		3341	continue;
		3342
		3343	if (bo->tiling != tiling \|\|
		3344	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3345	if (!gem_set_tiling(kgem->fd, bo->handle,
		3346	tiling, pitch))
		3347	continue;
		3348
		3349	bo->tiling = tiling;
		3350	bo->pitch = pitch;
		3351	}
		3352
		3353	if (flags & CREATE_INACTIVE && bo->rq) {
		3354	last = bo;
		3355	continue;
		3356	}
		3357
		3358	list_del(&bo->list);
		3359
		3360	bo->unique_id = kgem_get_unique_id(kgem);
		3361	DBG((" 1:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3362	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3363	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3364	assert_tiling(kgem, bo);
		3365	bo->refcnt = 1;
		3366	return bo;
		3367	}
		3368
		3369	if (last) {
		3370	list_del(&last->list);
		3371
		3372	last->unique_id = kgem_get_unique_id(kgem);
		3373	DBG((" 1:from scanout: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3374	last->pitch, last->tiling, last->handle, last->unique_id));
		3375	assert(last->pitch*kgem_aligned_height(kgem, height, last->tiling) <= kgem_bo_size(last));
		3376	assert_tiling(kgem, last);
		3377	last->refcnt = 1;
		3378	return last;
		3379	}
		3380
		3381	bo = NULL; //__kgem_bo_create_as_display(kgem, size, tiling, pitch);
		3382	if (bo)
		3383	return bo;
		3384	}
		3385
		3386	if (bucket >= NUM_CACHE_BUCKETS) {
		3387	DBG(("%s: large bo num pages=%d, bucket=%d\n",
		3388	__FUNCTION__, size, bucket));
		3389
		3390	if (flags & CREATE_INACTIVE)
		3391	goto large_inactive;
		3392
		3393	tiled_height = kgem_aligned_height(kgem, height, tiling);
		3394
		3395	list_for_each_entry(bo, &kgem->large, list) {
		3396	assert(!bo->purged);
		3397	assert(!bo->scanout);
		3398	assert(bo->refcnt == 0);
		3399	assert(bo->reusable);
		3400	assert_tiling(kgem, bo);
		3401
		3402	if (kgem->gen < 040) {
		3403	if (bo->pitch < pitch) {
		3404	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3405	bo->tiling, tiling,
		3406	bo->pitch, pitch));
		3407	continue;
		3408	}
		3409
		3410	if (bo->pitch * tiled_height > bytes(bo))
		3411	continue;
		3412	} else {
		3413	if (num_pages(bo) < size)
		3414	continue;
		3415
		3416	if (bo->pitch != pitch \|\| bo->tiling != tiling) {
		3417	if (!gem_set_tiling(kgem->fd, bo->handle,
		3418	tiling, pitch))
		3419	continue;
		3420
		3421	bo->pitch = pitch;
		3422	bo->tiling = tiling;
		3423	}
		3424	}
		3425
		3426	kgem_bo_remove_from_active(kgem, bo);
		3427
		3428	bo->unique_id = kgem_get_unique_id(kgem);
		3429	bo->delta = 0;
		3430	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3431	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3432	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3433	assert_tiling(kgem, bo);
		3434	bo->refcnt = 1;
		3435	bo->flush = true;
		3436	return bo;
		3437	}
		3438
		3439	large_inactive:
		3440	__kgem_throttle_retire(kgem, flags);
		3441	list_for_each_entry(bo, &kgem->large_inactive, list) {
		3442	assert(bo->refcnt == 0);
		3443	assert(bo->reusable);
		3444	assert(!bo->scanout);
		3445	assert_tiling(kgem, bo);
		3446
		3447	if (size > num_pages(bo))
		3448	continue;
		3449
		3450	if (bo->tiling != tiling \|\|
		3451	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3452	if (!gem_set_tiling(kgem->fd, bo->handle,
		3453	tiling, pitch))
		3454	continue;
		3455
		3456	bo->tiling = tiling;
		3457	bo->pitch = pitch;
		3458	}
		3459
		3460	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3461	kgem_bo_free(kgem, bo);
		3462	break;
		3463	}
		3464
		3465	list_del(&bo->list);
		3466
		3467	assert(bo->domain != DOMAIN_GPU);
		3468	bo->unique_id = kgem_get_unique_id(kgem);
		3469	bo->pitch = pitch;
		3470	bo->delta = 0;
		3471	DBG((" 1:from large inactive: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3472	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3473	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3474	assert_tiling(kgem, bo);
		3475	bo->refcnt = 1;
		3476	return bo;
		3477	}
		3478
		3479	goto create;
		3480	}
		3481
		3482	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3483	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3484	if (kgem->has_llc && tiling == I915_TILING_NONE)
		3485	for_cpu = 1;
		3486	/* We presume that we will need to upload to this bo,
		3487	* and so would prefer to have an active VMA.
		3488	*/
		3489	cache = &kgem->vma[for_cpu].inactive[bucket];
		3490	do {
		3491	list_for_each_entry(bo, cache, vma) {
		3492	assert(bucket(bo) == bucket);
		3493	assert(bo->refcnt == 0);
		3494	assert(!bo->scanout);
		3495	assert(bo->map);
		3496	assert(IS_CPU_MAP(bo->map) == for_cpu);
		3497	assert(bo->rq == NULL);
		3498	assert(list_is_empty(&bo->request));
		3499	assert(bo->flush == false);
		3500	assert_tiling(kgem, bo);
		3501
		3502	if (size > num_pages(bo)) {
		3503	DBG(("inactive too small: %d < %d\n",
		3504	num_pages(bo), size));
		3505	continue;
		3506	}
		3507
		3508	if (bo->tiling != tiling \|\|
		3509	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3510	DBG(("inactive vma with wrong tiling: %d < %d\n",
		3511	bo->tiling, tiling));
		3512	continue;
		3513	}
		3514
		3515	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3516	kgem_bo_free(kgem, bo);
		3517	break;
		3518	}
		3519
		3520	assert(bo->tiling == tiling);
		3521	bo->pitch = pitch;
		3522	bo->delta = 0;
		3523	bo->unique_id = kgem_get_unique_id(kgem);
		3524	bo->domain = DOMAIN_NONE;
		3525
		3526	kgem_bo_remove_from_inactive(kgem, bo);
		3527
		3528	DBG((" from inactive vma: pitch=%d, tiling=%d: handle=%d, id=%d\n",
		3529	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3530	assert(bo->reusable);
		3531	assert(bo->domain != DOMAIN_GPU);
		3532	ASSERT_IDLE(kgem, bo->handle);
		3533	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3534	assert_tiling(kgem, bo);
		3535	bo->refcnt = 1;
		3536	return bo;
		3537	}
		3538	} while (!list_is_empty(cache) &&
		3539	__kgem_throttle_retire(kgem, flags));
		3540
		3541	if (flags & CREATE_CPU_MAP && !kgem->has_llc) {
		3542	if (list_is_empty(&kgem->active[bucket][tiling]) &&
		3543	list_is_empty(&kgem->inactive[bucket]))
		3544	flags &= ~CREATE_CACHED;
		3545
		3546	goto create;
		3547	}
		3548	}
		3549
		3550	if (flags & CREATE_INACTIVE)
		3551	goto skip_active_search;
		3552
		3553	/* Best active match */
		3554	retry = NUM_CACHE_BUCKETS - bucket;
		3555	if (retry > 3 && (flags & CREATE_TEMPORARY) == 0)
		3556	retry = 3;
		3557	search_again:
		3558	assert(bucket < NUM_CACHE_BUCKETS);
		3559	cache = &kgem->active[bucket][tiling];
		3560	if (tiling) {
		3561	tiled_height = kgem_aligned_height(kgem, height, tiling);
		3562	list_for_each_entry(bo, cache, list) {
		3563	assert(!bo->purged);
		3564	assert(bo->refcnt == 0);
		3565	assert(bucket(bo) == bucket);
		3566	assert(bo->reusable);
		3567	assert(bo->tiling == tiling);
		3568	assert(bo->flush == false);
		3569	assert(!bo->scanout);
		3570	assert_tiling(kgem, bo);
		3571
		3572	if (kgem->gen < 040) {
		3573	if (bo->pitch < pitch) {
		3574	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3575	bo->tiling, tiling,
		3576	bo->pitch, pitch));
		3577	continue;
		3578	}
		3579
		3580	if (bo->pitch * tiled_height > bytes(bo))
		3581	continue;
		3582	} else {
		3583	if (num_pages(bo) < size)
		3584	continue;
		3585
		3586	if (bo->pitch != pitch) {
		3587	if (!gem_set_tiling(kgem->fd,
		3588	bo->handle,
		3589	tiling, pitch))
		3590	continue;
		3591
		3592	bo->pitch = pitch;
		3593	}
		3594	}
		3595
		3596	kgem_bo_remove_from_active(kgem, bo);
		3597
		3598	bo->unique_id = kgem_get_unique_id(kgem);
		3599	bo->delta = 0;
		3600	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3601	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3602	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3603	assert_tiling(kgem, bo);
		3604	bo->refcnt = 1;
		3605	return bo;
		3606	}
		3607	} else {
		3608	list_for_each_entry(bo, cache, list) {
		3609	assert(bucket(bo) == bucket);
		3610	assert(!bo->purged);
		3611	assert(bo->refcnt == 0);
		3612	assert(bo->reusable);
		3613	assert(!bo->scanout);
		3614	assert(bo->tiling == tiling);
		3615	assert(bo->flush == false);
		3616	assert_tiling(kgem, bo);
		3617
		3618	if (num_pages(bo) < size)
		3619	continue;
		3620
		3621	kgem_bo_remove_from_active(kgem, bo);
		3622
		3623	bo->pitch = pitch;
		3624	bo->unique_id = kgem_get_unique_id(kgem);
		3625	bo->delta = 0;
		3626	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3627	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3628	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3629	assert_tiling(kgem, bo);
		3630	bo->refcnt = 1;
		3631	return bo;
		3632	}
		3633	}
		3634
		3635	if (--retry && exact) {
		3636	if (kgem->gen >= 040) {
		3637	for (i = I915_TILING_NONE; i <= I915_TILING_Y; i++) {
		3638	if (i == tiling)
		3639	continue;
		3640
		3641	cache = &kgem->active[bucket][i];
		3642	list_for_each_entry(bo, cache, list) {
		3643	assert(!bo->purged);
		3644	assert(bo->refcnt == 0);
		3645	assert(bo->reusable);
		3646	assert(!bo->scanout);
		3647	assert(bo->flush == false);
		3648	assert_tiling(kgem, bo);
		3649
		3650	if (num_pages(bo) < size)
		3651	continue;
		3652
		3653	if (!gem_set_tiling(kgem->fd,
		3654	bo->handle,
		3655	tiling, pitch))
		3656	continue;
		3657
		3658	kgem_bo_remove_from_active(kgem, bo);
		3659
		3660	bo->unique_id = kgem_get_unique_id(kgem);
		3661	bo->pitch = pitch;
		3662	bo->tiling = tiling;
		3663	bo->delta = 0;
		3664	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3665	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3666	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3667	assert_tiling(kgem, bo);
		3668	bo->refcnt = 1;
		3669	return bo;
		3670	}
		3671	}
		3672	}
		3673
		3674	bucket++;
		3675	goto search_again;
		3676	}
		3677
		3678	if (!exact) { /* allow an active near-miss? */
		3679	i = tiling;
		3680	while (--i >= 0) {
		3681	tiled_height = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags,
		3682	width, height, bpp, tiling, &pitch);
		3683	cache = active(kgem, tiled_height / PAGE_SIZE, i);
		3684	tiled_height = kgem_aligned_height(kgem, height, i);
		3685	list_for_each_entry(bo, cache, list) {
		3686	assert(!bo->purged);
		3687	assert(bo->refcnt == 0);
		3688	assert(bo->reusable);
		3689	assert(!bo->scanout);
		3690	assert(bo->flush == false);
		3691	assert_tiling(kgem, bo);
		3692
		3693	if (bo->tiling) {
		3694	if (bo->pitch < pitch) {
		3695	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3696	bo->tiling, tiling,
		3697	bo->pitch, pitch));
		3698	continue;
		3699	}
		3700	} else
		3701	bo->pitch = pitch;
		3702
		3703	if (bo->pitch * tiled_height > bytes(bo))
		3704	continue;
		3705
		3706	kgem_bo_remove_from_active(kgem, bo);
		3707
		3708	bo->unique_id = kgem_get_unique_id(kgem);
		3709	bo->delta = 0;
		3710	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3711	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3712	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3713	assert_tiling(kgem, bo);
		3714	bo->refcnt = 1;
		3715	return bo;
		3716	}
		3717	}
		3718	}
		3719
		3720	skip_active_search:
		3721	bucket = cache_bucket(size);
		3722	retry = NUM_CACHE_BUCKETS - bucket;
		3723	if (retry > 3)
		3724	retry = 3;
		3725	search_inactive:
		3726	/* Now just look for a close match and prefer any currently active */
		3727	assert(bucket < NUM_CACHE_BUCKETS);
		3728	cache = &kgem->inactive[bucket];
		3729	list_for_each_entry(bo, cache, list) {
		3730	assert(bucket(bo) == bucket);
		3731	assert(bo->reusable);
		3732	assert(!bo->scanout);
		3733	assert(bo->flush == false);
		3734	assert_tiling(kgem, bo);
		3735
		3736	if (size > num_pages(bo)) {
		3737	DBG(("inactive too small: %d < %d\n",
		3738	num_pages(bo), size));
		3739	continue;
		3740	}
		3741
		3742	if (bo->tiling != tiling \|\|
		3743	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3744	if (!gem_set_tiling(kgem->fd, bo->handle,
		3745	tiling, pitch))
		3746	continue;
		3747
		3748	if (bo->map)
		3749	kgem_bo_release_map(kgem, bo);
		3750	}
		3751
		3752	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3753	kgem_bo_free(kgem, bo);
		3754	break;
		3755	}
		3756
		3757	kgem_bo_remove_from_inactive(kgem, bo);
		3758
		3759	bo->pitch = pitch;
		3760	bo->tiling = tiling;
		3761
		3762	bo->delta = 0;
		3763	bo->unique_id = kgem_get_unique_id(kgem);
		3764	assert(bo->pitch);
		3765	DBG((" from inactive: pitch=%d, tiling=%d: handle=%d, id=%d\n",
		3766	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3767	assert(bo->refcnt == 0);
		3768	assert(bo->reusable);
		3769	assert((flags & CREATE_INACTIVE) == 0 \|\| bo->domain != DOMAIN_GPU);
		3770	ASSERT_MAYBE_IDLE(kgem, bo->handle, flags & CREATE_INACTIVE);
		3771	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3772	assert_tiling(kgem, bo);
		3773	bo->refcnt = 1;
		3774	return bo;
		3775	}
		3776
		3777	if (flags & CREATE_INACTIVE &&
		3778	!list_is_empty(&kgem->active[bucket][tiling]) &&
		3779	__kgem_throttle_retire(kgem, flags)) {
		3780	flags &= ~CREATE_INACTIVE;
		3781	goto search_inactive;
		3782	}
		3783
		3784	if (--retry) {
		3785	bucket++;
		3786	flags &= ~CREATE_INACTIVE;
		3787	goto search_inactive;
		3788	}
		3789
		3790	create:
		3791	if (flags & CREATE_CACHED)
		3792	return NULL;
		3793
		3794	if (bucket >= NUM_CACHE_BUCKETS)
		3795	size = ALIGN(size, 1024);
		3796	handle = gem_create(kgem->fd, size);
		3797	if (handle == 0)
		3798	return NULL;
		3799
		3800	bo = __kgem_bo_alloc(handle, size);
		3801	if (!bo) {
		3802	gem_close(kgem->fd, handle);
		3803	return NULL;
		3804	}
		3805
		3806	if (bucket >= NUM_CACHE_BUCKETS) {
		3807	DBG(("%s: marking large bo for automatic flushing\n",
		3808	__FUNCTION__));
		3809	bo->flush = true;
		3810	}
		3811
		3812	bo->unique_id = kgem_get_unique_id(kgem);
		3813	if (tiling == I915_TILING_NONE \|\|
		3814	gem_set_tiling(kgem->fd, handle, tiling, pitch)) {
		3815	bo->tiling = tiling;
		3816	bo->pitch = pitch;
		3817	} else {
		3818	if (flags & CREATE_EXACT) {
		3819	if (bo->pitch != pitch \|\| bo->tiling != tiling) {
		3820	kgem_bo_free(kgem, bo);
		3821	return NULL;
		3822	}
		3823	}
		3824	}
		3825
		3826	assert(bytes(bo) >= bo->pitch * kgem_aligned_height(kgem, height, bo->tiling));
		3827	assert_tiling(kgem, bo);
		3828
		3829	debug_alloc__bo(kgem, bo);
		3830
		3831	DBG((" new pitch=%d, tiling=%d, handle=%d, id=%d, num_pages=%d [%d], bucket=%d\n",
		3832	bo->pitch, bo->tiling, bo->handle, bo->unique_id,
		3833	size, num_pages(bo), bucket(bo)));
		3834	return bo;
		3835	}
		3836
		3837	#if 0
		3838	struct kgem_bo kgem_create_cpu_2d(struct kgem kgem,
		3839	int width,
		3840	int height,
		3841	int bpp,
		3842	uint32_t flags)
		3843	{
		3844	struct kgem_bo *bo;
		3845	int stride, size;
		3846
		3847	if (DBG_NO_CPU)
		3848	return NULL;
		3849
		3850	DBG(("%s(%dx%d, bpp=%d)\n", __FUNCTION__, width, height, bpp));
		3851
		3852	if (kgem->has_llc) {
		3853	bo = kgem_create_2d(kgem, width, height, bpp,
		3854	I915_TILING_NONE, flags);
		3855	if (bo == NULL)
		3856	return bo;
		3857
		3858	assert(bo->tiling == I915_TILING_NONE);
		3859	assert_tiling(kgem, bo);
		3860
		3861	if (kgem_bo_map__cpu(kgem, bo) == NULL) {
		3862	kgem_bo_destroy(kgem, bo);
		3863	return NULL;
		3864	}
		3865
		3866	return bo;
		3867	}
		3868
		3869	assert(width > 0 && height > 0);
		3870	stride = ALIGN(width, 2) * bpp >> 3;
		3871	stride = ALIGN(stride, 4);
		3872	size = stride * ALIGN(height, 2);
		3873	assert(size >= PAGE_SIZE);
		3874
		3875	DBG(("%s: %dx%d, %d bpp, stride=%d\n",
		3876	__FUNCTION__, width, height, bpp, stride));
		3877
		3878	bo = search_snoop_cache(kgem, NUM_PAGES(size), 0);
		3879	if (bo) {
		3880	assert(bo->tiling == I915_TILING_NONE);
		3881	assert_tiling(kgem, bo);
		3882	assert(bo->snoop);
		3883	bo->refcnt = 1;
		3884	bo->pitch = stride;
		3885	bo->unique_id = kgem_get_unique_id(kgem);
		3886	return bo;
		3887	}
		3888
		3889	if (kgem->has_caching) {
		3890	bo = kgem_create_linear(kgem, size, flags);
		3891	if (bo == NULL)
		3892	return NULL;
		3893
		3894	assert(bo->tiling == I915_TILING_NONE);
		3895	assert_tiling(kgem, bo);
		3896
		3897	if (!gem_set_caching(kgem->fd, bo->handle, SNOOPED)) {
		3898	kgem_bo_destroy(kgem, bo);
		3899	return NULL;
		3900	}
		3901	bo->snoop = true;
		3902
		3903	if (kgem_bo_map__cpu(kgem, bo) == NULL) {
		3904	kgem_bo_destroy(kgem, bo);
		3905	return NULL;
		3906	}
		3907
		3908	bo->pitch = stride;
		3909	bo->unique_id = kgem_get_unique_id(kgem);
		3910	return bo;
		3911	}
		3912
		3913	if (kgem->has_userptr) {
		3914	void *ptr;
		3915
		3916	/* XXX */
		3917	//if (posix_memalign(&ptr, 64, ALIGN(size, 64)))
		3918	if (posix_memalign(&ptr, PAGE_SIZE, ALIGN(size, PAGE_SIZE)))
		3919	return NULL;
		3920
		3921	bo = kgem_create_map(kgem, ptr, size, false);
		3922	if (bo == NULL) {
		3923	free(ptr);
		3924	return NULL;
		3925	}
		3926
		3927	bo->pitch = stride;
		3928	bo->unique_id = kgem_get_unique_id(kgem);
		3929	return bo;
		3930	}
		3931
		3932	return NULL;
		3933	}
		3934	#endif
		3935
		3936	void _kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		3937	{
		3938	DBG(("%s: handle=%d, proxy? %d\n",
		3939	__FUNCTION__, bo->handle, bo->proxy != NULL));
		3940
		3941	if (bo->proxy) {
		3942	_list_del(&bo->vma);
		3943	_list_del(&bo->request);
		3944	if (bo->io && bo->exec == NULL)
		3945	_kgem_bo_delete_buffer(kgem, bo);
		3946	kgem_bo_unref(kgem, bo->proxy);
		3947	kgem_bo_binding_free(kgem, bo);
		3948	free(bo);
		3949	return;
		3950	}
		3951
		3952	__kgem_bo_destroy(kgem, bo);
		3953	}
		3954
		3955	static void __kgem_flush(struct kgem kgem, struct kgem_bo bo)
		3956	{
		3957	assert(bo->rq);
		3958	assert(bo->exec == NULL);
		3959	assert(bo->needs_flush);
		3960
		3961	/* The kernel will emit a flush and update its own flushing lists. */
		3962	if (!__kgem_busy(kgem, bo->handle))
		3963	__kgem_bo_clear_busy(bo);
		3964
		3965	DBG(("%s: handle=%d, busy?=%d\n",
		3966	__FUNCTION__, bo->handle, bo->rq != NULL));
		3967	}
		3968
		3969	void kgem_scanout_flush(struct kgem kgem, struct kgem_bo bo)
		3970	{
		3971	kgem_bo_submit(kgem, bo);
		3972	if (!bo->needs_flush)
		3973	return;
		3974
		3975	/* If the kernel fails to emit the flush, then it will be forced when
		3976	* we assume direct access. And as the usual failure is EIO, we do
		3977	* not actually care.
		3978	*/
		3979	assert(bo->exec == NULL);
		3980	if (bo->rq)
		3981	__kgem_flush(kgem, bo);
		3982
		3983	/* Whatever actually happens, we can regard the GTT write domain
		3984	* as being flushed.
		3985	*/
		3986	bo->gtt_dirty = false;
		3987	bo->needs_flush = false;
		3988	bo->domain = DOMAIN_NONE;
		3989	}
		3990
		3991	inline static bool needs_semaphore(struct kgem kgem, struct kgem_bo bo)
		3992	{
		3993	return kgem->nreloc && bo->rq && RQ_RING(bo->rq) != kgem->ring;
		3994	}
		3995
		3996	bool kgem_check_bo(struct kgem *kgem, ...)
		3997	{
		3998	va_list ap;
		3999	struct kgem_bo *bo;
		4000	int num_exec = 0;
		4001	int num_pages = 0;
		4002	bool flush = false;
		4003
		4004	va_start(ap, kgem);
		4005	while ((bo = va_arg(ap, struct kgem_bo *))) {
		4006	while (bo->proxy)
		4007	bo = bo->proxy;
		4008	if (bo->exec)
		4009	continue;
		4010
		4011	if (needs_semaphore(kgem, bo))
		4012	return false;
		4013
		4014	num_pages += num_pages(bo);
		4015	num_exec++;
		4016
		4017	flush \|= bo->flush;
		4018	}
		4019	va_end(ap);
		4020
		4021	DBG(("%s: num_pages=+%d, num_exec=+%d\n",
		4022	__FUNCTION__, num_pages, num_exec));
		4023
		4024	if (!num_pages)
		4025	return true;
		4026
		4027	if (kgem_flush(kgem, flush))
		4028	return false;
		4029
		4030	if (kgem->aperture > kgem->aperture_low &&
		4031	kgem_ring_is_idle(kgem, kgem->ring)) {
		4032	DBG(("%s: current aperture usage (%d) is greater than low water mark (%d)\n",
		4033	__FUNCTION__, kgem->aperture, kgem->aperture_low));
		4034	return false;
		4035	}
		4036
		4037	if (num_pages + kgem->aperture > kgem->aperture_high) {
		4038	DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n",
		4039	__FUNCTION__, num_pages + kgem->aperture, kgem->aperture_high));
		4040	return false;
		4041	}
		4042
		4043	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) {
		4044	DBG(("%s: out of exec slots (%d + %d / %d)\n", __FUNCTION__,
		4045	kgem->nexec, num_exec, KGEM_EXEC_SIZE(kgem)));
		4046	return false;
		4047	}
		4048
		4049	return true;
		4050	}
		4051
		4052
		4053
		4054
		4055
		4056
		4057
		4058
		4059
		4060
		4061
		4062
		4063
		4064
		4065
		4066
		4067
		4068
		4069
		4070
		4071
		4072
		4073
		4074
		4075
		4076
		4077
		4078
		4079
		4080
		4081	uint32_t kgem_add_reloc(struct kgem *kgem,
		4082	uint32_t pos,
		4083	struct kgem_bo *bo,
		4084	uint32_t read_write_domain,
		4085	uint32_t delta)
		4086	{
		4087	int index;
		4088
		4089	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
		4090	__FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
		4091
		4092	assert((read_write_domain & 0x7fff) == 0 \|\| bo != NULL);
		4093
		4094	if( bo != NULL && bo->handle == -2)
		4095	{
		4096	if (bo->exec == NULL)
		4097	kgem_add_bo(kgem, bo);
		4098
		4099	if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
		4100	__kgem_bo_mark_dirty(bo);
		4101	}
		4102	return 0;
		4103	};
		4104
		4105	index = kgem->nreloc++;
		4106	assert(index < ARRAY_SIZE(kgem->reloc));
		4107	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
		4108	if (bo) {
		4109	assert(bo->refcnt);
		4110	while (bo->proxy) {
		4111	DBG(("%s: adding proxy [delta=%d] for handle=%d\n",
		4112	__FUNCTION__, bo->delta, bo->handle));
		4113	delta += bo->delta;
		4114	assert(bo->handle == bo->proxy->handle);
		4115	/* need to release the cache upon batch submit */
		4116	if (bo->exec == NULL) {
		4117	list_move_tail(&bo->request,
		4118	&kgem->next_request->buffers);
		4119	bo->rq = MAKE_REQUEST(kgem->next_request,
		4120	kgem->ring);
		4121	bo->exec = &_kgem_dummy_exec;
		4122	}
		4123
		4124	if (read_write_domain & 0x7fff && !bo->gpu_dirty)
		4125	__kgem_bo_mark_dirty(bo);
		4126
		4127	bo = bo->proxy;
		4128	assert(bo->refcnt);
		4129	}
		4130	assert(bo->refcnt);
		4131
		4132	if (bo->exec == NULL)
		4133	kgem_add_bo(kgem, bo);
		4134	assert(bo->rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		4135	assert(RQ_RING(bo->rq) == kgem->ring);
		4136
		4137	if (kgem->gen < 040 && read_write_domain & KGEM_RELOC_FENCED) {
		4138	if (bo->tiling &&
		4139	(bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		4140	assert(kgem->nfence < kgem->fence_max);
		4141	kgem->aperture_fenced +=
		4142	kgem_bo_fenced_size(kgem, bo);
		4143	kgem->nfence++;
		4144	}
		4145	bo->exec->flags \|= EXEC_OBJECT_NEEDS_FENCE;
		4146	}
		4147
		4148	kgem->reloc[index].delta = delta;
		4149	kgem->reloc[index].target_handle = bo->target_handle;
		4150	kgem->reloc[index].presumed_offset = bo->presumed_offset;
		4151
		4152	if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
		4153	assert(!bo->snoop \|\| kgem->can_blt_cpu);
		4154	__kgem_bo_mark_dirty(bo);
		4155	}
		4156
		4157	delta += bo->presumed_offset;
		4158	} else {
		4159	kgem->reloc[index].delta = delta;
		4160	kgem->reloc[index].target_handle = ~0U;
		4161	kgem->reloc[index].presumed_offset = 0;
		4162	if (kgem->nreloc__self < 256)
		4163	kgem->reloc__self[kgem->nreloc__self++] = index;
		4164	}
		4165	kgem->reloc[index].read_domains = read_write_domain >> 16;
		4166	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
		4167
		4168	return delta;
		4169	}
		4170
		4171	static void kgem_trim_vma_cache(struct kgem *kgem, int type, int bucket)
		4172	{
		4173	int i, j;
		4174
		4175	DBG(("%s: type=%d, count=%d (bucket: %d)\n",
		4176	__FUNCTION__, type, kgem->vma[type].count, bucket));
		4177	if (kgem->vma[type].count <= 0)
		4178	return;
		4179
		4180	if (kgem->need_purge)
		4181	kgem_purge_cache(kgem);
		4182
		4183	/* vma are limited on a per-process basis to around 64k.
		4184	* This includes all malloc arenas as well as other file
		4185	* mappings. In order to be fair and not hog the cache,
		4186	* and more importantly not to exhaust that limit and to
		4187	* start failing mappings, we keep our own number of open
		4188	* vma to within a conservative value.
		4189	*/
		4190	i = 0;
		4191	while (kgem->vma[type].count > 0) {
		4192	struct kgem_bo *bo = NULL;
		4193
		4194	for (j = 0;
		4195	bo == NULL && j < ARRAY_SIZE(kgem->vma[type].inactive);
		4196	j++) {
		4197	struct list *head = &kgem->vma[type].inactive[i++%ARRAY_SIZE(kgem->vma[type].inactive)];
		4198	if (!list_is_empty(head))
		4199	bo = list_last_entry(head, struct kgem_bo, vma);
		4200	}
		4201	if (bo == NULL)
		4202	break;
		4203
		4204	DBG(("%s: discarding inactive %s vma cache for %d\n",
		4205	__FUNCTION__,
		4206	IS_CPU_MAP(bo->map) ? "CPU" : "GTT", bo->handle));
		4207	assert(IS_CPU_MAP(bo->map) == type);
		4208	assert(bo->map);
		4209	assert(bo->rq == NULL);
		4210
		4211	VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(bo->map), bytes(bo)));
		4212	// munmap(MAP(bo->map), bytes(bo));
		4213	bo->map = NULL;
		4214	list_del(&bo->vma);
		4215	kgem->vma[type].count--;
		4216
		4217	if (!bo->purged && !kgem_bo_set_purgeable(kgem, bo)) {
		4218	DBG(("%s: freeing unpurgeable old mapping\n",
		4219	__FUNCTION__));
		4220	kgem_bo_free(kgem, bo);
		4221	}
		4222	}
		4223	}
		4224
		4225	void kgem_bo_map__async(struct kgem kgem, struct kgem_bo *bo)
		4226	{
		4227	void *ptr;
		4228
		4229	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		4230	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		4231
		4232	assert(bo->proxy == NULL);
		4233	assert(list_is_empty(&bo->list));
		4234	assert(!IS_USER_MAP(bo->map));
		4235	assert_tiling(kgem, bo);
		4236
		4237	if (bo->tiling == I915_TILING_NONE && !bo->scanout && kgem->has_llc) {
		4238	DBG(("%s: converting request for GTT map into CPU map\n",
		4239	__FUNCTION__));
		4240	return kgem_bo_map__cpu(kgem, bo);
		4241	}
		4242
		4243	if (IS_CPU_MAP(bo->map))
		4244	kgem_bo_release_map(kgem, bo);
		4245
		4246	ptr = bo->map;
		4247	if (ptr == NULL) {
		4248	assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2);
		4249
		4250	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4251
		4252	ptr = __kgem_bo_map__gtt(kgem, bo);
		4253	if (ptr == NULL)
		4254	return NULL;
		4255
		4256	/* Cache this mapping to avoid the overhead of an
		4257	* excruciatingly slow GTT pagefault. This is more an
		4258	* issue with compositing managers which need to frequently
		4259	* flush CPU damage to their GPU bo.
		4260	*/
		4261	bo->map = ptr;
		4262	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4263	}
		4264
		4265	return ptr;
		4266	}
		4267
		4268	void kgem_bo_map(struct kgem kgem, struct kgem_bo *bo)
		4269	{
		4270	void *ptr;
		4271
		4272	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		4273	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		4274
		4275	assert(bo->proxy == NULL);
		4276	assert(list_is_empty(&bo->list));
		4277	assert(!IS_USER_MAP(bo->map));
		4278	assert(bo->exec == NULL);
		4279	assert_tiling(kgem, bo);
		4280
		4281	if (bo->tiling == I915_TILING_NONE && !bo->scanout &&
		4282	(kgem->has_llc \|\| bo->domain == DOMAIN_CPU)) {
		4283	DBG(("%s: converting request for GTT map into CPU map\n",
		4284	__FUNCTION__));
		4285	ptr = kgem_bo_map__cpu(kgem, bo);
		4286	if (ptr)
		4287	kgem_bo_sync__cpu(kgem, bo);
		4288	return ptr;
		4289	}
		4290
		4291	if (IS_CPU_MAP(bo->map))
		4292	kgem_bo_release_map(kgem, bo);
		4293
		4294	ptr = bo->map;
		4295	if (ptr == NULL) {
		4296	assert(kgem_bo_size(bo) <= kgem->aperture_mappable / 2);
		4297	assert(kgem->gen != 021 \|\| bo->tiling != I915_TILING_Y);
		4298
		4299	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4300
		4301	ptr = __kgem_bo_map__gtt(kgem, bo);
		4302	if (ptr == NULL)
		4303	return NULL;
		4304
		4305	/* Cache this mapping to avoid the overhead of an
		4306	* excruciatingly slow GTT pagefault. This is more an
		4307	* issue with compositing managers which need to frequently
		4308	* flush CPU damage to their GPU bo.
		4309	*/
		4310	bo->map = ptr;
		4311	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4312	}
		4313
		4314	if (bo->domain != DOMAIN_GTT \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_GTT)) {
		4315	struct drm_i915_gem_set_domain set_domain;
		4316
		4317	DBG(("%s: sync: needs_flush? %d, domain? %d, busy? %d\n", __FUNCTION__,
		4318	bo->needs_flush, bo->domain, __kgem_busy(kgem, bo->handle)));
		4319
		4320	/* XXX use PROT_READ to avoid the write flush? */
		4321
		4322	VG_CLEAR(set_domain);
		4323	set_domain.handle = bo->handle;
		4324	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		4325	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		4326	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4327	kgem_bo_retire(kgem, bo);
		4328	bo->domain = DOMAIN_GTT;
		4329	bo->gtt_dirty = true;
		4330	}
		4331	}
		4332
		4333	return ptr;
		4334	}
		4335
		4336	void kgem_bo_map__gtt(struct kgem kgem, struct kgem_bo *bo)
		4337	{
		4338	void *ptr;
		4339
		4340	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		4341	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		4342
		4343	assert(bo->exec == NULL);
		4344	assert(list_is_empty(&bo->list));
		4345	assert(!IS_USER_MAP(bo->map));
		4346	assert_tiling(kgem, bo);
		4347
		4348	if (IS_CPU_MAP(bo->map))
		4349	kgem_bo_release_map(kgem, bo);
		4350
		4351	ptr = bo->map;
		4352	if (ptr == NULL) {
		4353	assert(bytes(bo) <= kgem->aperture_mappable / 4);
		4354
		4355	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4356
		4357	ptr = __kgem_bo_map__gtt(kgem, bo);
		4358	if (ptr == NULL)
		4359	return NULL;
		4360
		4361	/* Cache this mapping to avoid the overhead of an
		4362	* excruciatingly slow GTT pagefault. This is more an
		4363	* issue with compositing managers which need to frequently
		4364	* flush CPU damage to their GPU bo.
		4365	*/
		4366	bo->map = ptr;
		4367	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4368	}
		4369
		4370	return ptr;
		4371	}
		4372
		4373	void kgem_bo_map__debug(struct kgem kgem, struct kgem_bo *bo)
		4374	{
		4375	if (bo->map)
		4376	return MAP(bo->map);
		4377
		4378	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4379	return bo->map = __kgem_bo_map__gtt(kgem, bo);
		4380	}
		4381
		4382	void kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		4383	{
		4384	struct drm_i915_gem_mmap mmap_arg;
		4385
		4386	DBG(("%s(handle=%d, size=%d, mapped? %d)\n",
		4387	__FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map)));
		4388	assert(!bo->purged);
		4389	assert(list_is_empty(&bo->list));
		4390	assert(bo->proxy == NULL);
		4391
		4392	if (IS_CPU_MAP(bo->map))
		4393	return MAP(bo->map);
		4394
		4395	if (bo->map)
		4396	kgem_bo_release_map(kgem, bo);
		4397
		4398	kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));
		4399
		4400	retry:
		4401	VG_CLEAR(mmap_arg);
		4402	mmap_arg.handle = bo->handle;
		4403	mmap_arg.offset = 0;
		4404	mmap_arg.size = bytes(bo);
		4405	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
		4406
		4407	if (__kgem_throttle_retire(kgem, 0))
		4408	goto retry;
		4409
		4410	if (kgem->need_expire) {
		4411	kgem_cleanup_cache(kgem);
		4412	goto retry;
		4413	}
		4414
		4415	ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain\n",
		4416	__FUNCTION__, bo->handle, bytes(bo));
		4417	return NULL;
		4418	}
		4419
		4420	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
		4421
		4422	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
		4423	bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr);
		4424	return (void *)(uintptr_t)mmap_arg.addr_ptr;
		4425	}
		4426
		4427	void __kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		4428	{
		4429	struct drm_i915_gem_mmap mmap_arg;
		4430
		4431	DBG(("%s(handle=%d, size=%d, mapped? %d)\n",
		4432	__FUNCTION__, bo->handle, bytes(bo), (int)__MAP_TYPE(bo->map)));
		4433	assert(bo->refcnt);
		4434	assert(!bo->purged);
		4435	assert(list_is_empty(&bo->list));
		4436	assert(bo->proxy == NULL);
		4437
		4438	if (IS_CPU_MAP(bo->map))
		4439	return MAP(bo->map);
		4440
		4441	retry:
		4442	VG_CLEAR(mmap_arg);
		4443	mmap_arg.handle = bo->handle;
		4444	mmap_arg.offset = 0;
		4445	mmap_arg.size = bytes(bo);
		4446	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
		4447	int err = errno;
		4448
		4449	assert(err != EINVAL);
		4450
		4451	if (__kgem_throttle_retire(kgem, 0))
		4452	goto retry;
		4453
		4454	if (kgem->need_expire) {
		4455	kgem_cleanup_cache(kgem);
		4456	goto retry;
		4457	}
		4458
		4459	ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n",
		4460	__FUNCTION__, bo->handle, bytes(bo), err);
		4461	return NULL;
		4462	}
		4463
		4464	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
		4465	if (bo->map && bo->domain == DOMAIN_CPU) {
		4466	DBG(("%s: discarding GTT vma for %d\n", __FUNCTION__, bo->handle));
		4467	kgem_bo_release_map(kgem, bo);
		4468	}
		4469	if (bo->map == NULL) {
		4470	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
		4471	bo->map = MAKE_CPU_MAP(mmap_arg.addr_ptr);
		4472	}
		4473	return (void *)(uintptr_t)mmap_arg.addr_ptr;
		4474	}
		4475	void kgem_bo_sync__cpu(struct kgem kgem, struct kgem_bo bo)
		4476	{
		4477	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		4478	assert(!bo->scanout);
		4479	kgem_bo_submit(kgem, bo);
		4480
		4481	/* SHM pixmaps use proxies for subpage offsets */
		4482	assert(!bo->purged);
		4483	while (bo->proxy)
		4484	bo = bo->proxy;
		4485	assert(!bo->purged);
		4486
		4487	if (bo->domain != DOMAIN_CPU \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_CPU)) {
		4488	struct drm_i915_gem_set_domain set_domain;
		4489
		4490	DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
		4491	__FUNCTION__, bo->handle,
		4492	bo->needs_flush, bo->domain,
		4493	__kgem_busy(kgem, bo->handle)));
		4494
		4495	VG_CLEAR(set_domain);
		4496	set_domain.handle = bo->handle;
		4497	set_domain.read_domains = I915_GEM_DOMAIN_CPU;
		4498	set_domain.write_domain = I915_GEM_DOMAIN_CPU;
		4499
		4500	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4501	kgem_bo_retire(kgem, bo);
		4502	bo->domain = DOMAIN_CPU;
		4503	}
		4504	}
		4505	}
		4506
		4507	void kgem_clear_dirty(struct kgem *kgem)
		4508	{
		4509	struct list * const buffers = &kgem->next_request->buffers;
		4510	struct kgem_bo *bo;
		4511
		4512	list_for_each_entry(bo, buffers, request) {
		4513	if (!bo->gpu_dirty)
		4514	break;
		4515
		4516	bo->gpu_dirty = false;
		4517	}
		4518	}
		4519
		4520	struct kgem_bo kgem_create_proxy(struct kgem kgem,
		4521	struct kgem_bo *target,
		4522	int offset, int length)
		4523	{
		4524	struct kgem_bo *bo;
		4525
		4526	DBG(("%s: target handle=%d [proxy? %d], offset=%d, length=%d, io=%d\n",
		4527	__FUNCTION__, target->handle, target->proxy ? target->proxy->delta : -1,
		4528	offset, length, target->io));
		4529
		4530	bo = __kgem_bo_alloc(target->handle, length);
		4531	if (bo == NULL)
		4532	return NULL;
		4533
		4534	bo->unique_id = kgem_get_unique_id(kgem);
		4535	bo->reusable = false;
		4536	bo->size.bytes = length;
		4537
		4538	bo->io = target->io && target->proxy == NULL;
		4539	bo->gpu_dirty = target->gpu_dirty;
		4540	bo->tiling = target->tiling;
		4541	bo->pitch = target->pitch;
		4542	bo->flush = target->flush;
		4543	bo->snoop = target->snoop;
		4544
		4545	assert(!bo->scanout);
		4546	bo->proxy = kgem_bo_reference(target);
		4547	bo->delta = offset;
		4548
		4549	if (target->exec) {
		4550	list_move_tail(&bo->request, &kgem->next_request->buffers);
		4551	bo->exec = &_kgem_dummy_exec;
		4552	}
		4553	bo->rq = target->rq;
		4554
		4555	return bo;
		4556	}
		4557
		4558	#if 0
		4559	static struct kgem_buffer *
		4560	buffer_alloc(void)
		4561	{
		4562	struct kgem_buffer *bo;
		4563
		4564	bo = malloc(sizeof(*bo));
		4565	if (bo == NULL)
		4566	return NULL;
		4567
		4568	bo->mem = NULL;
		4569	bo->need_io = false;
		4570	bo->mmapped = true;
		4571
		4572	return bo;
		4573	}
		4574
		4575	static struct kgem_buffer *
		4576	buffer_alloc_with_data(int num_pages)
		4577	{
		4578	struct kgem_buffer *bo;
		4579
		4580	bo = malloc(sizeof(bo) + 2UPLOAD_ALIGNMENT + num_pages * PAGE_SIZE);
		4581	if (bo == NULL)
		4582	return NULL;
		4583
		4584	bo->mem = (void )ALIGN((uintptr_t)bo + sizeof(bo), UPLOAD_ALIGNMENT);
		4585	bo->mmapped = false;
		4586	return bo;
		4587	}
		4588
		4589	static inline bool
		4590	use_snoopable_buffer(struct kgem *kgem, uint32_t flags)
		4591	{
		4592	if ((flags & KGEM_BUFFER_WRITE) == 0)
		4593	return kgem->gen >= 030;
		4594
		4595	return true;
		4596	}
		4597
		4598	static void
		4599	init_buffer_from_bo(struct kgem_buffer bo, struct kgem_bo old)
		4600	{
		4601	DBG(("%s: reusing handle=%d for buffer\n",
		4602	__FUNCTION__, old->handle));
		4603
		4604	assert(old->proxy == NULL);
		4605
		4606	memcpy(&bo->base, old, sizeof(*old));
		4607	if (old->rq)
		4608	list_replace(&old->request, &bo->base.request);
		4609	else
		4610	list_init(&bo->base.request);
		4611	list_replace(&old->vma, &bo->base.vma);
		4612	list_init(&bo->base.list);
		4613	free(old);
		4614
		4615	assert(bo->base.tiling == I915_TILING_NONE);
		4616
		4617	bo->base.refcnt = 1;
		4618	}
		4619
		4620	static struct kgem_buffer *
		4621	search_snoopable_buffer(struct kgem *kgem, unsigned alloc)
		4622	{
		4623	struct kgem_buffer *bo;
		4624	struct kgem_bo *old;
		4625
		4626	old = search_snoop_cache(kgem, alloc, 0);
		4627	if (old) {
		4628	if (!old->io) {
		4629	bo = buffer_alloc();
		4630	if (bo == NULL)
		4631	return NULL;
		4632
		4633	init_buffer_from_bo(bo, old);
		4634	} else {
		4635	bo = (struct kgem_buffer *)old;
		4636	bo->base.refcnt = 1;
		4637	}
		4638
		4639	DBG(("%s: created CPU handle=%d for buffer, size %d\n",
		4640	__FUNCTION__, bo->base.handle, num_pages(&bo->base)));
		4641
		4642	assert(bo->base.snoop);
		4643	assert(bo->base.tiling == I915_TILING_NONE);
		4644	assert(num_pages(&bo->base) >= alloc);
		4645	assert(bo->mmapped == true);
		4646	assert(bo->need_io == false);
		4647
		4648	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4649	if (bo->mem == NULL) {
		4650	bo->base.refcnt = 0;
		4651	kgem_bo_free(kgem, &bo->base);
		4652	bo = NULL;
		4653	}
		4654
		4655	return bo;
		4656	}
		4657
		4658	return NULL;
		4659	}
		4660
		4661	static struct kgem_buffer *
		4662	create_snoopable_buffer(struct kgem *kgem, unsigned alloc)
		4663	{
		4664	struct kgem_buffer *bo;
		4665	uint32_t handle;
		4666
		4667	if (kgem->has_llc) {
		4668	struct kgem_bo *old;
		4669
		4670	bo = buffer_alloc();
		4671	if (bo == NULL)
		4672	return NULL;
		4673
		4674	old = search_linear_cache(kgem, alloc,
		4675	CREATE_INACTIVE \| CREATE_CPU_MAP \| CREATE_EXACT);
		4676	if (old) {
		4677	init_buffer_from_bo(bo, old);
		4678	} else {
		4679	handle = gem_create(kgem->fd, alloc);
		4680	if (handle == 0) {
		4681	free(bo);
		4682	return NULL;
		4683	}
		4684
		4685	debug_alloc(kgem, alloc);
		4686	__kgem_bo_init(&bo->base, handle, alloc);
		4687	DBG(("%s: created CPU (LLC) handle=%d for buffer, size %d\n",
		4688	__FUNCTION__, bo->base.handle, alloc));
		4689	}
		4690
		4691	assert(bo->base.refcnt == 1);
		4692	assert(bo->mmapped == true);
		4693	assert(bo->need_io == false);
		4694
		4695	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4696	if (bo->mem != NULL)
		4697	return bo;
		4698
		4699	bo->base.refcnt = 0; /* for valgrind */
		4700	kgem_bo_free(kgem, &bo->base);
		4701	}
		4702
		4703	if (kgem->has_caching) {
		4704	struct kgem_bo *old;
		4705
		4706	bo = buffer_alloc();
		4707	if (bo == NULL)
		4708	return NULL;
		4709
		4710	old = search_linear_cache(kgem, alloc,
		4711	CREATE_INACTIVE \| CREATE_CPU_MAP \| CREATE_EXACT);
		4712	if (old) {
		4713	init_buffer_from_bo(bo, old);
		4714	} else {
		4715	handle = gem_create(kgem->fd, alloc);
		4716	if (handle == 0) {
		4717	free(bo);
		4718	return NULL;
		4719	}
		4720
		4721	debug_alloc(kgem, alloc);
		4722	__kgem_bo_init(&bo->base, handle, alloc);
		4723	DBG(("%s: created CPU handle=%d for buffer, size %d\n",
		4724	__FUNCTION__, bo->base.handle, alloc));
		4725	}
		4726
		4727	assert(bo->base.refcnt == 1);
		4728	assert(bo->mmapped == true);
		4729	assert(bo->need_io == false);
		4730
		4731	if (!gem_set_caching(kgem->fd, bo->base.handle, SNOOPED))
		4732	goto free_caching;
		4733
		4734	bo->base.snoop = true;
		4735
		4736	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4737	if (bo->mem == NULL)
		4738	goto free_caching;
		4739
		4740	return bo;
		4741
		4742	free_caching:
		4743	bo->base.refcnt = 0; /* for valgrind */
		4744	kgem_bo_free(kgem, &bo->base);
		4745	}
		4746
		4747	if (kgem->has_userptr) {
		4748	bo = buffer_alloc();
		4749	if (bo == NULL)
		4750	return NULL;
		4751
		4752	//if (posix_memalign(&ptr, 64, ALIGN(size, 64)))
		4753	if (posix_memalign(&bo->mem, PAGE_SIZE, alloc * PAGE_SIZE)) {
		4754	free(bo);
		4755	return NULL;
		4756	}
		4757
		4758	handle = gem_userptr(kgem->fd, bo->mem, alloc * PAGE_SIZE, false);
		4759	if (handle == 0) {
		4760	free(bo->mem);
		4761	free(bo);
		4762	return NULL;
		4763	}
		4764
		4765	debug_alloc(kgem, alloc);
		4766	__kgem_bo_init(&bo->base, handle, alloc);
		4767	DBG(("%s: created snoop handle=%d for buffer\n",
		4768	__FUNCTION__, bo->base.handle));
		4769
		4770	assert(bo->mmapped == true);
		4771	assert(bo->need_io == false);
		4772
		4773	bo->base.refcnt = 1;
		4774	bo->base.snoop = true;
		4775	bo->base.map = MAKE_USER_MAP(bo->mem);
		4776
		4777	return bo;
		4778	}
		4779
		4780	return NULL;
		4781	}
		4782
		4783	struct kgem_bo kgem_create_buffer(struct kgem kgem,
		4784	uint32_t size, uint32_t flags,
		4785	void **ret)
		4786	{
		4787	struct kgem_buffer *bo;
		4788	unsigned offset, alloc;
		4789	struct kgem_bo *old;
		4790
		4791	DBG(("%s: size=%d, flags=%x [write?=%d, inplace?=%d, last?=%d]\n",
		4792	__FUNCTION__, size, flags,
		4793	!!(flags & KGEM_BUFFER_WRITE),
		4794	!!(flags & KGEM_BUFFER_INPLACE),
		4795	!!(flags & KGEM_BUFFER_LAST)));
		4796	assert(size);
		4797	/* we should never be asked to create anything TOO large */
		4798	assert(size <= kgem->max_object_size);
		4799
		4800	#if !DBG_NO_UPLOAD_CACHE
		4801	list_for_each_entry(bo, &kgem->batch_buffers, base.list) {
		4802	assert(bo->base.io);
		4803	assert(bo->base.refcnt >= 1);
		4804
		4805	/* We can reuse any write buffer which we can fit */
		4806	if (flags == KGEM_BUFFER_LAST &&
		4807	bo->write == KGEM_BUFFER_WRITE &&
		4808	bo->base.refcnt == 1 && !bo->mmapped &&
		4809	size <= bytes(&bo->base)) {
		4810	DBG(("%s: reusing write buffer for read of %d bytes? used=%d, total=%d\n",
		4811	__FUNCTION__, size, bo->used, bytes(&bo->base)));
		4812	gem_write(kgem->fd, bo->base.handle,
		4813	0, bo->used, bo->mem);
		4814	kgem_buffer_release(kgem, bo);
		4815	bo->need_io = 0;
		4816	bo->write = 0;
		4817	offset = 0;
		4818	bo->used = size;
		4819	goto done;
		4820	}
		4821
		4822	if (flags & KGEM_BUFFER_WRITE) {
		4823	if ((bo->write & KGEM_BUFFER_WRITE) == 0 \|\|
		4824	(((bo->write & ~flags) & KGEM_BUFFER_INPLACE) &&
		4825	!bo->base.snoop)) {
		4826	DBG(("%s: skip write %x buffer, need %x\n",
		4827	__FUNCTION__, bo->write, flags));
		4828	continue;
		4829	}
		4830	assert(bo->mmapped \|\| bo->need_io);
		4831	} else {
		4832	if (bo->write & KGEM_BUFFER_WRITE) {
		4833	DBG(("%s: skip write %x buffer, need %x\n",
		4834	__FUNCTION__, bo->write, flags));
		4835	continue;
		4836	}
		4837	}
		4838
		4839	if (bo->used + size <= bytes(&bo->base)) {
		4840	DBG(("%s: reusing buffer? used=%d + size=%d, total=%d\n",
		4841	__FUNCTION__, bo->used, size, bytes(&bo->base)));
		4842	offset = bo->used;
		4843	bo->used += size;
		4844	goto done;
		4845	}
		4846	}
		4847
		4848	if (flags & KGEM_BUFFER_WRITE) {
		4849	list_for_each_entry(bo, &kgem->active_buffers, base.list) {
		4850	assert(bo->base.io);
		4851	assert(bo->base.refcnt >= 1);
		4852	assert(bo->mmapped);
		4853	assert(!IS_CPU_MAP(bo->base.map) \|\| kgem->has_llc \|\| bo->base.snoop);
		4854
		4855	if (!kgem->has_llc && (bo->write & ~flags) & KGEM_BUFFER_INPLACE) {
		4856	DBG(("%s: skip write %x buffer, need %x\n",
		4857	__FUNCTION__, bo->write, flags));
		4858	continue;
		4859	}
		4860
		4861	if (bo->used + size <= bytes(&bo->base)) {
		4862	DBG(("%s: reusing buffer? used=%d + size=%d, total=%d\n",
		4863	__FUNCTION__, bo->used, size, bytes(&bo->base)));
		4864	offset = bo->used;
		4865	bo->used += size;
		4866	list_move(&bo->base.list, &kgem->batch_buffers);
		4867	goto done;
		4868	}
		4869	}
		4870	}
		4871	#endif
		4872
		4873	#if !DBG_NO_MAP_UPLOAD
		4874	/* Be a little more generous and hope to hold fewer mmappings */
		4875	alloc = ALIGN(2*size, kgem->buffer_size);
		4876	if (alloc > MAX_CACHE_SIZE)
		4877	alloc = ALIGN(size, kgem->buffer_size);
		4878	if (alloc > MAX_CACHE_SIZE)
		4879	alloc = PAGE_ALIGN(size);
		4880	assert(alloc);
		4881
		4882	if (alloc > kgem->aperture_mappable / 4)
		4883	flags &= ~KGEM_BUFFER_INPLACE;
		4884	alloc /= PAGE_SIZE;
		4885
		4886	if (kgem->has_llc &&
		4887	(flags & KGEM_BUFFER_WRITE_INPLACE) != KGEM_BUFFER_WRITE_INPLACE) {
		4888	bo = buffer_alloc();
		4889	if (bo == NULL)
		4890	goto skip_llc;
		4891
		4892	old = NULL;
		4893	if ((flags & KGEM_BUFFER_WRITE) == 0)
		4894	old = search_linear_cache(kgem, alloc, CREATE_CPU_MAP);
		4895	if (old == NULL)
		4896	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE \| CREATE_CPU_MAP);
		4897	if (old == NULL)
		4898	old = search_linear_cache(kgem, NUM_PAGES(size), CREATE_INACTIVE \| CREATE_CPU_MAP);
		4899	if (old) {
		4900	DBG(("%s: found LLC handle=%d for buffer\n",
		4901	__FUNCTION__, old->handle));
		4902
		4903	init_buffer_from_bo(bo, old);
		4904	} else {
		4905	uint32_t handle = gem_create(kgem->fd, alloc);
		4906	if (handle == 0) {
		4907	free(bo);
		4908	goto skip_llc;
		4909	}
		4910	__kgem_bo_init(&bo->base, handle, alloc);
		4911	DBG(("%s: created LLC handle=%d for buffer\n",
		4912	__FUNCTION__, bo->base.handle));
		4913
		4914	debug_alloc(kgem, alloc);
		4915	}
		4916
		4917	assert(bo->mmapped);
		4918	assert(!bo->need_io);
		4919
		4920	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4921	if (bo->mem) {
		4922	if (flags & KGEM_BUFFER_WRITE)
		4923	kgem_bo_sync__cpu(kgem, &bo->base);
		4924	flags &= ~KGEM_BUFFER_INPLACE;
		4925	goto init;
		4926	} else {
		4927	bo->base.refcnt = 0; /* for valgrind */
		4928	kgem_bo_free(kgem, &bo->base);
		4929	}
		4930	}
		4931	skip_llc:
		4932
		4933	if ((flags & KGEM_BUFFER_WRITE_INPLACE) == KGEM_BUFFER_WRITE_INPLACE) {
		4934	/* The issue with using a GTT upload buffer is that we may
		4935	* cause eviction-stalls in order to free up some GTT space.
		4936	* An is-mappable? ioctl could help us detect when we are
		4937	* about to block, or some per-page magic in the kernel.
		4938	*
		4939	* XXX This is especially noticeable on memory constrained
		4940	* devices like gen2 or with relatively slow gpu like i3.
		4941	*/
		4942	DBG(("%s: searching for an inactive GTT map for upload\n",
		4943	__FUNCTION__));
		4944	old = search_linear_cache(kgem, alloc,
		4945	CREATE_EXACT \| CREATE_INACTIVE \| CREATE_GTT_MAP);
		4946	#if HAVE_I915_GEM_BUFFER_INFO
		4947	if (old) {
		4948	struct drm_i915_gem_buffer_info info;
		4949
		4950	/* An example of such a non-blocking ioctl might work */
		4951
		4952	VG_CLEAR(info);
		4953	info.handle = handle;
		4954	if (drmIoctl(kgem->fd,
		4955	DRM_IOCTL_I915_GEM_BUFFER_INFO,
		4956	&fino) == 0) {
		4957	old->presumed_offset = info.addr;
		4958	if ((info.flags & I915_GEM_MAPPABLE) == 0) {
		4959	kgem_bo_move_to_inactive(kgem, old);
		4960	old = NULL;
		4961	}
		4962	}
		4963	}
		4964	#endif
		4965	if (old == NULL)
		4966	old = search_linear_cache(kgem, NUM_PAGES(size),
		4967	CREATE_EXACT \| CREATE_INACTIVE \| CREATE_GTT_MAP);
		4968	if (old == NULL) {
		4969	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
		4970	if (old && !__kgem_bo_is_mappable(kgem, old)) {
		4971	_kgem_bo_destroy(kgem, old);
		4972	old = NULL;
		4973	}
		4974	}
		4975	if (old) {
		4976	DBG(("%s: reusing handle=%d for buffer\n",
		4977	__FUNCTION__, old->handle));
		4978	assert(__kgem_bo_is_mappable(kgem, old));
		4979	assert(!old->snoop);
		4980	assert(old->rq == NULL);
		4981
		4982	bo = buffer_alloc();
		4983	if (bo == NULL)
		4984	return NULL;
		4985
		4986	init_buffer_from_bo(bo, old);
		4987	assert(num_pages(&bo->base) >= NUM_PAGES(size));
		4988
		4989	assert(bo->mmapped);
		4990	assert(bo->base.refcnt == 1);
		4991
		4992	bo->mem = kgem_bo_map(kgem, &bo->base);
		4993	if (bo->mem) {
		4994	if (IS_CPU_MAP(bo->base.map))
		4995	flags &= ~KGEM_BUFFER_INPLACE;
		4996	goto init;
		4997	} else {
		4998	bo->base.refcnt = 0;
		4999	kgem_bo_free(kgem, &bo->base);
		5000	}
		5001	}
		5002	}
		5003	#else
		5004	flags &= ~KGEM_BUFFER_INPLACE;
		5005	#endif
		5006	/* Be more parsimonious with pwrite/pread/cacheable buffers */
		5007	if ((flags & KGEM_BUFFER_INPLACE) == 0)
		5008	alloc = NUM_PAGES(size);
		5009
		5010	if (use_snoopable_buffer(kgem, flags)) {
		5011	bo = search_snoopable_buffer(kgem, alloc);
		5012	if (bo) {
		5013	if (flags & KGEM_BUFFER_WRITE)
		5014	kgem_bo_sync__cpu(kgem, &bo->base);
		5015	flags &= ~KGEM_BUFFER_INPLACE;
		5016	goto init;
		5017	}
		5018
		5019	if ((flags & KGEM_BUFFER_INPLACE) == 0) {
		5020	bo = create_snoopable_buffer(kgem, alloc);
		5021	if (bo)
		5022	goto init;
		5023	}
		5024	}
		5025
		5026	flags &= ~KGEM_BUFFER_INPLACE;
		5027
		5028	old = NULL;
		5029	if ((flags & KGEM_BUFFER_WRITE) == 0)
		5030	old = search_linear_cache(kgem, alloc, 0);
		5031	if (old == NULL)
		5032	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
		5033	if (old) {
		5034	DBG(("%s: reusing ordinary handle %d for io\n",
		5035	__FUNCTION__, old->handle));
		5036	bo = buffer_alloc_with_data(num_pages(old));
		5037	if (bo == NULL)
		5038	return NULL;
		5039
		5040	init_buffer_from_bo(bo, old);
		5041	bo->need_io = flags & KGEM_BUFFER_WRITE;
		5042	} else {
		5043	unsigned hint;
		5044
		5045	if (use_snoopable_buffer(kgem, flags)) {
		5046	bo = create_snoopable_buffer(kgem, alloc);
		5047	if (bo)
		5048	goto init;
		5049	}
		5050
		5051	bo = buffer_alloc();
		5052	if (bo == NULL)
		5053	return NULL;
		5054
		5055	hint = CREATE_INACTIVE;
		5056	if (flags & KGEM_BUFFER_WRITE)
		5057	hint \|= CREATE_CPU_MAP;
		5058	old = search_linear_cache(kgem, alloc, hint);
		5059	if (old) {
		5060	DBG(("%s: reusing handle=%d for buffer\n",
		5061	__FUNCTION__, old->handle));
		5062
		5063	init_buffer_from_bo(bo, old);
		5064	} else {
		5065	uint32_t handle = gem_create(kgem->fd, alloc);
		5066	if (handle == 0) {
		5067	free(bo);
		5068	return NULL;
		5069	}
		5070
		5071	DBG(("%s: created handle=%d for buffer\n",
		5072	__FUNCTION__, handle));
		5073
		5074	__kgem_bo_init(&bo->base, handle, alloc);
		5075	debug_alloc(kgem, alloc * PAGE_SIZE);
		5076	}
		5077
		5078	assert(bo->mmapped);
		5079	assert(!bo->need_io);
		5080	assert(bo->base.refcnt == 1);
		5081
		5082	if (flags & KGEM_BUFFER_WRITE) {
		5083	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		5084	if (bo->mem != NULL) {
		5085	kgem_bo_sync__cpu(kgem, &bo->base);
		5086	goto init;
		5087	}
		5088	}
		5089
		5090	DBG(("%s: failing back to new pwrite buffer\n", __FUNCTION__));
		5091	old = &bo->base;
		5092	bo = buffer_alloc_with_data(num_pages(old));
		5093	if (bo == NULL) {
		5094	old->refcnt= 0;
		5095	kgem_bo_free(kgem, old);
		5096	return NULL;
		5097	}
		5098
		5099	init_buffer_from_bo(bo, old);
		5100
		5101	assert(bo->mem);
		5102	assert(!bo->mmapped);
		5103	assert(bo->base.refcnt == 1);
		5104
		5105	bo->need_io = flags & KGEM_BUFFER_WRITE;
		5106	}
		5107	init:
		5108	bo->base.io = true;
		5109	assert(bo->base.refcnt == 1);
		5110	assert(num_pages(&bo->base) >= NUM_PAGES(size));
		5111	assert(!bo->need_io \|\| !bo->base.needs_flush);
		5112	assert(!bo->need_io \|\| bo->base.domain != DOMAIN_GPU);
		5113	assert(bo->mem);
		5114	assert(!bo->mmapped \|\| bo->base.map != NULL);
		5115
		5116	bo->used = size;
		5117	bo->write = flags & KGEM_BUFFER_WRITE_INPLACE;
		5118	offset = 0;
		5119
		5120	assert(list_is_empty(&bo->base.list));
		5121	list_add(&bo->base.list, &kgem->batch_buffers);
		5122
		5123	DBG(("%s(pages=%d [%d]) new handle=%d, used=%d, write=%d\n",
		5124	__FUNCTION__, num_pages(&bo->base), alloc, bo->base.handle, bo->used, bo->write));
		5125
		5126	done:
		5127	bo->used = ALIGN(bo->used, UPLOAD_ALIGNMENT);
		5128	assert(bo->mem);
		5129	ret = (char )bo->mem + offset;
		5130	return kgem_create_proxy(kgem, &bo->base, offset, size);
		5131	}
		5132
		5133	bool kgem_buffer_is_inplace(struct kgem_bo *_bo)
		5134	{
		5135	struct kgem_buffer bo = (struct kgem_buffer )_bo->proxy;
		5136	return bo->write & KGEM_BUFFER_WRITE_INPLACE;
		5137	}
		5138
		5139	struct kgem_bo kgem_create_buffer_2d(struct kgem kgem,
		5140	int width, int height, int bpp,
		5141	uint32_t flags,
		5142	void **ret)
		5143	{
		5144	struct kgem_bo *bo;
		5145	int stride;
		5146
		5147	assert(width > 0 && height > 0);
		5148	assert(ret != NULL);
		5149	stride = ALIGN(width, 2) * bpp >> 3;
		5150	stride = ALIGN(stride, 4);
		5151
		5152	DBG(("%s: %dx%d, %d bpp, stride=%d\n",
		5153	__FUNCTION__, width, height, bpp, stride));
		5154
		5155	bo = kgem_create_buffer(kgem, stride * ALIGN(height, 2), flags, ret);
		5156	if (bo == NULL) {
		5157	DBG(("%s: allocation failure for upload buffer\n",
		5158	__FUNCTION__));
		5159	return NULL;
		5160	}
		5161	assert(*ret != NULL);
		5162	assert(bo->proxy != NULL);
		5163
		5164	if (height & 1) {
		5165	struct kgem_buffer io = (struct kgem_buffer )bo->proxy;
		5166	int min;
		5167
		5168	assert(io->used);
		5169
		5170	/* Having padded this surface to ensure that accesses to
		5171	* the last pair of rows is valid, remove the padding so
		5172	* that it can be allocated to other pixmaps.
		5173	*/
		5174	min = bo->delta + height * stride;
		5175	min = ALIGN(min, UPLOAD_ALIGNMENT);
		5176	if (io->used != min) {
		5177	DBG(("%s: trimming buffer from %d to %d\n",
		5178	__FUNCTION__, io->used, min));
		5179	io->used = min;
		5180	}
		5181	bo->size.bytes -= stride;
		5182	}
		5183
		5184	bo->map = MAKE_CPU_MAP(*ret);
		5185	bo->pitch = stride;
		5186	bo->unique_id = kgem_get_unique_id(kgem);
		5187	return bo;
		5188	}
		5189
		5190	struct kgem_bo kgem_upload_source_image(struct kgem kgem,
		5191	const void *data,
		5192	const BoxRec *box,
		5193	int stride, int bpp)
		5194	{
		5195	int width = box->x2 - box->x1;
		5196	int height = box->y2 - box->y1;
		5197	struct kgem_bo *bo;
		5198	void *dst;
		5199
		5200	if (!kgem_can_create_2d(kgem, width, height, bpp))
		5201	return NULL;
		5202
		5203	DBG(("%s : (%d, %d), (%d, %d), stride=%d, bpp=%d\n",
		5204	__FUNCTION__, box->x1, box->y1, box->x2, box->y2, stride, bpp));
		5205
		5206	assert(data);
		5207	assert(width > 0);
		5208	assert(height > 0);
		5209	assert(stride);
		5210	assert(bpp);
		5211
		5212	bo = kgem_create_buffer_2d(kgem,
		5213	width, height, bpp,
		5214	KGEM_BUFFER_WRITE_INPLACE, &dst);
		5215	if (bo)
		5216	memcpy_blt(data, dst, bpp,
		5217	stride, bo->pitch,
		5218	box->x1, box->y1,
		5219	0, 0,
		5220	width, height);
		5221
		5222	return bo;
		5223	}
		5224
		5225	void kgem_proxy_bo_attach(struct kgem_bo *bo,
		5226	struct kgem_bo **ptr)
		5227	{
		5228	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		5229	assert(bo->map == NULL \|\| IS_CPU_MAP(bo->map));
		5230	assert(bo->proxy);
		5231	list_add(&bo->vma, &bo->proxy->vma);
		5232	bo->map = ptr;
		5233	*ptr = kgem_bo_reference(bo);
		5234	}
		5235
		5236	void kgem_buffer_read_sync(struct kgem kgem, struct kgem_bo _bo)
		5237	{
		5238	struct kgem_buffer *bo;
		5239	uint32_t offset = _bo->delta, length = _bo->size.bytes;
		5240
		5241	/* We expect the caller to have already submitted the batch */
		5242	assert(_bo->io);
		5243	assert(_bo->exec == NULL);
		5244	assert(_bo->rq == NULL);
		5245	assert(_bo->proxy);
		5246
		5247	_bo = _bo->proxy;
		5248	assert(_bo->proxy == NULL);
		5249	assert(_bo->exec == NULL);
		5250
		5251	bo = (struct kgem_buffer *)_bo;
		5252
		5253	DBG(("%s(offset=%d, length=%d, snooped=%d)\n", __FUNCTION__,
		5254	offset, length, bo->base.snoop));
		5255
		5256	if (bo->mmapped) {
		5257	struct drm_i915_gem_set_domain set_domain;
		5258
		5259	DBG(("%s: sync: needs_flush? %d, domain? %d, busy? %d\n",
		5260	__FUNCTION__,
		5261	bo->base.needs_flush,
		5262	bo->base.domain,
		5263	__kgem_busy(kgem, bo->base.handle)));
		5264
		5265	assert(!IS_CPU_MAP(bo->base.map) \|\| bo->base.snoop \|\| kgem->has_llc);
		5266
		5267	VG_CLEAR(set_domain);
		5268	set_domain.handle = bo->base.handle;
		5269	set_domain.write_domain = 0;
		5270	set_domain.read_domains =
		5271	IS_CPU_MAP(bo->base.map) ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT;
		5272
		5273	if (drmIoctl(kgem->fd,
		5274	DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain))
		5275	return;
		5276	} else {
		5277	if (gem_read(kgem->fd,
		5278	bo->base.handle, (char *)bo->mem+offset,
		5279	offset, length))
		5280	return;
		5281	}
		5282	kgem_bo_retire(kgem, &bo->base);
		5283	bo->base.domain = DOMAIN_NONE;
		5284	}
		5285	#endif
		5286
		5287	uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format)
		5288	{
		5289	struct kgem_bo_binding *b;
		5290
		5291	for (b = &bo->binding; b && b->offset; b = b->next)
		5292	if (format == b->format)
		5293	return b->offset;
		5294
		5295	return 0;
		5296	}
		5297
		5298	void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset)
		5299	{
		5300	struct kgem_bo_binding *b;
		5301
		5302	for (b = &bo->binding; b; b = b->next) {
		5303	if (b->offset)
		5304	continue;
		5305
		5306	b->offset = offset;
		5307	b->format = format;
		5308
		5309	if (b->next)
		5310	b->next->offset = 0;
		5311
		5312	return;
		5313	}
		5314
		5315	b = malloc(sizeof(*b));
		5316	if (b) {
		5317	b->next = bo->binding.next;
		5318	b->format = format;
		5319	b->offset = offset;
		5320	bo->binding.next = b;
		5321	}
		5322	}
		5323
		5324	int kgem_init_fb(struct kgem kgem, struct sna_fb fb)
		5325	{
		5326	struct kgem_bo *bo;
		5327	size_t size;
		5328	int ret;
		5329
		5330	ret = drmIoctl(kgem->fd, SRV_FBINFO, fb);
		5331	if( ret != 0 )
		5332	return 0;
		5333
		5334	size = fb->pitch * fb->height / PAGE_SIZE;
		5335
		5336	bo = __kgem_bo_alloc(-2, size);
		5337	if (!bo) {
		5338	return 0;
		5339	}
		5340
		5341	bo->domain = DOMAIN_GTT;
		5342	bo->unique_id = kgem_get_unique_id(kgem);
		5343	bo->pitch = fb->pitch;
		5344	bo->tiling = I915_TILING_X;
		5345	bo->scanout = 1;
		5346	fb->fb_bo = bo;
		5347
		5348	// printf("fb width %d height %d pitch %d bo %p\n",
		5349	// fb->width, fb->height, fb->pitch, fb->fb_bo);
		5350
		5351	return 1;
		5352	};
		5353
		5354
		5355	int kgem_update_fb(struct kgem kgem, struct sna_fb fb)
		5356	{
		5357	struct kgem_bo *bo;
		5358	size_t size;
		5359	int ret;
		5360
		5361	bo = fb->fb_bo;
		5362
		5363	ret = drmIoctl(kgem->fd, SRV_FBINFO, fb);
		5364	if( ret != 0 )
		5365	return 0;
		5366
		5367	fb->fb_bo = bo;
		5368
		5369	size = fb->pitch * fb->height / PAGE_SIZE;
		5370
		5371	if((size != bo->size.pages.count) \|\|
		5372	(fb->pitch != bo->pitch))
		5373	{
		5374	bo->size.pages.count = size;
		5375	bo->pitch = fb->pitch;
		5376
		5377	printf("fb width %d height %d pitch %d bo %p\n",
		5378	fb->width, fb->height, fb->pitch, fb->fb_bo);
		5379
		5380	return 1;
		5381	}
		5382
		5383	return 0;
		5384	};
		5385
		5386	void sna_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		5387	{
		5388	kgem_bo_destroy(kgem, bo);
		5389	kgem_bo_free(kgem, bo);
		5390	}
		5391
		5392
		5393	void kgem_close_batches(struct kgem *kgem)
		5394	{
		5395	int n;
4368	Serge	5396	ENTER();
4304	Serge	5397	for (n = 0; n < ARRAY_SIZE(kgem->pinned_batches); n++) {
4368	Serge	5398	while (!list_is_empty(&kgem->pinned_batches[n]))
		5399	{
		5400	struct kgem_bo *bo =
4304	Serge	5401	list_first_entry(&kgem->pinned_batches[n],
4368	Serge	5402	struct kgem_bo, list);
		5403	list_del(&bo->list);
		5404	kgem_bo_destroy(kgem,bo);
4304	Serge	5405	}
		5406	}
4368	Serge	5407	LEAVE();
4304	Serge	5408	};
		5409
		5410	struct kgem_bo kgem_bo_from_handle(struct kgem kgem, int handle,
		5411	int pitch, int height)
		5412	{
		5413	struct kgem_bo *bo;
		5414	int size;
		5415
		5416	size = pitch * height / PAGE_SIZE;
		5417
		5418	bo = __kgem_bo_alloc(handle, size);
		5419	if(bo == NULL)
		5420	return NULL;
		5421
		5422	bo->domain = DOMAIN_GTT;
		5423	bo->unique_id = kgem_get_unique_id(kgem);
		5424	bo->pitch = pitch;
		5425	bo->tiling = I915_TILING_X;
		5426	bo->scanout = 0;
		5427
		5428	return bo;
		5429	}

Subversion Repositories Kolibri OS

(root)/contrib/sdk/sources/Intel-2D/sna/kgem.c @ 4411 – Rev 4368