WebSVN – Kolibri OS – Blame – /contrib/sdk/sources/Intel-2D/sna/kgem.c

Rev	Author	Line No.	Line
4304	Serge	1	/*
		2	* Copyright (c) 2011 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
		20	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		21	* SOFTWARE.
		22	*
		23	* Authors:
		24	* Chris Wilson
		25	*
		26	*/
		27
		28	#ifdef HAVE_CONFIG_H
		29	#include "config.h"
		30	#endif
		31
		32	#include "sna.h"
		33	#include "sna_reg.h"
		34
		35	#include
		36	#include
		37	#include
		38
		39	#ifdef HAVE_VALGRIND
		40	#include
		41	#include
		42	#endif
		43
		44	#ifdef HAVE_STRUCT_SYSINFO_TOTALRAM
		45	#include
		46	#endif
		47
		48	#include "sna_cpuid.h"
		49
		50	static struct kgem_bo *
		51	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
		52
		53	static struct kgem_bo *
		54	search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags);
		55
		56	#define DBG_NO_HW 0
		57	#define DBG_NO_TILING 0
		58	#define DBG_NO_CACHE 0
		59	#define DBG_NO_CACHE_LEVEL 0
		60	#define DBG_NO_CPU 0
		61	#define DBG_NO_CREATE2 1
4501	Serge	62	#define DBG_NO_USERPTR 1
4304	Serge	63	#define DBG_NO_UNSYNCHRONIZED_USERPTR 0
		64	#define DBG_NO_LLC 0
		65	#define DBG_NO_SEMAPHORES 0
		66	#define DBG_NO_MADV 1
		67	#define DBG_NO_UPLOAD_CACHE 0
		68	#define DBG_NO_UPLOAD_ACTIVE 0
		69	#define DBG_NO_MAP_UPLOAD 0
		70	#define DBG_NO_RELAXED_FENCING 0
		71	#define DBG_NO_SECURE_BATCHES 0
		72	#define DBG_NO_PINNED_BATCHES 0
		73	#define DBG_NO_FAST_RELOC 0
4501	Serge	74	#define DBG_NO_HANDLE_LUT 0
4304	Serge	75	#define DBG_NO_WT 0
		76	#define DBG_DUMP 0
		77
		78	#define FORCE_MMAP_SYNC 0 /* ((1 << DOMAIN_CPU) \| (1 << DOMAIN_GTT)) */
		79
		80	#ifndef DEBUG_SYNC
		81	#define DEBUG_SYNC 0
		82	#endif
		83
		84
		85	#if 0
		86	#define ASSERT_IDLE(kgem__, handle__) assert(!__kgem_busy(kgem__, handle__))
		87	#define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__) assert(!(expect__) \|\| !__kgem_busy(kgem__, handle__))
		88	#else
		89	#define ASSERT_IDLE(kgem__, handle__)
		90	#define ASSERT_MAYBE_IDLE(kgem__, handle__, expect__)
		91	#endif
		92
		93	/* Worst case seems to be 965gm where we cannot write within a cacheline that
		94	* is being simultaneously being read by the GPU, or within the sampler
		95	* prefetch. In general, the chipsets seem to have a requirement that sampler
		96	* offsets be aligned to a cacheline (64 bytes).
		97	*/
		98	#define UPLOAD_ALIGNMENT 128
		99
		100	#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
		101	#define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE)
		102
		103	#define MAX_GTT_VMA_CACHE 512
		104	#define MAX_CPU_VMA_CACHE INT16_MAX
		105	#define MAP_PRESERVE_TIME 10
		106
4501	Serge	107	#define MAKE_USER_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 1))
		108	#define IS_USER_MAP(ptr) ((uintptr_t)(ptr) & 1)
4304	Serge	109
		110	#define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) \| (ring)))
		111
		112	#define LOCAL_I915_PARAM_HAS_BLT 11
		113	#define LOCAL_I915_PARAM_HAS_RELAXED_FENCING 12
		114	#define LOCAL_I915_PARAM_HAS_RELAXED_DELTA 15
		115	#define LOCAL_I915_PARAM_HAS_SEMAPHORES 20
		116	#define LOCAL_I915_PARAM_HAS_SECURE_BATCHES 23
		117	#define LOCAL_I915_PARAM_HAS_PINNED_BATCHES 24
		118	#define LOCAL_I915_PARAM_HAS_NO_RELOC 25
		119	#define LOCAL_I915_PARAM_HAS_HANDLE_LUT 26
		120	#define LOCAL_I915_PARAM_HAS_WT 27
		121
		122	#define LOCAL_I915_EXEC_IS_PINNED (1<<10)
		123	#define LOCAL_I915_EXEC_NO_RELOC (1<<11)
		124	#define LOCAL_I915_EXEC_HANDLE_LUT (1<<12)
		125	struct local_i915_gem_userptr {
		126	uint64_t user_ptr;
		127	uint64_t user_size;
		128	uint32_t flags;
		129	#define I915_USERPTR_READ_ONLY (1<<0)
		130	#define I915_USERPTR_UNSYNCHRONIZED (1<<31)
		131	uint32_t handle;
		132	};
		133
		134	#define UNCACHED 0
		135	#define SNOOPED 1
		136	#define DISPLAY 2
		137
		138	struct local_i915_gem_caching {
		139	uint32_t handle;
		140	uint32_t caching;
		141	};
		142
		143	#define LOCAL_IOCTL_I915_GEM_SET_CACHING SRV_I915_GEM_SET_CACHING
		144
		145	struct local_fbinfo {
		146	int width;
		147	int height;
		148	int pitch;
		149	int tiling;
		150	};
		151
		152	struct kgem_buffer {
		153	struct kgem_bo base;
		154	void *mem;
		155	uint32_t used;
		156	uint32_t need_io : 1;
		157	uint32_t write : 2;
4501	Serge	158	uint32_t mmapped : 2;
4304	Serge	159	};
4501	Serge	160	enum {
		161	MMAPPED_NONE,
		162	MMAPPED_GTT,
		163	MMAPPED_CPU
		164	};
4304	Serge	165
		166	static struct kgem_bo *__kgem_freed_bo;
		167	static struct kgem_request *__kgem_freed_request;
		168	static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;
		169
		170	static inline int bytes(struct kgem_bo *bo)
		171	{
		172	return __kgem_bo_size(bo);
		173	}
		174
		175	#define bucket(B) (B)->size.pages.bucket
		176	#define num_pages(B) (B)->size.pages.count
		177
		178	#ifdef DEBUG_MEMORY
		179	static void debug_alloc(struct kgem *kgem, size_t size)
		180	{
		181	kgem->debug_memory.bo_allocs++;
		182	kgem->debug_memory.bo_bytes += size;
		183	}
		184	static void debug_alloc__bo(struct kgem kgem, struct kgem_bo bo)
		185	{
		186	debug_alloc(kgem, bytes(bo));
		187	}
		188	#else
		189	#define debug_alloc(k, b)
		190	#define debug_alloc__bo(k, b)
		191	#endif
		192
		193	#ifndef NDEBUG
		194	static void assert_tiling(struct kgem kgem, struct kgem_bo bo)
		195	{
		196	struct drm_i915_gem_get_tiling tiling;
		197
		198	assert(bo);
		199
		200	VG_CLEAR(tiling);
		201	tiling.handle = bo->handle;
		202	tiling.tiling_mode = -1;
		203	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_TILING, &tiling);
		204	assert(tiling.tiling_mode == bo->tiling);
		205	}
		206	#else
		207	#define assert_tiling(kgem, bo)
		208	#endif
		209
		210	static void kgem_sna_reset(struct kgem *kgem)
		211	{
		212	struct sna *sna = container_of(kgem, struct sna, kgem);
		213
		214	sna->render.reset(sna);
		215	sna->blt_state.fill_bo = 0;
		216	}
		217
		218	static void kgem_sna_flush(struct kgem *kgem)
		219	{
		220	struct sna *sna = container_of(kgem, struct sna, kgem);
		221
		222	sna->render.flush(sna);
		223
		224	// if (sna->render.solid_cache.dirty)
		225	// sna_render_flush_solid(sna);
		226	}
		227
		228	static bool gem_set_tiling(int fd, uint32_t handle, int tiling, int stride)
		229	{
		230	struct drm_i915_gem_set_tiling set_tiling;
		231	int ret;
		232
		233	if (DBG_NO_TILING)
		234	return false;
		235
		236	VG_CLEAR(set_tiling);
		237	do {
		238	set_tiling.handle = handle;
		239	set_tiling.tiling_mode = tiling;
		240	set_tiling.stride = stride;
		241
		242	ret = drmIoctl(fd, DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling);
		243	} while (ret != 0);
		244	return ret == 0;
		245	}
		246
		247	static bool gem_set_caching(int fd, uint32_t handle, int caching)
		248	{
		249	struct local_i915_gem_caching arg;
		250
		251	VG_CLEAR(arg);
		252	arg.handle = handle;
		253	arg.caching = caching;
		254	return drmIoctl(fd, LOCAL_IOCTL_I915_GEM_SET_CACHING, &arg) == 0;
		255	}
		256
4501	Serge	257	static uint32_t gem_userptr(int fd, void *ptr, int size, int read_only)
		258	{
		259	return 0;
		260	}
4304	Serge	261
		262	static bool __kgem_throttle_retire(struct kgem *kgem, unsigned flags)
		263	{
		264	if (flags & CREATE_NO_RETIRE) {
		265	DBG(("%s: not retiring per-request\n", __FUNCTION__));
		266	return false;
		267	}
		268
		269	if (!kgem->need_retire) {
		270	DBG(("%s: nothing to retire\n", __FUNCTION__));
		271	return false;
		272	}
		273
		274	if (kgem_retire(kgem))
		275	return true;
		276
		277	if (flags & CREATE_NO_THROTTLE \|\| !kgem->need_throttle) {
		278	DBG(("%s: not throttling\n", __FUNCTION__));
		279	return false;
		280	}
		281
		282	kgem_throttle(kgem);
		283	return kgem_retire(kgem);
		284	}
		285
		286	static void __kgem_bo_map__gtt(struct kgem kgem, struct kgem_bo *bo)
		287	{
		288	struct drm_i915_gem_mmap_gtt mmap_arg;
		289	void *ptr;
		290
		291	DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__,
		292	bo->handle, bytes(bo)));
		293	assert(bo->proxy == NULL);
		294	assert(!bo->snoop);
4501	Serge	295	assert(num_pages(bo) <= kgem->aperture_mappable / 4);
4304	Serge	296
		297	retry_gtt:
		298	VG_CLEAR(mmap_arg);
		299	mmap_arg.handle = bo->handle;
		300	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg)) {
4501	Serge	301	int err = 0;
4304	Serge	302
		303	(void)__kgem_throttle_retire(kgem, 0);
		304	if (kgem_expire_cache(kgem))
		305	goto retry_gtt;
		306
4501	Serge	307	if (kgem_cleanup_cache(kgem))
4304	Serge	308	goto retry_gtt;
		309
4501	Serge	310	ErrorF("%s: failed to retrieve GTT offset for handle=%d: %d\n",
		311	__FUNCTION__, bo->handle, err);
4304	Serge	312	return NULL;
		313	}
		314
		315	retry_mmap:
		316	ptr = (void*)(int)mmap_arg.offset;
		317	if (ptr == NULL) {
		318	ErrorF("%s: failed to mmap handle=%d, %d bytes, into GTT domain\n",
		319	__FUNCTION__, bo->handle, bytes(bo));
		320	ptr = NULL;
		321	}
		322
		323	return ptr;
		324	}
		325
4501	Serge	326	static int gem_write(int fd, uint32_t handle,
4304	Serge	327	int offset, int length,
		328	const void *src)
		329	{
		330	struct drm_i915_gem_pwrite pwrite;
		331
		332	DBG(("%s(handle=%d, offset=%d, len=%d)\n", __FUNCTION__,
		333	handle, offset, length));
		334
		335	VG_CLEAR(pwrite);
		336	pwrite.handle = handle;
		337	pwrite.offset = offset;
		338	pwrite.size = length;
		339	pwrite.data_ptr = (uintptr_t)src;
		340	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
		341	}
		342
4501	Serge	343	static int gem_write__cachealigned(int fd, uint32_t handle,
4304	Serge	344	int offset, int length,
		345	const void *src)
		346	{
		347	struct drm_i915_gem_pwrite pwrite;
		348
		349	DBG(("%s(handle=%d, offset=%d, len=%d)\n", __FUNCTION__,
		350	handle, offset, length));
		351
		352	VG_CLEAR(pwrite);
		353	pwrite.handle = handle;
		354	/* align the transfer to cachelines; fortuitously this is safe! */
		355	if ((offset \| length) & 63) {
		356	pwrite.offset = offset & ~63;
		357	pwrite.size = ALIGN(offset+length, 64) - pwrite.offset;
		358	pwrite.data_ptr = (uintptr_t)src + pwrite.offset - offset;
		359	} else {
		360	pwrite.offset = offset;
		361	pwrite.size = length;
		362	pwrite.data_ptr = (uintptr_t)src;
		363	}
		364	return drmIoctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite);
		365	}
		366
		367
		368	bool __kgem_busy(struct kgem *kgem, int handle)
		369	{
		370	struct drm_i915_gem_busy busy;
		371
		372	VG_CLEAR(busy);
		373	busy.handle = handle;
		374	busy.busy = !kgem->wedged;
		375	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
		376	DBG(("%s: handle=%d, busy=%d, wedged=%d\n",
		377	__FUNCTION__, handle, busy.busy, kgem->wedged));
		378
		379	return busy.busy;
		380	}
		381
		382	static void kgem_bo_retire(struct kgem kgem, struct kgem_bo bo)
		383	{
		384	DBG(("%s: retiring bo handle=%d (needed flush? %d), rq? %d [busy?=%d]\n",
		385	__FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL,
		386	__kgem_busy(kgem, bo->handle)));
		387	assert(bo->exec == NULL);
		388	assert(list_is_empty(&bo->vma));
		389
		390	if (bo->rq) {
		391	if (!__kgem_busy(kgem, bo->handle)) {
		392	__kgem_bo_clear_busy(bo);
		393	kgem_retire(kgem);
		394	}
		395	} else {
		396	assert(!bo->needs_flush);
		397	ASSERT_IDLE(kgem, bo->handle);
		398	}
		399	}
		400
		401	bool kgem_bo_write(struct kgem kgem, struct kgem_bo bo,
		402	const void *data, int length)
		403	{
		404	assert(bo->refcnt);
		405	assert(!bo->purged);
		406	assert(bo->proxy == NULL);
		407	ASSERT_IDLE(kgem, bo->handle);
		408
		409	assert(length <= bytes(bo));
		410	if (gem_write(kgem->fd, bo->handle, 0, length, data))
		411	return false;
		412
		413	DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain));
		414	if (bo->exec == NULL) {
		415	kgem_bo_retire(kgem, bo);
		416	bo->domain = DOMAIN_NONE;
		417	}
		418	bo->gtt_dirty = true;
		419	return true;
		420	}
		421
		422	static uint32_t gem_create(int fd, int num_pages)
		423	{
		424	struct drm_i915_gem_create create;
		425
		426	VG_CLEAR(create);
		427	create.handle = 0;
		428	create.size = PAGE_SIZE * num_pages;
		429	(void)drmIoctl(fd, DRM_IOCTL_I915_GEM_CREATE, &create);
		430
		431	return create.handle;
		432	}
		433
		434	static bool
		435	kgem_bo_set_purgeable(struct kgem kgem, struct kgem_bo bo)
		436	{
		437	#if DBG_NO_MADV
		438	return true;
		439	#else
		440	struct drm_i915_gem_madvise madv;
		441
		442	assert(bo->exec == NULL);
		443	assert(!bo->purged);
		444
		445	VG_CLEAR(madv);
		446	madv.handle = bo->handle;
		447	madv.madv = I915_MADV_DONTNEED;
		448	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
		449	bo->purged = 1;
		450	kgem->need_purge \|= !madv.retained && bo->domain == DOMAIN_GPU;
		451	return madv.retained;
		452	}
		453
		454	return true;
		455	#endif
		456	}
		457
		458	static bool
		459	kgem_bo_is_retained(struct kgem kgem, struct kgem_bo bo)
		460	{
		461	#if DBG_NO_MADV
		462	return true;
		463	#else
		464	struct drm_i915_gem_madvise madv;
		465
		466	if (!bo->purged)
		467	return true;
		468
		469	VG_CLEAR(madv);
		470	madv.handle = bo->handle;
		471	madv.madv = I915_MADV_DONTNEED;
		472	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0)
		473	return madv.retained;
		474
		475	return false;
		476	#endif
		477	}
		478
		479	static bool
		480	kgem_bo_clear_purgeable(struct kgem kgem, struct kgem_bo bo)
		481	{
		482	#if DBG_NO_MADV
		483	return true;
		484	#else
		485	struct drm_i915_gem_madvise madv;
		486
		487	assert(bo->purged);
		488
		489	VG_CLEAR(madv);
		490	madv.handle = bo->handle;
		491	madv.madv = I915_MADV_WILLNEED;
		492	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv) == 0) {
		493	bo->purged = !madv.retained;
		494	kgem->need_purge \|= !madv.retained && bo->domain == DOMAIN_GPU;
		495	return madv.retained;
		496	}
		497
		498	return false;
		499	#endif
		500	}
		501
		502	static void gem_close(int fd, uint32_t handle)
		503	{
		504	struct drm_gem_close close;
		505
		506	VG_CLEAR(close);
		507	close.handle = handle;
		508	(void)drmIoctl(fd, DRM_IOCTL_GEM_CLOSE, &close);
		509	}
		510
		511	constant inline static unsigned long __fls(unsigned long word)
		512	{
		513	#if defined(__GNUC__) && (defined(__i386__) \|\| defined(__x86__) \|\| defined(__x86_64__))
		514	asm("bsr %1,%0"
		515	: "=r" (word)
		516	: "rm" (word));
		517	return word;
		518	#else
		519	unsigned int v = 0;
		520
		521	while (word >>= 1)
		522	v++;
		523
		524	return v;
		525	#endif
		526	}
		527
		528	constant inline static int cache_bucket(int num_pages)
		529	{
		530	return __fls(num_pages);
		531	}
		532
		533	static struct kgem_bo __kgem_bo_init(struct kgem_bo bo,
		534	int handle, int num_pages)
		535	{
		536	assert(num_pages);
		537	memset(bo, 0, sizeof(*bo));
		538
		539	bo->refcnt = 1;
		540	bo->handle = handle;
		541	bo->target_handle = -1;
		542	num_pages(bo) = num_pages;
		543	bucket(bo) = cache_bucket(num_pages);
		544	bo->reusable = true;
		545	bo->domain = DOMAIN_CPU;
		546	list_init(&bo->request);
		547	list_init(&bo->list);
		548	list_init(&bo->vma);
		549
		550	return bo;
		551	}
		552
		553	static struct kgem_bo *__kgem_bo_alloc(int handle, int num_pages)
		554	{
		555	struct kgem_bo *bo;
		556
		557	if (__kgem_freed_bo) {
		558	bo = __kgem_freed_bo;
		559	__kgem_freed_bo = (struct kgem_bo *)bo;
		560	} else {
		561	bo = malloc(sizeof(*bo));
		562	if (bo == NULL)
		563	return NULL;
		564	}
		565
		566	return __kgem_bo_init(bo, handle, num_pages);
		567	}
		568
		569	static struct kgem_request __kgem_request_alloc(struct kgem kgem)
		570	{
		571	struct kgem_request *rq;
		572
		573	rq = __kgem_freed_request;
		574	if (rq) {
		575	__kgem_freed_request = (struct kgem_request *)rq;
		576	} else {
		577	rq = malloc(sizeof(*rq));
		578	if (rq == NULL)
		579	rq = &kgem->static_request;
		580	}
		581
		582	list_init(&rq->buffers);
		583	rq->bo = NULL;
		584	rq->ring = 0;
		585
		586	return rq;
		587	}
		588
		589	static void __kgem_request_free(struct kgem_request *rq)
		590	{
		591	_list_del(&rq->list);
		592	(struct kgem_request *)rq = __kgem_freed_request;
		593	__kgem_freed_request = rq;
		594	}
		595
		596	static struct list inactive(struct kgem kgem, int num_pages)
		597	{
		598	assert(num_pages < MAX_CACHE_SIZE / PAGE_SIZE);
		599	assert(cache_bucket(num_pages) < NUM_CACHE_BUCKETS);
		600	return &kgem->inactive[cache_bucket(num_pages)];
		601	}
		602
		603	static struct list active(struct kgem kgem, int num_pages, int tiling)
		604	{
		605	assert(num_pages < MAX_CACHE_SIZE / PAGE_SIZE);
		606	assert(cache_bucket(num_pages) < NUM_CACHE_BUCKETS);
		607	return &kgem->active[cache_bucket(num_pages)][tiling];
		608	}
		609
		610	static size_t
		611	agp_aperture_size(struct pci_device *dev, unsigned gen)
		612	{
		613	/* XXX assume that only future chipsets are unknown and follow
		614	* the post gen2 PCI layout.
		615	*/
		616	return 0;
		617	}
		618
		619	static size_t
		620	total_ram_size(void)
		621	{
		622	uint32_t data[9];
		623	size_t size = 0;
		624
		625	asm volatile("int $0x40"
		626	: "=a" (size)
		627	: "a" (18),"b"(20), "c" (data)
		628	: "memory");
		629
		630	return size != -1 ? size : 0;
		631	}
		632
		633	static unsigned
		634	cpu_cache_size__cpuid4(void)
		635	{
4501	Serge	636	/* Deterministic Cache Parameters (Function 04h)":
4304	Serge	637	* When EAX is initialized to a value of 4, the CPUID instruction
		638	* returns deterministic cache information in the EAX, EBX, ECX
		639	* and EDX registers. This function requires ECX be initialized
		640	* with an index which indicates which cache to return information
		641	* about. The OS is expected to call this function (CPUID.4) with
		642	* ECX = 0, 1, 2, until EAX[4:0] == 0, indicating no more caches.
		643	* The order in which the caches are returned is not specified
		644	* and may change at Intel's discretion.
		645	*
		646	* Calculating the Cache Size in bytes:
		647	* = (Ways +1) * (Partitions +1) * (Line Size +1) * (Sets +1)
		648	*/
		649
		650	unsigned int eax, ebx, ecx, edx;
		651	unsigned int llc_size = 0;
		652	int cnt = 0;
		653
		654	if (__get_cpuid_max(BASIC_CPUID, NULL) < 4)
		655	return 0;
		656
		657	do {
		658	unsigned associativity, line_partitions, line_size, sets;
		659
		660	__cpuid_count(4, cnt++, eax, ebx, ecx, edx);
		661
		662	if ((eax & 0x1f) == 0)
		663	break;
		664
		665	associativity = ((ebx >> 22) & 0x3ff) + 1;
		666	line_partitions = ((ebx >> 12) & 0x3ff) + 1;
		667	line_size = (ebx & 0xfff) + 1;
		668	sets = ecx + 1;
		669
		670	llc_size = associativity * line_partitions * line_size * sets;
		671	} while (1);
		672
		673	return llc_size;
		674	}
		675
		676	static int gem_param(struct kgem *kgem, int name)
		677	{
		678	drm_i915_getparam_t gp;
		679	int v = -1; /* No param uses the sign bit, reserve it for errors */
		680
		681	VG_CLEAR(gp);
		682	gp.param = name;
		683	gp.value = &v;
		684	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GETPARAM, &gp))
		685	return -1;
		686
		687	VG(VALGRIND_MAKE_MEM_DEFINED(&v, sizeof(v)));
		688	return v;
		689	}
		690
		691	static bool test_has_execbuffer2(struct kgem *kgem)
		692	{
		693	return 1;
		694	}
		695
		696	static bool test_has_no_reloc(struct kgem *kgem)
		697	{
		698	if (DBG_NO_FAST_RELOC)
		699	return false;
		700
		701	return gem_param(kgem, LOCAL_I915_PARAM_HAS_NO_RELOC) > 0;
		702	}
		703
		704	static bool test_has_handle_lut(struct kgem *kgem)
		705	{
		706	if (DBG_NO_HANDLE_LUT)
		707	return false;
		708
		709	return gem_param(kgem, LOCAL_I915_PARAM_HAS_HANDLE_LUT) > 0;
		710	}
		711
		712	static bool test_has_wt(struct kgem *kgem)
		713	{
		714	if (DBG_NO_WT)
		715	return false;
		716
		717	return gem_param(kgem, LOCAL_I915_PARAM_HAS_WT) > 0;
		718	}
		719
		720	static bool test_has_semaphores_enabled(struct kgem *kgem)
		721	{
		722	bool detected = false;
		723	int ret;
		724
		725	if (DBG_NO_SEMAPHORES)
		726	return false;
		727
		728	ret = gem_param(kgem, LOCAL_I915_PARAM_HAS_SEMAPHORES);
		729	if (ret != -1)
		730	return ret > 0;
		731
		732	return detected;
		733	}
		734
		735	static bool __kgem_throttle(struct kgem *kgem)
		736	{
		737	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_THROTTLE, NULL) == 0)
		738	return false;
		739
		740	return errno == EIO;
		741	}
		742
		743	static bool is_hw_supported(struct kgem *kgem,
		744	struct pci_device *dev)
		745	{
		746	if (DBG_NO_HW)
		747	return false;
		748
		749	if (!test_has_execbuffer2(kgem))
		750	return false;
		751
		752	if (kgem->gen == (unsigned)-1) /* unknown chipset, assume future gen */
		753	return kgem->has_blt;
		754
		755	/* Although pre-855gm the GMCH is fubar, it works mostly. So
		756	* let the user decide through "NoAccel" whether or not to risk
		757	* hw acceleration.
		758	*/
		759
4501	Serge	760	if (kgem->gen == 060 && dev && dev->revision < 8) {
4304	Serge	761	/* pre-production SNB with dysfunctional BLT */
		762	return false;
		763	}
		764
		765	if (kgem->gen >= 060) /* Only if the kernel supports the BLT ring */
		766	return kgem->has_blt;
		767
		768	return true;
		769	}
		770
		771	static bool test_has_relaxed_fencing(struct kgem *kgem)
		772	{
		773	if (kgem->gen < 040) {
		774	if (DBG_NO_RELAXED_FENCING)
		775	return false;
		776
		777	return gem_param(kgem, LOCAL_I915_PARAM_HAS_RELAXED_FENCING) > 0;
		778	} else
		779	return true;
		780	}
		781
		782	static bool test_has_llc(struct kgem *kgem)
		783	{
		784	int has_llc = -1;
		785
		786	if (DBG_NO_LLC)
		787	return false;
		788
		789	#if defined(I915_PARAM_HAS_LLC) /* Expected in libdrm-2.4.31 */
		790	has_llc = gem_param(kgem, I915_PARAM_HAS_LLC);
		791	#endif
		792	if (has_llc == -1) {
		793	DBG(("%s: no kernel/drm support for HAS_LLC, assuming support for LLC based on GPU generation\n", __FUNCTION__));
		794	has_llc = kgem->gen >= 060;
		795	}
		796
		797	return has_llc;
		798	}
		799
		800	static bool test_has_caching(struct kgem *kgem)
		801	{
		802	uint32_t handle;
		803	bool ret;
		804
		805	if (DBG_NO_CACHE_LEVEL)
		806	return false;
		807
		808	/* Incoherent blt and sampler hangs the GPU */
		809	if (kgem->gen == 040)
		810	return false;
		811
		812	handle = gem_create(kgem->fd, 1);
		813	if (handle == 0)
		814	return false;
		815
		816	ret = gem_set_caching(kgem->fd, handle, UNCACHED);
		817	gem_close(kgem->fd, handle);
		818	return ret;
		819	}
		820
		821	static bool test_has_userptr(struct kgem *kgem)
		822	{
		823	#if defined(USE_USERPTR)
		824	uint32_t handle;
		825	void *ptr;
		826
		827	if (DBG_NO_USERPTR)
		828	return false;
		829
		830	/* Incoherent blt and sampler hangs the GPU */
		831	if (kgem->gen == 040)
		832	return false;
		833
		834	if (posix_memalign(&ptr, PAGE_SIZE, PAGE_SIZE))
		835	return false;
		836
		837	handle = gem_userptr(kgem->fd, ptr, PAGE_SIZE, false);
		838	gem_close(kgem->fd, handle);
		839	free(ptr);
		840
		841	return handle != 0;
		842	#else
		843	return false;
		844	#endif
		845	}
		846
		847	static bool test_has_create2(struct kgem *kgem)
		848	{
		849	#if defined(USE_CREATE2)
		850	struct local_i915_gem_create2 args;
		851
		852	if (DBG_NO_CREATE2)
		853	return false;
		854
		855	memset(&args, 0, sizeof(args));
		856	args.size = PAGE_SIZE;
		857	args.caching = DISPLAY;
		858	if (drmIoctl(kgem->fd, LOCAL_IOCTL_I915_GEM_CREATE2, &args) == 0)
		859	gem_close(kgem->fd, args.handle);
		860
		861	return args.handle != 0;
		862	#else
		863	return false;
		864	#endif
		865	}
		866
		867	static bool test_has_secure_batches(struct kgem *kgem)
		868	{
		869	if (DBG_NO_SECURE_BATCHES)
		870	return false;
		871
		872	return gem_param(kgem, LOCAL_I915_PARAM_HAS_SECURE_BATCHES) > 0;
		873	}
		874
		875	static bool test_has_pinned_batches(struct kgem *kgem)
		876	{
		877	if (DBG_NO_PINNED_BATCHES)
		878	return false;
		879
		880	return gem_param(kgem, LOCAL_I915_PARAM_HAS_PINNED_BATCHES) > 0;
		881	}
		882
		883
		884	static bool kgem_init_pinned_batches(struct kgem *kgem)
		885	{
4501	Serge	886	int count[2] = { 4, 4 };
4304	Serge	887	int size[2] = { 1, 2 };
		888	int n, i;
		889
		890	if (kgem->wedged)
		891	return true;
		892
		893	for (n = 0; n < ARRAY_SIZE(count); n++) {
		894	for (i = 0; i < count[n]; i++) {
		895	struct drm_i915_gem_pin pin;
		896	struct kgem_bo *bo;
		897
		898	VG_CLEAR(pin);
		899
		900	pin.handle = gem_create(kgem->fd, size[n]);
		901	if (pin.handle == 0)
		902	goto err;
		903
		904	DBG(("%s: new handle=%d, num_pages=%d\n",
		905	__FUNCTION__, pin.handle, size[n]));
		906
		907	bo = __kgem_bo_alloc(pin.handle, size[n]);
		908	if (bo == NULL) {
		909	gem_close(kgem->fd, pin.handle);
		910	goto err;
		911	}
		912
		913	pin.alignment = 0;
		914	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_PIN, &pin)) {
		915	gem_close(kgem->fd, pin.handle);
4501	Serge	916	free(bo);
4304	Serge	917	goto err;
		918	}
		919	bo->presumed_offset = pin.offset;
		920	debug_alloc__bo(kgem, bo);
		921	list_add(&bo->list, &kgem->pinned_batches[n]);
		922	}
		923	}
		924
		925	return true;
		926
		927	err:
		928	for (n = 0; n < ARRAY_SIZE(kgem->pinned_batches); n++) {
		929	while (!list_is_empty(&kgem->pinned_batches[n])) {
		930	kgem_bo_destroy(kgem,
		931	list_first_entry(&kgem->pinned_batches[n],
		932	struct kgem_bo, list));
		933	}
		934	}
		935
		936	/* For simplicity populate the lists with a single unpinned bo */
		937	for (n = 0; n < ARRAY_SIZE(count); n++) {
		938	struct kgem_bo *bo;
		939	uint32_t handle;
		940
		941	handle = gem_create(kgem->fd, size[n]);
		942	if (handle == 0)
		943	break;
		944
		945	bo = __kgem_bo_alloc(handle, size[n]);
		946	if (bo == NULL) {
		947	gem_close(kgem->fd, handle);
		948	break;
		949	}
		950
		951	debug_alloc__bo(kgem, bo);
		952	list_add(&bo->list, &kgem->pinned_batches[n]);
		953	}
		954	return false;
		955	}
		956
		957	void kgem_init(struct kgem kgem, int fd, struct pci_device dev, unsigned gen)
		958	{
		959	struct drm_i915_gem_get_aperture aperture;
		960	size_t totalram;
		961	unsigned half_gpu_max;
		962	unsigned int i, j;
		963
		964	DBG(("%s: fd=%d, gen=%d\n", __FUNCTION__, fd, gen));
		965
		966	memset(kgem, 0, sizeof(*kgem));
		967
		968	kgem->fd = fd;
		969	kgem->gen = gen;
		970
		971	list_init(&kgem->requests[0]);
		972	list_init(&kgem->requests[1]);
		973	list_init(&kgem->batch_buffers);
		974	list_init(&kgem->active_buffers);
		975	list_init(&kgem->flushing);
		976	list_init(&kgem->large);
		977	list_init(&kgem->large_inactive);
		978	list_init(&kgem->snoop);
		979	list_init(&kgem->scanout);
		980	for (i = 0; i < ARRAY_SIZE(kgem->pinned_batches); i++)
		981	list_init(&kgem->pinned_batches[i]);
		982	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		983	list_init(&kgem->inactive[i]);
		984	for (i = 0; i < ARRAY_SIZE(kgem->active); i++) {
		985	for (j = 0; j < ARRAY_SIZE(kgem->active[i]); j++)
		986	list_init(&kgem->active[i][j]);
		987	}
		988	for (i = 0; i < ARRAY_SIZE(kgem->vma); i++) {
		989	for (j = 0; j < ARRAY_SIZE(kgem->vma[i].inactive); j++)
		990	list_init(&kgem->vma[i].inactive[j]);
		991	}
		992	kgem->vma[MAP_GTT].count = -MAX_GTT_VMA_CACHE;
		993	kgem->vma[MAP_CPU].count = -MAX_CPU_VMA_CACHE;
		994
		995	kgem->has_blt = gem_param(kgem, LOCAL_I915_PARAM_HAS_BLT) > 0;
		996	DBG(("%s: has BLT ring? %d\n", __FUNCTION__,
		997	kgem->has_blt));
		998
		999	kgem->has_relaxed_delta =
		1000	gem_param(kgem, LOCAL_I915_PARAM_HAS_RELAXED_DELTA) > 0;
		1001	DBG(("%s: has relaxed delta? %d\n", __FUNCTION__,
		1002	kgem->has_relaxed_delta));
		1003
		1004	kgem->has_relaxed_fencing = test_has_relaxed_fencing(kgem);
		1005	DBG(("%s: has relaxed fencing? %d\n", __FUNCTION__,
		1006	kgem->has_relaxed_fencing));
		1007
		1008	kgem->has_llc = test_has_llc(kgem);
		1009	DBG(("%s: has shared last-level-cache? %d\n", __FUNCTION__,
		1010	kgem->has_llc));
		1011
		1012	kgem->has_wt = test_has_wt(kgem);
		1013	DBG(("%s: has write-through caching for scanouts? %d\n", __FUNCTION__,
		1014	kgem->has_wt));
		1015
		1016	kgem->has_caching = test_has_caching(kgem);
		1017	DBG(("%s: has set-cache-level? %d\n", __FUNCTION__,
		1018	kgem->has_caching));
		1019
		1020	kgem->has_userptr = test_has_userptr(kgem);
		1021	DBG(("%s: has userptr? %d\n", __FUNCTION__,
		1022	kgem->has_userptr));
		1023
		1024	kgem->has_create2 = test_has_create2(kgem);
		1025	kgem->has_create2 = 0;
		1026	DBG(("%s: has create2? %d\n", __FUNCTION__,
		1027	kgem->has_create2));
		1028
		1029	kgem->has_no_reloc = test_has_no_reloc(kgem);
		1030	DBG(("%s: has no-reloc? %d\n", __FUNCTION__,
		1031	kgem->has_no_reloc));
		1032
		1033	kgem->has_handle_lut = test_has_handle_lut(kgem);
		1034	DBG(("%s: has handle-lut? %d\n", __FUNCTION__,
		1035	kgem->has_handle_lut));
		1036
		1037	kgem->has_semaphores = false;
		1038	if (kgem->has_blt && test_has_semaphores_enabled(kgem))
		1039	kgem->has_semaphores = true;
		1040	DBG(("%s: semaphores enabled? %d\n", __FUNCTION__,
		1041	kgem->has_semaphores));
		1042
		1043	kgem->can_blt_cpu = gen >= 030;
		1044	DBG(("%s: can blt to cpu? %d\n", __FUNCTION__,
		1045	kgem->can_blt_cpu));
		1046
4501	Serge	1047	kgem->can_render_y = gen != 021 && (gen >> 3) != 4;
		1048	DBG(("%s: can render to Y-tiled surfaces? %d\n", __FUNCTION__,
		1049	kgem->can_render_y));
		1050
4304	Serge	1051	kgem->has_secure_batches = test_has_secure_batches(kgem);
		1052	DBG(("%s: can use privileged batchbuffers? %d\n", __FUNCTION__,
		1053	kgem->has_secure_batches));
		1054
		1055	kgem->has_pinned_batches = test_has_pinned_batches(kgem);
		1056	DBG(("%s: can use pinned batchbuffers (to avoid CS w/a)? %d\n", __FUNCTION__,
		1057	kgem->has_pinned_batches));
		1058
		1059	if (!is_hw_supported(kgem, dev)) {
		1060	printf("Detected unsupported/dysfunctional hardware, disabling acceleration.\n");
		1061	kgem->wedged = 1;
		1062	} else if (__kgem_throttle(kgem)) {
		1063	printf("Detected a hung GPU, disabling acceleration.\n");
		1064	kgem->wedged = 1;
		1065	}
		1066
		1067	kgem->batch_size = ARRAY_SIZE(kgem->batch);
		1068	if (gen == 020 && !kgem->has_pinned_batches)
		1069	/* Limited to what we can pin */
		1070	kgem->batch_size = 4*1024;
		1071	if (gen == 022)
		1072	/* 865g cannot handle a batch spanning multiple pages */
		1073	kgem->batch_size = PAGE_SIZE / sizeof(uint32_t);
		1074	if ((gen >> 3) == 7)
		1075	kgem->batch_size = 16*1024;
		1076	if (!kgem->has_relaxed_delta && kgem->batch_size > 4*1024)
		1077	kgem->batch_size = 4*1024;
		1078
		1079	if (!kgem_init_pinned_batches(kgem) && gen == 020) {
		1080	printf("Unable to reserve memory for GPU, disabling acceleration.\n");
		1081	kgem->wedged = 1;
		1082	}
		1083
		1084	DBG(("%s: maximum batch size? %d\n", __FUNCTION__,
		1085	kgem->batch_size));
		1086
4403	Serge	1087	kgem->min_alignment = 16;
4304	Serge	1088	if (gen < 040)
		1089	kgem->min_alignment = 64;
		1090
		1091	kgem->half_cpu_cache_pages = cpu_cache_size() >> 13;
		1092	DBG(("%s: last-level cache size: %d bytes, threshold in pages: %d\n",
		1093	__FUNCTION__, cpu_cache_size(), kgem->half_cpu_cache_pages));
		1094
		1095	kgem->next_request = __kgem_request_alloc(kgem);
		1096
		1097	DBG(("%s: cpu bo enabled %d: llc? %d, set-cache-level? %d, userptr? %d\n", __FUNCTION__,
		1098	!DBG_NO_CPU && (kgem->has_llc \| kgem->has_userptr \| kgem->has_caching),
		1099	kgem->has_llc, kgem->has_caching, kgem->has_userptr));
		1100
		1101	VG_CLEAR(aperture);
		1102	aperture.aper_size = 0;
		1103	(void)drmIoctl(fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
		1104	if (aperture.aper_size == 0)
		1105	aperture.aper_size = 6410241024;
		1106
		1107	DBG(("%s: aperture size %lld, available now %lld\n",
		1108	__FUNCTION__,
		1109	(long long)aperture.aper_size,
		1110	(long long)aperture.aper_available_size));
		1111
		1112	kgem->aperture_total = aperture.aper_size;
		1113	kgem->aperture_high = aperture.aper_size * 3/4;
		1114	kgem->aperture_low = aperture.aper_size * 1/3;
		1115	if (gen < 033) {
		1116	/* Severe alignment penalties */
		1117	kgem->aperture_high /= 2;
		1118	kgem->aperture_low /= 2;
		1119	}
		1120	DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
		1121	kgem->aperture_low, kgem->aperture_low / (1024*1024),
		1122	kgem->aperture_high, kgem->aperture_high / (1024*1024)));
		1123
4501	Serge	1124	kgem->aperture_mappable = 256 * 1024 * 1024;
		1125	if (dev != NULL)
4304	Serge	1126	kgem->aperture_mappable = agp_aperture_size(dev, gen);
		1127	if (kgem->aperture_mappable == 0 \|\|
		1128	kgem->aperture_mappable > aperture.aper_size)
		1129	kgem->aperture_mappable = aperture.aper_size;
		1130	DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
		1131	kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));
		1132
		1133	kgem->buffer_size = 64 * 1024;
		1134	while (kgem->buffer_size < kgem->aperture_mappable >> 10)
		1135	kgem->buffer_size *= 2;
		1136	if (kgem->buffer_size >> 12 > kgem->half_cpu_cache_pages)
		1137	kgem->buffer_size = kgem->half_cpu_cache_pages << 12;
		1138	kgem->buffer_size = 1 << __fls(kgem->buffer_size);
		1139	DBG(("%s: buffer size=%d [%d KiB]\n", __FUNCTION__,
		1140	kgem->buffer_size, kgem->buffer_size / 1024));
		1141	assert(kgem->buffer_size);
		1142
		1143	kgem->max_object_size = 3 * (kgem->aperture_high >> 12) << 10;
		1144	kgem->max_gpu_size = kgem->max_object_size;
		1145	if (!kgem->has_llc && kgem->max_gpu_size > MAX_CACHE_SIZE)
		1146	kgem->max_gpu_size = MAX_CACHE_SIZE;
		1147
		1148	totalram = total_ram_size();
		1149	if (totalram == 0) {
		1150	DBG(("%s: total ram size unknown, assuming maximum of total aperture\n",
		1151	__FUNCTION__));
		1152	totalram = kgem->aperture_total;
		1153	}
		1154	DBG(("%s: total ram=%ld\n", __FUNCTION__, (long)totalram));
		1155	if (kgem->max_object_size > totalram / 2)
		1156	kgem->max_object_size = totalram / 2;
		1157	if (kgem->max_gpu_size > totalram / 4)
		1158	kgem->max_gpu_size = totalram / 4;
		1159
4501	Serge	1160	if (kgem->aperture_high > totalram / 2) {
		1161	kgem->aperture_high = totalram / 2;
		1162	kgem->aperture_low = kgem->aperture_high / 4;
		1163	DBG(("%s: reduced aperture watermaks to fit into ram; low=%d [%d], high=%d [%d]\n", __FUNCTION__,
		1164	kgem->aperture_low, kgem->aperture_low / (1024*1024),
		1165	kgem->aperture_high, kgem->aperture_high / (1024*1024)));
		1166	}
		1167
4304	Serge	1168	kgem->max_cpu_size = kgem->max_object_size;
		1169
		1170	half_gpu_max = kgem->max_gpu_size / 2;
		1171	kgem->max_copy_tile_size = (MAX_CACHE_SIZE + 1)/2;
		1172	if (kgem->max_copy_tile_size > half_gpu_max)
		1173	kgem->max_copy_tile_size = half_gpu_max;
		1174
		1175	if (kgem->has_llc)
		1176	kgem->max_upload_tile_size = kgem->max_copy_tile_size;
		1177	else
		1178	kgem->max_upload_tile_size = kgem->aperture_mappable / 4;
		1179	if (kgem->max_upload_tile_size > half_gpu_max)
		1180	kgem->max_upload_tile_size = half_gpu_max;
		1181	if (kgem->max_upload_tile_size > kgem->aperture_high/2)
		1182	kgem->max_upload_tile_size = kgem->aperture_high/2;
		1183	if (kgem->max_upload_tile_size > kgem->aperture_low)
		1184	kgem->max_upload_tile_size = kgem->aperture_low;
		1185	if (kgem->max_upload_tile_size < 16*PAGE_SIZE)
		1186	kgem->max_upload_tile_size = 16*PAGE_SIZE;
		1187
		1188	kgem->large_object_size = MAX_CACHE_SIZE;
		1189	if (kgem->large_object_size > half_gpu_max)
		1190	kgem->large_object_size = half_gpu_max;
		1191	if (kgem->max_copy_tile_size > kgem->aperture_high/2)
		1192	kgem->max_copy_tile_size = kgem->aperture_high/2;
		1193	if (kgem->max_copy_tile_size > kgem->aperture_low)
		1194	kgem->max_copy_tile_size = kgem->aperture_low;
		1195	if (kgem->max_copy_tile_size < 16*PAGE_SIZE)
		1196	kgem->max_copy_tile_size = 16*PAGE_SIZE;
		1197
		1198	if (kgem->has_llc \| kgem->has_caching \| kgem->has_userptr) {
		1199	if (kgem->large_object_size > kgem->max_cpu_size)
		1200	kgem->large_object_size = kgem->max_cpu_size;
		1201	} else
		1202	kgem->max_cpu_size = 0;
		1203	if (DBG_NO_CPU)
		1204	kgem->max_cpu_size = 0;
		1205
		1206	DBG(("%s: maximum object size=%d\n",
		1207	__FUNCTION__, kgem->max_object_size));
		1208	DBG(("%s: large object thresold=%d\n",
		1209	__FUNCTION__, kgem->large_object_size));
		1210	DBG(("%s: max object sizes (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
		1211	__FUNCTION__,
		1212	kgem->max_gpu_size, kgem->max_cpu_size,
		1213	kgem->max_upload_tile_size, kgem->max_copy_tile_size));
		1214
		1215	/* Convert the aperture thresholds to pages */
4501	Serge	1216	kgem->aperture_mappable /= PAGE_SIZE;
4304	Serge	1217	kgem->aperture_low /= PAGE_SIZE;
		1218	kgem->aperture_high /= PAGE_SIZE;
4501	Serge	1219	kgem->aperture_total /= PAGE_SIZE;
4304	Serge	1220
		1221	kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
		1222	if ((int)kgem->fence_max < 0)
		1223	kgem->fence_max = 5; /* minimum safe value for all hw */
		1224	DBG(("%s: max fences=%d\n", __FUNCTION__, kgem->fence_max));
		1225
		1226	kgem->batch_flags_base = 0;
		1227	if (kgem->has_no_reloc)
		1228	kgem->batch_flags_base \|= LOCAL_I915_EXEC_NO_RELOC;
		1229	if (kgem->has_handle_lut)
		1230	kgem->batch_flags_base \|= LOCAL_I915_EXEC_HANDLE_LUT;
		1231	if (kgem->has_pinned_batches)
		1232	kgem->batch_flags_base \|= LOCAL_I915_EXEC_IS_PINNED;
		1233	}
		1234
		1235	/* XXX hopefully a good approximation */
		1236	uint32_t kgem_get_unique_id(struct kgem *kgem)
		1237	{
		1238	uint32_t id;
		1239	id = ++kgem->unique_id;
		1240	if (id == 0)
		1241	id = ++kgem->unique_id;
		1242	return id;
		1243	}
		1244
		1245	inline static uint32_t kgem_pitch_alignment(struct kgem *kgem, unsigned flags)
		1246	{
		1247	if (flags & CREATE_PRIME)
		1248	return 256;
		1249	if (flags & CREATE_SCANOUT)
		1250	return 64;
		1251	return kgem->min_alignment;
		1252	}
		1253
4501	Serge	1254	void kgem_get_tile_size(struct kgem *kgem, int tiling, int pitch,
4304	Serge	1255	int tile_width, int tile_height, int *tile_size)
		1256	{
		1257	if (kgem->gen <= 030) {
		1258	if (tiling) {
		1259	if (kgem->gen < 030) {
		1260	*tile_width = 128;
		1261	*tile_height = 16;
		1262	*tile_size = 2048;
		1263	} else {
		1264	*tile_width = 512;
		1265	*tile_height = 8;
		1266	*tile_size = 4096;
		1267	}
		1268	} else {
		1269	*tile_width = 1;
		1270	*tile_height = 1;
		1271	*tile_size = 1;
		1272	}
		1273	} else switch (tiling) {
		1274	default:
		1275	case I915_TILING_NONE:
		1276	*tile_width = 1;
		1277	*tile_height = 1;
		1278	*tile_size = 1;
		1279	break;
		1280	case I915_TILING_X:
		1281	*tile_width = 512;
		1282	*tile_height = 8;
		1283	*tile_size = 4096;
		1284	break;
		1285	case I915_TILING_Y:
		1286	*tile_width = 128;
		1287	*tile_height = 32;
		1288	*tile_size = 4096;
		1289	break;
		1290	}
4501	Serge	1291
		1292	/* Force offset alignment to tile-row */
		1293	if (tiling && kgem->gen < 033)
		1294	*tile_width = pitch;
4304	Serge	1295	}
		1296
		1297	uint32_t kgem_surface_size(struct kgem *kgem,
		1298	bool relaxed_fencing,
		1299	unsigned flags,
		1300	uint32_t width,
		1301	uint32_t height,
		1302	uint32_t bpp,
		1303	uint32_t tiling,
		1304	uint32_t *pitch)
		1305	{
		1306	uint32_t tile_width, tile_height;
		1307	uint32_t size;
		1308
		1309	assert(width <= MAXSHORT);
		1310	assert(height <= MAXSHORT);
		1311	assert(bpp >= 8);
		1312
		1313	if (kgem->gen <= 030) {
		1314	if (tiling) {
		1315	if (kgem->gen < 030) {
		1316	tile_width = 128;
		1317	tile_height = 32;
		1318	} else {
		1319	tile_width = 512;
		1320	tile_height = 16;
		1321	}
		1322	} else {
		1323	tile_width = 2 * bpp >> 3;
		1324	tile_width = ALIGN(tile_width,
		1325	kgem_pitch_alignment(kgem, flags));
		1326	tile_height = 2;
		1327	}
		1328	} else switch (tiling) {
		1329	default:
		1330	case I915_TILING_NONE:
		1331	tile_width = 2 * bpp >> 3;
		1332	tile_width = ALIGN(tile_width,
		1333	kgem_pitch_alignment(kgem, flags));
		1334	tile_height = 2;
		1335	break;
		1336
		1337	/* XXX align to an even tile row */
		1338	case I915_TILING_X:
		1339	tile_width = 512;
		1340	tile_height = 16;
		1341	break;
		1342	case I915_TILING_Y:
		1343	tile_width = 128;
		1344	tile_height = 64;
		1345	break;
		1346	}
		1347
		1348	pitch = ALIGN(width bpp / 8, tile_width);
		1349	height = ALIGN(height, tile_height);
		1350	if (kgem->gen >= 040)
		1351	return PAGE_ALIGN(pitch height);
		1352
		1353	/* If it is too wide for the blitter, don't even bother. */
		1354	if (tiling != I915_TILING_NONE) {
		1355	if (*pitch > 8192)
		1356	return 0;
		1357
		1358	for (size = tile_width; size < *pitch; size <<= 1)
		1359	;
		1360	*pitch = size;
		1361	} else {
		1362	if (*pitch >= 32768)
		1363	return 0;
		1364	}
		1365
		1366	size = pitch height;
		1367	if (relaxed_fencing \|\| tiling == I915_TILING_NONE)
		1368	return PAGE_ALIGN(size);
		1369
		1370	/* We need to allocate a pot fence region for a tiled buffer. */
		1371	if (kgem->gen < 030)
		1372	tile_width = 512 * 1024;
		1373	else
		1374	tile_width = 1024 * 1024;
		1375	while (tile_width < size)
		1376	tile_width *= 2;
		1377	return tile_width;
		1378	}
		1379
		1380	static uint32_t kgem_aligned_height(struct kgem *kgem,
		1381	uint32_t height, uint32_t tiling)
		1382	{
		1383	uint32_t tile_height;
		1384
		1385	if (kgem->gen <= 030) {
		1386	tile_height = tiling ? kgem->gen < 030 ? 32 : 16 : 1;
		1387	} else switch (tiling) {
		1388	/* XXX align to an even tile row */
		1389	default:
		1390	case I915_TILING_NONE:
		1391	tile_height = 1;
		1392	break;
		1393	case I915_TILING_X:
		1394	tile_height = 16;
		1395	break;
		1396	case I915_TILING_Y:
		1397	tile_height = 64;
		1398	break;
		1399	}
		1400
		1401	return ALIGN(height, tile_height);
		1402	}
		1403
		1404	static struct drm_i915_gem_exec_object2 *
		1405	kgem_add_handle(struct kgem kgem, struct kgem_bo bo)
		1406	{
		1407	struct drm_i915_gem_exec_object2 *exec;
		1408
		1409	DBG(("%s: handle=%d, index=%d\n",
		1410	__FUNCTION__, bo->handle, kgem->nexec));
		1411
		1412	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		1413	bo->target_handle = kgem->has_handle_lut ? kgem->nexec : bo->handle;
		1414	exec = memset(&kgem->exec[kgem->nexec++], 0, sizeof(*exec));
		1415	exec->handle = bo->handle;
		1416	exec->offset = bo->presumed_offset;
		1417
		1418	kgem->aperture += num_pages(bo);
		1419
		1420	return exec;
		1421	}
		1422
		1423	static void kgem_add_bo(struct kgem kgem, struct kgem_bo bo)
		1424	{
4501	Serge	1425	assert(bo->refcnt);
		1426	assert(bo->proxy == NULL);
		1427
4304	Serge	1428	bo->exec = kgem_add_handle(kgem, bo);
		1429	bo->rq = MAKE_REQUEST(kgem->next_request, kgem->ring);
		1430
		1431	list_move_tail(&bo->request, &kgem->next_request->buffers);
4501	Serge	1432	if (bo->io && !list_is_empty(&bo->list))
		1433	list_move(&bo->list, &kgem->batch_buffers);
4304	Serge	1434
		1435	/* XXX is it worth working around gcc here? */
		1436	kgem->flush \|= bo->flush;
		1437	}
		1438
		1439	static uint32_t kgem_end_batch(struct kgem *kgem)
		1440	{
		1441	kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
		1442	if (kgem->nbatch & 1)
		1443	kgem->batch[kgem->nbatch++] = MI_NOOP;
		1444
		1445	return kgem->nbatch;
		1446	}
		1447
		1448	static void kgem_fixup_self_relocs(struct kgem kgem, struct kgem_bo bo)
		1449	{
		1450	int n;
		1451
		1452	assert(kgem->nreloc__self <= 256);
		1453	if (kgem->nreloc__self == 0)
		1454	return;
		1455
		1456	for (n = 0; n < kgem->nreloc__self; n++) {
		1457	int i = kgem->reloc__self[n];
		1458	assert(kgem->reloc[i].target_handle == ~0U);
		1459	kgem->reloc[i].target_handle = bo->target_handle;
		1460	kgem->reloc[i].presumed_offset = bo->presumed_offset;
		1461	kgem->batch[kgem->reloc[i].offset/sizeof(kgem->batch[0])] =
		1462	kgem->reloc[i].delta + bo->presumed_offset;
		1463	}
		1464
		1465	if (n == 256) {
		1466	for (n = kgem->reloc__self[255]; n < kgem->nreloc; n++) {
		1467	if (kgem->reloc[n].target_handle == ~0U) {
		1468	kgem->reloc[n].target_handle = bo->target_handle;
		1469	kgem->reloc[n].presumed_offset = bo->presumed_offset;
		1470	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		1471	kgem->reloc[n].delta + bo->presumed_offset;
		1472	}
		1473	}
		1474
		1475	}
		1476
		1477	}
		1478
		1479	static void kgem_bo_binding_free(struct kgem kgem, struct kgem_bo bo)
		1480	{
		1481	struct kgem_bo_binding *b;
		1482
		1483	b = bo->binding.next;
		1484	while (b) {
		1485	struct kgem_bo_binding *next = b->next;
4501	Serge	1486	free(b);
4304	Serge	1487	b = next;
		1488	}
		1489	}
		1490
		1491	static void kgem_bo_free(struct kgem kgem, struct kgem_bo bo)
		1492	{
		1493	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		1494	assert(bo->refcnt == 0);
		1495	assert(bo->proxy == NULL);
		1496	assert(bo->exec == NULL);
		1497	assert(!bo->snoop \|\| bo->rq == NULL);
		1498
		1499	#ifdef DEBUG_MEMORY
		1500	kgem->debug_memory.bo_allocs--;
		1501	kgem->debug_memory.bo_bytes -= bytes(bo);
		1502	#endif
		1503
		1504	kgem_bo_binding_free(kgem, bo);
		1505
4501	Serge	1506	if (IS_USER_MAP(bo->map__cpu)) {
4304	Serge	1507	assert(bo->rq == NULL);
		1508	assert(!__kgem_busy(kgem, bo->handle));
4501	Serge	1509	assert(MAP(bo->map__cpu) != bo \|\| bo->io \|\| bo->flush);
4304	Serge	1510	if (!(bo->io \|\| bo->flush)) {
		1511	DBG(("%s: freeing snooped base\n", __FUNCTION__));
4501	Serge	1512	assert(bo != MAP(bo->map__cpu));
		1513	free(MAP(bo->map__cpu));
4304	Serge	1514	}
4501	Serge	1515	bo->map__cpu = NULL;
4304	Serge	1516	}
		1517
4501	Serge	1518	DBG(("%s: releasing %p:%p vma for handle=%d, count=%d\n",
		1519	__FUNCTION__, bo->map__gtt, bo->map__cpu,
		1520	bo->handle, list_is_empty(&bo->vma) ? 0 : kgem->vma[bo->map__gtt == NULL].count));
		1521
		1522	if (!list_is_empty(&bo->vma)) {
		1523	_list_del(&bo->vma);
		1524	kgem->vma[bo->map__gtt == NULL].count--;
		1525	}
		1526
		1527	// if (bo->map__gtt)
		1528	// munmap(MAP(bo->map__gtt), bytes(bo));
		1529	// if (bo->map__cpu)
		1530	// munmap(MAP(bo->map__cpu), bytes(bo));
		1531
4304	Serge	1532	_list_del(&bo->list);
		1533	_list_del(&bo->request);
		1534	gem_close(kgem->fd, bo->handle);
		1535
		1536	if (!bo->io) {
		1537	(struct kgem_bo *)bo = __kgem_freed_bo;
		1538	__kgem_freed_bo = bo;
		1539	} else
		1540	free(bo);
		1541	}
		1542
		1543	inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
		1544	struct kgem_bo *bo)
		1545	{
		1546	DBG(("%s: moving handle=%d to inactive\n", __FUNCTION__, bo->handle));
		1547
		1548	assert(bo->refcnt == 0);
		1549	assert(bo->reusable);
		1550	assert(bo->rq == NULL);
		1551	assert(bo->exec == NULL);
		1552	assert(bo->domain != DOMAIN_GPU);
		1553	assert(!bo->proxy);
		1554	assert(!bo->io);
		1555	assert(!bo->scanout);
		1556	assert(!bo->snoop);
		1557	assert(!bo->flush);
		1558	assert(!bo->needs_flush);
		1559	assert(list_is_empty(&bo->vma));
		1560	assert_tiling(kgem, bo);
		1561	ASSERT_IDLE(kgem, bo->handle);
		1562
		1563	kgem->need_expire = true;
		1564
		1565	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
4501	Serge	1566	if (bo->map__gtt) {
		1567	// munmap(MAP(bo->map__gtt), bytes(bo));
		1568	bo->map__gtt = NULL;
4304	Serge	1569	}
		1570
4501	Serge	1571	list_move(&bo->list, &kgem->large_inactive);
		1572	} else {
4304	Serge	1573	assert(bo->flush == false);
		1574	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
4501	Serge	1575	if (bo->map__gtt) {
		1576	if (!kgem_bo_can_map(kgem, bo)) {
		1577	// munmap(MAP(bo->map__gtt), bytes(bo));
		1578	bo->map__gtt = NULL;
		1579	}
		1580	if (bo->map__gtt) {
		1581	list_add(&bo->vma, &kgem->vma[0].inactive[bucket(bo)]);
		1582	kgem->vma[0].count++;
		1583	}
4304	Serge	1584	}
4501	Serge	1585	if (bo->map__cpu && !bo->map__gtt) {
		1586	list_add(&bo->vma, &kgem->vma[1].inactive[bucket(bo)]);
		1587	kgem->vma[1].count++;
4304	Serge	1588	}
		1589	}
		1590	}
		1591
		1592	static struct kgem_bo kgem_bo_replace_io(struct kgem_bo bo)
		1593	{
		1594	struct kgem_bo *base;
		1595
		1596	if (!bo->io)
		1597	return bo;
		1598
		1599	assert(!bo->snoop);
4501	Serge	1600	if (__kgem_freed_bo) {
		1601	base = __kgem_freed_bo;
		1602	__kgem_freed_bo = (struct kgem_bo *)base;
		1603	} else
4304	Serge	1604	base = malloc(sizeof(*base));
		1605	if (base) {
		1606	DBG(("%s: transferring io handle=%d to bo\n",
		1607	__FUNCTION__, bo->handle));
		1608	/* transfer the handle to a minimum bo */
		1609	memcpy(base, bo, sizeof(*base));
		1610	base->io = false;
		1611	list_init(&base->list);
		1612	list_replace(&bo->request, &base->request);
		1613	list_replace(&bo->vma, &base->vma);
		1614	free(bo);
		1615	bo = base;
		1616	} else
		1617	bo->reusable = false;
		1618
		1619	return bo;
		1620	}
		1621
		1622	inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
		1623	struct kgem_bo *bo)
		1624	{
		1625	DBG(("%s: removing handle=%d from inactive\n", __FUNCTION__, bo->handle));
		1626
		1627	list_del(&bo->list);
		1628	assert(bo->rq == NULL);
		1629	assert(bo->exec == NULL);
4501	Serge	1630	if (!list_is_empty(&bo->vma)) {
		1631	assert(bo->map__gtt \|\| bo->map__cpu);
4304	Serge	1632	list_del(&bo->vma);
4501	Serge	1633	kgem->vma[bo->map__gtt == NULL].count--;
4304	Serge	1634	}
		1635	}
		1636
		1637	inline static void kgem_bo_remove_from_active(struct kgem *kgem,
		1638	struct kgem_bo *bo)
		1639	{
		1640	DBG(("%s: removing handle=%d from active\n", __FUNCTION__, bo->handle));
		1641
		1642	list_del(&bo->list);
		1643	assert(bo->rq != NULL);
4501	Serge	1644	if (RQ(bo->rq) == (void *)kgem) {
		1645	assert(bo->exec == NULL);
4304	Serge	1646	list_del(&bo->request);
4501	Serge	1647	}
4304	Serge	1648	assert(list_is_empty(&bo->vma));
		1649	}
		1650
		1651	static void _kgem_bo_delete_buffer(struct kgem kgem, struct kgem_bo bo)
		1652	{
		1653	struct kgem_buffer io = (struct kgem_buffer )bo->proxy;
		1654
		1655	DBG(("%s: size=%d, offset=%d, parent used=%d\n",
		1656	__FUNCTION__, bo->size.bytes, bo->delta, io->used));
		1657
		1658	if (ALIGN(bo->delta + bo->size.bytes, UPLOAD_ALIGNMENT) == io->used)
		1659	io->used = bo->delta;
		1660	}
		1661
		1662	static void kgem_bo_move_to_scanout(struct kgem kgem, struct kgem_bo bo)
		1663	{
		1664	assert(bo->refcnt == 0);
		1665	assert(bo->scanout);
		1666	assert(bo->delta);
		1667	assert(!bo->flush);
		1668	assert(!bo->snoop);
		1669	assert(!bo->io);
		1670
		1671	if (bo->purged) {
		1672	DBG(("%s: discarding purged scanout - external name?\n",
		1673	__FUNCTION__));
		1674	kgem_bo_free(kgem, bo);
		1675	return;
		1676	}
		1677
		1678	DBG(("%s: moving %d [fb %d] to scanout cache, active? %d\n",
		1679	__FUNCTION__, bo->handle, bo->delta, bo->rq != NULL));
		1680	if (bo->rq)
		1681	list_move_tail(&bo->list, &kgem->scanout);
		1682	else
		1683	list_move(&bo->list, &kgem->scanout);
		1684	}
		1685
		1686	static void kgem_bo_move_to_snoop(struct kgem kgem, struct kgem_bo bo)
		1687	{
		1688	assert(bo->reusable);
		1689	assert(!bo->flush);
		1690	assert(!bo->needs_flush);
		1691	assert(bo->refcnt == 0);
		1692	assert(bo->exec == NULL);
		1693
		1694	if (num_pages(bo) > kgem->max_cpu_size >> 13) {
		1695	DBG(("%s handle=%d discarding large CPU buffer (%d >%d pages)\n",
		1696	__FUNCTION__, bo->handle, num_pages(bo), kgem->max_cpu_size >> 13));
		1697	kgem_bo_free(kgem, bo);
		1698	return;
		1699	}
		1700
		1701	assert(bo->tiling == I915_TILING_NONE);
		1702	assert(bo->rq == NULL);
		1703
		1704	DBG(("%s: moving %d to snoop cachee\n", __FUNCTION__, bo->handle));
		1705	list_add(&bo->list, &kgem->snoop);
		1706	}
		1707
		1708	static struct kgem_bo *
		1709	search_snoop_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		1710	{
		1711	struct kgem_bo bo, first = NULL;
		1712
		1713	DBG(("%s: num_pages=%d, flags=%x\n", __FUNCTION__, num_pages, flags));
		1714
		1715	if ((kgem->has_caching \| kgem->has_userptr) == 0)
		1716	return NULL;
		1717
		1718	if (list_is_empty(&kgem->snoop)) {
		1719	DBG(("%s: inactive and cache empty\n", __FUNCTION__));
		1720	if (!__kgem_throttle_retire(kgem, flags)) {
		1721	DBG(("%s: nothing retired\n", __FUNCTION__));
		1722	return NULL;
		1723	}
		1724	}
		1725
		1726	list_for_each_entry(bo, &kgem->snoop, list) {
		1727	assert(bo->refcnt == 0);
		1728	assert(bo->snoop);
		1729	assert(!bo->scanout);
		1730	assert(!bo->purged);
		1731	assert(bo->proxy == NULL);
		1732	assert(bo->tiling == I915_TILING_NONE);
		1733	assert(bo->rq == NULL);
		1734	assert(bo->exec == NULL);
		1735
		1736	if (num_pages > num_pages(bo))
		1737	continue;
		1738
		1739	if (num_pages(bo) > 2*num_pages) {
		1740	if (first == NULL)
		1741	first = bo;
		1742	continue;
		1743	}
		1744
		1745	list_del(&bo->list);
		1746	bo->pitch = 0;
		1747	bo->delta = 0;
		1748
		1749	DBG((" %s: found handle=%d (num_pages=%d) in snoop cache\n",
		1750	__FUNCTION__, bo->handle, num_pages(bo)));
		1751	return bo;
		1752	}
		1753
		1754	if (first) {
		1755	list_del(&first->list);
		1756	first->pitch = 0;
		1757	first->delta = 0;
		1758
		1759	DBG((" %s: found handle=%d (num_pages=%d) in snoop cache\n",
		1760	__FUNCTION__, first->handle, num_pages(first)));
		1761	return first;
		1762	}
		1763
		1764	return NULL;
		1765	}
		1766
		1767	void kgem_bo_undo(struct kgem kgem, struct kgem_bo bo)
		1768	{
		1769	if (kgem->nexec != 1 \|\| bo->exec == NULL)
		1770	return;
		1771
4501	Serge	1772	assert(bo);
4304	Serge	1773	DBG(("%s: only handle in batch, discarding last operations for handle=%d\n",
		1774	__FUNCTION__, bo->handle));
		1775
		1776	assert(bo->exec == &kgem->exec[0]);
		1777	assert(kgem->exec[0].handle == bo->handle);
		1778	assert(RQ(bo->rq) == kgem->next_request);
		1779
		1780	bo->refcnt++;
		1781	kgem_reset(kgem);
		1782	bo->refcnt--;
4501	Serge	1783
		1784	assert(kgem->nreloc == 0);
		1785	assert(kgem->nexec == 0);
		1786	assert(bo->exec == NULL);
4304	Serge	1787	}
		1788
		1789	static void __kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		1790	{
		1791	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		1792
		1793	assert(list_is_empty(&bo->list));
		1794	assert(bo->refcnt == 0);
		1795	assert(!bo->purged \|\| !bo->reusable);
		1796	assert(bo->proxy == NULL);
		1797	assert_tiling(kgem, bo);
		1798
		1799	bo->binding.offset = 0;
		1800
		1801	if (DBG_NO_CACHE)
		1802	goto destroy;
		1803
		1804	if (bo->snoop && !bo->flush) {
		1805	DBG(("%s: handle=%d is snooped\n", __FUNCTION__, bo->handle));
		1806	assert(bo->reusable);
		1807	assert(list_is_empty(&bo->list));
		1808	if (bo->exec == NULL && bo->rq && !__kgem_busy(kgem, bo->handle))
		1809	__kgem_bo_clear_busy(bo);
		1810	if (bo->rq == NULL)
		1811	kgem_bo_move_to_snoop(kgem, bo);
		1812	return;
		1813	}
4501	Serge	1814	if (!IS_USER_MAP(bo->map__cpu))
4304	Serge	1815	bo->flush = false;
		1816
		1817	if (bo->scanout) {
		1818	kgem_bo_move_to_scanout(kgem, bo);
		1819	return;
		1820	}
		1821
		1822	if (bo->io)
		1823	bo = kgem_bo_replace_io(bo);
		1824	if (!bo->reusable) {
		1825	DBG(("%s: handle=%d, not reusable\n",
		1826	__FUNCTION__, bo->handle));
		1827	goto destroy;
		1828	}
		1829
		1830	assert(list_is_empty(&bo->vma));
		1831	assert(list_is_empty(&bo->list));
		1832	assert(bo->flush == false);
		1833	assert(bo->snoop == false);
		1834	assert(bo->io == false);
		1835	assert(bo->scanout == false);
		1836
		1837	kgem_bo_undo(kgem, bo);
		1838	assert(bo->refcnt == 0);
		1839
		1840	if (bo->rq && bo->exec == NULL && !__kgem_busy(kgem, bo->handle))
		1841	__kgem_bo_clear_busy(bo);
		1842
		1843	if (bo->rq) {
		1844	struct list *cache;
		1845
		1846	DBG(("%s: handle=%d -> active\n", __FUNCTION__, bo->handle));
		1847	if (bucket(bo) < NUM_CACHE_BUCKETS)
		1848	cache = &kgem->active[bucket(bo)][bo->tiling];
		1849	else
		1850	cache = &kgem->large;
		1851	list_add(&bo->list, cache);
		1852	return;
		1853	}
		1854
		1855	assert(bo->exec == NULL);
		1856	assert(list_is_empty(&bo->request));
		1857
4501	Serge	1858	if (bo->map__cpu == NULL \|\| bucket(bo) >= NUM_CACHE_BUCKETS) {
4304	Serge	1859	if (!kgem_bo_set_purgeable(kgem, bo))
		1860	goto destroy;
		1861
		1862	if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
		1863	goto destroy;
		1864
		1865	DBG(("%s: handle=%d, purged\n",
		1866	__FUNCTION__, bo->handle));
		1867	}
		1868
		1869	kgem_bo_move_to_inactive(kgem, bo);
		1870	return;
		1871
		1872	destroy:
		1873	if (!bo->exec)
		1874	kgem_bo_free(kgem, bo);
		1875	}
		1876
		1877	static void kgem_bo_unref(struct kgem kgem, struct kgem_bo bo)
		1878	{
		1879	assert(bo->refcnt);
		1880	if (--bo->refcnt == 0)
		1881	__kgem_bo_destroy(kgem, bo);
		1882	}
		1883
		1884	static void kgem_buffer_release(struct kgem kgem, struct kgem_buffer bo)
		1885	{
4501	Serge	1886	assert(bo->base.io);
4304	Serge	1887	while (!list_is_empty(&bo->base.vma)) {
		1888	struct kgem_bo *cached;
		1889
		1890	cached = list_first_entry(&bo->base.vma, struct kgem_bo, vma);
		1891	assert(cached->proxy == &bo->base);
4501	Serge	1892	assert(cached != &bo->base);
4304	Serge	1893	list_del(&cached->vma);
		1894
4501	Serge	1895	assert((struct kgem_bo *)cached->map__gtt == cached);
		1896	(struct kgem_bo *)cached->map__gtt = NULL;
		1897	cached->map__gtt = NULL;
4304	Serge	1898
		1899	kgem_bo_destroy(kgem, cached);
		1900	}
		1901	}
		1902
		1903	static bool kgem_retire__buffers(struct kgem *kgem)
		1904	{
		1905	bool retired = false;
		1906
		1907	while (!list_is_empty(&kgem->active_buffers)) {
		1908	struct kgem_buffer *bo =
		1909	list_last_entry(&kgem->active_buffers,
		1910	struct kgem_buffer,
		1911	base.list);
		1912
4501	Serge	1913	DBG(("%s: handle=%d, busy? %d [%d]\n",
		1914	__FUNCTION__, bo->base.handle, bo->base.rq != NULL, bo->base.exec != NULL));
		1915
		1916	assert(bo->base.exec == NULL \|\| RQ(bo->base.rq) == kgem->next_request);
4304	Serge	1917	if (bo->base.rq)
		1918	break;
		1919
		1920	DBG(("%s: releasing upload cache for handle=%d? %d\n",
		1921	__FUNCTION__, bo->base.handle, !list_is_empty(&bo->base.vma)));
		1922	list_del(&bo->base.list);
		1923	kgem_buffer_release(kgem, bo);
		1924	kgem_bo_unref(kgem, &bo->base);
		1925	retired = true;
		1926	}
		1927
		1928	return retired;
		1929	}
		1930
		1931	static bool kgem_retire__flushing(struct kgem *kgem)
		1932	{
		1933	struct kgem_bo bo, next;
		1934	bool retired = false;
		1935
		1936	list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
4501	Serge	1937	assert(RQ(bo->rq) == (void *)kgem);
4304	Serge	1938	assert(bo->exec == NULL);
		1939
		1940	if (__kgem_busy(kgem, bo->handle))
		1941	break;
		1942
		1943	__kgem_bo_clear_busy(bo);
		1944
		1945	if (bo->refcnt)
		1946	continue;
		1947
		1948	if (bo->snoop) {
		1949	kgem_bo_move_to_snoop(kgem, bo);
		1950	} else if (bo->scanout) {
		1951	kgem_bo_move_to_scanout(kgem, bo);
		1952	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		1953	kgem_bo_set_purgeable(kgem, bo)) {
		1954	kgem_bo_move_to_inactive(kgem, bo);
		1955	retired = true;
		1956	} else
		1957	kgem_bo_free(kgem, bo);
		1958	}
		1959	#if HAS_DEBUG_FULL
		1960	{
		1961	int count = 0;
		1962	list_for_each_entry(bo, &kgem->flushing, request)
		1963	count++;
		1964	ErrorF("%s: %d bo on flushing list\n", __FUNCTION__, count);
		1965	}
		1966	#endif
		1967
		1968	kgem->need_retire \|= !list_is_empty(&kgem->flushing);
		1969
		1970	return retired;
		1971	}
		1972
		1973
		1974	static bool __kgem_retire_rq(struct kgem kgem, struct kgem_request rq)
		1975	{
		1976	bool retired = false;
		1977
		1978	DBG(("%s: request %d complete\n",
		1979	__FUNCTION__, rq->bo->handle));
		1980
		1981	while (!list_is_empty(&rq->buffers)) {
		1982	struct kgem_bo *bo;
		1983
		1984	bo = list_first_entry(&rq->buffers,
		1985	struct kgem_bo,
		1986	request);
		1987
		1988	assert(RQ(bo->rq) == rq);
		1989	assert(bo->exec == NULL);
		1990	assert(bo->domain == DOMAIN_GPU \|\| bo->domain == DOMAIN_NONE);
		1991
		1992	list_del(&bo->request);
		1993
		1994	if (bo->needs_flush)
		1995	bo->needs_flush = __kgem_busy(kgem, bo->handle);
		1996	if (bo->needs_flush) {
		1997	DBG(("%s: moving %d to flushing\n",
		1998	__FUNCTION__, bo->handle));
		1999	list_add(&bo->request, &kgem->flushing);
4501	Serge	2000	bo->rq = MAKE_REQUEST(kgem, RQ_RING(bo->rq));
		2001	kgem->need_retire = true;
4304	Serge	2002	continue;
		2003	}
		2004
		2005	bo->domain = DOMAIN_NONE;
		2006	bo->rq = NULL;
		2007	if (bo->refcnt)
		2008	continue;
		2009
		2010	if (bo->snoop) {
		2011	kgem_bo_move_to_snoop(kgem, bo);
		2012	} else if (bo->scanout) {
		2013	kgem_bo_move_to_scanout(kgem, bo);
		2014	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		2015	kgem_bo_set_purgeable(kgem, bo)) {
		2016	kgem_bo_move_to_inactive(kgem, bo);
		2017	retired = true;
		2018	} else {
		2019	DBG(("%s: closing %d\n",
		2020	__FUNCTION__, bo->handle));
		2021	kgem_bo_free(kgem, bo);
		2022	}
		2023	}
		2024
		2025	assert(rq->bo->rq == NULL);
4501	Serge	2026	assert(rq->bo->exec == NULL);
4304	Serge	2027	assert(list_is_empty(&rq->bo->request));
		2028
		2029	if (--rq->bo->refcnt == 0) {
		2030	if (kgem_bo_set_purgeable(kgem, rq->bo)) {
		2031	kgem_bo_move_to_inactive(kgem, rq->bo);
		2032	retired = true;
		2033	} else {
		2034	DBG(("%s: closing %d\n",
		2035	__FUNCTION__, rq->bo->handle));
		2036	kgem_bo_free(kgem, rq->bo);
		2037	}
		2038	}
		2039
		2040	__kgem_request_free(rq);
		2041	return retired;
		2042	}
		2043
		2044	static bool kgem_retire__requests_ring(struct kgem *kgem, int ring)
		2045	{
		2046	bool retired = false;
		2047
		2048	while (!list_is_empty(&kgem->requests[ring])) {
		2049	struct kgem_request *rq;
		2050
		2051	rq = list_first_entry(&kgem->requests[ring],
		2052	struct kgem_request,
		2053	list);
		2054	if (__kgem_busy(kgem, rq->bo->handle))
		2055	break;
		2056
		2057	retired \|= __kgem_retire_rq(kgem, rq);
		2058	}
		2059
		2060	#if HAS_DEBUG_FULL
		2061	{
		2062	struct kgem_bo *bo;
		2063	int count = 0;
		2064
		2065	list_for_each_entry(bo, &kgem->requests[ring], request)
		2066	count++;
		2067
		2068	bo = NULL;
		2069	if (!list_is_empty(&kgem->requests[ring]))
		2070	bo = list_first_entry(&kgem->requests[ring],
		2071	struct kgem_request,
		2072	list)->bo;
		2073
		2074	ErrorF("%s: ring=%d, %d outstanding requests, oldest=%d\n",
		2075	__FUNCTION__, ring, count, bo ? bo->handle : 0);
		2076	}
		2077	#endif
		2078
		2079	return retired;
		2080	}
		2081
		2082	static bool kgem_retire__requests(struct kgem *kgem)
		2083	{
		2084	bool retired = false;
		2085	int n;
		2086
		2087	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2088	retired \|= kgem_retire__requests_ring(kgem, n);
		2089	kgem->need_retire \|= !list_is_empty(&kgem->requests[n]);
		2090	}
		2091
		2092	return retired;
		2093	}
		2094
		2095	bool kgem_retire(struct kgem *kgem)
		2096	{
		2097	bool retired = false;
		2098
4501	Serge	2099	DBG(("%s, need_retire?=%d\n", __FUNCTION__, kgem->need_retire));
4304	Serge	2100
		2101	kgem->need_retire = false;
		2102
		2103	retired \|= kgem_retire__flushing(kgem);
		2104	retired \|= kgem_retire__requests(kgem);
		2105	retired \|= kgem_retire__buffers(kgem);
		2106
		2107	DBG(("%s -- retired=%d, need_retire=%d\n",
		2108	__FUNCTION__, retired, kgem->need_retire));
		2109
		2110	kgem->retire(kgem);
		2111
		2112	return retired;
		2113	}
		2114
		2115	bool __kgem_ring_is_idle(struct kgem *kgem, int ring)
		2116	{
		2117	struct kgem_request *rq;
		2118
4501	Serge	2119	assert(ring < ARRAY_SIZE(kgem->requests));
4304	Serge	2120	assert(!list_is_empty(&kgem->requests[ring]));
		2121
		2122	rq = list_last_entry(&kgem->requests[ring],
		2123	struct kgem_request, list);
		2124	if (__kgem_busy(kgem, rq->bo->handle)) {
		2125	DBG(("%s: last requests handle=%d still busy\n",
		2126	__FUNCTION__, rq->bo->handle));
		2127	return false;
		2128	}
		2129
		2130	DBG(("%s: ring=%d idle (handle=%d)\n",
		2131	__FUNCTION__, ring, rq->bo->handle));
		2132
		2133	kgem_retire__requests_ring(kgem, ring);
4501	Serge	2134	kgem_retire__buffers(kgem);
		2135
4304	Serge	2136	assert(list_is_empty(&kgem->requests[ring]));
		2137	return true;
		2138	}
		2139
4501	Serge	2140	#ifndef NDEBUG
		2141	static void kgem_commit__check_buffers(struct kgem *kgem)
		2142	{
		2143	struct kgem_buffer *bo;
		2144
		2145	list_for_each_entry(bo, &kgem->active_buffers, base.list)
		2146	assert(bo->base.exec == NULL);
		2147	}
		2148	#else
		2149	#define kgem_commit__check_buffers(kgem)
		2150	#endif
		2151
4304	Serge	2152	static void kgem_commit(struct kgem *kgem)
		2153	{
		2154	struct kgem_request *rq = kgem->next_request;
		2155	struct kgem_bo bo, next;
		2156
		2157	list_for_each_entry_safe(bo, next, &rq->buffers, request) {
		2158	assert(next->request.prev == &bo->request);
		2159
		2160	DBG(("%s: release handle=%d (proxy? %d), dirty? %d flush? %d, snoop? %d -> offset=%x\n",
		2161	__FUNCTION__, bo->handle, bo->proxy != NULL,
		2162	bo->gpu_dirty, bo->needs_flush, bo->snoop,
		2163	(unsigned)bo->exec->offset));
		2164
		2165	assert(bo->exec);
		2166	assert(bo->proxy == NULL \|\| bo->exec == &_kgem_dummy_exec);
		2167	assert(RQ(bo->rq) == rq \|\| (RQ(bo->proxy->rq) == rq));
		2168
		2169	bo->presumed_offset = bo->exec->offset;
		2170	bo->exec = NULL;
		2171	bo->target_handle = -1;
		2172
		2173	if (!bo->refcnt && !bo->reusable) {
		2174	assert(!bo->snoop);
4501	Serge	2175	assert(!bo->proxy);
4304	Serge	2176	kgem_bo_free(kgem, bo);
		2177	continue;
		2178	}
		2179
		2180	bo->binding.offset = 0;
		2181	bo->domain = DOMAIN_GPU;
		2182	bo->gpu_dirty = false;
		2183
		2184	if (bo->proxy) {
		2185	/* proxies are not used for domain tracking */
		2186	__kgem_bo_clear_busy(bo);
		2187	}
		2188
		2189	kgem->scanout_busy \|= bo->scanout;
		2190	}
		2191
		2192	if (rq == &kgem->static_request) {
		2193	struct drm_i915_gem_set_domain set_domain;
		2194
		2195	DBG(("%s: syncing due to allocation failure\n", __FUNCTION__));
		2196
		2197	VG_CLEAR(set_domain);
		2198	set_domain.handle = rq->bo->handle;
		2199	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2200	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2201	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
		2202	DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
		2203	kgem_throttle(kgem);
		2204	}
		2205
		2206	kgem_retire(kgem);
		2207	assert(list_is_empty(&rq->buffers));
		2208
4501	Serge	2209	assert(rq->bo->map__gtt == NULL);
		2210	assert(rq->bo->map__cpu == NULL);
4304	Serge	2211	gem_close(kgem->fd, rq->bo->handle);
		2212	kgem_cleanup_cache(kgem);
		2213	} else {
		2214	list_add_tail(&rq->list, &kgem->requests[rq->ring]);
		2215	kgem->need_throttle = kgem->need_retire = 1;
		2216	}
		2217
		2218	kgem->next_request = NULL;
4501	Serge	2219
		2220	kgem_commit__check_buffers(kgem);
4304	Serge	2221	}
		2222
		2223	static void kgem_close_list(struct kgem kgem, struct list head)
		2224	{
		2225	while (!list_is_empty(head))
		2226	kgem_bo_free(kgem, list_first_entry(head, struct kgem_bo, list));
		2227	}
		2228
		2229	static void kgem_close_inactive(struct kgem *kgem)
		2230	{
		2231	unsigned int i;
		2232
		2233	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		2234	kgem_close_list(kgem, &kgem->inactive[i]);
		2235	}
		2236
		2237	static void kgem_finish_buffers(struct kgem *kgem)
		2238	{
		2239	struct kgem_buffer bo, next;
		2240
		2241	list_for_each_entry_safe(bo, next, &kgem->batch_buffers, base.list) {
4501	Serge	2242	DBG(("%s: buffer handle=%d, used=%d, exec?=%d, write=%d, mmapped=%s, refcnt=%d\n",
4304	Serge	2243	__FUNCTION__, bo->base.handle, bo->used, bo->base.exec!=NULL,
4501	Serge	2244	bo->write, bo->mmapped == MMAPPED_CPU ? "cpu" : bo->mmapped == MMAPPED_GTT ? "gtt" : "no",
		2245	bo->base.refcnt));
4304	Serge	2246
		2247	assert(next->base.list.prev == &bo->base.list);
		2248	assert(bo->base.io);
		2249	assert(bo->base.refcnt >= 1);
		2250
4501	Serge	2251	if (bo->base.refcnt > 1 && !bo->base.exec) {
		2252	DBG(("%s: skipping unattached handle=%d, used=%d, refcnt=%d\n",
		2253	__FUNCTION__, bo->base.handle, bo->used, bo->base.refcnt));
4304	Serge	2254	continue;
		2255	}
		2256
		2257	if (!bo->write) {
		2258	assert(bo->base.exec \|\| bo->base.refcnt > 1);
		2259	goto decouple;
		2260	}
		2261
		2262	if (bo->mmapped) {
4501	Serge	2263	uint32_t used;
4304	Serge	2264
		2265	assert(!bo->need_io);
		2266
		2267	used = ALIGN(bo->used, PAGE_SIZE);
		2268	if (!DBG_NO_UPLOAD_ACTIVE &&
		2269	used + PAGE_SIZE <= bytes(&bo->base) &&
4501	Serge	2270	(kgem->has_llc \|\| bo->mmapped == MMAPPED_GTT \|\| bo->base.snoop)) {
		2271	DBG(("%s: retaining upload buffer (%d/%d): used=%d, refcnt=%d\n",
		2272	__FUNCTION__, bo->used, bytes(&bo->base), used, bo->base.refcnt));
4304	Serge	2273	bo->used = used;
		2274	list_move(&bo->base.list,
		2275	&kgem->active_buffers);
4501	Serge	2276	kgem->need_retire = true;
4304	Serge	2277	continue;
		2278	}
		2279	DBG(("%s: discarding mmapped buffer, used=%d, map type=%d\n",
4501	Serge	2280	__FUNCTION__, bo->used, bo->mmapped));
4304	Serge	2281	goto decouple;
		2282	}
		2283
4501	Serge	2284	if (!bo->used \|\| !bo->base.exec) {
4304	Serge	2285	/* Unless we replace the handle in the execbuffer,
		2286	* then this bo will become active. So decouple it
		2287	* from the buffer list and track it in the normal
		2288	* manner.
		2289	*/
		2290	goto decouple;
		2291	}
		2292
		2293	assert(bo->need_io);
		2294	assert(bo->base.rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		2295	assert(bo->base.domain != DOMAIN_GPU);
		2296
		2297	if (bo->base.refcnt == 1 &&
		2298	bo->base.size.pages.count > 1 &&
		2299	bo->used < bytes(&bo->base) / 2) {
		2300	struct kgem_bo *shrink;
		2301	unsigned alloc = NUM_PAGES(bo->used);
		2302
		2303	shrink = search_snoop_cache(kgem, alloc,
		2304	CREATE_INACTIVE \| CREATE_NO_RETIRE);
		2305	if (shrink) {
		2306	void *map;
		2307	int n;
		2308
		2309	DBG(("%s: used=%d, shrinking %d to %d, handle %d to %d\n",
		2310	__FUNCTION__,
		2311	bo->used, bytes(&bo->base), bytes(shrink),
		2312	bo->base.handle, shrink->handle));
		2313
		2314	assert(bo->used <= bytes(shrink));
		2315	map = kgem_bo_map__cpu(kgem, shrink);
		2316	if (map) {
		2317	kgem_bo_sync__cpu(kgem, shrink);
		2318	memcpy(map, bo->mem, bo->used);
		2319
		2320	shrink->target_handle =
		2321	kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
		2322	for (n = 0; n < kgem->nreloc; n++) {
		2323	if (kgem->reloc[n].target_handle == bo->base.target_handle) {
		2324	kgem->reloc[n].target_handle = shrink->target_handle;
		2325	kgem->reloc[n].presumed_offset = shrink->presumed_offset;
		2326	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		2327	kgem->reloc[n].delta + shrink->presumed_offset;
		2328	}
		2329	}
		2330
		2331	bo->base.exec->handle = shrink->handle;
		2332	bo->base.exec->offset = shrink->presumed_offset;
		2333	shrink->exec = bo->base.exec;
		2334	shrink->rq = bo->base.rq;
		2335	list_replace(&bo->base.request,
		2336	&shrink->request);
		2337	list_init(&bo->base.request);
		2338	shrink->needs_flush = bo->base.gpu_dirty;
		2339
		2340	bo->base.exec = NULL;
		2341	bo->base.rq = NULL;
		2342	bo->base.gpu_dirty = false;
		2343	bo->base.needs_flush = false;
		2344	bo->used = 0;
		2345
		2346	goto decouple;
		2347	}
		2348
		2349	__kgem_bo_destroy(kgem, shrink);
		2350	}
		2351
		2352	shrink = search_linear_cache(kgem, alloc,
		2353	CREATE_INACTIVE \| CREATE_NO_RETIRE);
		2354	if (shrink) {
		2355	int n;
		2356
		2357	DBG(("%s: used=%d, shrinking %d to %d, handle %d to %d\n",
		2358	__FUNCTION__,
		2359	bo->used, bytes(&bo->base), bytes(shrink),
		2360	bo->base.handle, shrink->handle));
		2361
		2362	assert(bo->used <= bytes(shrink));
4501	Serge	2363	if (gem_write__cachealigned(kgem->fd, shrink->handle,
4304	Serge	2364	0, bo->used, bo->mem) == 0) {
		2365	shrink->target_handle =
		2366	kgem->has_handle_lut ? bo->base.target_handle : shrink->handle;
		2367	for (n = 0; n < kgem->nreloc; n++) {
		2368	if (kgem->reloc[n].target_handle == bo->base.target_handle) {
		2369	kgem->reloc[n].target_handle = shrink->target_handle;
		2370	kgem->reloc[n].presumed_offset = shrink->presumed_offset;
		2371	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		2372	kgem->reloc[n].delta + shrink->presumed_offset;
		2373	}
		2374	}
		2375
		2376	bo->base.exec->handle = shrink->handle;
		2377	bo->base.exec->offset = shrink->presumed_offset;
		2378	shrink->exec = bo->base.exec;
		2379	shrink->rq = bo->base.rq;
		2380	list_replace(&bo->base.request,
		2381	&shrink->request);
		2382	list_init(&bo->base.request);
		2383	shrink->needs_flush = bo->base.gpu_dirty;
		2384
		2385	bo->base.exec = NULL;
		2386	bo->base.rq = NULL;
		2387	bo->base.gpu_dirty = false;
		2388	bo->base.needs_flush = false;
		2389	bo->used = 0;
		2390
		2391	goto decouple;
		2392	}
		2393
		2394	__kgem_bo_destroy(kgem, shrink);
		2395	}
		2396	}
		2397
		2398	DBG(("%s: handle=%d, uploading %d/%d\n",
		2399	__FUNCTION__, bo->base.handle, bo->used, bytes(&bo->base)));
		2400	ASSERT_IDLE(kgem, bo->base.handle);
		2401	assert(bo->used <= bytes(&bo->base));
4501	Serge	2402	gem_write__cachealigned(kgem->fd, bo->base.handle,
4304	Serge	2403	0, bo->used, bo->mem);
		2404	bo->need_io = 0;
		2405
		2406	decouple:
		2407	DBG(("%s: releasing handle=%d\n",
		2408	__FUNCTION__, bo->base.handle));
		2409	list_del(&bo->base.list);
		2410	kgem_bo_unref(kgem, &bo->base);
		2411	}
		2412	}
		2413
		2414	static void kgem_cleanup(struct kgem *kgem)
		2415	{
		2416	int n;
		2417
		2418	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		2419	while (!list_is_empty(&kgem->requests[n])) {
		2420	struct kgem_request *rq;
		2421
		2422	rq = list_first_entry(&kgem->requests[n],
		2423	struct kgem_request,
		2424	list);
		2425	while (!list_is_empty(&rq->buffers)) {
		2426	struct kgem_bo *bo;
		2427
		2428	bo = list_first_entry(&rq->buffers,
		2429	struct kgem_bo,
		2430	request);
		2431
		2432	bo->exec = NULL;
		2433	bo->gpu_dirty = false;
		2434	__kgem_bo_clear_busy(bo);
		2435	if (bo->refcnt == 0)
		2436	kgem_bo_free(kgem, bo);
		2437	}
		2438
		2439	__kgem_request_free(rq);
		2440	}
		2441	}
		2442
		2443	kgem_close_inactive(kgem);
		2444	}
		2445
		2446	static int kgem_batch_write(struct kgem *kgem, uint32_t handle, uint32_t size)
		2447	{
		2448	int ret;
		2449
		2450	ASSERT_IDLE(kgem, handle);
		2451
4501	Serge	2452	retry:
4304	Serge	2453	/* If there is no surface data, just upload the batch */
4501	Serge	2454	if (kgem->surface == kgem->batch_size) {
		2455	if (gem_write__cachealigned(kgem->fd, handle,
4304	Serge	2456	0, sizeof(uint32_t)*kgem->nbatch,
4501	Serge	2457	kgem->batch) == 0)
		2458	return 0;
4304	Serge	2459
4501	Serge	2460	goto expire;
		2461	}
		2462
4304	Serge	2463	/* Are the batch pages conjoint with the surface pages? */
		2464	if (kgem->surface < kgem->nbatch + PAGE_SIZE/sizeof(uint32_t)) {
		2465	assert(size == PAGE_ALIGN(kgem->batch_size*sizeof(uint32_t)));
4501	Serge	2466	if (gem_write__cachealigned(kgem->fd, handle,
4304	Serge	2467	0, kgem->batch_size*sizeof(uint32_t),
4501	Serge	2468	kgem->batch) == 0)
		2469	return 0;
		2470
		2471	goto expire;
4304	Serge	2472	}
		2473
		2474	/* Disjoint surface/batch, upload separately */
4501	Serge	2475	if (gem_write__cachealigned(kgem->fd, handle,
4304	Serge	2476	0, sizeof(uint32_t)*kgem->nbatch,
4501	Serge	2477	kgem->batch))
		2478	goto expire;
4304	Serge	2479
		2480	ret = PAGE_ALIGN(sizeof(uint32_t) * kgem->batch_size);
		2481	ret -= sizeof(uint32_t) * kgem->surface;
		2482	assert(size-ret >= kgem->nbatch*sizeof(uint32_t));
4501	Serge	2483	if (gem_write(kgem->fd, handle,
4304	Serge	2484	size - ret, (kgem->batch_size - kgem->surface)*sizeof(uint32_t),
4501	Serge	2485	kgem->batch + kgem->surface))
		2486	goto expire;
		2487
		2488	return 0;
		2489
		2490	expire:
		2491	ret = errno;
		2492	assert(ret != EINVAL);
		2493
		2494	(void)__kgem_throttle_retire(kgem, 0);
		2495	if (kgem_expire_cache(kgem))
		2496	goto retry;
		2497
		2498	if (kgem_cleanup_cache(kgem))
		2499	goto retry;
		2500
		2501	ErrorF("%s: failed to write batch (handle=%d): %d\n",
		2502	__FUNCTION__, handle, ret);
		2503	return ret;
4304	Serge	2504	}
		2505
		2506	void kgem_reset(struct kgem *kgem)
		2507	{
		2508	if (kgem->next_request) {
		2509	struct kgem_request *rq = kgem->next_request;
		2510
		2511	while (!list_is_empty(&rq->buffers)) {
		2512	struct kgem_bo *bo =
		2513	list_first_entry(&rq->buffers,
		2514	struct kgem_bo,
		2515	request);
		2516	list_del(&bo->request);
		2517
		2518	assert(RQ(bo->rq) == rq);
		2519
		2520	bo->binding.offset = 0;
		2521	bo->exec = NULL;
		2522	bo->target_handle = -1;
		2523	bo->gpu_dirty = false;
		2524
		2525	if (bo->needs_flush && __kgem_busy(kgem, bo->handle)) {
		2526	assert(bo->domain == DOMAIN_GPU \|\| bo->domain == DOMAIN_NONE);
		2527	list_add(&bo->request, &kgem->flushing);
		2528	bo->rq = (void *)kgem;
4501	Serge	2529	kgem->need_retire = true;
4304	Serge	2530	} else
		2531	__kgem_bo_clear_busy(bo);
		2532
		2533	if (bo->refcnt \|\| bo->rq)
		2534	continue;
		2535
		2536	if (bo->snoop) {
		2537	kgem_bo_move_to_snoop(kgem, bo);
		2538	} else if (bo->scanout) {
		2539	kgem_bo_move_to_scanout(kgem, bo);
		2540	} else if ((bo = kgem_bo_replace_io(bo))->reusable &&
		2541	kgem_bo_set_purgeable(kgem, bo)) {
		2542	kgem_bo_move_to_inactive(kgem, bo);
		2543	} else {
		2544	DBG(("%s: closing %d\n",
		2545	__FUNCTION__, bo->handle));
		2546	kgem_bo_free(kgem, bo);
		2547	}
		2548	}
		2549
		2550	if (rq != &kgem->static_request) {
		2551	list_init(&rq->list);
		2552	__kgem_request_free(rq);
		2553	}
		2554	}
		2555
		2556	kgem->nfence = 0;
		2557	kgem->nexec = 0;
		2558	kgem->nreloc = 0;
		2559	kgem->nreloc__self = 0;
		2560	kgem->aperture = 0;
		2561	kgem->aperture_fenced = 0;
4501	Serge	2562	kgem->aperture_max_fence = 0;
4304	Serge	2563	kgem->nbatch = 0;
		2564	kgem->surface = kgem->batch_size;
		2565	kgem->mode = KGEM_NONE;
		2566	kgem->flush = 0;
		2567	kgem->batch_flags = kgem->batch_flags_base;
		2568
		2569	kgem->next_request = __kgem_request_alloc(kgem);
		2570
		2571	kgem_sna_reset(kgem);
		2572	}
		2573
		2574	static int compact_batch_surface(struct kgem *kgem)
		2575	{
		2576	int size, shrink, n;
		2577
		2578	if (!kgem->has_relaxed_delta)
		2579	return kgem->batch_size;
		2580
		2581	/* See if we can pack the contents into one or two pages */
		2582	n = ALIGN(kgem->batch_size, 1024);
		2583	size = n - kgem->surface + kgem->nbatch;
		2584	size = ALIGN(size, 1024);
		2585
		2586	shrink = n - size;
		2587	if (shrink) {
		2588	DBG(("shrinking from %d to %d\n", kgem->batch_size, size));
		2589
		2590	shrink *= sizeof(uint32_t);
		2591	for (n = 0; n < kgem->nreloc; n++) {
		2592	if (kgem->reloc[n].read_domains == I915_GEM_DOMAIN_INSTRUCTION &&
		2593	kgem->reloc[n].target_handle == ~0U)
		2594	kgem->reloc[n].delta -= shrink;
		2595
		2596	if (kgem->reloc[n].offset >= sizeof(uint32_t)*kgem->nbatch)
		2597	kgem->reloc[n].offset -= shrink;
		2598	}
		2599	}
		2600
		2601	return size * sizeof(uint32_t);
		2602	}
		2603
		2604	static struct kgem_bo *
		2605	kgem_create_batch(struct kgem *kgem, int size)
		2606	{
		2607	struct drm_i915_gem_set_domain set_domain;
		2608	struct kgem_bo *bo;
		2609
		2610	if (size <= 4096) {
		2611	bo = list_first_entry(&kgem->pinned_batches[0],
		2612	struct kgem_bo,
		2613	list);
		2614	if (!bo->rq) {
		2615	out_4096:
		2616	list_move_tail(&bo->list, &kgem->pinned_batches[0]);
		2617	return kgem_bo_reference(bo);
		2618	}
		2619
		2620	if (!__kgem_busy(kgem, bo->handle)) {
		2621	assert(RQ(bo->rq)->bo == bo);
		2622	__kgem_retire_rq(kgem, RQ(bo->rq));
		2623	goto out_4096;
		2624	}
		2625	}
		2626
		2627	if (size <= 16384) {
		2628	bo = list_first_entry(&kgem->pinned_batches[1],
		2629	struct kgem_bo,
		2630	list);
		2631	if (!bo->rq) {
		2632	out_16384:
		2633	list_move_tail(&bo->list, &kgem->pinned_batches[1]);
		2634	return kgem_bo_reference(bo);
		2635	}
		2636
		2637	if (!__kgem_busy(kgem, bo->handle)) {
		2638	assert(RQ(bo->rq)->bo == bo);
		2639	__kgem_retire_rq(kgem, RQ(bo->rq));
		2640	goto out_16384;
		2641	}
		2642	}
		2643
		2644	if (kgem->gen == 020 && !kgem->has_pinned_batches) {
		2645	assert(size <= 16384);
		2646
		2647	bo = list_first_entry(&kgem->pinned_batches[size > 4096],
		2648	struct kgem_bo,
		2649	list);
		2650	list_move_tail(&bo->list, &kgem->pinned_batches[size > 4096]);
		2651
		2652	DBG(("%s: syncing due to busy batches\n", __FUNCTION__));
		2653
		2654	VG_CLEAR(set_domain);
		2655	set_domain.handle = bo->handle;
		2656	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2657	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2658	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain)) {
		2659	DBG(("%s: sync: GPU hang detected\n", __FUNCTION__));
		2660	kgem_throttle(kgem);
		2661	return NULL;
		2662	}
		2663
		2664	kgem_retire(kgem);
		2665	assert(bo->rq == NULL);
		2666	return kgem_bo_reference(bo);
		2667	}
		2668
		2669	return kgem_create_linear(kgem, size, CREATE_NO_THROTTLE);
		2670	}
		2671
		2672	void _kgem_submit(struct kgem *kgem)
		2673	{
		2674	struct kgem_request *rq;
		2675	uint32_t batch_end;
		2676	int size;
		2677
		2678	assert(!DBG_NO_HW);
		2679	assert(!kgem->wedged);
		2680
		2681	assert(kgem->nbatch);
		2682	assert(kgem->nbatch <= KGEM_BATCH_SIZE(kgem));
		2683	assert(kgem->nbatch <= kgem->surface);
		2684
		2685	batch_end = kgem_end_batch(kgem);
		2686	kgem_sna_flush(kgem);
		2687
4501	Serge	2688	DBG(("batch[%d/%d, flags=%x]: %d %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d [fenced=%d]\n",
4304	Serge	2689	kgem->mode, kgem->ring, kgem->batch_flags,
		2690	batch_end, kgem->nbatch, kgem->surface, kgem->batch_size,
4501	Serge	2691	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced));
4304	Serge	2692
		2693	assert(kgem->nbatch <= kgem->batch_size);
		2694	assert(kgem->nbatch <= kgem->surface);
		2695	assert(kgem->nreloc <= ARRAY_SIZE(kgem->reloc));
		2696	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		2697	assert(kgem->nfence <= kgem->fence_max);
		2698
		2699	kgem_finish_buffers(kgem);
		2700
		2701	#if SHOW_BATCH
		2702	__kgem_batch_debug(kgem, batch_end);
		2703	#endif
		2704
		2705	rq = kgem->next_request;
		2706	if (kgem->surface != kgem->batch_size)
		2707	size = compact_batch_surface(kgem);
		2708	else
		2709	size = kgem->nbatch * sizeof(kgem->batch[0]);
		2710	rq->bo = kgem_create_batch(kgem, size);
		2711	if (rq->bo) {
		2712	uint32_t handle = rq->bo->handle;
		2713	int i;
		2714
		2715	assert(!rq->bo->needs_flush);
		2716
		2717	i = kgem->nexec++;
		2718	kgem->exec[i].handle = handle;
		2719	kgem->exec[i].relocation_count = kgem->nreloc;
		2720	kgem->exec[i].relocs_ptr = (uintptr_t)kgem->reloc;
		2721	kgem->exec[i].alignment = 0;
		2722	kgem->exec[i].offset = rq->bo->presumed_offset;
		2723	kgem->exec[i].flags = 0;
		2724	kgem->exec[i].rsvd1 = 0;
		2725	kgem->exec[i].rsvd2 = 0;
		2726
		2727	rq->bo->target_handle = kgem->has_handle_lut ? i : handle;
		2728	rq->bo->exec = &kgem->exec[i];
		2729	rq->bo->rq = MAKE_REQUEST(rq, kgem->ring); /* useful sanity check */
		2730	list_add(&rq->bo->request, &rq->buffers);
		2731	rq->ring = kgem->ring == KGEM_BLT;
		2732
		2733	kgem_fixup_self_relocs(kgem, rq->bo);
		2734
		2735	if (kgem_batch_write(kgem, handle, size) == 0) {
		2736	struct drm_i915_gem_execbuffer2 execbuf;
		2737	int ret, retry = 3;
		2738
		2739	memset(&execbuf, 0, sizeof(execbuf));
		2740	execbuf.buffers_ptr = (uintptr_t)kgem->exec;
		2741	execbuf.buffer_count = kgem->nexec;
		2742	execbuf.batch_len = batch_end*sizeof(uint32_t);
		2743	execbuf.flags = kgem->ring \| kgem->batch_flags;
		2744
		2745	if (DEBUG_DUMP)
		2746	{
		2747	int fd = open("/tmp1/1/batchbuffer.bin", O_CREAT\|O_WRONLY\|O_BINARY);
		2748	if (fd != -1) {
4501	Serge	2749	ret = write(fd, kgem->batch, batch_end*sizeof(uint32_t));
		2750	fd = close(fd);
4304	Serge	2751	}
		2752	else printf("SNA: failed to write batchbuffer\n");
		2753	asm volatile("int3");
		2754	}
		2755
		2756	ret = drmIoctl(kgem->fd,
		2757	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2758	&execbuf);
		2759	while (ret == -1 && errno == EBUSY && retry--) {
		2760	__kgem_throttle(kgem);
		2761	ret = drmIoctl(kgem->fd,
		2762	DRM_IOCTL_I915_GEM_EXECBUFFER2,
		2763	&execbuf);
		2764	}
		2765	if (DEBUG_SYNC && ret == 0) {
		2766	struct drm_i915_gem_set_domain set_domain;
		2767
		2768	VG_CLEAR(set_domain);
		2769	set_domain.handle = handle;
		2770	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		2771	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		2772
		2773	ret = drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
		2774	}
		2775	if (ret == -1) {
		2776	DBG(("%s: GPU hang detected [%d]\n",
		2777	__FUNCTION__, errno));
		2778	kgem_throttle(kgem);
		2779	kgem->wedged = true;
		2780
		2781	#if 0
		2782	ret = errno;
4501	Serge	2783	ErrorF("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d, fenced=%d, high=%d,%d: errno=%d\n",
4304	Serge	2784	kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
4501	Serge	2785	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture, kgem->aperture_fenced, kgem->aperture_high, kgem->aperture_total, errno);
4304	Serge	2786
		2787	for (i = 0; i < kgem->nexec; i++) {
		2788	struct kgem_bo bo, found = NULL;
		2789
		2790	list_for_each_entry(bo, &kgem->next_request->buffers, request) {
		2791	if (bo->handle == kgem->exec[i].handle) {
		2792	found = bo;
		2793	break;
		2794	}
		2795	}
		2796	ErrorF("exec[%d] = handle:%d, presumed offset: %x, size: %d, tiling %d, fenced %d, snooped %d, deleted %d\n",
		2797	i,
		2798	kgem->exec[i].handle,
		2799	(int)kgem->exec[i].offset,
		2800	found ? kgem_bo_size(found) : -1,
		2801	found ? found->tiling : -1,
		2802	(int)(kgem->exec[i].flags & EXEC_OBJECT_NEEDS_FENCE),
		2803	found ? found->snoop : -1,
		2804	found ? found->purged : -1);
		2805	}
		2806	for (i = 0; i < kgem->nreloc; i++) {
		2807	ErrorF("reloc[%d] = pos:%d, target:%d, delta:%d, read:%x, write:%x, offset:%x\n",
		2808	i,
		2809	(int)kgem->reloc[i].offset,
		2810	kgem->reloc[i].target_handle,
		2811	kgem->reloc[i].delta,
		2812	kgem->reloc[i].read_domains,
		2813	kgem->reloc[i].write_domain,
		2814	(int)kgem->reloc[i].presumed_offset);
		2815	}
		2816
		2817	if (DEBUG_SYNC) {
		2818	int fd = open("/tmp/batchbuffer", O_WRONLY \| O_CREAT \| O_APPEND, 0666);
		2819	if (fd != -1) {
		2820	write(fd, kgem->batch, batch_end*sizeof(uint32_t));
		2821	close(fd);
		2822	}
		2823
		2824	FatalError("SNA: failed to submit batchbuffer, errno=%d\n", ret);
		2825	}
		2826	#endif
		2827	}
		2828	}
		2829
		2830	kgem_commit(kgem);
		2831	}
		2832	if (kgem->wedged)
		2833	kgem_cleanup(kgem);
		2834
		2835	kgem_reset(kgem);
		2836
		2837	assert(kgem->next_request != NULL);
		2838	}
		2839
		2840	void kgem_throttle(struct kgem *kgem)
		2841	{
		2842	kgem->need_throttle = 0;
		2843	if (kgem->wedged)
		2844	return;
		2845
		2846	kgem->wedged = __kgem_throttle(kgem);
		2847	if (kgem->wedged) {
		2848	printf("Detected a hung GPU, disabling acceleration.\n");
		2849	printf("When reporting this, please include i915_error_state from debugfs and the full dmesg.\n");
		2850	}
		2851	}
		2852
4501	Serge	2853	static void kgem_purge_cache(struct kgem *kgem)
4304	Serge	2854	{
		2855	struct kgem_bo bo, next;
		2856	int i;
		2857
		2858	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2859	list_for_each_entry_safe(bo, next, &kgem->inactive[i], list) {
		2860	if (!kgem_bo_is_retained(kgem, bo)) {
		2861	DBG(("%s: purging %d\n",
		2862	__FUNCTION__, bo->handle));
		2863	kgem_bo_free(kgem, bo);
		2864	}
		2865	}
		2866	}
		2867
		2868	kgem->need_purge = false;
		2869	}
		2870
		2871
		2872	void kgem_clean_large_cache(struct kgem *kgem)
		2873	{
		2874	while (!list_is_empty(&kgem->large_inactive)) {
		2875	kgem_bo_free(kgem,
		2876	list_first_entry(&kgem->large_inactive,
		2877	struct kgem_bo, list));
		2878
		2879	}
		2880	}
		2881
		2882	bool kgem_expire_cache(struct kgem *kgem)
		2883	{
		2884	time_t now, expire;
		2885	struct kgem_bo *bo;
		2886	unsigned int size = 0, count = 0;
		2887	bool idle;
		2888	unsigned int i;
		2889
		2890	time(&now);
		2891
		2892	while (__kgem_freed_bo) {
		2893	bo = __kgem_freed_bo;
		2894	__kgem_freed_bo = (struct kgem_bo *)bo;
		2895	free(bo);
		2896	}
		2897
		2898	while (__kgem_freed_request) {
		2899	struct kgem_request *rq = __kgem_freed_request;
		2900	__kgem_freed_request = (struct kgem_request *)rq;
		2901	free(rq);
		2902	}
		2903
		2904	kgem_clean_large_cache(kgem);
		2905
		2906	expire = 0;
		2907	list_for_each_entry(bo, &kgem->snoop, list) {
		2908	if (bo->delta) {
		2909	expire = now - MAX_INACTIVE_TIME/2;
		2910	break;
		2911	}
		2912
		2913	bo->delta = now;
		2914	}
		2915	if (expire) {
		2916	while (!list_is_empty(&kgem->snoop)) {
		2917	bo = list_last_entry(&kgem->snoop, struct kgem_bo, list);
		2918
		2919	if (bo->delta > expire)
		2920	break;
		2921
		2922	kgem_bo_free(kgem, bo);
		2923	}
		2924	}
		2925	#ifdef DEBUG_MEMORY
		2926	{
		2927	long snoop_size = 0;
		2928	int snoop_count = 0;
		2929	list_for_each_entry(bo, &kgem->snoop, list)
		2930	snoop_count++, snoop_size += bytes(bo);
		2931	ErrorF("%s: still allocated %d bo, %ld bytes, in snoop cache\n",
		2932	__FUNCTION__, snoop_count, snoop_size);
		2933	}
		2934	#endif
		2935
		2936	kgem_retire(kgem);
		2937	if (kgem->wedged)
		2938	kgem_cleanup(kgem);
		2939
		2940	kgem->expire(kgem);
		2941
		2942	if (kgem->need_purge)
		2943	kgem_purge_cache(kgem);
		2944
		2945	expire = 0;
		2946
		2947	idle = !kgem->need_retire;
		2948	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2949	idle &= list_is_empty(&kgem->inactive[i]);
		2950	list_for_each_entry(bo, &kgem->inactive[i], list) {
		2951	if (bo->delta) {
		2952	expire = now - MAX_INACTIVE_TIME;
		2953	break;
		2954	}
		2955
		2956	bo->delta = now;
		2957	}
		2958	}
		2959	if (idle) {
		2960	DBG(("%s: idle\n", __FUNCTION__));
		2961	kgem->need_expire = false;
		2962	return false;
		2963	}
		2964	if (expire == 0)
		2965	return true;
		2966
		2967	idle = !kgem->need_retire;
		2968	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		2969	struct list preserve;
		2970
		2971	list_init(&preserve);
		2972	while (!list_is_empty(&kgem->inactive[i])) {
		2973	bo = list_last_entry(&kgem->inactive[i],
		2974	struct kgem_bo, list);
		2975
		2976	if (bo->delta > expire) {
		2977	idle = false;
		2978	break;
		2979	}
		2980
4501	Serge	2981	if (bo->map__cpu && bo->delta + MAP_PRESERVE_TIME > expire) {
4304	Serge	2982	idle = false;
		2983	list_move_tail(&bo->list, &preserve);
		2984	} else {
		2985	count++;
		2986	size += bytes(bo);
		2987	kgem_bo_free(kgem, bo);
		2988	DBG(("%s: expiring %d\n",
		2989	__FUNCTION__, bo->handle));
		2990	}
		2991	}
		2992	if (!list_is_empty(&preserve)) {
		2993	preserve.prev->next = kgem->inactive[i].next;
		2994	kgem->inactive[i].next->prev = preserve.prev;
		2995	kgem->inactive[i].next = preserve.next;
		2996	preserve.next->prev = &kgem->inactive[i];
		2997	}
		2998	}
		2999
		3000	#ifdef DEBUG_MEMORY
		3001	{
		3002	long inactive_size = 0;
		3003	int inactive_count = 0;
		3004	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		3005	list_for_each_entry(bo, &kgem->inactive[i], list)
		3006	inactive_count++, inactive_size += bytes(bo);
		3007	ErrorF("%s: still allocated %d bo, %ld bytes, in inactive cache\n",
		3008	__FUNCTION__, inactive_count, inactive_size);
		3009	}
		3010	#endif
		3011
		3012	DBG(("%s: expired %d objects, %d bytes, idle? %d\n",
		3013	__FUNCTION__, count, size, idle));
		3014
		3015	kgem->need_expire = !idle;
		3016	return !idle;
		3017	(void)count;
		3018	(void)size;
		3019	}
		3020
4501	Serge	3021	bool kgem_cleanup_cache(struct kgem *kgem)
4304	Serge	3022	{
		3023	unsigned int i;
		3024	int n;
		3025
		3026	/* sync to the most recent request */
		3027	for (n = 0; n < ARRAY_SIZE(kgem->requests); n++) {
		3028	if (!list_is_empty(&kgem->requests[n])) {
		3029	struct kgem_request *rq;
		3030	struct drm_i915_gem_set_domain set_domain;
		3031
		3032	rq = list_first_entry(&kgem->requests[n],
		3033	struct kgem_request,
		3034	list);
		3035
		3036	DBG(("%s: sync on cleanup\n", __FUNCTION__));
		3037
		3038	VG_CLEAR(set_domain);
		3039	set_domain.handle = rq->bo->handle;
		3040	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		3041	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		3042	(void)drmIoctl(kgem->fd,
		3043	DRM_IOCTL_I915_GEM_SET_DOMAIN,
		3044	&set_domain);
		3045	}
		3046	}
		3047
		3048	kgem_retire(kgem);
		3049	kgem_cleanup(kgem);
		3050
4501	Serge	3051	if (!kgem->need_expire)
		3052	return false;
		3053
4304	Serge	3054	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++) {
		3055	while (!list_is_empty(&kgem->inactive[i]))
		3056	kgem_bo_free(kgem,
		3057	list_last_entry(&kgem->inactive[i],
		3058	struct kgem_bo, list));
		3059	}
		3060
		3061	kgem_clean_large_cache(kgem);
		3062
		3063	while (!list_is_empty(&kgem->snoop))
		3064	kgem_bo_free(kgem,
		3065	list_last_entry(&kgem->snoop,
		3066	struct kgem_bo, list));
		3067
		3068	while (__kgem_freed_bo) {
		3069	struct kgem_bo *bo = __kgem_freed_bo;
		3070	__kgem_freed_bo = (struct kgem_bo *)bo;
		3071	free(bo);
		3072	}
		3073
		3074	kgem->need_purge = false;
		3075	kgem->need_expire = false;
4501	Serge	3076	return true;
4304	Serge	3077	}
		3078
		3079	static struct kgem_bo *
		3080	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		3081	{
		3082	struct kgem_bo bo, first = NULL;
		3083	bool use_active = (flags & CREATE_INACTIVE) == 0;
		3084	struct list *cache;
		3085
		3086	DBG(("%s: num_pages=%d, flags=%x, use_active? %d, use_large=%d [max=%d]\n",
		3087	__FUNCTION__, num_pages, flags, use_active,
		3088	num_pages >= MAX_CACHE_SIZE / PAGE_SIZE,
		3089	MAX_CACHE_SIZE / PAGE_SIZE));
		3090
		3091	assert(num_pages);
		3092
		3093	if (num_pages >= MAX_CACHE_SIZE / PAGE_SIZE) {
		3094	DBG(("%s: searching large buffers\n", __FUNCTION__));
		3095	retry_large:
		3096	cache = use_active ? &kgem->large : &kgem->large_inactive;
		3097	list_for_each_entry_safe(bo, first, cache, list) {
		3098	assert(bo->refcnt == 0);
		3099	assert(bo->reusable);
		3100	assert(!bo->scanout);
		3101
		3102	if (num_pages > num_pages(bo))
		3103	goto discard;
		3104
		3105	if (bo->tiling != I915_TILING_NONE) {
		3106	if (use_active)
		3107	goto discard;
		3108
		3109	if (!gem_set_tiling(kgem->fd, bo->handle,
		3110	I915_TILING_NONE, 0))
		3111	goto discard;
		3112
		3113	bo->tiling = I915_TILING_NONE;
		3114	bo->pitch = 0;
		3115	}
		3116
		3117	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo))
		3118	goto discard;
		3119
		3120	list_del(&bo->list);
4501	Serge	3121	if (RQ(bo->rq) == (void *)kgem) {
		3122	assert(bo->exec == NULL);
4304	Serge	3123	list_del(&bo->request);
4501	Serge	3124	}
4304	Serge	3125
		3126	bo->delta = 0;
		3127	assert_tiling(kgem, bo);
		3128	return bo;
		3129
		3130	discard:
		3131	if (!use_active)
		3132	kgem_bo_free(kgem, bo);
		3133	}
		3134
		3135	if (use_active) {
		3136	use_active = false;
		3137	goto retry_large;
		3138	}
		3139
		3140	if (__kgem_throttle_retire(kgem, flags))
		3141	goto retry_large;
		3142
		3143	return NULL;
		3144	}
		3145
		3146	if (!use_active && list_is_empty(inactive(kgem, num_pages))) {
		3147	DBG(("%s: inactive and cache bucket empty\n",
		3148	__FUNCTION__));
		3149
		3150	if (flags & CREATE_NO_RETIRE) {
		3151	DBG(("%s: can not retire\n", __FUNCTION__));
		3152	return NULL;
		3153	}
		3154
		3155	if (list_is_empty(active(kgem, num_pages, I915_TILING_NONE))) {
		3156	DBG(("%s: active cache bucket empty\n", __FUNCTION__));
		3157	return NULL;
		3158	}
		3159
		3160	if (!__kgem_throttle_retire(kgem, flags)) {
		3161	DBG(("%s: nothing retired\n", __FUNCTION__));
		3162	return NULL;
		3163	}
		3164
		3165	if (list_is_empty(inactive(kgem, num_pages))) {
		3166	DBG(("%s: active cache bucket still empty after retire\n",
		3167	__FUNCTION__));
		3168	return NULL;
		3169	}
		3170	}
		3171
		3172	if (!use_active && flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3173	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3174	DBG(("%s: searching for inactive %s map\n",
		3175	__FUNCTION__, for_cpu ? "cpu" : "gtt"));
		3176	cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
		3177	list_for_each_entry(bo, cache, vma) {
4501	Serge	3178	assert(for_cpu ? bo->map__cpu : bo->map__gtt);
4304	Serge	3179	assert(bucket(bo) == cache_bucket(num_pages));
		3180	assert(bo->proxy == NULL);
		3181	assert(bo->rq == NULL);
		3182	assert(bo->exec == NULL);
		3183	assert(!bo->scanout);
		3184
		3185	if (num_pages > num_pages(bo)) {
		3186	DBG(("inactive too small: %d < %d\n",
		3187	num_pages(bo), num_pages));
		3188	continue;
		3189	}
		3190
		3191	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3192	kgem_bo_free(kgem, bo);
		3193	break;
		3194	}
		3195
		3196	if (I915_TILING_NONE != bo->tiling &&
		3197	!gem_set_tiling(kgem->fd, bo->handle,
		3198	I915_TILING_NONE, 0))
		3199	continue;
		3200
		3201	kgem_bo_remove_from_inactive(kgem, bo);
4501	Serge	3202	assert(list_is_empty(&bo->vma));
		3203	assert(list_is_empty(&bo->list));
4304	Serge	3204
		3205	bo->tiling = I915_TILING_NONE;
		3206	bo->pitch = 0;
		3207	bo->delta = 0;
		3208	DBG((" %s: found handle=%d (num_pages=%d) in linear vma cache\n",
		3209	__FUNCTION__, bo->handle, num_pages(bo)));
		3210	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		3211	assert(!bo->needs_flush);
		3212	assert_tiling(kgem, bo);
		3213	ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
		3214	return bo;
		3215	}
		3216
		3217	if (flags & CREATE_EXACT)
		3218	return NULL;
		3219
		3220	if (flags & CREATE_CPU_MAP && !kgem->has_llc)
		3221	return NULL;
		3222	}
		3223
		3224	cache = use_active ? active(kgem, num_pages, I915_TILING_NONE) : inactive(kgem, num_pages);
		3225	list_for_each_entry(bo, cache, list) {
		3226	assert(bo->refcnt == 0);
		3227	assert(bo->reusable);
		3228	assert(!!bo->rq == !!use_active);
		3229	assert(bo->proxy == NULL);
		3230	assert(!bo->scanout);
		3231
		3232	if (num_pages > num_pages(bo))
		3233	continue;
		3234
		3235	if (use_active &&
		3236	kgem->gen <= 040 &&
		3237	bo->tiling != I915_TILING_NONE)
		3238	continue;
		3239
		3240	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3241	kgem_bo_free(kgem, bo);
		3242	break;
		3243	}
		3244
		3245	if (I915_TILING_NONE != bo->tiling) {
		3246	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP))
		3247	continue;
		3248
		3249	if (first)
		3250	continue;
		3251
		3252	if (!gem_set_tiling(kgem->fd, bo->handle,
		3253	I915_TILING_NONE, 0))
		3254	continue;
		3255
		3256	bo->tiling = I915_TILING_NONE;
		3257	bo->pitch = 0;
		3258	}
		3259
4501	Serge	3260	if (bo->map__gtt \|\| bo->map__cpu) {
4304	Serge	3261	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3262	int for_cpu = !!(flags & CREATE_CPU_MAP);
4501	Serge	3263	if (for_cpu ? bo->map__cpu : bo->map__gtt){
4304	Serge	3264	if (first != NULL)
		3265	break;
		3266
		3267	first = bo;
		3268	continue;
		3269	}
		3270	} else {
		3271	if (first != NULL)
		3272	break;
		3273
		3274	first = bo;
		3275	continue;
		3276	}
		3277	} else {
4501	Serge	3278	if (flags & CREATE_GTT_MAP && !kgem_bo_can_map(kgem, bo))
		3279	continue;
		3280
4304	Serge	3281	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3282	if (first != NULL)
		3283	break;
		3284
		3285	first = bo;
		3286	continue;
		3287	}
		3288	}
		3289
		3290	if (use_active)
		3291	kgem_bo_remove_from_active(kgem, bo);
		3292	else
		3293	kgem_bo_remove_from_inactive(kgem, bo);
		3294
		3295	assert(bo->tiling == I915_TILING_NONE);
		3296	bo->pitch = 0;
		3297	bo->delta = 0;
		3298	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		3299	__FUNCTION__, bo->handle, num_pages(bo),
		3300	use_active ? "active" : "inactive"));
		3301	assert(list_is_empty(&bo->list));
4501	Serge	3302	assert(list_is_empty(&bo->vma));
4304	Serge	3303	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		3304	assert(!bo->needs_flush \|\| use_active);
		3305	assert_tiling(kgem, bo);
		3306	ASSERT_MAYBE_IDLE(kgem, bo->handle, !use_active);
		3307	return bo;
		3308	}
		3309
		3310	if (first) {
		3311	assert(first->tiling == I915_TILING_NONE);
		3312
		3313	if (use_active)
		3314	kgem_bo_remove_from_active(kgem, first);
		3315	else
		3316	kgem_bo_remove_from_inactive(kgem, first);
		3317
		3318	first->pitch = 0;
		3319	first->delta = 0;
		3320	DBG((" %s: found handle=%d (near-miss) (num_pages=%d) in linear %s cache\n",
		3321	__FUNCTION__, first->handle, num_pages(first),
		3322	use_active ? "active" : "inactive"));
		3323	assert(list_is_empty(&first->list));
4501	Serge	3324	assert(list_is_empty(&first->vma));
4304	Serge	3325	assert(use_active \|\| first->domain != DOMAIN_GPU);
		3326	assert(!first->needs_flush \|\| use_active);
		3327	ASSERT_MAYBE_IDLE(kgem, first->handle, !use_active);
		3328	return first;
		3329	}
		3330
		3331	return NULL;
		3332	}
		3333
		3334
		3335	struct kgem_bo kgem_create_linear(struct kgem kgem, int size, unsigned flags)
		3336	{
		3337	struct kgem_bo *bo;
		3338	uint32_t handle;
		3339
		3340	DBG(("%s(%d)\n", __FUNCTION__, size));
		3341	assert(size);
		3342
		3343	if (flags & CREATE_GTT_MAP && kgem->has_llc) {
		3344	flags &= ~CREATE_GTT_MAP;
		3345	flags \|= CREATE_CPU_MAP;
		3346	}
		3347
		3348	size = NUM_PAGES(size);
		3349	bo = search_linear_cache(kgem, size, CREATE_INACTIVE \| flags);
		3350	if (bo) {
		3351	assert(bo->domain != DOMAIN_GPU);
		3352	ASSERT_IDLE(kgem, bo->handle);
		3353	bo->refcnt = 1;
		3354	return bo;
		3355	}
		3356
		3357	if (flags & CREATE_CACHED)
		3358	return NULL;
		3359
		3360	handle = gem_create(kgem->fd, size);
		3361	if (handle == 0)
		3362	return NULL;
		3363
		3364	DBG(("%s: new handle=%d, num_pages=%d\n", __FUNCTION__, handle, size));
		3365	bo = __kgem_bo_alloc(handle, size);
		3366	if (bo == NULL) {
		3367	gem_close(kgem->fd, handle);
		3368	return NULL;
		3369	}
		3370
		3371	debug_alloc__bo(kgem, bo);
		3372	return bo;
		3373	}
		3374
		3375	inline int kgem_bo_fenced_size(struct kgem kgem, struct kgem_bo bo)
		3376	{
		3377	unsigned int size;
		3378
		3379	assert(bo->tiling);
		3380	assert_tiling(kgem, bo);
		3381	assert(kgem->gen < 040);
		3382
		3383	if (kgem->gen < 030)
4501	Serge	3384	size = 512 * 1024 / PAGE_SIZE;
4304	Serge	3385	else
4501	Serge	3386	size = 1024 * 1024 / PAGE_SIZE;
		3387	while (size < num_pages(bo))
		3388	size <<= 1;
4304	Serge	3389
		3390	return size;
		3391	}
		3392
		3393	struct kgem_bo kgem_create_2d(struct kgem kgem,
		3394	int width,
		3395	int height,
		3396	int bpp,
		3397	int tiling,
		3398	uint32_t flags)
		3399	{
		3400	struct list *cache;
		3401	struct kgem_bo *bo;
		3402	uint32_t pitch, tiled_height, size;
		3403	uint32_t handle;
		3404	int i, bucket, retry;
		3405	bool exact = flags & (CREATE_EXACT \| CREATE_SCANOUT);
		3406
		3407	if (tiling < 0)
		3408	exact = true, tiling = -tiling;
		3409
		3410	DBG(("%s(%dx%d, bpp=%d, tiling=%d, exact=%d, inactive=%d, cpu-mapping=%d, gtt-mapping=%d, scanout?=%d, prime?=%d, temp?=%d)\n", __FUNCTION__,
		3411	width, height, bpp, tiling, exact,
		3412	!!(flags & CREATE_INACTIVE),
		3413	!!(flags & CREATE_CPU_MAP),
		3414	!!(flags & CREATE_GTT_MAP),
		3415	!!(flags & CREATE_SCANOUT),
		3416	!!(flags & CREATE_PRIME),
		3417	!!(flags & CREATE_TEMPORARY)));
		3418
		3419	size = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags,
		3420	width, height, bpp, tiling, &pitch);
		3421	assert(size && size <= kgem->max_object_size);
		3422	size /= PAGE_SIZE;
		3423	bucket = cache_bucket(size);
		3424
		3425	if (bucket >= NUM_CACHE_BUCKETS) {
		3426	DBG(("%s: large bo num pages=%d, bucket=%d\n",
		3427	__FUNCTION__, size, bucket));
		3428
		3429	if (flags & CREATE_INACTIVE)
		3430	goto large_inactive;
		3431
		3432	tiled_height = kgem_aligned_height(kgem, height, tiling);
		3433
		3434	list_for_each_entry(bo, &kgem->large, list) {
		3435	assert(!bo->purged);
		3436	assert(!bo->scanout);
		3437	assert(bo->refcnt == 0);
		3438	assert(bo->reusable);
		3439	assert_tiling(kgem, bo);
		3440
		3441	if (kgem->gen < 040) {
		3442	if (bo->pitch < pitch) {
		3443	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3444	bo->tiling, tiling,
		3445	bo->pitch, pitch));
		3446	continue;
		3447	}
		3448
		3449	if (bo->pitch * tiled_height > bytes(bo))
		3450	continue;
		3451	} else {
		3452	if (num_pages(bo) < size)
		3453	continue;
		3454
		3455	if (bo->pitch != pitch \|\| bo->tiling != tiling) {
		3456	if (!gem_set_tiling(kgem->fd, bo->handle,
		3457	tiling, pitch))
		3458	continue;
		3459
		3460	bo->pitch = pitch;
		3461	bo->tiling = tiling;
		3462	}
		3463	}
		3464
		3465	kgem_bo_remove_from_active(kgem, bo);
		3466
		3467	bo->unique_id = kgem_get_unique_id(kgem);
		3468	bo->delta = 0;
		3469	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3470	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3471	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3472	assert_tiling(kgem, bo);
		3473	bo->refcnt = 1;
		3474	return bo;
		3475	}
		3476
		3477	large_inactive:
		3478	__kgem_throttle_retire(kgem, flags);
		3479	list_for_each_entry(bo, &kgem->large_inactive, list) {
		3480	assert(bo->refcnt == 0);
		3481	assert(bo->reusable);
		3482	assert(!bo->scanout);
		3483	assert_tiling(kgem, bo);
		3484
		3485	if (size > num_pages(bo))
		3486	continue;
		3487
		3488	if (bo->tiling != tiling \|\|
		3489	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3490	if (!gem_set_tiling(kgem->fd, bo->handle,
		3491	tiling, pitch))
		3492	continue;
		3493
		3494	bo->tiling = tiling;
		3495	bo->pitch = pitch;
		3496	}
		3497
		3498	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3499	kgem_bo_free(kgem, bo);
		3500	break;
		3501	}
		3502
		3503	list_del(&bo->list);
		3504
		3505	assert(bo->domain != DOMAIN_GPU);
		3506	bo->unique_id = kgem_get_unique_id(kgem);
		3507	bo->pitch = pitch;
		3508	bo->delta = 0;
		3509	DBG((" 1:from large inactive: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3510	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3511	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3512	assert_tiling(kgem, bo);
		3513	bo->refcnt = 1;
		3514	return bo;
		3515	}
		3516
		3517	goto create;
		3518	}
		3519
		3520	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		3521	int for_cpu = !!(flags & CREATE_CPU_MAP);
		3522	if (kgem->has_llc && tiling == I915_TILING_NONE)
		3523	for_cpu = 1;
		3524	/* We presume that we will need to upload to this bo,
		3525	* and so would prefer to have an active VMA.
		3526	*/
		3527	cache = &kgem->vma[for_cpu].inactive[bucket];
		3528	do {
		3529	list_for_each_entry(bo, cache, vma) {
		3530	assert(bucket(bo) == bucket);
		3531	assert(bo->refcnt == 0);
		3532	assert(!bo->scanout);
4501	Serge	3533	assert(for_cpu ? bo->map__cpu : bo->map__gtt);
4304	Serge	3534	assert(bo->rq == NULL);
4501	Serge	3535	assert(bo->exec == NULL);
4304	Serge	3536	assert(list_is_empty(&bo->request));
		3537	assert(bo->flush == false);
		3538	assert_tiling(kgem, bo);
		3539
		3540	if (size > num_pages(bo)) {
		3541	DBG(("inactive too small: %d < %d\n",
		3542	num_pages(bo), size));
		3543	continue;
		3544	}
		3545
		3546	if (bo->tiling != tiling \|\|
		3547	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3548	DBG(("inactive vma with wrong tiling: %d < %d\n",
		3549	bo->tiling, tiling));
		3550	continue;
		3551	}
		3552
		3553	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3554	kgem_bo_free(kgem, bo);
		3555	break;
		3556	}
		3557
		3558	assert(bo->tiling == tiling);
		3559	bo->pitch = pitch;
		3560	bo->delta = 0;
		3561	bo->unique_id = kgem_get_unique_id(kgem);
		3562	bo->domain = DOMAIN_NONE;
		3563
		3564	kgem_bo_remove_from_inactive(kgem, bo);
4501	Serge	3565	assert(list_is_empty(&bo->list));
		3566	assert(list_is_empty(&bo->vma));
4304	Serge	3567
		3568	DBG((" from inactive vma: pitch=%d, tiling=%d: handle=%d, id=%d\n",
		3569	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3570	assert(bo->reusable);
		3571	assert(bo->domain != DOMAIN_GPU);
		3572	ASSERT_IDLE(kgem, bo->handle);
		3573	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3574	assert_tiling(kgem, bo);
		3575	bo->refcnt = 1;
		3576	return bo;
		3577	}
		3578	} while (!list_is_empty(cache) &&
		3579	__kgem_throttle_retire(kgem, flags));
		3580
		3581	if (flags & CREATE_CPU_MAP && !kgem->has_llc) {
		3582	if (list_is_empty(&kgem->active[bucket][tiling]) &&
		3583	list_is_empty(&kgem->inactive[bucket]))
		3584	flags &= ~CREATE_CACHED;
		3585
		3586	goto create;
		3587	}
		3588	}
		3589
		3590	if (flags & CREATE_INACTIVE)
		3591	goto skip_active_search;
		3592
		3593	/* Best active match */
		3594	retry = NUM_CACHE_BUCKETS - bucket;
		3595	if (retry > 3 && (flags & CREATE_TEMPORARY) == 0)
		3596	retry = 3;
		3597	search_again:
		3598	assert(bucket < NUM_CACHE_BUCKETS);
		3599	cache = &kgem->active[bucket][tiling];
		3600	if (tiling) {
		3601	tiled_height = kgem_aligned_height(kgem, height, tiling);
		3602	list_for_each_entry(bo, cache, list) {
		3603	assert(!bo->purged);
		3604	assert(bo->refcnt == 0);
		3605	assert(bucket(bo) == bucket);
		3606	assert(bo->reusable);
		3607	assert(bo->tiling == tiling);
		3608	assert(bo->flush == false);
		3609	assert(!bo->scanout);
		3610	assert_tiling(kgem, bo);
		3611
		3612	if (kgem->gen < 040) {
		3613	if (bo->pitch < pitch) {
		3614	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3615	bo->tiling, tiling,
		3616	bo->pitch, pitch));
		3617	continue;
		3618	}
		3619
		3620	if (bo->pitch * tiled_height > bytes(bo))
		3621	continue;
		3622	} else {
		3623	if (num_pages(bo) < size)
		3624	continue;
		3625
		3626	if (bo->pitch != pitch) {
		3627	if (!gem_set_tiling(kgem->fd,
		3628	bo->handle,
		3629	tiling, pitch))
		3630	continue;
		3631
		3632	bo->pitch = pitch;
		3633	}
		3634	}
		3635
		3636	kgem_bo_remove_from_active(kgem, bo);
		3637
		3638	bo->unique_id = kgem_get_unique_id(kgem);
		3639	bo->delta = 0;
		3640	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3641	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3642	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3643	assert_tiling(kgem, bo);
		3644	bo->refcnt = 1;
		3645	return bo;
		3646	}
		3647	} else {
		3648	list_for_each_entry(bo, cache, list) {
		3649	assert(bucket(bo) == bucket);
		3650	assert(!bo->purged);
		3651	assert(bo->refcnt == 0);
		3652	assert(bo->reusable);
		3653	assert(!bo->scanout);
		3654	assert(bo->tiling == tiling);
		3655	assert(bo->flush == false);
		3656	assert_tiling(kgem, bo);
		3657
		3658	if (num_pages(bo) < size)
		3659	continue;
		3660
		3661	kgem_bo_remove_from_active(kgem, bo);
		3662
		3663	bo->pitch = pitch;
		3664	bo->unique_id = kgem_get_unique_id(kgem);
		3665	bo->delta = 0;
		3666	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3667	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3668	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3669	assert_tiling(kgem, bo);
		3670	bo->refcnt = 1;
		3671	return bo;
		3672	}
		3673	}
		3674
		3675	if (--retry && exact) {
		3676	if (kgem->gen >= 040) {
		3677	for (i = I915_TILING_NONE; i <= I915_TILING_Y; i++) {
		3678	if (i == tiling)
		3679	continue;
		3680
		3681	cache = &kgem->active[bucket][i];
		3682	list_for_each_entry(bo, cache, list) {
		3683	assert(!bo->purged);
		3684	assert(bo->refcnt == 0);
		3685	assert(bo->reusable);
		3686	assert(!bo->scanout);
		3687	assert(bo->flush == false);
		3688	assert_tiling(kgem, bo);
		3689
		3690	if (num_pages(bo) < size)
		3691	continue;
		3692
		3693	if (!gem_set_tiling(kgem->fd,
		3694	bo->handle,
		3695	tiling, pitch))
		3696	continue;
		3697
		3698	kgem_bo_remove_from_active(kgem, bo);
		3699
		3700	bo->unique_id = kgem_get_unique_id(kgem);
		3701	bo->pitch = pitch;
		3702	bo->tiling = tiling;
		3703	bo->delta = 0;
		3704	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3705	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3706	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3707	assert_tiling(kgem, bo);
		3708	bo->refcnt = 1;
		3709	return bo;
		3710	}
		3711	}
		3712	}
		3713
		3714	bucket++;
		3715	goto search_again;
		3716	}
		3717
		3718	if (!exact) { /* allow an active near-miss? */
		3719	i = tiling;
		3720	while (--i >= 0) {
		3721	tiled_height = kgem_surface_size(kgem, kgem->has_relaxed_fencing, flags,
		3722	width, height, bpp, tiling, &pitch);
		3723	cache = active(kgem, tiled_height / PAGE_SIZE, i);
		3724	tiled_height = kgem_aligned_height(kgem, height, i);
		3725	list_for_each_entry(bo, cache, list) {
		3726	assert(!bo->purged);
		3727	assert(bo->refcnt == 0);
		3728	assert(bo->reusable);
		3729	assert(!bo->scanout);
		3730	assert(bo->flush == false);
		3731	assert_tiling(kgem, bo);
		3732
		3733	if (bo->tiling) {
		3734	if (bo->pitch < pitch) {
		3735	DBG(("tiled and pitch too small: tiling=%d, (want %d), pitch=%d, need %d\n",
		3736	bo->tiling, tiling,
		3737	bo->pitch, pitch));
		3738	continue;
		3739	}
		3740	} else
		3741	bo->pitch = pitch;
		3742
		3743	if (bo->pitch * tiled_height > bytes(bo))
		3744	continue;
		3745
		3746	kgem_bo_remove_from_active(kgem, bo);
		3747
		3748	bo->unique_id = kgem_get_unique_id(kgem);
		3749	bo->delta = 0;
		3750	DBG((" 1:from active: pitch=%d, tiling=%d, handle=%d, id=%d\n",
		3751	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3752	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3753	assert_tiling(kgem, bo);
		3754	bo->refcnt = 1;
		3755	return bo;
		3756	}
		3757	}
		3758	}
		3759
		3760	skip_active_search:
		3761	bucket = cache_bucket(size);
		3762	retry = NUM_CACHE_BUCKETS - bucket;
		3763	if (retry > 3)
		3764	retry = 3;
		3765	search_inactive:
		3766	/* Now just look for a close match and prefer any currently active */
		3767	assert(bucket < NUM_CACHE_BUCKETS);
		3768	cache = &kgem->inactive[bucket];
		3769	list_for_each_entry(bo, cache, list) {
		3770	assert(bucket(bo) == bucket);
		3771	assert(bo->reusable);
		3772	assert(!bo->scanout);
		3773	assert(bo->flush == false);
		3774	assert_tiling(kgem, bo);
		3775
		3776	if (size > num_pages(bo)) {
		3777	DBG(("inactive too small: %d < %d\n",
		3778	num_pages(bo), size));
		3779	continue;
		3780	}
		3781
		3782	if (bo->tiling != tiling \|\|
		3783	(tiling != I915_TILING_NONE && bo->pitch != pitch)) {
		3784	if (!gem_set_tiling(kgem->fd, bo->handle,
		3785	tiling, pitch))
		3786	continue;
		3787	}
		3788
		3789	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		3790	kgem_bo_free(kgem, bo);
		3791	break;
		3792	}
		3793
		3794	kgem_bo_remove_from_inactive(kgem, bo);
4501	Serge	3795	assert(list_is_empty(&bo->list));
		3796	assert(list_is_empty(&bo->vma));
4304	Serge	3797
		3798	bo->pitch = pitch;
		3799	bo->tiling = tiling;
		3800
		3801	bo->delta = 0;
		3802	bo->unique_id = kgem_get_unique_id(kgem);
		3803	assert(bo->pitch);
		3804	DBG((" from inactive: pitch=%d, tiling=%d: handle=%d, id=%d\n",
		3805	bo->pitch, bo->tiling, bo->handle, bo->unique_id));
		3806	assert(bo->refcnt == 0);
		3807	assert(bo->reusable);
		3808	assert((flags & CREATE_INACTIVE) == 0 \|\| bo->domain != DOMAIN_GPU);
		3809	ASSERT_MAYBE_IDLE(kgem, bo->handle, flags & CREATE_INACTIVE);
		3810	assert(bo->pitch*kgem_aligned_height(kgem, height, bo->tiling) <= kgem_bo_size(bo));
		3811	assert_tiling(kgem, bo);
		3812	bo->refcnt = 1;
		3813	return bo;
		3814	}
		3815
		3816	if (flags & CREATE_INACTIVE &&
		3817	!list_is_empty(&kgem->active[bucket][tiling]) &&
		3818	__kgem_throttle_retire(kgem, flags)) {
		3819	flags &= ~CREATE_INACTIVE;
		3820	goto search_inactive;
		3821	}
		3822
		3823	if (--retry) {
		3824	bucket++;
		3825	flags &= ~CREATE_INACTIVE;
		3826	goto search_inactive;
		3827	}
		3828
		3829	create:
		3830	if (flags & CREATE_CACHED)
		3831	return NULL;
		3832
		3833	if (bucket >= NUM_CACHE_BUCKETS)
		3834	size = ALIGN(size, 1024);
		3835	handle = gem_create(kgem->fd, size);
		3836	if (handle == 0)
		3837	return NULL;
		3838
		3839	bo = __kgem_bo_alloc(handle, size);
		3840	if (!bo) {
		3841	gem_close(kgem->fd, handle);
		3842	return NULL;
		3843	}
		3844
		3845	bo->unique_id = kgem_get_unique_id(kgem);
		3846	if (tiling == I915_TILING_NONE \|\|
		3847	gem_set_tiling(kgem->fd, handle, tiling, pitch)) {
		3848	bo->tiling = tiling;
		3849	bo->pitch = pitch;
		3850	} else {
		3851	if (flags & CREATE_EXACT) {
		3852	if (bo->pitch != pitch \|\| bo->tiling != tiling) {
		3853	kgem_bo_free(kgem, bo);
		3854	return NULL;
		3855	}
		3856	}
		3857	}
		3858
		3859	assert(bytes(bo) >= bo->pitch * kgem_aligned_height(kgem, height, bo->tiling));
		3860	assert_tiling(kgem, bo);
		3861
		3862	debug_alloc__bo(kgem, bo);
		3863
		3864	DBG((" new pitch=%d, tiling=%d, handle=%d, id=%d, num_pages=%d [%d], bucket=%d\n",
		3865	bo->pitch, bo->tiling, bo->handle, bo->unique_id,
		3866	size, num_pages(bo), bucket(bo)));
		3867	return bo;
		3868	}
		3869
		3870	#if 0
		3871	struct kgem_bo kgem_create_cpu_2d(struct kgem kgem,
		3872	int width,
		3873	int height,
		3874	int bpp,
		3875	uint32_t flags)
		3876	{
		3877	struct kgem_bo *bo;
		3878	int stride, size;
		3879
		3880	if (DBG_NO_CPU)
		3881	return NULL;
		3882
		3883	DBG(("%s(%dx%d, bpp=%d)\n", __FUNCTION__, width, height, bpp));
		3884
		3885	if (kgem->has_llc) {
		3886	bo = kgem_create_2d(kgem, width, height, bpp,
		3887	I915_TILING_NONE, flags);
		3888	if (bo == NULL)
		3889	return bo;
		3890
		3891	assert(bo->tiling == I915_TILING_NONE);
		3892	assert_tiling(kgem, bo);
		3893
		3894	if (kgem_bo_map__cpu(kgem, bo) == NULL) {
		3895	kgem_bo_destroy(kgem, bo);
		3896	return NULL;
		3897	}
		3898
		3899	return bo;
		3900	}
		3901
		3902	assert(width > 0 && height > 0);
		3903	stride = ALIGN(width, 2) * bpp >> 3;
		3904	stride = ALIGN(stride, 4);
		3905	size = stride * ALIGN(height, 2);
		3906	assert(size >= PAGE_SIZE);
		3907
		3908	DBG(("%s: %dx%d, %d bpp, stride=%d\n",
		3909	__FUNCTION__, width, height, bpp, stride));
		3910
		3911	bo = search_snoop_cache(kgem, NUM_PAGES(size), 0);
		3912	if (bo) {
		3913	assert(bo->tiling == I915_TILING_NONE);
		3914	assert_tiling(kgem, bo);
		3915	assert(bo->snoop);
		3916	bo->refcnt = 1;
		3917	bo->pitch = stride;
		3918	bo->unique_id = kgem_get_unique_id(kgem);
		3919	return bo;
		3920	}
		3921
		3922	if (kgem->has_caching) {
		3923	bo = kgem_create_linear(kgem, size, flags);
		3924	if (bo == NULL)
		3925	return NULL;
		3926
		3927	assert(bo->tiling == I915_TILING_NONE);
		3928	assert_tiling(kgem, bo);
		3929
		3930	if (!gem_set_caching(kgem->fd, bo->handle, SNOOPED)) {
		3931	kgem_bo_destroy(kgem, bo);
		3932	return NULL;
		3933	}
		3934	bo->snoop = true;
		3935
		3936	if (kgem_bo_map__cpu(kgem, bo) == NULL) {
		3937	kgem_bo_destroy(kgem, bo);
		3938	return NULL;
		3939	}
		3940
		3941	bo->pitch = stride;
		3942	bo->unique_id = kgem_get_unique_id(kgem);
		3943	return bo;
		3944	}
		3945
		3946	if (kgem->has_userptr) {
		3947	void *ptr;
		3948
		3949	/* XXX */
		3950	//if (posix_memalign(&ptr, 64, ALIGN(size, 64)))
		3951	if (posix_memalign(&ptr, PAGE_SIZE, ALIGN(size, PAGE_SIZE)))
		3952	return NULL;
		3953
		3954	bo = kgem_create_map(kgem, ptr, size, false);
		3955	if (bo == NULL) {
		3956	free(ptr);
		3957	return NULL;
		3958	}
		3959
		3960	bo->pitch = stride;
		3961	bo->unique_id = kgem_get_unique_id(kgem);
		3962	return bo;
		3963	}
		3964
		3965	return NULL;
		3966	}
		3967	#endif
		3968
		3969	void _kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		3970	{
		3971	DBG(("%s: handle=%d, proxy? %d\n",
		3972	__FUNCTION__, bo->handle, bo->proxy != NULL));
		3973
		3974	if (bo->proxy) {
4501	Serge	3975	assert(!bo->reusable);
		3976	kgem_bo_binding_free(kgem, bo);
		3977
		3978	assert(list_is_empty(&bo->list));
4304	Serge	3979	_list_del(&bo->vma);
		3980	_list_del(&bo->request);
4501	Serge	3981
		3982	if (bo->io && bo->domain == DOMAIN_CPU)
4304	Serge	3983	_kgem_bo_delete_buffer(kgem, bo);
4501	Serge	3984
4304	Serge	3985	kgem_bo_unref(kgem, bo->proxy);
		3986
4501	Serge	3987	(struct kgem_bo *)bo = __kgem_freed_bo;
		3988	__kgem_freed_bo = bo;
		3989	} else
4304	Serge	3990	__kgem_bo_destroy(kgem, bo);
		3991	}
		3992
		3993	static void __kgem_flush(struct kgem kgem, struct kgem_bo bo)
		3994	{
		3995	assert(bo->rq);
		3996	assert(bo->exec == NULL);
		3997	assert(bo->needs_flush);
		3998
		3999	/* The kernel will emit a flush and update its own flushing lists. */
		4000	if (!__kgem_busy(kgem, bo->handle))
		4001	__kgem_bo_clear_busy(bo);
		4002
		4003	DBG(("%s: handle=%d, busy?=%d\n",
		4004	__FUNCTION__, bo->handle, bo->rq != NULL));
		4005	}
		4006
		4007	void kgem_scanout_flush(struct kgem kgem, struct kgem_bo bo)
		4008	{
		4009	kgem_bo_submit(kgem, bo);
		4010	if (!bo->needs_flush)
		4011	return;
		4012
		4013	/* If the kernel fails to emit the flush, then it will be forced when
		4014	* we assume direct access. And as the usual failure is EIO, we do
		4015	* not actually care.
		4016	*/
		4017	assert(bo->exec == NULL);
		4018	if (bo->rq)
		4019	__kgem_flush(kgem, bo);
		4020
		4021	/* Whatever actually happens, we can regard the GTT write domain
		4022	* as being flushed.
		4023	*/
		4024	bo->gtt_dirty = false;
		4025	bo->needs_flush = false;
		4026	bo->domain = DOMAIN_NONE;
		4027	}
		4028
		4029	inline static bool needs_semaphore(struct kgem kgem, struct kgem_bo bo)
		4030	{
		4031	return kgem->nreloc && bo->rq && RQ_RING(bo->rq) != kgem->ring;
		4032	}
		4033
4501	Serge	4034	static bool aperture_check(struct kgem *kgem, unsigned num_pages)
		4035	{
		4036	if (kgem->aperture) {
		4037	struct drm_i915_gem_get_aperture aperture;
		4038
		4039	VG_CLEAR(aperture);
		4040	aperture.aper_available_size = kgem->aperture_high;
		4041	aperture.aper_available_size *= PAGE_SIZE;
		4042	(void)drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_GET_APERTURE, &aperture);
		4043
		4044	DBG(("%s: aperture required %ld bytes, available %ld bytes\n",
		4045	__FUNCTION__,
		4046	(long)num_pages * PAGE_SIZE,
		4047	(long)aperture.aper_available_size));
		4048
		4049	/* Leave some space in case of alignment issues */
		4050	aperture.aper_available_size -= 1024 * 1024;
		4051	aperture.aper_available_size -= kgem->aperture_mappable * PAGE_SIZE / 2;
		4052	if (kgem->gen < 033)
		4053	aperture.aper_available_size -= kgem->aperture_max_fence * PAGE_SIZE;
		4054	if (!kgem->has_llc)
		4055	aperture.aper_available_size -= 2 * kgem->nexec * PAGE_SIZE;
		4056
		4057	DBG(("%s: num_pages=%d, estimated max usable=%ld\n",
		4058	__FUNCTION__, num_pages, (long)(aperture.aper_available_size/PAGE_SIZE)));
		4059
		4060	if (num_pages <= aperture.aper_available_size / PAGE_SIZE)
		4061	return true;
		4062	}
		4063
		4064	return false;
		4065	}
		4066
		4067	static inline bool kgem_flush(struct kgem *kgem, bool flush)
		4068	{
		4069	if (unlikely(kgem->wedged))
		4070	return false;
		4071
		4072	if (kgem->nreloc == 0)
		4073	return true;
		4074
		4075	if (container_of(kgem, struct sna, kgem)->flags & SNA_POWERSAVE)
		4076	return true;
		4077
		4078	if (kgem->flush == flush && kgem->aperture < kgem->aperture_low)
		4079	return true;
		4080
		4081	DBG(("%s: opportunistic flushing? flush=%d,%d, aperture=%d/%d, idle?=%d\n",
		4082	__FUNCTION__, kgem->flush, flush, kgem->aperture, kgem->aperture_low, kgem_ring_is_idle(kgem, kgem->ring)));
		4083	return !kgem_ring_is_idle(kgem, kgem->ring);
		4084	}
		4085
4304	Serge	4086	bool kgem_check_bo(struct kgem *kgem, ...)
		4087	{
		4088	va_list ap;
		4089	struct kgem_bo *bo;
		4090	int num_exec = 0;
		4091	int num_pages = 0;
		4092	bool flush = false;
4501	Serge	4093	bool busy = true;
4304	Serge	4094
		4095	va_start(ap, kgem);
		4096	while ((bo = va_arg(ap, struct kgem_bo *))) {
		4097	while (bo->proxy)
		4098	bo = bo->proxy;
		4099	if (bo->exec)
		4100	continue;
		4101
4501	Serge	4102	if (needs_semaphore(kgem, bo)) {
		4103	DBG(("%s: flushing for required semaphore\n", __FUNCTION__));
4304	Serge	4104	return false;
4501	Serge	4105	}
4304	Serge	4106
		4107	num_pages += num_pages(bo);
		4108	num_exec++;
		4109
		4110	flush \|= bo->flush;
4501	Serge	4111	busy &= bo->rq != NULL;
4304	Serge	4112	}
		4113	va_end(ap);
		4114
		4115	DBG(("%s: num_pages=+%d, num_exec=+%d\n",
		4116	__FUNCTION__, num_pages, num_exec));
		4117
		4118	if (!num_pages)
		4119	return true;
		4120
4501	Serge	4121	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem)) {
		4122	DBG(("%s: out of exec slots (%d + %d / %d)\n", __FUNCTION__,
		4123	kgem->nexec, num_exec, KGEM_EXEC_SIZE(kgem)));
4304	Serge	4124	return false;
		4125	}
		4126
		4127	if (num_pages + kgem->aperture > kgem->aperture_high) {
		4128	DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n",
		4129	__FUNCTION__, num_pages + kgem->aperture, kgem->aperture_high));
4501	Serge	4130	if (!aperture_check(kgem, num_pages + kgem->aperture))
4304	Serge	4131	return false;
		4132	}
		4133
4501	Serge	4134	if (busy)
		4135	return true;
4304	Serge	4136
4501	Serge	4137	return kgem_flush(kgem, flush);
4304	Serge	4138	}
		4139
4501	Serge	4140	#if 0
		4141	bool kgem_check_bo_fenced(struct kgem kgem, struct kgem_bo bo)
		4142	{
		4143	assert(bo->refcnt);
		4144	while (bo->proxy)
		4145	bo = bo->proxy;
		4146	assert(bo->refcnt);
4304	Serge	4147
4501	Serge	4148	if (bo->exec) {
		4149	if (kgem->gen < 040 &&
		4150	bo->tiling != I915_TILING_NONE &&
		4151	(bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		4152	uint32_t size;
4304	Serge	4153
4501	Serge	4154	assert(bo->tiling == I915_TILING_X);
4304	Serge	4155
4501	Serge	4156	if (kgem->nfence >= kgem->fence_max)
		4157	return false;
4304	Serge	4158
4501	Serge	4159	if (kgem->aperture_fenced) {
		4160	size = 3*kgem->aperture_fenced;
		4161	if (kgem->aperture_total == kgem->aperture_mappable)
		4162	size += kgem->aperture;
		4163	if (size > kgem->aperture_mappable &&
		4164	kgem_ring_is_idle(kgem, kgem->ring)) {
		4165	DBG(("%s: opportunistic fence flush\n", __FUNCTION__));
		4166	return false;
		4167	}
		4168	}
4304	Serge	4169
4501	Serge	4170	size = kgem_bo_fenced_size(kgem, bo);
		4171	if (size > kgem->aperture_max_fence)
		4172	kgem->aperture_max_fence = size;
		4173	size += kgem->aperture_fenced;
		4174	if (kgem->gen < 033)
		4175	size += kgem->aperture_max_fence;
		4176	if (kgem->aperture_total == kgem->aperture_mappable)
		4177	size += kgem->aperture;
		4178	if (size > kgem->aperture_mappable) {
		4179	DBG(("%s: estimated fence space required [%d] exceed aperture [%d]\n",
		4180	__FUNCTION__, size, kgem->aperture_mappable));
		4181	return false;
		4182	}
		4183	}
4304	Serge	4184
4501	Serge	4185	return true;
		4186	}
4304	Serge	4187
4501	Serge	4188	if (kgem->nexec >= KGEM_EXEC_SIZE(kgem) - 1)
		4189	return false;
4304	Serge	4190
4501	Serge	4191	if (needs_semaphore(kgem, bo)) {
		4192	DBG(("%s: flushing for required semaphore\n", __FUNCTION__));
		4193	return false;
		4194	}
4304	Serge	4195
4501	Serge	4196	assert_tiling(kgem, bo);
		4197	if (kgem->gen < 040 && bo->tiling != I915_TILING_NONE) {
		4198	uint32_t size;
4304	Serge	4199
4501	Serge	4200	assert(bo->tiling == I915_TILING_X);
4304	Serge	4201
4501	Serge	4202	if (kgem->nfence >= kgem->fence_max)
		4203	return false;
4304	Serge	4204
4501	Serge	4205	if (kgem->aperture_fenced) {
		4206	size = 3*kgem->aperture_fenced;
		4207	if (kgem->aperture_total == kgem->aperture_mappable)
		4208	size += kgem->aperture;
		4209	if (size > kgem->aperture_mappable &&
		4210	kgem_ring_is_idle(kgem, kgem->ring)) {
		4211	DBG(("%s: opportunistic fence flush\n", __FUNCTION__));
		4212	return false;
		4213	}
		4214	}
4304	Serge	4215
4501	Serge	4216	size = kgem_bo_fenced_size(kgem, bo);
		4217	if (size > kgem->aperture_max_fence)
		4218	kgem->aperture_max_fence = size;
		4219	size += kgem->aperture_fenced;
		4220	if (kgem->gen < 033)
		4221	size += kgem->aperture_max_fence;
		4222	if (kgem->aperture_total == kgem->aperture_mappable)
		4223	size += kgem->aperture;
		4224	if (size > kgem->aperture_mappable) {
		4225	DBG(("%s: estimated fence space required [%d] exceed aperture [%d]\n",
		4226	__FUNCTION__, size, kgem->aperture_mappable));
		4227	return false;
		4228	}
		4229	}
4304	Serge	4230
4501	Serge	4231	if (kgem->aperture + kgem->aperture_fenced + num_pages(bo) > kgem->aperture_high) {
		4232	DBG(("%s: final aperture usage (%d) is greater than high water mark (%d)\n",
		4233	__FUNCTION__, num_pages(bo) + kgem->aperture, kgem->aperture_high));
		4234	if (!aperture_check(kgem, num_pages(bo) + kgem->aperture + kgem->aperture_fenced))
		4235	return false;
		4236	}
4304	Serge	4237
4501	Serge	4238	if (bo->rq)
		4239	return true;
4304	Serge	4240
4501	Serge	4241	return kgem_flush(kgem, bo->flush);
		4242	}
		4243	#endif
4304	Serge	4244
		4245
		4246
		4247
		4248
		4249
		4250
		4251
		4252
		4253
		4254
		4255
		4256
4501	Serge	4257
		4258
		4259
		4260
4304	Serge	4261	uint32_t kgem_add_reloc(struct kgem *kgem,
		4262	uint32_t pos,
		4263	struct kgem_bo *bo,
		4264	uint32_t read_write_domain,
		4265	uint32_t delta)
		4266	{
		4267	int index;
		4268
		4269	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
		4270	__FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
		4271
4501	Serge	4272	assert(kgem->gen < 0100);
4304	Serge	4273	assert((read_write_domain & 0x7fff) == 0 \|\| bo != NULL);
		4274
		4275	index = kgem->nreloc++;
		4276	assert(index < ARRAY_SIZE(kgem->reloc));
		4277	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
		4278	if (bo) {
4501	Serge	4279	assert(kgem->mode != KGEM_NONE);
4304	Serge	4280	assert(bo->refcnt);
		4281	while (bo->proxy) {
		4282	DBG(("%s: adding proxy [delta=%d] for handle=%d\n",
		4283	__FUNCTION__, bo->delta, bo->handle));
		4284	delta += bo->delta;
		4285	assert(bo->handle == bo->proxy->handle);
		4286	/* need to release the cache upon batch submit */
		4287	if (bo->exec == NULL) {
		4288	list_move_tail(&bo->request,
		4289	&kgem->next_request->buffers);
		4290	bo->rq = MAKE_REQUEST(kgem->next_request,
		4291	kgem->ring);
		4292	bo->exec = &_kgem_dummy_exec;
4501	Serge	4293	bo->domain = DOMAIN_GPU;
4304	Serge	4294	}
		4295
		4296	if (read_write_domain & 0x7fff && !bo->gpu_dirty)
		4297	__kgem_bo_mark_dirty(bo);
		4298
		4299	bo = bo->proxy;
		4300	assert(bo->refcnt);
		4301	}
		4302	assert(bo->refcnt);
		4303
		4304	if (bo->exec == NULL)
		4305	kgem_add_bo(kgem, bo);
		4306	assert(bo->rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		4307	assert(RQ_RING(bo->rq) == kgem->ring);
		4308
		4309	if (kgem->gen < 040 && read_write_domain & KGEM_RELOC_FENCED) {
		4310	if (bo->tiling &&
		4311	(bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
4501	Serge	4312	assert(bo->tiling == I915_TILING_X);
4304	Serge	4313	assert(kgem->nfence < kgem->fence_max);
		4314	kgem->aperture_fenced +=
		4315	kgem_bo_fenced_size(kgem, bo);
		4316	kgem->nfence++;
		4317	}
		4318	bo->exec->flags \|= EXEC_OBJECT_NEEDS_FENCE;
		4319	}
		4320
		4321	kgem->reloc[index].delta = delta;
		4322	kgem->reloc[index].target_handle = bo->target_handle;
		4323	kgem->reloc[index].presumed_offset = bo->presumed_offset;
		4324
		4325	if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
		4326	assert(!bo->snoop \|\| kgem->can_blt_cpu);
		4327	__kgem_bo_mark_dirty(bo);
		4328	}
		4329
		4330	delta += bo->presumed_offset;
		4331	} else {
		4332	kgem->reloc[index].delta = delta;
		4333	kgem->reloc[index].target_handle = ~0U;
		4334	kgem->reloc[index].presumed_offset = 0;
		4335	if (kgem->nreloc__self < 256)
		4336	kgem->reloc__self[kgem->nreloc__self++] = index;
		4337	}
		4338	kgem->reloc[index].read_domains = read_write_domain >> 16;
		4339	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
		4340
		4341	return delta;
		4342	}
		4343
4501	Serge	4344	uint64_t kgem_add_reloc64(struct kgem *kgem,
		4345	uint32_t pos,
		4346	struct kgem_bo *bo,
		4347	uint32_t read_write_domain,
		4348	uint64_t delta)
		4349	{
		4350	int index;
		4351
		4352	DBG(("%s: handle=%d, pos=%d, delta=%ld, domains=%08x\n",
		4353	__FUNCTION__, bo ? bo->handle : 0, pos, (long)delta, read_write_domain));
		4354
		4355	assert(kgem->gen >= 0100);
		4356	assert((read_write_domain & 0x7fff) == 0 \|\| bo != NULL);
		4357
		4358	index = kgem->nreloc++;
		4359	assert(index < ARRAY_SIZE(kgem->reloc));
		4360	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
		4361	if (bo) {
		4362	assert(kgem->mode != KGEM_NONE);
		4363	assert(bo->refcnt);
		4364	while (bo->proxy) {
		4365	DBG(("%s: adding proxy [delta=%ld] for handle=%d\n",
		4366	__FUNCTION__, (long)bo->delta, bo->handle));
		4367	delta += bo->delta;
		4368	assert(bo->handle == bo->proxy->handle);
		4369	/* need to release the cache upon batch submit */
		4370	if (bo->exec == NULL) {
		4371	list_move_tail(&bo->request,
		4372	&kgem->next_request->buffers);
		4373	bo->rq = MAKE_REQUEST(kgem->next_request,
		4374	kgem->ring);
		4375	bo->exec = &_kgem_dummy_exec;
		4376	bo->domain = DOMAIN_GPU;
		4377	}
		4378
		4379	if (read_write_domain & 0x7fff && !bo->gpu_dirty)
		4380	__kgem_bo_mark_dirty(bo);
		4381
		4382	bo = bo->proxy;
		4383	assert(bo->refcnt);
		4384	}
		4385	assert(bo->refcnt);
		4386
		4387	if (bo->exec == NULL)
		4388	kgem_add_bo(kgem, bo);
		4389	assert(bo->rq == MAKE_REQUEST(kgem->next_request, kgem->ring));
		4390	assert(RQ_RING(bo->rq) == kgem->ring);
		4391
		4392	kgem->reloc[index].delta = delta;
		4393	kgem->reloc[index].target_handle = bo->target_handle;
		4394	kgem->reloc[index].presumed_offset = bo->presumed_offset;
		4395
		4396	if (read_write_domain & 0x7fff && !bo->gpu_dirty) {
		4397	assert(!bo->snoop \|\| kgem->can_blt_cpu);
		4398	__kgem_bo_mark_dirty(bo);
		4399	}
		4400
		4401	delta += bo->presumed_offset;
		4402	} else {
		4403	kgem->reloc[index].delta = delta;
		4404	kgem->reloc[index].target_handle = ~0U;
		4405	kgem->reloc[index].presumed_offset = 0;
		4406	if (kgem->nreloc__self < 256)
		4407	kgem->reloc__self[kgem->nreloc__self++] = index;
		4408	}
		4409	kgem->reloc[index].read_domains = read_write_domain >> 16;
		4410	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
		4411
		4412	return delta;
		4413	}
		4414
4304	Serge	4415	static void kgem_trim_vma_cache(struct kgem *kgem, int type, int bucket)
		4416	{
		4417	int i, j;
		4418
		4419	DBG(("%s: type=%d, count=%d (bucket: %d)\n",
		4420	__FUNCTION__, type, kgem->vma[type].count, bucket));
		4421	if (kgem->vma[type].count <= 0)
		4422	return;
		4423
		4424	if (kgem->need_purge)
		4425	kgem_purge_cache(kgem);
		4426
		4427	/* vma are limited on a per-process basis to around 64k.
		4428	* This includes all malloc arenas as well as other file
		4429	* mappings. In order to be fair and not hog the cache,
		4430	* and more importantly not to exhaust that limit and to
		4431	* start failing mappings, we keep our own number of open
		4432	* vma to within a conservative value.
		4433	*/
		4434	i = 0;
		4435	while (kgem->vma[type].count > 0) {
		4436	struct kgem_bo *bo = NULL;
4501	Serge	4437	void **ptr;
4304	Serge	4438
		4439	for (j = 0;
		4440	bo == NULL && j < ARRAY_SIZE(kgem->vma[type].inactive);
		4441	j++) {
		4442	struct list *head = &kgem->vma[type].inactive[i++%ARRAY_SIZE(kgem->vma[type].inactive)];
		4443	if (!list_is_empty(head))
		4444	bo = list_last_entry(head, struct kgem_bo, vma);
		4445	}
		4446	if (bo == NULL)
		4447	break;
		4448
		4449	DBG(("%s: discarding inactive %s vma cache for %d\n",
4501	Serge	4450	__FUNCTION__, type ? "CPU" : "GTT", bo->handle));
		4451
		4452	ptr = type ? &bo->map__cpu : &bo->map__gtt;
4304	Serge	4453	assert(bo->rq == NULL);
		4454
4501	Serge	4455	VG(if (type) VALGRIND_MAKE_MEM_NOACCESS(MAP(*ptr), bytes(bo)));
		4456	// munmap(MAP(*ptr), bytes(bo));
		4457	*ptr = NULL;
4304	Serge	4458	list_del(&bo->vma);
		4459	kgem->vma[type].count--;
		4460
		4461	if (!bo->purged && !kgem_bo_set_purgeable(kgem, bo)) {
		4462	DBG(("%s: freeing unpurgeable old mapping\n",
		4463	__FUNCTION__));
		4464	kgem_bo_free(kgem, bo);
		4465	}
		4466	}
		4467	}
		4468
		4469	void kgem_bo_map__async(struct kgem kgem, struct kgem_bo *bo)
		4470	{
		4471	void *ptr;
		4472
4501	Serge	4473	DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__,
		4474	bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
4304	Serge	4475
		4476	assert(bo->proxy == NULL);
		4477	assert(list_is_empty(&bo->list));
		4478	assert_tiling(kgem, bo);
		4479
		4480	if (bo->tiling == I915_TILING_NONE && !bo->scanout && kgem->has_llc) {
		4481	DBG(("%s: converting request for GTT map into CPU map\n",
		4482	__FUNCTION__));
		4483	return kgem_bo_map__cpu(kgem, bo);
		4484	}
		4485
4501	Serge	4486	ptr = MAP(bo->map__gtt);
4304	Serge	4487	if (ptr == NULL) {
4501	Serge	4488	assert(num_pages(bo) <= kgem->aperture_mappable / 2);
4304	Serge	4489
		4490	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4491
		4492	ptr = __kgem_bo_map__gtt(kgem, bo);
		4493	if (ptr == NULL)
		4494	return NULL;
		4495
		4496	/* Cache this mapping to avoid the overhead of an
		4497	* excruciatingly slow GTT pagefault. This is more an
		4498	* issue with compositing managers which need to frequently
		4499	* flush CPU damage to their GPU bo.
		4500	*/
4501	Serge	4501	bo->map__gtt = ptr;
4304	Serge	4502	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4503	}
		4504
		4505	return ptr;
		4506	}
		4507
		4508	void kgem_bo_map(struct kgem kgem, struct kgem_bo *bo)
		4509	{
		4510	void *ptr;
		4511
4501	Serge	4512	DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__,
		4513	bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
4304	Serge	4514
		4515	assert(bo->proxy == NULL);
		4516	assert(list_is_empty(&bo->list));
		4517	assert(bo->exec == NULL);
		4518	assert_tiling(kgem, bo);
		4519
		4520	if (bo->tiling == I915_TILING_NONE && !bo->scanout &&
		4521	(kgem->has_llc \|\| bo->domain == DOMAIN_CPU)) {
		4522	DBG(("%s: converting request for GTT map into CPU map\n",
		4523	__FUNCTION__));
		4524	ptr = kgem_bo_map__cpu(kgem, bo);
		4525	if (ptr)
		4526	kgem_bo_sync__cpu(kgem, bo);
		4527	return ptr;
		4528	}
		4529
4501	Serge	4530	ptr = MAP(bo->map__gtt);
4304	Serge	4531	if (ptr == NULL) {
4501	Serge	4532	assert(num_pages(bo) <= kgem->aperture_mappable / 2);
4304	Serge	4533	assert(kgem->gen != 021 \|\| bo->tiling != I915_TILING_Y);
		4534
		4535	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4536
		4537	ptr = __kgem_bo_map__gtt(kgem, bo);
		4538	if (ptr == NULL)
		4539	return NULL;
		4540
		4541	/* Cache this mapping to avoid the overhead of an
		4542	* excruciatingly slow GTT pagefault. This is more an
		4543	* issue with compositing managers which need to frequently
		4544	* flush CPU damage to their GPU bo.
		4545	*/
4501	Serge	4546	bo->map__gtt = ptr;
4304	Serge	4547	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4548	}
		4549
		4550	if (bo->domain != DOMAIN_GTT \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_GTT)) {
		4551	struct drm_i915_gem_set_domain set_domain;
		4552
		4553	DBG(("%s: sync: needs_flush? %d, domain? %d, busy? %d\n", __FUNCTION__,
		4554	bo->needs_flush, bo->domain, __kgem_busy(kgem, bo->handle)));
		4555
		4556	/* XXX use PROT_READ to avoid the write flush? */
		4557
		4558	VG_CLEAR(set_domain);
		4559	set_domain.handle = bo->handle;
		4560	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		4561	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		4562	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4563	kgem_bo_retire(kgem, bo);
		4564	bo->domain = DOMAIN_GTT;
		4565	bo->gtt_dirty = true;
		4566	}
		4567	}
		4568
		4569	return ptr;
		4570	}
		4571
		4572	void kgem_bo_map__gtt(struct kgem kgem, struct kgem_bo *bo)
		4573	{
		4574	void *ptr;
		4575
4501	Serge	4576	DBG(("%s: handle=%d, offset=%ld, tiling=%d, map=%p:%p, domain=%d\n", __FUNCTION__,
		4577	bo->handle, (long)bo->presumed_offset, bo->tiling, bo->map__gtt, bo->map__cpu, bo->domain));
4304	Serge	4578
		4579	assert(bo->exec == NULL);
		4580	assert(list_is_empty(&bo->list));
		4581	assert_tiling(kgem, bo);
		4582
4501	Serge	4583	ptr = MAP(bo->map__gtt);
4304	Serge	4584	if (ptr == NULL) {
4501	Serge	4585	assert(num_pages(bo) <= kgem->aperture_mappable / 4);
4304	Serge	4586
		4587	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		4588
		4589	ptr = __kgem_bo_map__gtt(kgem, bo);
		4590	if (ptr == NULL)
		4591	return NULL;
		4592
		4593	/* Cache this mapping to avoid the overhead of an
		4594	* excruciatingly slow GTT pagefault. This is more an
		4595	* issue with compositing managers which need to frequently
		4596	* flush CPU damage to their GPU bo.
		4597	*/
4501	Serge	4598	bo->map__gtt = ptr;
4304	Serge	4599	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		4600	}
		4601
		4602	return ptr;
		4603	}
		4604
		4605	void kgem_bo_map__debug(struct kgem kgem, struct kgem_bo *bo)
		4606	{
4501	Serge	4607	return kgem_bo_map__async(kgem, bo);
4304	Serge	4608	}
		4609
		4610	void kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		4611	{
		4612	struct drm_i915_gem_mmap mmap_arg;
		4613
4501	Serge	4614	DBG(("%s(handle=%d, size=%d, map=%p:%p)\n",
		4615	__FUNCTION__, bo->handle, bytes(bo), bo->map__gtt, bo->map__cpu));
4304	Serge	4616	assert(!bo->purged);
		4617	assert(list_is_empty(&bo->list));
		4618	assert(bo->proxy == NULL);
		4619
4501	Serge	4620	if (bo->map__cpu)
		4621	return MAP(bo->map__cpu);
4304	Serge	4622
		4623	kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));
		4624
		4625	retry:
		4626	VG_CLEAR(mmap_arg);
		4627	mmap_arg.handle = bo->handle;
		4628	mmap_arg.offset = 0;
		4629	mmap_arg.size = bytes(bo);
		4630	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
4501	Serge	4631	int err = 0;
4304	Serge	4632
4501	Serge	4633
4304	Serge	4634	if (__kgem_throttle_retire(kgem, 0))
		4635	goto retry;
		4636
4501	Serge	4637	if (kgem_cleanup_cache(kgem))
4304	Serge	4638	goto retry;
		4639
4501	Serge	4640	ErrorF("%s: failed to mmap handle=%d, %d bytes, into CPU domain: %d\n",
		4641	__FUNCTION__, bo->handle, bytes(bo), err);
4304	Serge	4642	return NULL;
		4643	}
		4644
		4645	VG(VALGRIND_MAKE_MEM_DEFINED(mmap_arg.addr_ptr, bytes(bo)));
		4646
		4647	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
4501	Serge	4648	return bo->map__cpu = (void *)(uintptr_t)mmap_arg.addr_ptr;
4304	Serge	4649	}
		4650
4501	Serge	4651
		4652	/*
		4653	struct kgem_bo kgem_create_map(struct kgem kgem,
		4654	void *ptr, uint32_t size,
		4655	bool read_only)
4304	Serge	4656	{
4501	Serge	4657	struct kgem_bo *bo;
		4658	uintptr_t first_page, last_page;
		4659	uint32_t handle;
4304	Serge	4660
4501	Serge	4661	assert(MAP(ptr) == ptr);
4304	Serge	4662
4501	Serge	4663	if (!kgem->has_userptr)
		4664	return NULL;
4304	Serge	4665
4501	Serge	4666	first_page = (uintptr_t)ptr;
		4667	last_page = first_page + size + PAGE_SIZE - 1;
4304	Serge	4668
4501	Serge	4669	first_page &= ~(PAGE_SIZE-1);
		4670	last_page &= ~(PAGE_SIZE-1);
		4671	assert(last_page > first_page);
4304	Serge	4672
4501	Serge	4673	handle = gem_userptr(kgem->fd,
		4674	(void *)first_page, last_page-first_page,
		4675	read_only);
		4676	if (handle == 0)
		4677	return NULL;
4304	Serge	4678
4501	Serge	4679	bo = __kgem_bo_alloc(handle, (last_page - first_page) / PAGE_SIZE);
		4680	if (bo == NULL) {
		4681	gem_close(kgem->fd, handle);
		4682	return NULL;
		4683	}
4304	Serge	4684
4501	Serge	4685	bo->snoop = !kgem->has_llc;
		4686	debug_alloc__bo(kgem, bo);
		4687
		4688	if (first_page != (uintptr_t)ptr) {
		4689	struct kgem_bo *proxy;
		4690
		4691	proxy = kgem_create_proxy(kgem, bo,
		4692	(uintptr_t)ptr - first_page, size);
		4693	kgem_bo_destroy(kgem, bo);
		4694	if (proxy == NULL)
4304	Serge	4695	return NULL;
4501	Serge	4696
		4697	bo = proxy;
4304	Serge	4698	}
		4699
4501	Serge	4700	bo->map__cpu = MAKE_USER_MAP(ptr);
		4701
		4702	DBG(("%s(ptr=%p, size=%d, pages=%d, read_only=%d) => handle=%d (proxy? %d)\n",
		4703	__FUNCTION__, ptr, size, NUM_PAGES(size), read_only, handle, bo->proxy != NULL));
		4704	return bo;
4304	Serge	4705	}
4501	Serge	4706	*/
		4707
4304	Serge	4708	void kgem_bo_sync__cpu(struct kgem kgem, struct kgem_bo bo)
		4709	{
		4710	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		4711	assert(!bo->scanout);
		4712	kgem_bo_submit(kgem, bo);
		4713
		4714	/* SHM pixmaps use proxies for subpage offsets */
		4715	assert(!bo->purged);
		4716	while (bo->proxy)
		4717	bo = bo->proxy;
		4718	assert(!bo->purged);
		4719
		4720	if (bo->domain != DOMAIN_CPU \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_CPU)) {
		4721	struct drm_i915_gem_set_domain set_domain;
		4722
		4723	DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
		4724	__FUNCTION__, bo->handle,
		4725	bo->needs_flush, bo->domain,
		4726	__kgem_busy(kgem, bo->handle)));
		4727
		4728	VG_CLEAR(set_domain);
		4729	set_domain.handle = bo->handle;
		4730	set_domain.read_domains = I915_GEM_DOMAIN_CPU;
		4731	set_domain.write_domain = I915_GEM_DOMAIN_CPU;
		4732
		4733	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4734	kgem_bo_retire(kgem, bo);
		4735	bo->domain = DOMAIN_CPU;
		4736	}
		4737	}
		4738	}
		4739
4501	Serge	4740	void kgem_bo_sync__cpu_full(struct kgem kgem, struct kgem_bo bo, bool write)
		4741	{
		4742	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		4743	assert(!bo->scanout \|\| !write);
		4744
		4745	if (write \|\| bo->needs_flush)
		4746	kgem_bo_submit(kgem, bo);
		4747
		4748	/* SHM pixmaps use proxies for subpage offsets */
		4749	assert(!bo->purged);
		4750	assert(bo->refcnt);
		4751	while (bo->proxy)
		4752	bo = bo->proxy;
		4753	assert(bo->refcnt);
		4754	assert(!bo->purged);
		4755
		4756	if (bo->domain != DOMAIN_CPU \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_CPU)) {
		4757	struct drm_i915_gem_set_domain set_domain;
		4758
		4759	DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
		4760	__FUNCTION__, bo->handle,
		4761	bo->needs_flush, bo->domain,
		4762	__kgem_busy(kgem, bo->handle)));
		4763
		4764	VG_CLEAR(set_domain);
		4765	set_domain.handle = bo->handle;
		4766	set_domain.read_domains = I915_GEM_DOMAIN_CPU;
		4767	set_domain.write_domain = write ? I915_GEM_DOMAIN_CPU : 0;
		4768
		4769	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4770	if (bo->exec == NULL)
		4771	kgem_bo_retire(kgem, bo);
		4772	bo->domain = write ? DOMAIN_CPU : DOMAIN_NONE;
		4773	}
		4774	}
		4775	}
		4776
		4777	void kgem_bo_sync__gtt(struct kgem kgem, struct kgem_bo bo)
		4778	{
		4779	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		4780	assert(bo->refcnt);
		4781	assert(bo->proxy == NULL);
		4782
		4783	kgem_bo_submit(kgem, bo);
		4784
		4785	if (bo->domain != DOMAIN_GTT \|\| FORCE_MMAP_SYNC & (1 << DOMAIN_GTT)) {
		4786	struct drm_i915_gem_set_domain set_domain;
		4787
		4788	DBG(("%s: SYNC: handle=%d, needs_flush? %d, domain? %d, busy? %d\n",
		4789	__FUNCTION__, bo->handle,
		4790	bo->needs_flush, bo->domain,
		4791	__kgem_busy(kgem, bo->handle)));
		4792
		4793	VG_CLEAR(set_domain);
		4794	set_domain.handle = bo->handle;
		4795	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		4796	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		4797
		4798	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain) == 0) {
		4799	kgem_bo_retire(kgem, bo);
		4800	bo->domain = DOMAIN_GTT;
		4801	bo->gtt_dirty = true;
		4802	}
		4803	}
		4804	}
		4805
4304	Serge	4806	void kgem_clear_dirty(struct kgem *kgem)
		4807	{
		4808	struct list * const buffers = &kgem->next_request->buffers;
		4809	struct kgem_bo *bo;
		4810
		4811	list_for_each_entry(bo, buffers, request) {
		4812	if (!bo->gpu_dirty)
		4813	break;
		4814
		4815	bo->gpu_dirty = false;
		4816	}
		4817	}
		4818
		4819	struct kgem_bo kgem_create_proxy(struct kgem kgem,
		4820	struct kgem_bo *target,
		4821	int offset, int length)
		4822	{
		4823	struct kgem_bo *bo;
		4824
		4825	DBG(("%s: target handle=%d [proxy? %d], offset=%d, length=%d, io=%d\n",
		4826	__FUNCTION__, target->handle, target->proxy ? target->proxy->delta : -1,
		4827	offset, length, target->io));
		4828
		4829	bo = __kgem_bo_alloc(target->handle, length);
		4830	if (bo == NULL)
		4831	return NULL;
		4832
		4833	bo->unique_id = kgem_get_unique_id(kgem);
		4834	bo->reusable = false;
		4835	bo->size.bytes = length;
		4836
		4837	bo->io = target->io && target->proxy == NULL;
		4838	bo->gpu_dirty = target->gpu_dirty;
		4839	bo->tiling = target->tiling;
		4840	bo->pitch = target->pitch;
		4841	bo->flush = target->flush;
		4842	bo->snoop = target->snoop;
		4843
		4844	assert(!bo->scanout);
		4845	bo->proxy = kgem_bo_reference(target);
		4846	bo->delta = offset;
		4847
4501	Serge	4848	if (target->exec && !bo->io) {
4304	Serge	4849	list_move_tail(&bo->request, &kgem->next_request->buffers);
		4850	bo->exec = &_kgem_dummy_exec;
		4851	}
		4852	bo->rq = target->rq;
		4853
		4854	return bo;
		4855	}
		4856
		4857	#if 0
		4858	static struct kgem_buffer *
		4859	buffer_alloc(void)
		4860	{
		4861	struct kgem_buffer *bo;
		4862
		4863	bo = malloc(sizeof(*bo));
		4864	if (bo == NULL)
		4865	return NULL;
		4866
		4867	bo->mem = NULL;
		4868	bo->need_io = false;
4501	Serge	4869	bo->mmapped = MMAPPED_CPU;
4304	Serge	4870
		4871	return bo;
		4872	}
		4873
		4874	static struct kgem_buffer *
		4875	buffer_alloc_with_data(int num_pages)
		4876	{
		4877	struct kgem_buffer *bo;
		4878
		4879	bo = malloc(sizeof(bo) + 2UPLOAD_ALIGNMENT + num_pages * PAGE_SIZE);
		4880	if (bo == NULL)
		4881	return NULL;
		4882
		4883	bo->mem = (void )ALIGN((uintptr_t)bo + sizeof(bo), UPLOAD_ALIGNMENT);
		4884	bo->mmapped = false;
		4885	return bo;
		4886	}
		4887
		4888	static inline bool
		4889	use_snoopable_buffer(struct kgem *kgem, uint32_t flags)
		4890	{
		4891	if ((flags & KGEM_BUFFER_WRITE) == 0)
		4892	return kgem->gen >= 030;
		4893
		4894	return true;
		4895	}
		4896
		4897	static void
		4898	init_buffer_from_bo(struct kgem_buffer bo, struct kgem_bo old)
		4899	{
		4900	DBG(("%s: reusing handle=%d for buffer\n",
		4901	__FUNCTION__, old->handle));
		4902
		4903	assert(old->proxy == NULL);
		4904
		4905	memcpy(&bo->base, old, sizeof(*old));
		4906	if (old->rq)
		4907	list_replace(&old->request, &bo->base.request);
		4908	else
		4909	list_init(&bo->base.request);
		4910	list_replace(&old->vma, &bo->base.vma);
		4911	list_init(&bo->base.list);
		4912	free(old);
		4913
		4914	assert(bo->base.tiling == I915_TILING_NONE);
		4915
		4916	bo->base.refcnt = 1;
		4917	}
		4918
		4919	static struct kgem_buffer *
		4920	search_snoopable_buffer(struct kgem *kgem, unsigned alloc)
		4921	{
		4922	struct kgem_buffer *bo;
		4923	struct kgem_bo *old;
		4924
		4925	old = search_snoop_cache(kgem, alloc, 0);
		4926	if (old) {
		4927	if (!old->io) {
		4928	bo = buffer_alloc();
		4929	if (bo == NULL)
		4930	return NULL;
		4931
		4932	init_buffer_from_bo(bo, old);
		4933	} else {
		4934	bo = (struct kgem_buffer *)old;
		4935	bo->base.refcnt = 1;
		4936	}
		4937
		4938	DBG(("%s: created CPU handle=%d for buffer, size %d\n",
		4939	__FUNCTION__, bo->base.handle, num_pages(&bo->base)));
		4940
		4941	assert(bo->base.snoop);
		4942	assert(bo->base.tiling == I915_TILING_NONE);
		4943	assert(num_pages(&bo->base) >= alloc);
4501	Serge	4944	assert(bo->mmapped == MMAPPED_CPU);
4304	Serge	4945	assert(bo->need_io == false);
		4946
		4947	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4948	if (bo->mem == NULL) {
		4949	bo->base.refcnt = 0;
		4950	kgem_bo_free(kgem, &bo->base);
		4951	bo = NULL;
		4952	}
		4953
		4954	return bo;
		4955	}
		4956
		4957	return NULL;
		4958	}
		4959
		4960	static struct kgem_buffer *
		4961	create_snoopable_buffer(struct kgem *kgem, unsigned alloc)
		4962	{
		4963	struct kgem_buffer *bo;
		4964	uint32_t handle;
		4965
		4966	if (kgem->has_llc) {
		4967	struct kgem_bo *old;
		4968
		4969	bo = buffer_alloc();
		4970	if (bo == NULL)
		4971	return NULL;
		4972
		4973	old = search_linear_cache(kgem, alloc,
		4974	CREATE_INACTIVE \| CREATE_CPU_MAP \| CREATE_EXACT);
		4975	if (old) {
		4976	init_buffer_from_bo(bo, old);
		4977	} else {
		4978	handle = gem_create(kgem->fd, alloc);
		4979	if (handle == 0) {
		4980	free(bo);
		4981	return NULL;
		4982	}
		4983
		4984	debug_alloc(kgem, alloc);
		4985	__kgem_bo_init(&bo->base, handle, alloc);
		4986	DBG(("%s: created CPU (LLC) handle=%d for buffer, size %d\n",
		4987	__FUNCTION__, bo->base.handle, alloc));
		4988	}
		4989
		4990	assert(bo->base.refcnt == 1);
4501	Serge	4991	assert(bo->mmapped == MMAPPED_CPU);
4304	Serge	4992	assert(bo->need_io == false);
		4993
		4994	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		4995	if (bo->mem != NULL)
		4996	return bo;
		4997
		4998	bo->base.refcnt = 0; /* for valgrind */
		4999	kgem_bo_free(kgem, &bo->base);
		5000	}
		5001
		5002	if (kgem->has_caching) {
		5003	struct kgem_bo *old;
		5004
		5005	bo = buffer_alloc();
		5006	if (bo == NULL)
		5007	return NULL;
		5008
		5009	old = search_linear_cache(kgem, alloc,
		5010	CREATE_INACTIVE \| CREATE_CPU_MAP \| CREATE_EXACT);
		5011	if (old) {
		5012	init_buffer_from_bo(bo, old);
		5013	} else {
		5014	handle = gem_create(kgem->fd, alloc);
		5015	if (handle == 0) {
		5016	free(bo);
		5017	return NULL;
		5018	}
		5019
		5020	debug_alloc(kgem, alloc);
		5021	__kgem_bo_init(&bo->base, handle, alloc);
		5022	DBG(("%s: created CPU handle=%d for buffer, size %d\n",
		5023	__FUNCTION__, bo->base.handle, alloc));
		5024	}
		5025
		5026	assert(bo->base.refcnt == 1);
4501	Serge	5027	assert(bo->mmapped == MMAPPED_CPU);
4304	Serge	5028	assert(bo->need_io == false);
		5029
		5030	if (!gem_set_caching(kgem->fd, bo->base.handle, SNOOPED))
		5031	goto free_caching;
		5032
		5033	bo->base.snoop = true;
		5034
		5035	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		5036	if (bo->mem == NULL)
		5037	goto free_caching;
		5038
		5039	return bo;
		5040
		5041	free_caching:
		5042	bo->base.refcnt = 0; /* for valgrind */
		5043	kgem_bo_free(kgem, &bo->base);
		5044	}
		5045
		5046	if (kgem->has_userptr) {
		5047	bo = buffer_alloc();
		5048	if (bo == NULL)
		5049	return NULL;
		5050
		5051	//if (posix_memalign(&ptr, 64, ALIGN(size, 64)))
		5052	if (posix_memalign(&bo->mem, PAGE_SIZE, alloc * PAGE_SIZE)) {
		5053	free(bo);
		5054	return NULL;
		5055	}
		5056
		5057	handle = gem_userptr(kgem->fd, bo->mem, alloc * PAGE_SIZE, false);
		5058	if (handle == 0) {
		5059	free(bo->mem);
		5060	free(bo);
		5061	return NULL;
		5062	}
		5063
		5064	debug_alloc(kgem, alloc);
		5065	__kgem_bo_init(&bo->base, handle, alloc);
		5066	DBG(("%s: created snoop handle=%d for buffer\n",
		5067	__FUNCTION__, bo->base.handle));
		5068
4501	Serge	5069	assert(bo->mmapped == MMAPPED_CPU);
4304	Serge	5070	assert(bo->need_io == false);
		5071
		5072	bo->base.refcnt = 1;
		5073	bo->base.snoop = true;
4501	Serge	5074	bo->base.map__cpu = MAKE_USER_MAP(bo->mem);
4304	Serge	5075
		5076	return bo;
		5077	}
		5078
		5079	return NULL;
		5080	}
		5081
		5082	struct kgem_bo kgem_create_buffer(struct kgem kgem,
		5083	uint32_t size, uint32_t flags,
		5084	void **ret)
		5085	{
		5086	struct kgem_buffer *bo;
		5087	unsigned offset, alloc;
		5088	struct kgem_bo *old;
		5089
		5090	DBG(("%s: size=%d, flags=%x [write?=%d, inplace?=%d, last?=%d]\n",
		5091	__FUNCTION__, size, flags,
		5092	!!(flags & KGEM_BUFFER_WRITE),
		5093	!!(flags & KGEM_BUFFER_INPLACE),
		5094	!!(flags & KGEM_BUFFER_LAST)));
		5095	assert(size);
		5096	/* we should never be asked to create anything TOO large */
		5097	assert(size <= kgem->max_object_size);
		5098
		5099	#if !DBG_NO_UPLOAD_CACHE
		5100	list_for_each_entry(bo, &kgem->batch_buffers, base.list) {
		5101	assert(bo->base.io);
		5102	assert(bo->base.refcnt >= 1);
		5103
		5104	/* We can reuse any write buffer which we can fit */
		5105	if (flags == KGEM_BUFFER_LAST &&
		5106	bo->write == KGEM_BUFFER_WRITE &&
4501	Serge	5107	bo->base.refcnt == 1 &&
		5108	bo->mmapped == MMAPPED_NONE &&
4304	Serge	5109	size <= bytes(&bo->base)) {
		5110	DBG(("%s: reusing write buffer for read of %d bytes? used=%d, total=%d\n",
		5111	__FUNCTION__, size, bo->used, bytes(&bo->base)));
4501	Serge	5112	gem_write__cachealigned(kgem->fd, bo->base.handle,
4304	Serge	5113	0, bo->used, bo->mem);
		5114	kgem_buffer_release(kgem, bo);
		5115	bo->need_io = 0;
		5116	bo->write = 0;
		5117	offset = 0;
		5118	bo->used = size;
		5119	goto done;
		5120	}
		5121
		5122	if (flags & KGEM_BUFFER_WRITE) {
		5123	if ((bo->write & KGEM_BUFFER_WRITE) == 0 \|\|
		5124	(((bo->write & ~flags) & KGEM_BUFFER_INPLACE) &&
		5125	!bo->base.snoop)) {
		5126	DBG(("%s: skip write %x buffer, need %x\n",
		5127	__FUNCTION__, bo->write, flags));
		5128	continue;
		5129	}
		5130	assert(bo->mmapped \|\| bo->need_io);
		5131	} else {
		5132	if (bo->write & KGEM_BUFFER_WRITE) {
		5133	DBG(("%s: skip write %x buffer, need %x\n",
		5134	__FUNCTION__, bo->write, flags));
		5135	continue;
		5136	}
		5137	}
		5138
		5139	if (bo->used + size <= bytes(&bo->base)) {
		5140	DBG(("%s: reusing buffer? used=%d + size=%d, total=%d\n",
		5141	__FUNCTION__, bo->used, size, bytes(&bo->base)));
		5142	offset = bo->used;
		5143	bo->used += size;
		5144	goto done;
		5145	}
		5146	}
		5147
		5148	if (flags & KGEM_BUFFER_WRITE) {
		5149	list_for_each_entry(bo, &kgem->active_buffers, base.list) {
		5150	assert(bo->base.io);
		5151	assert(bo->base.refcnt >= 1);
4501	Serge	5152	assert(bo->base.exec == NULL);
4304	Serge	5153	assert(bo->mmapped);
4501	Serge	5154	assert(bo->mmapped == MMAPPED_GTT \|\| kgem->has_llc \|\| bo->base.snoop);
4304	Serge	5155
4501	Serge	5156	if ((bo->write & ~flags) & KGEM_BUFFER_INPLACE && !bo->base.snoop) {
4304	Serge	5157	DBG(("%s: skip write %x buffer, need %x\n",
		5158	__FUNCTION__, bo->write, flags));
		5159	continue;
		5160	}
		5161
		5162	if (bo->used + size <= bytes(&bo->base)) {
		5163	DBG(("%s: reusing buffer? used=%d + size=%d, total=%d\n",
		5164	__FUNCTION__, bo->used, size, bytes(&bo->base)));
		5165	offset = bo->used;
		5166	bo->used += size;
		5167	list_move(&bo->base.list, &kgem->batch_buffers);
		5168	goto done;
		5169	}
4501	Serge	5170
		5171	if (size <= bytes(&bo->base) &&
		5172	(bo->base.rq == NULL \|\|
		5173	!__kgem_busy(kgem, bo->base.handle))) {
		5174	DBG(("%s: reusing whole buffer? size=%d, total=%d\n",
		5175	__FUNCTION__, size, bytes(&bo->base)));
		5176	__kgem_bo_clear_busy(&bo->base);
		5177	kgem_buffer_release(kgem, bo);
		5178
		5179	switch (bo->mmapped) {
		5180	case MMAPPED_CPU:
		5181	kgem_bo_sync__cpu(kgem, &bo->base);
		5182	break;
		5183	case MMAPPED_GTT:
		5184	kgem_bo_sync__gtt(kgem, &bo->base);
		5185	break;
		5186	}
		5187
		5188	offset = 0;
		5189	bo->used = size;
		5190	list_move(&bo->base.list, &kgem->batch_buffers);
		5191	goto done;
		5192	}
4304	Serge	5193	}
		5194	}
		5195	#endif
		5196
		5197	#if !DBG_NO_MAP_UPLOAD
		5198	/* Be a little more generous and hope to hold fewer mmappings */
		5199	alloc = ALIGN(2*size, kgem->buffer_size);
		5200	if (alloc > MAX_CACHE_SIZE)
		5201	alloc = ALIGN(size, kgem->buffer_size);
		5202	if (alloc > MAX_CACHE_SIZE)
		5203	alloc = PAGE_ALIGN(size);
		5204	assert(alloc);
		5205
4501	Serge	5206	alloc /= PAGE_SIZE;
4304	Serge	5207	if (alloc > kgem->aperture_mappable / 4)
		5208	flags &= ~KGEM_BUFFER_INPLACE;
		5209
		5210	if (kgem->has_llc &&
		5211	(flags & KGEM_BUFFER_WRITE_INPLACE) != KGEM_BUFFER_WRITE_INPLACE) {
		5212	bo = buffer_alloc();
		5213	if (bo == NULL)
		5214	goto skip_llc;
		5215
		5216	old = NULL;
		5217	if ((flags & KGEM_BUFFER_WRITE) == 0)
		5218	old = search_linear_cache(kgem, alloc, CREATE_CPU_MAP);
		5219	if (old == NULL)
		5220	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE \| CREATE_CPU_MAP);
		5221	if (old == NULL)
		5222	old = search_linear_cache(kgem, NUM_PAGES(size), CREATE_INACTIVE \| CREATE_CPU_MAP);
		5223	if (old) {
		5224	DBG(("%s: found LLC handle=%d for buffer\n",
		5225	__FUNCTION__, old->handle));
		5226
		5227	init_buffer_from_bo(bo, old);
		5228	} else {
		5229	uint32_t handle = gem_create(kgem->fd, alloc);
		5230	if (handle == 0) {
		5231	free(bo);
		5232	goto skip_llc;
		5233	}
		5234	__kgem_bo_init(&bo->base, handle, alloc);
		5235	DBG(("%s: created LLC handle=%d for buffer\n",
		5236	__FUNCTION__, bo->base.handle));
		5237
		5238	debug_alloc(kgem, alloc);
		5239	}
		5240
		5241	assert(bo->mmapped);
		5242	assert(!bo->need_io);
		5243
		5244	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		5245	if (bo->mem) {
		5246	if (flags & KGEM_BUFFER_WRITE)
		5247	kgem_bo_sync__cpu(kgem, &bo->base);
		5248	flags &= ~KGEM_BUFFER_INPLACE;
		5249	goto init;
		5250	} else {
		5251	bo->base.refcnt = 0; /* for valgrind */
		5252	kgem_bo_free(kgem, &bo->base);
		5253	}
		5254	}
		5255	skip_llc:
		5256
		5257	if ((flags & KGEM_BUFFER_WRITE_INPLACE) == KGEM_BUFFER_WRITE_INPLACE) {
		5258	/* The issue with using a GTT upload buffer is that we may
		5259	* cause eviction-stalls in order to free up some GTT space.
		5260	* An is-mappable? ioctl could help us detect when we are
		5261	* about to block, or some per-page magic in the kernel.
		5262	*
		5263	* XXX This is especially noticeable on memory constrained
		5264	* devices like gen2 or with relatively slow gpu like i3.
		5265	*/
		5266	DBG(("%s: searching for an inactive GTT map for upload\n",
		5267	__FUNCTION__));
		5268	old = search_linear_cache(kgem, alloc,
		5269	CREATE_EXACT \| CREATE_INACTIVE \| CREATE_GTT_MAP);
		5270	#if HAVE_I915_GEM_BUFFER_INFO
		5271	if (old) {
		5272	struct drm_i915_gem_buffer_info info;
		5273
		5274	/* An example of such a non-blocking ioctl might work */
		5275
		5276	VG_CLEAR(info);
		5277	info.handle = handle;
		5278	if (drmIoctl(kgem->fd,
		5279	DRM_IOCTL_I915_GEM_BUFFER_INFO,
		5280	&fino) == 0) {
		5281	old->presumed_offset = info.addr;
		5282	if ((info.flags & I915_GEM_MAPPABLE) == 0) {
		5283	kgem_bo_move_to_inactive(kgem, old);
		5284	old = NULL;
		5285	}
		5286	}
		5287	}
		5288	#endif
		5289	if (old == NULL)
		5290	old = search_linear_cache(kgem, NUM_PAGES(size),
		5291	CREATE_EXACT \| CREATE_INACTIVE \| CREATE_GTT_MAP);
		5292	if (old == NULL) {
		5293	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
4501	Serge	5294	if (old && !kgem_bo_can_map(kgem, old)) {
4304	Serge	5295	_kgem_bo_destroy(kgem, old);
		5296	old = NULL;
		5297	}
		5298	}
		5299	if (old) {
		5300	DBG(("%s: reusing handle=%d for buffer\n",
		5301	__FUNCTION__, old->handle));
4501	Serge	5302	assert(kgem_bo_can_map(kgem, old));
4304	Serge	5303	assert(!old->snoop);
		5304	assert(old->rq == NULL);
		5305
		5306	bo = buffer_alloc();
		5307	if (bo == NULL)
		5308	return NULL;
		5309
		5310	init_buffer_from_bo(bo, old);
		5311	assert(num_pages(&bo->base) >= NUM_PAGES(size));
		5312
		5313	assert(bo->mmapped);
		5314	assert(bo->base.refcnt == 1);
		5315
		5316	bo->mem = kgem_bo_map(kgem, &bo->base);
		5317	if (bo->mem) {
4501	Serge	5318	if (bo->mem == MAP(bo->base.map__cpu))
4304	Serge	5319	flags &= ~KGEM_BUFFER_INPLACE;
4501	Serge	5320	else
		5321	bo->mmapped = MMAPPED_GTT;
4304	Serge	5322	goto init;
		5323	} else {
		5324	bo->base.refcnt = 0;
		5325	kgem_bo_free(kgem, &bo->base);
		5326	}
		5327	}
		5328	}
		5329	#else
		5330	flags &= ~KGEM_BUFFER_INPLACE;
		5331	#endif
		5332	/* Be more parsimonious with pwrite/pread/cacheable buffers */
		5333	if ((flags & KGEM_BUFFER_INPLACE) == 0)
		5334	alloc = NUM_PAGES(size);
		5335
		5336	if (use_snoopable_buffer(kgem, flags)) {
		5337	bo = search_snoopable_buffer(kgem, alloc);
		5338	if (bo) {
		5339	if (flags & KGEM_BUFFER_WRITE)
		5340	kgem_bo_sync__cpu(kgem, &bo->base);
		5341	flags &= ~KGEM_BUFFER_INPLACE;
		5342	goto init;
		5343	}
		5344
		5345	if ((flags & KGEM_BUFFER_INPLACE) == 0) {
		5346	bo = create_snoopable_buffer(kgem, alloc);
		5347	if (bo)
		5348	goto init;
		5349	}
		5350	}
		5351
		5352	flags &= ~KGEM_BUFFER_INPLACE;
		5353
		5354	old = NULL;
		5355	if ((flags & KGEM_BUFFER_WRITE) == 0)
		5356	old = search_linear_cache(kgem, alloc, 0);
		5357	if (old == NULL)
		5358	old = search_linear_cache(kgem, alloc, CREATE_INACTIVE);
		5359	if (old) {
		5360	DBG(("%s: reusing ordinary handle %d for io\n",
		5361	__FUNCTION__, old->handle));
		5362	bo = buffer_alloc_with_data(num_pages(old));
		5363	if (bo == NULL)
		5364	return NULL;
		5365
		5366	init_buffer_from_bo(bo, old);
		5367	bo->need_io = flags & KGEM_BUFFER_WRITE;
		5368	} else {
		5369	unsigned hint;
		5370
		5371	if (use_snoopable_buffer(kgem, flags)) {
		5372	bo = create_snoopable_buffer(kgem, alloc);
		5373	if (bo)
		5374	goto init;
		5375	}
		5376
		5377	bo = buffer_alloc();
		5378	if (bo == NULL)
		5379	return NULL;
		5380
		5381	hint = CREATE_INACTIVE;
		5382	if (flags & KGEM_BUFFER_WRITE)
		5383	hint \|= CREATE_CPU_MAP;
		5384	old = search_linear_cache(kgem, alloc, hint);
		5385	if (old) {
		5386	DBG(("%s: reusing handle=%d for buffer\n",
		5387	__FUNCTION__, old->handle));
		5388
		5389	init_buffer_from_bo(bo, old);
		5390	} else {
		5391	uint32_t handle = gem_create(kgem->fd, alloc);
		5392	if (handle == 0) {
		5393	free(bo);
		5394	return NULL;
		5395	}
		5396
		5397	DBG(("%s: created handle=%d for buffer\n",
		5398	__FUNCTION__, handle));
		5399
		5400	__kgem_bo_init(&bo->base, handle, alloc);
		5401	debug_alloc(kgem, alloc * PAGE_SIZE);
		5402	}
		5403
		5404	assert(bo->mmapped);
		5405	assert(!bo->need_io);
		5406	assert(bo->base.refcnt == 1);
		5407
		5408	if (flags & KGEM_BUFFER_WRITE) {
		5409	bo->mem = kgem_bo_map__cpu(kgem, &bo->base);
		5410	if (bo->mem != NULL) {
		5411	kgem_bo_sync__cpu(kgem, &bo->base);
		5412	goto init;
		5413	}
		5414	}
		5415
		5416	DBG(("%s: failing back to new pwrite buffer\n", __FUNCTION__));
		5417	old = &bo->base;
		5418	bo = buffer_alloc_with_data(num_pages(old));
		5419	if (bo == NULL) {
		5420	old->refcnt= 0;
		5421	kgem_bo_free(kgem, old);
		5422	return NULL;
		5423	}
		5424
		5425	init_buffer_from_bo(bo, old);
		5426
		5427	assert(bo->mem);
		5428	assert(!bo->mmapped);
		5429	assert(bo->base.refcnt == 1);
		5430
		5431	bo->need_io = flags & KGEM_BUFFER_WRITE;
		5432	}
		5433	init:
		5434	bo->base.io = true;
		5435	assert(bo->base.refcnt == 1);
		5436	assert(num_pages(&bo->base) >= NUM_PAGES(size));
		5437	assert(!bo->need_io \|\| !bo->base.needs_flush);
		5438	assert(!bo->need_io \|\| bo->base.domain != DOMAIN_GPU);
		5439	assert(bo->mem);
4501	Serge	5440	assert(bo->mmapped != MMAPPED_GTT \|\| MAP(bo->base.map__gtt) == bo->mem);
		5441	assert(bo->mmapped != MMAPPED_CPU \|\| MAP(bo->base.map__cpu) == bo->mem);
4304	Serge	5442
		5443	bo->used = size;
		5444	bo->write = flags & KGEM_BUFFER_WRITE_INPLACE;
		5445	offset = 0;
		5446
		5447	assert(list_is_empty(&bo->base.list));
		5448	list_add(&bo->base.list, &kgem->batch_buffers);
		5449
		5450	DBG(("%s(pages=%d [%d]) new handle=%d, used=%d, write=%d\n",
		5451	__FUNCTION__, num_pages(&bo->base), alloc, bo->base.handle, bo->used, bo->write));
		5452
		5453	done:
		5454	bo->used = ALIGN(bo->used, UPLOAD_ALIGNMENT);
4501	Serge	5455	assert(bo->used && bo->used <= bytes(&bo->base));
4304	Serge	5456	assert(bo->mem);
		5457	ret = (char )bo->mem + offset;
		5458	return kgem_create_proxy(kgem, &bo->base, offset, size);
		5459	}
		5460
		5461	bool kgem_buffer_is_inplace(struct kgem_bo *_bo)
		5462	{
		5463	struct kgem_buffer bo = (struct kgem_buffer )_bo->proxy;
		5464	return bo->write & KGEM_BUFFER_WRITE_INPLACE;
		5465	}
		5466
		5467	struct kgem_bo kgem_create_buffer_2d(struct kgem kgem,
		5468	int width, int height, int bpp,
		5469	uint32_t flags,
		5470	void **ret)
		5471	{
		5472	struct kgem_bo *bo;
		5473	int stride;
		5474
		5475	assert(width > 0 && height > 0);
		5476	assert(ret != NULL);
		5477	stride = ALIGN(width, 2) * bpp >> 3;
		5478	stride = ALIGN(stride, 4);
		5479
		5480	DBG(("%s: %dx%d, %d bpp, stride=%d\n",
		5481	__FUNCTION__, width, height, bpp, stride));
		5482
		5483	bo = kgem_create_buffer(kgem, stride * ALIGN(height, 2), flags, ret);
		5484	if (bo == NULL) {
		5485	DBG(("%s: allocation failure for upload buffer\n",
		5486	__FUNCTION__));
		5487	return NULL;
		5488	}
		5489	assert(*ret != NULL);
		5490	assert(bo->proxy != NULL);
		5491
		5492	if (height & 1) {
		5493	struct kgem_buffer io = (struct kgem_buffer )bo->proxy;
		5494	int min;
		5495
		5496	assert(io->used);
		5497
		5498	/* Having padded this surface to ensure that accesses to
		5499	* the last pair of rows is valid, remove the padding so
		5500	* that it can be allocated to other pixmaps.
		5501	*/
		5502	min = bo->delta + height * stride;
		5503	min = ALIGN(min, UPLOAD_ALIGNMENT);
		5504	if (io->used != min) {
		5505	DBG(("%s: trimming buffer from %d to %d\n",
		5506	__FUNCTION__, io->used, min));
		5507	io->used = min;
		5508	}
		5509	bo->size.bytes -= stride;
		5510	}
		5511
4501	Serge	5512	bo->map__cpu = *ret;
4304	Serge	5513	bo->pitch = stride;
		5514	bo->unique_id = kgem_get_unique_id(kgem);
		5515	return bo;
		5516	}
		5517
		5518	struct kgem_bo kgem_upload_source_image(struct kgem kgem,
		5519	const void *data,
		5520	const BoxRec *box,
		5521	int stride, int bpp)
		5522	{
		5523	int width = box->x2 - box->x1;
		5524	int height = box->y2 - box->y1;
		5525	struct kgem_bo *bo;
		5526	void *dst;
		5527
		5528	if (!kgem_can_create_2d(kgem, width, height, bpp))
		5529	return NULL;
		5530
		5531	DBG(("%s : (%d, %d), (%d, %d), stride=%d, bpp=%d\n",
		5532	__FUNCTION__, box->x1, box->y1, box->x2, box->y2, stride, bpp));
		5533
		5534	assert(data);
		5535	assert(width > 0);
		5536	assert(height > 0);
		5537	assert(stride);
		5538	assert(bpp);
		5539
		5540	bo = kgem_create_buffer_2d(kgem,
		5541	width, height, bpp,
		5542	KGEM_BUFFER_WRITE_INPLACE, &dst);
		5543	if (bo)
		5544	memcpy_blt(data, dst, bpp,
		5545	stride, bo->pitch,
		5546	box->x1, box->y1,
		5547	0, 0,
		5548	width, height);
		5549
		5550	return bo;
		5551	}
		5552
		5553	void kgem_proxy_bo_attach(struct kgem_bo *bo,
		5554	struct kgem_bo **ptr)
		5555	{
		5556	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
4501	Serge	5557	assert(bo->map__gtt == NULL);
4304	Serge	5558	assert(bo->proxy);
		5559	list_add(&bo->vma, &bo->proxy->vma);
4501	Serge	5560	bo->map__gtt = ptr;
4304	Serge	5561	*ptr = kgem_bo_reference(bo);
		5562	}
		5563
		5564	void kgem_buffer_read_sync(struct kgem kgem, struct kgem_bo _bo)
		5565	{
		5566	struct kgem_buffer *bo;
		5567	uint32_t offset = _bo->delta, length = _bo->size.bytes;
		5568
		5569	/* We expect the caller to have already submitted the batch */
		5570	assert(_bo->io);
		5571	assert(_bo->exec == NULL);
		5572	assert(_bo->rq == NULL);
		5573	assert(_bo->proxy);
		5574
		5575	_bo = _bo->proxy;
		5576	assert(_bo->proxy == NULL);
		5577	assert(_bo->exec == NULL);
		5578
		5579	bo = (struct kgem_buffer *)_bo;
		5580
		5581	DBG(("%s(offset=%d, length=%d, snooped=%d)\n", __FUNCTION__,
		5582	offset, length, bo->base.snoop));
		5583
		5584	if (bo->mmapped) {
		5585	struct drm_i915_gem_set_domain set_domain;
		5586
		5587	DBG(("%s: sync: needs_flush? %d, domain? %d, busy? %d\n",
		5588	__FUNCTION__,
		5589	bo->base.needs_flush,
		5590	bo->base.domain,
		5591	__kgem_busy(kgem, bo->base.handle)));
		5592
4501	Serge	5593	assert(bo->mmapped == MMAPPED_GTT \|\| bo->base.snoop \|\| kgem->has_llc);
4304	Serge	5594
		5595	VG_CLEAR(set_domain);
		5596	set_domain.handle = bo->base.handle;
		5597	set_domain.write_domain = 0;
		5598	set_domain.read_domains =
4501	Serge	5599	bo->mmapped == MMAPPED_CPU ? I915_GEM_DOMAIN_CPU : I915_GEM_DOMAIN_GTT;
4304	Serge	5600
		5601	if (drmIoctl(kgem->fd,
		5602	DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain))
		5603	return;
		5604	} else {
		5605	if (gem_read(kgem->fd,
		5606	bo->base.handle, (char *)bo->mem+offset,
		5607	offset, length))
		5608	return;
		5609	}
		5610	kgem_bo_retire(kgem, &bo->base);
		5611	bo->base.domain = DOMAIN_NONE;
		5612	}
		5613	#endif
		5614
		5615	uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format)
		5616	{
		5617	struct kgem_bo_binding *b;
		5618
		5619	for (b = &bo->binding; b && b->offset; b = b->next)
		5620	if (format == b->format)
		5621	return b->offset;
		5622
		5623	return 0;
		5624	}
		5625
		5626	void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset)
		5627	{
		5628	struct kgem_bo_binding *b;
		5629
		5630	for (b = &bo->binding; b; b = b->next) {
		5631	if (b->offset)
		5632	continue;
		5633
		5634	b->offset = offset;
		5635	b->format = format;
		5636
		5637	if (b->next)
		5638	b->next->offset = 0;
		5639
		5640	return;
		5641	}
		5642
		5643	b = malloc(sizeof(*b));
		5644	if (b) {
		5645	b->next = bo->binding.next;
		5646	b->format = format;
		5647	b->offset = offset;
		5648	bo->binding.next = b;
		5649	}
		5650	}
		5651
		5652	int kgem_init_fb(struct kgem kgem, struct sna_fb fb)
		5653	{
		5654	struct kgem_bo *bo;
4397	Serge	5655	struct drm_gem_open open_arg;
		5656	struct drm_i915_gem_get_tiling get_tiling;
		5657
4304	Serge	5658	size_t size;
		5659	int ret;
		5660
		5661	ret = drmIoctl(kgem->fd, SRV_FBINFO, fb);
		5662	if( ret != 0 )
		5663	return 0;
		5664
4397	Serge	5665	open_arg.name = fb->name;
		5666	ret = drmIoctl(kgem->fd, DRM_IOCTL_GEM_OPEN, &open_arg);
		5667	if (ret != 0) {
		5668	printf("Couldn't reference %s handle 0x%08x\n",
		5669	fb->name, fb->name);
		5670	return NULL;
		5671	}
		5672	size = open_arg.size / PAGE_SIZE;
4304	Serge	5673
4397	Serge	5674	bo = __kgem_bo_alloc(open_arg.handle, size);
4304	Serge	5675	if (!bo) {
		5676	return 0;
		5677	}
		5678
4397	Serge	5679	get_tiling.handle = bo->handle;
		5680	ret = drmIoctl(kgem->fd,DRM_IOCTL_I915_GEM_GET_TILING,&get_tiling);
		5681	if (ret != 0) {
		5682	printf("%s: couldn't get tiling for handle %d\n", __FUNCTION__, bo->handle);
		5683	// drm_intel_gem_bo_unreference(&bo_gem->bo);
		5684	return 0;
		5685	}
		5686
4304	Serge	5687	bo->domain = DOMAIN_GTT;
		5688	bo->unique_id = kgem_get_unique_id(kgem);
		5689	bo->pitch = fb->pitch;
4397	Serge	5690	bo->tiling = get_tiling.tiling_mode;
4304	Serge	5691	bo->scanout = 1;
		5692	fb->fb_bo = bo;
		5693
4397	Serge	5694	printf("fb handle %d w: %d h: %d pitch %d tilng %d bo %p\n",
		5695	bo->handle, fb->width, fb->height, fb->pitch, fb->tiling, fb->fb_bo);
4304	Serge	5696
		5697	return 1;
		5698	};
		5699
		5700
		5701	int kgem_update_fb(struct kgem kgem, struct sna_fb fb)
		5702	{
		5703	struct kgem_bo *bo;
		5704	size_t size;
		5705	int ret;
		5706
		5707	bo = fb->fb_bo;
		5708
		5709	ret = drmIoctl(kgem->fd, SRV_FBINFO, fb);
		5710	if( ret != 0 )
		5711	return 0;
		5712
		5713	fb->fb_bo = bo;
		5714
		5715	size = fb->pitch * fb->height / PAGE_SIZE;
		5716
		5717	if((size != bo->size.pages.count) \|\|
		5718	(fb->pitch != bo->pitch))
		5719	{
		5720	bo->size.pages.count = size;
		5721	bo->pitch = fb->pitch;
		5722
		5723	printf("fb width %d height %d pitch %d bo %p\n",
		5724	fb->width, fb->height, fb->pitch, fb->fb_bo);
		5725
		5726	return 1;
		5727	}
		5728
		5729	return 0;
		5730	};
		5731
		5732	void sna_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		5733	{
		5734	kgem_bo_destroy(kgem, bo);
		5735	kgem_bo_free(kgem, bo);
		5736	}
		5737
		5738
		5739	void kgem_close_batches(struct kgem *kgem)
		5740	{
		5741	int n;
		5742	for (n = 0; n < ARRAY_SIZE(kgem->pinned_batches); n++) {
4368	Serge	5743	while (!list_is_empty(&kgem->pinned_batches[n]))
		5744	{
		5745	struct kgem_bo *bo =
4304	Serge	5746	list_first_entry(&kgem->pinned_batches[n],
4368	Serge	5747	struct kgem_bo, list);
		5748	list_del(&bo->list);
		5749	kgem_bo_destroy(kgem,bo);
4304	Serge	5750	}
		5751	}
		5752	};
		5753
		5754	struct kgem_bo kgem_bo_from_handle(struct kgem kgem, int handle,
		5755	int pitch, int height)
		5756	{
		5757	struct kgem_bo *bo;
		5758	int size;
		5759
		5760	size = pitch * height / PAGE_SIZE;
		5761
		5762	bo = __kgem_bo_alloc(handle, size);
		5763	if(bo == NULL)
		5764	return NULL;
		5765
		5766	bo->domain = DOMAIN_GTT;
		5767	bo->unique_id = kgem_get_unique_id(kgem);
		5768	bo->pitch = pitch;
		5769	bo->tiling = I915_TILING_X;
		5770	bo->scanout = 0;
		5771
		5772	return bo;
		5773	}

Subversion Repositories Kolibri OS

(root)/contrib/sdk/sources/Intel-2D/sna/kgem.c – Rev 4501