WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/sna/kgem.c

Rev	Author	Line No.	Line
2351	Serge	1	/*
		2	* Copyright (c) 2011 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
		20	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		21	* SOFTWARE.
		22	*
		23	* Authors:
		24	* Chris Wilson
		25	*
		26	*/
		27
		28
		29	#ifdef HAVE_CONFIG_H
		30	#include "config.h"
		31	#endif
		32
		33	#include
		34	#include
		35	#include "i915_drm.h"
		36	#include "i915_drv.h"
		37	#include "intel_drv.h"
		38
		39	#include
		40	#include "../bitmap.h"
		41	#include "sna.h"
		42	//#include "sna_reg.h"
		43	//#include
		44	//#include
		45
		46	#define NO_CACHE 1
		47
		48	#define list_is_empty list_empty
		49	#define list_init INIT_LIST_HEAD
		50
		51	extern struct drm_device *main_device;
		52
		53	static struct kgem_bo *
		54	search_linear_cache(struct kgem *kgem, unsigned int num_pages,
		55	unsigned flags);
		56
		57	#define INT16_MAX (32767)
		58
		59	#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
		60	#define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE)
		61
		62	#define MAX_GTT_VMA_CACHE 512
		63	#define MAX_CPU_VMA_CACHE INT16_MAX
		64	#define MAP_PRESERVE_TIME 10
		65
		66	#define CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) & ~1))
		67	#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 1))
		68
		69	struct kgem_partial_bo {
		70	struct kgem_bo base;
		71	void *mem;
		72	uint32_t used;
		73	uint32_t need_io : 1;
		74	uint32_t write : 2;
		75	uint32_t mmapped : 1;
		76	};
		77
		78	static struct kgem_bo *__kgem_freed_bo;
		79	static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;
		80
		81	static inline int bytes(struct kgem_bo *bo)
		82	{
		83	return kgem_bo_size(bo);
		84	}
		85
		86	#define bucket(B) (B)->size.pages.bucket
		87	#define num_pages(B) (B)->size.pages.count
		88
		89	static void kgem_sna_reset(struct kgem *kgem)
		90	{
		91	struct sna *sna = container_of(kgem, struct sna, kgem);
		92
		93	sna->render.reset(sna);
		94	sna->blt_state.fill_bo = 0;
		95	}
		96
		97	static void kgem_sna_flush(struct kgem *kgem)
		98	{
		99	struct sna *sna = container_of(kgem, struct sna, kgem);
		100
		101	sna->render.flush(sna);
		102
		103	if (sna->render.solid_cache.dirty)
		104	sna_render_flush_solid(sna);
		105	}
		106
		107	static int __gem_write(int fd, uint32_t handle,
		108	int offset, int length,
		109	const void *src)
		110	{
		111	DBG(("%s(handle=%x, offset=%d, len=%d)\n", __FUNCTION__,
		112	handle, offset, length));
		113
		114	write_gem_object(handle, offset, length, src);
		115	return 0;
		116	}
		117
		118
		119	static int gem_write(int fd, uint32_t handle,
		120	int offset, int length,
		121	const void *src)
		122	{
		123	u32 _offset;
		124	u32 _size;
		125	u8 *data_ptr;
		126
		127	DBG(("%s(handle=%x, offset=%d, len=%d)\n", __FUNCTION__,
		128	handle, offset, length));
		129
		130	/* align the transfer to cachelines; fortuitously this is safe! */
		131	if ((offset \| length) & 63) {
		132	_offset = offset & ~63;
		133	_size = ALIGN(offset+length, 64) - _offset;
		134	data_ptr = (u8*)src + _offset - offset;
		135	} else {
		136	_offset = offset;
		137	_size = length;
		138	data_ptr = (u8*)src;
		139	}
		140
		141	write_gem_object(handle, _offset, _size, data_ptr);
		142	return 0;
		143	}
		144
		145	static void kgem_bo_retire(struct kgem kgem, struct kgem_bo bo)
		146	{
		147	DBG(("%s: handle=%x, domain=%d\n",
		148	__FUNCTION__, bo->handle, bo->domain));
		149	assert(!kgem_busy(kgem, bo->handle));
		150
		151	if (bo->domain == DOMAIN_GPU)
		152	kgem_retire(kgem);
		153
		154	if (bo->exec == NULL) {
		155	DBG(("%s: retiring bo handle=%x (needed flush? %d), rq? %d\n",
		156	__FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL));
		157	bo->rq = NULL;
		158	list_del(&bo->request);
		159	bo->needs_flush = bo->flush;
		160	}
		161	}
		162
		163	Bool kgem_bo_write(struct kgem kgem, struct kgem_bo bo,
		164	const void *data, int length)
		165	{
		166	assert(bo->refcnt);
		167	assert(!bo->purged);
		168	assert(!kgem_busy(kgem, bo->handle));
		169
		170	assert(length <= bytes(bo));
		171	if (gem_write(kgem->fd, bo->handle, 0, length, data))
		172	return FALSE;
		173
		174	DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain));
		175	kgem_bo_retire(kgem, bo);
		176	bo->domain = DOMAIN_NONE;
		177	return TRUE;
		178	}
		179
		180	static uint32_t gem_create(int fd, int num_pages)
		181	{
		182	struct drm_i915_gem_object *obj;
		183	int ret;
		184
		185	/* Allocate the new object */
		186	obj = i915_gem_alloc_object(main_device,
		187	PAGE_SIZE * num_pages);
		188	if (obj == NULL)
		189	goto err1;
		190
		191	ret = i915_gem_object_pin(obj, 4096, true);
		192	if (ret)
		193	goto err2;
		194
		195	return (uint32_t)obj;
		196
		197	err2:
		198	drm_gem_object_unreference(&obj->base);
		199	err1:
		200	return 0;
		201	}
		202
		203	static bool
		204	kgem_bo_set_purgeable(struct kgem kgem, struct kgem_bo bo)
		205	{
		206	return true;
		207	}
		208
		209
		210	static bool
		211	kgem_bo_clear_purgeable(struct kgem kgem, struct kgem_bo bo)
		212	{
		213	return true;
		214	}
		215
		216	static void gem_close(int fd, uint32_t handle)
		217	{
		218	destroy_gem_object(handle);
		219	}
		220
		221
		222	/*
		223	constant inline static unsigned long __fls(unsigned long word)
		224	{
		225	asm("bsr %1,%0"
		226	: "=r" (word)
		227	: "rm" (word));
		228	return word;
		229	}
		230	*/
		231
		232	constant inline static int cache_bucket(int num_pages)
		233	{
		234	return __fls(num_pages);
		235	}
		236
		237	static struct kgem_bo __kgem_bo_init(struct kgem_bo bo,
		238	int handle, int num_pages)
		239	{
		240	assert(num_pages);
		241	memset(bo, 0, sizeof(*bo));
		242
		243	bo->refcnt = 1;
		244	bo->handle = handle;
		245	num_pages(bo) = num_pages;
		246	bucket(bo) = cache_bucket(num_pages);
		247	bo->reusable = true;
		248	bo->domain = DOMAIN_CPU;
		249	list_init(&bo->request);
		250	list_init(&bo->list);
		251	list_init(&bo->vma);
		252
		253	return bo;
		254	}
		255
		256	static struct kgem_bo *__kgem_bo_alloc(int handle, int num_pages)
		257	{
		258	struct kgem_bo *bo;
		259
		260	if (__kgem_freed_bo) {
		261	bo = __kgem_freed_bo;
		262	__kgem_freed_bo = (struct kgem_bo *)bo;
		263	} else {
		264	bo = malloc(sizeof(*bo));
		265	if (bo == NULL)
		266	return NULL;
		267	}
		268
		269	return __kgem_bo_init(bo, handle, num_pages);
		270	}
		271
		272	static struct kgem_request _kgem_static_request;
		273
		274	static struct kgem_request *__kgem_request_alloc(void)
		275	{
		276	struct kgem_request *rq;
		277
		278	rq = malloc(sizeof(*rq));
		279	if (rq == NULL)
		280	rq = &_kgem_static_request;
		281
		282	list_init(&rq->buffers);
		283
		284	return rq;
		285	}
		286
		287	static struct list_head inactive(struct kgem kgem, int num_pages)
		288	{
		289	return &kgem->inactive[cache_bucket(num_pages)];
		290	}
		291
		292	static struct list_head active(struct kgem kgem, int num_pages, int tiling)
		293	{
		294	return &kgem->active[cache_bucket(num_pages)][tiling];
		295	}
		296
		297
		298
		299	void kgem_init(struct kgem *kgem, int gen)
		300	{
		301	struct drm_i915_gem_get_aperture aperture;
		302	struct drm_i915_gem_object *obj;
		303
		304	size_t totalram;
		305	unsigned int i, j;
		306	int ret;
		307
		308	memset(kgem, 0, sizeof(*kgem));
		309
		310	kgem->gen = gen;
		311	kgem->wedged = 0;
		312	// kgem->wedged \|= DBG_NO_HW;
		313
		314	obj = i915_gem_alloc_object(main_device, 4096*4);
		315	if (obj == NULL)
		316	goto err2;
		317
		318	ret = i915_gem_object_pin(obj, 4096, true);
		319	if (ret)
		320	goto err3;
		321
		322	kgem->batch_ptr = drm_intel_bo_map(obj, true);
		323	kgem->batch = kgem->batch_ptr;
		324	kgem->batch_idx = 0;
		325	kgem->batch_obj = obj;
		326
		327	kgem->max_batch_size = 1024; //ARRAY_SIZE(kgem->batch);
		328
		329	kgem->half_cpu_cache_pages = (2048*1024) >> 13;
		330
		331	list_init(&kgem->partial);
		332	list_init(&kgem->requests);
		333	list_init(&kgem->flushing);
		334	list_init(&kgem->large);
		335	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		336	list_init(&kgem->inactive[i]);
		337	for (i = 0; i < ARRAY_SIZE(kgem->active); i++) {
		338	for (j = 0; j < ARRAY_SIZE(kgem->active[i]); j++)
		339	list_init(&kgem->active[i][j]);
		340	}
		341	for (i = 0; i < ARRAY_SIZE(kgem->vma); i++) {
		342	for (j = 0; j < ARRAY_SIZE(kgem->vma[i].inactive); j++)
		343	list_init(&kgem->vma[i].inactive[j]);
		344	}
		345	kgem->vma[MAP_GTT].count = -MAX_GTT_VMA_CACHE;
		346	kgem->vma[MAP_CPU].count = -MAX_CPU_VMA_CACHE;
		347
		348	kgem->next_request = __kgem_request_alloc();
		349
		350	//#if defined(USE_VMAP) && defined(I915_PARAM_HAS_VMAP)
		351	// if (!DBG_NO_VMAP)
		352	// kgem->has_vmap = gem_param(kgem, I915_PARAM_HAS_VMAP) > 0;
		353	//#endif
		354	// DBG(("%s: using vmap=%d\n", __FUNCTION__, kgem->has_vmap));
		355
		356	if (gen < 40) {
		357	// if (!DBG_NO_RELAXED_FENCING) {
		358	// kgem->has_relaxed_fencing =
		359	// gem_param(kgem, I915_PARAM_HAS_RELAXED_FENCING) > 0;
		360	// }
		361	} else
		362	kgem->has_relaxed_fencing = 1;
		363	DBG(("%s: has relaxed fencing? %d\n", __FUNCTION__,
		364	kgem->has_relaxed_fencing));
		365
		366	kgem->has_llc = (gen >= 60)?true:false;
		367	kgem->has_cpu_bo = kgem->has_llc;
		368	DBG(("%s: cpu bo enabled %d: llc? %d\n", __FUNCTION__,
		369	kgem->has_cpu_bo, kgem->has_llc));
		370
		371	kgem->has_semaphores = false;
		372	// if (gen >= 60 && semaphores_enabled())
		373	// kgem->has_semaphores = true;
		374	// DBG(("%s: semaphores enabled? %d\n", __FUNCTION__,
		375	// kgem->has_semaphores));
		376
		377	VG_CLEAR(aperture);
		378	aperture.aper_size = 6410241024;
		379	i915_gem_get_aperture_ioctl(main_device, &aperture, NULL);
		380	kgem->aperture_total = aperture.aper_size;
		381	kgem->aperture_high = aperture.aper_size * 3/4;
		382	kgem->aperture_low = aperture.aper_size * 1/3;
		383	DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
		384	kgem->aperture_low, kgem->aperture_low / (1024*1024),
		385	kgem->aperture_high, kgem->aperture_high / (1024*1024)));
		386
		387	kgem->aperture_mappable = aperture.aper_size;
		388	DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
		389	kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));
		390
		391	kgem->partial_buffer_size = 64 * 1024;
		392	while (kgem->partial_buffer_size < kgem->aperture_mappable >> 10)
		393	kgem->partial_buffer_size *= 2;
		394	DBG(("%s: partial buffer size=%d [%d KiB]\n", __FUNCTION__,
		395	kgem->partial_buffer_size, kgem->partial_buffer_size / 1024));
		396
		397	kgem->min_alignment = 4;
		398	if (gen < 60)
		399	/* XXX workaround an issue where we appear to fail to
		400	* disable dual-stream mode */
		401	kgem->min_alignment = 64;
		402
		403	kgem->max_object_size = 2 * kgem->aperture_total / 3;
		404	kgem->max_cpu_size = kgem->max_object_size;
		405	kgem->max_gpu_size = kgem->max_object_size;
		406	if (!kgem->has_llc)
		407	kgem->max_gpu_size = MAX_CACHE_SIZE;
		408	if (gen < 40) {
		409	/* If we have to use fences for blitting, we have to make
		410	* sure we can fit them into the aperture.
		411	*/
		412	kgem->max_gpu_size = kgem->aperture_mappable / 2;
		413	if (kgem->max_gpu_size > kgem->aperture_low)
		414	kgem->max_gpu_size = kgem->aperture_low;
		415	}
		416	if (kgem->max_gpu_size > kgem->max_cpu_size)
		417	kgem->max_gpu_size = kgem->max_cpu_size;
		418
		419	kgem->max_upload_tile_size = kgem->aperture_mappable / 2;
		420	if (kgem->max_upload_tile_size > kgem->max_gpu_size / 2)
		421	kgem->max_upload_tile_size = kgem->max_gpu_size / 2;
		422
		423	kgem->max_copy_tile_size = (MAX_CACHE_SIZE + 1)/2;
		424	if (kgem->max_copy_tile_size > kgem->max_gpu_size / 2)
		425	kgem->max_copy_tile_size = kgem->max_gpu_size / 2;
		426
		427	totalram = 1024*1024; //total_ram_size();
		428	if (totalram == 0) {
		429	DBG(("%s: total ram size unknown, assuming maximum of total aperture\n",
		430	__FUNCTION__));
		431	totalram = kgem->aperture_total;
		432	}
		433	if (kgem->max_object_size > totalram / 2)
		434	kgem->max_object_size = totalram / 2;
		435	if (kgem->max_cpu_size > totalram / 2)
		436	kgem->max_cpu_size = totalram / 2;
		437	if (kgem->max_gpu_size > totalram / 4)
		438	kgem->max_gpu_size = totalram / 4;
		439
		440	kgem->large_object_size = MAX_CACHE_SIZE;
		441	if (kgem->large_object_size > kgem->max_gpu_size)
		442	kgem->large_object_size = kgem->max_gpu_size;
		443
		444	DBG(("%s: large object thresold=%d\n",
		445	__FUNCTION__, kgem->large_object_size));
		446	DBG(("%s: max object size (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
		447	__FUNCTION__,
		448	kgem->max_gpu_size, kgem->max_cpu_size,
		449	kgem->max_upload_tile_size, kgem->max_copy_tile_size));
		450
		451	/* Convert the aperture thresholds to pages */
		452	kgem->aperture_low /= PAGE_SIZE;
		453	kgem->aperture_high /= PAGE_SIZE;
		454
		455	// kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
		456	// if ((int)kgem->fence_max < 0)
		457	kgem->fence_max = 5; /* minimum safe value for all hw */
		458	DBG(("%s: max fences=%d\n", __FUNCTION__, kgem->fence_max));
		459	err3:
		460	err2:
		461	return;
		462	}
		463
		464	static struct drm_i915_gem_exec_object2 *
		465	kgem_add_handle(struct kgem kgem, struct kgem_bo bo)
		466	{
		467	struct drm_i915_gem_exec_object2 *exec;
		468
		469	DBG(("%s: handle=%d, index=%d\n",
		470	__FUNCTION__, bo->handle, kgem->nexec));
		471
		472	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		473	exec = memset(&kgem->exec[kgem->nexec++], 0, sizeof(*exec));
		474	exec->handle = bo->handle;
		475	exec->offset = bo->presumed_offset;
		476
		477	kgem->aperture += num_pages(bo);
		478
		479	return exec;
		480	}
		481
		482	void _kgem_add_bo(struct kgem kgem, struct kgem_bo bo)
		483	{
		484	bo->exec = kgem_add_handle(kgem, bo);
		485	bo->rq = kgem->next_request;
		486
		487	list_move(&bo->request, &kgem->next_request->buffers);
		488
		489	/* XXX is it worth working around gcc here? */
		490	kgem->flush \|= bo->flush;
		491	kgem->sync \|= bo->sync;
		492	kgem->scanout \|= bo->scanout;
		493	}
		494
		495	static uint32_t kgem_end_batch(struct kgem *kgem)
		496	{
		497	// kgem->context_switch(kgem, KGEM_NONE);
		498
		499	kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
		500	if (kgem->nbatch & 1)
		501	kgem->batch[kgem->nbatch++] = MI_NOOP;
		502
		503	return kgem->nbatch;
		504	}
		505
		506	static void kgem_fixup_self_relocs(struct kgem kgem, struct kgem_bo bo)
		507	{
		508	int n;
		509
		510	for (n = 0; n < kgem->nreloc; n++)
		511	{
		512	if (kgem->reloc[n].target_handle == 0)
		513	{
		514	kgem->reloc[n].target_handle = bo->handle;
		515	kgem->reloc[n].presumed_offset = bo->presumed_offset;
		516	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		517	kgem->reloc[n].delta + bo->presumed_offset;
		518
		519	dbgprintf("fixup reloc %d pos %d handle %d delta %x \n",
		520	n, kgem->reloc[n].offset/sizeof(kgem->batch[0]),
		521	bo->handle, kgem->reloc[n].delta);
		522	}
		523	}
		524	}
		525
		526	static void kgem_bo_binding_free(struct kgem kgem, struct kgem_bo bo)
		527	{
		528	struct kgem_bo_binding *b;
		529
		530	b = bo->binding.next;
		531	while (b) {
		532	struct kgem_bo_binding *next = b->next;
		533	free (b);
		534	b = next;
		535	}
		536	}
		537
		538	static void kgem_bo_release_map(struct kgem kgem, struct kgem_bo bo)
		539	{
		540	int type = IS_CPU_MAP(bo->map);
		541
		542	DBG(("%s: releasing %s vma for handle=%d, count=%d\n",
		543	__FUNCTION__, type ? "CPU" : "GTT",
		544	bo->handle, kgem->vma[type].count));
		545
		546	VG(if (type) VALGRIND_FREELIKE_BLOCK(CPU_MAP(bo->map), 0));
		547	// munmap(CPU_MAP(bo->map), bytes(bo));
		548	bo->map = NULL;
		549
		550	if (!list_is_empty(&bo->vma)) {
		551	list_del(&bo->vma);
		552	kgem->vma[type].count--;
		553	}
		554	}
		555
		556	static void kgem_bo_free(struct kgem kgem, struct kgem_bo bo)
		557	{
		558	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		559	assert(bo->refcnt == 0);
		560	assert(bo->exec == NULL);
		561
		562	kgem_bo_binding_free(kgem, bo);
		563
		564	if (bo->map)
		565	kgem_bo_release_map(kgem, bo);
		566	assert(list_is_empty(&bo->vma));
		567
		568	list_del(&bo->list);
		569	list_del(&bo->request);
		570	gem_close(kgem->fd, bo->handle);
		571
		572	if (!bo->io) {
		573	(struct kgem_bo *)bo = __kgem_freed_bo;
		574	__kgem_freed_bo = bo;
		575	} else
		576	free(bo);
		577	}
		578
		579	inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
		580	struct kgem_bo *bo)
		581	{
		582	assert(!kgem_busy(kgem, bo->handle));
		583	assert(!bo->proxy);
		584	assert(!bo->io);
		585	assert(!bo->needs_flush);
		586	assert(bo->rq == NULL);
		587	assert(bo->domain != DOMAIN_GPU);
		588
		589	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
		590	kgem_bo_free(kgem, bo);
		591	return;
		592	}
		593
		594	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
		595	if (bo->map) {
		596	int type = IS_CPU_MAP(bo->map);
		597	if (bucket(bo) >= NUM_CACHE_BUCKETS \|\|
		598	(!type && !kgem_bo_is_mappable(kgem, bo))) {
		599	list_del(&bo->vma);
		600	// munmap(CPU_MAP(bo->map), bytes(bo));
		601	bo->map = NULL;
		602	}
		603	if (bo->map) {
		604	list_move(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]);
		605	kgem->vma[type].count++;
		606	}
		607	}
		608
		609	kgem->need_expire = true;
		610	}
		611
		612	inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
		613	struct kgem_bo *bo)
		614	{
		615	list_del(&bo->list);
		616	assert(bo->rq == NULL);
		617	if (bo->map) {
		618	assert(!list_is_empty(&bo->vma));
		619	list_del(&bo->vma);
		620	kgem->vma[IS_CPU_MAP(bo->map)].count--;
		621	}
		622	}
		623
		624	inline static void kgem_bo_remove_from_active(struct kgem *kgem,
		625	struct kgem_bo *bo)
		626	{
		627	list_del(&bo->list);
		628	if (bo->rq == &_kgem_static_request)
		629	list_del(&bo->request);
		630	assert(list_is_empty(&bo->vma));
		631	}
		632
		633	static void __kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		634	{
		635	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		636
		637	assert(list_is_empty(&bo->list));
		638	assert(bo->refcnt == 0);
		639
		640	bo->binding.offset = 0;
		641
		642	if (NO_CACHE)
		643	goto destroy;
		644
		645	if (bo->io) {
		646	struct kgem_bo *base;
		647
		648	base = malloc(sizeof(*base));
		649	if (base) {
		650	DBG(("%s: transferring io handle=%d to bo\n",
		651	__FUNCTION__, bo->handle));
		652	/* transfer the handle to a minimum bo */
		653	memcpy(base, bo, sizeof (*base));
		654	base->reusable = true;
		655	base->io = false;
		656	list_init(&base->list);
		657	list_replace(&bo->request, &base->request);
		658	list_replace(&bo->vma, &base->vma);
		659	free(bo);
		660	bo = base;
		661	}
		662	}
		663
		664	if (!bo->reusable) {
		665	DBG(("%s: handle=%d, not reusable\n",
		666	__FUNCTION__, bo->handle));
		667	goto destroy;
		668	}
		669
		670	if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU)
		671	kgem_bo_release_map(kgem, bo);
		672
		673	assert(list_is_empty(&bo->vma));
		674	assert(list_is_empty(&bo->list));
		675	assert(bo->vmap == false && bo->sync == false);
		676	assert(bo->io == false);
		677
		678	bo->scanout = bo->flush = false;
		679	if (bo->rq) {
		680	struct list *cache;
		681
		682	DBG(("%s: handle=%d -> active\n", __FUNCTION__, bo->handle));
		683	if (bucket(bo) < NUM_CACHE_BUCKETS)
		684	cache = &kgem->active[bucket(bo)][bo->tiling];
		685	else
		686	cache = &kgem->large;
		687	list_add(&bo->list, cache);
		688	return;
		689	}
		690
		691	assert(bo->exec == NULL);
		692	assert(list_is_empty(&bo->request));
		693	/*
		694	if (bo->needs_flush) {
		695	if ((bo->needs_flush = kgem_busy(kgem, bo->handle))) {
		696	struct list *cache;
		697
		698	DBG(("%s: handle=%d -> flushing\n",
		699	__FUNCTION__, bo->handle));
		700
		701	list_add(&bo->request, &kgem->flushing);
		702	if (bucket(bo) < NUM_CACHE_BUCKETS)
		703	cache = &kgem->active[bucket(bo)][bo->tiling];
		704	else
		705	cache = &kgem->large;
		706	list_add(&bo->list, cache);
		707	bo->rq = &_kgem_static_request;
		708	return;
		709	}
		710
		711	bo->domain = DOMAIN_NONE;
		712	}
		713	*/
		714	if (!IS_CPU_MAP(bo->map)) {
		715	if (!kgem_bo_set_purgeable(kgem, bo))
		716	goto destroy;
		717
		718	if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
		719	goto destroy;
		720
		721	DBG(("%s: handle=%d, purged\n",
		722	__FUNCTION__, bo->handle));
		723	}
		724
		725	DBG(("%s: handle=%d -> inactive\n", __FUNCTION__, bo->handle));
		726	kgem_bo_move_to_inactive(kgem, bo);
		727	return;
		728
		729	destroy:
		730	if (!bo->exec)
		731	kgem_bo_free(kgem, bo);
		732	}
		733
		734
		735
		736
		737
		738	bool kgem_retire(struct kgem *kgem)
		739	{
		740	struct kgem_bo bo, next;
		741	bool retired = false;
		742
		743	DBG(("%s\n", __FUNCTION__));
		744
		745	list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
		746	assert(bo->refcnt == 0);
		747	assert(bo->rq == &_kgem_static_request);
		748	assert(bo->exec == NULL);
		749
		750	// if (kgem_busy(kgem, bo->handle))
		751	// break;
		752
		753	DBG(("%s: moving %d from flush to inactive\n",
		754	__FUNCTION__, bo->handle));
		755	if (kgem_bo_set_purgeable(kgem, bo)) {
		756	bo->needs_flush = false;
		757	bo->domain = DOMAIN_NONE;
		758	bo->rq = NULL;
		759	list_del(&bo->request);
		760	kgem_bo_move_to_inactive(kgem, bo);
		761	} else
		762	kgem_bo_free(kgem, bo);
		763
		764	retired = true;
		765	}
		766
		767	while (!list_is_empty(&kgem->requests)) {
		768	struct kgem_request *rq;
		769
		770	rq = list_first_entry(&kgem->requests,
		771	struct kgem_request,
		772	list);
		773	// if (kgem_busy(kgem, rq->bo->handle))
		774	// break;
		775
		776	DBG(("%s: request %d complete\n",
		777	__FUNCTION__, rq->bo->handle));
		778
		779	while (!list_is_empty(&rq->buffers)) {
		780	bo = list_first_entry(&rq->buffers,
		781	struct kgem_bo,
		782	request);
		783
		784	assert(bo->rq == rq);
		785	assert(bo->exec == NULL);
		786	assert(bo->domain == DOMAIN_GPU);
		787
		788	list_del(&bo->request);
		789	bo->rq = NULL;
		790
		791	// if (bo->needs_flush)
		792	// bo->needs_flush = kgem_busy(kgem, bo->handle);
		793	if (!bo->needs_flush)
		794	bo->domain = DOMAIN_NONE;
		795
		796	if (bo->refcnt)
		797	continue;
		798
		799	if (!bo->reusable) {
		800	DBG(("%s: closing %d\n",
		801	__FUNCTION__, bo->handle));
		802	kgem_bo_free(kgem, bo);
		803	continue;
		804	}
		805
		806	if (bo->needs_flush) {
		807	DBG(("%s: moving %d to flushing\n",
		808	__FUNCTION__, bo->handle));
		809	list_add(&bo->request, &kgem->flushing);
		810	bo->rq = &_kgem_static_request;
		811	} else if (kgem_bo_set_purgeable(kgem, bo)) {
		812	DBG(("%s: moving %d to inactive\n",
		813	__FUNCTION__, bo->handle));
		814	kgem_bo_move_to_inactive(kgem, bo);
		815	retired = true;
		816	} else {
		817	DBG(("%s: closing %d\n",
		818	__FUNCTION__, bo->handle));
		819	kgem_bo_free(kgem, bo);
		820	}
		821	}
		822
		823	rq->bo->refcnt--;
		824	assert(rq->bo->refcnt == 0);
		825	assert(rq->bo->rq == NULL);
		826	assert(list_is_empty(&rq->bo->request));
		827	if (kgem_bo_set_purgeable(kgem, rq->bo)) {
		828	kgem_bo_move_to_inactive(kgem, rq->bo);
		829	retired = true;
		830	} else {
		831	DBG(("%s: closing %d\n",
		832	__FUNCTION__, rq->bo->handle));
		833	kgem_bo_free(kgem, rq->bo);
		834	}
		835
		836	list_del(&rq->list);
		837	free(rq);
		838	}
		839
		840	kgem->need_retire = !list_is_empty(&kgem->requests);
		841	DBG(("%s -- need_retire=%d\n", __FUNCTION__, kgem->need_retire));
		842
		843	kgem->retire(kgem);
		844
		845	return retired;
		846	}
		847
		848
		849
		850
		851
		852
		853
		854
		855
		856
		857
		858
		859
		860
		861
		862
		863
		864
		865
		866
		867
		868
		869
		870
		871
		872
		873
		874
		875
		876
		877
		878
		879
		880
		881
		882
		883
		884
		885
		886
		887
		888	static int kgem_batch_write(struct kgem *kgem, uint32_t handle, uint32_t size)
		889	{
		890	int ret;
		891
		892	assert(!kgem_busy(kgem, handle));
		893
		894	/* If there is no surface data, just upload the batch */
		895	if (kgem->surface == kgem->max_batch_size)
		896	return gem_write(kgem->fd, handle,
		897	0, sizeof(uint32_t)*kgem->nbatch,
		898	kgem->batch);
		899
		900	/* Are the batch pages conjoint with the surface pages? */
		901	if (kgem->surface < kgem->nbatch + PAGE_SIZE/4) {
		902	assert(size == sizeof(kgem->batch));
		903	return gem_write(kgem->fd, handle,
		904	0, sizeof(kgem->batch),
		905	kgem->batch);
		906	}
		907
		908	/* Disjoint surface/batch, upload separately */
		909	ret = gem_write(kgem->fd, handle,
		910	0, sizeof(uint32_t)*kgem->nbatch,
		911	kgem->batch);
		912	if (ret)
		913	return ret;
		914
		915	assert(kgem->nbatch*sizeof(uint32_t) <=
		916	sizeof(uint32_t)*kgem->surface - (sizeof(kgem->batch)-size));
		917	return __gem_write(kgem->fd, handle,
		918	sizeof(uint32_t)*kgem->surface - (sizeof(kgem->batch)-size),
		919	sizeof(kgem->batch) - sizeof(uint32_t)*kgem->surface,
		920	kgem->batch + kgem->surface);
		921	}
		922
		923	void kgem_reset(struct kgem *kgem)
		924	{
		925	// ENTER();
		926
		927	kgem->nfence = 0;
		928	kgem->nexec = 0;
		929	kgem->nreloc = 0;
		930	kgem->aperture = 0;
		931	kgem->aperture_fenced = 0;
		932	kgem->nbatch = 0;
		933	kgem->surface = kgem->max_batch_size;
		934	kgem->mode = KGEM_NONE;
		935	kgem->flush = 0;
		936	kgem->scanout = 0;
		937
		938	kgem->batch = kgem->batch_ptr+1024*kgem->batch_idx;
		939
		940	kgem->next_request = __kgem_request_alloc();
		941
		942	kgem_sna_reset(kgem);
		943	// dbgprintf("surface %x\n", kgem->surface);
		944	// LEAVE();
		945	}
		946
		947	static int compact_batch_surface(struct kgem *kgem)
		948	{
		949	int size, shrink, n;
		950
		951	/* See if we can pack the contents into one or two pages */
		952	size = kgem->max_batch_size - kgem->surface + kgem->nbatch;
		953	if (size > 2048)
		954	return sizeof(kgem->batch);
		955	else if (size > 1024)
		956	size = 8192, shrink = 2*4096;
		957	else
		958	size = 4096, shrink = 3*4096;
		959
		960
		961	for (n = 0; n < kgem->nreloc; n++) {
		962	if (kgem->reloc[n].read_domains == I915_GEM_DOMAIN_INSTRUCTION &&
		963	kgem->reloc[n].target_handle == 0)
		964	kgem->reloc[n].delta -= shrink;
		965
		966	if (kgem->reloc[n].offset >= size)
		967	kgem->reloc[n].offset -= shrink;
		968	}
		969
		970	return size;
		971	}
		972
		973	void execute_buffer (struct drm_i915_gem_object *buffer, uint32_t offset,
		974	int size);
		975
		976	void _kgem_submit(struct kgem *kgem)
		977	{
		978	struct kgem_request *rq;
		979	uint32_t batch_end;
		980	int size;
		981
		982	assert(!DBG_NO_HW);
		983
		984	assert(kgem->nbatch);
		985	assert(kgem->nbatch <= KGEM_BATCH_SIZE(kgem));
		986	assert(kgem->nbatch <= kgem->surface);
		987
		988	batch_end = kgem_end_batch(kgem);
		989	kgem_sna_flush(kgem);
		990
		991	DBG(("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n",
		992	kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
		993	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture));
		994
		995	assert(kgem->nbatch <= kgem->max_batch_size);
		996	assert(kgem->nbatch <= kgem->surface);
		997	assert(kgem->nreloc <= ARRAY_SIZE(kgem->reloc));
		998	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		999	assert(kgem->nfence <= kgem->fence_max);
		1000
		1001	// kgem_finish_partials(kgem);
		1002
		1003	rq = kgem->next_request;
		1004	// if (kgem->surface != kgem->max_batch_size)
		1005	// size = compact_batch_surface(kgem);
		1006	// else
		1007	size = kgem->nbatch * sizeof(kgem->batch[0]);
		1008	#if 0
		1009	{
		1010	int i;
		1011
		1012	dbgprintf("\nDump batch\n\n");
		1013
		1014	for(i=0; i < kgem->nbatch; i++)
		1015	{
		1016	dbgprintf("\t0x%08x,\t/* %d */\n",
		1017	kgem->batch[i], i);
		1018	}
		1019	dbgprintf("\ndone\n");
		1020	};
		1021	#endif
		1022
		1023	execute_buffer(kgem->batch_obj, kgem->batch_idx4096, sizeof(uint32_t)kgem->nbatch);
		1024
		1025	// if (kgem->wedged)
		1026	// kgem_cleanup(kgem);
		1027
		1028	kgem->batch_idx++;
		1029	kgem->batch_idx&= 3;
		1030
		1031	kgem->flush_now = kgem->scanout;
		1032	kgem_reset(kgem);
		1033
		1034	assert(kgem->next_request != NULL);
		1035	}
		1036
		1037	static struct kgem_bo *
		1038	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		1039	{
		1040	struct kgem_bo bo, first = NULL;
		1041	bool use_active = (flags & CREATE_INACTIVE) == 0;
		1042	struct list_head *cache;
		1043
		1044	if (num_pages >= MAX_CACHE_SIZE / PAGE_SIZE)
		1045	return NULL;
		1046
		1047	if (!use_active &&
		1048	list_is_empty(inactive(kgem, num_pages)) &&
		1049	!list_is_empty(active(kgem, num_pages, I915_TILING_NONE)) &&
		1050	!kgem_retire(kgem))
		1051	return NULL;
		1052
		1053	if (!use_active && flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		1054	int for_cpu = !!(flags & CREATE_CPU_MAP);
		1055	cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
		1056	list_for_each_entry(bo, cache, vma) {
		1057	assert(IS_CPU_MAP(bo->map) == for_cpu);
		1058	assert(bucket(bo) == cache_bucket(num_pages));
		1059
		1060	if (num_pages > num_pages(bo)) {
		1061	DBG(("inactive too small: %d < %d\n",
		1062	num_pages(bo), num_pages));
		1063	continue;
		1064	}
		1065
		1066	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		1067	kgem_bo_free(kgem, bo);
		1068	break;
		1069	}
		1070
		1071	// if (I915_TILING_NONE != bo->tiling &&
		1072	// gem_set_tiling(kgem->fd, bo->handle,
		1073	// I915_TILING_NONE, 0) != I915_TILING_NONE)
		1074	// continue;
		1075
		1076	kgem_bo_remove_from_inactive(kgem, bo);
		1077
		1078	bo->tiling = I915_TILING_NONE;
		1079	bo->pitch = 0;
		1080	bo->delta = 0;
		1081	DBG((" %s: found handle=%d (num_pages=%d) in linear vma cache\n",
		1082	__FUNCTION__, bo->handle, num_pages(bo)));
		1083	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		1084	assert(!bo->needs_flush);
		1085	//assert(!kgem_busy(kgem, bo->handle));
		1086	return bo;
		1087	}
		1088	}
		1089
		1090	cache = use_active ? active(kgem, num_pages, I915_TILING_NONE) : inactive(kgem, num_pages);
		1091	list_for_each_entry(bo, cache, list) {
		1092	assert(bo->refcnt == 0);
		1093	assert(bo->reusable);
		1094	assert(!!bo->rq == !!use_active);
		1095
		1096	if (num_pages > num_pages(bo))
		1097	continue;
		1098
		1099	if (use_active && bo->tiling != I915_TILING_NONE)
		1100	continue;
		1101
		1102	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		1103	kgem_bo_free(kgem, bo);
		1104	break;
		1105	}
		1106	/*
		1107	if (I915_TILING_NONE != bo->tiling) {
		1108	if (use_active)
		1109	continue;
		1110
		1111	if (gem_set_tiling(kgem->fd, bo->handle,
		1112	I915_TILING_NONE, 0) != I915_TILING_NONE)
		1113	continue;
		1114
		1115	bo->tiling = I915_TILING_NONE;
		1116	}
		1117	*/
		1118	if (bo->map) {
		1119	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		1120	int for_cpu = !!(flags & CREATE_CPU_MAP);
		1121	if (IS_CPU_MAP(bo->map) != for_cpu) {
		1122	if (first != NULL)
		1123	break;
		1124
		1125	first = bo;
		1126	continue;
		1127	}
		1128	} else {
		1129	if (first != NULL)
		1130	break;
		1131
		1132	first = bo;
		1133	continue;
		1134	}
		1135	} else {
		1136	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		1137	if (first != NULL)
		1138	break;
		1139
		1140	first = bo;
		1141	continue;
		1142	}
		1143	}
		1144
		1145	if (use_active)
		1146	kgem_bo_remove_from_active(kgem, bo);
		1147	else
		1148	kgem_bo_remove_from_inactive(kgem, bo);
		1149
		1150	assert(bo->tiling == I915_TILING_NONE);
		1151	bo->pitch = 0;
		1152	bo->delta = 0;
		1153	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		1154	__FUNCTION__, bo->handle, num_pages(bo),
		1155	use_active ? "active" : "inactive"));
		1156	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		1157	assert(!bo->needs_flush \|\| use_active);
		1158	//assert(use_active \|\| !kgem_busy(kgem, bo->handle));
		1159	return bo;
		1160	}
		1161
		1162	if (first) {
		1163	assert(first->tiling == I915_TILING_NONE);
		1164
		1165	if (use_active)
		1166	kgem_bo_remove_from_active(kgem, first);
		1167	else
		1168	kgem_bo_remove_from_inactive(kgem, first);
		1169
		1170	first->pitch = 0;
		1171	first->delta = 0;
		1172	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		1173	__FUNCTION__, first->handle, num_pages(first),
		1174	use_active ? "active" : "inactive"));
		1175	assert(use_active \|\| first->domain != DOMAIN_GPU);
		1176	assert(!first->needs_flush \|\| use_active);
		1177	//assert(use_active \|\| !kgem_busy(kgem, first->handle));
		1178	return first;
		1179	}
		1180
		1181	return NULL;
		1182	}
		1183
		1184
		1185
		1186
		1187
		1188
		1189	struct kgem_bo kgem_create_linear(struct kgem kgem, int size)
		1190	{
		1191	struct kgem_bo *bo;
		1192	uint32_t handle;
		1193
		1194	DBG(("%s(%d)\n", __FUNCTION__, size));
		1195
		1196	size = (size + PAGE_SIZE - 1) / PAGE_SIZE;
		1197	bo = search_linear_cache(kgem, size, CREATE_INACTIVE);
		1198	if (bo)
		1199	return kgem_bo_reference(bo);
		1200
		1201	handle = gem_create(kgem->fd, size);
		1202	if (handle == 0)
		1203	return NULL;
		1204
		1205	DBG(("%s: new handle=%x\n", __FUNCTION__, handle));
		1206	bo = __kgem_bo_alloc(handle, size);
		1207	if (bo == NULL) {
		1208	gem_close(kgem->fd, handle);
		1209	return NULL;
		1210	}
		1211	struct drm_i915_gem_object *obj;
		1212	obj = (void*)handle;
		1213
		1214	bo->gaddr = obj->gtt_offset;
		1215	return bo;
		1216	}
		1217
		1218
		1219
		1220
		1221
		1222
		1223	inline int kgem_bo_fenced_size(struct kgem kgem, struct kgem_bo bo)
		1224	{
		1225	unsigned int size;
		1226
		1227	assert(bo->tiling);
		1228	assert(kgem->gen < 40);
		1229
		1230	if (kgem->gen < 30)
		1231	size = 512 * 1024;
		1232	else
		1233	size = 1024 * 1024;
		1234	while (size < bytes(bo))
		1235	size *= 2;
		1236
		1237	return size;
		1238	}
		1239
		1240
		1241
		1242
		1243
		1244
		1245
		1246
		1247
		1248
		1249
		1250
		1251
		1252
		1253
		1254
		1255
		1256
		1257
		1258
		1259
		1260
		1261
		1262
		1263
		1264
		1265
		1266
		1267
		1268
		1269
		1270
		1271
		1272
		1273
		1274
		1275
		1276
		1277
		1278
		1279
		1280
		1281
		1282
		1283
		1284
		1285
		1286
		1287	void _kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		1288	{
		1289	// if (bo->proxy) {
		1290	// assert(bo->map == NULL);
		1291	// if (bo->io && bo->exec == NULL)
		1292	// _kgem_bo_delete_partial(kgem, bo);
		1293	// kgem_bo_unref(kgem, bo->proxy);
		1294	// kgem_bo_binding_free(kgem, bo);
		1295	// _list_del(&bo->request);
		1296	// free(bo);
		1297	// return;
		1298	// }
		1299
		1300	// if (bo->vmap)
		1301	// kgem_bo_sync__cpu(kgem, bo);
		1302
		1303	__kgem_bo_destroy(kgem, bo);
		1304	}
		1305
		1306	void __kgem_flush(struct kgem kgem, struct kgem_bo bo)
		1307	{
		1308	/* The kernel will emit a flush and update its own flushing lists. */
		1309	// kgem_busy(kgem, bo->handle);
		1310	}
		1311
		1312	bool kgem_check_bo(struct kgem *kgem, ...)
		1313	{
		1314	va_list ap;
		1315	struct kgem_bo *bo;
		1316	int num_exec = 0;
		1317	int num_pages = 0;
		1318
		1319	va_start(ap, kgem);
		1320	while ((bo = va_arg(ap, struct kgem_bo *))) {
		1321	if (bo->exec)
		1322	continue;
		1323
		1324	if (bo->proxy) {
		1325	bo = bo->proxy;
		1326	if (bo->exec)
		1327	continue;
		1328	}
		1329	num_pages += num_pages(bo);
		1330	num_exec++;
		1331	}
		1332	va_end(ap);
		1333
		1334	if (!num_pages)
		1335	return true;
		1336
		1337	if (kgem->aperture > kgem->aperture_low)
		1338	return false;
		1339
		1340	if (num_pages + kgem->aperture > kgem->aperture_high)
		1341	return false;
		1342
		1343	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem))
		1344	return false;
		1345
		1346	return true;
		1347	}
		1348
		1349	/*
		1350	bool kgem_check_bo_fenced(struct kgem *kgem, ...)
		1351	{
		1352	va_list ap;
		1353	struct kgem_bo *bo;
		1354	int num_fence = 0;
		1355	int num_exec = 0;
		1356	int num_pages = 0;
		1357	int fenced_size = 0;
		1358
		1359	va_start(ap, kgem);
		1360	while ((bo = va_arg(ap, struct kgem_bo *))) {
		1361	if (bo->proxy)
		1362	bo = bo->proxy;
		1363	if (bo->exec) {
		1364	if (kgem->gen >= 40 \|\| bo->tiling == I915_TILING_NONE)
		1365	continue;
		1366
		1367	if ((bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		1368	fenced_size += kgem_bo_fenced_size(kgem, bo);
		1369	num_fence++;
		1370	}
		1371
		1372	continue;
		1373	}
		1374
		1375	num_pages += num_pages(bo);
		1376	num_exec++;
		1377	if (kgem->gen < 40 && bo->tiling) {
		1378	fenced_size += kgem_bo_fenced_size(kgem, bo);
		1379	num_fence++;
		1380	}
		1381	}
		1382	va_end(ap);
		1383
		1384	if (fenced_size + kgem->aperture_fenced > kgem->aperture_mappable)
		1385	return false;
		1386
		1387	if (kgem->nfence + num_fence > kgem->fence_max)
		1388	return false;
		1389
		1390	if (!num_pages)
		1391	return true;
		1392
		1393	if (kgem->aperture > kgem->aperture_low)
		1394	return false;
		1395
		1396	if (num_pages + kgem->aperture > kgem->aperture_high)
		1397	return false;
		1398
		1399	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem))
		1400	return false;
		1401
		1402	return true;
		1403	}
		1404	*/
		1405	#if 0
		1406	uint32_t kgem_add_reloc(struct kgem *kgem,
		1407	uint32_t pos,
		1408	struct kgem_bo *bo,
		1409	uint32_t read_write_domain,
		1410	uint32_t delta)
		1411	{
		1412	int index;
		1413
		1414	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
		1415	__FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
		1416
		1417	assert((read_write_domain & 0x7fff) == 0 \|\| bo != NULL);
		1418
		1419	index = kgem->nreloc++;
		1420	assert(index < ARRAY_SIZE(kgem->reloc));
		1421	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
		1422	if (bo) {
		1423	assert(bo->refcnt);
		1424	assert(!bo->purged);
		1425
		1426	delta += bo->delta;
		1427	if (bo->proxy) {
		1428	DBG(("%s: adding proxy for handle=%d\n",
		1429	__FUNCTION__, bo->handle));
		1430	assert(bo->handle == bo->proxy->handle);
		1431	/* need to release the cache upon batch submit */
		1432	list_move(&bo->request, &kgem->next_request->buffers);
		1433	bo->exec = &_kgem_dummy_exec;
		1434	bo = bo->proxy;
		1435	}
		1436
		1437	assert(!bo->purged);
		1438
		1439	// if (bo->exec == NULL)
		1440	// _kgem_add_bo(kgem, bo);
		1441
		1442	// if (kgem->gen < 40 && read_write_domain & KGEM_RELOC_FENCED) {
		1443	// if (bo->tiling &&
		1444	// (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		1445	// assert(kgem->nfence < kgem->fence_max);
		1446	// kgem->aperture_fenced +=
		1447	// kgem_bo_fenced_size(kgem, bo);
		1448	// kgem->nfence++;
		1449	// }
		1450	// bo->exec->flags \|= EXEC_OBJECT_NEEDS_FENCE;
		1451	// }
		1452
		1453	kgem->reloc[index].delta = delta;
		1454	kgem->reloc[index].target_handle = bo->handle;
		1455	kgem->reloc[index].presumed_offset = bo->presumed_offset;
		1456
		1457	if (read_write_domain & 0x7fff) {
		1458	DBG(("%s: marking handle=%d dirty\n",
		1459	__FUNCTION__, bo->handle));
		1460	bo->needs_flush = bo->dirty = true;
		1461	}
		1462
		1463	delta += bo->presumed_offset;
		1464	} else {
		1465	kgem->reloc[index].delta = delta;
		1466	kgem->reloc[index].target_handle = 0;
		1467	kgem->reloc[index].presumed_offset = 0;
		1468	}
		1469	kgem->reloc[index].read_domains = read_write_domain >> 16;
		1470	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
		1471
		1472	return delta;
		1473	}
		1474	#endif
		1475
		1476
		1477
		1478
		1479
		1480
		1481
		1482
		1483
		1484	void kgem_bo_map(struct kgem kgem, struct kgem_bo *bo)
		1485	{
		1486	void *ptr;
		1487
		1488	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		1489	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		1490
		1491	assert(!bo->purged);
		1492	assert(bo->exec == NULL);
		1493	assert(list_is_empty(&bo->list));
		1494
		1495	// if (bo->tiling == I915_TILING_NONE &&
		1496	// (kgem->has_llc \|\| bo->domain == bo->presumed_offset)) {
		1497	DBG(("%s: converting request for GTT map into CPU map\n",
		1498	__FUNCTION__));
		1499	ptr = kgem_bo_map__cpu(kgem, bo);
		1500	// kgem_bo_sync__cpu(kgem, bo);
		1501	return ptr;
		1502	// }
		1503
		1504	#if 0
		1505
		1506	if (IS_CPU_MAP(bo->map))
		1507	kgem_bo_release_map(kgem, bo);
		1508
		1509	ptr = bo->map;
		1510	if (ptr == NULL) {
		1511	assert(bytes(bo) <= kgem->aperture_mappable / 4);
		1512
		1513	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		1514
		1515	ptr = gem_mmap(kgem->fd, bo->handle, bytes(bo),
		1516	PROT_READ \| PROT_WRITE);
		1517	if (ptr == NULL)
		1518	return NULL;
		1519
		1520	/* Cache this mapping to avoid the overhead of an
		1521	* excruciatingly slow GTT pagefault. This is more an
		1522	* issue with compositing managers which need to frequently
		1523	* flush CPU damage to their GPU bo.
		1524	*/
		1525	bo->map = ptr;
		1526	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		1527	}
		1528
		1529	if (bo->domain != DOMAIN_GTT) {
		1530	struct drm_i915_gem_set_domain set_domain;
		1531
		1532	DBG(("%s: sync: needs_flush? %d, domain? %d\n", __FUNCTION__,
		1533	bo->needs_flush, bo->domain));
		1534
		1535	/* XXX use PROT_READ to avoid the write flush? */
		1536
		1537	VG_CLEAR(set_domain);
		1538	set_domain.handle = bo->handle;
		1539	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		1540	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		1541	drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
		1542
		1543	kgem_bo_retire(kgem, bo);
		1544	bo->domain = DOMAIN_GTT;
		1545	}
		1546	#endif
		1547
		1548	return ptr;
		1549	}
		1550
		1551	void kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		1552	{
		1553	// struct drm_i915_gem_mmap mmap_arg;
		1554
		1555	DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__, bo->handle, bytes(bo)));
		1556	assert(!bo->purged);
		1557	assert(list_is_empty(&bo->list));
		1558
		1559	if (IS_CPU_MAP(bo->map))
		1560	return CPU_MAP(bo->map);
		1561
		1562	struct drm_i915_gem_object obj = (void)bo->handle;
		1563	u8 *dst;
		1564	int ret;
		1565
		1566	if(obj->pin_count == 0)
		1567	{
		1568	ret = i915_gem_object_pin(obj, 4096, true);
		1569	if (ret)
		1570	return NULL;
		1571	};
		1572
		1573	dst = drm_intel_bo_map(obj, true);
		1574	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
		1575	bo->map = MAKE_CPU_MAP(dst);
		1576	return (void *)dst;
		1577
		1578
		1579	#if 0
		1580	if (bo->map)
		1581	kgem_bo_release_map(kgem, bo);
		1582
		1583	kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));
		1584
		1585	VG_CLEAR(mmap_arg);
		1586	mmap_arg.handle = bo->handle;
		1587	mmap_arg.offset = 0;
		1588	mmap_arg.size = bytes(bo);
		1589	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
		1590	ErrorF("%s: failed to mmap %d, %d bytes, into CPU domain\n",
		1591	__FUNCTION__, bo->handle, bytes(bo));
		1592	return NULL;
		1593	}
		1594
		1595	VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, bytes(bo), 0, 1));
		1596	#endif
		1597
		1598	}
		1599
		1600
		1601
		1602
		1603
		1604
		1605
		1606
		1607
		1608
		1609
		1610
		1611
		1612
		1613
		1614
		1615
		1616
		1617
		1618	void kgem_clear_dirty(struct kgem *kgem)
		1619	{
		1620	struct kgem_request *rq = kgem->next_request;
		1621	struct kgem_bo *bo;
		1622
		1623	list_for_each_entry(bo, &rq->buffers, request)
		1624	bo->dirty = false;
		1625	}
		1626
		1627	struct kgem_bo kgem_create_proxy(struct kgem_bo target,
		1628	int offset, int length)
		1629	{
		1630	struct kgem_bo *bo;
		1631
		1632	DBG(("%s: target handle=%d, offset=%d, length=%d, io=%d\n",
		1633	__FUNCTION__, target->handle, offset, length, target->io));
		1634
		1635	bo = __kgem_bo_alloc(target->handle, length);
		1636	if (bo == NULL)
		1637	return NULL;
		1638
		1639	bo->reusable = false;
		1640	bo->size.bytes = length;
		1641
		1642	bo->io = target->io;
		1643	bo->dirty = target->dirty;
		1644	bo->tiling = target->tiling;
		1645	bo->pitch = target->pitch;
		1646
		1647	if (target->proxy) {
		1648	offset += target->delta;
		1649	target = target->proxy;
		1650	}
		1651	bo->proxy = kgem_bo_reference(target);
		1652	bo->delta = offset;
		1653	bo->gaddr = offset + target->gaddr;
		1654	return bo;
		1655	}
		1656
		1657
		1658
		1659
		1660
		1661
		1662
		1663
		1664
		1665
		1666	uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format)
		1667	{
		1668	struct kgem_bo_binding *b;
		1669
		1670	for (b = &bo->binding; b && b->offset; b = b->next)
		1671	if (format == b->format)
		1672	return b->offset;
		1673
		1674	return 0;
		1675	}
		1676
		1677	void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset)
		1678	{
		1679	struct kgem_bo_binding *b;
		1680
		1681	for (b = &bo->binding; b; b = b->next) {
		1682	if (b->offset)
		1683	continue;
		1684
		1685	b->offset = offset;
		1686	b->format = format;
		1687
		1688	if (b->next)
		1689	b->next->offset = 0;
		1690
		1691	return;
		1692	}
		1693
		1694	b = malloc(sizeof(*b));
		1695	if (b) {
		1696	b->next = bo->binding.next;
		1697	b->format = format;
		1698	b->offset = offset;
		1699	bo->binding.next = b;
		1700	}
		1701	}
		1702
		1703
		1704	struct kgem_bo create_bo(bitmap_t bitmap)
		1705	{
		1706	struct kgem_bo *bo;
		1707
		1708	bo = __kgem_bo_alloc(bitmap->obj, 10247684/4096);
		1709	bo->gaddr = bitmap->gaddr;
		1710	bo->pitch = bitmap->pitch;
		1711	bo->tiling = 0;
		1712	return bo;
		1713
		1714	};

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/sna/kgem.c – Rev 2351