WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/sna/kgem.c

Rev	Author	Line No.	Line
2351	Serge	1	/*
		2	* Copyright (c) 2011 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
		20	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		21	* SOFTWARE.
		22	*
		23	* Authors:
		24	* Chris Wilson
		25	*
		26	*/
		27
		28
		29	#ifdef HAVE_CONFIG_H
		30	#include "config.h"
		31	#endif
		32
		33	#include
		34	#include
		35	#include "i915_drm.h"
		36	#include "i915_drv.h"
		37	#include "intel_drv.h"
		38
		39	#include
		40	#include "../bitmap.h"
		41	#include "sna.h"
		42	//#include "sna_reg.h"
		43	//#include
		44	//#include
		45
		46	#define NO_CACHE 1
		47
		48	#define list_is_empty list_empty
		49	#define list_init INIT_LIST_HEAD
		50
		51	extern struct drm_device *main_device;
		52
		53	static struct kgem_bo *
		54	search_linear_cache(struct kgem *kgem, unsigned int num_pages,
		55	unsigned flags);
		56
		57	#define INT16_MAX (32767)
		58
		59	#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
		60	#define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE)
		61
		62	#define MAX_GTT_VMA_CACHE 512
		63	#define MAX_CPU_VMA_CACHE INT16_MAX
		64	#define MAP_PRESERVE_TIME 10
		65
		66	#define CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) & ~1))
		67	#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) \| 1))
		68
		69	struct kgem_partial_bo {
		70	struct kgem_bo base;
		71	void *mem;
		72	uint32_t used;
		73	uint32_t need_io : 1;
		74	uint32_t write : 2;
		75	uint32_t mmapped : 1;
		76	};
		77
		78	static struct kgem_bo *__kgem_freed_bo;
		79	static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;
		80
		81	static inline int bytes(struct kgem_bo *bo)
		82	{
		83	return kgem_bo_size(bo);
		84	}
		85
		86	#define bucket(B) (B)->size.pages.bucket
		87	#define num_pages(B) (B)->size.pages.count
		88
		89	static void kgem_sna_reset(struct kgem *kgem)
		90	{
		91	struct sna *sna = container_of(kgem, struct sna, kgem);
		92
		93	sna->render.reset(sna);
		94	sna->blt_state.fill_bo = 0;
		95	}
		96
		97	static void kgem_sna_flush(struct kgem *kgem)
		98	{
		99	struct sna *sna = container_of(kgem, struct sna, kgem);
		100
		101	sna->render.flush(sna);
		102
		103	if (sna->render.solid_cache.dirty)
		104	sna_render_flush_solid(sna);
		105	}
		106
		107	static int __gem_write(int fd, uint32_t handle,
		108	int offset, int length,
		109	const void *src)
		110	{
		111	DBG(("%s(handle=%x, offset=%d, len=%d)\n", __FUNCTION__,
		112	handle, offset, length));
		113
		114	write_gem_object(handle, offset, length, src);
		115	return 0;
		116	}
		117
		118
		119	static int gem_write(int fd, uint32_t handle,
		120	int offset, int length,
		121	const void *src)
		122	{
		123	u32 _offset;
		124	u32 _size;
		125	u8 *data_ptr;
		126
		127	DBG(("%s(handle=%x, offset=%d, len=%d)\n", __FUNCTION__,
		128	handle, offset, length));
		129
		130	/* align the transfer to cachelines; fortuitously this is safe! */
		131	if ((offset \| length) & 63) {
		132	_offset = offset & ~63;
		133	_size = ALIGN(offset+length, 64) - _offset;
		134	data_ptr = (u8*)src + _offset - offset;
		135	} else {
		136	_offset = offset;
		137	_size = length;
		138	data_ptr = (u8*)src;
		139	}
		140
		141	write_gem_object(handle, _offset, _size, data_ptr);
		142	return 0;
		143	}
		144
		145	static void kgem_bo_retire(struct kgem kgem, struct kgem_bo bo)
		146	{
		147	DBG(("%s: handle=%x, domain=%d\n",
		148	__FUNCTION__, bo->handle, bo->domain));
		149	assert(!kgem_busy(kgem, bo->handle));
		150
		151	if (bo->domain == DOMAIN_GPU)
		152	kgem_retire(kgem);
		153
		154	if (bo->exec == NULL) {
		155	DBG(("%s: retiring bo handle=%x (needed flush? %d), rq? %d\n",
		156	__FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL));
		157	bo->rq = NULL;
		158	list_del(&bo->request);
		159	bo->needs_flush = bo->flush;
		160	}
		161	}
		162
		163	Bool kgem_bo_write(struct kgem kgem, struct kgem_bo bo,
		164	const void *data, int length)
		165	{
		166	assert(bo->refcnt);
		167	assert(!bo->purged);
		168	assert(!kgem_busy(kgem, bo->handle));
		169
		170	assert(length <= bytes(bo));
		171	if (gem_write(kgem->fd, bo->handle, 0, length, data))
		172	return FALSE;
		173
		174	DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain));
		175	kgem_bo_retire(kgem, bo);
		176	bo->domain = DOMAIN_NONE;
		177	return TRUE;
		178	}
		179
		180	static uint32_t gem_create(int fd, int num_pages)
		181	{
		182	struct drm_i915_gem_object *obj;
		183	int ret;
		184
		185	/* Allocate the new object */
		186	obj = i915_gem_alloc_object(main_device,
		187	PAGE_SIZE * num_pages);
		188	if (obj == NULL)
		189	goto err1;
		190
		191	ret = i915_gem_object_pin(obj, 4096, true);
		192	if (ret)
		193	goto err2;
		194
		195	return (uint32_t)obj;
		196
		197	err2:
		198	drm_gem_object_unreference(&obj->base);
		199	err1:
		200	return 0;
		201	}
		202
		203	static bool
		204	kgem_bo_set_purgeable(struct kgem kgem, struct kgem_bo bo)
		205	{
		206	return true;
		207	}
		208
		209
		210	static bool
		211	kgem_bo_clear_purgeable(struct kgem kgem, struct kgem_bo bo)
		212	{
		213	return true;
		214	}
		215
		216	static void gem_close(int fd, uint32_t handle)
		217	{
		218	destroy_gem_object(handle);
		219	}
		220
		221
		222	/*
		223	constant inline static unsigned long __fls(unsigned long word)
		224	{
		225	asm("bsr %1,%0"
		226	: "=r" (word)
		227	: "rm" (word));
		228	return word;
		229	}
		230	*/
		231
		232	constant inline static int cache_bucket(int num_pages)
		233	{
		234	return __fls(num_pages);
		235	}
		236
		237	static struct kgem_bo __kgem_bo_init(struct kgem_bo bo,
		238	int handle, int num_pages)
		239	{
		240	assert(num_pages);
		241	memset(bo, 0, sizeof(*bo));
		242
		243	bo->refcnt = 1;
		244	bo->handle = handle;
		245	num_pages(bo) = num_pages;
		246	bucket(bo) = cache_bucket(num_pages);
		247	bo->reusable = true;
		248	bo->domain = DOMAIN_CPU;
		249	list_init(&bo->request);
		250	list_init(&bo->list);
		251	list_init(&bo->vma);
		252
		253	return bo;
		254	}
		255
		256	static struct kgem_bo *__kgem_bo_alloc(int handle, int num_pages)
		257	{
		258	struct kgem_bo *bo;
		259
		260	if (__kgem_freed_bo) {
		261	bo = __kgem_freed_bo;
		262	__kgem_freed_bo = (struct kgem_bo *)bo;
		263	} else {
		264	bo = malloc(sizeof(*bo));
		265	if (bo == NULL)
		266	return NULL;
		267	}
		268
		269	return __kgem_bo_init(bo, handle, num_pages);
		270	}
		271
		272	static struct kgem_request _kgem_static_request;
		273
		274	static struct kgem_request *__kgem_request_alloc(void)
		275	{
		276	struct kgem_request *rq;
		277
		278	rq = malloc(sizeof(*rq));
		279	if (rq == NULL)
		280	rq = &_kgem_static_request;
		281
		282	list_init(&rq->buffers);
		283
		284	return rq;
		285	}
		286
		287	static struct list_head inactive(struct kgem kgem, int num_pages)
		288	{
		289	return &kgem->inactive[cache_bucket(num_pages)];
		290	}
		291
		292	static struct list_head active(struct kgem kgem, int num_pages, int tiling)
		293	{
		294	return &kgem->active[cache_bucket(num_pages)][tiling];
		295	}
		296
		297
		298
		299	void kgem_init(struct kgem *kgem, int gen)
		300	{
		301	struct drm_i915_gem_get_aperture aperture;
		302	struct drm_i915_gem_object *obj;
		303
		304	size_t totalram;
		305	unsigned int i, j;
		306	int ret;
		307
		308	memset(kgem, 0, sizeof(*kgem));
		309
		310	kgem->gen = gen;
		311	kgem->wedged = 0;
		312	// kgem->wedged \|= DBG_NO_HW;
		313
		314	obj = i915_gem_alloc_object(main_device, 4096*4);
		315	if (obj == NULL)
		316	goto err2;
		317
		318	ret = i915_gem_object_pin(obj, 4096, true);
		319	if (ret)
		320	goto err3;
		321
		322	kgem->batch_ptr = drm_intel_bo_map(obj, true);
		323	kgem->batch = kgem->batch_ptr;
		324	kgem->batch_idx = 0;
		325	kgem->batch_obj = obj;
		326
		327	kgem->max_batch_size = 1024; //ARRAY_SIZE(kgem->batch);
		328
		329	kgem->half_cpu_cache_pages = (2048*1024) >> 13;
		330
		331	list_init(&kgem->partial);
		332	list_init(&kgem->requests);
		333	list_init(&kgem->flushing);
		334	list_init(&kgem->large);
		335	for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
		336	list_init(&kgem->inactive[i]);
		337	for (i = 0; i < ARRAY_SIZE(kgem->active); i++) {
		338	for (j = 0; j < ARRAY_SIZE(kgem->active[i]); j++)
		339	list_init(&kgem->active[i][j]);
		340	}
		341	for (i = 0; i < ARRAY_SIZE(kgem->vma); i++) {
		342	for (j = 0; j < ARRAY_SIZE(kgem->vma[i].inactive); j++)
		343	list_init(&kgem->vma[i].inactive[j]);
		344	}
		345	kgem->vma[MAP_GTT].count = -MAX_GTT_VMA_CACHE;
		346	kgem->vma[MAP_CPU].count = -MAX_CPU_VMA_CACHE;
		347
		348	kgem->next_request = __kgem_request_alloc();
		349
		350	//#if defined(USE_VMAP) && defined(I915_PARAM_HAS_VMAP)
		351	// if (!DBG_NO_VMAP)
		352	// kgem->has_vmap = gem_param(kgem, I915_PARAM_HAS_VMAP) > 0;
		353	//#endif
		354	// DBG(("%s: using vmap=%d\n", __FUNCTION__, kgem->has_vmap));
		355
		356	if (gen < 40) {
		357	// if (!DBG_NO_RELAXED_FENCING) {
		358	// kgem->has_relaxed_fencing =
		359	// gem_param(kgem, I915_PARAM_HAS_RELAXED_FENCING) > 0;
		360	// }
		361	} else
		362	kgem->has_relaxed_fencing = 1;
		363	DBG(("%s: has relaxed fencing? %d\n", __FUNCTION__,
		364	kgem->has_relaxed_fencing));
		365
		366	kgem->has_llc = (gen >= 60)?true:false;
		367	kgem->has_cpu_bo = kgem->has_llc;
		368	DBG(("%s: cpu bo enabled %d: llc? %d\n", __FUNCTION__,
		369	kgem->has_cpu_bo, kgem->has_llc));
		370
		371	kgem->has_semaphores = false;
		372	// if (gen >= 60 && semaphores_enabled())
		373	// kgem->has_semaphores = true;
		374	// DBG(("%s: semaphores enabled? %d\n", __FUNCTION__,
		375	// kgem->has_semaphores));
		376
		377	VG_CLEAR(aperture);
		378	aperture.aper_size = 6410241024;
		379	i915_gem_get_aperture_ioctl(main_device, &aperture, NULL);
		380	kgem->aperture_total = aperture.aper_size;
		381	kgem->aperture_high = aperture.aper_size * 3/4;
		382	kgem->aperture_low = aperture.aper_size * 1/3;
		383	DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
		384	kgem->aperture_low, kgem->aperture_low / (1024*1024),
		385	kgem->aperture_high, kgem->aperture_high / (1024*1024)));
		386
		387	kgem->aperture_mappable = aperture.aper_size;
		388	DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
		389	kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));
		390
		391	kgem->partial_buffer_size = 64 * 1024;
		392	while (kgem->partial_buffer_size < kgem->aperture_mappable >> 10)
		393	kgem->partial_buffer_size *= 2;
		394	DBG(("%s: partial buffer size=%d [%d KiB]\n", __FUNCTION__,
		395	kgem->partial_buffer_size, kgem->partial_buffer_size / 1024));
		396
		397	kgem->min_alignment = 4;
		398	if (gen < 60)
		399	/* XXX workaround an issue where we appear to fail to
		400	* disable dual-stream mode */
		401	kgem->min_alignment = 64;
		402
		403	kgem->max_object_size = 2 * kgem->aperture_total / 3;
		404	kgem->max_cpu_size = kgem->max_object_size;
		405	kgem->max_gpu_size = kgem->max_object_size;
		406	if (!kgem->has_llc)
		407	kgem->max_gpu_size = MAX_CACHE_SIZE;
		408	if (gen < 40) {
		409	/* If we have to use fences for blitting, we have to make
		410	* sure we can fit them into the aperture.
		411	*/
		412	kgem->max_gpu_size = kgem->aperture_mappable / 2;
		413	if (kgem->max_gpu_size > kgem->aperture_low)
		414	kgem->max_gpu_size = kgem->aperture_low;
		415	}
		416	if (kgem->max_gpu_size > kgem->max_cpu_size)
		417	kgem->max_gpu_size = kgem->max_cpu_size;
		418
		419	kgem->max_upload_tile_size = kgem->aperture_mappable / 2;
		420	if (kgem->max_upload_tile_size > kgem->max_gpu_size / 2)
		421	kgem->max_upload_tile_size = kgem->max_gpu_size / 2;
		422
		423	kgem->max_copy_tile_size = (MAX_CACHE_SIZE + 1)/2;
		424	if (kgem->max_copy_tile_size > kgem->max_gpu_size / 2)
		425	kgem->max_copy_tile_size = kgem->max_gpu_size / 2;
		426
		427	totalram = 1024*1024; //total_ram_size();
		428	if (totalram == 0) {
		429	DBG(("%s: total ram size unknown, assuming maximum of total aperture\n",
		430	__FUNCTION__));
		431	totalram = kgem->aperture_total;
		432	}
		433	if (kgem->max_object_size > totalram / 2)
		434	kgem->max_object_size = totalram / 2;
		435	if (kgem->max_cpu_size > totalram / 2)
		436	kgem->max_cpu_size = totalram / 2;
		437	if (kgem->max_gpu_size > totalram / 4)
		438	kgem->max_gpu_size = totalram / 4;
		439
		440	kgem->large_object_size = MAX_CACHE_SIZE;
		441	if (kgem->large_object_size > kgem->max_gpu_size)
		442	kgem->large_object_size = kgem->max_gpu_size;
		443
		444	DBG(("%s: large object thresold=%d\n",
		445	__FUNCTION__, kgem->large_object_size));
		446	DBG(("%s: max object size (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
		447	__FUNCTION__,
		448	kgem->max_gpu_size, kgem->max_cpu_size,
		449	kgem->max_upload_tile_size, kgem->max_copy_tile_size));
		450
		451	/* Convert the aperture thresholds to pages */
		452	kgem->aperture_low /= PAGE_SIZE;
		453	kgem->aperture_high /= PAGE_SIZE;
		454
		455	// kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
		456	// if ((int)kgem->fence_max < 0)
		457	kgem->fence_max = 5; /* minimum safe value for all hw */
		458	DBG(("%s: max fences=%d\n", __FUNCTION__, kgem->fence_max));
		459	err3:
		460	err2:
		461	return;
		462	}
		463
		464	static struct drm_i915_gem_exec_object2 *
		465	kgem_add_handle(struct kgem kgem, struct kgem_bo bo)
		466	{
		467	struct drm_i915_gem_exec_object2 *exec;
		468
		469	DBG(("%s: handle=%d, index=%d\n",
		470	__FUNCTION__, bo->handle, kgem->nexec));
		471
		472	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		473	exec = memset(&kgem->exec[kgem->nexec++], 0, sizeof(*exec));
		474	exec->handle = bo->handle;
		475	exec->offset = bo->presumed_offset;
		476
		477	kgem->aperture += num_pages(bo);
		478
		479	return exec;
		480	}
		481
		482	void _kgem_add_bo(struct kgem kgem, struct kgem_bo bo)
		483	{
		484	bo->exec = kgem_add_handle(kgem, bo);
		485	bo->rq = kgem->next_request;
		486
		487	list_move(&bo->request, &kgem->next_request->buffers);
		488
		489	/* XXX is it worth working around gcc here? */
		490	kgem->flush \|= bo->flush;
		491	kgem->sync \|= bo->sync;
		492	kgem->scanout \|= bo->scanout;
		493	}
		494
		495	static uint32_t kgem_end_batch(struct kgem *kgem)
		496	{
		497	// kgem->context_switch(kgem, KGEM_NONE);
		498
		499	kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
		500	if (kgem->nbatch & 1)
		501	kgem->batch[kgem->nbatch++] = MI_NOOP;
		502
		503	return kgem->nbatch;
		504	}
		505
		506	static void kgem_fixup_self_relocs(struct kgem kgem, struct kgem_bo bo)
		507	{
		508	int n;
		509
		510	for (n = 0; n < kgem->nreloc; n++)
		511	{
		512	if (kgem->reloc[n].target_handle == 0)
		513	{
		514	kgem->reloc[n].target_handle = bo->handle;
		515	kgem->reloc[n].presumed_offset = bo->presumed_offset;
		516	kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
		517	kgem->reloc[n].delta + bo->presumed_offset;
		518
		519	dbgprintf("fixup reloc %d pos %d handle %d delta %x \n",
		520	n, kgem->reloc[n].offset/sizeof(kgem->batch[0]),
		521	bo->handle, kgem->reloc[n].delta);
		522	}
		523	}
		524	}
		525
		526	static void kgem_bo_binding_free(struct kgem kgem, struct kgem_bo bo)
		527	{
		528	struct kgem_bo_binding *b;
		529
		530	b = bo->binding.next;
		531	while (b) {
		532	struct kgem_bo_binding *next = b->next;
		533	free (b);
		534	b = next;
		535	}
		536	}
		537
		538	static void kgem_bo_release_map(struct kgem kgem, struct kgem_bo bo)
		539	{
		540	int type = IS_CPU_MAP(bo->map);
		541
		542	DBG(("%s: releasing %s vma for handle=%d, count=%d\n",
		543	__FUNCTION__, type ? "CPU" : "GTT",
		544	bo->handle, kgem->vma[type].count));
		545
		546	VG(if (type) VALGRIND_FREELIKE_BLOCK(CPU_MAP(bo->map), 0));
		547	// munmap(CPU_MAP(bo->map), bytes(bo));
		548	bo->map = NULL;
		549
		550	if (!list_is_empty(&bo->vma)) {
		551	list_del(&bo->vma);
		552	kgem->vma[type].count--;
		553	}
		554	}
		555
		556	static void kgem_bo_free(struct kgem kgem, struct kgem_bo bo)
		557	{
		558	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		559	assert(bo->refcnt == 0);
		560	assert(bo->exec == NULL);
		561
		562	kgem_bo_binding_free(kgem, bo);
		563
		564	if (bo->map)
		565	kgem_bo_release_map(kgem, bo);
		566	assert(list_is_empty(&bo->vma));
		567
		568	list_del(&bo->list);
		569	list_del(&bo->request);
		570	gem_close(kgem->fd, bo->handle);
		571
		572	if (!bo->io) {
		573	(struct kgem_bo *)bo = __kgem_freed_bo;
		574	__kgem_freed_bo = bo;
		575	} else
		576	free(bo);
		577	}
		578
		579	inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
		580	struct kgem_bo *bo)
		581	{
		582	assert(!kgem_busy(kgem, bo->handle));
		583	assert(!bo->proxy);
		584	assert(!bo->io);
		585	assert(!bo->needs_flush);
		586	assert(bo->rq == NULL);
		587	assert(bo->domain != DOMAIN_GPU);
		588
		589	if (bucket(bo) >= NUM_CACHE_BUCKETS) {
		590	kgem_bo_free(kgem, bo);
		591	return;
		592	}
		593
		594	list_move(&bo->list, &kgem->inactive[bucket(bo)]);
		595	if (bo->map) {
		596	int type = IS_CPU_MAP(bo->map);
		597	if (bucket(bo) >= NUM_CACHE_BUCKETS \|\|
		598	(!type && !kgem_bo_is_mappable(kgem, bo))) {
		599	list_del(&bo->vma);
		600	// munmap(CPU_MAP(bo->map), bytes(bo));
		601	bo->map = NULL;
		602	}
		603	if (bo->map) {
		604	list_move(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]);
		605	kgem->vma[type].count++;
		606	}
		607	}
		608
		609	kgem->need_expire = true;
		610	}
		611
		612	inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
		613	struct kgem_bo *bo)
		614	{
		615	list_del(&bo->list);
		616	assert(bo->rq == NULL);
		617	if (bo->map) {
		618	assert(!list_is_empty(&bo->vma));
		619	list_del(&bo->vma);
		620	kgem->vma[IS_CPU_MAP(bo->map)].count--;
		621	}
		622	}
		623
		624	inline static void kgem_bo_remove_from_active(struct kgem *kgem,
		625	struct kgem_bo *bo)
		626	{
		627	list_del(&bo->list);
		628	if (bo->rq == &_kgem_static_request)
		629	list_del(&bo->request);
		630	assert(list_is_empty(&bo->vma));
		631	}
		632
		633	static void __kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		634	{
		635	DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
		636
		637	assert(list_is_empty(&bo->list));
		638	assert(bo->refcnt == 0);
		639
		640	bo->binding.offset = 0;
		641
		642	if (NO_CACHE)
		643	goto destroy;
		644
		645	if (bo->io) {
		646	struct kgem_bo *base;
		647
		648	base = malloc(sizeof(*base));
		649	if (base) {
		650	DBG(("%s: transferring io handle=%d to bo\n",
		651	__FUNCTION__, bo->handle));
		652	/* transfer the handle to a minimum bo */
		653	memcpy(base, bo, sizeof (*base));
		654	base->reusable = true;
		655	base->io = false;
		656	list_init(&base->list);
		657	list_replace(&bo->request, &base->request);
		658	list_replace(&bo->vma, &base->vma);
		659	free(bo);
		660	bo = base;
		661	}
		662	}
		663
		664	if (!bo->reusable) {
		665	DBG(("%s: handle=%d, not reusable\n",
		666	__FUNCTION__, bo->handle));
		667	goto destroy;
		668	}
		669
		670	if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU)
		671	kgem_bo_release_map(kgem, bo);
		672
		673	assert(list_is_empty(&bo->vma));
		674	assert(list_is_empty(&bo->list));
		675	assert(bo->vmap == false && bo->sync == false);
		676	assert(bo->io == false);
		677
		678	bo->scanout = bo->flush = false;
		679	if (bo->rq) {
		680	struct list *cache;
		681
		682	DBG(("%s: handle=%d -> active\n", __FUNCTION__, bo->handle));
		683	if (bucket(bo) < NUM_CACHE_BUCKETS)
		684	cache = &kgem->active[bucket(bo)][bo->tiling];
		685	else
		686	cache = &kgem->large;
		687	list_add(&bo->list, cache);
		688	return;
		689	}
		690
		691	assert(bo->exec == NULL);
		692	assert(list_is_empty(&bo->request));
		693	/*
		694	if (bo->needs_flush) {
		695	if ((bo->needs_flush = kgem_busy(kgem, bo->handle))) {
		696	struct list *cache;
		697
		698	DBG(("%s: handle=%d -> flushing\n",
		699	__FUNCTION__, bo->handle));
		700
		701	list_add(&bo->request, &kgem->flushing);
		702	if (bucket(bo) < NUM_CACHE_BUCKETS)
		703	cache = &kgem->active[bucket(bo)][bo->tiling];
		704	else
		705	cache = &kgem->large;
		706	list_add(&bo->list, cache);
		707	bo->rq = &_kgem_static_request;
		708	return;
		709	}
		710
		711	bo->domain = DOMAIN_NONE;
		712	}
		713	*/
		714	if (!IS_CPU_MAP(bo->map)) {
		715	if (!kgem_bo_set_purgeable(kgem, bo))
		716	goto destroy;
		717
		718	if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
		719	goto destroy;
		720
		721	DBG(("%s: handle=%d, purged\n",
		722	__FUNCTION__, bo->handle));
		723	}
		724
		725	DBG(("%s: handle=%d -> inactive\n", __FUNCTION__, bo->handle));
		726	kgem_bo_move_to_inactive(kgem, bo);
		727	return;
		728
		729	destroy:
		730	if (!bo->exec)
		731	kgem_bo_free(kgem, bo);
		732	}
		733
		734
		735
		736
		737
		738	bool kgem_retire(struct kgem *kgem)
		739	{
		740	struct kgem_bo bo, next;
		741	bool retired = false;
		742
		743	DBG(("%s\n", __FUNCTION__));
		744
		745	list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
		746	assert(bo->refcnt == 0);
		747	assert(bo->rq == &_kgem_static_request);
		748	assert(bo->exec == NULL);
		749
		750	// if (kgem_busy(kgem, bo->handle))
		751	// break;
		752
		753	DBG(("%s: moving %d from flush to inactive\n",
		754	__FUNCTION__, bo->handle));
		755	if (kgem_bo_set_purgeable(kgem, bo)) {
		756	bo->needs_flush = false;
		757	bo->domain = DOMAIN_NONE;
		758	bo->rq = NULL;
		759	list_del(&bo->request);
		760	kgem_bo_move_to_inactive(kgem, bo);
		761	} else
		762	kgem_bo_free(kgem, bo);
		763
		764	retired = true;
		765	}
		766
		767	while (!list_is_empty(&kgem->requests)) {
		768	struct kgem_request *rq;
		769
		770	rq = list_first_entry(&kgem->requests,
		771	struct kgem_request,
		772	list);
		773	// if (kgem_busy(kgem, rq->bo->handle))
		774	// break;
		775
		776	DBG(("%s: request %d complete\n",
		777	__FUNCTION__, rq->bo->handle));
		778
		779	while (!list_is_empty(&rq->buffers)) {
		780	bo = list_first_entry(&rq->buffers,
		781	struct kgem_bo,
		782	request);
		783
		784	assert(bo->rq == rq);
		785	assert(bo->exec == NULL);
		786	assert(bo->domain == DOMAIN_GPU);
		787
		788	list_del(&bo->request);
		789	bo->rq = NULL;
		790
		791	// if (bo->needs_flush)
		792	// bo->needs_flush = kgem_busy(kgem, bo->handle);
		793	if (!bo->needs_flush)
		794	bo->domain = DOMAIN_NONE;
		795
		796	if (bo->refcnt)
		797	continue;
		798
		799	if (!bo->reusable) {
		800	DBG(("%s: closing %d\n",
		801	__FUNCTION__, bo->handle));
		802	kgem_bo_free(kgem, bo);
		803	continue;
		804	}
		805
		806	if (bo->needs_flush) {
		807	DBG(("%s: moving %d to flushing\n",
		808	__FUNCTION__, bo->handle));
		809	list_add(&bo->request, &kgem->flushing);
		810	bo->rq = &_kgem_static_request;
		811	} else if (kgem_bo_set_purgeable(kgem, bo)) {
		812	DBG(("%s: moving %d to inactive\n",
		813	__FUNCTION__, bo->handle));
		814	kgem_bo_move_to_inactive(kgem, bo);
		815	retired = true;
		816	} else {
		817	DBG(("%s: closing %d\n",
		818	__FUNCTION__, bo->handle));
		819	kgem_bo_free(kgem, bo);
		820	}
		821	}
		822
		823	rq->bo->refcnt--;
		824	assert(rq->bo->refcnt == 0);
		825	assert(rq->bo->rq == NULL);
		826	assert(list_is_empty(&rq->bo->request));
		827	if (kgem_bo_set_purgeable(kgem, rq->bo)) {
		828	kgem_bo_move_to_inactive(kgem, rq->bo);
		829	retired = true;
		830	} else {
		831	DBG(("%s: closing %d\n",
		832	__FUNCTION__, rq->bo->handle));
		833	kgem_bo_free(kgem, rq->bo);
		834	}
		835
		836	list_del(&rq->list);
		837	free(rq);
		838	}
		839
		840	kgem->need_retire = !list_is_empty(&kgem->requests);
		841	DBG(("%s -- need_retire=%d\n", __FUNCTION__, kgem->need_retire));
		842
		843	kgem->retire(kgem);
		844
		845	return retired;
		846	}
		847
		848
		849
		850
		851
		852
		853
		854
		855
		856
		857
		858
		859
		860
		861
		862
		863
		864
		865
		866
		867
		868
		869
		870
		871
		872
		873
		874
		875
		876
		877
		878
		879
		880
		881
		882
		883
		884
		885
		886
		887
		888	static int kgem_batch_write(struct kgem *kgem, uint32_t handle, uint32_t size)
		889	{
		890	int ret;
		891
		892	assert(!kgem_busy(kgem, handle));
		893
		894	/* If there is no surface data, just upload the batch */
		895	if (kgem->surface == kgem->max_batch_size)
		896	return gem_write(kgem->fd, handle,
		897	0, sizeof(uint32_t)*kgem->nbatch,
		898	kgem->batch);
		899
		900	/* Are the batch pages conjoint with the surface pages? */
		901	if (kgem->surface < kgem->nbatch + PAGE_SIZE/4) {
		902	assert(size == sizeof(kgem->batch));
		903	return gem_write(kgem->fd, handle,
		904	0, sizeof(kgem->batch),
		905	kgem->batch);
		906	}
		907
		908	/* Disjoint surface/batch, upload separately */
		909	ret = gem_write(kgem->fd, handle,
		910	0, sizeof(uint32_t)*kgem->nbatch,
		911	kgem->batch);
		912	if (ret)
		913	return ret;
		914
		915	assert(kgem->nbatch*sizeof(uint32_t) <=
		916	sizeof(uint32_t)*kgem->surface - (sizeof(kgem->batch)-size));
		917	return __gem_write(kgem->fd, handle,
		918	sizeof(uint32_t)*kgem->surface - (sizeof(kgem->batch)-size),
		919	sizeof(kgem->batch) - sizeof(uint32_t)*kgem->surface,
		920	kgem->batch + kgem->surface);
		921	}
		922
		923	void kgem_reset(struct kgem *kgem)
		924	{
		925	// ENTER();
		926
		927	kgem->nfence = 0;
		928	kgem->nexec = 0;
		929	kgem->nreloc = 0;
		930	kgem->aperture = 0;
		931	kgem->aperture_fenced = 0;
		932	kgem->nbatch = 0;
		933	kgem->surface = kgem->max_batch_size;
		934	kgem->mode = KGEM_NONE;
		935	kgem->flush = 0;
		936	kgem->scanout = 0;
		937
		938	kgem->batch = kgem->batch_ptr+1024*kgem->batch_idx;
		939
		940	kgem->next_request = __kgem_request_alloc();
		941
		942	kgem_sna_reset(kgem);
		943	// dbgprintf("surface %x\n", kgem->surface);
		944	// LEAVE();
		945	}
		946
		947	static int compact_batch_surface(struct kgem *kgem)
		948	{
		949	int size, shrink, n;
		950
		951	/* See if we can pack the contents into one or two pages */
		952	size = kgem->max_batch_size - kgem->surface + kgem->nbatch;
		953	if (size > 2048)
		954	return sizeof(kgem->batch);
		955	else if (size > 1024)
		956	size = 8192, shrink = 2*4096;
		957	else
		958	size = 4096, shrink = 3*4096;
		959
		960
		961	for (n = 0; n < kgem->nreloc; n++) {
		962	if (kgem->reloc[n].read_domains == I915_GEM_DOMAIN_INSTRUCTION &&
		963	kgem->reloc[n].target_handle == 0)
		964	kgem->reloc[n].delta -= shrink;
		965
		966	if (kgem->reloc[n].offset >= size)
		967	kgem->reloc[n].offset -= shrink;
		968	}
		969
		970	return size;
		971	}
		972
2352	Serge	973	int exec_batch(struct drm_device dev, struct intel_ring_buffer ring,
		974	batchbuffer_t *exec);
2351	Serge	975
2352	Serge	976	void _kgem_submit(struct kgem kgem, batchbuffer_t exb)
2351	Serge	977	{
		978	struct kgem_request *rq;
		979	uint32_t batch_end;
		980	int size;
		981
		982	assert(!DBG_NO_HW);
		983
		984	assert(kgem->nbatch);
		985	assert(kgem->nbatch <= KGEM_BATCH_SIZE(kgem));
		986	assert(kgem->nbatch <= kgem->surface);
		987
		988	batch_end = kgem_end_batch(kgem);
		989	kgem_sna_flush(kgem);
		990
		991	DBG(("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n",
		992	kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
		993	kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture));
		994
		995	assert(kgem->nbatch <= kgem->max_batch_size);
		996	assert(kgem->nbatch <= kgem->surface);
		997	assert(kgem->nreloc <= ARRAY_SIZE(kgem->reloc));
		998	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
		999	assert(kgem->nfence <= kgem->fence_max);
		1000
		1001	// kgem_finish_partials(kgem);
		1002
		1003	rq = kgem->next_request;
		1004	// if (kgem->surface != kgem->max_batch_size)
		1005	// size = compact_batch_surface(kgem);
		1006	// else
		1007	size = kgem->nbatch * sizeof(kgem->batch[0]);
		1008	#if 0
		1009	{
		1010	int i;
		1011
		1012	dbgprintf("\nDump batch\n\n");
		1013
		1014	for(i=0; i < kgem->nbatch; i++)
		1015	{
		1016	dbgprintf("\t0x%08x,\t/* %d */\n",
		1017	kgem->batch[i], i);
		1018	}
		1019	dbgprintf("\ndone\n");
		1020	};
		1021	#endif
		1022
2352	Serge	1023	exb->batch = kgem->batch_obj;
		1024	exb->exec_start = kgem->batch_obj->gtt_offset+kgem->batch_idx*4096;
		1025	exb->exec_len = sizeof(uint32_t)*kgem->nbatch;
2351	Serge	1026
2352	Serge	1027	exec_batch(main_device, NULL, exb);
		1028
2351	Serge	1029	// if (kgem->wedged)
		1030	// kgem_cleanup(kgem);
		1031
		1032	kgem->batch_idx++;
		1033	kgem->batch_idx&= 3;
		1034
		1035	kgem->flush_now = kgem->scanout;
		1036	kgem_reset(kgem);
		1037
		1038	assert(kgem->next_request != NULL);
		1039	}
		1040
		1041	static struct kgem_bo *
		1042	search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
		1043	{
		1044	struct kgem_bo bo, first = NULL;
		1045	bool use_active = (flags & CREATE_INACTIVE) == 0;
		1046	struct list_head *cache;
		1047
		1048	if (num_pages >= MAX_CACHE_SIZE / PAGE_SIZE)
		1049	return NULL;
		1050
		1051	if (!use_active &&
		1052	list_is_empty(inactive(kgem, num_pages)) &&
		1053	!list_is_empty(active(kgem, num_pages, I915_TILING_NONE)) &&
		1054	!kgem_retire(kgem))
		1055	return NULL;
		1056
		1057	if (!use_active && flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		1058	int for_cpu = !!(flags & CREATE_CPU_MAP);
		1059	cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
		1060	list_for_each_entry(bo, cache, vma) {
		1061	assert(IS_CPU_MAP(bo->map) == for_cpu);
		1062	assert(bucket(bo) == cache_bucket(num_pages));
		1063
		1064	if (num_pages > num_pages(bo)) {
		1065	DBG(("inactive too small: %d < %d\n",
		1066	num_pages(bo), num_pages));
		1067	continue;
		1068	}
		1069
		1070	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		1071	kgem_bo_free(kgem, bo);
		1072	break;
		1073	}
		1074
		1075	// if (I915_TILING_NONE != bo->tiling &&
		1076	// gem_set_tiling(kgem->fd, bo->handle,
		1077	// I915_TILING_NONE, 0) != I915_TILING_NONE)
		1078	// continue;
		1079
		1080	kgem_bo_remove_from_inactive(kgem, bo);
		1081
		1082	bo->tiling = I915_TILING_NONE;
		1083	bo->pitch = 0;
		1084	bo->delta = 0;
		1085	DBG((" %s: found handle=%d (num_pages=%d) in linear vma cache\n",
		1086	__FUNCTION__, bo->handle, num_pages(bo)));
		1087	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		1088	assert(!bo->needs_flush);
		1089	//assert(!kgem_busy(kgem, bo->handle));
		1090	return bo;
		1091	}
		1092	}
		1093
		1094	cache = use_active ? active(kgem, num_pages, I915_TILING_NONE) : inactive(kgem, num_pages);
		1095	list_for_each_entry(bo, cache, list) {
		1096	assert(bo->refcnt == 0);
		1097	assert(bo->reusable);
		1098	assert(!!bo->rq == !!use_active);
		1099
		1100	if (num_pages > num_pages(bo))
		1101	continue;
		1102
		1103	if (use_active && bo->tiling != I915_TILING_NONE)
		1104	continue;
		1105
		1106	if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
		1107	kgem_bo_free(kgem, bo);
		1108	break;
		1109	}
		1110	/*
		1111	if (I915_TILING_NONE != bo->tiling) {
		1112	if (use_active)
		1113	continue;
		1114
		1115	if (gem_set_tiling(kgem->fd, bo->handle,
		1116	I915_TILING_NONE, 0) != I915_TILING_NONE)
		1117	continue;
		1118
		1119	bo->tiling = I915_TILING_NONE;
		1120	}
		1121	*/
		1122	if (bo->map) {
		1123	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		1124	int for_cpu = !!(flags & CREATE_CPU_MAP);
		1125	if (IS_CPU_MAP(bo->map) != for_cpu) {
		1126	if (first != NULL)
		1127	break;
		1128
		1129	first = bo;
		1130	continue;
		1131	}
		1132	} else {
		1133	if (first != NULL)
		1134	break;
		1135
		1136	first = bo;
		1137	continue;
		1138	}
		1139	} else {
		1140	if (flags & (CREATE_CPU_MAP \| CREATE_GTT_MAP)) {
		1141	if (first != NULL)
		1142	break;
		1143
		1144	first = bo;
		1145	continue;
		1146	}
		1147	}
		1148
		1149	if (use_active)
		1150	kgem_bo_remove_from_active(kgem, bo);
		1151	else
		1152	kgem_bo_remove_from_inactive(kgem, bo);
		1153
		1154	assert(bo->tiling == I915_TILING_NONE);
		1155	bo->pitch = 0;
		1156	bo->delta = 0;
		1157	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		1158	__FUNCTION__, bo->handle, num_pages(bo),
		1159	use_active ? "active" : "inactive"));
		1160	assert(use_active \|\| bo->domain != DOMAIN_GPU);
		1161	assert(!bo->needs_flush \|\| use_active);
		1162	//assert(use_active \|\| !kgem_busy(kgem, bo->handle));
		1163	return bo;
		1164	}
		1165
		1166	if (first) {
		1167	assert(first->tiling == I915_TILING_NONE);
		1168
		1169	if (use_active)
		1170	kgem_bo_remove_from_active(kgem, first);
		1171	else
		1172	kgem_bo_remove_from_inactive(kgem, first);
		1173
		1174	first->pitch = 0;
		1175	first->delta = 0;
		1176	DBG((" %s: found handle=%d (num_pages=%d) in linear %s cache\n",
		1177	__FUNCTION__, first->handle, num_pages(first),
		1178	use_active ? "active" : "inactive"));
		1179	assert(use_active \|\| first->domain != DOMAIN_GPU);
		1180	assert(!first->needs_flush \|\| use_active);
		1181	//assert(use_active \|\| !kgem_busy(kgem, first->handle));
		1182	return first;
		1183	}
		1184
		1185	return NULL;
		1186	}
		1187
		1188
		1189
		1190
		1191
		1192
		1193	struct kgem_bo kgem_create_linear(struct kgem kgem, int size)
		1194	{
		1195	struct kgem_bo *bo;
		1196	uint32_t handle;
		1197
		1198	DBG(("%s(%d)\n", __FUNCTION__, size));
		1199
		1200	size = (size + PAGE_SIZE - 1) / PAGE_SIZE;
		1201	bo = search_linear_cache(kgem, size, CREATE_INACTIVE);
		1202	if (bo)
		1203	return kgem_bo_reference(bo);
		1204
		1205	handle = gem_create(kgem->fd, size);
		1206	if (handle == 0)
		1207	return NULL;
		1208
		1209	DBG(("%s: new handle=%x\n", __FUNCTION__, handle));
		1210	bo = __kgem_bo_alloc(handle, size);
		1211	if (bo == NULL) {
		1212	gem_close(kgem->fd, handle);
		1213	return NULL;
		1214	}
		1215	struct drm_i915_gem_object *obj;
		1216	obj = (void*)handle;
		1217
		1218	bo->gaddr = obj->gtt_offset;
		1219	return bo;
		1220	}
		1221
		1222
		1223
		1224
		1225
		1226
		1227	inline int kgem_bo_fenced_size(struct kgem kgem, struct kgem_bo bo)
		1228	{
		1229	unsigned int size;
		1230
		1231	assert(bo->tiling);
		1232	assert(kgem->gen < 40);
		1233
		1234	if (kgem->gen < 30)
		1235	size = 512 * 1024;
		1236	else
		1237	size = 1024 * 1024;
		1238	while (size < bytes(bo))
		1239	size *= 2;
		1240
		1241	return size;
		1242	}
		1243
		1244
		1245
		1246
		1247
		1248
		1249
		1250
		1251
		1252
		1253
		1254
		1255
		1256
		1257
		1258
		1259
		1260
		1261
		1262
		1263
		1264
		1265
		1266
		1267
		1268
		1269
		1270
		1271
		1272
		1273
		1274
		1275
		1276
		1277
		1278
		1279
		1280
		1281
		1282
		1283
		1284
		1285
		1286
		1287
		1288
		1289
		1290
		1291	void _kgem_bo_destroy(struct kgem kgem, struct kgem_bo bo)
		1292	{
		1293	// if (bo->proxy) {
		1294	// assert(bo->map == NULL);
		1295	// if (bo->io && bo->exec == NULL)
		1296	// _kgem_bo_delete_partial(kgem, bo);
		1297	// kgem_bo_unref(kgem, bo->proxy);
		1298	// kgem_bo_binding_free(kgem, bo);
		1299	// _list_del(&bo->request);
		1300	// free(bo);
		1301	// return;
		1302	// }
		1303
		1304	// if (bo->vmap)
		1305	// kgem_bo_sync__cpu(kgem, bo);
		1306
		1307	__kgem_bo_destroy(kgem, bo);
		1308	}
		1309
		1310	void __kgem_flush(struct kgem kgem, struct kgem_bo bo)
		1311	{
		1312	/* The kernel will emit a flush and update its own flushing lists. */
		1313	// kgem_busy(kgem, bo->handle);
		1314	}
		1315
		1316	bool kgem_check_bo(struct kgem *kgem, ...)
		1317	{
		1318	va_list ap;
		1319	struct kgem_bo *bo;
		1320	int num_exec = 0;
		1321	int num_pages = 0;
		1322
		1323	va_start(ap, kgem);
		1324	while ((bo = va_arg(ap, struct kgem_bo *))) {
		1325	if (bo->exec)
		1326	continue;
		1327
		1328	if (bo->proxy) {
		1329	bo = bo->proxy;
		1330	if (bo->exec)
		1331	continue;
		1332	}
		1333	num_pages += num_pages(bo);
		1334	num_exec++;
		1335	}
		1336	va_end(ap);
		1337
		1338	if (!num_pages)
		1339	return true;
		1340
		1341	if (kgem->aperture > kgem->aperture_low)
		1342	return false;
		1343
		1344	if (num_pages + kgem->aperture > kgem->aperture_high)
		1345	return false;
		1346
		1347	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem))
		1348	return false;
		1349
		1350	return true;
		1351	}
		1352
		1353	/*
		1354	bool kgem_check_bo_fenced(struct kgem *kgem, ...)
		1355	{
		1356	va_list ap;
		1357	struct kgem_bo *bo;
		1358	int num_fence = 0;
		1359	int num_exec = 0;
		1360	int num_pages = 0;
		1361	int fenced_size = 0;
		1362
		1363	va_start(ap, kgem);
		1364	while ((bo = va_arg(ap, struct kgem_bo *))) {
		1365	if (bo->proxy)
		1366	bo = bo->proxy;
		1367	if (bo->exec) {
		1368	if (kgem->gen >= 40 \|\| bo->tiling == I915_TILING_NONE)
		1369	continue;
		1370
		1371	if ((bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		1372	fenced_size += kgem_bo_fenced_size(kgem, bo);
		1373	num_fence++;
		1374	}
		1375
		1376	continue;
		1377	}
		1378
		1379	num_pages += num_pages(bo);
		1380	num_exec++;
		1381	if (kgem->gen < 40 && bo->tiling) {
		1382	fenced_size += kgem_bo_fenced_size(kgem, bo);
		1383	num_fence++;
		1384	}
		1385	}
		1386	va_end(ap);
		1387
		1388	if (fenced_size + kgem->aperture_fenced > kgem->aperture_mappable)
		1389	return false;
		1390
		1391	if (kgem->nfence + num_fence > kgem->fence_max)
		1392	return false;
		1393
		1394	if (!num_pages)
		1395	return true;
		1396
		1397	if (kgem->aperture > kgem->aperture_low)
		1398	return false;
		1399
		1400	if (num_pages + kgem->aperture > kgem->aperture_high)
		1401	return false;
		1402
		1403	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem))
		1404	return false;
		1405
		1406	return true;
		1407	}
		1408	*/
		1409	#if 0
		1410	uint32_t kgem_add_reloc(struct kgem *kgem,
		1411	uint32_t pos,
		1412	struct kgem_bo *bo,
		1413	uint32_t read_write_domain,
		1414	uint32_t delta)
		1415	{
		1416	int index;
		1417
		1418	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
		1419	__FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));
		1420
		1421	assert((read_write_domain & 0x7fff) == 0 \|\| bo != NULL);
		1422
		1423	index = kgem->nreloc++;
		1424	assert(index < ARRAY_SIZE(kgem->reloc));
		1425	kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
		1426	if (bo) {
		1427	assert(bo->refcnt);
		1428	assert(!bo->purged);
		1429
		1430	delta += bo->delta;
		1431	if (bo->proxy) {
		1432	DBG(("%s: adding proxy for handle=%d\n",
		1433	__FUNCTION__, bo->handle));
		1434	assert(bo->handle == bo->proxy->handle);
		1435	/* need to release the cache upon batch submit */
		1436	list_move(&bo->request, &kgem->next_request->buffers);
		1437	bo->exec = &_kgem_dummy_exec;
		1438	bo = bo->proxy;
		1439	}
		1440
		1441	assert(!bo->purged);
		1442
		1443	// if (bo->exec == NULL)
		1444	// _kgem_add_bo(kgem, bo);
		1445
		1446	// if (kgem->gen < 40 && read_write_domain & KGEM_RELOC_FENCED) {
		1447	// if (bo->tiling &&
		1448	// (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
		1449	// assert(kgem->nfence < kgem->fence_max);
		1450	// kgem->aperture_fenced +=
		1451	// kgem_bo_fenced_size(kgem, bo);
		1452	// kgem->nfence++;
		1453	// }
		1454	// bo->exec->flags \|= EXEC_OBJECT_NEEDS_FENCE;
		1455	// }
		1456
		1457	kgem->reloc[index].delta = delta;
		1458	kgem->reloc[index].target_handle = bo->handle;
		1459	kgem->reloc[index].presumed_offset = bo->presumed_offset;
		1460
		1461	if (read_write_domain & 0x7fff) {
		1462	DBG(("%s: marking handle=%d dirty\n",
		1463	__FUNCTION__, bo->handle));
		1464	bo->needs_flush = bo->dirty = true;
		1465	}
		1466
		1467	delta += bo->presumed_offset;
		1468	} else {
		1469	kgem->reloc[index].delta = delta;
		1470	kgem->reloc[index].target_handle = 0;
		1471	kgem->reloc[index].presumed_offset = 0;
		1472	}
		1473	kgem->reloc[index].read_domains = read_write_domain >> 16;
		1474	kgem->reloc[index].write_domain = read_write_domain & 0x7fff;
		1475
		1476	return delta;
		1477	}
		1478	#endif
		1479
		1480
		1481
		1482
		1483
		1484
		1485
		1486
		1487
		1488	void kgem_bo_map(struct kgem kgem, struct kgem_bo *bo)
		1489	{
		1490	void *ptr;
		1491
		1492	DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
		1493	bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));
		1494
		1495	assert(!bo->purged);
		1496	assert(bo->exec == NULL);
		1497	assert(list_is_empty(&bo->list));
		1498
		1499	// if (bo->tiling == I915_TILING_NONE &&
		1500	// (kgem->has_llc \|\| bo->domain == bo->presumed_offset)) {
		1501	DBG(("%s: converting request for GTT map into CPU map\n",
		1502	__FUNCTION__));
		1503	ptr = kgem_bo_map__cpu(kgem, bo);
		1504	// kgem_bo_sync__cpu(kgem, bo);
		1505	return ptr;
		1506	// }
		1507
		1508	#if 0
		1509
		1510	if (IS_CPU_MAP(bo->map))
		1511	kgem_bo_release_map(kgem, bo);
		1512
		1513	ptr = bo->map;
		1514	if (ptr == NULL) {
		1515	assert(bytes(bo) <= kgem->aperture_mappable / 4);
		1516
		1517	kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));
		1518
		1519	ptr = gem_mmap(kgem->fd, bo->handle, bytes(bo),
		1520	PROT_READ \| PROT_WRITE);
		1521	if (ptr == NULL)
		1522	return NULL;
		1523
		1524	/* Cache this mapping to avoid the overhead of an
		1525	* excruciatingly slow GTT pagefault. This is more an
		1526	* issue with compositing managers which need to frequently
		1527	* flush CPU damage to their GPU bo.
		1528	*/
		1529	bo->map = ptr;
		1530	DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
		1531	}
		1532
		1533	if (bo->domain != DOMAIN_GTT) {
		1534	struct drm_i915_gem_set_domain set_domain;
		1535
		1536	DBG(("%s: sync: needs_flush? %d, domain? %d\n", __FUNCTION__,
		1537	bo->needs_flush, bo->domain));
		1538
		1539	/* XXX use PROT_READ to avoid the write flush? */
		1540
		1541	VG_CLEAR(set_domain);
		1542	set_domain.handle = bo->handle;
		1543	set_domain.read_domains = I915_GEM_DOMAIN_GTT;
		1544	set_domain.write_domain = I915_GEM_DOMAIN_GTT;
		1545	drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);
		1546
		1547	kgem_bo_retire(kgem, bo);
		1548	bo->domain = DOMAIN_GTT;
		1549	}
		1550	#endif
		1551
		1552	return ptr;
		1553	}
		1554
		1555	void kgem_bo_map__cpu(struct kgem kgem, struct kgem_bo *bo)
		1556	{
		1557	// struct drm_i915_gem_mmap mmap_arg;
		1558
		1559	DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__, bo->handle, bytes(bo)));
		1560	assert(!bo->purged);
		1561	assert(list_is_empty(&bo->list));
		1562
		1563	if (IS_CPU_MAP(bo->map))
		1564	return CPU_MAP(bo->map);
		1565
		1566	struct drm_i915_gem_object obj = (void)bo->handle;
		1567	u8 *dst;
		1568	int ret;
		1569
		1570	if(obj->pin_count == 0)
		1571	{
		1572	ret = i915_gem_object_pin(obj, 4096, true);
		1573	if (ret)
		1574	return NULL;
		1575	};
		1576
		1577	dst = drm_intel_bo_map(obj, true);
		1578	DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
		1579	bo->map = MAKE_CPU_MAP(dst);
		1580	return (void *)dst;
		1581
		1582
		1583	#if 0
		1584	if (bo->map)
		1585	kgem_bo_release_map(kgem, bo);
		1586
		1587	kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));
		1588
		1589	VG_CLEAR(mmap_arg);
		1590	mmap_arg.handle = bo->handle;
		1591	mmap_arg.offset = 0;
		1592	mmap_arg.size = bytes(bo);
		1593	if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
		1594	ErrorF("%s: failed to mmap %d, %d bytes, into CPU domain\n",
		1595	__FUNCTION__, bo->handle, bytes(bo));
		1596	return NULL;
		1597	}
		1598
		1599	VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, bytes(bo), 0, 1));
		1600	#endif
		1601
		1602	}
		1603
		1604
		1605
		1606
		1607
		1608
		1609
		1610
		1611
		1612
		1613
		1614
		1615
		1616
		1617
		1618
		1619
		1620
		1621
		1622	void kgem_clear_dirty(struct kgem *kgem)
		1623	{
		1624	struct kgem_request *rq = kgem->next_request;
		1625	struct kgem_bo *bo;
		1626
		1627	list_for_each_entry(bo, &rq->buffers, request)
		1628	bo->dirty = false;
		1629	}
		1630
		1631	struct kgem_bo kgem_create_proxy(struct kgem_bo target,
		1632	int offset, int length)
		1633	{
		1634	struct kgem_bo *bo;
		1635
		1636	DBG(("%s: target handle=%d, offset=%d, length=%d, io=%d\n",
		1637	__FUNCTION__, target->handle, offset, length, target->io));
		1638
		1639	bo = __kgem_bo_alloc(target->handle, length);
		1640	if (bo == NULL)
		1641	return NULL;
		1642
		1643	bo->reusable = false;
		1644	bo->size.bytes = length;
		1645
		1646	bo->io = target->io;
		1647	bo->dirty = target->dirty;
		1648	bo->tiling = target->tiling;
		1649	bo->pitch = target->pitch;
		1650
		1651	if (target->proxy) {
		1652	offset += target->delta;
		1653	target = target->proxy;
		1654	}
		1655	bo->proxy = kgem_bo_reference(target);
		1656	bo->delta = offset;
		1657	bo->gaddr = offset + target->gaddr;
		1658	return bo;
		1659	}
		1660
		1661
		1662
		1663
		1664
		1665
		1666
		1667
		1668
		1669
		1670	uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format)
		1671	{
		1672	struct kgem_bo_binding *b;
		1673
		1674	for (b = &bo->binding; b && b->offset; b = b->next)
		1675	if (format == b->format)
		1676	return b->offset;
		1677
		1678	return 0;
		1679	}
		1680
		1681	void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset)
		1682	{
		1683	struct kgem_bo_binding *b;
		1684
		1685	for (b = &bo->binding; b; b = b->next) {
		1686	if (b->offset)
		1687	continue;
		1688
		1689	b->offset = offset;
		1690	b->format = format;
		1691
		1692	if (b->next)
		1693	b->next->offset = 0;
		1694
		1695	return;
		1696	}
		1697
		1698	b = malloc(sizeof(*b));
		1699	if (b) {
		1700	b->next = bo->binding.next;
		1701	b->format = format;
		1702	b->offset = offset;
		1703	bo->binding.next = b;
		1704	}
		1705	}
		1706
		1707
		1708	struct kgem_bo create_bo(bitmap_t bitmap)
		1709	{
		1710	struct kgem_bo *bo;
		1711
		1712	bo = __kgem_bo_alloc(bitmap->obj, 10247684/4096);
		1713	bo->gaddr = bitmap->gaddr;
		1714	bo->pitch = bitmap->pitch;
		1715	bo->tiling = 0;
		1716	return bo;
		1717
		1718	};

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/sna/kgem.c – Rev 2352