Subversion Repositories Kolibri OS

/**************************************************************************

 *

 * Copyright © 2007 Red Hat Inc.

 * Copyright © 2007-2012 Intel Corporation

 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA

 * All Rights Reserved.

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the

 * "Software"), to deal in the Software without restriction, including

 * without limitation the rights to use, copy, modify, merge, publish,

 * distribute, sub license, and/or sell copies of the Software, and to

 * permit persons to whom the Software is furnished to do so, subject to

 * the following conditions:

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL

 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,

 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR

 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE

 * USE OR OTHER DEALINGS IN THE SOFTWARE.

 *

 * The above copyright notice and this permission notice (including the

 * next paragraph) shall be included in all copies or substantial portions

 * of the Software.

 *

 *

 **************************************************************************/

/*

 * Authors: Thomas Hellström 

 *          Keith Whitwell 

 *	    Eric Anholt 

 *	    Dave Airlie 

 */

#ifdef HAVE_CONFIG_H

#include "config.h"

#endif

#include 

#include 

#include 

#include 

#include 

#include 

#include 

#include 

//#include 

#include 

#include 

#include 

#include "errno.h"

#ifndef ETIME

#define ETIME ETIMEDOUT

#endif

#include "libdrm_macros.h"

#include "libdrm_lists.h"

#include "intel_bufmgr.h"

#include "intel_bufmgr_priv.h"

#include "intel_chipset.h"

#include "string.h"

#include "i915_drm.h"

#ifdef HAVE_VALGRIND

#include 

#include 

#define VG(x) x

#else

#define VG(x)

#endif

#define memclear(s) memset(&s, 0, sizeof(s))

#if 0

#define DBG(...) do {					\

	if (bufmgr_gem->bufmgr.debug)			\

		fprintf(stderr, __VA_ARGS__);		\

} while (0)

#else

#define DBG(...)

#endif

#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))

#define MAX2(A, B) ((A) > (B) ? (A) : (B))

/**

 * upper_32_bits - return bits 32-63 of a number

 * @n: the number we're accessing

 *

 * A basic shift-right of a 64- or 32-bit quantity.  Use this to suppress

 * the "right shift count >= width of type" warning when that quantity is

 * 32-bits.

 */

#define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))

/**

 * lower_32_bits - return bits 0-31 of a number

 * @n: the number we're accessing

 */

#define lower_32_bits(n) ((__u32)(n))

typedef struct _drm_intel_bo_gem drm_intel_bo_gem;

struct drm_intel_gem_bo_bucket {

	drmMMListHead head;

	unsigned long size;

};

typedef struct _drm_intel_bufmgr_gem {

	drm_intel_bufmgr bufmgr;

	atomic_t refcount;

	int fd;

	int max_relocs;

//	pthread_mutex_t lock;

	struct drm_i915_gem_exec_object *exec_objects;

	struct drm_i915_gem_exec_object2 *exec2_objects;

	drm_intel_bo **exec_bos;

	int exec_size;

	int exec_count;

	/** Array of lists of cached gem objects of power-of-two sizes */

	struct drm_intel_gem_bo_bucket cache_bucket[14 * 4];

	int num_buckets;

	time_t time;

	drmMMListHead managers;

	drmMMListHead named;

	drmMMListHead vma_cache;

	int vma_count, vma_open, vma_max;

	uint64_t gtt_size;

	int available_fences;

	int pci_device;

	int gen;

	unsigned int has_bsd : 1;

	unsigned int has_blt : 1;

	unsigned int has_relaxed_fencing : 1;

	unsigned int has_llc : 1;

	unsigned int has_wait_timeout : 1;

	unsigned int bo_reuse : 1;

	unsigned int no_exec : 1;

	unsigned int has_vebox : 1;

	bool fenced_relocs;

	struct {

		void *ptr;

		uint32_t handle;

	} userptr_active;

} drm_intel_bufmgr_gem;

#define DRM_INTEL_RELOC_FENCE (1<<0)

typedef struct _drm_intel_reloc_target_info {

	drm_intel_bo *bo;

	int flags;

} drm_intel_reloc_target;

struct _drm_intel_bo_gem {

	drm_intel_bo bo;

	atomic_t refcount;

	uint32_t gem_handle;

	const char *name;

	/**

	 * Kenel-assigned global name for this object

         *

         * List contains both flink named and prime fd'd objects

	 */

	unsigned int global_name;

	drmMMListHead name_list;

	/**

	 * Index of the buffer within the validation list while preparing a

	 * batchbuffer execution.

	 */

	int validate_index;

	/**

	 * Current tiling mode

	 */

	uint32_t tiling_mode;

	uint32_t swizzle_mode;

	unsigned long stride;

	time_t free_time;

	/** Array passed to the DRM containing relocation information. */

	struct drm_i915_gem_relocation_entry *relocs;

	/**

	 * Array of info structs corresponding to relocs[i].target_handle etc

	 */

	drm_intel_reloc_target *reloc_target_info;

	/** Number of entries in relocs */

	int reloc_count;

	/** Array of BOs that are referenced by this buffer and will be softpinned */

	drm_intel_bo **softpin_target;

	/** Number softpinned BOs that are referenced by this buffer */

	int softpin_target_count;

	/** Maximum amount of softpinned BOs that are referenced by this buffer */

	int softpin_target_size;

	/** Mapped address for the buffer, saved across map/unmap cycles */

	void *mem_virtual;

	/** GTT virtual address for the buffer, saved across map/unmap cycles */

	void *gtt_virtual;

	/**

	 * Virtual address of the buffer allocated by user, used for userptr

	 * objects only.

	 */

	void *user_virtual;

	int map_count;

	drmMMListHead vma_list;

	/** BO cache list */

	drmMMListHead head;

	/**

	 * Boolean of whether this BO and its children have been included in

	 * the current drm_intel_bufmgr_check_aperture_space() total.

	 */

	bool included_in_check_aperture;

	/**

	 * Boolean of whether this buffer has been used as a relocation

	 * target and had its size accounted for, and thus can't have any

	 * further relocations added to it.

	 */

	bool used_as_reloc_target;

	/**

	 * Boolean of whether we have encountered an error whilst building the relocation tree.

	 */

	bool has_error;

	/**

	 * Boolean of whether this buffer can be re-used

	 */

	bool reusable;

	/**

	 * Boolean of whether the GPU is definitely not accessing the buffer.

	 *

	 * This is only valid when reusable, since non-reusable

	 * buffers are those that have been shared wth other

	 * processes, so we don't know their state.

	 */

	bool idle;

	/**

	 * Boolean of whether this buffer was allocated with userptr

	 */

	bool is_userptr;

	/**

	 * Boolean of whether this buffer can be placed in the full 48-bit

	 * address range on gen8+.

	 *

	 * By default, buffers will be keep in a 32-bit range, unless this

	 * flag is explicitly set.

	 */

	bool use_48b_address_range;

	/**

	 * Whether this buffer is softpinned at offset specified by the user

	 */

	bool is_softpin;

	/**

	 * Size in bytes of this buffer and its relocation descendents.

	 *

	 * Used to avoid costly tree walking in

	 * drm_intel_bufmgr_check_aperture in the common case.

	 */

	int reloc_tree_size;

	/**

	 * Number of potential fence registers required by this buffer and its

	 * relocations.

	 */

	int reloc_tree_fences;

	/** Flags that we may need to do the SW_FINSIH ioctl on unmap. */

	bool mapped_cpu_write;

};

static unsigned int

drm_intel_gem_estimate_batch_space(drm_intel_bo ** bo_array, int count);

static unsigned int

drm_intel_gem_compute_batch_space(drm_intel_bo ** bo_array, int count);

static int

drm_intel_gem_bo_get_tiling(drm_intel_bo *bo, uint32_t * tiling_mode,

			    uint32_t * swizzle_mode);

static int

drm_intel_gem_bo_set_tiling_internal(drm_intel_bo *bo,

				     uint32_t tiling_mode,

				     uint32_t stride);

static void drm_intel_gem_bo_unreference_locked_timed(drm_intel_bo *bo,

						      time_t time);

static void drm_intel_gem_bo_unreference(drm_intel_bo *bo);

static void drm_intel_gem_bo_free(drm_intel_bo *bo);

static inline drm_intel_bo_gem *to_bo_gem(drm_intel_bo *bo)

{

        return (drm_intel_bo_gem *)bo;

}

static unsigned long

drm_intel_gem_bo_tile_size(drm_intel_bufmgr_gem *bufmgr_gem, unsigned long size,

			   uint32_t *tiling_mode)

{

	unsigned long min_size, max_size;

	unsigned long i;

	if (*tiling_mode == I915_TILING_NONE)

		return size;

	/* 965+ just need multiples of page size for tiling */

	if (bufmgr_gem->gen >= 4)

		return ROUND_UP_TO(size, 4096);

	/* Older chips need powers of two, of at least 512k or 1M */

	if (bufmgr_gem->gen == 3) {

		min_size = 1024*1024;

		max_size = 128*1024*1024;

	} else {

		min_size = 512*1024;

		max_size = 64*1024*1024;

	}

	if (size > max_size) {

		*tiling_mode = I915_TILING_NONE;

		return size;

	}

	/* Do we need to allocate every page for the fence? */

	if (bufmgr_gem->has_relaxed_fencing)

		return ROUND_UP_TO(size, 4096);

	for (i = min_size; i < size; i <<= 1)

		;

	return i;

}

/*

 * Round a given pitch up to the minimum required for X tiling on a

 * given chip.  We use 512 as the minimum to allow for a later tiling

 * change.

 */

static unsigned long

drm_intel_gem_bo_tile_pitch(drm_intel_bufmgr_gem *bufmgr_gem,

			    unsigned long pitch, uint32_t *tiling_mode)

{

	unsigned long tile_width;

	unsigned long i;

	/* If untiled, then just align it so that we can do rendering

	 * to it with the 3D engine.

	 */

	if (*tiling_mode == I915_TILING_NONE)

		return ALIGN(pitch, 64);

	if (*tiling_mode == I915_TILING_X

			|| (IS_915(bufmgr_gem->pci_device)

			    && *tiling_mode == I915_TILING_Y))

		tile_width = 512;

	else

		tile_width = 128;

	/* 965 is flexible */

	if (bufmgr_gem->gen >= 4)

		return ROUND_UP_TO(pitch, tile_width);

	/* The older hardware has a maximum pitch of 8192 with tiled

	 * surfaces, so fallback to untiled if it's too large.

	 */

	if (pitch > 8192) {

		*tiling_mode = I915_TILING_NONE;

		return ALIGN(pitch, 64);

	}

	/* Pre-965 needs power of two tile width */

	for (i = tile_width; i < pitch; i <<= 1)

		;

	return i;

}

static struct drm_intel_gem_bo_bucket *

drm_intel_gem_bo_bucket_for_size(drm_intel_bufmgr_gem *bufmgr_gem,

				 unsigned long size)

{

	int i;

	for (i = 0; i < bufmgr_gem->num_buckets; i++) {

		struct drm_intel_gem_bo_bucket *bucket =

		    &bufmgr_gem->cache_bucket[i];

		if (bucket->size >= size) {

			return bucket;

		}

	}

	return NULL;

}

static void

drm_intel_gem_dump_validation_list(drm_intel_bufmgr_gem *bufmgr_gem)

{

	int i, j;

	for (i = 0; i < bufmgr_gem->exec_count; i++) {

		drm_intel_bo *bo = bufmgr_gem->exec_bos[i];

		drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

		if (bo_gem->relocs == NULL && bo_gem->softpin_target == NULL) {

			DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,

			    bo_gem->is_softpin ? "*" : "",

			    bo_gem->name);

			continue;

		}

		for (j = 0; j < bo_gem->reloc_count; j++) {

			drm_intel_bo *target_bo = bo_gem->reloc_target_info[j].bo;

			drm_intel_bo_gem *target_gem =

			    (drm_intel_bo_gem *) target_bo;

			DBG("%2d: %d %s(%s)@0x%08x %08x -> "

			    "%d (%s)@0x%08x %08x + 0x%08x\n",

			    i,

			    bo_gem->gem_handle,

			    bo_gem->is_softpin ? "*" : "",

			    bo_gem->name,

			    upper_32_bits(bo_gem->relocs[j].offset),

			    lower_32_bits(bo_gem->relocs[j].offset),

			    target_gem->gem_handle,

			    target_gem->name,

			    upper_32_bits(target_bo->offset64),

			    lower_32_bits(target_bo->offset64),

			    bo_gem->relocs[j].delta);

		}

		for (j = 0; j < bo_gem->softpin_target_count; j++) {

			drm_intel_bo *target_bo = bo_gem->softpin_target[j];

			drm_intel_bo_gem *target_gem =

			    (drm_intel_bo_gem *) target_bo;

			DBG("%2d: %d %s(%s) -> "

			    "%d *(%s)@0x%08x %08x\n",

			    i,

			    bo_gem->gem_handle,

			    bo_gem->is_softpin ? "*" : "",

			    bo_gem->name,

			    target_gem->gem_handle,

			    target_gem->name,

			    upper_32_bits(target_bo->offset64),

			    lower_32_bits(target_bo->offset64));

		}

	}

}

static inline void

drm_intel_gem_bo_reference(drm_intel_bo *bo)

{

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	atomic_inc(&bo_gem->refcount);

}

/**

 * Adds the given buffer to the list of buffers to be validated (moved into the

 * appropriate memory type) with the next batch submission.

 *

 * If a buffer is validated multiple times in a batch submission, it ends up

 * with the intersection of the memory type flags and the union of the

 * access flags.

 */

static void

drm_intel_add_validate_buffer(drm_intel_bo *bo)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	int index;

	if (bo_gem->validate_index != -1)

		return;

	/* Extend the array of validation entries as necessary. */

	if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {

		int new_size = bufmgr_gem->exec_size * 2;

		if (new_size == 0)

			new_size = 5;

		bufmgr_gem->exec_objects =

		    realloc(bufmgr_gem->exec_objects,

			    sizeof(*bufmgr_gem->exec_objects) * new_size);

		bufmgr_gem->exec_bos =

		    realloc(bufmgr_gem->exec_bos,

			    sizeof(*bufmgr_gem->exec_bos) * new_size);

		bufmgr_gem->exec_size = new_size;

	}

	index = bufmgr_gem->exec_count;

	bo_gem->validate_index = index;

	/* Fill in array entry */

	bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;

	bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;

	bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;

	bufmgr_gem->exec_objects[index].alignment = bo->align;

	bufmgr_gem->exec_objects[index].offset = 0;

	bufmgr_gem->exec_bos[index] = bo;

	bufmgr_gem->exec_count++;

}

static void

drm_intel_add_validate_buffer2(drm_intel_bo *bo, int need_fence)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bo->bufmgr;

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;

	int index;

	int flags = 0;

	if (need_fence)

		flags |= EXEC_OBJECT_NEEDS_FENCE;

	if (bo_gem->use_48b_address_range)

		flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;

	if (bo_gem->is_softpin)

		flags |= EXEC_OBJECT_PINNED;

	if (bo_gem->validate_index != -1) {

		bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= flags;

		return;

	}

	/* Extend the array of validation entries as necessary. */

	if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {

		int new_size = bufmgr_gem->exec_size * 2;

		if (new_size == 0)

			new_size = 5;

		bufmgr_gem->exec2_objects =

			realloc(bufmgr_gem->exec2_objects,

				sizeof(*bufmgr_gem->exec2_objects) * new_size);

		bufmgr_gem->exec_bos =

			realloc(bufmgr_gem->exec_bos,

				sizeof(*bufmgr_gem->exec_bos) * new_size);

		bufmgr_gem->exec_size = new_size;

	}

	index = bufmgr_gem->exec_count;

	bo_gem->validate_index = index;

	/* Fill in array entry */

	bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;

	bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;

	bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;

	bufmgr_gem->exec2_objects[index].alignment = bo->align;

	bufmgr_gem->exec2_objects[index].offset = bo_gem->is_softpin ?

		bo->offset64 : 0;

	bufmgr_gem->exec_bos[index] = bo;

	bufmgr_gem->exec2_objects[index].flags = flags;

	bufmgr_gem->exec2_objects[index].rsvd1 = 0;

	bufmgr_gem->exec2_objects[index].rsvd2 = 0;

	bufmgr_gem->exec_count++;

}

#define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \

	sizeof(uint32_t))

static void

drm_intel_bo_gem_set_in_aperture_size(drm_intel_bufmgr_gem *bufmgr_gem,

				      drm_intel_bo_gem *bo_gem,

				      unsigned int alignment)

{

	unsigned int size;

	assert(!bo_gem->used_as_reloc_target);

	/* The older chipsets are far-less flexible in terms of tiling,

	 * and require tiled buffer to be size aligned in the aperture.

	 * This means that in the worst possible case we will need a hole

	 * twice as large as the object in order for it to fit into the

	 * aperture. Optimal packing is for wimps.

	 */

	size = bo_gem->bo.size;

	if (bufmgr_gem->gen < 4 && bo_gem->tiling_mode != I915_TILING_NONE) {

		unsigned int min_size;

		if (bufmgr_gem->has_relaxed_fencing) {

			if (bufmgr_gem->gen == 3)

				min_size = 1024*1024;

			else

				min_size = 512*1024;

			while (min_size < size)

				min_size *= 2;

		} else

			min_size = size;

		/* Account for worst-case alignment. */

		alignment = MAX2(alignment, min_size);

	}

	bo_gem->reloc_tree_size = size + alignment;

}

static int

drm_intel_setup_reloc_list(drm_intel_bo *bo)

{

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;

	unsigned int max_relocs = bufmgr_gem->max_relocs;

	if (bo->size / 4 < max_relocs)

		max_relocs = bo->size / 4;

	bo_gem->relocs = malloc(max_relocs *

				sizeof(struct drm_i915_gem_relocation_entry));

	bo_gem->reloc_target_info = malloc(max_relocs *

					   sizeof(drm_intel_reloc_target));

	if (bo_gem->relocs == NULL || bo_gem->reloc_target_info == NULL) {

		bo_gem->has_error = true;

		free (bo_gem->relocs);

		bo_gem->relocs = NULL;

		free (bo_gem->reloc_target_info);

		bo_gem->reloc_target_info = NULL;

		return 1;

	}

	return 0;

}

static int

drm_intel_gem_bo_busy(drm_intel_bo *bo)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	struct drm_i915_gem_busy busy;

	int ret;

	if (bo_gem->reusable && bo_gem->idle)

		return false;

	memclear(busy);

	busy.handle = bo_gem->gem_handle;

	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);

	if (ret == 0) {

		bo_gem->idle = !busy.busy;

		return busy.busy;

	} else {

		return false;

	}

	return (ret == 0 && busy.busy);

}

static int

drm_intel_gem_bo_madvise_internal(drm_intel_bufmgr_gem *bufmgr_gem,

				  drm_intel_bo_gem *bo_gem, int state)

{

	struct drm_i915_gem_madvise madv;

	memclear(madv);

	madv.handle = bo_gem->gem_handle;

	madv.madv = state;

	madv.retained = 1;

//	drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);

	return madv.retained;

}

static int

drm_intel_gem_bo_madvise(drm_intel_bo *bo, int madv)

{

	return drm_intel_gem_bo_madvise_internal

		((drm_intel_bufmgr_gem *) bo->bufmgr,

		 (drm_intel_bo_gem *) bo,

		 madv);

}

/* drop the oldest entries that have been purged by the kernel */

static void

drm_intel_gem_bo_cache_purge_bucket(drm_intel_bufmgr_gem *bufmgr_gem,

				    struct drm_intel_gem_bo_bucket *bucket)

{

	while (!DRMLISTEMPTY(&bucket->head)) {

		drm_intel_bo_gem *bo_gem;

		bo_gem = DRMLISTENTRY(drm_intel_bo_gem,

				      bucket->head.next, head);

		if (drm_intel_gem_bo_madvise_internal

		    (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))

			break;

		DRMLISTDEL(&bo_gem->head);

		drm_intel_gem_bo_free(&bo_gem->bo);

	}

}

static drm_intel_bo *

drm_intel_gem_bo_alloc_internal(drm_intel_bufmgr *bufmgr,

				const char *name,

				unsigned long size,

				unsigned long flags,

				uint32_t tiling_mode,

				unsigned long stride,

				unsigned int alignment)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;

	drm_intel_bo_gem *bo_gem;

	unsigned int page_size = 4096;

	int ret;

	struct drm_intel_gem_bo_bucket *bucket;

	bool alloc_from_cache;

	unsigned long bo_size;

	bool for_render = false;

	if (flags & BO_ALLOC_FOR_RENDER)

		for_render = true;

	/* Round the allocated size up to a power of two number of pages. */

	bucket = drm_intel_gem_bo_bucket_for_size(bufmgr_gem, size);

	/* If we don't have caching at this size, don't actually round the

	 * allocation up.

	 */

	if (bucket == NULL) {

		bo_size = size;

		if (bo_size < page_size)

			bo_size = page_size;

	} else {

		bo_size = bucket->size;

	}

//	pthread_mutex_lock(&bufmgr_gem->lock);

	/* Get a buffer out of the cache if available */

retry:

	alloc_from_cache = false;

	if (bucket != NULL && !DRMLISTEMPTY(&bucket->head)) {

		if (for_render) {

			/* Allocate new render-target BOs from the tail (MRU)

			 * of the list, as it will likely be hot in the GPU

			 * cache and in the aperture for us.

			 */

			bo_gem = DRMLISTENTRY(drm_intel_bo_gem,

					      bucket->head.prev, head);

			DRMLISTDEL(&bo_gem->head);

			alloc_from_cache = true;

			bo_gem->bo.align = alignment;

		} else {

			assert(alignment == 0);

			/* For non-render-target BOs (where we're probably

			 * going to map it first thing in order to fill it

			 * with data), check if the last BO in the cache is

			 * unbusy, and only reuse in that case. Otherwise,

			 * allocating a new buffer is probably faster than

			 * waiting for the GPU to finish.

			 */

			bo_gem = DRMLISTENTRY(drm_intel_bo_gem,

					      bucket->head.next, head);

			if (!drm_intel_gem_bo_busy(&bo_gem->bo)) {

				alloc_from_cache = true;

				DRMLISTDEL(&bo_gem->head);

			}

		}

		if (alloc_from_cache) {

			if (!drm_intel_gem_bo_madvise_internal

			    (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {

				drm_intel_gem_bo_free(&bo_gem->bo);

				drm_intel_gem_bo_cache_purge_bucket(bufmgr_gem,

								    bucket);

				goto retry;

			}

			if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,

								 tiling_mode,

								 stride)) {

				drm_intel_gem_bo_free(&bo_gem->bo);

				goto retry;

			}

		}

	}

//	pthread_mutex_unlock(&bufmgr_gem->lock);

	if (!alloc_from_cache) {

		struct drm_i915_gem_create create;

		bo_gem = calloc(1, sizeof(*bo_gem));

		if (!bo_gem)

			return NULL;

		bo_gem->bo.size = bo_size;

		memclear(create);

		create.size = bo_size;

		ret = drmIoctl(bufmgr_gem->fd,

			       DRM_IOCTL_I915_GEM_CREATE,

			       &create);

		bo_gem->gem_handle = create.handle;

		bo_gem->bo.handle = bo_gem->gem_handle;

		if (ret != 0) {

			free(bo_gem);

			return NULL;

		}

		bo_gem->bo.bufmgr = bufmgr;

		bo_gem->bo.align = alignment;

		bo_gem->tiling_mode = I915_TILING_NONE;

		bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;

		bo_gem->stride = 0;

		/* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized

		   list (vma_list), so better set the list head here */

		DRMINITLISTHEAD(&bo_gem->name_list);

		DRMINITLISTHEAD(&bo_gem->vma_list);

		if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,

							 tiling_mode,

							 stride)) {

		    drm_intel_gem_bo_free(&bo_gem->bo);

		    return NULL;

		}

	}

	bo_gem->name = name;

	atomic_set(&bo_gem->refcount, 1);

	bo_gem->validate_index = -1;

	bo_gem->reloc_tree_fences = 0;

	bo_gem->used_as_reloc_target = false;

	bo_gem->has_error = false;

	bo_gem->reusable = true;

	bo_gem->use_48b_address_range = false;

	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alignment);

	DBG("bo_create: buf %d (%s) %ldb\n",

	    bo_gem->gem_handle, bo_gem->name, size);

	return &bo_gem->bo;

}

static drm_intel_bo *

drm_intel_gem_bo_alloc_for_render(drm_intel_bufmgr *bufmgr,

				  const char *name,

				  unsigned long size,

				  unsigned int alignment)

{

	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size,

					       BO_ALLOC_FOR_RENDER,

					       I915_TILING_NONE, 0,

					       alignment);

}

static drm_intel_bo *

drm_intel_gem_bo_alloc(drm_intel_bufmgr *bufmgr,

		       const char *name,

		       unsigned long size,

		       unsigned int alignment)

{

	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, 0,

					       I915_TILING_NONE, 0, 0);

}

static drm_intel_bo *

drm_intel_gem_bo_alloc_tiled(drm_intel_bufmgr *bufmgr, const char *name,

			     int x, int y, int cpp, uint32_t *tiling_mode,

			     unsigned long *pitch, unsigned long flags)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bufmgr;

	unsigned long size, stride;

	uint32_t tiling;

	do {

		unsigned long aligned_y, height_alignment;

		tiling = *tiling_mode;

		/* If we're tiled, our allocations are in 8 or 32-row blocks,

		 * so failure to align our height means that we won't allocate

		 * enough pages.

		 *

		 * If we're untiled, we still have to align to 2 rows high

		 * because the data port accesses 2x2 blocks even if the

		 * bottom row isn't to be rendered, so failure to align means

		 * we could walk off the end of the GTT and fault.  This is

		 * documented on 965, and may be the case on older chipsets

		 * too so we try to be careful.

		 */

		aligned_y = y;

		height_alignment = 2;

		if ((bufmgr_gem->gen == 2) && tiling != I915_TILING_NONE)

			height_alignment = 16;

		else if (tiling == I915_TILING_X

			|| (IS_915(bufmgr_gem->pci_device)

			    && tiling == I915_TILING_Y))

			height_alignment = 8;

		else if (tiling == I915_TILING_Y)

			height_alignment = 32;

		aligned_y = ALIGN(y, height_alignment);

		stride = x * cpp;

		stride = drm_intel_gem_bo_tile_pitch(bufmgr_gem, stride, tiling_mode);

		size = stride * aligned_y;

		size = drm_intel_gem_bo_tile_size(bufmgr_gem, size, tiling_mode);

	} while (*tiling_mode != tiling);

	*pitch = stride;

	if (tiling == I915_TILING_NONE)

		stride = 0;

	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, flags,

					       tiling, stride, 0);

}

#if 0

static drm_intel_bo *

drm_intel_gem_bo_alloc_userptr(drm_intel_bufmgr *bufmgr,

				const char *name,

				void *addr,

				uint32_t tiling_mode,

				uint32_t stride,

				unsigned long size,

				unsigned long flags)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;

	drm_intel_bo_gem *bo_gem;

	int ret;

	struct drm_i915_gem_userptr userptr;

	/* Tiling with userptr surfaces is not supported

	 * on all hardware so refuse it for time being.

	 */

	if (tiling_mode != I915_TILING_NONE)

		return NULL;

	bo_gem = calloc(1, sizeof(*bo_gem));

	if (!bo_gem)

		return NULL;

	bo_gem->bo.size = size;

	memclear(userptr);

	userptr.user_ptr = (__u64)((unsigned long)addr);

	userptr.user_size = size;

	userptr.flags = flags;

	ret = drmIoctl(bufmgr_gem->fd,

			DRM_IOCTL_I915_GEM_USERPTR,

			&userptr);

	if (ret != 0) {

		DBG("bo_create_userptr: "

		    "ioctl failed with user ptr %p size 0x%lx, "

		    "user flags 0x%lx\n", addr, size, flags);

		free(bo_gem);

		return NULL;

	}

	bo_gem->gem_handle = userptr.handle;

	bo_gem->bo.handle = bo_gem->gem_handle;

	bo_gem->bo.bufmgr    = bufmgr;

	bo_gem->is_userptr   = true;

	bo_gem->bo.virtual   = addr;

	/* Save the address provided by user */

	bo_gem->user_virtual = addr;

	bo_gem->tiling_mode  = I915_TILING_NONE;

	bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;

	bo_gem->stride       = 0;

	DRMINITLISTHEAD(&bo_gem->name_list);

	DRMINITLISTHEAD(&bo_gem->vma_list);

	bo_gem->name = name;

	atomic_set(&bo_gem->refcount, 1);

	bo_gem->validate_index = -1;

	bo_gem->reloc_tree_fences = 0;

	bo_gem->used_as_reloc_target = false;

	bo_gem->has_error = false;

	bo_gem->reusable = false;

	bo_gem->use_48b_address_range = false;

	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);

	DBG("bo_create_userptr: "

	    "ptr %p buf %d (%s) size %ldb, stride 0x%x, tile mode %d\n",

		addr, bo_gem->gem_handle, bo_gem->name,

		size, stride, tiling_mode);

	return &bo_gem->bo;

}

static bool

has_userptr(drm_intel_bufmgr_gem *bufmgr_gem)

{

	int ret;

	void *ptr;

	long pgsz;

	struct drm_i915_gem_userptr userptr;

	pgsz = sysconf(_SC_PAGESIZE);

	assert(pgsz > 0);

	ret = posix_memalign(&ptr, pgsz, pgsz);

	if (ret) {

		DBG("Failed to get a page (%ld) for userptr detection!\n",

			pgsz);

		return false;

	}

	memclear(userptr);

	userptr.user_ptr = (__u64)(unsigned long)ptr;

	userptr.user_size = pgsz;

retry:

	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_USERPTR, &userptr);

	if (ret) {

		if (errno == ENODEV && userptr.flags == 0) {

			userptr.flags = I915_USERPTR_UNSYNCHRONIZED;

			goto retry;

		}

		free(ptr);

		return false;

	}

	/* We don't release the userptr bo here as we want to keep the

	 * kernel mm tracking alive for our lifetime. The first time we

	 * create a userptr object the kernel has to install a mmu_notifer

	 * which is a heavyweight operation (e.g. it requires taking all

	 * mm_locks and stop_machine()).

	 */

	bufmgr_gem->userptr_active.ptr = ptr;

	bufmgr_gem->userptr_active.handle = userptr.handle;

	return true;

}

#endif

static drm_intel_bo *

check_bo_alloc_userptr(drm_intel_bufmgr *bufmgr,

		       const char *name,

		       void *addr,

		       uint32_t tiling_mode,

		       uint32_t stride,

		       unsigned long size,

		       unsigned long flags)

{

		bufmgr->bo_alloc_userptr = NULL;

	return drm_intel_bo_alloc_userptr(bufmgr, name, addr,

					  tiling_mode, stride, size, flags);

}

/**

 * Returns a drm_intel_bo wrapping the given buffer object handle.

 *

 * This can be used when one application needs to pass a buffer object

 * to another.

 */

drm_intel_bo *

drm_intel_bo_gem_create_from_name(drm_intel_bufmgr *bufmgr,

				  const char *name,

				  unsigned int handle)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;

	drm_intel_bo_gem *bo_gem;

	int ret;

	struct drm_gem_open open_arg;

	struct drm_i915_gem_get_tiling get_tiling;

	drmMMListHead *list;

	/* At the moment most applications only have a few named bo.

	 * For instance, in a DRI client only the render buffers passed

	 * between X and the client are named. And since X returns the

	 * alternating names for the front/back buffer a linear search

	 * provides a sufficiently fast match.

	 */

	for (list = bufmgr_gem->named.next;

	     list != &bufmgr_gem->named;

	     list = list->next) {

		bo_gem = DRMLISTENTRY(drm_intel_bo_gem, list, name_list);

		if (bo_gem->global_name == handle) {

			drm_intel_gem_bo_reference(&bo_gem->bo);

			return &bo_gem->bo;

		}

	}

	memclear(open_arg);

	open_arg.name = handle;

	ret = drmIoctl(bufmgr_gem->fd,

		       DRM_IOCTL_GEM_OPEN,

		       &open_arg);