Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 5077 → Rev 5078

/drivers/video/drm/ttm/ttm_bo.c
39,92 → 39,35
#include <linux/mm.h>
#include <linux/module.h>
 
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
 
#define TTM_ASSERT_LOCKED(param)
#define TTM_DEBUG(fmt, arg...)
#define TTM_BO_HASH_ORDER 13
 
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
 
int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
 
static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
{
int i;
 
mutex_lock(&man->io_reserve_mutex);
for (i = 0; i <= TTM_PL_PRIV5; i++)
if (flags & (1 << i)) {
*mem_type = i;
return 0;
}
 
void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
{
if (likely(man->io_reserve_fastpath))
return;
 
mutex_unlock(&man->io_reserve_mutex);
return -EINVAL;
}
 
 
#if 0
static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
{
struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 
pr_err(" has_type: %d\n", man->has_type);
pr_err(" use_type: %d\n", man->use_type);
pr_err(" flags: 0x%08X\n", man->flags);
pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset);
pr_err(" size: %llu\n", man->size);
pr_err(" available_caching: 0x%08X\n", man->available_caching);
pr_err(" default_caching: 0x%08X\n", man->default_caching);
if (mem_type != TTM_PL_SYSTEM)
(*man->func->debug)(man, TTM_PFX);
}
 
static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
struct ttm_placement *placement)
{
int i, ret, mem_type;
 
pr_err("No space for %p (%lu pages, %luK, %luM)\n",
bo, bo->mem.num_pages, bo->mem.size >> 10,
bo->mem.size >> 20);
for (i = 0; i < placement->num_placement; i++) {
ret = ttm_mem_type_from_flags(placement->placement[i],
&mem_type);
if (ret)
return;
pr_err(" placement[%d]=0x%08X (%d)\n",
i, placement->placement[i], mem_type);
ttm_mem_type_debug(bo->bdev, mem_type);
}
}
 
static ssize_t ttm_bo_global_show(struct kobject *kobj,
struct attribute *attr,
char *buffer)
{
struct ttm_bo_global *glob =
container_of(kobj, struct ttm_bo_global, kobj);
 
return snprintf(buffer, PAGE_SIZE, "%lu\n",
(unsigned long) atomic_read(&glob->bo_count));
}
 
static struct attribute *ttm_bo_global_attrs[] = {
&ttm_bo_count,
NULL
};
 
static const struct sysfs_ops ttm_bo_global_ops = {
.show = &ttm_bo_global_show
};
 
static struct kobj_type ttm_bo_glob_kobj_type = {
.release = &ttm_bo_global_kobj_release,
.sysfs_ops = &ttm_bo_global_ops,
.default_attrs = ttm_bo_global_attrs
};
#endif
 
 
static inline uint32_t ttm_bo_type_flags(unsigned type)
{
return 1 << (type);
148,12 → 91,14
if (bo->ttm)
ttm_tt_destroy(bo->ttm);
atomic_dec(&bo->glob->bo_count);
if (bo->resv == &bo->ttm_resv)
reservation_object_fini(&bo->ttm_resv);
mutex_destroy(&bo->wu_mutex);
if (bo->destroy)
bo->destroy(bo);
else {
kfree(bo);
}
ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
}
 
void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
161,7 → 106,7
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_type_manager *man;
 
// BUG_ON(!ttm_bo_is_reserved(bo));
lockdep_assert_held(&bo->resv->lock.base);
 
if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 
268,7 → 213,6
return ret;
}
 
#if 0
static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
struct ttm_mem_reg *mem,
bool evict, bool interruptible,
348,9 → 292,11
 
moved:
if (bo->evicted) {
if (bdev->driver->invalidate_caches) {
ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
if (ret)
pr_err("Can not flush read caches\n");
}
bo->evicted = false;
}
 
407,7 → 353,7
int ret;
 
spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
ret = __ttm_bo_reserve(bo, false, true, false, NULL);
 
spin_lock(&bdev->fence_lock);
(void) ttm_bo_wait(bo, false, false, true);
438,7 → 384,7
ttm_bo_add_to_lru(bo);
}
 
ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
}
 
kref_get(&bo->list_kref);
449,8 → 395,8
driver->sync_obj_flush(sync_obj);
driver->sync_obj_unref(&sync_obj);
}
schedule_delayed_work(&bdev->wq,
((HZ / 100) < 1) ? 1 : HZ / 100);
// schedule_delayed_work(&bdev->wq,
// ((HZ / 100) < 1) ? 1 : HZ / 100);
}
 
/**
489,7 → 435,7
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
 
ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
spin_unlock(&glob->lru_lock);
 
ret = driver->sync_obj_wait(sync_obj, false, interruptible);
509,7 → 455,7
return ret;
 
spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
ret = __ttm_bo_reserve(bo, false, true, false, NULL);
 
/*
* We raced, and lost, someone else holds the reservation now,
527,7 → 473,7
spin_unlock(&bdev->fence_lock);
 
if (ret || unlikely(list_empty(&bo->ddestroy))) {
ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
spin_unlock(&glob->lru_lock);
return ret;
}
572,11 → 518,11
kref_get(&nentry->list_kref);
}
 
ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);
ret = __ttm_bo_reserve(entry, false, true, false, NULL);
if (remove_all && ret) {
spin_unlock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(entry, false, false,
false, 0);
ret = __ttm_bo_reserve(entry, false, false,
false, NULL);
spin_lock(&glob->lru_lock);
}
 
615,7 → 561,6
((HZ / 100) < 1) ? 1 : HZ / 100);
}
}
#endif
 
static void ttm_bo_release(struct kref *kref)
{
626,10 → 571,10
 
drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
ttm_mem_io_lock(man, false);
// ttm_mem_io_free_vm(bo);
ttm_mem_io_free_vm(bo);
ttm_mem_io_unlock(man);
// ttm_bo_cleanup_refs_or_queue(bo);
// kref_put(&bo->list_kref, ttm_bo_release_list);
ttm_bo_cleanup_refs_or_queue(bo);
kref_put(&bo->list_kref, ttm_bo_release_list);
}
 
void ttm_bo_unref(struct ttm_buffer_object **p_bo)
641,121 → 586,6
}
EXPORT_SYMBOL(ttm_bo_unref);
 
#if 0
int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
{
return cancel_delayed_work_sync(&bdev->wq);
}
EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 
void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
{
if (resched)
schedule_delayed_work(&bdev->wq,
((HZ / 100) < 1) ? 1 : HZ / 100);
}
EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 
static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
bool no_wait_gpu)
{
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_reg evict_mem;
struct ttm_placement placement;
int ret = 0;
 
spin_lock(&bdev->fence_lock);
ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
spin_unlock(&bdev->fence_lock);
 
if (unlikely(ret != 0)) {
if (ret != -ERESTARTSYS) {
pr_err("Failed to expire sync object before buffer eviction\n");
}
goto out;
}
 
// BUG_ON(!ttm_bo_is_reserved(bo));
 
evict_mem = bo->mem;
evict_mem.mm_node = NULL;
evict_mem.bus.io_reserved_vm = false;
evict_mem.bus.io_reserved_count = 0;
 
placement.fpfn = 0;
placement.lpfn = 0;
placement.num_placement = 0;
placement.num_busy_placement = 0;
bdev->driver->evict_flags(bo, &placement);
ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
no_wait_gpu);
if (ret) {
if (ret != -ERESTARTSYS) {
pr_err("Failed to find memory space for buffer 0x%p eviction\n",
bo);
ttm_bo_mem_space_debug(bo, &placement);
}
goto out;
}
 
ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
no_wait_gpu);
if (ret) {
if (ret != -ERESTARTSYS)
pr_err("Buffer eviction failed\n");
ttm_bo_mem_put(bo, &evict_mem);
goto out;
}
bo->evicted = true;
out:
return ret;
}
 
static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
uint32_t mem_type,
bool interruptible,
bool no_wait_gpu)
{
struct ttm_bo_global *glob = bdev->glob;
struct ttm_mem_type_manager *man = &bdev->man[mem_type];
struct ttm_buffer_object *bo;
int ret = -EBUSY, put_count;
 
spin_lock(&glob->lru_lock);
list_for_each_entry(bo, &man->lru, lru) {
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
if (!ret)
break;
}
 
if (ret) {
spin_unlock(&glob->lru_lock);
return ret;
}
 
kref_get(&bo->list_kref);
 
if (!list_empty(&bo->ddestroy)) {
ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,
no_wait_gpu);
kref_put(&bo->list_kref, ttm_bo_release_list);
return ret;
}
 
put_count = ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock);
 
BUG_ON(ret != 0);
 
ttm_bo_list_ref_sub(bo, put_count, true);
 
ret = ttm_bo_evict(bo, interruptible, no_wait_gpu);
ttm_bo_unreserve(bo);
 
kref_put(&bo->list_kref, ttm_bo_release_list);
return ret;
}
 
void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
{
struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
781,15 → 611,15
int ret;
 
do {
ret = (*man->func->get_node)(man, bo, placement, mem);
ret = (*man->func->get_node)(man, bo, placement, 0, mem);
if (unlikely(ret != 0))
return ret;
if (mem->mm_node)
break;
ret = ttm_mem_evict_first(bdev, mem_type,
interruptible, no_wait_gpu);
if (unlikely(ret != 0))
return ret;
// ret = ttm_mem_evict_first(bdev, mem_type,
// interruptible, no_wait_gpu);
// if (unlikely(ret != 0))
// return ret;
} while (1);
if (mem->mm_node == NULL)
return -ENOMEM;
894,7 → 724,8
 
if (man->has_type && man->use_type) {
type_found = true;
ret = (*man->func->get_node)(man, bo, placement, mem);
ret = (*man->func->get_node)(man, bo, placement,
cur_flags, mem);
if (unlikely(ret))
return ret;
}
934,7 → 765,6
ttm_flag_masked(&cur_flags, placement->busy_placement[i],
~TTM_PL_MASK_MEMTYPE);
 
 
if (mem_type == TTM_PL_SYSTEM) {
mem->mem_type = mem_type;
mem->placement = cur_flags;
965,7 → 795,7
struct ttm_mem_reg mem;
struct ttm_bo_device *bdev = bo->bdev;
 
// BUG_ON(!ttm_bo_is_reserved(bo));
lockdep_assert_held(&bo->resv->lock.base);
 
/*
* FIXME: It's possible to pipeline buffer moves.
996,7 → 826,6
ttm_bo_mem_put(bo, &mem);
return ret;
}
#endif
 
static bool ttm_bo_mem_compat(struct ttm_placement *placement,
struct ttm_mem_reg *mem,
1034,7 → 863,7
int ret;
uint32_t new_flags;
 
// BUG_ON(!ttm_bo_is_reserved(bo));
lockdep_assert_held(&bo->resv->lock.base);
/* Check that range is valid */
if (placement->lpfn || placement->fpfn)
if (placement->fpfn > placement->lpfn ||
1044,8 → 873,8
* Check whether we need to move buffer.
*/
if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
// ret = ttm_bo_move_buffer(bo, placement, interruptible,
// no_wait_gpu);
ret = ttm_bo_move_buffer(bo, placement, interruptible,
no_wait_gpu);
if (ret)
return ret;
} else {
1091,19 → 920,8
{
int ret = 0;
unsigned long num_pages;
struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
bool locked;
 
// ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
if (ret) {
pr_err("Out of kernel memory\n");
if (destroy)
(*destroy)(bo);
else
kfree(bo);
return -ENOMEM;
}
 
num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (num_pages == 0) {
pr_err("Illegal buffer object size\n");
1111,7 → 929,6
(*destroy)(bo);
else
kfree(bo);
// ttm_mem_global_free(mem_glob, acc_size);
return -EINVAL;
}
bo->destroy = destroy;
1141,7 → 958,7
bo->acc_size = acc_size;
bo->sg = sg;
bo->resv = &bo->ttm_resv;
// reservation_object_init(bo->resv);
reservation_object_init(bo->resv);
atomic_inc(&bo->glob->bo_count);
drm_vma_node_reset(&bo->vma_node);
 
1151,19 → 968,23
* For ttm_bo_type_device buffers, allocate
* address space from the device.
*/
// if (likely(!ret) &&
// (bo->type == ttm_bo_type_device ||
// bo->type == ttm_bo_type_sg))
// ret = ttm_bo_setup_vm(bo);
if (likely(!ret) &&
(bo->type == ttm_bo_type_device ||
bo->type == ttm_bo_type_sg))
ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
bo->mem.num_pages);
 
// if (likely(!ret))
// ret = ttm_bo_validate(bo, placement, interruptible, false);
locked = ww_mutex_trylock(&bo->resv->lock);
WARN_ON(!locked);
 
// ttm_bo_unreserve(bo);
if (likely(!ret))
ret = ttm_bo_validate(bo, placement, interruptible, false);
 
// if (unlikely(ret))
// ttm_bo_unref(&bo);
ttm_bo_unreserve(bo);
 
if (unlikely(ret))
ttm_bo_unref(&bo);
 
return ret;
}
EXPORT_SYMBOL(ttm_bo_init);
1197,53 → 1018,6
}
EXPORT_SYMBOL(ttm_bo_dma_acc_size);
 
int ttm_bo_create(struct ttm_bo_device *bdev,
unsigned long size,
enum ttm_bo_type type,
struct ttm_placement *placement,
uint32_t page_alignment,
bool interruptible,
struct file *persistent_swap_storage,
struct ttm_buffer_object **p_bo)
{
struct ttm_buffer_object *bo;
size_t acc_size;
int ret;
 
bo = kzalloc(sizeof(*bo), GFP_KERNEL);
if (unlikely(bo == NULL))
return -ENOMEM;
 
acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
interruptible, persistent_swap_storage, acc_size,
NULL, NULL);
if (likely(ret == 0))
*p_bo = bo;
 
return ret;
}
EXPORT_SYMBOL(ttm_bo_create);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
unsigned long p_size)
{
1250,8 → 1024,6
int ret = -EINVAL;
struct ttm_mem_type_manager *man;
 
ENTER();
 
BUG_ON(type >= TTM_NUM_MEM_TYPES);
man = &bdev->man[type];
BUG_ON(man->has_type);
1277,12 → 1049,9
 
INIT_LIST_HEAD(&man->lru);
 
LEAVE();
 
return 0;
}
 
 
EXPORT_SYMBOL(ttm_bo_init_mm);
void ttm_bo_global_release(struct drm_global_reference *ref)
{
struct ttm_bo_global *glob = ref->object;
1297,12 → 1066,10
struct ttm_bo_global *glob = ref->object;
int ret;
 
ENTER();
 
mutex_init(&glob->device_list_mutex);
spin_lock_init(&glob->lru_lock);
glob->mem_glob = bo_ref->mem_glob;
glob->dummy_read_page = AllocPage();
glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
 
if (unlikely(glob->dummy_read_page == NULL)) {
ret = -ENOMEM;
1314,8 → 1081,6
 
atomic_set(&glob->bo_count, 0);
 
LEAVE();
 
return 0;
 
out_no_drp:
1324,17 → 1089,15
}
EXPORT_SYMBOL(ttm_bo_global_init);
 
 
int ttm_bo_device_init(struct ttm_bo_device *bdev,
struct ttm_bo_global *glob,
struct ttm_bo_driver *driver,
struct address_space *mapping,
uint64_t file_page_offset,
bool need_dma32)
{
int ret = -EINVAL;
 
ENTER();
 
bdev->driver = driver;
 
memset(bdev->man, 0, sizeof(bdev->man));
1349,9 → 1112,9
 
drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
0x10000000);
// INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
INIT_LIST_HEAD(&bdev->ddestroy);
bdev->dev_mapping = NULL;
bdev->dev_mapping = mapping;
bdev->glob = glob;
bdev->need_dma32 = need_dma32;
bdev->val_seq = 0;
1360,8 → 1123,6
list_add_tail(&bdev->device_list, &glob->device_list);
mutex_unlock(&glob->device_list_mutex);
 
LEAVE();
 
return 0;
out_no_sys:
return ret;
1389,6 → 1150,28
return true;
}
 
void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
{
struct ttm_bo_device *bdev = bo->bdev;
 
drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
ttm_mem_io_free_vm(bo);
}
 
void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
{
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 
ttm_mem_io_lock(man, false);
ttm_bo_unmap_virtual_locked(bo);
ttm_mem_io_unlock(man);
}
 
 
EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 
 
int ttm_bo_wait(struct ttm_buffer_object *bo,
bool lazy, bool interruptible, bool no_wait)
{
1400,25 → 1183,47
if (likely(bo->sync_obj == NULL))
return 0;
 
return 0;
while (bo->sync_obj) {
 
if (driver->sync_obj_signaled(bo->sync_obj)) {
void *tmp_obj = bo->sync_obj;
bo->sync_obj = NULL;
clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&tmp_obj);
spin_lock(&bdev->fence_lock);
continue;
}
EXPORT_SYMBOL(ttm_bo_wait);
 
int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
{
struct ttm_bo_device *bdev = bo->bdev;
int ret = 0;
if (no_wait)
return -EBUSY;
 
/*
* Using ttm_bo_reserve makes sure the lru lists are updated.
*/
 
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
ret = driver->sync_obj_wait(sync_obj,
lazy, interruptible);
if (unlikely(ret != 0)) {
driver->sync_obj_unref(&sync_obj);
spin_lock(&bdev->fence_lock);
return ret;
}
EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
spin_lock(&bdev->fence_lock);
if (likely(bo->sync_obj == sync_obj)) {
void *tmp_obj = bo->sync_obj;
bo->sync_obj = NULL;
clear_bit(TTM_BO_PRIV_FLAG_MOVING,
&bo->priv_flags);
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&sync_obj);
driver->sync_obj_unref(&tmp_obj);
spin_lock(&bdev->fence_lock);
} else {
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&sync_obj);
spin_lock(&bdev->fence_lock);
}
}
return 0;
}
EXPORT_SYMBOL(ttm_bo_wait);
 
void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
{
atomic_dec(&bo->cpu_writers);
}
EXPORT_SYMBOL(ttm_bo_synccpu_write_release);