/drivers/video/drm/ttm/ttm_bo.c |
---|
27,54 → 27,1228 |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
/* Notes: |
#define pr_fmt(fmt) "[TTM] " fmt |
#include <drm/ttm/ttm_module.h> |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_placement.h> |
#include <linux/jiffies.h> |
#include <linux/slab.h> |
#include <linux/sched.h> |
#include <linux/mm.h> |
#include <linux/module.h> |
#define pr_err(fmt, ...) \ |
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) |
{ |
mutex_lock(&man->io_reserve_mutex); |
return 0; |
} |
void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) |
{ |
if (likely(man->io_reserve_fastpath)) |
return; |
mutex_unlock(&man->io_reserve_mutex); |
} |
#if 0 |
static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) |
{ |
struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
pr_err(" has_type: %d\n", man->has_type); |
pr_err(" use_type: %d\n", man->use_type); |
pr_err(" flags: 0x%08X\n", man->flags); |
pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); |
pr_err(" size: %llu\n", man->size); |
pr_err(" available_caching: 0x%08X\n", man->available_caching); |
pr_err(" default_caching: 0x%08X\n", man->default_caching); |
if (mem_type != TTM_PL_SYSTEM) |
(*man->func->debug)(man, TTM_PFX); |
} |
static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, |
struct ttm_placement *placement) |
{ |
int i, ret, mem_type; |
pr_err("No space for %p (%lu pages, %luK, %luM)\n", |
bo, bo->mem.num_pages, bo->mem.size >> 10, |
bo->mem.size >> 20); |
for (i = 0; i < placement->num_placement; i++) { |
ret = ttm_mem_type_from_flags(placement->placement[i], |
&mem_type); |
if (ret) |
return; |
pr_err(" placement[%d]=0x%08X (%d)\n", |
i, placement->placement[i], mem_type); |
ttm_mem_type_debug(bo->bdev, mem_type); |
} |
} |
static ssize_t ttm_bo_global_show(struct kobject *kobj, |
struct attribute *attr, |
char *buffer) |
{ |
struct ttm_bo_global *glob = |
container_of(kobj, struct ttm_bo_global, kobj); |
return snprintf(buffer, PAGE_SIZE, "%lu\n", |
(unsigned long) atomic_read(&glob->bo_count)); |
} |
static struct attribute *ttm_bo_global_attrs[] = { |
&ttm_bo_count, |
NULL |
}; |
static const struct sysfs_ops ttm_bo_global_ops = { |
.show = &ttm_bo_global_show |
}; |
static struct kobj_type ttm_bo_glob_kobj_type = { |
.release = &ttm_bo_global_kobj_release, |
.sysfs_ops = &ttm_bo_global_ops, |
.default_attrs = ttm_bo_global_attrs |
}; |
#endif |
static inline uint32_t ttm_bo_type_flags(unsigned type) |
{ |
return 1 << (type); |
} |
static void ttm_bo_release_list(struct kref *list_kref) |
{ |
struct ttm_buffer_object *bo = |
container_of(list_kref, struct ttm_buffer_object, list_kref); |
struct ttm_bo_device *bdev = bo->bdev; |
size_t acc_size = bo->acc_size; |
BUG_ON(atomic_read(&bo->list_kref.refcount)); |
BUG_ON(atomic_read(&bo->kref.refcount)); |
BUG_ON(atomic_read(&bo->cpu_writers)); |
BUG_ON(bo->sync_obj != NULL); |
BUG_ON(bo->mem.mm_node != NULL); |
BUG_ON(!list_empty(&bo->lru)); |
BUG_ON(!list_empty(&bo->ddestroy)); |
if (bo->ttm) |
ttm_tt_destroy(bo->ttm); |
atomic_dec(&bo->glob->bo_count); |
if (bo->destroy) |
bo->destroy(bo); |
else { |
kfree(bo); |
} |
ttm_mem_global_free(bdev->glob->mem_glob, acc_size); |
} |
void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man; |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { |
BUG_ON(!list_empty(&bo->lru)); |
man = &bdev->man[bo->mem.mem_type]; |
list_add_tail(&bo->lru, &man->lru); |
kref_get(&bo->list_kref); |
if (bo->ttm != NULL) { |
list_add_tail(&bo->swap, &bo->glob->swap_lru); |
kref_get(&bo->list_kref); |
} |
} |
} |
EXPORT_SYMBOL(ttm_bo_add_to_lru); |
int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) |
{ |
int put_count = 0; |
if (!list_empty(&bo->swap)) { |
list_del_init(&bo->swap); |
++put_count; |
} |
if (!list_empty(&bo->lru)) { |
list_del_init(&bo->lru); |
++put_count; |
} |
/* |
* TODO: Add a driver hook to delete from |
* driver-specific LRU's here. |
*/ |
return put_count; |
} |
static void ttm_bo_ref_bug(struct kref *list_kref) |
{ |
BUG(); |
} |
void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, |
bool never_free) |
{ |
// kref_sub(&bo->list_kref, count, |
// (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); |
} |
void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) |
{ |
int put_count; |
spin_lock(&bo->glob->lru_lock); |
put_count = ttm_bo_del_from_lru(bo); |
spin_unlock(&bo->glob->lru_lock); |
ttm_bo_list_ref_sub(bo, put_count, true); |
} |
EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); |
/* |
* Call bo->mutex locked. |
*/ |
static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_bo_global *glob = bo->glob; |
int ret = 0; |
uint32_t page_flags = 0; |
// TTM_ASSERT_LOCKED(&bo->mutex); |
bo->ttm = NULL; |
if (bdev->need_dma32) |
page_flags |= TTM_PAGE_FLAG_DMA32; |
switch (bo->type) { |
case ttm_bo_type_device: |
if (zero_alloc) |
page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; |
case ttm_bo_type_kernel: |
bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, |
page_flags, glob->dummy_read_page); |
if (unlikely(bo->ttm == NULL)) |
ret = -ENOMEM; |
break; |
case ttm_bo_type_sg: |
bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, |
page_flags | TTM_PAGE_FLAG_SG, |
glob->dummy_read_page); |
if (unlikely(bo->ttm == NULL)) { |
ret = -ENOMEM; |
break; |
} |
bo->ttm->sg = bo->sg; |
break; |
default: |
pr_err("Illegal buffer object type\n"); |
ret = -EINVAL; |
break; |
} |
return ret; |
} |
#if 0 |
static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, |
struct ttm_mem_reg *mem, |
bool evict, bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); |
bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); |
struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; |
struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; |
int ret = 0; |
if (old_is_pci || new_is_pci || |
((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { |
ret = ttm_mem_io_lock(old_man, true); |
if (unlikely(ret != 0)) |
goto out_err; |
ttm_bo_unmap_virtual_locked(bo); |
ttm_mem_io_unlock(old_man); |
} |
/* |
* Create and bind a ttm if required. |
*/ |
if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { |
if (bo->ttm == NULL) { |
bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); |
ret = ttm_bo_add_ttm(bo, zero); |
if (ret) |
goto out_err; |
} |
ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); |
if (ret) |
goto out_err; |
if (mem->mem_type != TTM_PL_SYSTEM) { |
ret = ttm_tt_bind(bo->ttm, mem); |
if (ret) |
goto out_err; |
} |
if (bo->mem.mem_type == TTM_PL_SYSTEM) { |
if (bdev->driver->move_notify) |
bdev->driver->move_notify(bo, mem); |
bo->mem = *mem; |
mem->mm_node = NULL; |
goto moved; |
} |
} |
if (bdev->driver->move_notify) |
bdev->driver->move_notify(bo, mem); |
if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && |
!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) |
ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); |
else if (bdev->driver->move) |
ret = bdev->driver->move(bo, evict, interruptible, |
no_wait_gpu, mem); |
else |
ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); |
if (ret) { |
if (bdev->driver->move_notify) { |
struct ttm_mem_reg tmp_mem = *mem; |
*mem = bo->mem; |
bo->mem = tmp_mem; |
bdev->driver->move_notify(bo, mem); |
bo->mem = *mem; |
*mem = tmp_mem; |
} |
goto out_err; |
} |
moved: |
if (bo->evicted) { |
ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); |
if (ret) |
pr_err("Can not flush read caches\n"); |
bo->evicted = false; |
} |
if (bo->mem.mm_node) { |
bo->offset = (bo->mem.start << PAGE_SHIFT) + |
bdev->man[bo->mem.mem_type].gpu_offset; |
bo->cur_placement = bo->mem.placement; |
} else |
bo->offset = 0; |
return 0; |
out_err: |
new_man = &bdev->man[bo->mem.mem_type]; |
if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { |
ttm_tt_unbind(bo->ttm); |
ttm_tt_destroy(bo->ttm); |
bo->ttm = NULL; |
} |
return ret; |
} |
/** |
* Call bo::reserved. |
* Will release GPU memory type usage on destruction. |
* This is the place to put in driver specific hooks to release |
* driver private resources. |
* Will release the bo::reserved lock. |
*/ |
static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) |
{ |
if (bo->bdev->driver->move_notify) |
bo->bdev->driver->move_notify(bo, NULL); |
if (bo->ttm) { |
ttm_tt_unbind(bo->ttm); |
ttm_tt_destroy(bo->ttm); |
bo->ttm = NULL; |
} |
ttm_bo_mem_put(bo, &bo->mem); |
ww_mutex_unlock (&bo->resv->lock); |
} |
static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_bo_global *glob = bo->glob; |
struct ttm_bo_driver *driver = bdev->driver; |
void *sync_obj = NULL; |
int put_count; |
int ret; |
spin_lock(&glob->lru_lock); |
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
spin_lock(&bdev->fence_lock); |
(void) ttm_bo_wait(bo, false, false, true); |
if (!ret && !bo->sync_obj) { |
spin_unlock(&bdev->fence_lock); |
put_count = ttm_bo_del_from_lru(bo); |
spin_unlock(&glob->lru_lock); |
ttm_bo_cleanup_memtype_use(bo); |
ttm_bo_list_ref_sub(bo, put_count, true); |
return; |
} |
if (bo->sync_obj) |
sync_obj = driver->sync_obj_ref(bo->sync_obj); |
spin_unlock(&bdev->fence_lock); |
if (!ret) |
ww_mutex_unlock(&bo->resv->lock); |
kref_get(&bo->list_kref); |
list_add_tail(&bo->ddestroy, &bdev->ddestroy); |
spin_unlock(&glob->lru_lock); |
if (sync_obj) { |
driver->sync_obj_flush(sync_obj); |
driver->sync_obj_unref(&sync_obj); |
} |
schedule_delayed_work(&bdev->wq, |
((HZ / 100) < 1) ? 1 : HZ / 100); |
} |
/** |
* function ttm_bo_cleanup_refs_and_unlock |
* If bo idle, remove from delayed- and lru lists, and unref. |
* If not idle, do nothing. |
* |
* We store bo pointer in drm_mm_node struct so we know which bo own a |
* specific node. There is no protection on the pointer, thus to make |
* sure things don't go berserk you have to access this pointer while |
* holding the global lru lock and make sure anytime you free a node you |
* reset the pointer to NULL. |
* Must be called with lru_lock and reservation held, this function |
* will drop both before returning. |
* |
* @interruptible Any sleeps should occur interruptibly. |
* @no_wait_gpu Never wait for gpu. Return -EBUSY instead. |
*/ |
#include "ttm/ttm_module.h" |
#include "ttm/ttm_bo_driver.h" |
#include "ttm/ttm_placement.h" |
#include <linux/module.h> |
static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, |
bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_bo_driver *driver = bdev->driver; |
struct ttm_bo_global *glob = bo->glob; |
int put_count; |
int ret; |
spin_lock(&bdev->fence_lock); |
ret = ttm_bo_wait(bo, false, false, true); |
if (ret && !no_wait_gpu) { |
void *sync_obj; |
int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, |
unsigned long p_size) |
/* |
* Take a reference to the fence and unreserve, |
* at this point the buffer should be dead, so |
* no new sync objects can be attached. |
*/ |
sync_obj = driver->sync_obj_ref(bo->sync_obj); |
spin_unlock(&bdev->fence_lock); |
ww_mutex_unlock(&bo->resv->lock); |
spin_unlock(&glob->lru_lock); |
ret = driver->sync_obj_wait(sync_obj, false, interruptible); |
driver->sync_obj_unref(&sync_obj); |
if (ret) |
return ret; |
/* |
* remove sync_obj with ttm_bo_wait, the wait should be |
* finished, and no new wait object should have been added. |
*/ |
spin_lock(&bdev->fence_lock); |
ret = ttm_bo_wait(bo, false, false, true); |
WARN_ON(ret); |
spin_unlock(&bdev->fence_lock); |
if (ret) |
return ret; |
spin_lock(&glob->lru_lock); |
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
/* |
* We raced, and lost, someone else holds the reservation now, |
* and is probably busy in ttm_bo_cleanup_memtype_use. |
* |
* Even if it's not the case, because we finished waiting any |
* delayed destruction would succeed, so just return success |
* here. |
*/ |
if (ret) { |
spin_unlock(&glob->lru_lock); |
return 0; |
} |
} else |
spin_unlock(&bdev->fence_lock); |
if (ret || unlikely(list_empty(&bo->ddestroy))) { |
ww_mutex_unlock(&bo->resv->lock); |
spin_unlock(&glob->lru_lock); |
return ret; |
} |
put_count = ttm_bo_del_from_lru(bo); |
list_del_init(&bo->ddestroy); |
++put_count; |
spin_unlock(&glob->lru_lock); |
ttm_bo_cleanup_memtype_use(bo); |
ttm_bo_list_ref_sub(bo, put_count, true); |
return 0; |
} |
/** |
* Traverse the delayed list, and call ttm_bo_cleanup_refs on all |
* encountered buffers. |
*/ |
static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) |
{ |
int ret = -EINVAL; |
struct ttm_bo_global *glob = bdev->glob; |
struct ttm_buffer_object *entry = NULL; |
int ret = 0; |
spin_lock(&glob->lru_lock); |
if (list_empty(&bdev->ddestroy)) |
goto out_unlock; |
entry = list_first_entry(&bdev->ddestroy, |
struct ttm_buffer_object, ddestroy); |
kref_get(&entry->list_kref); |
for (;;) { |
struct ttm_buffer_object *nentry = NULL; |
if (entry->ddestroy.next != &bdev->ddestroy) { |
nentry = list_first_entry(&entry->ddestroy, |
struct ttm_buffer_object, ddestroy); |
kref_get(&nentry->list_kref); |
} |
ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); |
if (remove_all && ret) { |
spin_unlock(&glob->lru_lock); |
ret = ttm_bo_reserve_nolru(entry, false, false, |
false, 0); |
spin_lock(&glob->lru_lock); |
} |
if (!ret) |
ret = ttm_bo_cleanup_refs_and_unlock(entry, false, |
!remove_all); |
else |
spin_unlock(&glob->lru_lock); |
kref_put(&entry->list_kref, ttm_bo_release_list); |
entry = nentry; |
if (ret || !entry) |
goto out; |
spin_lock(&glob->lru_lock); |
if (list_empty(&entry->ddestroy)) |
break; |
} |
out_unlock: |
spin_unlock(&glob->lru_lock); |
out: |
if (entry) |
kref_put(&entry->list_kref, ttm_bo_release_list); |
return ret; |
} |
static void ttm_bo_delayed_workqueue(struct work_struct *work) |
{ |
struct ttm_bo_device *bdev = |
container_of(work, struct ttm_bo_device, wq.work); |
if (ttm_bo_delayed_delete(bdev, false)) { |
schedule_delayed_work(&bdev->wq, |
((HZ / 100) < 1) ? 1 : HZ / 100); |
} |
} |
#endif |
static void ttm_bo_release(struct kref *kref) |
{ |
struct ttm_buffer_object *bo = |
container_of(kref, struct ttm_buffer_object, kref); |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; |
write_lock(&bdev->vm_lock); |
if (likely(bo->vm_node != NULL)) { |
// rb_erase(&bo->vm_rb, &bdev->addr_space_rb); |
drm_mm_put_block(bo->vm_node); |
bo->vm_node = NULL; |
} |
write_unlock(&bdev->vm_lock); |
ttm_mem_io_lock(man, false); |
// ttm_mem_io_free_vm(bo); |
ttm_mem_io_unlock(man); |
// ttm_bo_cleanup_refs_or_queue(bo); |
// kref_put(&bo->list_kref, ttm_bo_release_list); |
} |
void ttm_bo_unref(struct ttm_buffer_object **p_bo) |
{ |
struct ttm_buffer_object *bo = *p_bo; |
*p_bo = NULL; |
kref_put(&bo->kref, ttm_bo_release); |
} |
EXPORT_SYMBOL(ttm_bo_unref); |
#if 0 |
int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) |
{ |
return cancel_delayed_work_sync(&bdev->wq); |
} |
EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); |
void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) |
{ |
if (resched) |
schedule_delayed_work(&bdev->wq, |
((HZ / 100) < 1) ? 1 : HZ / 100); |
} |
EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); |
static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_reg evict_mem; |
struct ttm_placement placement; |
int ret = 0; |
spin_lock(&bdev->fence_lock); |
ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
spin_unlock(&bdev->fence_lock); |
if (unlikely(ret != 0)) { |
if (ret != -ERESTARTSYS) { |
pr_err("Failed to expire sync object before buffer eviction\n"); |
} |
goto out; |
} |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
evict_mem = bo->mem; |
evict_mem.mm_node = NULL; |
evict_mem.bus.io_reserved_vm = false; |
evict_mem.bus.io_reserved_count = 0; |
placement.fpfn = 0; |
placement.lpfn = 0; |
placement.num_placement = 0; |
placement.num_busy_placement = 0; |
bdev->driver->evict_flags(bo, &placement); |
ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, |
no_wait_gpu); |
if (ret) { |
if (ret != -ERESTARTSYS) { |
pr_err("Failed to find memory space for buffer 0x%p eviction\n", |
bo); |
ttm_bo_mem_space_debug(bo, &placement); |
} |
goto out; |
} |
ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, |
no_wait_gpu); |
if (ret) { |
if (ret != -ERESTARTSYS) |
pr_err("Buffer eviction failed\n"); |
ttm_bo_mem_put(bo, &evict_mem); |
goto out; |
} |
bo->evicted = true; |
out: |
return ret; |
} |
static int ttm_mem_evict_first(struct ttm_bo_device *bdev, |
uint32_t mem_type, |
bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_global *glob = bdev->glob; |
struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
struct ttm_buffer_object *bo; |
int ret = -EBUSY, put_count; |
spin_lock(&glob->lru_lock); |
list_for_each_entry(bo, &man->lru, lru) { |
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
if (!ret) |
break; |
} |
if (ret) { |
spin_unlock(&glob->lru_lock); |
return ret; |
} |
kref_get(&bo->list_kref); |
if (!list_empty(&bo->ddestroy)) { |
ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, |
no_wait_gpu); |
kref_put(&bo->list_kref, ttm_bo_release_list); |
return ret; |
} |
put_count = ttm_bo_del_from_lru(bo); |
spin_unlock(&glob->lru_lock); |
BUG_ON(ret != 0); |
ttm_bo_list_ref_sub(bo, put_count, true); |
ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); |
ttm_bo_unreserve(bo); |
kref_put(&bo->list_kref, ttm_bo_release_list); |
return ret; |
} |
void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) |
{ |
struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; |
if (mem->mm_node) |
(*man->func->put_node)(man, mem); |
} |
EXPORT_SYMBOL(ttm_bo_mem_put); |
/** |
* Repeatedly evict memory from the LRU for @mem_type until we create enough |
* space, or we've evicted everything and there isn't enough space. |
*/ |
static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, |
uint32_t mem_type, |
struct ttm_placement *placement, |
struct ttm_mem_reg *mem, |
bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
int ret; |
do { |
ret = (*man->func->get_node)(man, bo, placement, mem); |
if (unlikely(ret != 0)) |
return ret; |
if (mem->mm_node) |
break; |
ret = ttm_mem_evict_first(bdev, mem_type, |
interruptible, no_wait_gpu); |
if (unlikely(ret != 0)) |
return ret; |
} while (1); |
if (mem->mm_node == NULL) |
return -ENOMEM; |
mem->mem_type = mem_type; |
return 0; |
} |
static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, |
uint32_t cur_placement, |
uint32_t proposed_placement) |
{ |
uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; |
uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; |
/** |
* Keep current caching if possible. |
*/ |
if ((cur_placement & caching) != 0) |
result |= (cur_placement & caching); |
else if ((man->default_caching & caching) != 0) |
result |= man->default_caching; |
else if ((TTM_PL_FLAG_CACHED & caching) != 0) |
result |= TTM_PL_FLAG_CACHED; |
else if ((TTM_PL_FLAG_WC & caching) != 0) |
result |= TTM_PL_FLAG_WC; |
else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) |
result |= TTM_PL_FLAG_UNCACHED; |
return result; |
} |
static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, |
uint32_t mem_type, |
uint32_t proposed_placement, |
uint32_t *masked_placement) |
{ |
uint32_t cur_flags = ttm_bo_type_flags(mem_type); |
if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) |
return false; |
if ((proposed_placement & man->available_caching) == 0) |
return false; |
cur_flags |= (proposed_placement & man->available_caching); |
*masked_placement = cur_flags; |
return true; |
} |
/** |
* Creates space for memory region @mem according to its type. |
* |
* This function first searches for free space in compatible memory types in |
* the priority order defined by the driver. If free space isn't found, then |
* ttm_bo_mem_force_space is attempted in priority order to evict and find |
* space. |
*/ |
int ttm_bo_mem_space(struct ttm_buffer_object *bo, |
struct ttm_placement *placement, |
struct ttm_mem_reg *mem, |
bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man; |
uint32_t mem_type = TTM_PL_SYSTEM; |
uint32_t cur_flags = 0; |
bool type_found = false; |
bool type_ok = false; |
bool has_erestartsys = false; |
int i, ret; |
if (type >= TTM_NUM_MEM_TYPES) { |
printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type); |
mem->mm_node = NULL; |
for (i = 0; i < placement->num_placement; ++i) { |
ret = ttm_mem_type_from_flags(placement->placement[i], |
&mem_type); |
if (ret) |
return ret; |
man = &bdev->man[mem_type]; |
type_ok = ttm_bo_mt_compatible(man, |
mem_type, |
placement->placement[i], |
&cur_flags); |
if (!type_ok) |
continue; |
cur_flags = ttm_bo_select_caching(man, bo->mem.placement, |
cur_flags); |
/* |
* Use the access and other non-mapping-related flag bits from |
* the memory placement flags to the current flags |
*/ |
ttm_flag_masked(&cur_flags, placement->placement[i], |
~TTM_PL_MASK_MEMTYPE); |
if (mem_type == TTM_PL_SYSTEM) |
break; |
if (man->has_type && man->use_type) { |
type_found = true; |
ret = (*man->func->get_node)(man, bo, placement, mem); |
if (unlikely(ret)) |
return ret; |
} |
if (mem->mm_node) |
break; |
} |
man = &bdev->man[type]; |
if (man->has_type) { |
printk(KERN_ERR TTM_PFX |
"Memory manager already initialized for type %d\n", |
type); |
if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { |
mem->mem_type = mem_type; |
mem->placement = cur_flags; |
return 0; |
} |
if (!type_found) |
return -EINVAL; |
for (i = 0; i < placement->num_busy_placement; ++i) { |
ret = ttm_mem_type_from_flags(placement->busy_placement[i], |
&mem_type); |
if (ret) |
return ret; |
man = &bdev->man[mem_type]; |
if (!man->has_type) |
continue; |
if (!ttm_bo_mt_compatible(man, |
mem_type, |
placement->busy_placement[i], |
&cur_flags)) |
continue; |
cur_flags = ttm_bo_select_caching(man, bo->mem.placement, |
cur_flags); |
/* |
* Use the access and other non-mapping-related flag bits from |
* the memory placement flags to the current flags |
*/ |
ttm_flag_masked(&cur_flags, placement->busy_placement[i], |
~TTM_PL_MASK_MEMTYPE); |
if (mem_type == TTM_PL_SYSTEM) { |
mem->mem_type = mem_type; |
mem->placement = cur_flags; |
mem->mm_node = NULL; |
return 0; |
} |
ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, |
interruptible, no_wait_gpu); |
if (ret == 0 && mem->mm_node) { |
mem->placement = cur_flags; |
return 0; |
} |
if (ret == -ERESTARTSYS) |
has_erestartsys = true; |
} |
ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_mem_space); |
int ttm_bo_move_buffer(struct ttm_buffer_object *bo, |
struct ttm_placement *placement, |
bool interruptible, |
bool no_wait_gpu) |
{ |
int ret = 0; |
struct ttm_mem_reg mem; |
struct ttm_bo_device *bdev = bo->bdev; |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
/* |
* FIXME: It's possible to pipeline buffer moves. |
* Have the driver move function wait for idle when necessary, |
* instead of doing it here. |
*/ |
spin_lock(&bdev->fence_lock); |
ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
spin_unlock(&bdev->fence_lock); |
if (ret) |
return ret; |
mem.num_pages = bo->num_pages; |
mem.size = mem.num_pages << PAGE_SHIFT; |
mem.page_alignment = bo->mem.page_alignment; |
mem.bus.io_reserved_vm = false; |
mem.bus.io_reserved_count = 0; |
/* |
* Determine where to move the buffer. |
*/ |
ret = ttm_bo_mem_space(bo, placement, &mem, |
interruptible, no_wait_gpu); |
if (ret) |
goto out_unlock; |
ret = ttm_bo_handle_move_mem(bo, &mem, false, |
interruptible, no_wait_gpu); |
out_unlock: |
if (ret && mem.mm_node) |
ttm_bo_mem_put(bo, &mem); |
return ret; |
} |
#endif |
static int ttm_bo_mem_compat(struct ttm_placement *placement, |
struct ttm_mem_reg *mem) |
{ |
int i; |
if (mem->mm_node && placement->lpfn != 0 && |
(mem->start < placement->fpfn || |
mem->start + mem->num_pages > placement->lpfn)) |
return -1; |
for (i = 0; i < placement->num_placement; i++) { |
if ((placement->placement[i] & mem->placement & |
TTM_PL_MASK_CACHING) && |
(placement->placement[i] & mem->placement & |
TTM_PL_MASK_MEM)) |
return i; |
} |
return -1; |
} |
int ttm_bo_validate(struct ttm_buffer_object *bo, |
struct ttm_placement *placement, |
bool interruptible, |
bool no_wait_gpu) |
{ |
int ret; |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
/* Check that range is valid */ |
if (placement->lpfn || placement->fpfn) |
if (placement->fpfn > placement->lpfn || |
(placement->lpfn - placement->fpfn) < bo->num_pages) |
return -EINVAL; |
/* |
* Check whether we need to move buffer. |
*/ |
ret = ttm_bo_mem_compat(placement, &bo->mem); |
if (ret < 0) { |
// ret = ttm_bo_move_buffer(bo, placement, interruptible, |
// no_wait_gpu); |
if (ret) |
return ret; |
} else { |
/* |
* Use the access and other non-mapping-related flag bits from |
* the compatible memory placement flags to the active flags |
*/ |
ttm_flag_masked(&bo->mem.placement, placement->placement[ret], |
~TTM_PL_MASK_MEMTYPE); |
} |
/* |
* We might need to add a TTM. |
*/ |
if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { |
ret = ttm_bo_add_ttm(bo, true); |
if (ret) |
return ret; |
} |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_validate); |
int ttm_bo_check_placement(struct ttm_buffer_object *bo, |
struct ttm_placement *placement) |
{ |
BUG_ON((placement->fpfn || placement->lpfn) && |
(bo->mem.num_pages > (placement->lpfn - placement->fpfn))); |
return 0; |
} |
int ttm_bo_init(struct ttm_bo_device *bdev, |
struct ttm_buffer_object *bo, |
unsigned long size, |
enum ttm_bo_type type, |
struct ttm_placement *placement, |
uint32_t page_alignment, |
bool interruptible, |
struct file *persistent_swap_storage, |
size_t acc_size, |
struct sg_table *sg, |
void (*destroy) (struct ttm_buffer_object *)) |
{ |
int ret = 0; |
unsigned long num_pages; |
struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; |
bool locked; |
// ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); |
if (ret) { |
pr_err("Out of kernel memory\n"); |
if (destroy) |
(*destroy)(bo); |
else |
kfree(bo); |
return -ENOMEM; |
} |
num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
if (num_pages == 0) { |
pr_err("Illegal buffer object size\n"); |
if (destroy) |
(*destroy)(bo); |
else |
kfree(bo); |
// ttm_mem_global_free(mem_glob, acc_size); |
return -EINVAL; |
} |
bo->destroy = destroy; |
kref_init(&bo->kref); |
kref_init(&bo->list_kref); |
atomic_set(&bo->cpu_writers, 0); |
INIT_LIST_HEAD(&bo->lru); |
INIT_LIST_HEAD(&bo->ddestroy); |
INIT_LIST_HEAD(&bo->swap); |
INIT_LIST_HEAD(&bo->io_reserve_lru); |
bo->bdev = bdev; |
bo->glob = bdev->glob; |
bo->type = type; |
bo->num_pages = num_pages; |
bo->mem.size = num_pages << PAGE_SHIFT; |
bo->mem.mem_type = TTM_PL_SYSTEM; |
bo->mem.num_pages = bo->num_pages; |
bo->mem.mm_node = NULL; |
bo->mem.page_alignment = page_alignment; |
bo->mem.bus.io_reserved_vm = false; |
bo->mem.bus.io_reserved_count = 0; |
bo->priv_flags = 0; |
bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); |
bo->persistent_swap_storage = persistent_swap_storage; |
bo->acc_size = acc_size; |
bo->sg = sg; |
bo->resv = &bo->ttm_resv; |
// reservation_object_init(bo->resv); |
atomic_inc(&bo->glob->bo_count); |
ret = ttm_bo_check_placement(bo, placement); |
/* |
* For ttm_bo_type_device buffers, allocate |
* address space from the device. |
*/ |
// if (likely(!ret) && |
// (bo->type == ttm_bo_type_device || |
// bo->type == ttm_bo_type_sg)) |
// ret = ttm_bo_setup_vm(bo); |
// if (likely(!ret)) |
// ret = ttm_bo_validate(bo, placement, interruptible, false); |
// ttm_bo_unreserve(bo); |
// if (unlikely(ret)) |
// ttm_bo_unref(&bo); |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_init); |
size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, |
unsigned long bo_size, |
unsigned struct_size) |
{ |
unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; |
size_t size = 0; |
size += ttm_round_pot(struct_size); |
size += PAGE_ALIGN(npages * sizeof(void *)); |
size += ttm_round_pot(sizeof(struct ttm_tt)); |
return size; |
} |
EXPORT_SYMBOL(ttm_bo_acc_size); |
size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, |
unsigned long bo_size, |
unsigned struct_size) |
{ |
unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; |
size_t size = 0; |
size += ttm_round_pot(struct_size); |
size += PAGE_ALIGN(npages * sizeof(void *)); |
size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); |
size += ttm_round_pot(sizeof(struct ttm_dma_tt)); |
return size; |
} |
EXPORT_SYMBOL(ttm_bo_dma_acc_size); |
int ttm_bo_create(struct ttm_bo_device *bdev, |
unsigned long size, |
enum ttm_bo_type type, |
struct ttm_placement *placement, |
uint32_t page_alignment, |
bool interruptible, |
struct file *persistent_swap_storage, |
struct ttm_buffer_object **p_bo) |
{ |
struct ttm_buffer_object *bo; |
size_t acc_size; |
int ret; |
bo = kzalloc(sizeof(*bo), GFP_KERNEL); |
if (unlikely(bo == NULL)) |
return -ENOMEM; |
acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); |
ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, |
interruptible, persistent_swap_storage, acc_size, |
NULL, NULL); |
if (likely(ret == 0)) |
*p_bo = bo; |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_create); |
int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, |
unsigned long p_size) |
{ |
int ret = -EINVAL; |
struct ttm_mem_type_manager *man; |
ENTER(); |
BUG_ON(type >= TTM_NUM_MEM_TYPES); |
man = &bdev->man[type]; |
BUG_ON(man->has_type); |
man->io_reserve_fastpath = true; |
man->use_io_reserve_lru = false; |
mutex_init(&man->io_reserve_mutex); |
INIT_LIST_HEAD(&man->io_reserve_lru); |
ret = bdev->driver->init_mem_type(bdev, type, man); |
if (ret) |
return ret; |
man->bdev = bdev; |
ret = 0; |
if (type != TTM_PL_SYSTEM) { |
if (!p_size) { |
printk(KERN_ERR TTM_PFX |
"Zero size memory manager type %d\n", |
type); |
return ret; |
} |
ret = drm_mm_init(&man->manager, 0, p_size); |
ret = (*man->func->init)(man, p_size); |
if (ret) |
return ret; |
} |
84,21 → 1258,32 |
INIT_LIST_HEAD(&man->lru); |
LEAVE(); |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_init_mm); |
int ttm_bo_global_init(struct ttm_global_reference *ref) |
void ttm_bo_global_release(struct drm_global_reference *ref) |
{ |
struct ttm_bo_global *glob = ref->object; |
} |
EXPORT_SYMBOL(ttm_bo_global_release); |
int ttm_bo_global_init(struct drm_global_reference *ref) |
{ |
struct ttm_bo_global_ref *bo_ref = |
container_of(ref, struct ttm_bo_global_ref, ref); |
struct ttm_bo_global *glob = ref->object; |
int ret; |
// mutex_init(&glob->device_list_mutex); |
// spin_lock_init(&glob->lru_lock); |
ENTER(); |
mutex_init(&glob->device_list_mutex); |
spin_lock_init(&glob->lru_lock); |
glob->mem_glob = bo_ref->mem_glob; |
// glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); |
glob->dummy_read_page = AllocPage(); |
if (unlikely(glob->dummy_read_page == NULL)) { |
ret = -ENOMEM; |
108,34 → 1293,81 |
INIT_LIST_HEAD(&glob->swap_lru); |
INIT_LIST_HEAD(&glob->device_list); |
// ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); |
ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); |
if (unlikely(ret != 0)) { |
printk(KERN_ERR TTM_PFX |
"Could not register buffer object swapout.\n"); |
goto out_no_shrink; |
} |
atomic_set(&glob->bo_count, 0); |
glob->ttm_bo_extra_size = |
ttm_round_pot(sizeof(struct ttm_tt)) + |
ttm_round_pot(sizeof(struct ttm_backend)); |
LEAVE(); |
glob->ttm_bo_size = glob->ttm_bo_extra_size + |
ttm_round_pot(sizeof(struct ttm_buffer_object)); |
return 0; |
atomic_set(&glob->bo_count, 0); |
// kobject_init(&glob->kobj, &ttm_bo_glob_kobj_type); |
// ret = kobject_add(&glob->kobj, ttm_get_kobj(), "buffer_objects"); |
// if (unlikely(ret != 0)) |
// kobject_put(&glob->kobj); |
return ret; |
out_no_shrink: |
__free_page(glob->dummy_read_page); |
out_no_drp: |
kfree(glob); |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_global_init); |
int ttm_bo_device_init(struct ttm_bo_device *bdev, |
struct ttm_bo_global *glob, |
struct ttm_bo_driver *driver, |
uint64_t file_page_offset, |
bool need_dma32) |
{ |
int ret = -EINVAL; |
ENTER(); |
// rwlock_init(&bdev->vm_lock); |
bdev->driver = driver; |
memset(bdev->man, 0, sizeof(bdev->man)); |
/* |
* Initialize the system memory buffer type. |
* Other types need to be driver / IOCTL initialized. |
*/ |
ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); |
if (unlikely(ret != 0)) |
goto out_no_sys; |
bdev->addr_space_rb = RB_ROOT; |
drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); |
// INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); |
INIT_LIST_HEAD(&bdev->ddestroy); |
bdev->dev_mapping = NULL; |
bdev->glob = glob; |
bdev->need_dma32 = need_dma32; |
bdev->val_seq = 0; |
spin_lock_init(&bdev->fence_lock); |
mutex_lock(&glob->device_list_mutex); |
list_add_tail(&bdev->device_list, &glob->device_list); |
mutex_unlock(&glob->device_list_mutex); |
LEAVE(); |
return 0; |
out_no_sys: |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_device_init); |
/* |
* buffer object vm functions. |
*/ |
bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) |
{ |
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { |
if (mem->mem_type == TTM_PL_SYSTEM) |
return false; |
if (man->flags & TTM_MEMTYPE_FLAG_CMA) |
return false; |
if (mem->placement & TTM_PL_FLAG_CACHED) |
return false; |
} |
return true; |
} |
/drivers/video/drm/ttm/ttm_bo_manager.c |
---|
0,0 → 1,151 |
/************************************************************************** |
* |
* Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
#include <drm/ttm/ttm_module.h> |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_placement.h> |
#include <drm/drm_mm.h> |
#include <linux/slab.h> |
#include <linux/spinlock.h> |
#include <linux/module.h> |
/** |
* Currently we use a spinlock for the lock, but a mutex *may* be |
* more appropriate to reduce scheduling latency if the range manager |
* ends up with very fragmented allocation patterns. |
*/ |
struct ttm_range_manager { |
struct drm_mm mm; |
spinlock_t lock; |
}; |
static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, |
struct ttm_buffer_object *bo, |
struct ttm_placement *placement, |
struct ttm_mem_reg *mem) |
{ |
struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; |
struct drm_mm *mm = &rman->mm; |
struct drm_mm_node *node = NULL; |
unsigned long lpfn; |
int ret; |
lpfn = placement->lpfn; |
if (!lpfn) |
lpfn = man->size; |
do { |
ret = drm_mm_pre_get(mm); |
if (unlikely(ret)) |
return ret; |
spin_lock(&rman->lock); |
node = drm_mm_search_free_in_range(mm, |
mem->num_pages, mem->page_alignment, |
placement->fpfn, lpfn, 1); |
if (unlikely(node == NULL)) { |
spin_unlock(&rman->lock); |
return 0; |
} |
node = drm_mm_get_block_atomic_range(node, mem->num_pages, |
mem->page_alignment, |
placement->fpfn, |
lpfn); |
spin_unlock(&rman->lock); |
} while (node == NULL); |
mem->mm_node = node; |
mem->start = node->start; |
return 0; |
} |
static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, |
struct ttm_mem_reg *mem) |
{ |
struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; |
if (mem->mm_node) { |
spin_lock(&rman->lock); |
drm_mm_put_block(mem->mm_node); |
spin_unlock(&rman->lock); |
mem->mm_node = NULL; |
} |
} |
static int ttm_bo_man_init(struct ttm_mem_type_manager *man, |
unsigned long p_size) |
{ |
struct ttm_range_manager *rman; |
rman = kzalloc(sizeof(*rman), GFP_KERNEL); |
if (!rman) |
return -ENOMEM; |
drm_mm_init(&rman->mm, 0, p_size); |
spin_lock_init(&rman->lock); |
man->priv = rman; |
return 0; |
} |
static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man) |
{ |
struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; |
struct drm_mm *mm = &rman->mm; |
spin_lock(&rman->lock); |
if (drm_mm_clean(mm)) { |
drm_mm_takedown(mm); |
spin_unlock(&rman->lock); |
kfree(rman); |
man->priv = NULL; |
return 0; |
} |
spin_unlock(&rman->lock); |
return -EBUSY; |
} |
static void ttm_bo_man_debug(struct ttm_mem_type_manager *man, |
const char *prefix) |
{ |
struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; |
spin_lock(&rman->lock); |
drm_mm_debug_table(&rman->mm, prefix); |
spin_unlock(&rman->lock); |
} |
const struct ttm_mem_type_manager_func ttm_bo_manager_func = { |
ttm_bo_man_init, |
ttm_bo_man_takedown, |
ttm_bo_man_get_node, |
ttm_bo_man_put_node, |
ttm_bo_man_debug |
}; |
EXPORT_SYMBOL(ttm_bo_manager_func); |
/drivers/video/drm/ttm/ttm_bo_util.c |
---|
0,0 → 1,706 |
/************************************************************************** |
* |
* Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_placement.h> |
#include <linux/io.h> |
#include <linux/highmem.h> |
#include <linux/wait.h> |
#include <linux/slab.h> |
#include <linux/vmalloc.h> |
#include <linux/module.h> |
void ttm_bo_free_old_node(struct ttm_buffer_object *bo) |
{ |
ttm_bo_mem_put(bo, &bo->mem); |
} |
int ttm_bo_move_ttm(struct ttm_buffer_object *bo, |
bool evict, |
bool no_wait_gpu, struct ttm_mem_reg *new_mem) |
{ |
struct ttm_tt *ttm = bo->ttm; |
struct ttm_mem_reg *old_mem = &bo->mem; |
int ret; |
if (old_mem->mem_type != TTM_PL_SYSTEM) { |
ttm_tt_unbind(ttm); |
ttm_bo_free_old_node(bo); |
ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, |
TTM_PL_MASK_MEM); |
old_mem->mem_type = TTM_PL_SYSTEM; |
} |
ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); |
if (unlikely(ret != 0)) |
return ret; |
if (new_mem->mem_type != TTM_PL_SYSTEM) { |
ret = ttm_tt_bind(ttm, new_mem); |
if (unlikely(ret != 0)) |
return ret; |
} |
*old_mem = *new_mem; |
new_mem->mm_node = NULL; |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_move_ttm); |
int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) |
{ |
if (likely(man->io_reserve_fastpath)) |
return 0; |
if (interruptible) |
return mutex_lock_interruptible(&man->io_reserve_mutex); |
mutex_lock(&man->io_reserve_mutex); |
return 0; |
} |
EXPORT_SYMBOL(ttm_mem_io_lock); |
void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) |
{ |
if (likely(man->io_reserve_fastpath)) |
return; |
mutex_unlock(&man->io_reserve_mutex); |
} |
EXPORT_SYMBOL(ttm_mem_io_unlock); |
static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) |
{ |
struct ttm_buffer_object *bo; |
if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) |
return -EAGAIN; |
bo = list_first_entry(&man->io_reserve_lru, |
struct ttm_buffer_object, |
io_reserve_lru); |
list_del_init(&bo->io_reserve_lru); |
ttm_bo_unmap_virtual_locked(bo); |
return 0; |
} |
int ttm_mem_io_reserve(struct ttm_bo_device *bdev, |
struct ttm_mem_reg *mem) |
{ |
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
int ret = 0; |
if (!bdev->driver->io_mem_reserve) |
return 0; |
if (likely(man->io_reserve_fastpath)) |
return bdev->driver->io_mem_reserve(bdev, mem); |
if (bdev->driver->io_mem_reserve && |
mem->bus.io_reserved_count++ == 0) { |
retry: |
ret = bdev->driver->io_mem_reserve(bdev, mem); |
if (ret == -EAGAIN) { |
ret = ttm_mem_io_evict(man); |
if (ret == 0) |
goto retry; |
} |
} |
return ret; |
} |
EXPORT_SYMBOL(ttm_mem_io_reserve); |
void ttm_mem_io_free(struct ttm_bo_device *bdev, |
struct ttm_mem_reg *mem) |
{ |
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
if (likely(man->io_reserve_fastpath)) |
return; |
if (bdev->driver->io_mem_reserve && |
--mem->bus.io_reserved_count == 0 && |
bdev->driver->io_mem_free) |
bdev->driver->io_mem_free(bdev, mem); |
} |
EXPORT_SYMBOL(ttm_mem_io_free); |
int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) |
{ |
struct ttm_mem_reg *mem = &bo->mem; |
int ret; |
if (!mem->bus.io_reserved_vm) { |
struct ttm_mem_type_manager *man = |
&bo->bdev->man[mem->mem_type]; |
ret = ttm_mem_io_reserve(bo->bdev, mem); |
if (unlikely(ret != 0)) |
return ret; |
mem->bus.io_reserved_vm = true; |
if (man->use_io_reserve_lru) |
list_add_tail(&bo->io_reserve_lru, |
&man->io_reserve_lru); |
} |
return 0; |
} |
void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) |
{ |
struct ttm_mem_reg *mem = &bo->mem; |
if (mem->bus.io_reserved_vm) { |
mem->bus.io_reserved_vm = false; |
list_del_init(&bo->io_reserve_lru); |
ttm_mem_io_free(bo->bdev, mem); |
} |
} |
int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, |
void **virtual) |
{ |
struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
int ret; |
void *addr; |
*virtual = NULL; |
(void) ttm_mem_io_lock(man, false); |
ret = ttm_mem_io_reserve(bdev, mem); |
ttm_mem_io_unlock(man); |
if (ret || !mem->bus.is_iomem) |
return ret; |
if (mem->bus.addr) { |
addr = mem->bus.addr; |
} else { |
if (mem->placement & TTM_PL_FLAG_WC) |
addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); |
else |
addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); |
if (!addr) { |
(void) ttm_mem_io_lock(man, false); |
ttm_mem_io_free(bdev, mem); |
ttm_mem_io_unlock(man); |
return -ENOMEM; |
} |
} |
*virtual = addr; |
return 0; |
} |
void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, |
void *virtual) |
{ |
struct ttm_mem_type_manager *man; |
man = &bdev->man[mem->mem_type]; |
if (virtual && mem->bus.addr == NULL) |
iounmap(virtual); |
(void) ttm_mem_io_lock(man, false); |
ttm_mem_io_free(bdev, mem); |
ttm_mem_io_unlock(man); |
} |
static int ttm_copy_io_page(void *dst, void *src, unsigned long page) |
{ |
uint32_t *dstP = |
(uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); |
uint32_t *srcP = |
(uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); |
int i; |
for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) |
iowrite32(ioread32(srcP++), dstP++); |
return 0; |
} |
static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, |
unsigned long page, |
pgprot_t prot) |
{ |
struct page *d = ttm->pages[page]; |
void *dst; |
if (!d) |
return -ENOMEM; |
src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); |
#ifdef CONFIG_X86 |
dst = kmap_atomic_prot(d, prot); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
dst = vmap(&d, 1, 0, prot); |
else |
dst = kmap(d); |
#endif |
if (!dst) |
return -ENOMEM; |
memcpy_fromio(dst, src, PAGE_SIZE); |
#ifdef CONFIG_X86 |
kunmap_atomic(dst); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
vunmap(dst); |
else |
kunmap(d); |
#endif |
return 0; |
} |
static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, |
unsigned long page, |
pgprot_t prot) |
{ |
struct page *s = ttm->pages[page]; |
void *src; |
if (!s) |
return -ENOMEM; |
dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); |
#ifdef CONFIG_X86 |
src = kmap_atomic_prot(s, prot); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
src = vmap(&s, 1, 0, prot); |
else |
src = kmap(s); |
#endif |
if (!src) |
return -ENOMEM; |
memcpy_toio(dst, src, PAGE_SIZE); |
#ifdef CONFIG_X86 |
kunmap_atomic(src); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
vunmap(src); |
else |
kunmap(s); |
#endif |
return 0; |
} |
int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, |
bool evict, bool no_wait_gpu, |
struct ttm_mem_reg *new_mem) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; |
struct ttm_tt *ttm = bo->ttm; |
struct ttm_mem_reg *old_mem = &bo->mem; |
struct ttm_mem_reg old_copy = *old_mem; |
void *old_iomap; |
void *new_iomap; |
int ret; |
unsigned long i; |
unsigned long page; |
unsigned long add = 0; |
int dir; |
ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); |
if (ret) |
return ret; |
ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); |
if (ret) |
goto out; |
if (old_iomap == NULL && new_iomap == NULL) |
goto out2; |
if (old_iomap == NULL && ttm == NULL) |
goto out2; |
if (ttm->state == tt_unpopulated) { |
ret = ttm->bdev->driver->ttm_tt_populate(ttm); |
if (ret) { |
/* if we fail here don't nuke the mm node |
* as the bo still owns it */ |
old_copy.mm_node = NULL; |
goto out1; |
} |
} |
add = 0; |
dir = 1; |
if ((old_mem->mem_type == new_mem->mem_type) && |
(new_mem->start < old_mem->start + old_mem->size)) { |
dir = -1; |
add = new_mem->num_pages - 1; |
} |
for (i = 0; i < new_mem->num_pages; ++i) { |
page = i * dir + add; |
if (old_iomap == NULL) { |
pgprot_t prot = ttm_io_prot(old_mem->placement, |
PAGE_KERNEL); |
ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, |
prot); |
} else if (new_iomap == NULL) { |
pgprot_t prot = ttm_io_prot(new_mem->placement, |
PAGE_KERNEL); |
ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, |
prot); |
} else |
ret = ttm_copy_io_page(new_iomap, old_iomap, page); |
if (ret) { |
/* failing here, means keep old copy as-is */ |
old_copy.mm_node = NULL; |
goto out1; |
} |
} |
mb(); |
out2: |
old_copy = *old_mem; |
*old_mem = *new_mem; |
new_mem->mm_node = NULL; |
if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { |
ttm_tt_unbind(ttm); |
ttm_tt_destroy(ttm); |
bo->ttm = NULL; |
} |
out1: |
ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); |
out: |
ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); |
ttm_bo_mem_put(bo, &old_copy); |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_move_memcpy); |
static void ttm_transfered_destroy(struct ttm_buffer_object *bo) |
{ |
kfree(bo); |
} |
/** |
* ttm_buffer_object_transfer |
* |
* @bo: A pointer to a struct ttm_buffer_object. |
* @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, |
* holding the data of @bo with the old placement. |
* |
* This is a utility function that may be called after an accelerated move |
* has been scheduled. A new buffer object is created as a placeholder for |
* the old data while it's being copied. When that buffer object is idle, |
* it can be destroyed, releasing the space of the old placement. |
* Returns: |
* !0: Failure. |
*/ |
static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, |
struct ttm_buffer_object **new_obj) |
{ |
struct ttm_buffer_object *fbo; |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_bo_driver *driver = bdev->driver; |
int ret; |
fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); |
if (!fbo) |
return -ENOMEM; |
*fbo = *bo; |
/** |
* Fix up members that we shouldn't copy directly: |
* TODO: Explicit member copy would probably be better here. |
*/ |
INIT_LIST_HEAD(&fbo->ddestroy); |
INIT_LIST_HEAD(&fbo->lru); |
INIT_LIST_HEAD(&fbo->swap); |
INIT_LIST_HEAD(&fbo->io_reserve_lru); |
fbo->vm_node = NULL; |
atomic_set(&fbo->cpu_writers, 0); |
spin_lock(&bdev->fence_lock); |
if (bo->sync_obj) |
fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); |
else |
fbo->sync_obj = NULL; |
spin_unlock(&bdev->fence_lock); |
kref_init(&fbo->list_kref); |
kref_init(&fbo->kref); |
fbo->destroy = &ttm_transfered_destroy; |
fbo->acc_size = 0; |
fbo->resv = &fbo->ttm_resv; |
reservation_object_init(fbo->resv); |
ret = ww_mutex_trylock(&fbo->resv->lock); |
WARN_ON(!ret); |
*new_obj = fbo; |
return 0; |
} |
pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) |
{ |
#if defined(__i386__) || defined(__x86_64__) |
if (caching_flags & TTM_PL_FLAG_WC) |
tmp = pgprot_writecombine(tmp); |
else if (boot_cpu_data.x86 > 3) |
tmp = pgprot_noncached(tmp); |
#elif defined(__powerpc__) |
if (!(caching_flags & TTM_PL_FLAG_CACHED)) { |
pgprot_val(tmp) |= _PAGE_NO_CACHE; |
if (caching_flags & TTM_PL_FLAG_UNCACHED) |
pgprot_val(tmp) |= _PAGE_GUARDED; |
} |
#endif |
#if defined(__ia64__) |
if (caching_flags & TTM_PL_FLAG_WC) |
tmp = pgprot_writecombine(tmp); |
else |
tmp = pgprot_noncached(tmp); |
#endif |
#if defined(__sparc__) || defined(__mips__) |
if (!(caching_flags & TTM_PL_FLAG_CACHED)) |
tmp = pgprot_noncached(tmp); |
#endif |
return tmp; |
} |
EXPORT_SYMBOL(ttm_io_prot); |
static int ttm_bo_ioremap(struct ttm_buffer_object *bo, |
unsigned long offset, |
unsigned long size, |
struct ttm_bo_kmap_obj *map) |
{ |
struct ttm_mem_reg *mem = &bo->mem; |
if (bo->mem.bus.addr) { |
map->bo_kmap_type = ttm_bo_map_premapped; |
map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); |
} else { |
map->bo_kmap_type = ttm_bo_map_iomap; |
if (mem->placement & TTM_PL_FLAG_WC) |
map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, |
size); |
else |
map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset, |
size); |
} |
return (!map->virtual) ? -ENOMEM : 0; |
} |
static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, |
unsigned long start_page, |
unsigned long num_pages, |
struct ttm_bo_kmap_obj *map) |
{ |
struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; |
struct ttm_tt *ttm = bo->ttm; |
int ret; |
BUG_ON(!ttm); |
if (ttm->state == tt_unpopulated) { |
ret = ttm->bdev->driver->ttm_tt_populate(ttm); |
if (ret) |
return ret; |
} |
if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { |
/* |
* We're mapping a single page, and the desired |
* page protection is consistent with the bo. |
*/ |
map->bo_kmap_type = ttm_bo_map_kmap; |
map->page = ttm->pages[start_page]; |
map->virtual = kmap(map->page); |
} else { |
/* |
* We need to use vmap to get the desired page protection |
* or to make the buffer object look contiguous. |
*/ |
prot = (mem->placement & TTM_PL_FLAG_CACHED) ? |
PAGE_KERNEL : |
ttm_io_prot(mem->placement, PAGE_KERNEL); |
map->bo_kmap_type = ttm_bo_map_vmap; |
map->virtual = vmap(ttm->pages + start_page, num_pages, |
0, prot); |
} |
return (!map->virtual) ? -ENOMEM : 0; |
} |
int ttm_bo_kmap(struct ttm_buffer_object *bo, |
unsigned long start_page, unsigned long num_pages, |
struct ttm_bo_kmap_obj *map) |
{ |
struct ttm_mem_type_manager *man = |
&bo->bdev->man[bo->mem.mem_type]; |
unsigned long offset, size; |
int ret; |
BUG_ON(!list_empty(&bo->swap)); |
map->virtual = NULL; |
map->bo = bo; |
if (num_pages > bo->num_pages) |
return -EINVAL; |
if (start_page > bo->num_pages) |
return -EINVAL; |
#if 0 |
if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) |
return -EPERM; |
#endif |
(void) ttm_mem_io_lock(man, false); |
ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); |
ttm_mem_io_unlock(man); |
if (ret) |
return ret; |
if (!bo->mem.bus.is_iomem) { |
return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); |
} else { |
offset = start_page << PAGE_SHIFT; |
size = num_pages << PAGE_SHIFT; |
return ttm_bo_ioremap(bo, offset, size, map); |
} |
} |
EXPORT_SYMBOL(ttm_bo_kmap); |
void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) |
{ |
struct ttm_buffer_object *bo = map->bo; |
struct ttm_mem_type_manager *man = |
&bo->bdev->man[bo->mem.mem_type]; |
if (!map->virtual) |
return; |
switch (map->bo_kmap_type) { |
case ttm_bo_map_iomap: |
iounmap(map->virtual); |
break; |
case ttm_bo_map_vmap: |
vunmap(map->virtual); |
break; |
case ttm_bo_map_kmap: |
kunmap(map->page); |
break; |
case ttm_bo_map_premapped: |
break; |
default: |
BUG(); |
} |
(void) ttm_mem_io_lock(man, false); |
ttm_mem_io_free(map->bo->bdev, &map->bo->mem); |
ttm_mem_io_unlock(man); |
map->virtual = NULL; |
map->page = NULL; |
} |
EXPORT_SYMBOL(ttm_bo_kunmap); |
int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, |
void *sync_obj, |
bool evict, |
bool no_wait_gpu, |
struct ttm_mem_reg *new_mem) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_bo_driver *driver = bdev->driver; |
struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; |
struct ttm_mem_reg *old_mem = &bo->mem; |
int ret; |
struct ttm_buffer_object *ghost_obj; |
void *tmp_obj = NULL; |
spin_lock(&bdev->fence_lock); |
if (bo->sync_obj) { |
tmp_obj = bo->sync_obj; |
bo->sync_obj = NULL; |
} |
bo->sync_obj = driver->sync_obj_ref(sync_obj); |
if (evict) { |
ret = ttm_bo_wait(bo, false, false, false); |
spin_unlock(&bdev->fence_lock); |
if (tmp_obj) |
driver->sync_obj_unref(&tmp_obj); |
if (ret) |
return ret; |
if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && |
(bo->ttm != NULL)) { |
ttm_tt_unbind(bo->ttm); |
ttm_tt_destroy(bo->ttm); |
bo->ttm = NULL; |
} |
ttm_bo_free_old_node(bo); |
} else { |
/** |
* This should help pipeline ordinary buffer moves. |
* |
* Hang old buffer memory on a new buffer object, |
* and leave it to be released when the GPU |
* operation has completed. |
*/ |
set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); |
spin_unlock(&bdev->fence_lock); |
if (tmp_obj) |
driver->sync_obj_unref(&tmp_obj); |
ret = ttm_buffer_object_transfer(bo, &ghost_obj); |
if (ret) |
return ret; |
/** |
* If we're not moving to fixed memory, the TTM object |
* needs to stay alive. Otherwhise hang it on the ghost |
* bo to be unbound and destroyed. |
*/ |
if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) |
ghost_obj->ttm = NULL; |
else |
bo->ttm = NULL; |
ttm_bo_unreserve(ghost_obj); |
ttm_bo_unref(&ghost_obj); |
} |
*old_mem = *new_mem; |
new_mem->mm_node = NULL; |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); |
/drivers/video/drm/ttm/ttm_execbuf_util.c |
---|
0,0 → 1,235 |
/************************************************************************** |
* |
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
struct ww_acquire_ctx{}; |
#include <drm/ttm/ttm_execbuf_util.h> |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_placement.h> |
#include <linux/wait.h> |
#include <linux/sched.h> |
#include <linux/module.h> |
static void ttm_eu_backoff_reservation_locked(struct list_head *list, |
struct ww_acquire_ctx *ticket) |
{ |
struct ttm_validate_buffer *entry; |
list_for_each_entry(entry, list, head) { |
struct ttm_buffer_object *bo = entry->bo; |
if (!entry->reserved) |
continue; |
entry->reserved = false; |
if (entry->removed) { |
ttm_bo_add_to_lru(bo); |
entry->removed = false; |
} |
// ww_mutex_unlock(&bo->resv->lock); |
} |
} |
static void ttm_eu_del_from_lru_locked(struct list_head *list) |
{ |
struct ttm_validate_buffer *entry; |
list_for_each_entry(entry, list, head) { |
struct ttm_buffer_object *bo = entry->bo; |
if (!entry->reserved) |
continue; |
if (!entry->removed) { |
entry->put_count = ttm_bo_del_from_lru(bo); |
entry->removed = true; |
} |
} |
} |
static void ttm_eu_list_ref_sub(struct list_head *list) |
{ |
struct ttm_validate_buffer *entry; |
list_for_each_entry(entry, list, head) { |
struct ttm_buffer_object *bo = entry->bo; |
if (entry->put_count) { |
ttm_bo_list_ref_sub(bo, entry->put_count, true); |
entry->put_count = 0; |
} |
} |
} |
void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, |
struct list_head *list) |
{ |
struct ttm_validate_buffer *entry; |
struct ttm_bo_global *glob; |
if (list_empty(list)) |
return; |
entry = list_first_entry(list, struct ttm_validate_buffer, head); |
glob = entry->bo->glob; |
spin_lock(&glob->lru_lock); |
ttm_eu_backoff_reservation_locked(list, ticket); |
// ww_acquire_fini(ticket); |
spin_unlock(&glob->lru_lock); |
} |
EXPORT_SYMBOL(ttm_eu_backoff_reservation); |
/* |
* Reserve buffers for validation. |
* |
* If a buffer in the list is marked for CPU access, we back off and |
* wait for that buffer to become free for GPU access. |
* |
* If a buffer is reserved for another validation, the validator with |
* the highest validation sequence backs off and waits for that buffer |
* to become unreserved. This prevents deadlocks when validating multiple |
* buffers in different orders. |
*/ |
int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, |
struct list_head *list) |
{ |
struct ttm_bo_global *glob; |
struct ttm_validate_buffer *entry; |
int ret; |
if (list_empty(list)) |
return 0; |
list_for_each_entry(entry, list, head) { |
entry->reserved = false; |
entry->put_count = 0; |
entry->removed = false; |
} |
entry = list_first_entry(list, struct ttm_validate_buffer, head); |
glob = entry->bo->glob; |
// ww_acquire_init(ticket, &reservation_ww_class); |
retry: |
list_for_each_entry(entry, list, head) { |
struct ttm_buffer_object *bo = entry->bo; |
/* already slowpath reserved? */ |
if (entry->reserved) |
continue; |
ret = ttm_bo_reserve_nolru(bo, true, false, true, ticket); |
if (ret == -EDEADLK) { |
/* uh oh, we lost out, drop every reservation and try |
* to only reserve this buffer, then start over if |
* this succeeds. |
*/ |
spin_lock(&glob->lru_lock); |
ttm_eu_backoff_reservation_locked(list, ticket); |
spin_unlock(&glob->lru_lock); |
ttm_eu_list_ref_sub(list); |
// ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, |
// ticket); |
if (unlikely(ret != 0)) { |
if (ret == -EINTR) |
ret = -ERESTARTSYS; |
goto err_fini; |
} |
entry->reserved = true; |
if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { |
ret = -EBUSY; |
goto err; |
} |
goto retry; |
} else if (ret) |
goto err; |
entry->reserved = true; |
if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { |
ret = -EBUSY; |
goto err; |
} |
} |
// ww_acquire_done(ticket); |
spin_lock(&glob->lru_lock); |
ttm_eu_del_from_lru_locked(list); |
spin_unlock(&glob->lru_lock); |
ttm_eu_list_ref_sub(list); |
return 0; |
err: |
spin_lock(&glob->lru_lock); |
ttm_eu_backoff_reservation_locked(list, ticket); |
spin_unlock(&glob->lru_lock); |
ttm_eu_list_ref_sub(list); |
err_fini: |
// ww_acquire_done(ticket); |
// ww_acquire_fini(ticket); |
return ret; |
} |
EXPORT_SYMBOL(ttm_eu_reserve_buffers); |
void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, |
struct list_head *list, void *sync_obj) |
{ |
struct ttm_validate_buffer *entry; |
struct ttm_buffer_object *bo; |
struct ttm_bo_global *glob; |
struct ttm_bo_device *bdev; |
struct ttm_bo_driver *driver; |
if (list_empty(list)) |
return; |
bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo; |
bdev = bo->bdev; |
driver = bdev->driver; |
glob = bo->glob; |
spin_lock(&glob->lru_lock); |
spin_lock(&bdev->fence_lock); |
list_for_each_entry(entry, list, head) { |
bo = entry->bo; |
entry->old_sync_obj = bo->sync_obj; |
bo->sync_obj = driver->sync_obj_ref(sync_obj); |
ttm_bo_add_to_lru(bo); |
// ww_mutex_unlock(&bo->resv->lock); |
entry->reserved = false; |
} |
spin_unlock(&bdev->fence_lock); |
spin_unlock(&glob->lru_lock); |
// ww_acquire_fini(ticket); |
list_for_each_entry(entry, list, head) { |
if (entry->old_sync_obj) |
driver->sync_obj_unref(&entry->old_sync_obj); |
} |
} |
EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); |
/drivers/video/drm/ttm/ttm_memory.c |
---|
0,0 → 1,605 |
/************************************************************************** |
* |
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
#define pr_fmt(fmt) "[TTM] " fmt |
#include <drm/ttm/ttm_memory.h> |
#include <drm/ttm/ttm_module.h> |
#include <drm/ttm/ttm_page_alloc.h> |
#include <linux/spinlock.h> |
#include <linux/sched.h> |
#include <linux/wait.h> |
#include <linux/mm.h> |
#include <linux/module.h> |
#include <linux/slab.h> |
struct sysinfo { |
u32_t totalram; /* Total usable main memory size */ |
u32_t freeram; /* Available memory size */ |
u32_t sharedram; /* Amount of shared memory */ |
u32_t bufferram; /* Memory used by buffers */ |
u32_t totalswap; /* Total swap space size */ |
u32_t freeswap; /* swap space still available */ |
u32_t totalhigh; /* Total high memory size */ |
u32_t freehigh; /* Available high memory size */ |
u32_t mem_unit; /* Memory unit size in bytes */ |
}; |
#define TTM_MEMORY_ALLOC_RETRIES 4 |
struct ttm_mem_zone { |
struct kobject kobj; |
struct ttm_mem_global *glob; |
const char *name; |
uint64_t zone_mem; |
uint64_t emer_mem; |
uint64_t max_mem; |
uint64_t swap_limit; |
uint64_t used_mem; |
}; |
#if 0 |
static struct attribute ttm_mem_sys = { |
.name = "zone_memory", |
.mode = S_IRUGO |
}; |
static struct attribute ttm_mem_emer = { |
.name = "emergency_memory", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_mem_max = { |
.name = "available_memory", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_mem_swap = { |
.name = "swap_limit", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_mem_used = { |
.name = "used_memory", |
.mode = S_IRUGO |
}; |
#endif |
static void ttm_mem_zone_kobj_release(struct kobject *kobj) |
{ |
struct ttm_mem_zone *zone = |
container_of(kobj, struct ttm_mem_zone, kobj); |
pr_info("Zone %7s: Used memory at exit: %llu kiB\n", |
zone->name, (unsigned long long)zone->used_mem >> 10); |
kfree(zone); |
} |
#if 0 |
static ssize_t ttm_mem_zone_show(struct kobject *kobj, |
struct attribute *attr, |
char *buffer) |
{ |
struct ttm_mem_zone *zone = |
container_of(kobj, struct ttm_mem_zone, kobj); |
uint64_t val = 0; |
spin_lock(&zone->glob->lock); |
if (attr == &ttm_mem_sys) |
val = zone->zone_mem; |
else if (attr == &ttm_mem_emer) |
val = zone->emer_mem; |
else if (attr == &ttm_mem_max) |
val = zone->max_mem; |
else if (attr == &ttm_mem_swap) |
val = zone->swap_limit; |
else if (attr == &ttm_mem_used) |
val = zone->used_mem; |
spin_unlock(&zone->glob->lock); |
return snprintf(buffer, PAGE_SIZE, "%llu\n", |
(unsigned long long) val >> 10); |
} |
static void ttm_check_swapping(struct ttm_mem_global *glob); |
static ssize_t ttm_mem_zone_store(struct kobject *kobj, |
struct attribute *attr, |
const char *buffer, |
size_t size) |
{ |
struct ttm_mem_zone *zone = |
container_of(kobj, struct ttm_mem_zone, kobj); |
int chars; |
unsigned long val; |
uint64_t val64; |
chars = sscanf(buffer, "%lu", &val); |
if (chars == 0) |
return size; |
val64 = val; |
val64 <<= 10; |
spin_lock(&zone->glob->lock); |
if (val64 > zone->zone_mem) |
val64 = zone->zone_mem; |
if (attr == &ttm_mem_emer) { |
zone->emer_mem = val64; |
if (zone->max_mem > val64) |
zone->max_mem = val64; |
} else if (attr == &ttm_mem_max) { |
zone->max_mem = val64; |
if (zone->emer_mem < val64) |
zone->emer_mem = val64; |
} else if (attr == &ttm_mem_swap) |
zone->swap_limit = val64; |
spin_unlock(&zone->glob->lock); |
ttm_check_swapping(zone->glob); |
return size; |
} |
#endif |
//static struct attribute *ttm_mem_zone_attrs[] = { |
// &ttm_mem_sys, |
// &ttm_mem_emer, |
// &ttm_mem_max, |
// &ttm_mem_swap, |
// &ttm_mem_used, |
// NULL |
//}; |
//static const struct sysfs_ops ttm_mem_zone_ops = { |
// .show = &ttm_mem_zone_show, |
// .store = &ttm_mem_zone_store |
//}; |
static struct kobj_type ttm_mem_zone_kobj_type = { |
.release = &ttm_mem_zone_kobj_release, |
// .sysfs_ops = &ttm_mem_zone_ops, |
// .default_attrs = ttm_mem_zone_attrs, |
}; |
static void ttm_mem_global_kobj_release(struct kobject *kobj) |
{ |
struct ttm_mem_global *glob = |
container_of(kobj, struct ttm_mem_global, kobj); |
kfree(glob); |
} |
static struct kobj_type ttm_mem_glob_kobj_type = { |
.release = &ttm_mem_global_kobj_release, |
}; |
#if 0 |
static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, |
bool from_wq, uint64_t extra) |
{ |
unsigned int i; |
struct ttm_mem_zone *zone; |
uint64_t target; |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (from_wq) |
target = zone->swap_limit; |
else if (capable(CAP_SYS_ADMIN)) |
target = zone->emer_mem; |
else |
target = zone->max_mem; |
target = (extra > target) ? 0ULL : target; |
if (zone->used_mem > target) |
return true; |
} |
return false; |
} |
/** |
* At this point we only support a single shrink callback. |
* Extend this if needed, perhaps using a linked list of callbacks. |
* Note that this function is reentrant: |
* many threads may try to swap out at any given time. |
*/ |
static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, |
uint64_t extra) |
{ |
int ret; |
struct ttm_mem_shrink *shrink; |
spin_lock(&glob->lock); |
if (glob->shrink == NULL) |
goto out; |
while (ttm_zones_above_swap_target(glob, from_wq, extra)) { |
shrink = glob->shrink; |
spin_unlock(&glob->lock); |
ret = shrink->do_shrink(shrink); |
spin_lock(&glob->lock); |
if (unlikely(ret != 0)) |
goto out; |
} |
out: |
spin_unlock(&glob->lock); |
} |
static void ttm_shrink_work(struct work_struct *work) |
{ |
struct ttm_mem_global *glob = |
container_of(work, struct ttm_mem_global, work); |
ttm_shrink(glob, true, 0ULL); |
} |
#endif |
static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, |
const struct sysinfo *si) |
{ |
struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
uint64_t mem; |
int ret; |
if (unlikely(!zone)) |
return -ENOMEM; |
// mem = si->totalram - si->totalhigh; |
// mem *= si->mem_unit; |
zone->name = "kernel"; |
zone->zone_mem = mem; |
zone->max_mem = mem >> 1; |
zone->emer_mem = (mem >> 1) + (mem >> 2); |
zone->swap_limit = zone->max_mem - (mem >> 3); |
zone->used_mem = 0; |
zone->glob = glob; |
glob->zone_kernel = zone; |
ret = kobject_init_and_add( |
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
if (unlikely(ret != 0)) { |
kobject_put(&zone->kobj); |
return ret; |
} |
glob->zones[glob->num_zones++] = zone; |
return 0; |
} |
#if 0 |
#ifdef CONFIG_HIGHMEM |
static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, |
const struct sysinfo *si) |
{ |
struct ttm_mem_zone *zone; |
uint64_t mem; |
int ret; |
if (si->totalhigh == 0) |
return 0; |
zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
if (unlikely(!zone)) |
return -ENOMEM; |
mem = si->totalram; |
mem *= si->mem_unit; |
zone->name = "highmem"; |
zone->zone_mem = mem; |
zone->max_mem = mem >> 1; |
zone->emer_mem = (mem >> 1) + (mem >> 2); |
zone->swap_limit = zone->max_mem - (mem >> 3); |
zone->used_mem = 0; |
zone->glob = glob; |
glob->zone_highmem = zone; |
ret = kobject_init_and_add( |
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
if (unlikely(ret != 0)) { |
kobject_put(&zone->kobj); |
return ret; |
} |
glob->zones[glob->num_zones++] = zone; |
return 0; |
} |
#else |
static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, |
const struct sysinfo *si) |
{ |
struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
uint64_t mem; |
int ret; |
if (unlikely(!zone)) |
return -ENOMEM; |
mem = si->totalram; |
mem *= si->mem_unit; |
/** |
* No special dma32 zone needed. |
*/ |
if (mem <= ((uint64_t) 1ULL << 32)) { |
kfree(zone); |
return 0; |
} |
/* |
* Limit max dma32 memory to 4GB for now |
* until we can figure out how big this |
* zone really is. |
*/ |
mem = ((uint64_t) 1ULL << 32); |
zone->name = "dma32"; |
zone->zone_mem = mem; |
zone->max_mem = mem >> 1; |
zone->emer_mem = (mem >> 1) + (mem >> 2); |
zone->swap_limit = zone->max_mem - (mem >> 3); |
zone->used_mem = 0; |
zone->glob = glob; |
glob->zone_dma32 = zone; |
ret = kobject_init_and_add( |
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
if (unlikely(ret != 0)) { |
kobject_put(&zone->kobj); |
return ret; |
} |
glob->zones[glob->num_zones++] = zone; |
return 0; |
} |
#endif |
void ttm_mem_global_release(struct ttm_mem_global *glob) |
{ |
unsigned int i; |
struct ttm_mem_zone *zone; |
/* let the page allocator first stop the shrink work. */ |
ttm_page_alloc_fini(); |
ttm_dma_page_alloc_fini(); |
flush_workqueue(glob->swap_queue); |
destroy_workqueue(glob->swap_queue); |
glob->swap_queue = NULL; |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
kobject_del(&zone->kobj); |
kobject_put(&zone->kobj); |
} |
kobject_del(&glob->kobj); |
kobject_put(&glob->kobj); |
} |
EXPORT_SYMBOL(ttm_mem_global_release); |
static void ttm_check_swapping(struct ttm_mem_global *glob) |
{ |
bool needs_swapping = false; |
unsigned int i; |
struct ttm_mem_zone *zone; |
spin_lock(&glob->lock); |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (zone->used_mem > zone->swap_limit) { |
needs_swapping = true; |
break; |
} |
} |
spin_unlock(&glob->lock); |
if (unlikely(needs_swapping)) |
(void)queue_work(glob->swap_queue, &glob->work); |
} |
static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, |
struct ttm_mem_zone *single_zone, |
uint64_t amount) |
{ |
unsigned int i; |
struct ttm_mem_zone *zone; |
spin_lock(&glob->lock); |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (single_zone && zone != single_zone) |
continue; |
zone->used_mem -= amount; |
} |
spin_unlock(&glob->lock); |
} |
void ttm_mem_global_free(struct ttm_mem_global *glob, |
uint64_t amount) |
{ |
return ttm_mem_global_free_zone(glob, NULL, amount); |
} |
EXPORT_SYMBOL(ttm_mem_global_free); |
static int ttm_mem_global_reserve(struct ttm_mem_global *glob, |
struct ttm_mem_zone *single_zone, |
uint64_t amount, bool reserve) |
{ |
uint64_t limit; |
int ret = -ENOMEM; |
unsigned int i; |
struct ttm_mem_zone *zone; |
spin_lock(&glob->lock); |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (single_zone && zone != single_zone) |
continue; |
limit = zone->emer_mem; |
if (zone->used_mem > limit) |
goto out_unlock; |
} |
if (reserve) { |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (single_zone && zone != single_zone) |
continue; |
zone->used_mem += amount; |
} |
} |
ret = 0; |
out_unlock: |
spin_unlock(&glob->lock); |
ttm_check_swapping(glob); |
return ret; |
} |
static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, |
struct ttm_mem_zone *single_zone, |
uint64_t memory, |
bool no_wait, bool interruptible) |
{ |
int count = TTM_MEMORY_ALLOC_RETRIES; |
while (unlikely(ttm_mem_global_reserve(glob, |
single_zone, |
memory, true) |
!= 0)) { |
if (no_wait) |
return -ENOMEM; |
if (unlikely(count-- == 0)) |
return -ENOMEM; |
ttm_shrink(glob, false, memory + (memory >> 2) + 16); |
} |
return 0; |
} |
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, |
bool no_wait, bool interruptible) |
{ |
/** |
* Normal allocations of kernel memory are registered in |
* all zones. |
*/ |
return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, |
interruptible); |
} |
EXPORT_SYMBOL(ttm_mem_global_alloc); |
int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, |
struct page *page, |
bool no_wait, bool interruptible) |
{ |
struct ttm_mem_zone *zone = NULL; |
/** |
* Page allocations may be registed in a single zone |
* only if highmem or !dma32. |
*/ |
#ifdef CONFIG_HIGHMEM |
if (PageHighMem(page) && glob->zone_highmem != NULL) |
zone = glob->zone_highmem; |
#else |
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) |
zone = glob->zone_kernel; |
#endif |
return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait, |
interruptible); |
} |
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) |
{ |
struct ttm_mem_zone *zone = NULL; |
#ifdef CONFIG_HIGHMEM |
if (PageHighMem(page) && glob->zone_highmem != NULL) |
zone = glob->zone_highmem; |
#else |
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) |
zone = glob->zone_kernel; |
#endif |
ttm_mem_global_free_zone(glob, zone, PAGE_SIZE); |
} |
#endif |
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) |
{ |
} |
size_t ttm_round_pot(size_t size) |
{ |
if ((size & (size - 1)) == 0) |
return size; |
else if (size > PAGE_SIZE) |
return PAGE_ALIGN(size); |
else { |
size_t tmp_size = 4; |
while (tmp_size < size) |
tmp_size <<= 1; |
return tmp_size; |
} |
return 0; |
} |
EXPORT_SYMBOL(ttm_round_pot); |
void ttm_mem_global_free(struct ttm_mem_global *glob, |
uint64_t amount) |
{ |
return 0; |
} |
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, |
bool no_wait, bool interruptible) |
{ |
return 0; |
} |
EXPORT_SYMBOL(ttm_mem_global_alloc); |
int ttm_mem_global_init(struct ttm_mem_global *glob) |
{ |
return 0; |
} |
EXPORT_SYMBOL(ttm_mem_global_init); |
/drivers/video/drm/ttm/ttm_object.c |
---|
0,0 → 1,462 |
/************************************************************************** |
* |
* Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
/** @file ttm_ref_object.c |
* |
* Base- and reference object implementation for the various |
* ttm objects. Implements reference counting, minimal security checks |
* and release on file close. |
*/ |
/** |
* struct ttm_object_file |
* |
* @tdev: Pointer to the ttm_object_device. |
* |
* @lock: Lock that protects the ref_list list and the |
* ref_hash hash tables. |
* |
* @ref_list: List of ttm_ref_objects to be destroyed at |
* file release. |
* |
* @ref_hash: Hash tables of ref objects, one per ttm_ref_type, |
* for fast lookup of ref objects given a base object. |
*/ |
#define pr_fmt(fmt) "[TTM] " fmt |
#include <drm/ttm/ttm_object.h> |
#include <drm/ttm/ttm_module.h> |
#include <linux/list.h> |
#include <linux/spinlock.h> |
#include <linux/slab.h> |
#include <linux/module.h> |
//#include <linux/atomic.h> |
static inline int __must_check kref_get_unless_zero(struct kref *kref) |
{ |
return atomic_add_unless(&kref->refcount, 1, 0); |
} |
#define pr_err(fmt, ...) \ |
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
struct ttm_object_file { |
struct ttm_object_device *tdev; |
rwlock_t lock; |
struct list_head ref_list; |
struct drm_open_hash ref_hash[TTM_REF_NUM]; |
struct kref refcount; |
}; |
/** |
* struct ttm_object_device |
* |
* @object_lock: lock that protects the object_hash hash table. |
* |
* @object_hash: hash table for fast lookup of object global names. |
* |
* @object_count: Per device object count. |
* |
* This is the per-device data structure needed for ttm object management. |
*/ |
struct ttm_object_device { |
spinlock_t object_lock; |
struct drm_open_hash object_hash; |
atomic_t object_count; |
struct ttm_mem_global *mem_glob; |
}; |
/** |
* struct ttm_ref_object |
* |
* @hash: Hash entry for the per-file object reference hash. |
* |
* @head: List entry for the per-file list of ref-objects. |
* |
* @kref: Ref count. |
* |
* @obj: Base object this ref object is referencing. |
* |
* @ref_type: Type of ref object. |
* |
* This is similar to an idr object, but it also has a hash table entry |
* that allows lookup with a pointer to the referenced object as a key. In |
* that way, one can easily detect whether a base object is referenced by |
* a particular ttm_object_file. It also carries a ref count to avoid creating |
* multiple ref objects if a ttm_object_file references the same base |
* object more than once. |
*/ |
struct ttm_ref_object { |
struct drm_hash_item hash; |
struct list_head head; |
struct kref kref; |
enum ttm_ref_type ref_type; |
struct ttm_base_object *obj; |
struct ttm_object_file *tfile; |
}; |
static inline struct ttm_object_file * |
ttm_object_file_ref(struct ttm_object_file *tfile) |
{ |
kref_get(&tfile->refcount); |
return tfile; |
} |
static void ttm_object_file_destroy(struct kref *kref) |
{ |
struct ttm_object_file *tfile = |
container_of(kref, struct ttm_object_file, refcount); |
kfree(tfile); |
} |
static inline void ttm_object_file_unref(struct ttm_object_file **p_tfile) |
{ |
struct ttm_object_file *tfile = *p_tfile; |
*p_tfile = NULL; |
kref_put(&tfile->refcount, ttm_object_file_destroy); |
} |
int ttm_base_object_init(struct ttm_object_file *tfile, |
struct ttm_base_object *base, |
bool shareable, |
enum ttm_object_type object_type, |
void (*refcount_release) (struct ttm_base_object **), |
void (*ref_obj_release) (struct ttm_base_object *, |
enum ttm_ref_type ref_type)) |
{ |
struct ttm_object_device *tdev = tfile->tdev; |
int ret; |
base->shareable = shareable; |
base->tfile = ttm_object_file_ref(tfile); |
base->refcount_release = refcount_release; |
base->ref_obj_release = ref_obj_release; |
base->object_type = object_type; |
kref_init(&base->refcount); |
spin_lock(&tdev->object_lock); |
ret = drm_ht_just_insert_please_rcu(&tdev->object_hash, |
&base->hash, |
(unsigned long)base, 31, 0, 0); |
spin_unlock(&tdev->object_lock); |
if (unlikely(ret != 0)) |
goto out_err0; |
ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL); |
if (unlikely(ret != 0)) |
goto out_err1; |
ttm_base_object_unref(&base); |
return 0; |
out_err1: |
spin_lock(&tdev->object_lock); |
(void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); |
spin_unlock(&tdev->object_lock); |
out_err0: |
return ret; |
} |
EXPORT_SYMBOL(ttm_base_object_init); |
static void ttm_release_base(struct kref *kref) |
{ |
struct ttm_base_object *base = |
container_of(kref, struct ttm_base_object, refcount); |
struct ttm_object_device *tdev = base->tfile->tdev; |
spin_lock(&tdev->object_lock); |
(void)drm_ht_remove_item_rcu(&tdev->object_hash, &base->hash); |
spin_unlock(&tdev->object_lock); |
/* |
* Note: We don't use synchronize_rcu() here because it's far |
* too slow. It's up to the user to free the object using |
* call_rcu() or ttm_base_object_kfree(). |
*/ |
if (base->refcount_release) { |
ttm_object_file_unref(&base->tfile); |
base->refcount_release(&base); |
} |
} |
void ttm_base_object_unref(struct ttm_base_object **p_base) |
{ |
struct ttm_base_object *base = *p_base; |
*p_base = NULL; |
kref_put(&base->refcount, ttm_release_base); |
} |
EXPORT_SYMBOL(ttm_base_object_unref); |
struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile, |
uint32_t key) |
{ |
struct ttm_object_device *tdev = tfile->tdev; |
struct ttm_base_object *base; |
struct drm_hash_item *hash; |
int ret; |
// rcu_read_lock(); |
ret = drm_ht_find_item_rcu(&tdev->object_hash, key, &hash); |
if (likely(ret == 0)) { |
base = drm_hash_entry(hash, struct ttm_base_object, hash); |
ret = kref_get_unless_zero(&base->refcount) ? 0 : -EINVAL; |
} |
// rcu_read_unlock(); |
if (unlikely(ret != 0)) |
return NULL; |
if (tfile != base->tfile && !base->shareable) { |
pr_err("Attempted access of non-shareable object\n"); |
ttm_base_object_unref(&base); |
return NULL; |
} |
return base; |
} |
EXPORT_SYMBOL(ttm_base_object_lookup); |
int ttm_ref_object_add(struct ttm_object_file *tfile, |
struct ttm_base_object *base, |
enum ttm_ref_type ref_type, bool *existed) |
{ |
struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; |
struct ttm_ref_object *ref; |
struct drm_hash_item *hash; |
struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; |
int ret = -EINVAL; |
if (existed != NULL) |
*existed = true; |
while (ret == -EINVAL) { |
read_lock(&tfile->lock); |
ret = drm_ht_find_item(ht, base->hash.key, &hash); |
if (ret == 0) { |
ref = drm_hash_entry(hash, struct ttm_ref_object, hash); |
kref_get(&ref->kref); |
read_unlock(&tfile->lock); |
break; |
} |
read_unlock(&tfile->lock); |
ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), |
false, false); |
if (unlikely(ret != 0)) |
return ret; |
ref = kmalloc(sizeof(*ref), GFP_KERNEL); |
if (unlikely(ref == NULL)) { |
ttm_mem_global_free(mem_glob, sizeof(*ref)); |
return -ENOMEM; |
} |
ref->hash.key = base->hash.key; |
ref->obj = base; |
ref->tfile = tfile; |
ref->ref_type = ref_type; |
kref_init(&ref->kref); |
write_lock(&tfile->lock); |
ret = drm_ht_insert_item(ht, &ref->hash); |
if (likely(ret == 0)) { |
list_add_tail(&ref->head, &tfile->ref_list); |
kref_get(&base->refcount); |
write_unlock(&tfile->lock); |
if (existed != NULL) |
*existed = false; |
break; |
} |
write_unlock(&tfile->lock); |
BUG_ON(ret != -EINVAL); |
ttm_mem_global_free(mem_glob, sizeof(*ref)); |
kfree(ref); |
} |
return ret; |
} |
EXPORT_SYMBOL(ttm_ref_object_add); |
static void ttm_ref_object_release(struct kref *kref) |
{ |
struct ttm_ref_object *ref = |
container_of(kref, struct ttm_ref_object, kref); |
struct ttm_base_object *base = ref->obj; |
struct ttm_object_file *tfile = ref->tfile; |
struct drm_open_hash *ht; |
struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; |
ht = &tfile->ref_hash[ref->ref_type]; |
(void)drm_ht_remove_item(ht, &ref->hash); |
list_del(&ref->head); |
write_unlock(&tfile->lock); |
if (ref->ref_type != TTM_REF_USAGE && base->ref_obj_release) |
base->ref_obj_release(base, ref->ref_type); |
ttm_base_object_unref(&ref->obj); |
ttm_mem_global_free(mem_glob, sizeof(*ref)); |
kfree(ref); |
write_lock(&tfile->lock); |
} |
int ttm_ref_object_base_unref(struct ttm_object_file *tfile, |
unsigned long key, enum ttm_ref_type ref_type) |
{ |
struct drm_open_hash *ht = &tfile->ref_hash[ref_type]; |
struct ttm_ref_object *ref; |
struct drm_hash_item *hash; |
int ret; |
write_lock(&tfile->lock); |
ret = drm_ht_find_item(ht, key, &hash); |
if (unlikely(ret != 0)) { |
write_unlock(&tfile->lock); |
return -EINVAL; |
} |
ref = drm_hash_entry(hash, struct ttm_ref_object, hash); |
kref_put(&ref->kref, ttm_ref_object_release); |
write_unlock(&tfile->lock); |
return 0; |
} |
EXPORT_SYMBOL(ttm_ref_object_base_unref); |
void ttm_object_file_release(struct ttm_object_file **p_tfile) |
{ |
struct ttm_ref_object *ref; |
struct list_head *list; |
unsigned int i; |
struct ttm_object_file *tfile = *p_tfile; |
*p_tfile = NULL; |
write_lock(&tfile->lock); |
/* |
* Since we release the lock within the loop, we have to |
* restart it from the beginning each time. |
*/ |
while (!list_empty(&tfile->ref_list)) { |
list = tfile->ref_list.next; |
ref = list_entry(list, struct ttm_ref_object, head); |
ttm_ref_object_release(&ref->kref); |
} |
for (i = 0; i < TTM_REF_NUM; ++i) |
drm_ht_remove(&tfile->ref_hash[i]); |
write_unlock(&tfile->lock); |
ttm_object_file_unref(&tfile); |
} |
EXPORT_SYMBOL(ttm_object_file_release); |
struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev, |
unsigned int hash_order) |
{ |
struct ttm_object_file *tfile = kmalloc(sizeof(*tfile), GFP_KERNEL); |
unsigned int i; |
unsigned int j = 0; |
int ret; |
if (unlikely(tfile == NULL)) |
return NULL; |
rwlock_init(&tfile->lock); |
tfile->tdev = tdev; |
kref_init(&tfile->refcount); |
INIT_LIST_HEAD(&tfile->ref_list); |
for (i = 0; i < TTM_REF_NUM; ++i) { |
ret = drm_ht_create(&tfile->ref_hash[i], hash_order); |
if (ret) { |
j = i; |
goto out_err; |
} |
} |
return tfile; |
out_err: |
for (i = 0; i < j; ++i) |
drm_ht_remove(&tfile->ref_hash[i]); |
kfree(tfile); |
return NULL; |
} |
EXPORT_SYMBOL(ttm_object_file_init); |
struct ttm_object_device *ttm_object_device_init(struct ttm_mem_global |
*mem_glob, |
unsigned int hash_order) |
{ |
struct ttm_object_device *tdev = kmalloc(sizeof(*tdev), GFP_KERNEL); |
int ret; |
if (unlikely(tdev == NULL)) |
return NULL; |
tdev->mem_glob = mem_glob; |
spin_lock_init(&tdev->object_lock); |
atomic_set(&tdev->object_count, 0); |
ret = drm_ht_create(&tdev->object_hash, hash_order); |
if (likely(ret == 0)) |
return tdev; |
kfree(tdev); |
return NULL; |
} |
EXPORT_SYMBOL(ttm_object_device_init); |
void ttm_object_device_release(struct ttm_object_device **p_tdev) |
{ |
struct ttm_object_device *tdev = *p_tdev; |
*p_tdev = NULL; |
spin_lock(&tdev->object_lock); |
drm_ht_remove(&tdev->object_hash); |
spin_unlock(&tdev->object_lock); |
kfree(tdev); |
} |
EXPORT_SYMBOL(ttm_object_device_release); |
/drivers/video/drm/ttm/ttm_page_alloc.c |
---|
0,0 → 1,923 |
/* |
* Copyright (c) Red Hat Inc. |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sub license, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
* DEALINGS IN THE SOFTWARE. |
* |
* Authors: Dave Airlie <airlied@redhat.com> |
* Jerome Glisse <jglisse@redhat.com> |
* Pauli Nieminen <suokkos@gmail.com> |
*/ |
/* simple list based uncached page pool |
* - Pool collects resently freed pages for reuse |
* - Use page->lru to keep a free list |
* - doesn't track currently in use pages |
*/ |
#define pr_fmt(fmt) "[TTM] " fmt |
#include <linux/list.h> |
#include <linux/spinlock.h> |
//#include <linux/highmem.h> |
//#include <linux/mm_types.h> |
#include <linux/module.h> |
#include <linux/mm.h> |
#include <linux/seq_file.h> /* for seq_printf */ |
#include <linux/slab.h> |
//#include <linux/dma-mapping.h> |
//#include <linux/atomic.h> |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_page_alloc.h> |
#ifdef TTM_HAS_AGP |
#include <asm/agp.h> |
#endif |
#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) |
#define SMALL_ALLOCATION 16 |
#define FREE_ALL_PAGES (~0U) |
/* times are in msecs */ |
#define PAGE_FREE_INTERVAL 1000 |
#define pr_err(fmt, ...) \ |
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
#if 0 |
/** |
* struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. |
* |
* @lock: Protects the shared pool from concurrnet access. Must be used with |
* irqsave/irqrestore variants because pool allocator maybe called from |
* delayed work. |
* @fill_lock: Prevent concurrent calls to fill. |
* @list: Pool of free uc/wc pages for fast reuse. |
* @gfp_flags: Flags to pass for alloc_page. |
* @npages: Number of pages in pool. |
*/ |
struct ttm_page_pool { |
spinlock_t lock; |
bool fill_lock; |
struct list_head list; |
gfp_t gfp_flags; |
unsigned npages; |
char *name; |
unsigned long nfrees; |
unsigned long nrefills; |
}; |
/** |
* Limits for the pool. They are handled without locks because only place where |
* they may change is in sysfs store. They won't have immediate effect anyway |
* so forcing serialization to access them is pointless. |
*/ |
struct ttm_pool_opts { |
unsigned alloc_size; |
unsigned max_size; |
unsigned small; |
}; |
#define NUM_POOLS 4 |
/** |
* struct ttm_pool_manager - Holds memory pools for fst allocation |
* |
* Manager is read only object for pool code so it doesn't need locking. |
* |
* @free_interval: minimum number of jiffies between freeing pages from pool. |
* @page_alloc_inited: reference counting for pool allocation. |
* @work: Work that is used to shrink the pool. Work is only run when there is |
* some pages to free. |
* @small_allocation: Limit in number of pages what is small allocation. |
* |
* @pools: All pool objects in use. |
**/ |
struct ttm_pool_manager { |
struct kobject kobj; |
struct shrinker mm_shrink; |
struct ttm_pool_opts options; |
union { |
struct ttm_page_pool pools[NUM_POOLS]; |
struct { |
struct ttm_page_pool wc_pool; |
struct ttm_page_pool uc_pool; |
struct ttm_page_pool wc_pool_dma32; |
struct ttm_page_pool uc_pool_dma32; |
} ; |
}; |
}; |
static struct attribute ttm_page_pool_max = { |
.name = "pool_max_size", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_page_pool_small = { |
.name = "pool_small_allocation", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_page_pool_alloc_size = { |
.name = "pool_allocation_size", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute *ttm_pool_attrs[] = { |
&ttm_page_pool_max, |
&ttm_page_pool_small, |
&ttm_page_pool_alloc_size, |
NULL |
}; |
static void ttm_pool_kobj_release(struct kobject *kobj) |
{ |
struct ttm_pool_manager *m = |
container_of(kobj, struct ttm_pool_manager, kobj); |
kfree(m); |
} |
static ssize_t ttm_pool_store(struct kobject *kobj, |
struct attribute *attr, const char *buffer, size_t size) |
{ |
struct ttm_pool_manager *m = |
container_of(kobj, struct ttm_pool_manager, kobj); |
int chars; |
unsigned val; |
chars = sscanf(buffer, "%u", &val); |
if (chars == 0) |
return size; |
/* Convert kb to number of pages */ |
val = val / (PAGE_SIZE >> 10); |
if (attr == &ttm_page_pool_max) |
m->options.max_size = val; |
else if (attr == &ttm_page_pool_small) |
m->options.small = val; |
else if (attr == &ttm_page_pool_alloc_size) { |
if (val > NUM_PAGES_TO_ALLOC*8) { |
pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", |
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), |
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); |
return size; |
} else if (val > NUM_PAGES_TO_ALLOC) { |
pr_warn("Setting allocation size to larger than %lu is not recommended\n", |
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); |
} |
m->options.alloc_size = val; |
} |
return size; |
} |
static ssize_t ttm_pool_show(struct kobject *kobj, |
struct attribute *attr, char *buffer) |
{ |
struct ttm_pool_manager *m = |
container_of(kobj, struct ttm_pool_manager, kobj); |
unsigned val = 0; |
if (attr == &ttm_page_pool_max) |
val = m->options.max_size; |
else if (attr == &ttm_page_pool_small) |
val = m->options.small; |
else if (attr == &ttm_page_pool_alloc_size) |
val = m->options.alloc_size; |
val = val * (PAGE_SIZE >> 10); |
return snprintf(buffer, PAGE_SIZE, "%u\n", val); |
} |
static const struct sysfs_ops ttm_pool_sysfs_ops = { |
.show = &ttm_pool_show, |
.store = &ttm_pool_store, |
}; |
static struct kobj_type ttm_pool_kobj_type = { |
.release = &ttm_pool_kobj_release, |
.sysfs_ops = &ttm_pool_sysfs_ops, |
.default_attrs = ttm_pool_attrs, |
}; |
static struct ttm_pool_manager *_manager; |
#ifndef CONFIG_X86 |
static int set_pages_array_wb(struct page **pages, int addrinarray) |
{ |
#ifdef TTM_HAS_AGP |
int i; |
for (i = 0; i < addrinarray; i++) |
unmap_page_from_agp(pages[i]); |
#endif |
return 0; |
} |
static int set_pages_array_wc(struct page **pages, int addrinarray) |
{ |
#ifdef TTM_HAS_AGP |
int i; |
for (i = 0; i < addrinarray; i++) |
map_page_into_agp(pages[i]); |
#endif |
return 0; |
} |
static int set_pages_array_uc(struct page **pages, int addrinarray) |
{ |
#ifdef TTM_HAS_AGP |
int i; |
for (i = 0; i < addrinarray; i++) |
map_page_into_agp(pages[i]); |
#endif |
return 0; |
} |
#endif |
/** |
* Select the right pool or requested caching state and ttm flags. */ |
static struct ttm_page_pool *ttm_get_pool(int flags, |
enum ttm_caching_state cstate) |
{ |
int pool_index; |
if (cstate == tt_cached) |
return NULL; |
if (cstate == tt_wc) |
pool_index = 0x0; |
else |
pool_index = 0x1; |
if (flags & TTM_PAGE_FLAG_DMA32) |
pool_index |= 0x2; |
return &_manager->pools[pool_index]; |
} |
/* set memory back to wb and free the pages. */ |
static void ttm_pages_put(struct page *pages[], unsigned npages) |
{ |
unsigned i; |
if (set_pages_array_wb(pages, npages)) |
pr_err("Failed to set %d pages to wb!\n", npages); |
for (i = 0; i < npages; ++i) |
__free_page(pages[i]); |
} |
static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, |
unsigned freed_pages) |
{ |
pool->npages -= freed_pages; |
pool->nfrees += freed_pages; |
} |
/** |
* Free pages from pool. |
* |
* To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC |
* number of pages in one go. |
* |
* @pool: to free the pages from |
* @free_all: If set to true will free all pages in pool |
**/ |
static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) |
{ |
unsigned long irq_flags; |
struct page *p; |
struct page **pages_to_free; |
unsigned freed_pages = 0, |
npages_to_free = nr_free; |
if (NUM_PAGES_TO_ALLOC < nr_free) |
npages_to_free = NUM_PAGES_TO_ALLOC; |
pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), |
GFP_KERNEL); |
if (!pages_to_free) { |
pr_err("Failed to allocate memory for pool free operation\n"); |
return 0; |
} |
restart: |
spin_lock_irqsave(&pool->lock, irq_flags); |
list_for_each_entry_reverse(p, &pool->list, lru) { |
if (freed_pages >= npages_to_free) |
break; |
pages_to_free[freed_pages++] = p; |
/* We can only remove NUM_PAGES_TO_ALLOC at a time. */ |
if (freed_pages >= NUM_PAGES_TO_ALLOC) { |
/* remove range of pages from the pool */ |
__list_del(p->lru.prev, &pool->list); |
ttm_pool_update_free_locked(pool, freed_pages); |
/** |
* Because changing page caching is costly |
* we unlock the pool to prevent stalling. |
*/ |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
ttm_pages_put(pages_to_free, freed_pages); |
if (likely(nr_free != FREE_ALL_PAGES)) |
nr_free -= freed_pages; |
if (NUM_PAGES_TO_ALLOC >= nr_free) |
npages_to_free = nr_free; |
else |
npages_to_free = NUM_PAGES_TO_ALLOC; |
freed_pages = 0; |
/* free all so restart the processing */ |
if (nr_free) |
goto restart; |
/* Not allowed to fall through or break because |
* following context is inside spinlock while we are |
* outside here. |
*/ |
goto out; |
} |
} |
/* remove range of pages from the pool */ |
if (freed_pages) { |
__list_del(&p->lru, &pool->list); |
ttm_pool_update_free_locked(pool, freed_pages); |
nr_free -= freed_pages; |
} |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
if (freed_pages) |
ttm_pages_put(pages_to_free, freed_pages); |
out: |
kfree(pages_to_free); |
return nr_free; |
} |
/* Get good estimation how many pages are free in pools */ |
static int ttm_pool_get_num_unused_pages(void) |
{ |
unsigned i; |
int total = 0; |
for (i = 0; i < NUM_POOLS; ++i) |
total += _manager->pools[i].npages; |
return total; |
} |
/** |
* Callback for mm to request pool to reduce number of page held. |
*/ |
static int ttm_pool_mm_shrink(struct shrinker *shrink, |
struct shrink_control *sc) |
{ |
static atomic_t start_pool = ATOMIC_INIT(0); |
unsigned i; |
unsigned pool_offset = atomic_add_return(1, &start_pool); |
struct ttm_page_pool *pool; |
int shrink_pages = sc->nr_to_scan; |
pool_offset = pool_offset % NUM_POOLS; |
/* select start pool in round robin fashion */ |
for (i = 0; i < NUM_POOLS; ++i) { |
unsigned nr_free = shrink_pages; |
if (shrink_pages == 0) |
break; |
pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; |
shrink_pages = ttm_page_pool_free(pool, nr_free); |
} |
/* return estimated number of unused pages in pool */ |
return ttm_pool_get_num_unused_pages(); |
} |
static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) |
{ |
manager->mm_shrink.shrink = &ttm_pool_mm_shrink; |
manager->mm_shrink.seeks = 1; |
register_shrinker(&manager->mm_shrink); |
} |
static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) |
{ |
unregister_shrinker(&manager->mm_shrink); |
} |
static int ttm_set_pages_caching(struct page **pages, |
enum ttm_caching_state cstate, unsigned cpages) |
{ |
int r = 0; |
/* Set page caching */ |
switch (cstate) { |
case tt_uncached: |
r = set_pages_array_uc(pages, cpages); |
if (r) |
pr_err("Failed to set %d pages to uc!\n", cpages); |
break; |
case tt_wc: |
r = set_pages_array_wc(pages, cpages); |
if (r) |
pr_err("Failed to set %d pages to wc!\n", cpages); |
break; |
default: |
break; |
} |
return r; |
} |
/** |
* Free pages the pages that failed to change the caching state. If there is |
* any pages that have changed their caching state already put them to the |
* pool. |
*/ |
static void ttm_handle_caching_state_failure(struct list_head *pages, |
int ttm_flags, enum ttm_caching_state cstate, |
struct page **failed_pages, unsigned cpages) |
{ |
unsigned i; |
/* Failed pages have to be freed */ |
for (i = 0; i < cpages; ++i) { |
list_del(&failed_pages[i]->lru); |
__free_page(failed_pages[i]); |
} |
} |
/** |
* Allocate new pages with correct caching. |
* |
* This function is reentrant if caller updates count depending on number of |
* pages returned in pages array. |
*/ |
static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, |
int ttm_flags, enum ttm_caching_state cstate, unsigned count) |
{ |
struct page **caching_array; |
struct page *p; |
int r = 0; |
unsigned i, cpages; |
unsigned max_cpages = min(count, |
(unsigned)(PAGE_SIZE/sizeof(struct page *))); |
/* allocate array for page caching change */ |
caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); |
if (!caching_array) { |
pr_err("Unable to allocate table for new pages\n"); |
return -ENOMEM; |
} |
for (i = 0, cpages = 0; i < count; ++i) { |
p = alloc_page(gfp_flags); |
if (!p) { |
pr_err("Unable to get page %u\n", i); |
/* store already allocated pages in the pool after |
* setting the caching state */ |
if (cpages) { |
r = ttm_set_pages_caching(caching_array, |
cstate, cpages); |
if (r) |
ttm_handle_caching_state_failure(pages, |
ttm_flags, cstate, |
caching_array, cpages); |
} |
r = -ENOMEM; |
goto out; |
} |
#ifdef CONFIG_HIGHMEM |
/* gfp flags of highmem page should never be dma32 so we |
* we should be fine in such case |
*/ |
if (!PageHighMem(p)) |
#endif |
{ |
caching_array[cpages++] = p; |
if (cpages == max_cpages) { |
r = ttm_set_pages_caching(caching_array, |
cstate, cpages); |
if (r) { |
ttm_handle_caching_state_failure(pages, |
ttm_flags, cstate, |
caching_array, cpages); |
goto out; |
} |
cpages = 0; |
} |
} |
list_add(&p->lru, pages); |
} |
if (cpages) { |
r = ttm_set_pages_caching(caching_array, cstate, cpages); |
if (r) |
ttm_handle_caching_state_failure(pages, |
ttm_flags, cstate, |
caching_array, cpages); |
} |
out: |
kfree(caching_array); |
return r; |
} |
/** |
* Fill the given pool if there aren't enough pages and the requested number of |
* pages is small. |
*/ |
static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, |
int ttm_flags, enum ttm_caching_state cstate, unsigned count, |
unsigned long *irq_flags) |
{ |
struct page *p; |
int r; |
unsigned cpages = 0; |
/** |
* Only allow one pool fill operation at a time. |
* If pool doesn't have enough pages for the allocation new pages are |
* allocated from outside of pool. |
*/ |
if (pool->fill_lock) |
return; |
pool->fill_lock = true; |
/* If allocation request is small and there are not enough |
* pages in a pool we fill the pool up first. */ |
if (count < _manager->options.small |
&& count > pool->npages) { |
struct list_head new_pages; |
unsigned alloc_size = _manager->options.alloc_size; |
/** |
* Can't change page caching if in irqsave context. We have to |
* drop the pool->lock. |
*/ |
spin_unlock_irqrestore(&pool->lock, *irq_flags); |
INIT_LIST_HEAD(&new_pages); |
r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, |
cstate, alloc_size); |
spin_lock_irqsave(&pool->lock, *irq_flags); |
if (!r) { |
list_splice(&new_pages, &pool->list); |
++pool->nrefills; |
pool->npages += alloc_size; |
} else { |
pr_err("Failed to fill pool (%p)\n", pool); |
/* If we have any pages left put them to the pool. */ |
list_for_each_entry(p, &pool->list, lru) { |
++cpages; |
} |
list_splice(&new_pages, &pool->list); |
pool->npages += cpages; |
} |
} |
pool->fill_lock = false; |
} |
/** |
* Cut 'count' number of pages from the pool and put them on the return list. |
* |
* @return count of pages still required to fulfill the request. |
*/ |
static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, |
struct list_head *pages, |
int ttm_flags, |
enum ttm_caching_state cstate, |
unsigned count) |
{ |
unsigned long irq_flags; |
struct list_head *p; |
unsigned i; |
spin_lock_irqsave(&pool->lock, irq_flags); |
ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); |
if (count >= pool->npages) { |
/* take all pages from the pool */ |
list_splice_init(&pool->list, pages); |
count -= pool->npages; |
pool->npages = 0; |
goto out; |
} |
/* find the last pages to include for requested number of pages. Split |
* pool to begin and halve it to reduce search space. */ |
if (count <= pool->npages/2) { |
i = 0; |
list_for_each(p, &pool->list) { |
if (++i == count) |
break; |
} |
} else { |
i = pool->npages + 1; |
list_for_each_prev(p, &pool->list) { |
if (--i == count) |
break; |
} |
} |
/* Cut 'count' number of pages from the pool */ |
list_cut_position(pages, &pool->list, p); |
pool->npages -= count; |
count = 0; |
out: |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
return count; |
} |
#endif |
/* Put all pages in pages list to correct pool to wait for reuse */ |
static void ttm_put_pages(struct page **pages, unsigned npages, int flags, |
enum ttm_caching_state cstate) |
{ |
unsigned long irq_flags; |
// struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
unsigned i; |
for (i = 0; i < npages; i++) { |
if (pages[i]) { |
// if (page_count(pages[i]) != 1) |
// pr_err("Erroneous page count. Leaking pages.\n"); |
FreePage(pages[i]); |
pages[i] = NULL; |
} |
} |
return; |
#if 0 |
if (pool == NULL) { |
/* No pool for this memory type so free the pages */ |
for (i = 0; i < npages; i++) { |
if (pages[i]) { |
if (page_count(pages[i]) != 1) |
pr_err("Erroneous page count. Leaking pages.\n"); |
__free_page(pages[i]); |
pages[i] = NULL; |
} |
} |
return; |
} |
spin_lock_irqsave(&pool->lock, irq_flags); |
for (i = 0; i < npages; i++) { |
if (pages[i]) { |
if (page_count(pages[i]) != 1) |
pr_err("Erroneous page count. Leaking pages.\n"); |
list_add_tail(&pages[i]->lru, &pool->list); |
pages[i] = NULL; |
pool->npages++; |
} |
} |
/* Check that we don't go over the pool limit */ |
npages = 0; |
if (pool->npages > _manager->options.max_size) { |
npages = pool->npages - _manager->options.max_size; |
/* free at least NUM_PAGES_TO_ALLOC number of pages |
* to reduce calls to set_memory_wb */ |
if (npages < NUM_PAGES_TO_ALLOC) |
npages = NUM_PAGES_TO_ALLOC; |
} |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
if (npages) |
ttm_page_pool_free(pool, npages); |
#endif |
} |
/* |
* On success pages list will hold count number of correctly |
* cached pages. |
*/ |
static int ttm_get_pages(struct page **pages, unsigned npages, int flags, |
enum ttm_caching_state cstate) |
{ |
// struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
struct list_head plist; |
struct page *p = NULL; |
// gfp_t gfp_flags = GFP_USER; |
unsigned count; |
int r; |
for (r = 0; r < npages; ++r) { |
p = AllocPage(); |
if (!p) { |
pr_err("Unable to allocate page\n"); |
return -ENOMEM; |
} |
pages[r] = p; |
} |
return 0; |
#if 0 |
/* set zero flag for page allocation if required */ |
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) |
gfp_flags |= __GFP_ZERO; |
/* No pool for cached pages */ |
if (pool == NULL) { |
if (flags & TTM_PAGE_FLAG_DMA32) |
gfp_flags |= GFP_DMA32; |
else |
gfp_flags |= GFP_HIGHUSER; |
for (r = 0; r < npages; ++r) { |
p = alloc_page(gfp_flags); |
if (!p) { |
pr_err("Unable to allocate page\n"); |
return -ENOMEM; |
} |
pages[r] = p; |
} |
return 0; |
} |
/* combine zero flag to pool flags */ |
gfp_flags |= pool->gfp_flags; |
/* First we take pages from the pool */ |
INIT_LIST_HEAD(&plist); |
npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); |
count = 0; |
list_for_each_entry(p, &plist, lru) { |
pages[count++] = p; |
} |
/* clear the pages coming from the pool if requested */ |
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { |
list_for_each_entry(p, &plist, lru) { |
if (PageHighMem(p)) |
clear_highpage(p); |
else |
clear_page(page_address(p)); |
} |
} |
/* If pool didn't have enough pages allocate new one. */ |
if (npages > 0) { |
/* ttm_alloc_new_pages doesn't reference pool so we can run |
* multiple requests in parallel. |
**/ |
INIT_LIST_HEAD(&plist); |
r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); |
list_for_each_entry(p, &plist, lru) { |
pages[count++] = p; |
} |
if (r) { |
/* If there is any pages in the list put them back to |
* the pool. */ |
pr_err("Failed to allocate extra pages for large request\n"); |
ttm_put_pages(pages, count, flags, cstate); |
return r; |
} |
} |
#endif |
return 0; |
} |
#if 0 |
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, |
char *name) |
{ |
spin_lock_init(&pool->lock); |
pool->fill_lock = false; |
INIT_LIST_HEAD(&pool->list); |
pool->npages = pool->nfrees = 0; |
pool->gfp_flags = flags; |
pool->name = name; |
} |
int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) |
{ |
int ret; |
WARN_ON(_manager); |
pr_info("Initializing pool allocator\n"); |
_manager = kzalloc(sizeof(*_manager), GFP_KERNEL); |
ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); |
ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); |
ttm_page_pool_init_locked(&_manager->wc_pool_dma32, |
GFP_USER | GFP_DMA32, "wc dma"); |
ttm_page_pool_init_locked(&_manager->uc_pool_dma32, |
GFP_USER | GFP_DMA32, "uc dma"); |
_manager->options.max_size = max_pages; |
_manager->options.small = SMALL_ALLOCATION; |
_manager->options.alloc_size = NUM_PAGES_TO_ALLOC; |
ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, |
&glob->kobj, "pool"); |
if (unlikely(ret != 0)) { |
kobject_put(&_manager->kobj); |
_manager = NULL; |
return ret; |
} |
ttm_pool_mm_shrink_init(_manager); |
return 0; |
} |
void ttm_page_alloc_fini(void) |
{ |
int i; |
pr_info("Finalizing pool allocator\n"); |
ttm_pool_mm_shrink_fini(_manager); |
for (i = 0; i < NUM_POOLS; ++i) |
ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); |
kobject_put(&_manager->kobj); |
_manager = NULL; |
} |
#endif |
int ttm_pool_populate(struct ttm_tt *ttm) |
{ |
struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; |
unsigned i; |
int ret; |
if (ttm->state != tt_unpopulated) |
return 0; |
for (i = 0; i < ttm->num_pages; ++i) { |
ret = ttm_get_pages(&ttm->pages[i], 1, |
ttm->page_flags, |
ttm->caching_state); |
if (ret != 0) { |
ttm_pool_unpopulate(ttm); |
return -ENOMEM; |
} |
} |
ttm->state = tt_unbound; |
return 0; |
} |
EXPORT_SYMBOL(ttm_pool_populate); |
void ttm_pool_unpopulate(struct ttm_tt *ttm) |
{ |
unsigned i; |
for (i = 0; i < ttm->num_pages; ++i) { |
if (ttm->pages[i]) { |
ttm_mem_global_free_page(ttm->glob->mem_glob, |
ttm->pages[i]); |
ttm_put_pages(&ttm->pages[i], 1, |
ttm->page_flags, |
ttm->caching_state); |
} |
} |
ttm->state = tt_unpopulated; |
} |
EXPORT_SYMBOL(ttm_pool_unpopulate); |
/drivers/video/drm/ttm/ttm_tt.c |
---|
0,0 → 1,387 |
/************************************************************************** |
* |
* Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
#define pr_fmt(fmt) "[TTM] " fmt |
#include <syscall.h> |
#include <linux/sched.h> |
//#include <linux/highmem.h> |
//#include <linux/pagemap.h> |
#include <linux/shmem_fs.h> |
//#include <linux/file.h> |
//#include <linux/swap.h> |
#include <linux/slab.h> |
#include <linux/export.h> |
//#include <drm/drm_cache.h> |
#include <drm/drm_mem_util.h> |
#include <drm/ttm/ttm_module.h> |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_placement.h> |
#include <drm/ttm/ttm_page_alloc.h> |
/** |
* Allocates storage for pointers to the pages that back the ttm. |
*/ |
static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) |
{ |
ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); |
} |
static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) |
{ |
ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); |
ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, |
sizeof(*ttm->dma_address)); |
} |
#ifdef CONFIG_X86 |
static inline int ttm_tt_set_page_caching(struct page *p, |
enum ttm_caching_state c_old, |
enum ttm_caching_state c_new) |
{ |
int ret = 0; |
if (PageHighMem(p)) |
return 0; |
if (c_old != tt_cached) { |
/* p isn't in the default caching state, set it to |
* writeback first to free its current memtype. */ |
ret = set_pages_wb(p, 1); |
if (ret) |
return ret; |
} |
if (c_new == tt_wc) |
ret = set_memory_wc((unsigned long) page_address(p), 1); |
else if (c_new == tt_uncached) |
ret = set_pages_uc(p, 1); |
return ret; |
} |
#else /* CONFIG_X86 */ |
static inline int ttm_tt_set_page_caching(struct page *p, |
enum ttm_caching_state c_old, |
enum ttm_caching_state c_new) |
{ |
return 0; |
} |
#endif /* CONFIG_X86 */ |
/* |
* Change caching policy for the linear kernel map |
* for range of pages in a ttm. |
*/ |
static int ttm_tt_set_caching(struct ttm_tt *ttm, |
enum ttm_caching_state c_state) |
{ |
int i, j; |
struct page *cur_page; |
int ret; |
if (ttm->caching_state == c_state) |
return 0; |
if (ttm->state == tt_unpopulated) { |
/* Change caching but don't populate */ |
ttm->caching_state = c_state; |
return 0; |
} |
// if (ttm->caching_state == tt_cached) |
// drm_clflush_pages(ttm->pages, ttm->num_pages); |
for (i = 0; i < ttm->num_pages; ++i) { |
cur_page = ttm->pages[i]; |
if (likely(cur_page != NULL)) { |
ret = ttm_tt_set_page_caching(cur_page, |
ttm->caching_state, |
c_state); |
if (unlikely(ret != 0)) |
goto out_err; |
} |
} |
ttm->caching_state = c_state; |
return 0; |
out_err: |
for (j = 0; j < i; ++j) { |
cur_page = ttm->pages[j]; |
if (likely(cur_page != NULL)) { |
(void)ttm_tt_set_page_caching(cur_page, c_state, |
ttm->caching_state); |
} |
} |
return ret; |
} |
int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) |
{ |
enum ttm_caching_state state; |
if (placement & TTM_PL_FLAG_WC) |
state = tt_wc; |
else if (placement & TTM_PL_FLAG_UNCACHED) |
state = tt_uncached; |
else |
state = tt_cached; |
return ttm_tt_set_caching(ttm, state); |
} |
EXPORT_SYMBOL(ttm_tt_set_placement_caching); |
void ttm_tt_destroy(struct ttm_tt *ttm) |
{ |
if (unlikely(ttm == NULL)) |
return; |
if (ttm->state == tt_bound) { |
ttm_tt_unbind(ttm); |
} |
if (likely(ttm->pages != NULL)) { |
ttm->bdev->driver->ttm_tt_unpopulate(ttm); |
} |
// if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && |
// ttm->swap_storage) |
// fput(ttm->swap_storage); |
ttm->swap_storage = NULL; |
ttm->func->destroy(ttm); |
} |
int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, |
unsigned long size, uint32_t page_flags, |
struct page *dummy_read_page) |
{ |
ttm->bdev = bdev; |
ttm->glob = bdev->glob; |
ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
ttm->caching_state = tt_cached; |
ttm->page_flags = page_flags; |
ttm->dummy_read_page = dummy_read_page; |
ttm->state = tt_unpopulated; |
ttm->swap_storage = NULL; |
ttm_tt_alloc_page_directory(ttm); |
if (!ttm->pages) { |
ttm_tt_destroy(ttm); |
printf("Failed allocating page table\n"); |
return -ENOMEM; |
} |
return 0; |
} |
EXPORT_SYMBOL(ttm_tt_init); |
void ttm_tt_fini(struct ttm_tt *ttm) |
{ |
drm_free_large(ttm->pages); |
ttm->pages = NULL; |
} |
EXPORT_SYMBOL(ttm_tt_fini); |
int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, |
unsigned long size, uint32_t page_flags, |
struct page *dummy_read_page) |
{ |
struct ttm_tt *ttm = &ttm_dma->ttm; |
ttm->bdev = bdev; |
ttm->glob = bdev->glob; |
ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
ttm->caching_state = tt_cached; |
ttm->page_flags = page_flags; |
ttm->dummy_read_page = dummy_read_page; |
ttm->state = tt_unpopulated; |
ttm->swap_storage = NULL; |
INIT_LIST_HEAD(&ttm_dma->pages_list); |
ttm_dma_tt_alloc_page_directory(ttm_dma); |
if (!ttm->pages || !ttm_dma->dma_address) { |
ttm_tt_destroy(ttm); |
printf("Failed allocating page table\n"); |
return -ENOMEM; |
} |
return 0; |
} |
EXPORT_SYMBOL(ttm_dma_tt_init); |
void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) |
{ |
struct ttm_tt *ttm = &ttm_dma->ttm; |
drm_free_large(ttm->pages); |
ttm->pages = NULL; |
drm_free_large(ttm_dma->dma_address); |
ttm_dma->dma_address = NULL; |
} |
EXPORT_SYMBOL(ttm_dma_tt_fini); |
void ttm_tt_unbind(struct ttm_tt *ttm) |
{ |
int ret; |
if (ttm->state == tt_bound) { |
ret = ttm->func->unbind(ttm); |
BUG_ON(ret); |
ttm->state = tt_unbound; |
} |
} |
#if 0 |
int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) |
{ |
int ret = 0; |
if (!ttm) |
return -EINVAL; |
if (ttm->state == tt_bound) |
return 0; |
ret = ttm->bdev->driver->ttm_tt_populate(ttm); |
if (ret) |
return ret; |
ret = ttm->func->bind(ttm, bo_mem); |
if (unlikely(ret != 0)) |
return ret; |
ttm->state = tt_bound; |
return 0; |
} |
EXPORT_SYMBOL(ttm_tt_bind); |
#endif |
/* |
int ttm_tt_swapin(struct ttm_tt *ttm) |
{ |
struct address_space *swap_space; |
struct file *swap_storage; |
struct page *from_page; |
struct page *to_page; |
int i; |
int ret = -ENOMEM; |
swap_storage = ttm->swap_storage; |
BUG_ON(swap_storage == NULL); |
swap_space = file_inode(swap_storage)->i_mapping; |
for (i = 0; i < ttm->num_pages; ++i) { |
from_page = shmem_read_mapping_page(swap_space, i); |
if (IS_ERR(from_page)) { |
ret = PTR_ERR(from_page); |
goto out_err; |
} |
to_page = ttm->pages[i]; |
if (unlikely(to_page == NULL)) |
goto out_err; |
copy_highpage(to_page, from_page); |
page_cache_release(from_page); |
} |
if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) |
fput(swap_storage); |
ttm->swap_storage = NULL; |
ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; |
return 0; |
out_err: |
return ret; |
} |
int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) |
{ |
struct address_space *swap_space; |
struct file *swap_storage; |
struct page *from_page; |
struct page *to_page; |
int i; |
int ret = -ENOMEM; |
BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); |
BUG_ON(ttm->caching_state != tt_cached); |
if (!persistent_swap_storage) { |
swap_storage = shmem_file_setup("ttm swap", |
ttm->num_pages << PAGE_SHIFT, |
0); |
if (unlikely(IS_ERR(swap_storage))) { |
pr_err("Failed allocating swap storage\n"); |
return PTR_ERR(swap_storage); |
} |
} else |
swap_storage = persistent_swap_storage; |
swap_space = file_inode(swap_storage)->i_mapping; |
for (i = 0; i < ttm->num_pages; ++i) { |
from_page = ttm->pages[i]; |
if (unlikely(from_page == NULL)) |
continue; |
to_page = shmem_read_mapping_page(swap_space, i); |
if (unlikely(IS_ERR(to_page))) { |
ret = PTR_ERR(to_page); |
goto out_err; |
} |
copy_highpage(to_page, from_page); |
set_page_dirty(to_page); |
mark_page_accessed(to_page); |
page_cache_release(to_page); |
} |
ttm->bdev->driver->ttm_tt_unpopulate(ttm); |
ttm->swap_storage = swap_storage; |
ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; |
if (persistent_swap_storage) |
ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; |
return 0; |
out_err: |
if (!persistent_swap_storage) |
fput(swap_storage); |
return ret; |
} |
*/ |