/drivers/video/drm/ttm/ttm_bo.c |
---|
39,92 → 39,35 |
#include <linux/mm.h> |
#include <linux/module.h> |
#define pr_err(fmt, ...) \ |
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
#define TTM_ASSERT_LOCKED(param) |
#define TTM_DEBUG(fmt, arg...) |
#define TTM_BO_HASH_ORDER 13 |
#define pr_err(fmt, ...) \ |
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) |
static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) |
{ |
int i; |
mutex_lock(&man->io_reserve_mutex); |
for (i = 0; i <= TTM_PL_PRIV5; i++) |
if (flags & (1 << i)) { |
*mem_type = i; |
return 0; |
} |
void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) |
{ |
if (likely(man->io_reserve_fastpath)) |
return; |
mutex_unlock(&man->io_reserve_mutex); |
return -EINVAL; |
} |
#if 0 |
static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) |
{ |
struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
pr_err(" has_type: %d\n", man->has_type); |
pr_err(" use_type: %d\n", man->use_type); |
pr_err(" flags: 0x%08X\n", man->flags); |
pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); |
pr_err(" size: %llu\n", man->size); |
pr_err(" available_caching: 0x%08X\n", man->available_caching); |
pr_err(" default_caching: 0x%08X\n", man->default_caching); |
if (mem_type != TTM_PL_SYSTEM) |
(*man->func->debug)(man, TTM_PFX); |
} |
static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, |
struct ttm_placement *placement) |
{ |
int i, ret, mem_type; |
pr_err("No space for %p (%lu pages, %luK, %luM)\n", |
bo, bo->mem.num_pages, bo->mem.size >> 10, |
bo->mem.size >> 20); |
for (i = 0; i < placement->num_placement; i++) { |
ret = ttm_mem_type_from_flags(placement->placement[i], |
&mem_type); |
if (ret) |
return; |
pr_err(" placement[%d]=0x%08X (%d)\n", |
i, placement->placement[i], mem_type); |
ttm_mem_type_debug(bo->bdev, mem_type); |
} |
} |
static ssize_t ttm_bo_global_show(struct kobject *kobj, |
struct attribute *attr, |
char *buffer) |
{ |
struct ttm_bo_global *glob = |
container_of(kobj, struct ttm_bo_global, kobj); |
return snprintf(buffer, PAGE_SIZE, "%lu\n", |
(unsigned long) atomic_read(&glob->bo_count)); |
} |
static struct attribute *ttm_bo_global_attrs[] = { |
&ttm_bo_count, |
NULL |
}; |
static const struct sysfs_ops ttm_bo_global_ops = { |
.show = &ttm_bo_global_show |
}; |
static struct kobj_type ttm_bo_glob_kobj_type = { |
.release = &ttm_bo_global_kobj_release, |
.sysfs_ops = &ttm_bo_global_ops, |
.default_attrs = ttm_bo_global_attrs |
}; |
#endif |
static inline uint32_t ttm_bo_type_flags(unsigned type) |
{ |
return 1 << (type); |
148,12 → 91,14 |
if (bo->ttm) |
ttm_tt_destroy(bo->ttm); |
atomic_dec(&bo->glob->bo_count); |
if (bo->resv == &bo->ttm_resv) |
reservation_object_fini(&bo->ttm_resv); |
mutex_destroy(&bo->wu_mutex); |
if (bo->destroy) |
bo->destroy(bo); |
else { |
kfree(bo); |
} |
ttm_mem_global_free(bdev->glob->mem_glob, acc_size); |
} |
void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) |
161,7 → 106,7 |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man; |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
lockdep_assert_held(&bo->resv->lock.base); |
if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { |
268,7 → 213,6 |
return ret; |
} |
#if 0 |
static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, |
struct ttm_mem_reg *mem, |
bool evict, bool interruptible, |
348,9 → 292,11 |
moved: |
if (bo->evicted) { |
if (bdev->driver->invalidate_caches) { |
ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); |
if (ret) |
pr_err("Can not flush read caches\n"); |
} |
bo->evicted = false; |
} |
407,7 → 353,7 |
int ret; |
spin_lock(&glob->lru_lock); |
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
ret = __ttm_bo_reserve(bo, false, true, false, NULL); |
spin_lock(&bdev->fence_lock); |
(void) ttm_bo_wait(bo, false, false, true); |
438,7 → 384,7 |
ttm_bo_add_to_lru(bo); |
} |
ww_mutex_unlock(&bo->resv->lock); |
__ttm_bo_unreserve(bo); |
} |
kref_get(&bo->list_kref); |
449,8 → 395,8 |
driver->sync_obj_flush(sync_obj); |
driver->sync_obj_unref(&sync_obj); |
} |
schedule_delayed_work(&bdev->wq, |
((HZ / 100) < 1) ? 1 : HZ / 100); |
// schedule_delayed_work(&bdev->wq, |
// ((HZ / 100) < 1) ? 1 : HZ / 100); |
} |
/** |
489,7 → 435,7 |
sync_obj = driver->sync_obj_ref(bo->sync_obj); |
spin_unlock(&bdev->fence_lock); |
ww_mutex_unlock(&bo->resv->lock); |
__ttm_bo_unreserve(bo); |
spin_unlock(&glob->lru_lock); |
ret = driver->sync_obj_wait(sync_obj, false, interruptible); |
509,7 → 455,7 |
return ret; |
spin_lock(&glob->lru_lock); |
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
ret = __ttm_bo_reserve(bo, false, true, false, NULL); |
/* |
* We raced, and lost, someone else holds the reservation now, |
527,7 → 473,7 |
spin_unlock(&bdev->fence_lock); |
if (ret || unlikely(list_empty(&bo->ddestroy))) { |
ww_mutex_unlock(&bo->resv->lock); |
__ttm_bo_unreserve(bo); |
spin_unlock(&glob->lru_lock); |
return ret; |
} |
572,11 → 518,11 |
kref_get(&nentry->list_kref); |
} |
ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); |
ret = __ttm_bo_reserve(entry, false, true, false, NULL); |
if (remove_all && ret) { |
spin_unlock(&glob->lru_lock); |
ret = ttm_bo_reserve_nolru(entry, false, false, |
false, 0); |
ret = __ttm_bo_reserve(entry, false, false, |
false, NULL); |
spin_lock(&glob->lru_lock); |
} |
615,7 → 561,6 |
((HZ / 100) < 1) ? 1 : HZ / 100); |
} |
} |
#endif |
static void ttm_bo_release(struct kref *kref) |
{ |
626,10 → 571,10 |
drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); |
ttm_mem_io_lock(man, false); |
// ttm_mem_io_free_vm(bo); |
ttm_mem_io_free_vm(bo); |
ttm_mem_io_unlock(man); |
// ttm_bo_cleanup_refs_or_queue(bo); |
// kref_put(&bo->list_kref, ttm_bo_release_list); |
ttm_bo_cleanup_refs_or_queue(bo); |
kref_put(&bo->list_kref, ttm_bo_release_list); |
} |
void ttm_bo_unref(struct ttm_buffer_object **p_bo) |
641,121 → 586,6 |
} |
EXPORT_SYMBOL(ttm_bo_unref); |
#if 0 |
int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) |
{ |
return cancel_delayed_work_sync(&bdev->wq); |
} |
EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); |
void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) |
{ |
if (resched) |
schedule_delayed_work(&bdev->wq, |
((HZ / 100) < 1) ? 1 : HZ / 100); |
} |
EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); |
static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_reg evict_mem; |
struct ttm_placement placement; |
int ret = 0; |
spin_lock(&bdev->fence_lock); |
ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); |
spin_unlock(&bdev->fence_lock); |
if (unlikely(ret != 0)) { |
if (ret != -ERESTARTSYS) { |
pr_err("Failed to expire sync object before buffer eviction\n"); |
} |
goto out; |
} |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
evict_mem = bo->mem; |
evict_mem.mm_node = NULL; |
evict_mem.bus.io_reserved_vm = false; |
evict_mem.bus.io_reserved_count = 0; |
placement.fpfn = 0; |
placement.lpfn = 0; |
placement.num_placement = 0; |
placement.num_busy_placement = 0; |
bdev->driver->evict_flags(bo, &placement); |
ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, |
no_wait_gpu); |
if (ret) { |
if (ret != -ERESTARTSYS) { |
pr_err("Failed to find memory space for buffer 0x%p eviction\n", |
bo); |
ttm_bo_mem_space_debug(bo, &placement); |
} |
goto out; |
} |
ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, |
no_wait_gpu); |
if (ret) { |
if (ret != -ERESTARTSYS) |
pr_err("Buffer eviction failed\n"); |
ttm_bo_mem_put(bo, &evict_mem); |
goto out; |
} |
bo->evicted = true; |
out: |
return ret; |
} |
static int ttm_mem_evict_first(struct ttm_bo_device *bdev, |
uint32_t mem_type, |
bool interruptible, |
bool no_wait_gpu) |
{ |
struct ttm_bo_global *glob = bdev->glob; |
struct ttm_mem_type_manager *man = &bdev->man[mem_type]; |
struct ttm_buffer_object *bo; |
int ret = -EBUSY, put_count; |
spin_lock(&glob->lru_lock); |
list_for_each_entry(bo, &man->lru, lru) { |
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); |
if (!ret) |
break; |
} |
if (ret) { |
spin_unlock(&glob->lru_lock); |
return ret; |
} |
kref_get(&bo->list_kref); |
if (!list_empty(&bo->ddestroy)) { |
ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, |
no_wait_gpu); |
kref_put(&bo->list_kref, ttm_bo_release_list); |
return ret; |
} |
put_count = ttm_bo_del_from_lru(bo); |
spin_unlock(&glob->lru_lock); |
BUG_ON(ret != 0); |
ttm_bo_list_ref_sub(bo, put_count, true); |
ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); |
ttm_bo_unreserve(bo); |
kref_put(&bo->list_kref, ttm_bo_release_list); |
return ret; |
} |
void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) |
{ |
struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; |
781,15 → 611,15 |
int ret; |
do { |
ret = (*man->func->get_node)(man, bo, placement, mem); |
ret = (*man->func->get_node)(man, bo, placement, 0, mem); |
if (unlikely(ret != 0)) |
return ret; |
if (mem->mm_node) |
break; |
ret = ttm_mem_evict_first(bdev, mem_type, |
interruptible, no_wait_gpu); |
if (unlikely(ret != 0)) |
return ret; |
// ret = ttm_mem_evict_first(bdev, mem_type, |
// interruptible, no_wait_gpu); |
// if (unlikely(ret != 0)) |
// return ret; |
} while (1); |
if (mem->mm_node == NULL) |
return -ENOMEM; |
894,7 → 724,8 |
if (man->has_type && man->use_type) { |
type_found = true; |
ret = (*man->func->get_node)(man, bo, placement, mem); |
ret = (*man->func->get_node)(man, bo, placement, |
cur_flags, mem); |
if (unlikely(ret)) |
return ret; |
} |
934,7 → 765,6 |
ttm_flag_masked(&cur_flags, placement->busy_placement[i], |
~TTM_PL_MASK_MEMTYPE); |
if (mem_type == TTM_PL_SYSTEM) { |
mem->mem_type = mem_type; |
mem->placement = cur_flags; |
965,7 → 795,7 |
struct ttm_mem_reg mem; |
struct ttm_bo_device *bdev = bo->bdev; |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
lockdep_assert_held(&bo->resv->lock.base); |
/* |
* FIXME: It's possible to pipeline buffer moves. |
996,7 → 826,6 |
ttm_bo_mem_put(bo, &mem); |
return ret; |
} |
#endif |
static bool ttm_bo_mem_compat(struct ttm_placement *placement, |
struct ttm_mem_reg *mem, |
1034,7 → 863,7 |
int ret; |
uint32_t new_flags; |
// BUG_ON(!ttm_bo_is_reserved(bo)); |
lockdep_assert_held(&bo->resv->lock.base); |
/* Check that range is valid */ |
if (placement->lpfn || placement->fpfn) |
if (placement->fpfn > placement->lpfn || |
1044,8 → 873,8 |
* Check whether we need to move buffer. |
*/ |
if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { |
// ret = ttm_bo_move_buffer(bo, placement, interruptible, |
// no_wait_gpu); |
ret = ttm_bo_move_buffer(bo, placement, interruptible, |
no_wait_gpu); |
if (ret) |
return ret; |
} else { |
1091,19 → 920,8 |
{ |
int ret = 0; |
unsigned long num_pages; |
struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; |
bool locked; |
// ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); |
if (ret) { |
pr_err("Out of kernel memory\n"); |
if (destroy) |
(*destroy)(bo); |
else |
kfree(bo); |
return -ENOMEM; |
} |
num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
if (num_pages == 0) { |
pr_err("Illegal buffer object size\n"); |
1111,7 → 929,6 |
(*destroy)(bo); |
else |
kfree(bo); |
// ttm_mem_global_free(mem_glob, acc_size); |
return -EINVAL; |
} |
bo->destroy = destroy; |
1141,7 → 958,7 |
bo->acc_size = acc_size; |
bo->sg = sg; |
bo->resv = &bo->ttm_resv; |
// reservation_object_init(bo->resv); |
reservation_object_init(bo->resv); |
atomic_inc(&bo->glob->bo_count); |
drm_vma_node_reset(&bo->vma_node); |
1151,19 → 968,23 |
* For ttm_bo_type_device buffers, allocate |
* address space from the device. |
*/ |
// if (likely(!ret) && |
// (bo->type == ttm_bo_type_device || |
// bo->type == ttm_bo_type_sg)) |
// ret = ttm_bo_setup_vm(bo); |
if (likely(!ret) && |
(bo->type == ttm_bo_type_device || |
bo->type == ttm_bo_type_sg)) |
ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, |
bo->mem.num_pages); |
// if (likely(!ret)) |
// ret = ttm_bo_validate(bo, placement, interruptible, false); |
locked = ww_mutex_trylock(&bo->resv->lock); |
WARN_ON(!locked); |
// ttm_bo_unreserve(bo); |
if (likely(!ret)) |
ret = ttm_bo_validate(bo, placement, interruptible, false); |
// if (unlikely(ret)) |
// ttm_bo_unref(&bo); |
ttm_bo_unreserve(bo); |
if (unlikely(ret)) |
ttm_bo_unref(&bo); |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_init); |
1197,53 → 1018,6 |
} |
EXPORT_SYMBOL(ttm_bo_dma_acc_size); |
int ttm_bo_create(struct ttm_bo_device *bdev, |
unsigned long size, |
enum ttm_bo_type type, |
struct ttm_placement *placement, |
uint32_t page_alignment, |
bool interruptible, |
struct file *persistent_swap_storage, |
struct ttm_buffer_object **p_bo) |
{ |
struct ttm_buffer_object *bo; |
size_t acc_size; |
int ret; |
bo = kzalloc(sizeof(*bo), GFP_KERNEL); |
if (unlikely(bo == NULL)) |
return -ENOMEM; |
acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); |
ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, |
interruptible, persistent_swap_storage, acc_size, |
NULL, NULL); |
if (likely(ret == 0)) |
*p_bo = bo; |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_create); |
int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, |
unsigned long p_size) |
{ |
1250,8 → 1024,6 |
int ret = -EINVAL; |
struct ttm_mem_type_manager *man; |
ENTER(); |
BUG_ON(type >= TTM_NUM_MEM_TYPES); |
man = &bdev->man[type]; |
BUG_ON(man->has_type); |
1277,12 → 1049,9 |
INIT_LIST_HEAD(&man->lru); |
LEAVE(); |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_init_mm); |
void ttm_bo_global_release(struct drm_global_reference *ref) |
{ |
struct ttm_bo_global *glob = ref->object; |
1297,12 → 1066,10 |
struct ttm_bo_global *glob = ref->object; |
int ret; |
ENTER(); |
mutex_init(&glob->device_list_mutex); |
spin_lock_init(&glob->lru_lock); |
glob->mem_glob = bo_ref->mem_glob; |
glob->dummy_read_page = AllocPage(); |
glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); |
if (unlikely(glob->dummy_read_page == NULL)) { |
ret = -ENOMEM; |
1314,8 → 1081,6 |
atomic_set(&glob->bo_count, 0); |
LEAVE(); |
return 0; |
out_no_drp: |
1324,17 → 1089,15 |
} |
EXPORT_SYMBOL(ttm_bo_global_init); |
int ttm_bo_device_init(struct ttm_bo_device *bdev, |
struct ttm_bo_global *glob, |
struct ttm_bo_driver *driver, |
struct address_space *mapping, |
uint64_t file_page_offset, |
bool need_dma32) |
{ |
int ret = -EINVAL; |
ENTER(); |
bdev->driver = driver; |
memset(bdev->man, 0, sizeof(bdev->man)); |
1349,9 → 1112,9 |
drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, |
0x10000000); |
// INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); |
INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); |
INIT_LIST_HEAD(&bdev->ddestroy); |
bdev->dev_mapping = NULL; |
bdev->dev_mapping = mapping; |
bdev->glob = glob; |
bdev->need_dma32 = need_dma32; |
bdev->val_seq = 0; |
1360,8 → 1123,6 |
list_add_tail(&bdev->device_list, &glob->device_list); |
mutex_unlock(&glob->device_list_mutex); |
LEAVE(); |
return 0; |
out_no_sys: |
return ret; |
1389,6 → 1150,28 |
return true; |
} |
void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); |
ttm_mem_io_free_vm(bo); |
} |
void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; |
ttm_mem_io_lock(man, false); |
ttm_bo_unmap_virtual_locked(bo); |
ttm_mem_io_unlock(man); |
} |
EXPORT_SYMBOL(ttm_bo_unmap_virtual); |
int ttm_bo_wait(struct ttm_buffer_object *bo, |
bool lazy, bool interruptible, bool no_wait) |
{ |
1400,25 → 1183,47 |
if (likely(bo->sync_obj == NULL)) |
return 0; |
return 0; |
while (bo->sync_obj) { |
if (driver->sync_obj_signaled(bo->sync_obj)) { |
void *tmp_obj = bo->sync_obj; |
bo->sync_obj = NULL; |
clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); |
spin_unlock(&bdev->fence_lock); |
driver->sync_obj_unref(&tmp_obj); |
spin_lock(&bdev->fence_lock); |
continue; |
} |
EXPORT_SYMBOL(ttm_bo_wait); |
int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) |
{ |
struct ttm_bo_device *bdev = bo->bdev; |
int ret = 0; |
if (no_wait) |
return -EBUSY; |
/* |
* Using ttm_bo_reserve makes sure the lru lists are updated. |
*/ |
sync_obj = driver->sync_obj_ref(bo->sync_obj); |
spin_unlock(&bdev->fence_lock); |
ret = driver->sync_obj_wait(sync_obj, |
lazy, interruptible); |
if (unlikely(ret != 0)) { |
driver->sync_obj_unref(&sync_obj); |
spin_lock(&bdev->fence_lock); |
return ret; |
} |
EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); |
spin_lock(&bdev->fence_lock); |
if (likely(bo->sync_obj == sync_obj)) { |
void *tmp_obj = bo->sync_obj; |
bo->sync_obj = NULL; |
clear_bit(TTM_BO_PRIV_FLAG_MOVING, |
&bo->priv_flags); |
spin_unlock(&bdev->fence_lock); |
driver->sync_obj_unref(&sync_obj); |
driver->sync_obj_unref(&tmp_obj); |
spin_lock(&bdev->fence_lock); |
} else { |
spin_unlock(&bdev->fence_lock); |
driver->sync_obj_unref(&sync_obj); |
spin_lock(&bdev->fence_lock); |
} |
} |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_wait); |
void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) |
{ |
atomic_dec(&bo->cpu_writers); |
} |
EXPORT_SYMBOL(ttm_bo_synccpu_write_release); |
/drivers/video/drm/ttm/ttm_bo_manager.c |
---|
50,11 → 50,13 |
static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, |
struct ttm_buffer_object *bo, |
struct ttm_placement *placement, |
uint32_t flags, |
struct ttm_mem_reg *mem) |
{ |
struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; |
struct drm_mm *mm = &rman->mm; |
struct drm_mm_node *node = NULL; |
enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT; |
unsigned long lpfn; |
int ret; |
66,11 → 68,15 |
if (!node) |
return -ENOMEM; |
if (flags & TTM_PL_FLAG_TOPDOWN) |
aflags = DRM_MM_CREATE_TOP; |
spin_lock(&rman->lock); |
ret = drm_mm_insert_node_in_range(mm, node, mem->num_pages, |
mem->page_alignment, |
ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages, |
mem->page_alignment, 0, |
placement->fpfn, lpfn, |
DRM_MM_SEARCH_BEST); |
DRM_MM_SEARCH_BEST, |
aflags); |
spin_unlock(&rman->lock); |
if (unlikely(ret)) { |
/drivers/video/drm/ttm/ttm_bo_util.c |
---|
27,17 → 27,25 |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
#define iowrite32(v, addr) writel((v), (addr)) |
#define ioread32(addr) readl(addr) |
#include <drm/ttm/ttm_bo_driver.h> |
#include <drm/ttm/ttm_placement.h> |
#include <drm/drm_vma_manager.h> |
#include <linux/io.h> |
#include <linux/highmem.h> |
//#include <linux/io.h> |
//#include <linux/highmem.h> |
#include <linux/wait.h> |
#include <linux/slab.h> |
#include <linux/vmalloc.h> |
//#include <linux/vmalloc.h> |
#include <linux/module.h> |
#define __pgprot(x) ((pgprot_t) { (x) } ) |
#define PAGE_KERNEL __pgprot(3) |
void *vmap(struct page **pages, unsigned int count, |
unsigned long flags, pgprot_t prot); |
void ttm_bo_free_old_node(struct ttm_buffer_object *bo) |
{ |
ttm_bo_mem_put(bo, &bo->mem); |
156,6 → 164,7 |
} |
EXPORT_SYMBOL(ttm_mem_io_free); |
#if 0 |
int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) |
{ |
struct ttm_mem_reg *mem = &bo->mem; |
175,6 → 184,7 |
} |
return 0; |
} |
#endif |
void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) |
{ |
207,7 → 217,7 |
if (mem->placement & TTM_PL_FLAG_WC) |
addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); |
else |
addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); |
addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size); |
if (!addr) { |
(void) ttm_mem_io_lock(man, false); |
ttm_mem_io_free(bdev, mem); |
258,27 → 268,14 |
src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); |
#ifdef CONFIG_X86 |
dst = kmap_atomic_prot(d, prot); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
dst = vmap(&d, 1, 0, prot); |
else |
dst = kmap(d); |
#endif |
dst = (void*)MapIoMem((addr_t)d, 4096, PG_SW); |
if (!dst) |
return -ENOMEM; |
memcpy_fromio(dst, src, PAGE_SIZE); |
memcpy(dst, src, PAGE_SIZE); |
#ifdef CONFIG_X86 |
kunmap_atomic(dst); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
vunmap(dst); |
else |
kunmap(d); |
#endif |
FreeKernelSpace(dst); |
return 0; |
} |
294,27 → 291,15 |
return -ENOMEM; |
dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); |
#ifdef CONFIG_X86 |
src = kmap_atomic_prot(s, prot); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
src = vmap(&s, 1, 0, prot); |
else |
src = kmap(s); |
#endif |
src = (void*)MapIoMem((addr_t)s, 4096, PG_SW); |
if (!src) |
return -ENOMEM; |
memcpy_toio(dst, src, PAGE_SIZE); |
memcpy(dst, src, PAGE_SIZE); |
#ifdef CONFIG_X86 |
kunmap_atomic(src); |
#else |
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) |
vunmap(src); |
else |
kunmap(s); |
#endif |
FreeKernelSpace(src); |
return 0; |
} |
352,8 → 337,12 |
/* |
* Don't move nonexistent data. Clear destination instead. |
*/ |
if (old_iomap == NULL && ttm == NULL) |
if (old_iomap == NULL && |
(ttm == NULL || (ttm->state == tt_unpopulated && |
!(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { |
memset(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); |
goto out2; |
} |
/* |
* TTM might be null for moves within the same region. |
483,29 → 472,6 |
pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) |
{ |
#if defined(__i386__) || defined(__x86_64__) |
if (caching_flags & TTM_PL_FLAG_WC) |
tmp = pgprot_writecombine(tmp); |
else if (boot_cpu_data.x86 > 3) |
tmp = pgprot_noncached(tmp); |
#elif defined(__powerpc__) |
if (!(caching_flags & TTM_PL_FLAG_CACHED)) { |
pgprot_val(tmp) |= _PAGE_NO_CACHE; |
if (caching_flags & TTM_PL_FLAG_UNCACHED) |
pgprot_val(tmp) |= _PAGE_GUARDED; |
} |
#endif |
#if defined(__ia64__) |
if (caching_flags & TTM_PL_FLAG_WC) |
tmp = pgprot_writecombine(tmp); |
else |
tmp = pgprot_noncached(tmp); |
#endif |
#if defined(__sparc__) || defined(__mips__) |
if (!(caching_flags & TTM_PL_FLAG_CACHED)) |
tmp = pgprot_noncached(tmp); |
#endif |
return tmp; |
} |
EXPORT_SYMBOL(ttm_io_prot); |
526,7 → 492,7 |
map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, |
size); |
else |
map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset, |
map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset, |
size); |
} |
return (!map->virtual) ? -ENOMEM : 0; |
557,7 → 523,7 |
map->bo_kmap_type = ttm_bo_map_kmap; |
map->page = ttm->pages[start_page]; |
map->virtual = kmap(map->page); |
map->virtual = (void*)MapIoMem(page_to_phys(map->page), 4096, PG_SW); |
} else { |
/* |
* We need to use vmap to get the desired page protection |
621,10 → 587,8 |
iounmap(map->virtual); |
break; |
case ttm_bo_map_vmap: |
vunmap(map->virtual); |
break; |
case ttm_bo_map_kmap: |
kunmap(map->page); |
FreeKernelSpace(map->virtual); |
break; |
case ttm_bo_map_premapped: |
break; |
713,3 → 677,25 |
return 0; |
} |
EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); |
void *vmap(struct page **pages, unsigned int count, |
unsigned long flags, pgprot_t prot) |
{ |
void *vaddr; |
char *tmp; |
int i; |
vaddr = AllocKernelSpace(count << 12); |
if(vaddr == NULL) |
return NULL; |
for(i = 0, tmp = vaddr; i < count; i++) |
{ |
MapPage(tmp, page_to_phys(pages[i]), PG_SW); |
tmp+= 4096; |
}; |
return vaddr; |
}; |
/drivers/video/drm/ttm/ttm_execbuf_util.c |
---|
24,7 → 24,6 |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
struct ww_acquire_ctx{}; |
#include <drm/ttm/ttm_execbuf_util.h> |
#include <drm/ttm/ttm_bo_driver.h> |
33,8 → 32,9 |
#include <linux/sched.h> |
#include <linux/module.h> |
static void ttm_eu_backoff_reservation_locked(struct list_head *list, |
struct ww_acquire_ctx *ticket) |
DEFINE_WW_CLASS(reservation_ww_class); |
static void ttm_eu_backoff_reservation_locked(struct list_head *list) |
{ |
struct ttm_validate_buffer *entry; |
48,7 → 48,7 |
ttm_bo_add_to_lru(bo); |
entry->removed = false; |
} |
// ww_mutex_unlock(&bo->resv->lock); |
__ttm_bo_unreserve(bo); |
} |
} |
94,8 → 94,9 |
entry = list_first_entry(list, struct ttm_validate_buffer, head); |
glob = entry->bo->glob; |
spin_lock(&glob->lru_lock); |
ttm_eu_backoff_reservation_locked(list, ticket); |
// ww_acquire_fini(ticket); |
ttm_eu_backoff_reservation_locked(list); |
if (ticket) |
ww_acquire_fini(ticket); |
spin_unlock(&glob->lru_lock); |
} |
EXPORT_SYMBOL(ttm_eu_backoff_reservation); |
131,7 → 132,8 |
entry = list_first_entry(list, struct ttm_validate_buffer, head); |
glob = entry->bo->glob; |
// ww_acquire_init(ticket, &reservation_ww_class); |
if (ticket) |
ww_acquire_init(ticket, &reservation_ww_class); |
retry: |
list_for_each_entry(entry, list, head) { |
struct ttm_buffer_object *bo = entry->bo; |
140,20 → 142,21 |
if (entry->reserved) |
continue; |
ret = __ttm_bo_reserve(bo, true, (ticket == NULL), true, |
ticket); |
ret = ttm_bo_reserve_nolru(bo, true, false, true, ticket); |
if (ret == -EDEADLK) { |
/* uh oh, we lost out, drop every reservation and try |
* to only reserve this buffer, then start over if |
* this succeeds. |
*/ |
BUG_ON(ticket == NULL); |
spin_lock(&glob->lru_lock); |
ttm_eu_backoff_reservation_locked(list, ticket); |
ttm_eu_backoff_reservation_locked(list); |
spin_unlock(&glob->lru_lock); |
ttm_eu_list_ref_sub(list); |
// ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, |
// ticket); |
ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, |
ticket); |
if (unlikely(ret != 0)) { |
if (ret == -EINTR) |
ret = -ERESTARTSYS; |
176,7 → 179,8 |
} |
} |
// ww_acquire_done(ticket); |
if (ticket) |
ww_acquire_done(ticket); |
spin_lock(&glob->lru_lock); |
ttm_eu_del_from_lru_locked(list); |
spin_unlock(&glob->lru_lock); |
185,12 → 189,14 |
err: |
spin_lock(&glob->lru_lock); |
ttm_eu_backoff_reservation_locked(list, ticket); |
ttm_eu_backoff_reservation_locked(list); |
spin_unlock(&glob->lru_lock); |
ttm_eu_list_ref_sub(list); |
err_fini: |
// ww_acquire_done(ticket); |
// ww_acquire_fini(ticket); |
if (ticket) { |
ww_acquire_done(ticket); |
ww_acquire_fini(ticket); |
} |
return ret; |
} |
EXPORT_SYMBOL(ttm_eu_reserve_buffers); |
220,12 → 226,13 |
entry->old_sync_obj = bo->sync_obj; |
bo->sync_obj = driver->sync_obj_ref(sync_obj); |
ttm_bo_add_to_lru(bo); |
// ww_mutex_unlock(&bo->resv->lock); |
__ttm_bo_unreserve(bo); |
entry->reserved = false; |
} |
spin_unlock(&bdev->fence_lock); |
spin_unlock(&glob->lru_lock); |
// ww_acquire_fini(ticket); |
if (ticket) |
ww_acquire_fini(ticket); |
list_for_each_entry(entry, list, head) { |
if (entry->old_sync_obj) |
/drivers/video/drm/ttm/ttm_lock.c |
---|
0,0 → 1,286 |
/************************************************************************** |
* |
* Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA |
* All Rights Reserved. |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the |
* "Software"), to deal in the Software without restriction, including |
* without limitation the rights to use, copy, modify, merge, publish, |
* distribute, sub license, and/or sell copies of the Software, and to |
* permit persons to whom the Software is furnished to do so, subject to |
* the following conditions: |
* |
* The above copyright notice and this permission notice (including the |
* next paragraph) shall be included in all copies or substantial portions |
* of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
* USE OR OTHER DEALINGS IN THE SOFTWARE. |
* |
**************************************************************************/ |
/* |
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
*/ |
#include <linux/mutex.h> |
#include <drm/ttm/ttm_lock.h> |
#include <drm/ttm/ttm_module.h> |
//#include <linux/atomic.h> |
#include <linux/errno.h> |
#include <linux/wait.h> |
#include <linux/sched.h> |
#include <linux/module.h> |
#define TTM_WRITE_LOCK_PENDING (1 << 0) |
#define TTM_VT_LOCK_PENDING (1 << 1) |
#define TTM_SUSPEND_LOCK_PENDING (1 << 2) |
#define TTM_VT_LOCK (1 << 3) |
#define TTM_SUSPEND_LOCK (1 << 4) |
void ttm_lock_init(struct ttm_lock *lock) |
{ |
spin_lock_init(&lock->lock); |
init_waitqueue_head(&lock->queue); |
lock->rw = 0; |
lock->flags = 0; |
} |
EXPORT_SYMBOL(ttm_lock_init); |
void ttm_read_unlock(struct ttm_lock *lock) |
{ |
spin_lock(&lock->lock); |
if (--lock->rw == 0) |
wake_up_all(&lock->queue); |
spin_unlock(&lock->lock); |
} |
EXPORT_SYMBOL(ttm_read_unlock); |
static bool __ttm_read_lock(struct ttm_lock *lock) |
{ |
bool locked = false; |
spin_lock(&lock->lock); |
if (lock->rw >= 0 && lock->flags == 0) { |
++lock->rw; |
locked = true; |
} |
spin_unlock(&lock->lock); |
return locked; |
} |
int ttm_read_lock(struct ttm_lock *lock, bool interruptible) |
{ |
int ret = 0; |
if (interruptible) |
ret = wait_event_interruptible(lock->queue, |
__ttm_read_lock(lock)); |
else |
wait_event(lock->queue, __ttm_read_lock(lock)); |
return ret; |
} |
EXPORT_SYMBOL(ttm_read_lock); |
static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked) |
{ |
bool block = true; |
*locked = false; |
spin_lock(&lock->lock); |
if (lock->rw >= 0 && lock->flags == 0) { |
++lock->rw; |
block = false; |
*locked = true; |
} else if (lock->flags == 0) { |
block = false; |
} |
spin_unlock(&lock->lock); |
return !block; |
} |
int ttm_read_trylock(struct ttm_lock *lock, bool interruptible) |
{ |
int ret = 0; |
bool locked; |
if (interruptible) |
ret = wait_event_interruptible |
(lock->queue, __ttm_read_trylock(lock, &locked)); |
else |
wait_event(lock->queue, __ttm_read_trylock(lock, &locked)); |
if (unlikely(ret != 0)) { |
BUG_ON(locked); |
return ret; |
} |
return (locked) ? 0 : -EBUSY; |
} |
void ttm_write_unlock(struct ttm_lock *lock) |
{ |
spin_lock(&lock->lock); |
lock->rw = 0; |
wake_up_all(&lock->queue); |
spin_unlock(&lock->lock); |
} |
EXPORT_SYMBOL(ttm_write_unlock); |
static bool __ttm_write_lock(struct ttm_lock *lock) |
{ |
bool locked = false; |
spin_lock(&lock->lock); |
if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) { |
lock->rw = -1; |
lock->flags &= ~TTM_WRITE_LOCK_PENDING; |
locked = true; |
} else { |
lock->flags |= TTM_WRITE_LOCK_PENDING; |
} |
spin_unlock(&lock->lock); |
return locked; |
} |
int ttm_write_lock(struct ttm_lock *lock, bool interruptible) |
{ |
int ret = 0; |
if (interruptible) { |
ret = wait_event_interruptible(lock->queue, |
__ttm_write_lock(lock)); |
if (unlikely(ret != 0)) { |
spin_lock(&lock->lock); |
lock->flags &= ~TTM_WRITE_LOCK_PENDING; |
wake_up_all(&lock->queue); |
spin_unlock(&lock->lock); |
} |
} else |
wait_event(lock->queue, __ttm_read_lock(lock)); |
return ret; |
} |
EXPORT_SYMBOL(ttm_write_lock); |
static int __ttm_vt_unlock(struct ttm_lock *lock) |
{ |
int ret = 0; |
spin_lock(&lock->lock); |
if (unlikely(!(lock->flags & TTM_VT_LOCK))) |
ret = -EINVAL; |
lock->flags &= ~TTM_VT_LOCK; |
wake_up_all(&lock->queue); |
spin_unlock(&lock->lock); |
return ret; |
} |
static void ttm_vt_lock_remove(struct ttm_base_object **p_base) |
{ |
struct ttm_base_object *base = *p_base; |
struct ttm_lock *lock = container_of(base, struct ttm_lock, base); |
int ret; |
*p_base = NULL; |
ret = __ttm_vt_unlock(lock); |
BUG_ON(ret != 0); |
} |
static bool __ttm_vt_lock(struct ttm_lock *lock) |
{ |
bool locked = false; |
spin_lock(&lock->lock); |
if (lock->rw == 0) { |
lock->flags &= ~TTM_VT_LOCK_PENDING; |
lock->flags |= TTM_VT_LOCK; |
locked = true; |
} else { |
lock->flags |= TTM_VT_LOCK_PENDING; |
} |
spin_unlock(&lock->lock); |
return locked; |
} |
int ttm_vt_lock(struct ttm_lock *lock, |
bool interruptible, |
struct ttm_object_file *tfile) |
{ |
int ret = 0; |
if (interruptible) { |
ret = wait_event_interruptible(lock->queue, |
__ttm_vt_lock(lock)); |
if (unlikely(ret != 0)) { |
spin_lock(&lock->lock); |
lock->flags &= ~TTM_VT_LOCK_PENDING; |
wake_up_all(&lock->queue); |
spin_unlock(&lock->lock); |
return ret; |
} |
} else |
wait_event(lock->queue, __ttm_vt_lock(lock)); |
/* |
* Add a base-object, the destructor of which will |
* make sure the lock is released if the client dies |
* while holding it. |
*/ |
ret = ttm_base_object_init(tfile, &lock->base, false, |
ttm_lock_type, &ttm_vt_lock_remove, NULL); |
if (ret) |
(void)__ttm_vt_unlock(lock); |
else |
lock->vt_holder = tfile; |
return ret; |
} |
EXPORT_SYMBOL(ttm_vt_lock); |
int ttm_vt_unlock(struct ttm_lock *lock) |
{ |
return ttm_ref_object_base_unref(lock->vt_holder, |
lock->base.hash.key, TTM_REF_USAGE); |
} |
EXPORT_SYMBOL(ttm_vt_unlock); |
void ttm_suspend_unlock(struct ttm_lock *lock) |
{ |
spin_lock(&lock->lock); |
lock->flags &= ~TTM_SUSPEND_LOCK; |
wake_up_all(&lock->queue); |
spin_unlock(&lock->lock); |
} |
EXPORT_SYMBOL(ttm_suspend_unlock); |
static bool __ttm_suspend_lock(struct ttm_lock *lock) |
{ |
bool locked = false; |
spin_lock(&lock->lock); |
if (lock->rw == 0) { |
lock->flags &= ~TTM_SUSPEND_LOCK_PENDING; |
lock->flags |= TTM_SUSPEND_LOCK; |
locked = true; |
} else { |
lock->flags |= TTM_SUSPEND_LOCK_PENDING; |
} |
spin_unlock(&lock->lock); |
return locked; |
} |
void ttm_suspend_lock(struct ttm_lock *lock) |
{ |
wait_event(lock->queue, __ttm_suspend_lock(lock)); |
} |
EXPORT_SYMBOL(ttm_suspend_lock); |
/drivers/video/drm/ttm/ttm_memory.c |
---|
37,534 → 37,40 |
#include <linux/module.h> |
#include <linux/slab.h> |
struct sysinfo { |
u32_t totalram; /* Total usable main memory size */ |
u32_t freeram; /* Available memory size */ |
u32_t sharedram; /* Amount of shared memory */ |
u32_t bufferram; /* Memory used by buffers */ |
u32_t totalswap; /* Total swap space size */ |
u32_t freeswap; /* swap space still available */ |
u32_t totalhigh; /* Total high memory size */ |
u32_t freehigh; /* Available high memory size */ |
u32_t mem_unit; /* Memory unit size in bytes */ |
}; |
#define TTM_MEMORY_ALLOC_RETRIES 4 |
struct ttm_mem_zone { |
struct kobject kobj; |
struct ttm_mem_global *glob; |
const char *name; |
uint64_t zone_mem; |
uint64_t emer_mem; |
uint64_t max_mem; |
uint64_t swap_limit; |
uint64_t used_mem; |
}; |
#if 0 |
static struct attribute ttm_mem_sys = { |
.name = "zone_memory", |
.mode = S_IRUGO |
}; |
static struct attribute ttm_mem_emer = { |
.name = "emergency_memory", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_mem_max = { |
.name = "available_memory", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_mem_swap = { |
.name = "swap_limit", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_mem_used = { |
.name = "used_memory", |
.mode = S_IRUGO |
}; |
#endif |
static void ttm_mem_zone_kobj_release(struct kobject *kobj) |
int ttm_mem_global_init(struct ttm_mem_global *glob) |
{ |
struct ttm_mem_zone *zone = |
container_of(kobj, struct ttm_mem_zone, kobj); |
pr_info("Zone %7s: Used memory at exit: %llu kiB\n", |
zone->name, (unsigned long long)zone->used_mem >> 10); |
kfree(zone); |
} |
#if 0 |
static ssize_t ttm_mem_zone_show(struct kobject *kobj, |
struct attribute *attr, |
char *buffer) |
{ |
struct ttm_mem_zone *zone = |
container_of(kobj, struct ttm_mem_zone, kobj); |
uint64_t val = 0; |
spin_lock(&zone->glob->lock); |
if (attr == &ttm_mem_sys) |
val = zone->zone_mem; |
else if (attr == &ttm_mem_emer) |
val = zone->emer_mem; |
else if (attr == &ttm_mem_max) |
val = zone->max_mem; |
else if (attr == &ttm_mem_swap) |
val = zone->swap_limit; |
else if (attr == &ttm_mem_used) |
val = zone->used_mem; |
spin_unlock(&zone->glob->lock); |
return snprintf(buffer, PAGE_SIZE, "%llu\n", |
(unsigned long long) val >> 10); |
} |
static void ttm_check_swapping(struct ttm_mem_global *glob); |
static ssize_t ttm_mem_zone_store(struct kobject *kobj, |
struct attribute *attr, |
const char *buffer, |
size_t size) |
{ |
struct ttm_mem_zone *zone = |
container_of(kobj, struct ttm_mem_zone, kobj); |
int chars; |
unsigned long val; |
uint64_t val64; |
chars = sscanf(buffer, "%lu", &val); |
if (chars == 0) |
return size; |
val64 = val; |
val64 <<= 10; |
spin_lock(&zone->glob->lock); |
if (val64 > zone->zone_mem) |
val64 = zone->zone_mem; |
if (attr == &ttm_mem_emer) { |
zone->emer_mem = val64; |
if (zone->max_mem > val64) |
zone->max_mem = val64; |
} else if (attr == &ttm_mem_max) { |
zone->max_mem = val64; |
if (zone->emer_mem < val64) |
zone->emer_mem = val64; |
} else if (attr == &ttm_mem_swap) |
zone->swap_limit = val64; |
spin_unlock(&zone->glob->lock); |
ttm_check_swapping(zone->glob); |
return size; |
} |
#endif |
//static struct attribute *ttm_mem_zone_attrs[] = { |
// &ttm_mem_sys, |
// &ttm_mem_emer, |
// &ttm_mem_max, |
// &ttm_mem_swap, |
// &ttm_mem_used, |
// NULL |
//}; |
//static const struct sysfs_ops ttm_mem_zone_ops = { |
// .show = &ttm_mem_zone_show, |
// .store = &ttm_mem_zone_store |
//}; |
static struct kobj_type ttm_mem_zone_kobj_type = { |
.release = &ttm_mem_zone_kobj_release, |
// .sysfs_ops = &ttm_mem_zone_ops, |
// .default_attrs = ttm_mem_zone_attrs, |
}; |
static void ttm_mem_global_kobj_release(struct kobject *kobj) |
{ |
struct ttm_mem_global *glob = |
container_of(kobj, struct ttm_mem_global, kobj); |
kfree(glob); |
} |
static struct kobj_type ttm_mem_glob_kobj_type = { |
.release = &ttm_mem_global_kobj_release, |
}; |
#if 0 |
static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, |
bool from_wq, uint64_t extra) |
{ |
unsigned int i; |
struct ttm_mem_zone *zone; |
uint64_t target; |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (from_wq) |
target = zone->swap_limit; |
else if (capable(CAP_SYS_ADMIN)) |
target = zone->emer_mem; |
else |
target = zone->max_mem; |
target = (extra > target) ? 0ULL : target; |
if (zone->used_mem > target) |
return true; |
} |
return false; |
} |
/** |
* At this point we only support a single shrink callback. |
* Extend this if needed, perhaps using a linked list of callbacks. |
* Note that this function is reentrant: |
* many threads may try to swap out at any given time. |
*/ |
static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, |
uint64_t extra) |
{ |
int ret; |
struct ttm_mem_shrink *shrink; |
int i; |
spin_lock(&glob->lock); |
if (glob->shrink == NULL) |
goto out; |
spin_lock_init(&glob->lock); |
while (ttm_zones_above_swap_target(glob, from_wq, extra)) { |
shrink = glob->shrink; |
spin_unlock(&glob->lock); |
ret = shrink->do_shrink(shrink); |
spin_lock(&glob->lock); |
if (unlikely(ret != 0)) |
goto out; |
} |
out: |
spin_unlock(&glob->lock); |
} |
ttm_page_alloc_init(glob, 4*1024); |
static void ttm_shrink_work(struct work_struct *work) |
{ |
struct ttm_mem_global *glob = |
container_of(work, struct ttm_mem_global, work); |
ttm_shrink(glob, true, 0ULL); |
} |
#endif |
static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, |
const struct sysinfo *si) |
{ |
struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
uint64_t mem; |
int ret; |
if (unlikely(!zone)) |
return -ENOMEM; |
// mem = si->totalram - si->totalhigh; |
// mem *= si->mem_unit; |
zone->name = "kernel"; |
zone->zone_mem = mem; |
zone->max_mem = mem >> 1; |
zone->emer_mem = (mem >> 1) + (mem >> 2); |
zone->swap_limit = zone->max_mem - (mem >> 3); |
zone->used_mem = 0; |
zone->glob = glob; |
glob->zone_kernel = zone; |
ret = kobject_init_and_add( |
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
if (unlikely(ret != 0)) { |
kobject_put(&zone->kobj); |
return ret; |
} |
glob->zones[glob->num_zones++] = zone; |
return 0; |
} |
#if 0 |
#ifdef CONFIG_HIGHMEM |
static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, |
const struct sysinfo *si) |
{ |
struct ttm_mem_zone *zone; |
uint64_t mem; |
int ret; |
if (si->totalhigh == 0) |
return 0; |
zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
if (unlikely(!zone)) |
return -ENOMEM; |
mem = si->totalram; |
mem *= si->mem_unit; |
zone->name = "highmem"; |
zone->zone_mem = mem; |
zone->max_mem = mem >> 1; |
zone->emer_mem = (mem >> 1) + (mem >> 2); |
zone->swap_limit = zone->max_mem - (mem >> 3); |
zone->used_mem = 0; |
zone->glob = glob; |
glob->zone_highmem = zone; |
ret = kobject_init_and_add( |
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
if (unlikely(ret != 0)) { |
kobject_put(&zone->kobj); |
out_no_zone: |
ttm_mem_global_release(glob); |
return ret; |
} |
glob->zones[glob->num_zones++] = zone; |
return 0; |
} |
#else |
static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, |
const struct sysinfo *si) |
{ |
struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
uint64_t mem; |
int ret; |
EXPORT_SYMBOL(ttm_mem_global_init); |
if (unlikely(!zone)) |
return -ENOMEM; |
mem = si->totalram; |
mem *= si->mem_unit; |
/** |
* No special dma32 zone needed. |
*/ |
if (mem <= ((uint64_t) 1ULL << 32)) { |
kfree(zone); |
return 0; |
} |
/* |
* Limit max dma32 memory to 4GB for now |
* until we can figure out how big this |
* zone really is. |
*/ |
mem = ((uint64_t) 1ULL << 32); |
zone->name = "dma32"; |
zone->zone_mem = mem; |
zone->max_mem = mem >> 1; |
zone->emer_mem = (mem >> 1) + (mem >> 2); |
zone->swap_limit = zone->max_mem - (mem >> 3); |
zone->used_mem = 0; |
zone->glob = glob; |
glob->zone_dma32 = zone; |
ret = kobject_init_and_add( |
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); |
if (unlikely(ret != 0)) { |
kobject_put(&zone->kobj); |
return ret; |
} |
glob->zones[glob->num_zones++] = zone; |
return 0; |
} |
#endif |
void ttm_mem_global_release(struct ttm_mem_global *glob) |
{ |
unsigned int i; |
struct ttm_mem_zone *zone; |
/* let the page allocator first stop the shrink work. */ |
ttm_page_alloc_fini(); |
ttm_dma_page_alloc_fini(); |
// ttm_page_alloc_fini(); |
// ttm_dma_page_alloc_fini(); |
flush_workqueue(glob->swap_queue); |
destroy_workqueue(glob->swap_queue); |
glob->swap_queue = NULL; |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
kobject_del(&zone->kobj); |
kobject_put(&zone->kobj); |
} |
kobject_del(&glob->kobj); |
kobject_put(&glob->kobj); |
} |
EXPORT_SYMBOL(ttm_mem_global_release); |
static void ttm_check_swapping(struct ttm_mem_global *glob) |
{ |
bool needs_swapping = false; |
unsigned int i; |
struct ttm_mem_zone *zone; |
spin_lock(&glob->lock); |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (zone->used_mem > zone->swap_limit) { |
needs_swapping = true; |
break; |
} |
} |
spin_unlock(&glob->lock); |
if (unlikely(needs_swapping)) |
(void)queue_work(glob->swap_queue, &glob->work); |
} |
static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, |
struct ttm_mem_zone *single_zone, |
uint64_t amount) |
{ |
unsigned int i; |
struct ttm_mem_zone *zone; |
spin_lock(&glob->lock); |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (single_zone && zone != single_zone) |
continue; |
zone->used_mem -= amount; |
} |
spin_unlock(&glob->lock); |
} |
void ttm_mem_global_free(struct ttm_mem_global *glob, |
uint64_t amount) |
{ |
return ttm_mem_global_free_zone(glob, NULL, amount); |
} |
EXPORT_SYMBOL(ttm_mem_global_free); |
static int ttm_mem_global_reserve(struct ttm_mem_global *glob, |
struct ttm_mem_zone *single_zone, |
uint64_t amount, bool reserve) |
{ |
uint64_t limit; |
int ret = -ENOMEM; |
unsigned int i; |
struct ttm_mem_zone *zone; |
spin_lock(&glob->lock); |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (single_zone && zone != single_zone) |
continue; |
limit = zone->emer_mem; |
if (zone->used_mem > limit) |
goto out_unlock; |
} |
if (reserve) { |
for (i = 0; i < glob->num_zones; ++i) { |
zone = glob->zones[i]; |
if (single_zone && zone != single_zone) |
continue; |
zone->used_mem += amount; |
} |
} |
ret = 0; |
out_unlock: |
spin_unlock(&glob->lock); |
ttm_check_swapping(glob); |
return ret; |
} |
static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, |
struct ttm_mem_zone *single_zone, |
uint64_t memory, |
bool no_wait, bool interruptible) |
{ |
int count = TTM_MEMORY_ALLOC_RETRIES; |
while (unlikely(ttm_mem_global_reserve(glob, |
single_zone, |
memory, true) |
!= 0)) { |
if (no_wait) |
return -ENOMEM; |
if (unlikely(count-- == 0)) |
return -ENOMEM; |
ttm_shrink(glob, false, memory + (memory >> 2) + 16); |
} |
return 0; |
} |
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, |
bool no_wait, bool interruptible) |
{ |
/** |
* Normal allocations of kernel memory are registered in |
* all zones. |
*/ |
return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, |
interruptible); |
} |
EXPORT_SYMBOL(ttm_mem_global_alloc); |
int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, |
struct page *page, |
bool no_wait, bool interruptible) |
{ |
struct ttm_mem_zone *zone = NULL; |
/** |
* Page allocations may be registed in a single zone |
* only if highmem or !dma32. |
*/ |
#ifdef CONFIG_HIGHMEM |
if (PageHighMem(page) && glob->zone_highmem != NULL) |
zone = glob->zone_highmem; |
#else |
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) |
zone = glob->zone_kernel; |
#endif |
return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait, |
interruptible); |
} |
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) |
{ |
struct ttm_mem_zone *zone = NULL; |
#ifdef CONFIG_HIGHMEM |
if (PageHighMem(page) && glob->zone_highmem != NULL) |
zone = glob->zone_highmem; |
#else |
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) |
zone = glob->zone_kernel; |
#endif |
ttm_mem_global_free_zone(glob, zone, PAGE_SIZE); |
} |
#endif |
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page) |
{ |
} |
size_t ttm_round_pot(size_t size) |
{ |
if ((size & (size - 1)) == 0) |
582,24 → 88,3 |
return 0; |
} |
EXPORT_SYMBOL(ttm_round_pot); |
void ttm_mem_global_free(struct ttm_mem_global *glob, |
uint64_t amount) |
{ |
return 0; |
} |
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, |
bool no_wait, bool interruptible) |
{ |
return 0; |
} |
EXPORT_SYMBOL(ttm_mem_global_alloc); |
int ttm_mem_global_init(struct ttm_mem_global *glob) |
{ |
return 0; |
} |
EXPORT_SYMBOL(ttm_mem_global_init); |
/drivers/video/drm/ttm/ttm_page_alloc.c |
---|
41,7 → 41,7 |
#include <linux/mm.h> |
#include <linux/seq_file.h> /* for seq_printf */ |
#include <linux/slab.h> |
#include <linux/dma-mapping.h> |
//#include <linux/dma-mapping.h> |
//#include <linux/atomic.h> |
58,12 → 58,6 |
/* times are in msecs */ |
#define PAGE_FREE_INTERVAL 1000 |
#define pr_err(fmt, ...) \ |
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) |
#if 0 |
/** |
* struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. |
* |
115,7 → 109,6 |
**/ |
struct ttm_pool_manager { |
struct kobject kobj; |
struct shrinker mm_shrink; |
struct ttm_pool_opts options; |
union { |
129,134 → 122,10 |
}; |
}; |
static struct attribute ttm_page_pool_max = { |
.name = "pool_max_size", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_page_pool_small = { |
.name = "pool_small_allocation", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute ttm_page_pool_alloc_size = { |
.name = "pool_allocation_size", |
.mode = S_IRUGO | S_IWUSR |
}; |
static struct attribute *ttm_pool_attrs[] = { |
&ttm_page_pool_max, |
&ttm_page_pool_small, |
&ttm_page_pool_alloc_size, |
NULL |
}; |
static void ttm_pool_kobj_release(struct kobject *kobj) |
{ |
struct ttm_pool_manager *m = |
container_of(kobj, struct ttm_pool_manager, kobj); |
kfree(m); |
} |
static ssize_t ttm_pool_store(struct kobject *kobj, |
struct attribute *attr, const char *buffer, size_t size) |
{ |
struct ttm_pool_manager *m = |
container_of(kobj, struct ttm_pool_manager, kobj); |
int chars; |
unsigned val; |
chars = sscanf(buffer, "%u", &val); |
if (chars == 0) |
return size; |
/* Convert kb to number of pages */ |
val = val / (PAGE_SIZE >> 10); |
if (attr == &ttm_page_pool_max) |
m->options.max_size = val; |
else if (attr == &ttm_page_pool_small) |
m->options.small = val; |
else if (attr == &ttm_page_pool_alloc_size) { |
if (val > NUM_PAGES_TO_ALLOC*8) { |
pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", |
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), |
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); |
return size; |
} else if (val > NUM_PAGES_TO_ALLOC) { |
pr_warn("Setting allocation size to larger than %lu is not recommended\n", |
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); |
} |
m->options.alloc_size = val; |
} |
return size; |
} |
static ssize_t ttm_pool_show(struct kobject *kobj, |
struct attribute *attr, char *buffer) |
{ |
struct ttm_pool_manager *m = |
container_of(kobj, struct ttm_pool_manager, kobj); |
unsigned val = 0; |
if (attr == &ttm_page_pool_max) |
val = m->options.max_size; |
else if (attr == &ttm_page_pool_small) |
val = m->options.small; |
else if (attr == &ttm_page_pool_alloc_size) |
val = m->options.alloc_size; |
val = val * (PAGE_SIZE >> 10); |
return snprintf(buffer, PAGE_SIZE, "%u\n", val); |
} |
static const struct sysfs_ops ttm_pool_sysfs_ops = { |
.show = &ttm_pool_show, |
.store = &ttm_pool_store, |
}; |
static struct kobj_type ttm_pool_kobj_type = { |
.release = &ttm_pool_kobj_release, |
.sysfs_ops = &ttm_pool_sysfs_ops, |
.default_attrs = ttm_pool_attrs, |
}; |
static struct ttm_pool_manager *_manager; |
#ifndef CONFIG_X86 |
static int set_pages_array_wb(struct page **pages, int addrinarray) |
{ |
#ifdef TTM_HAS_AGP |
int i; |
for (i = 0; i < addrinarray; i++) |
unmap_page_from_agp(pages[i]); |
#endif |
return 0; |
} |
static int set_pages_array_wc(struct page **pages, int addrinarray) |
{ |
#ifdef TTM_HAS_AGP |
int i; |
for (i = 0; i < addrinarray; i++) |
map_page_into_agp(pages[i]); |
#endif |
return 0; |
} |
static int set_pages_array_uc(struct page **pages, int addrinarray) |
{ |
#ifdef TTM_HAS_AGP |
int i; |
for (i = 0; i < addrinarray; i++) |
map_page_into_agp(pages[i]); |
#endif |
return 0; |
} |
#endif |
/** |
* Select the right pool or requested caching state and ttm flags. */ |
static struct ttm_page_pool *ttm_get_pool(int flags, |
282,8 → 151,6 |
static void ttm_pages_put(struct page *pages[], unsigned npages) |
{ |
unsigned i; |
if (set_pages_array_wb(pages, npages)) |
pr_err("Failed to set %d pages to wb!\n", npages); |
for (i = 0; i < npages; ++i) |
__free_page(pages[i]); |
} |
295,407 → 162,21 |
pool->nfrees += freed_pages; |
} |
/** |
* Free pages from pool. |
* |
* To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC |
* number of pages in one go. |
* |
* @pool: to free the pages from |
* @free_all: If set to true will free all pages in pool |
**/ |
static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) |
{ |
unsigned long irq_flags; |
struct page *p; |
struct page **pages_to_free; |
unsigned freed_pages = 0, |
npages_to_free = nr_free; |
if (NUM_PAGES_TO_ALLOC < nr_free) |
npages_to_free = NUM_PAGES_TO_ALLOC; |
pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), |
GFP_KERNEL); |
if (!pages_to_free) { |
pr_err("Failed to allocate memory for pool free operation\n"); |
return 0; |
} |
restart: |
spin_lock_irqsave(&pool->lock, irq_flags); |
list_for_each_entry_reverse(p, &pool->list, lru) { |
if (freed_pages >= npages_to_free) |
break; |
pages_to_free[freed_pages++] = p; |
/* We can only remove NUM_PAGES_TO_ALLOC at a time. */ |
if (freed_pages >= NUM_PAGES_TO_ALLOC) { |
/* remove range of pages from the pool */ |
__list_del(p->lru.prev, &pool->list); |
ttm_pool_update_free_locked(pool, freed_pages); |
/** |
* Because changing page caching is costly |
* we unlock the pool to prevent stalling. |
*/ |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
ttm_pages_put(pages_to_free, freed_pages); |
if (likely(nr_free != FREE_ALL_PAGES)) |
nr_free -= freed_pages; |
if (NUM_PAGES_TO_ALLOC >= nr_free) |
npages_to_free = nr_free; |
else |
npages_to_free = NUM_PAGES_TO_ALLOC; |
freed_pages = 0; |
/* free all so restart the processing */ |
if (nr_free) |
goto restart; |
/* Not allowed to fall through or break because |
* following context is inside spinlock while we are |
* outside here. |
*/ |
goto out; |
} |
} |
/* remove range of pages from the pool */ |
if (freed_pages) { |
__list_del(&p->lru, &pool->list); |
ttm_pool_update_free_locked(pool, freed_pages); |
nr_free -= freed_pages; |
} |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
if (freed_pages) |
ttm_pages_put(pages_to_free, freed_pages); |
out: |
kfree(pages_to_free); |
return nr_free; |
} |
/** |
* Callback for mm to request pool to reduce number of page held. |
* |
* XXX: (dchinner) Deadlock warning! |
* |
* ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means |
* this can deadlock when called a sc->gfp_mask that is not equal to |
* GFP_KERNEL. |
* |
* This code is crying out for a shrinker per pool.... |
*/ |
static unsigned long |
ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) |
{ |
static atomic_t start_pool = ATOMIC_INIT(0); |
unsigned i; |
unsigned pool_offset = atomic_add_return(1, &start_pool); |
struct ttm_page_pool *pool; |
int shrink_pages = sc->nr_to_scan; |
unsigned long freed = 0; |
pool_offset = pool_offset % NUM_POOLS; |
/* select start pool in round robin fashion */ |
for (i = 0; i < NUM_POOLS; ++i) { |
unsigned nr_free = shrink_pages; |
if (shrink_pages == 0) |
break; |
pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; |
shrink_pages = ttm_page_pool_free(pool, nr_free); |
freed += nr_free - shrink_pages; |
} |
return freed; |
} |
static unsigned long |
ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) |
{ |
unsigned i; |
unsigned long count = 0; |
for (i = 0; i < NUM_POOLS; ++i) |
count += _manager->pools[i].npages; |
return count; |
} |
static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) |
{ |
manager->mm_shrink.count_objects = ttm_pool_shrink_count; |
manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; |
manager->mm_shrink.seeks = 1; |
register_shrinker(&manager->mm_shrink); |
} |
static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) |
{ |
unregister_shrinker(&manager->mm_shrink); |
} |
static int ttm_set_pages_caching(struct page **pages, |
enum ttm_caching_state cstate, unsigned cpages) |
{ |
int r = 0; |
/* Set page caching */ |
switch (cstate) { |
case tt_uncached: |
r = set_pages_array_uc(pages, cpages); |
if (r) |
pr_err("Failed to set %d pages to uc!\n", cpages); |
break; |
case tt_wc: |
r = set_pages_array_wc(pages, cpages); |
if (r) |
pr_err("Failed to set %d pages to wc!\n", cpages); |
break; |
default: |
break; |
} |
return r; |
} |
/** |
* Free pages the pages that failed to change the caching state. If there is |
* any pages that have changed their caching state already put them to the |
* pool. |
*/ |
static void ttm_handle_caching_state_failure(struct list_head *pages, |
int ttm_flags, enum ttm_caching_state cstate, |
struct page **failed_pages, unsigned cpages) |
{ |
unsigned i; |
/* Failed pages have to be freed */ |
for (i = 0; i < cpages; ++i) { |
list_del(&failed_pages[i]->lru); |
__free_page(failed_pages[i]); |
} |
} |
/** |
* Allocate new pages with correct caching. |
* |
* This function is reentrant if caller updates count depending on number of |
* pages returned in pages array. |
*/ |
static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, |
int ttm_flags, enum ttm_caching_state cstate, unsigned count) |
{ |
struct page **caching_array; |
struct page *p; |
int r = 0; |
unsigned i, cpages; |
unsigned max_cpages = min(count, |
(unsigned)(PAGE_SIZE/sizeof(struct page *))); |
/* allocate array for page caching change */ |
caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); |
if (!caching_array) { |
pr_err("Unable to allocate table for new pages\n"); |
return -ENOMEM; |
} |
for (i = 0, cpages = 0; i < count; ++i) { |
p = alloc_page(gfp_flags); |
if (!p) { |
pr_err("Unable to get page %u\n", i); |
/* store already allocated pages in the pool after |
* setting the caching state */ |
if (cpages) { |
r = ttm_set_pages_caching(caching_array, |
cstate, cpages); |
if (r) |
ttm_handle_caching_state_failure(pages, |
ttm_flags, cstate, |
caching_array, cpages); |
} |
r = -ENOMEM; |
goto out; |
} |
#ifdef CONFIG_HIGHMEM |
/* gfp flags of highmem page should never be dma32 so we |
* we should be fine in such case |
*/ |
if (!PageHighMem(p)) |
#endif |
{ |
caching_array[cpages++] = p; |
if (cpages == max_cpages) { |
r = ttm_set_pages_caching(caching_array, |
cstate, cpages); |
if (r) { |
ttm_handle_caching_state_failure(pages, |
ttm_flags, cstate, |
caching_array, cpages); |
goto out; |
} |
cpages = 0; |
} |
} |
list_add(&p->lru, pages); |
} |
if (cpages) { |
r = ttm_set_pages_caching(caching_array, cstate, cpages); |
if (r) |
ttm_handle_caching_state_failure(pages, |
ttm_flags, cstate, |
caching_array, cpages); |
} |
out: |
kfree(caching_array); |
return r; |
} |
/** |
* Fill the given pool if there aren't enough pages and the requested number of |
* pages is small. |
*/ |
static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, |
int ttm_flags, enum ttm_caching_state cstate, unsigned count, |
unsigned long *irq_flags) |
{ |
struct page *p; |
int r; |
unsigned cpages = 0; |
/** |
* Only allow one pool fill operation at a time. |
* If pool doesn't have enough pages for the allocation new pages are |
* allocated from outside of pool. |
*/ |
if (pool->fill_lock) |
return; |
pool->fill_lock = true; |
/* If allocation request is small and there are not enough |
* pages in a pool we fill the pool up first. */ |
if (count < _manager->options.small |
&& count > pool->npages) { |
struct list_head new_pages; |
unsigned alloc_size = _manager->options.alloc_size; |
/** |
* Can't change page caching if in irqsave context. We have to |
* drop the pool->lock. |
*/ |
spin_unlock_irqrestore(&pool->lock, *irq_flags); |
INIT_LIST_HEAD(&new_pages); |
r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, |
cstate, alloc_size); |
spin_lock_irqsave(&pool->lock, *irq_flags); |
if (!r) { |
list_splice(&new_pages, &pool->list); |
++pool->nrefills; |
pool->npages += alloc_size; |
} else { |
pr_err("Failed to fill pool (%p)\n", pool); |
/* If we have any pages left put them to the pool. */ |
list_for_each_entry(p, &pool->list, lru) { |
++cpages; |
} |
list_splice(&new_pages, &pool->list); |
pool->npages += cpages; |
} |
} |
pool->fill_lock = false; |
} |
/** |
* Cut 'count' number of pages from the pool and put them on the return list. |
* |
* @return count of pages still required to fulfill the request. |
*/ |
static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, |
struct list_head *pages, |
int ttm_flags, |
enum ttm_caching_state cstate, |
unsigned count) |
{ |
unsigned long irq_flags; |
struct list_head *p; |
unsigned i; |
spin_lock_irqsave(&pool->lock, irq_flags); |
ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); |
if (count >= pool->npages) { |
/* take all pages from the pool */ |
list_splice_init(&pool->list, pages); |
count -= pool->npages; |
pool->npages = 0; |
goto out; |
} |
/* find the last pages to include for requested number of pages. Split |
* pool to begin and halve it to reduce search space. */ |
if (count <= pool->npages/2) { |
i = 0; |
list_for_each(p, &pool->list) { |
if (++i == count) |
break; |
} |
} else { |
i = pool->npages + 1; |
list_for_each_prev(p, &pool->list) { |
if (--i == count) |
break; |
} |
} |
/* Cut 'count' number of pages from the pool */ |
list_cut_position(pages, &pool->list, p); |
pool->npages -= count; |
count = 0; |
out: |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
return count; |
} |
#endif |
/* Put all pages in pages list to correct pool to wait for reuse */ |
static void ttm_put_pages(struct page **pages, unsigned npages, int flags, |
enum ttm_caching_state cstate) |
{ |
unsigned long irq_flags; |
// struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
unsigned i; |
for (i = 0; i < npages; i++) { |
if (pages[i]) { |
// if (page_count(pages[i]) != 1) |
// pr_err("Erroneous page count. Leaking pages.\n"); |
FreePage(pages[i]); |
pages[i] = NULL; |
} |
} |
return; |
#if 0 |
if (pool == NULL) { |
if (1) { |
/* No pool for this memory type so free the pages */ |
for (i = 0; i < npages; i++) { |
if (pages[i]) { |
if (page_count(pages[i]) != 1) |
pr_err("Erroneous page count. Leaking pages.\n"); |
__free_page(pages[i]); |
pages[i] = NULL; |
} |
703,32 → 184,8 |
return; |
} |
spin_lock_irqsave(&pool->lock, irq_flags); |
for (i = 0; i < npages; i++) { |
if (pages[i]) { |
if (page_count(pages[i]) != 1) |
pr_err("Erroneous page count. Leaking pages.\n"); |
list_add_tail(&pages[i]->lru, &pool->list); |
pages[i] = NULL; |
pool->npages++; |
} |
} |
/* Check that we don't go over the pool limit */ |
npages = 0; |
if (pool->npages > _manager->options.max_size) { |
npages = pool->npages - _manager->options.max_size; |
/* free at least NUM_PAGES_TO_ALLOC number of pages |
* to reduce calls to set_memory_wb */ |
if (npages < NUM_PAGES_TO_ALLOC) |
npages = NUM_PAGES_TO_ALLOC; |
} |
spin_unlock_irqrestore(&pool->lock, irq_flags); |
if (npages) |
ttm_page_pool_free(pool, npages); |
#endif |
} |
/* |
* On success pages list will hold count number of correctly |
* cached pages. |
736,44 → 193,21 |
static int ttm_get_pages(struct page **pages, unsigned npages, int flags, |
enum ttm_caching_state cstate) |
{ |
// struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); |
struct list_head plist; |
struct page *p = NULL; |
// gfp_t gfp_flags = GFP_USER; |
gfp_t gfp_flags = 0; |
unsigned count; |
int r; |
for (r = 0; r < npages; ++r) { |
p = AllocPage(); |
if (!p) { |
pr_err("Unable to allocate page\n"); |
return -ENOMEM; |
} |
pages[r] = p; |
} |
return 0; |
#if 0 |
/* set zero flag for page allocation if required */ |
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) |
gfp_flags |= __GFP_ZERO; |
/* No pool for cached pages */ |
if (pool == NULL) { |
if (flags & TTM_PAGE_FLAG_DMA32) |
gfp_flags |= GFP_DMA32; |
else |
gfp_flags |= GFP_HIGHUSER; |
if (1) { |
for (r = 0; r < npages; ++r) { |
p = alloc_page(gfp_flags); |
if (!p) { |
pr_err("Unable to allocate page\n"); |
return -ENOMEM; |
} |
782,52 → 216,12 |
return 0; |
} |
/* combine zero flag to pool flags */ |
gfp_flags |= pool->gfp_flags; |
/* First we take pages from the pool */ |
INIT_LIST_HEAD(&plist); |
npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); |
count = 0; |
list_for_each_entry(p, &plist, lru) { |
pages[count++] = p; |
} |
/* clear the pages coming from the pool if requested */ |
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { |
list_for_each_entry(p, &plist, lru) { |
if (PageHighMem(p)) |
clear_highpage(p); |
else |
clear_page(page_address(p)); |
} |
} |
/* If pool didn't have enough pages allocate new one. */ |
if (npages > 0) { |
/* ttm_alloc_new_pages doesn't reference pool so we can run |
* multiple requests in parallel. |
**/ |
INIT_LIST_HEAD(&plist); |
r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); |
list_for_each_entry(p, &plist, lru) { |
pages[count++] = p; |
} |
if (r) { |
/* If there is any pages in the list put them back to |
* the pool. */ |
pr_err("Failed to allocate extra pages for large request\n"); |
ttm_put_pages(pages, count, flags, cstate); |
return r; |
} |
} |
#endif |
return 0; |
} |
#if 0 |
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, |
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, |
char *name) |
{ |
spin_lock_init(&pool->lock); |
844,34 → 238,12 |
WARN_ON(_manager); |
pr_info("Initializing pool allocator\n"); |
_manager = kzalloc(sizeof(*_manager), GFP_KERNEL); |
ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); |
ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); |
ttm_page_pool_init_locked(&_manager->wc_pool_dma32, |
GFP_USER | GFP_DMA32, "wc dma"); |
ttm_page_pool_init_locked(&_manager->uc_pool_dma32, |
GFP_USER | GFP_DMA32, "uc dma"); |
_manager->options.max_size = max_pages; |
_manager->options.small = SMALL_ALLOCATION; |
_manager->options.alloc_size = NUM_PAGES_TO_ALLOC; |
ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, |
&glob->kobj, "pool"); |
if (unlikely(ret != 0)) { |
kobject_put(&_manager->kobj); |
_manager = NULL; |
return ret; |
} |
ttm_pool_mm_shrink_init(_manager); |
return 0; |
} |
879,18 → 251,9 |
{ |
int i; |
pr_info("Finalizing pool allocator\n"); |
ttm_pool_mm_shrink_fini(_manager); |
for (i = 0; i < NUM_POOLS; ++i) |
ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); |
kobject_put(&_manager->kobj); |
_manager = NULL; |
} |
#endif |
int ttm_pool_populate(struct ttm_tt *ttm) |
{ |
struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; |
908,9 → 271,7 |
ttm_pool_unpopulate(ttm); |
return -ENOMEM; |
} |
} |
ttm->state = tt_unbound; |
return 0; |
} |
922,8 → 283,6 |
for (i = 0; i < ttm->num_pages; ++i) { |
if (ttm->pages[i]) { |
ttm_mem_global_free_page(ttm->glob->mem_glob, |
ttm->pages[i]); |
ttm_put_pages(&ttm->pages[i], 1, |
ttm->page_flags, |
ttm->caching_state); |
/drivers/video/drm/ttm/ttm_tt.c |
---|
57,9 → 57,12 |
static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) |
{ |
ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); |
ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, |
sizeof(*ttm->dma_address)); |
ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, |
sizeof(*ttm->ttm.pages) + |
sizeof(*ttm->dma_address) + |
sizeof(*ttm->cpu_address)); |
ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); |
ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages); |
} |
#ifdef CONFIG_X86 |
118,8 → 121,8 |
return 0; |
} |
// if (ttm->caching_state == tt_cached) |
// drm_clflush_pages(ttm->pages, ttm->num_pages); |
if (ttm->caching_state == tt_cached) |
drm_clflush_pages(ttm->pages, ttm->num_pages); |
for (i = 0; i < ttm->num_pages; ++i) { |
cur_page = ttm->pages[i]; |
172,8 → 175,8 |
ttm_tt_unbind(ttm); |
} |
// if (ttm->state == tt_unbound) |
// ttm_tt_unpopulate(ttm); |
if (ttm->state == tt_unbound) |
ttm_tt_unpopulate(ttm); |
// if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && |
// ttm->swap_storage) |
230,7 → 233,7 |
INIT_LIST_HEAD(&ttm_dma->pages_list); |
ttm_dma_tt_alloc_page_directory(ttm_dma); |
if (!ttm->pages || !ttm_dma->dma_address) { |
if (!ttm->pages) { |
ttm_tt_destroy(ttm); |
printf("Failed allocating page table\n"); |
return -ENOMEM; |
245,7 → 248,7 |
drm_free_large(ttm->pages); |
ttm->pages = NULL; |
drm_free_large(ttm_dma->dma_address); |
ttm_dma->cpu_address = NULL; |
ttm_dma->dma_address = NULL; |
} |
EXPORT_SYMBOL(ttm_dma_tt_fini); |
261,8 → 264,6 |
} |
} |
#if 0 |
int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) |
{ |
int ret = 0; |
286,9 → 287,8 |
return 0; |
} |
EXPORT_SYMBOL(ttm_tt_bind); |
#endif |
/* |
#if 0 |
int ttm_tt_swapin(struct ttm_tt *ttm) |
{ |
struct address_space *swap_space; |
380,7 → 380,20 |
return ret; |
} |
#endif |
*/ |
static void ttm_tt_clear_mapping(struct ttm_tt *ttm) |
{ |
pgoff_t i; |
struct page **page = ttm->pages; |
} |
void ttm_tt_unpopulate(struct ttm_tt *ttm) |
{ |
if (ttm->state == tt_unpopulated) |
return; |
ttm_tt_clear_mapping(ttm); |
ttm->bdev->driver->ttm_tt_unpopulate(ttm); |
} |