Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 5077 → Rev 5078

/drivers/video/drm/ttm/ttm_bo.c
39,92 → 39,35
#include <linux/mm.h>
#include <linux/module.h>
 
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
 
#define TTM_ASSERT_LOCKED(param)
#define TTM_DEBUG(fmt, arg...)
#define TTM_BO_HASH_ORDER 13
 
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
 
int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
 
static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
{
int i;
 
mutex_lock(&man->io_reserve_mutex);
for (i = 0; i <= TTM_PL_PRIV5; i++)
if (flags & (1 << i)) {
*mem_type = i;
return 0;
}
 
void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
{
if (likely(man->io_reserve_fastpath))
return;
 
mutex_unlock(&man->io_reserve_mutex);
return -EINVAL;
}
 
 
#if 0
static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
{
struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 
pr_err(" has_type: %d\n", man->has_type);
pr_err(" use_type: %d\n", man->use_type);
pr_err(" flags: 0x%08X\n", man->flags);
pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset);
pr_err(" size: %llu\n", man->size);
pr_err(" available_caching: 0x%08X\n", man->available_caching);
pr_err(" default_caching: 0x%08X\n", man->default_caching);
if (mem_type != TTM_PL_SYSTEM)
(*man->func->debug)(man, TTM_PFX);
}
 
static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
struct ttm_placement *placement)
{
int i, ret, mem_type;
 
pr_err("No space for %p (%lu pages, %luK, %luM)\n",
bo, bo->mem.num_pages, bo->mem.size >> 10,
bo->mem.size >> 20);
for (i = 0; i < placement->num_placement; i++) {
ret = ttm_mem_type_from_flags(placement->placement[i],
&mem_type);
if (ret)
return;
pr_err(" placement[%d]=0x%08X (%d)\n",
i, placement->placement[i], mem_type);
ttm_mem_type_debug(bo->bdev, mem_type);
}
}
 
static ssize_t ttm_bo_global_show(struct kobject *kobj,
struct attribute *attr,
char *buffer)
{
struct ttm_bo_global *glob =
container_of(kobj, struct ttm_bo_global, kobj);
 
return snprintf(buffer, PAGE_SIZE, "%lu\n",
(unsigned long) atomic_read(&glob->bo_count));
}
 
static struct attribute *ttm_bo_global_attrs[] = {
&ttm_bo_count,
NULL
};
 
static const struct sysfs_ops ttm_bo_global_ops = {
.show = &ttm_bo_global_show
};
 
static struct kobj_type ttm_bo_glob_kobj_type = {
.release = &ttm_bo_global_kobj_release,
.sysfs_ops = &ttm_bo_global_ops,
.default_attrs = ttm_bo_global_attrs
};
#endif
 
 
static inline uint32_t ttm_bo_type_flags(unsigned type)
{
return 1 << (type);
148,12 → 91,14
if (bo->ttm)
ttm_tt_destroy(bo->ttm);
atomic_dec(&bo->glob->bo_count);
if (bo->resv == &bo->ttm_resv)
reservation_object_fini(&bo->ttm_resv);
mutex_destroy(&bo->wu_mutex);
if (bo->destroy)
bo->destroy(bo);
else {
kfree(bo);
}
ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
}
 
void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
161,7 → 106,7
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_type_manager *man;
 
// BUG_ON(!ttm_bo_is_reserved(bo));
lockdep_assert_held(&bo->resv->lock.base);
 
if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 
268,7 → 213,6
return ret;
}
 
#if 0
static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
struct ttm_mem_reg *mem,
bool evict, bool interruptible,
348,9 → 292,11
 
moved:
if (bo->evicted) {
if (bdev->driver->invalidate_caches) {
ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
if (ret)
pr_err("Can not flush read caches\n");
}
bo->evicted = false;
}
 
407,7 → 353,7
int ret;
 
spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
ret = __ttm_bo_reserve(bo, false, true, false, NULL);
 
spin_lock(&bdev->fence_lock);
(void) ttm_bo_wait(bo, false, false, true);
438,7 → 384,7
ttm_bo_add_to_lru(bo);
}
 
ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
}
 
kref_get(&bo->list_kref);
449,8 → 395,8
driver->sync_obj_flush(sync_obj);
driver->sync_obj_unref(&sync_obj);
}
schedule_delayed_work(&bdev->wq,
((HZ / 100) < 1) ? 1 : HZ / 100);
// schedule_delayed_work(&bdev->wq,
// ((HZ / 100) < 1) ? 1 : HZ / 100);
}
 
/**
489,7 → 435,7
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
 
ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
spin_unlock(&glob->lru_lock);
 
ret = driver->sync_obj_wait(sync_obj, false, interruptible);
509,7 → 455,7
return ret;
 
spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
ret = __ttm_bo_reserve(bo, false, true, false, NULL);
 
/*
* We raced, and lost, someone else holds the reservation now,
527,7 → 473,7
spin_unlock(&bdev->fence_lock);
 
if (ret || unlikely(list_empty(&bo->ddestroy))) {
ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
spin_unlock(&glob->lru_lock);
return ret;
}
572,11 → 518,11
kref_get(&nentry->list_kref);
}
 
ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);
ret = __ttm_bo_reserve(entry, false, true, false, NULL);
if (remove_all && ret) {
spin_unlock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(entry, false, false,
false, 0);
ret = __ttm_bo_reserve(entry, false, false,
false, NULL);
spin_lock(&glob->lru_lock);
}
 
615,7 → 561,6
((HZ / 100) < 1) ? 1 : HZ / 100);
}
}
#endif
 
static void ttm_bo_release(struct kref *kref)
{
626,10 → 571,10
 
drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
ttm_mem_io_lock(man, false);
// ttm_mem_io_free_vm(bo);
ttm_mem_io_free_vm(bo);
ttm_mem_io_unlock(man);
// ttm_bo_cleanup_refs_or_queue(bo);
// kref_put(&bo->list_kref, ttm_bo_release_list);
ttm_bo_cleanup_refs_or_queue(bo);
kref_put(&bo->list_kref, ttm_bo_release_list);
}
 
void ttm_bo_unref(struct ttm_buffer_object **p_bo)
641,121 → 586,6
}
EXPORT_SYMBOL(ttm_bo_unref);
 
#if 0
int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
{
return cancel_delayed_work_sync(&bdev->wq);
}
EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 
void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
{
if (resched)
schedule_delayed_work(&bdev->wq,
((HZ / 100) < 1) ? 1 : HZ / 100);
}
EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 
static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
bool no_wait_gpu)
{
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_reg evict_mem;
struct ttm_placement placement;
int ret = 0;
 
spin_lock(&bdev->fence_lock);
ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
spin_unlock(&bdev->fence_lock);
 
if (unlikely(ret != 0)) {
if (ret != -ERESTARTSYS) {
pr_err("Failed to expire sync object before buffer eviction\n");
}
goto out;
}
 
// BUG_ON(!ttm_bo_is_reserved(bo));
 
evict_mem = bo->mem;
evict_mem.mm_node = NULL;
evict_mem.bus.io_reserved_vm = false;
evict_mem.bus.io_reserved_count = 0;
 
placement.fpfn = 0;
placement.lpfn = 0;
placement.num_placement = 0;
placement.num_busy_placement = 0;
bdev->driver->evict_flags(bo, &placement);
ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
no_wait_gpu);
if (ret) {
if (ret != -ERESTARTSYS) {
pr_err("Failed to find memory space for buffer 0x%p eviction\n",
bo);
ttm_bo_mem_space_debug(bo, &placement);
}
goto out;
}
 
ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
no_wait_gpu);
if (ret) {
if (ret != -ERESTARTSYS)
pr_err("Buffer eviction failed\n");
ttm_bo_mem_put(bo, &evict_mem);
goto out;
}
bo->evicted = true;
out:
return ret;
}
 
static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
uint32_t mem_type,
bool interruptible,
bool no_wait_gpu)
{
struct ttm_bo_global *glob = bdev->glob;
struct ttm_mem_type_manager *man = &bdev->man[mem_type];
struct ttm_buffer_object *bo;
int ret = -EBUSY, put_count;
 
spin_lock(&glob->lru_lock);
list_for_each_entry(bo, &man->lru, lru) {
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
if (!ret)
break;
}
 
if (ret) {
spin_unlock(&glob->lru_lock);
return ret;
}
 
kref_get(&bo->list_kref);
 
if (!list_empty(&bo->ddestroy)) {
ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,
no_wait_gpu);
kref_put(&bo->list_kref, ttm_bo_release_list);
return ret;
}
 
put_count = ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock);
 
BUG_ON(ret != 0);
 
ttm_bo_list_ref_sub(bo, put_count, true);
 
ret = ttm_bo_evict(bo, interruptible, no_wait_gpu);
ttm_bo_unreserve(bo);
 
kref_put(&bo->list_kref, ttm_bo_release_list);
return ret;
}
 
void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
{
struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
781,15 → 611,15
int ret;
 
do {
ret = (*man->func->get_node)(man, bo, placement, mem);
ret = (*man->func->get_node)(man, bo, placement, 0, mem);
if (unlikely(ret != 0))
return ret;
if (mem->mm_node)
break;
ret = ttm_mem_evict_first(bdev, mem_type,
interruptible, no_wait_gpu);
if (unlikely(ret != 0))
return ret;
// ret = ttm_mem_evict_first(bdev, mem_type,
// interruptible, no_wait_gpu);
// if (unlikely(ret != 0))
// return ret;
} while (1);
if (mem->mm_node == NULL)
return -ENOMEM;
894,7 → 724,8
 
if (man->has_type && man->use_type) {
type_found = true;
ret = (*man->func->get_node)(man, bo, placement, mem);
ret = (*man->func->get_node)(man, bo, placement,
cur_flags, mem);
if (unlikely(ret))
return ret;
}
934,7 → 765,6
ttm_flag_masked(&cur_flags, placement->busy_placement[i],
~TTM_PL_MASK_MEMTYPE);
 
 
if (mem_type == TTM_PL_SYSTEM) {
mem->mem_type = mem_type;
mem->placement = cur_flags;
965,7 → 795,7
struct ttm_mem_reg mem;
struct ttm_bo_device *bdev = bo->bdev;
 
// BUG_ON(!ttm_bo_is_reserved(bo));
lockdep_assert_held(&bo->resv->lock.base);
 
/*
* FIXME: It's possible to pipeline buffer moves.
996,7 → 826,6
ttm_bo_mem_put(bo, &mem);
return ret;
}
#endif
 
static bool ttm_bo_mem_compat(struct ttm_placement *placement,
struct ttm_mem_reg *mem,
1034,7 → 863,7
int ret;
uint32_t new_flags;
 
// BUG_ON(!ttm_bo_is_reserved(bo));
lockdep_assert_held(&bo->resv->lock.base);
/* Check that range is valid */
if (placement->lpfn || placement->fpfn)
if (placement->fpfn > placement->lpfn ||
1044,8 → 873,8
* Check whether we need to move buffer.
*/
if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
// ret = ttm_bo_move_buffer(bo, placement, interruptible,
// no_wait_gpu);
ret = ttm_bo_move_buffer(bo, placement, interruptible,
no_wait_gpu);
if (ret)
return ret;
} else {
1091,19 → 920,8
{
int ret = 0;
unsigned long num_pages;
struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
bool locked;
 
// ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
if (ret) {
pr_err("Out of kernel memory\n");
if (destroy)
(*destroy)(bo);
else
kfree(bo);
return -ENOMEM;
}
 
num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (num_pages == 0) {
pr_err("Illegal buffer object size\n");
1111,7 → 929,6
(*destroy)(bo);
else
kfree(bo);
// ttm_mem_global_free(mem_glob, acc_size);
return -EINVAL;
}
bo->destroy = destroy;
1141,7 → 958,7
bo->acc_size = acc_size;
bo->sg = sg;
bo->resv = &bo->ttm_resv;
// reservation_object_init(bo->resv);
reservation_object_init(bo->resv);
atomic_inc(&bo->glob->bo_count);
drm_vma_node_reset(&bo->vma_node);
 
1151,19 → 968,23
* For ttm_bo_type_device buffers, allocate
* address space from the device.
*/
// if (likely(!ret) &&
// (bo->type == ttm_bo_type_device ||
// bo->type == ttm_bo_type_sg))
// ret = ttm_bo_setup_vm(bo);
if (likely(!ret) &&
(bo->type == ttm_bo_type_device ||
bo->type == ttm_bo_type_sg))
ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
bo->mem.num_pages);
 
// if (likely(!ret))
// ret = ttm_bo_validate(bo, placement, interruptible, false);
locked = ww_mutex_trylock(&bo->resv->lock);
WARN_ON(!locked);
 
// ttm_bo_unreserve(bo);
if (likely(!ret))
ret = ttm_bo_validate(bo, placement, interruptible, false);
 
// if (unlikely(ret))
// ttm_bo_unref(&bo);
ttm_bo_unreserve(bo);
 
if (unlikely(ret))
ttm_bo_unref(&bo);
 
return ret;
}
EXPORT_SYMBOL(ttm_bo_init);
1197,53 → 1018,6
}
EXPORT_SYMBOL(ttm_bo_dma_acc_size);
 
int ttm_bo_create(struct ttm_bo_device *bdev,
unsigned long size,
enum ttm_bo_type type,
struct ttm_placement *placement,
uint32_t page_alignment,
bool interruptible,
struct file *persistent_swap_storage,
struct ttm_buffer_object **p_bo)
{
struct ttm_buffer_object *bo;
size_t acc_size;
int ret;
 
bo = kzalloc(sizeof(*bo), GFP_KERNEL);
if (unlikely(bo == NULL))
return -ENOMEM;
 
acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
interruptible, persistent_swap_storage, acc_size,
NULL, NULL);
if (likely(ret == 0))
*p_bo = bo;
 
return ret;
}
EXPORT_SYMBOL(ttm_bo_create);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
unsigned long p_size)
{
1250,8 → 1024,6
int ret = -EINVAL;
struct ttm_mem_type_manager *man;
 
ENTER();
 
BUG_ON(type >= TTM_NUM_MEM_TYPES);
man = &bdev->man[type];
BUG_ON(man->has_type);
1277,12 → 1049,9
 
INIT_LIST_HEAD(&man->lru);
 
LEAVE();
 
return 0;
}
 
 
EXPORT_SYMBOL(ttm_bo_init_mm);
void ttm_bo_global_release(struct drm_global_reference *ref)
{
struct ttm_bo_global *glob = ref->object;
1297,12 → 1066,10
struct ttm_bo_global *glob = ref->object;
int ret;
 
ENTER();
 
mutex_init(&glob->device_list_mutex);
spin_lock_init(&glob->lru_lock);
glob->mem_glob = bo_ref->mem_glob;
glob->dummy_read_page = AllocPage();
glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
 
if (unlikely(glob->dummy_read_page == NULL)) {
ret = -ENOMEM;
1314,8 → 1081,6
 
atomic_set(&glob->bo_count, 0);
 
LEAVE();
 
return 0;
 
out_no_drp:
1324,17 → 1089,15
}
EXPORT_SYMBOL(ttm_bo_global_init);
 
 
int ttm_bo_device_init(struct ttm_bo_device *bdev,
struct ttm_bo_global *glob,
struct ttm_bo_driver *driver,
struct address_space *mapping,
uint64_t file_page_offset,
bool need_dma32)
{
int ret = -EINVAL;
 
ENTER();
 
bdev->driver = driver;
 
memset(bdev->man, 0, sizeof(bdev->man));
1349,9 → 1112,9
 
drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
0x10000000);
// INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
INIT_LIST_HEAD(&bdev->ddestroy);
bdev->dev_mapping = NULL;
bdev->dev_mapping = mapping;
bdev->glob = glob;
bdev->need_dma32 = need_dma32;
bdev->val_seq = 0;
1360,8 → 1123,6
list_add_tail(&bdev->device_list, &glob->device_list);
mutex_unlock(&glob->device_list_mutex);
 
LEAVE();
 
return 0;
out_no_sys:
return ret;
1389,6 → 1150,28
return true;
}
 
void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
{
struct ttm_bo_device *bdev = bo->bdev;
 
drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
ttm_mem_io_free_vm(bo);
}
 
void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
{
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 
ttm_mem_io_lock(man, false);
ttm_bo_unmap_virtual_locked(bo);
ttm_mem_io_unlock(man);
}
 
 
EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 
 
int ttm_bo_wait(struct ttm_buffer_object *bo,
bool lazy, bool interruptible, bool no_wait)
{
1400,25 → 1183,47
if (likely(bo->sync_obj == NULL))
return 0;
 
return 0;
while (bo->sync_obj) {
 
if (driver->sync_obj_signaled(bo->sync_obj)) {
void *tmp_obj = bo->sync_obj;
bo->sync_obj = NULL;
clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&tmp_obj);
spin_lock(&bdev->fence_lock);
continue;
}
EXPORT_SYMBOL(ttm_bo_wait);
 
int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
{
struct ttm_bo_device *bdev = bo->bdev;
int ret = 0;
if (no_wait)
return -EBUSY;
 
/*
* Using ttm_bo_reserve makes sure the lru lists are updated.
*/
 
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
ret = driver->sync_obj_wait(sync_obj,
lazy, interruptible);
if (unlikely(ret != 0)) {
driver->sync_obj_unref(&sync_obj);
spin_lock(&bdev->fence_lock);
return ret;
}
EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
spin_lock(&bdev->fence_lock);
if (likely(bo->sync_obj == sync_obj)) {
void *tmp_obj = bo->sync_obj;
bo->sync_obj = NULL;
clear_bit(TTM_BO_PRIV_FLAG_MOVING,
&bo->priv_flags);
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&sync_obj);
driver->sync_obj_unref(&tmp_obj);
spin_lock(&bdev->fence_lock);
} else {
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&sync_obj);
spin_lock(&bdev->fence_lock);
}
}
return 0;
}
EXPORT_SYMBOL(ttm_bo_wait);
 
void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
{
atomic_dec(&bo->cpu_writers);
}
EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
/drivers/video/drm/ttm/ttm_bo_manager.c
50,11 → 50,13
static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man,
struct ttm_buffer_object *bo,
struct ttm_placement *placement,
uint32_t flags,
struct ttm_mem_reg *mem)
{
struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
struct drm_mm *mm = &rman->mm;
struct drm_mm_node *node = NULL;
enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT;
unsigned long lpfn;
int ret;
 
66,11 → 68,15
if (!node)
return -ENOMEM;
 
if (flags & TTM_PL_FLAG_TOPDOWN)
aflags = DRM_MM_CREATE_TOP;
 
spin_lock(&rman->lock);
ret = drm_mm_insert_node_in_range(mm, node, mem->num_pages,
mem->page_alignment,
ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages,
mem->page_alignment, 0,
placement->fpfn, lpfn,
DRM_MM_SEARCH_BEST);
DRM_MM_SEARCH_BEST,
aflags);
spin_unlock(&rman->lock);
 
if (unlikely(ret)) {
/drivers/video/drm/ttm/ttm_bo_util.c
27,17 → 27,25
/*
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
*/
#define iowrite32(v, addr) writel((v), (addr))
#define ioread32(addr) readl(addr)
 
#include <drm/ttm/ttm_bo_driver.h>
#include <drm/ttm/ttm_placement.h>
#include <drm/drm_vma_manager.h>
#include <linux/io.h>
#include <linux/highmem.h>
//#include <linux/io.h>
//#include <linux/highmem.h>
#include <linux/wait.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
//#include <linux/vmalloc.h>
#include <linux/module.h>
 
#define __pgprot(x) ((pgprot_t) { (x) } )
#define PAGE_KERNEL __pgprot(3)
 
void *vmap(struct page **pages, unsigned int count,
unsigned long flags, pgprot_t prot);
 
void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
{
ttm_bo_mem_put(bo, &bo->mem);
156,6 → 164,7
}
EXPORT_SYMBOL(ttm_mem_io_free);
 
#if 0
int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
{
struct ttm_mem_reg *mem = &bo->mem;
175,6 → 184,7
}
return 0;
}
#endif
 
void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
{
207,7 → 217,7
if (mem->placement & TTM_PL_FLAG_WC)
addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
else
addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size);
if (!addr) {
(void) ttm_mem_io_lock(man, false);
ttm_mem_io_free(bdev, mem);
258,27 → 268,14
 
src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 
#ifdef CONFIG_X86
dst = kmap_atomic_prot(d, prot);
#else
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
dst = vmap(&d, 1, 0, prot);
else
dst = kmap(d);
#endif
dst = (void*)MapIoMem((addr_t)d, 4096, PG_SW);
 
if (!dst)
return -ENOMEM;
 
memcpy_fromio(dst, src, PAGE_SIZE);
memcpy(dst, src, PAGE_SIZE);
 
#ifdef CONFIG_X86
kunmap_atomic(dst);
#else
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
vunmap(dst);
else
kunmap(d);
#endif
FreeKernelSpace(dst);
 
return 0;
}
294,27 → 291,15
return -ENOMEM;
 
dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
#ifdef CONFIG_X86
src = kmap_atomic_prot(s, prot);
#else
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
src = vmap(&s, 1, 0, prot);
else
src = kmap(s);
#endif
 
src = (void*)MapIoMem((addr_t)s, 4096, PG_SW);
 
if (!src)
return -ENOMEM;
 
memcpy_toio(dst, src, PAGE_SIZE);
memcpy(dst, src, PAGE_SIZE);
 
#ifdef CONFIG_X86
kunmap_atomic(src);
#else
if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
vunmap(src);
else
kunmap(s);
#endif
FreeKernelSpace(src);
 
return 0;
}
352,8 → 337,12
/*
* Don't move nonexistent data. Clear destination instead.
*/
if (old_iomap == NULL && ttm == NULL)
if (old_iomap == NULL &&
(ttm == NULL || (ttm->state == tt_unpopulated &&
!(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
memset(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
goto out2;
}
 
/*
* TTM might be null for moves within the same region.
483,29 → 472,6
 
pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
{
#if defined(__i386__) || defined(__x86_64__)
if (caching_flags & TTM_PL_FLAG_WC)
tmp = pgprot_writecombine(tmp);
else if (boot_cpu_data.x86 > 3)
tmp = pgprot_noncached(tmp);
 
#elif defined(__powerpc__)
if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
pgprot_val(tmp) |= _PAGE_NO_CACHE;
if (caching_flags & TTM_PL_FLAG_UNCACHED)
pgprot_val(tmp) |= _PAGE_GUARDED;
}
#endif
#if defined(__ia64__)
if (caching_flags & TTM_PL_FLAG_WC)
tmp = pgprot_writecombine(tmp);
else
tmp = pgprot_noncached(tmp);
#endif
#if defined(__sparc__) || defined(__mips__)
if (!(caching_flags & TTM_PL_FLAG_CACHED))
tmp = pgprot_noncached(tmp);
#endif
return tmp;
}
EXPORT_SYMBOL(ttm_io_prot);
526,7 → 492,7
map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
size);
else
map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset,
size);
}
return (!map->virtual) ? -ENOMEM : 0;
557,7 → 523,7
 
map->bo_kmap_type = ttm_bo_map_kmap;
map->page = ttm->pages[start_page];
map->virtual = kmap(map->page);
map->virtual = (void*)MapIoMem(page_to_phys(map->page), 4096, PG_SW);
} else {
/*
* We need to use vmap to get the desired page protection
621,10 → 587,8
iounmap(map->virtual);
break;
case ttm_bo_map_vmap:
vunmap(map->virtual);
break;
case ttm_bo_map_kmap:
kunmap(map->page);
FreeKernelSpace(map->virtual);
break;
case ttm_bo_map_premapped:
break;
713,3 → 677,25
return 0;
}
EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 
 
void *vmap(struct page **pages, unsigned int count,
unsigned long flags, pgprot_t prot)
{
void *vaddr;
char *tmp;
int i;
 
vaddr = AllocKernelSpace(count << 12);
if(vaddr == NULL)
return NULL;
 
for(i = 0, tmp = vaddr; i < count; i++)
{
MapPage(tmp, page_to_phys(pages[i]), PG_SW);
tmp+= 4096;
};
 
return vaddr;
};
 
/drivers/video/drm/ttm/ttm_execbuf_util.c
24,7 → 24,6
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
struct ww_acquire_ctx{};
 
#include <drm/ttm/ttm_execbuf_util.h>
#include <drm/ttm/ttm_bo_driver.h>
33,8 → 32,9
#include <linux/sched.h>
#include <linux/module.h>
 
static void ttm_eu_backoff_reservation_locked(struct list_head *list,
struct ww_acquire_ctx *ticket)
DEFINE_WW_CLASS(reservation_ww_class);
 
static void ttm_eu_backoff_reservation_locked(struct list_head *list)
{
struct ttm_validate_buffer *entry;
 
48,7 → 48,7
ttm_bo_add_to_lru(bo);
entry->removed = false;
}
// ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
}
}
 
94,8 → 94,9
entry = list_first_entry(list, struct ttm_validate_buffer, head);
glob = entry->bo->glob;
spin_lock(&glob->lru_lock);
ttm_eu_backoff_reservation_locked(list, ticket);
// ww_acquire_fini(ticket);
ttm_eu_backoff_reservation_locked(list);
if (ticket)
ww_acquire_fini(ticket);
spin_unlock(&glob->lru_lock);
}
EXPORT_SYMBOL(ttm_eu_backoff_reservation);
131,7 → 132,8
entry = list_first_entry(list, struct ttm_validate_buffer, head);
glob = entry->bo->glob;
 
// ww_acquire_init(ticket, &reservation_ww_class);
if (ticket)
ww_acquire_init(ticket, &reservation_ww_class);
retry:
list_for_each_entry(entry, list, head) {
struct ttm_buffer_object *bo = entry->bo;
140,20 → 142,21
if (entry->reserved)
continue;
 
ret = __ttm_bo_reserve(bo, true, (ticket == NULL), true,
ticket);
 
ret = ttm_bo_reserve_nolru(bo, true, false, true, ticket);
 
if (ret == -EDEADLK) {
/* uh oh, we lost out, drop every reservation and try
* to only reserve this buffer, then start over if
* this succeeds.
*/
BUG_ON(ticket == NULL);
spin_lock(&glob->lru_lock);
ttm_eu_backoff_reservation_locked(list, ticket);
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
// ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
// ticket);
ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
ticket);
if (unlikely(ret != 0)) {
if (ret == -EINTR)
ret = -ERESTARTSYS;
176,7 → 179,8
}
}
 
// ww_acquire_done(ticket);
if (ticket)
ww_acquire_done(ticket);
spin_lock(&glob->lru_lock);
ttm_eu_del_from_lru_locked(list);
spin_unlock(&glob->lru_lock);
185,12 → 189,14
 
err:
spin_lock(&glob->lru_lock);
ttm_eu_backoff_reservation_locked(list, ticket);
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
err_fini:
// ww_acquire_done(ticket);
// ww_acquire_fini(ticket);
if (ticket) {
ww_acquire_done(ticket);
ww_acquire_fini(ticket);
}
return ret;
}
EXPORT_SYMBOL(ttm_eu_reserve_buffers);
220,12 → 226,13
entry->old_sync_obj = bo->sync_obj;
bo->sync_obj = driver->sync_obj_ref(sync_obj);
ttm_bo_add_to_lru(bo);
// ww_mutex_unlock(&bo->resv->lock);
__ttm_bo_unreserve(bo);
entry->reserved = false;
}
spin_unlock(&bdev->fence_lock);
spin_unlock(&glob->lru_lock);
// ww_acquire_fini(ticket);
if (ticket)
ww_acquire_fini(ticket);
 
list_for_each_entry(entry, list, head) {
if (entry->old_sync_obj)
/drivers/video/drm/ttm/ttm_lock.c
0,0 → 1,286
/**************************************************************************
*
* Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
* All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
/*
* Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
*/
 
#include <linux/mutex.h>
#include <drm/ttm/ttm_lock.h>
#include <drm/ttm/ttm_module.h>
//#include <linux/atomic.h>
#include <linux/errno.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <linux/module.h>
 
#define TTM_WRITE_LOCK_PENDING (1 << 0)
#define TTM_VT_LOCK_PENDING (1 << 1)
#define TTM_SUSPEND_LOCK_PENDING (1 << 2)
#define TTM_VT_LOCK (1 << 3)
#define TTM_SUSPEND_LOCK (1 << 4)
 
void ttm_lock_init(struct ttm_lock *lock)
{
spin_lock_init(&lock->lock);
init_waitqueue_head(&lock->queue);
lock->rw = 0;
lock->flags = 0;
}
EXPORT_SYMBOL(ttm_lock_init);
 
void ttm_read_unlock(struct ttm_lock *lock)
{
spin_lock(&lock->lock);
if (--lock->rw == 0)
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
}
EXPORT_SYMBOL(ttm_read_unlock);
 
static bool __ttm_read_lock(struct ttm_lock *lock)
{
bool locked = false;
 
spin_lock(&lock->lock);
if (lock->rw >= 0 && lock->flags == 0) {
++lock->rw;
locked = true;
}
spin_unlock(&lock->lock);
return locked;
}
 
int ttm_read_lock(struct ttm_lock *lock, bool interruptible)
{
int ret = 0;
 
if (interruptible)
ret = wait_event_interruptible(lock->queue,
__ttm_read_lock(lock));
else
wait_event(lock->queue, __ttm_read_lock(lock));
return ret;
}
EXPORT_SYMBOL(ttm_read_lock);
 
static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked)
{
bool block = true;
 
*locked = false;
 
spin_lock(&lock->lock);
if (lock->rw >= 0 && lock->flags == 0) {
++lock->rw;
block = false;
*locked = true;
} else if (lock->flags == 0) {
block = false;
}
spin_unlock(&lock->lock);
 
return !block;
}
 
int ttm_read_trylock(struct ttm_lock *lock, bool interruptible)
{
int ret = 0;
bool locked;
 
if (interruptible)
ret = wait_event_interruptible
(lock->queue, __ttm_read_trylock(lock, &locked));
else
wait_event(lock->queue, __ttm_read_trylock(lock, &locked));
 
if (unlikely(ret != 0)) {
BUG_ON(locked);
return ret;
}
 
return (locked) ? 0 : -EBUSY;
}
 
void ttm_write_unlock(struct ttm_lock *lock)
{
spin_lock(&lock->lock);
lock->rw = 0;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
}
EXPORT_SYMBOL(ttm_write_unlock);
 
static bool __ttm_write_lock(struct ttm_lock *lock)
{
bool locked = false;
 
spin_lock(&lock->lock);
if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) {
lock->rw = -1;
lock->flags &= ~TTM_WRITE_LOCK_PENDING;
locked = true;
} else {
lock->flags |= TTM_WRITE_LOCK_PENDING;
}
spin_unlock(&lock->lock);
return locked;
}
 
int ttm_write_lock(struct ttm_lock *lock, bool interruptible)
{
int ret = 0;
 
if (interruptible) {
ret = wait_event_interruptible(lock->queue,
__ttm_write_lock(lock));
if (unlikely(ret != 0)) {
spin_lock(&lock->lock);
lock->flags &= ~TTM_WRITE_LOCK_PENDING;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
}
} else
wait_event(lock->queue, __ttm_read_lock(lock));
 
return ret;
}
EXPORT_SYMBOL(ttm_write_lock);
 
static int __ttm_vt_unlock(struct ttm_lock *lock)
{
int ret = 0;
 
spin_lock(&lock->lock);
if (unlikely(!(lock->flags & TTM_VT_LOCK)))
ret = -EINVAL;
lock->flags &= ~TTM_VT_LOCK;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
 
return ret;
}
 
static void ttm_vt_lock_remove(struct ttm_base_object **p_base)
{
struct ttm_base_object *base = *p_base;
struct ttm_lock *lock = container_of(base, struct ttm_lock, base);
int ret;
 
*p_base = NULL;
ret = __ttm_vt_unlock(lock);
BUG_ON(ret != 0);
}
 
static bool __ttm_vt_lock(struct ttm_lock *lock)
{
bool locked = false;
 
spin_lock(&lock->lock);
if (lock->rw == 0) {
lock->flags &= ~TTM_VT_LOCK_PENDING;
lock->flags |= TTM_VT_LOCK;
locked = true;
} else {
lock->flags |= TTM_VT_LOCK_PENDING;
}
spin_unlock(&lock->lock);
return locked;
}
 
int ttm_vt_lock(struct ttm_lock *lock,
bool interruptible,
struct ttm_object_file *tfile)
{
int ret = 0;
 
if (interruptible) {
ret = wait_event_interruptible(lock->queue,
__ttm_vt_lock(lock));
if (unlikely(ret != 0)) {
spin_lock(&lock->lock);
lock->flags &= ~TTM_VT_LOCK_PENDING;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
return ret;
}
} else
wait_event(lock->queue, __ttm_vt_lock(lock));
 
/*
* Add a base-object, the destructor of which will
* make sure the lock is released if the client dies
* while holding it.
*/
 
ret = ttm_base_object_init(tfile, &lock->base, false,
ttm_lock_type, &ttm_vt_lock_remove, NULL);
if (ret)
(void)__ttm_vt_unlock(lock);
else
lock->vt_holder = tfile;
 
return ret;
}
EXPORT_SYMBOL(ttm_vt_lock);
 
int ttm_vt_unlock(struct ttm_lock *lock)
{
return ttm_ref_object_base_unref(lock->vt_holder,
lock->base.hash.key, TTM_REF_USAGE);
}
EXPORT_SYMBOL(ttm_vt_unlock);
 
void ttm_suspend_unlock(struct ttm_lock *lock)
{
spin_lock(&lock->lock);
lock->flags &= ~TTM_SUSPEND_LOCK;
wake_up_all(&lock->queue);
spin_unlock(&lock->lock);
}
EXPORT_SYMBOL(ttm_suspend_unlock);
 
static bool __ttm_suspend_lock(struct ttm_lock *lock)
{
bool locked = false;
 
spin_lock(&lock->lock);
if (lock->rw == 0) {
lock->flags &= ~TTM_SUSPEND_LOCK_PENDING;
lock->flags |= TTM_SUSPEND_LOCK;
locked = true;
} else {
lock->flags |= TTM_SUSPEND_LOCK_PENDING;
}
spin_unlock(&lock->lock);
return locked;
}
 
void ttm_suspend_lock(struct ttm_lock *lock)
{
wait_event(lock->queue, __ttm_suspend_lock(lock));
}
EXPORT_SYMBOL(ttm_suspend_lock);
/drivers/video/drm/ttm/ttm_memory.c
37,534 → 37,40
#include <linux/module.h>
#include <linux/slab.h>
 
struct sysinfo {
u32_t totalram; /* Total usable main memory size */
u32_t freeram; /* Available memory size */
u32_t sharedram; /* Amount of shared memory */
u32_t bufferram; /* Memory used by buffers */
u32_t totalswap; /* Total swap space size */
u32_t freeswap; /* swap space still available */
u32_t totalhigh; /* Total high memory size */
u32_t freehigh; /* Available high memory size */
u32_t mem_unit; /* Memory unit size in bytes */
};
 
 
#define TTM_MEMORY_ALLOC_RETRIES 4
 
struct ttm_mem_zone {
struct kobject kobj;
struct ttm_mem_global *glob;
const char *name;
uint64_t zone_mem;
uint64_t emer_mem;
uint64_t max_mem;
uint64_t swap_limit;
uint64_t used_mem;
};
 
#if 0
 
static struct attribute ttm_mem_sys = {
.name = "zone_memory",
.mode = S_IRUGO
};
static struct attribute ttm_mem_emer = {
.name = "emergency_memory",
.mode = S_IRUGO | S_IWUSR
};
static struct attribute ttm_mem_max = {
.name = "available_memory",
.mode = S_IRUGO | S_IWUSR
};
static struct attribute ttm_mem_swap = {
.name = "swap_limit",
.mode = S_IRUGO | S_IWUSR
};
static struct attribute ttm_mem_used = {
.name = "used_memory",
.mode = S_IRUGO
};
#endif
 
static void ttm_mem_zone_kobj_release(struct kobject *kobj)
int ttm_mem_global_init(struct ttm_mem_global *glob)
{
struct ttm_mem_zone *zone =
container_of(kobj, struct ttm_mem_zone, kobj);
 
pr_info("Zone %7s: Used memory at exit: %llu kiB\n",
zone->name, (unsigned long long)zone->used_mem >> 10);
kfree(zone);
}
 
#if 0
static ssize_t ttm_mem_zone_show(struct kobject *kobj,
struct attribute *attr,
char *buffer)
{
struct ttm_mem_zone *zone =
container_of(kobj, struct ttm_mem_zone, kobj);
uint64_t val = 0;
 
spin_lock(&zone->glob->lock);
if (attr == &ttm_mem_sys)
val = zone->zone_mem;
else if (attr == &ttm_mem_emer)
val = zone->emer_mem;
else if (attr == &ttm_mem_max)
val = zone->max_mem;
else if (attr == &ttm_mem_swap)
val = zone->swap_limit;
else if (attr == &ttm_mem_used)
val = zone->used_mem;
spin_unlock(&zone->glob->lock);
 
return snprintf(buffer, PAGE_SIZE, "%llu\n",
(unsigned long long) val >> 10);
}
 
static void ttm_check_swapping(struct ttm_mem_global *glob);
 
static ssize_t ttm_mem_zone_store(struct kobject *kobj,
struct attribute *attr,
const char *buffer,
size_t size)
{
struct ttm_mem_zone *zone =
container_of(kobj, struct ttm_mem_zone, kobj);
int chars;
unsigned long val;
uint64_t val64;
 
chars = sscanf(buffer, "%lu", &val);
if (chars == 0)
return size;
 
val64 = val;
val64 <<= 10;
 
spin_lock(&zone->glob->lock);
if (val64 > zone->zone_mem)
val64 = zone->zone_mem;
if (attr == &ttm_mem_emer) {
zone->emer_mem = val64;
if (zone->max_mem > val64)
zone->max_mem = val64;
} else if (attr == &ttm_mem_max) {
zone->max_mem = val64;
if (zone->emer_mem < val64)
zone->emer_mem = val64;
} else if (attr == &ttm_mem_swap)
zone->swap_limit = val64;
spin_unlock(&zone->glob->lock);
 
ttm_check_swapping(zone->glob);
 
return size;
}
#endif
 
//static struct attribute *ttm_mem_zone_attrs[] = {
// &ttm_mem_sys,
// &ttm_mem_emer,
// &ttm_mem_max,
// &ttm_mem_swap,
// &ttm_mem_used,
// NULL
//};
 
//static const struct sysfs_ops ttm_mem_zone_ops = {
// .show = &ttm_mem_zone_show,
// .store = &ttm_mem_zone_store
//};
 
static struct kobj_type ttm_mem_zone_kobj_type = {
.release = &ttm_mem_zone_kobj_release,
// .sysfs_ops = &ttm_mem_zone_ops,
// .default_attrs = ttm_mem_zone_attrs,
};
 
static void ttm_mem_global_kobj_release(struct kobject *kobj)
{
struct ttm_mem_global *glob =
container_of(kobj, struct ttm_mem_global, kobj);
 
kfree(glob);
}
 
static struct kobj_type ttm_mem_glob_kobj_type = {
.release = &ttm_mem_global_kobj_release,
};
 
#if 0
static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
bool from_wq, uint64_t extra)
{
unsigned int i;
struct ttm_mem_zone *zone;
uint64_t target;
 
for (i = 0; i < glob->num_zones; ++i) {
zone = glob->zones[i];
 
if (from_wq)
target = zone->swap_limit;
else if (capable(CAP_SYS_ADMIN))
target = zone->emer_mem;
else
target = zone->max_mem;
 
target = (extra > target) ? 0ULL : target;
 
if (zone->used_mem > target)
return true;
}
return false;
}
 
/**
* At this point we only support a single shrink callback.
* Extend this if needed, perhaps using a linked list of callbacks.
* Note that this function is reentrant:
* many threads may try to swap out at any given time.
*/
 
static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,
uint64_t extra)
{
int ret;
struct ttm_mem_shrink *shrink;
int i;
 
spin_lock(&glob->lock);
if (glob->shrink == NULL)
goto out;
spin_lock_init(&glob->lock);
 
while (ttm_zones_above_swap_target(glob, from_wq, extra)) {
shrink = glob->shrink;
spin_unlock(&glob->lock);
ret = shrink->do_shrink(shrink);
spin_lock(&glob->lock);
if (unlikely(ret != 0))
goto out;
}
out:
spin_unlock(&glob->lock);
}
 
 
ttm_page_alloc_init(glob, 4*1024);
 
static void ttm_shrink_work(struct work_struct *work)
{
struct ttm_mem_global *glob =
container_of(work, struct ttm_mem_global, work);
 
ttm_shrink(glob, true, 0ULL);
}
#endif
 
static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,
const struct sysinfo *si)
{
struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
uint64_t mem;
int ret;
 
if (unlikely(!zone))
return -ENOMEM;
 
// mem = si->totalram - si->totalhigh;
// mem *= si->mem_unit;
 
zone->name = "kernel";
zone->zone_mem = mem;
zone->max_mem = mem >> 1;
zone->emer_mem = (mem >> 1) + (mem >> 2);
zone->swap_limit = zone->max_mem - (mem >> 3);
zone->used_mem = 0;
zone->glob = glob;
glob->zone_kernel = zone;
ret = kobject_init_and_add(
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
if (unlikely(ret != 0)) {
kobject_put(&zone->kobj);
return ret;
}
glob->zones[glob->num_zones++] = zone;
return 0;
}
 
#if 0
#ifdef CONFIG_HIGHMEM
static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,
const struct sysinfo *si)
{
struct ttm_mem_zone *zone;
uint64_t mem;
int ret;
 
if (si->totalhigh == 0)
return 0;
 
zone = kzalloc(sizeof(*zone), GFP_KERNEL);
if (unlikely(!zone))
return -ENOMEM;
 
mem = si->totalram;
mem *= si->mem_unit;
 
zone->name = "highmem";
zone->zone_mem = mem;
zone->max_mem = mem >> 1;
zone->emer_mem = (mem >> 1) + (mem >> 2);
zone->swap_limit = zone->max_mem - (mem >> 3);
zone->used_mem = 0;
zone->glob = glob;
glob->zone_highmem = zone;
ret = kobject_init_and_add(
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
if (unlikely(ret != 0)) {
kobject_put(&zone->kobj);
out_no_zone:
ttm_mem_global_release(glob);
return ret;
}
glob->zones[glob->num_zones++] = zone;
return 0;
}
#else
static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,
const struct sysinfo *si)
{
struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
uint64_t mem;
int ret;
EXPORT_SYMBOL(ttm_mem_global_init);
 
if (unlikely(!zone))
return -ENOMEM;
 
mem = si->totalram;
mem *= si->mem_unit;
 
/**
* No special dma32 zone needed.
*/
 
if (mem <= ((uint64_t) 1ULL << 32)) {
kfree(zone);
return 0;
}
 
/*
* Limit max dma32 memory to 4GB for now
* until we can figure out how big this
* zone really is.
*/
 
mem = ((uint64_t) 1ULL << 32);
zone->name = "dma32";
zone->zone_mem = mem;
zone->max_mem = mem >> 1;
zone->emer_mem = (mem >> 1) + (mem >> 2);
zone->swap_limit = zone->max_mem - (mem >> 3);
zone->used_mem = 0;
zone->glob = glob;
glob->zone_dma32 = zone;
ret = kobject_init_and_add(
&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
if (unlikely(ret != 0)) {
kobject_put(&zone->kobj);
return ret;
}
glob->zones[glob->num_zones++] = zone;
return 0;
}
#endif
 
 
 
void ttm_mem_global_release(struct ttm_mem_global *glob)
{
unsigned int i;
struct ttm_mem_zone *zone;
 
/* let the page allocator first stop the shrink work. */
ttm_page_alloc_fini();
ttm_dma_page_alloc_fini();
// ttm_page_alloc_fini();
// ttm_dma_page_alloc_fini();
 
flush_workqueue(glob->swap_queue);
destroy_workqueue(glob->swap_queue);
glob->swap_queue = NULL;
for (i = 0; i < glob->num_zones; ++i) {
zone = glob->zones[i];
kobject_del(&zone->kobj);
kobject_put(&zone->kobj);
 
}
kobject_del(&glob->kobj);
kobject_put(&glob->kobj);
}
EXPORT_SYMBOL(ttm_mem_global_release);
 
static void ttm_check_swapping(struct ttm_mem_global *glob)
{
bool needs_swapping = false;
unsigned int i;
struct ttm_mem_zone *zone;
 
spin_lock(&glob->lock);
for (i = 0; i < glob->num_zones; ++i) {
zone = glob->zones[i];
if (zone->used_mem > zone->swap_limit) {
needs_swapping = true;
break;
}
}
 
spin_unlock(&glob->lock);
 
if (unlikely(needs_swapping))
(void)queue_work(glob->swap_queue, &glob->work);
 
}
 
static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,
struct ttm_mem_zone *single_zone,
uint64_t amount)
{
unsigned int i;
struct ttm_mem_zone *zone;
 
spin_lock(&glob->lock);
for (i = 0; i < glob->num_zones; ++i) {
zone = glob->zones[i];
if (single_zone && zone != single_zone)
continue;
zone->used_mem -= amount;
}
spin_unlock(&glob->lock);
}
 
void ttm_mem_global_free(struct ttm_mem_global *glob,
uint64_t amount)
{
return ttm_mem_global_free_zone(glob, NULL, amount);
}
EXPORT_SYMBOL(ttm_mem_global_free);
 
static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
struct ttm_mem_zone *single_zone,
uint64_t amount, bool reserve)
{
uint64_t limit;
int ret = -ENOMEM;
unsigned int i;
struct ttm_mem_zone *zone;
 
spin_lock(&glob->lock);
for (i = 0; i < glob->num_zones; ++i) {
zone = glob->zones[i];
if (single_zone && zone != single_zone)
continue;
 
limit = zone->emer_mem;
 
if (zone->used_mem > limit)
goto out_unlock;
}
 
if (reserve) {
for (i = 0; i < glob->num_zones; ++i) {
zone = glob->zones[i];
if (single_zone && zone != single_zone)
continue;
zone->used_mem += amount;
}
}
 
ret = 0;
out_unlock:
spin_unlock(&glob->lock);
ttm_check_swapping(glob);
 
return ret;
}
 
 
static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
struct ttm_mem_zone *single_zone,
uint64_t memory,
bool no_wait, bool interruptible)
{
int count = TTM_MEMORY_ALLOC_RETRIES;
 
while (unlikely(ttm_mem_global_reserve(glob,
single_zone,
memory, true)
!= 0)) {
if (no_wait)
return -ENOMEM;
if (unlikely(count-- == 0))
return -ENOMEM;
ttm_shrink(glob, false, memory + (memory >> 2) + 16);
}
 
return 0;
}
 
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
bool no_wait, bool interruptible)
{
/**
* Normal allocations of kernel memory are registered in
* all zones.
*/
 
return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait,
interruptible);
}
EXPORT_SYMBOL(ttm_mem_global_alloc);
 
int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
struct page *page,
bool no_wait, bool interruptible)
{
 
struct ttm_mem_zone *zone = NULL;
 
/**
* Page allocations may be registed in a single zone
* only if highmem or !dma32.
*/
 
#ifdef CONFIG_HIGHMEM
if (PageHighMem(page) && glob->zone_highmem != NULL)
zone = glob->zone_highmem;
#else
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
zone = glob->zone_kernel;
#endif
return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait,
interruptible);
}
 
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
{
struct ttm_mem_zone *zone = NULL;
 
#ifdef CONFIG_HIGHMEM
if (PageHighMem(page) && glob->zone_highmem != NULL)
zone = glob->zone_highmem;
#else
if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
zone = glob->zone_kernel;
#endif
ttm_mem_global_free_zone(glob, zone, PAGE_SIZE);
}
 
#endif
 
void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
{
 
}
 
size_t ttm_round_pot(size_t size)
{
if ((size & (size - 1)) == 0)
582,24 → 88,3
return 0;
}
EXPORT_SYMBOL(ttm_round_pot);
 
void ttm_mem_global_free(struct ttm_mem_global *glob,
uint64_t amount)
{
return 0;
}
 
 
int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
bool no_wait, bool interruptible)
{
return 0;
}
 
EXPORT_SYMBOL(ttm_mem_global_alloc);
 
int ttm_mem_global_init(struct ttm_mem_global *glob)
{
return 0;
}
EXPORT_SYMBOL(ttm_mem_global_init);
/drivers/video/drm/ttm/ttm_page_alloc.c
41,7 → 41,7
#include <linux/mm.h>
#include <linux/seq_file.h> /* for seq_printf */
#include <linux/slab.h>
#include <linux/dma-mapping.h>
//#include <linux/dma-mapping.h>
 
//#include <linux/atomic.h>
 
58,12 → 58,6
/* times are in msecs */
#define PAGE_FREE_INTERVAL 1000
 
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
 
 
 
#if 0
/**
* struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
*
115,7 → 109,6
**/
struct ttm_pool_manager {
struct kobject kobj;
struct shrinker mm_shrink;
struct ttm_pool_opts options;
 
union {
129,134 → 122,10
};
};
 
static struct attribute ttm_page_pool_max = {
.name = "pool_max_size",
.mode = S_IRUGO | S_IWUSR
};
static struct attribute ttm_page_pool_small = {
.name = "pool_small_allocation",
.mode = S_IRUGO | S_IWUSR
};
static struct attribute ttm_page_pool_alloc_size = {
.name = "pool_allocation_size",
.mode = S_IRUGO | S_IWUSR
};
 
static struct attribute *ttm_pool_attrs[] = {
&ttm_page_pool_max,
&ttm_page_pool_small,
&ttm_page_pool_alloc_size,
NULL
};
 
static void ttm_pool_kobj_release(struct kobject *kobj)
{
struct ttm_pool_manager *m =
container_of(kobj, struct ttm_pool_manager, kobj);
kfree(m);
}
 
static ssize_t ttm_pool_store(struct kobject *kobj,
struct attribute *attr, const char *buffer, size_t size)
{
struct ttm_pool_manager *m =
container_of(kobj, struct ttm_pool_manager, kobj);
int chars;
unsigned val;
chars = sscanf(buffer, "%u", &val);
if (chars == 0)
return size;
 
/* Convert kb to number of pages */
val = val / (PAGE_SIZE >> 10);
 
if (attr == &ttm_page_pool_max)
m->options.max_size = val;
else if (attr == &ttm_page_pool_small)
m->options.small = val;
else if (attr == &ttm_page_pool_alloc_size) {
if (val > NUM_PAGES_TO_ALLOC*8) {
pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
return size;
} else if (val > NUM_PAGES_TO_ALLOC) {
pr_warn("Setting allocation size to larger than %lu is not recommended\n",
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
}
m->options.alloc_size = val;
}
 
return size;
}
 
static ssize_t ttm_pool_show(struct kobject *kobj,
struct attribute *attr, char *buffer)
{
struct ttm_pool_manager *m =
container_of(kobj, struct ttm_pool_manager, kobj);
unsigned val = 0;
 
if (attr == &ttm_page_pool_max)
val = m->options.max_size;
else if (attr == &ttm_page_pool_small)
val = m->options.small;
else if (attr == &ttm_page_pool_alloc_size)
val = m->options.alloc_size;
 
val = val * (PAGE_SIZE >> 10);
 
return snprintf(buffer, PAGE_SIZE, "%u\n", val);
}
 
static const struct sysfs_ops ttm_pool_sysfs_ops = {
.show = &ttm_pool_show,
.store = &ttm_pool_store,
};
 
static struct kobj_type ttm_pool_kobj_type = {
.release = &ttm_pool_kobj_release,
.sysfs_ops = &ttm_pool_sysfs_ops,
.default_attrs = ttm_pool_attrs,
};
 
static struct ttm_pool_manager *_manager;
 
#ifndef CONFIG_X86
static int set_pages_array_wb(struct page **pages, int addrinarray)
{
#ifdef TTM_HAS_AGP
int i;
 
for (i = 0; i < addrinarray; i++)
unmap_page_from_agp(pages[i]);
#endif
return 0;
}
 
static int set_pages_array_wc(struct page **pages, int addrinarray)
{
#ifdef TTM_HAS_AGP
int i;
 
for (i = 0; i < addrinarray; i++)
map_page_into_agp(pages[i]);
#endif
return 0;
}
 
static int set_pages_array_uc(struct page **pages, int addrinarray)
{
#ifdef TTM_HAS_AGP
int i;
 
for (i = 0; i < addrinarray; i++)
map_page_into_agp(pages[i]);
#endif
return 0;
}
#endif
 
/**
* Select the right pool or requested caching state and ttm flags. */
static struct ttm_page_pool *ttm_get_pool(int flags,
282,8 → 151,6
static void ttm_pages_put(struct page *pages[], unsigned npages)
{
unsigned i;
if (set_pages_array_wb(pages, npages))
pr_err("Failed to set %d pages to wb!\n", npages);
for (i = 0; i < npages; ++i)
__free_page(pages[i]);
}
295,407 → 162,21
pool->nfrees += freed_pages;
}
 
/**
* Free pages from pool.
*
* To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
* number of pages in one go.
*
* @pool: to free the pages from
* @free_all: If set to true will free all pages in pool
**/
static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
{
unsigned long irq_flags;
struct page *p;
struct page **pages_to_free;
unsigned freed_pages = 0,
npages_to_free = nr_free;
 
if (NUM_PAGES_TO_ALLOC < nr_free)
npages_to_free = NUM_PAGES_TO_ALLOC;
 
pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
GFP_KERNEL);
if (!pages_to_free) {
pr_err("Failed to allocate memory for pool free operation\n");
return 0;
}
 
restart:
spin_lock_irqsave(&pool->lock, irq_flags);
 
list_for_each_entry_reverse(p, &pool->list, lru) {
if (freed_pages >= npages_to_free)
break;
 
pages_to_free[freed_pages++] = p;
/* We can only remove NUM_PAGES_TO_ALLOC at a time. */
if (freed_pages >= NUM_PAGES_TO_ALLOC) {
/* remove range of pages from the pool */
__list_del(p->lru.prev, &pool->list);
 
ttm_pool_update_free_locked(pool, freed_pages);
/**
* Because changing page caching is costly
* we unlock the pool to prevent stalling.
*/
spin_unlock_irqrestore(&pool->lock, irq_flags);
 
ttm_pages_put(pages_to_free, freed_pages);
if (likely(nr_free != FREE_ALL_PAGES))
nr_free -= freed_pages;
 
if (NUM_PAGES_TO_ALLOC >= nr_free)
npages_to_free = nr_free;
else
npages_to_free = NUM_PAGES_TO_ALLOC;
 
freed_pages = 0;
 
/* free all so restart the processing */
if (nr_free)
goto restart;
 
/* Not allowed to fall through or break because
* following context is inside spinlock while we are
* outside here.
*/
goto out;
 
}
}
 
/* remove range of pages from the pool */
if (freed_pages) {
__list_del(&p->lru, &pool->list);
 
ttm_pool_update_free_locked(pool, freed_pages);
nr_free -= freed_pages;
}
 
spin_unlock_irqrestore(&pool->lock, irq_flags);
 
if (freed_pages)
ttm_pages_put(pages_to_free, freed_pages);
out:
kfree(pages_to_free);
return nr_free;
}
 
/**
* Callback for mm to request pool to reduce number of page held.
*
* XXX: (dchinner) Deadlock warning!
*
* ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means
* this can deadlock when called a sc->gfp_mask that is not equal to
* GFP_KERNEL.
*
* This code is crying out for a shrinker per pool....
*/
static unsigned long
ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
{
static atomic_t start_pool = ATOMIC_INIT(0);
unsigned i;
unsigned pool_offset = atomic_add_return(1, &start_pool);
struct ttm_page_pool *pool;
int shrink_pages = sc->nr_to_scan;
unsigned long freed = 0;
 
pool_offset = pool_offset % NUM_POOLS;
/* select start pool in round robin fashion */
for (i = 0; i < NUM_POOLS; ++i) {
unsigned nr_free = shrink_pages;
if (shrink_pages == 0)
break;
pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
shrink_pages = ttm_page_pool_free(pool, nr_free);
freed += nr_free - shrink_pages;
}
return freed;
}
 
 
static unsigned long
ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
unsigned i;
unsigned long count = 0;
 
for (i = 0; i < NUM_POOLS; ++i)
count += _manager->pools[i].npages;
 
return count;
}
 
static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
{
manager->mm_shrink.count_objects = ttm_pool_shrink_count;
manager->mm_shrink.scan_objects = ttm_pool_shrink_scan;
manager->mm_shrink.seeks = 1;
register_shrinker(&manager->mm_shrink);
}
 
static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
{
unregister_shrinker(&manager->mm_shrink);
}
 
static int ttm_set_pages_caching(struct page **pages,
enum ttm_caching_state cstate, unsigned cpages)
{
int r = 0;
/* Set page caching */
switch (cstate) {
case tt_uncached:
r = set_pages_array_uc(pages, cpages);
if (r)
pr_err("Failed to set %d pages to uc!\n", cpages);
break;
case tt_wc:
r = set_pages_array_wc(pages, cpages);
if (r)
pr_err("Failed to set %d pages to wc!\n", cpages);
break;
default:
break;
}
return r;
}
 
/**
* Free pages the pages that failed to change the caching state. If there is
* any pages that have changed their caching state already put them to the
* pool.
*/
static void ttm_handle_caching_state_failure(struct list_head *pages,
int ttm_flags, enum ttm_caching_state cstate,
struct page **failed_pages, unsigned cpages)
{
unsigned i;
/* Failed pages have to be freed */
for (i = 0; i < cpages; ++i) {
list_del(&failed_pages[i]->lru);
__free_page(failed_pages[i]);
}
}
 
/**
* Allocate new pages with correct caching.
*
* This function is reentrant if caller updates count depending on number of
* pages returned in pages array.
*/
static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
int ttm_flags, enum ttm_caching_state cstate, unsigned count)
{
struct page **caching_array;
struct page *p;
int r = 0;
unsigned i, cpages;
unsigned max_cpages = min(count,
(unsigned)(PAGE_SIZE/sizeof(struct page *)));
 
/* allocate array for page caching change */
caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
 
if (!caching_array) {
pr_err("Unable to allocate table for new pages\n");
return -ENOMEM;
}
 
for (i = 0, cpages = 0; i < count; ++i) {
p = alloc_page(gfp_flags);
 
if (!p) {
pr_err("Unable to get page %u\n", i);
 
/* store already allocated pages in the pool after
* setting the caching state */
if (cpages) {
r = ttm_set_pages_caching(caching_array,
cstate, cpages);
if (r)
ttm_handle_caching_state_failure(pages,
ttm_flags, cstate,
caching_array, cpages);
}
r = -ENOMEM;
goto out;
}
 
#ifdef CONFIG_HIGHMEM
/* gfp flags of highmem page should never be dma32 so we
* we should be fine in such case
*/
if (!PageHighMem(p))
#endif
{
caching_array[cpages++] = p;
if (cpages == max_cpages) {
 
r = ttm_set_pages_caching(caching_array,
cstate, cpages);
if (r) {
ttm_handle_caching_state_failure(pages,
ttm_flags, cstate,
caching_array, cpages);
goto out;
}
cpages = 0;
}
}
 
list_add(&p->lru, pages);
}
 
if (cpages) {
r = ttm_set_pages_caching(caching_array, cstate, cpages);
if (r)
ttm_handle_caching_state_failure(pages,
ttm_flags, cstate,
caching_array, cpages);
}
out:
kfree(caching_array);
 
return r;
}
 
/**
* Fill the given pool if there aren't enough pages and the requested number of
* pages is small.
*/
static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
int ttm_flags, enum ttm_caching_state cstate, unsigned count,
unsigned long *irq_flags)
{
struct page *p;
int r;
unsigned cpages = 0;
/**
* Only allow one pool fill operation at a time.
* If pool doesn't have enough pages for the allocation new pages are
* allocated from outside of pool.
*/
if (pool->fill_lock)
return;
 
pool->fill_lock = true;
 
/* If allocation request is small and there are not enough
* pages in a pool we fill the pool up first. */
if (count < _manager->options.small
&& count > pool->npages) {
struct list_head new_pages;
unsigned alloc_size = _manager->options.alloc_size;
 
/**
* Can't change page caching if in irqsave context. We have to
* drop the pool->lock.
*/
spin_unlock_irqrestore(&pool->lock, *irq_flags);
 
INIT_LIST_HEAD(&new_pages);
r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
cstate, alloc_size);
spin_lock_irqsave(&pool->lock, *irq_flags);
 
if (!r) {
list_splice(&new_pages, &pool->list);
++pool->nrefills;
pool->npages += alloc_size;
} else {
pr_err("Failed to fill pool (%p)\n", pool);
/* If we have any pages left put them to the pool. */
list_for_each_entry(p, &pool->list, lru) {
++cpages;
}
list_splice(&new_pages, &pool->list);
pool->npages += cpages;
}
 
}
pool->fill_lock = false;
}
 
/**
* Cut 'count' number of pages from the pool and put them on the return list.
*
* @return count of pages still required to fulfill the request.
*/
static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
struct list_head *pages,
int ttm_flags,
enum ttm_caching_state cstate,
unsigned count)
{
unsigned long irq_flags;
struct list_head *p;
unsigned i;
 
spin_lock_irqsave(&pool->lock, irq_flags);
ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
 
if (count >= pool->npages) {
/* take all pages from the pool */
list_splice_init(&pool->list, pages);
count -= pool->npages;
pool->npages = 0;
goto out;
}
/* find the last pages to include for requested number of pages. Split
* pool to begin and halve it to reduce search space. */
if (count <= pool->npages/2) {
i = 0;
list_for_each(p, &pool->list) {
if (++i == count)
break;
}
} else {
i = pool->npages + 1;
list_for_each_prev(p, &pool->list) {
if (--i == count)
break;
}
}
/* Cut 'count' number of pages from the pool */
list_cut_position(pages, &pool->list, p);
pool->npages -= count;
count = 0;
out:
spin_unlock_irqrestore(&pool->lock, irq_flags);
return count;
}
#endif
 
/* Put all pages in pages list to correct pool to wait for reuse */
static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate)
{
unsigned long irq_flags;
// struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
unsigned i;
 
for (i = 0; i < npages; i++) {
if (pages[i]) {
// if (page_count(pages[i]) != 1)
// pr_err("Erroneous page count. Leaking pages.\n");
FreePage(pages[i]);
pages[i] = NULL;
}
}
return;
 
#if 0
if (pool == NULL) {
if (1) {
/* No pool for this memory type so free the pages */
for (i = 0; i < npages; i++) {
if (pages[i]) {
if (page_count(pages[i]) != 1)
pr_err("Erroneous page count. Leaking pages.\n");
__free_page(pages[i]);
pages[i] = NULL;
}
703,32 → 184,8
return;
}
 
spin_lock_irqsave(&pool->lock, irq_flags);
for (i = 0; i < npages; i++) {
if (pages[i]) {
if (page_count(pages[i]) != 1)
pr_err("Erroneous page count. Leaking pages.\n");
list_add_tail(&pages[i]->lru, &pool->list);
pages[i] = NULL;
pool->npages++;
}
}
/* Check that we don't go over the pool limit */
npages = 0;
if (pool->npages > _manager->options.max_size) {
npages = pool->npages - _manager->options.max_size;
/* free at least NUM_PAGES_TO_ALLOC number of pages
* to reduce calls to set_memory_wb */
if (npages < NUM_PAGES_TO_ALLOC)
npages = NUM_PAGES_TO_ALLOC;
}
spin_unlock_irqrestore(&pool->lock, irq_flags);
if (npages)
ttm_page_pool_free(pool, npages);
#endif
 
}
 
/*
* On success pages list will hold count number of correctly
* cached pages.
736,44 → 193,21
static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate)
{
// struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
struct list_head plist;
struct page *p = NULL;
// gfp_t gfp_flags = GFP_USER;
gfp_t gfp_flags = 0;
unsigned count;
int r;
 
for (r = 0; r < npages; ++r) {
p = AllocPage();
if (!p) {
 
pr_err("Unable to allocate page\n");
return -ENOMEM;
}
 
pages[r] = p;
}
return 0;
 
#if 0
 
 
/* set zero flag for page allocation if required */
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
gfp_flags |= __GFP_ZERO;
 
/* No pool for cached pages */
if (pool == NULL) {
if (flags & TTM_PAGE_FLAG_DMA32)
gfp_flags |= GFP_DMA32;
else
gfp_flags |= GFP_HIGHUSER;
if (1) {
 
for (r = 0; r < npages; ++r) {
p = alloc_page(gfp_flags);
if (!p) {
 
pr_err("Unable to allocate page\n");
return -ENOMEM;
}
 
782,52 → 216,12
return 0;
}
 
/* combine zero flag to pool flags */
gfp_flags |= pool->gfp_flags;
 
/* First we take pages from the pool */
INIT_LIST_HEAD(&plist);
npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
count = 0;
list_for_each_entry(p, &plist, lru) {
pages[count++] = p;
}
 
/* clear the pages coming from the pool if requested */
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
list_for_each_entry(p, &plist, lru) {
if (PageHighMem(p))
clear_highpage(p);
else
clear_page(page_address(p));
}
}
 
/* If pool didn't have enough pages allocate new one. */
if (npages > 0) {
/* ttm_alloc_new_pages doesn't reference pool so we can run
* multiple requests in parallel.
**/
INIT_LIST_HEAD(&plist);
r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
list_for_each_entry(p, &plist, lru) {
pages[count++] = p;
}
if (r) {
/* If there is any pages in the list put them back to
* the pool. */
pr_err("Failed to allocate extra pages for large request\n");
ttm_put_pages(pages, count, flags, cstate);
return r;
}
}
#endif
 
return 0;
}
 
#if 0
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,
char *name)
{
spin_lock_init(&pool->lock);
844,34 → 238,12
 
WARN_ON(_manager);
 
pr_info("Initializing pool allocator\n");
 
_manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
 
ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
 
ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
 
ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
GFP_USER | GFP_DMA32, "wc dma");
 
ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
GFP_USER | GFP_DMA32, "uc dma");
 
_manager->options.max_size = max_pages;
_manager->options.small = SMALL_ALLOCATION;
_manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
 
ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
&glob->kobj, "pool");
if (unlikely(ret != 0)) {
kobject_put(&_manager->kobj);
_manager = NULL;
return ret;
}
 
ttm_pool_mm_shrink_init(_manager);
 
return 0;
}
 
879,18 → 251,9
{
int i;
 
pr_info("Finalizing pool allocator\n");
ttm_pool_mm_shrink_fini(_manager);
 
for (i = 0; i < NUM_POOLS; ++i)
ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
 
kobject_put(&_manager->kobj);
_manager = NULL;
}
 
#endif
 
int ttm_pool_populate(struct ttm_tt *ttm)
{
struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
908,9 → 271,7
ttm_pool_unpopulate(ttm);
return -ENOMEM;
}
 
}
 
ttm->state = tt_unbound;
return 0;
}
922,8 → 283,6
 
for (i = 0; i < ttm->num_pages; ++i) {
if (ttm->pages[i]) {
ttm_mem_global_free_page(ttm->glob->mem_glob,
ttm->pages[i]);
ttm_put_pages(&ttm->pages[i], 1,
ttm->page_flags,
ttm->caching_state);
/drivers/video/drm/ttm/ttm_tt.c
57,9 → 57,12
 
static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
{
ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*));
ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages,
sizeof(*ttm->dma_address));
ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages,
sizeof(*ttm->ttm.pages) +
sizeof(*ttm->dma_address) +
sizeof(*ttm->cpu_address));
ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);
ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages);
}
 
#ifdef CONFIG_X86
118,8 → 121,8
return 0;
}
 
// if (ttm->caching_state == tt_cached)
// drm_clflush_pages(ttm->pages, ttm->num_pages);
if (ttm->caching_state == tt_cached)
drm_clflush_pages(ttm->pages, ttm->num_pages);
 
for (i = 0; i < ttm->num_pages; ++i) {
cur_page = ttm->pages[i];
172,8 → 175,8
ttm_tt_unbind(ttm);
}
 
// if (ttm->state == tt_unbound)
// ttm_tt_unpopulate(ttm);
if (ttm->state == tt_unbound)
ttm_tt_unpopulate(ttm);
 
// if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
// ttm->swap_storage)
230,7 → 233,7
 
INIT_LIST_HEAD(&ttm_dma->pages_list);
ttm_dma_tt_alloc_page_directory(ttm_dma);
if (!ttm->pages || !ttm_dma->dma_address) {
if (!ttm->pages) {
ttm_tt_destroy(ttm);
printf("Failed allocating page table\n");
return -ENOMEM;
245,7 → 248,7
 
drm_free_large(ttm->pages);
ttm->pages = NULL;
drm_free_large(ttm_dma->dma_address);
ttm_dma->cpu_address = NULL;
ttm_dma->dma_address = NULL;
}
EXPORT_SYMBOL(ttm_dma_tt_fini);
261,8 → 264,6
}
}
 
#if 0
 
int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
{
int ret = 0;
286,9 → 287,8
return 0;
}
EXPORT_SYMBOL(ttm_tt_bind);
#endif
 
/*
#if 0
int ttm_tt_swapin(struct ttm_tt *ttm)
{
struct address_space *swap_space;
380,7 → 380,20
 
return ret;
}
#endif
 
*/
static void ttm_tt_clear_mapping(struct ttm_tt *ttm)
{
pgoff_t i;
struct page **page = ttm->pages;
 
}
 
void ttm_tt_unpopulate(struct ttm_tt *ttm)
{
if (ttm->state == tt_unpopulated)
return;
 
ttm_tt_clear_mapping(ttm);
ttm->bdev->driver->ttm_tt_unpopulate(ttm);
}