Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 5270 → Rev 5271

/drivers/video/drm/ttm/ttm_bo.c
37,11 → 37,11
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/file.h>
#include <linux/module.h>
#include <linux/atomic.h>
#include <linux/reservation.h>
 
#define pr_err(fmt, ...) \
printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__)
 
#define TTM_ASSERT_LOCKED(param)
#define TTM_DEBUG(fmt, arg...)
#define TTM_BO_HASH_ORDER 13
48,12 → 48,13
 
 
 
static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
static inline int ttm_mem_type_from_place(const struct ttm_place *place,
uint32_t *mem_type)
{
int i;
 
for (i = 0; i <= TTM_PL_PRIV5; i++)
if (flags & (1 << i)) {
if (place->flags & (1 << i)) {
*mem_type = i;
return 0;
}
83,7 → 84,6
BUG_ON(atomic_read(&bo->list_kref.refcount));
BUG_ON(atomic_read(&bo->kref.refcount));
BUG_ON(atomic_read(&bo->cpu_writers));
BUG_ON(bo->sync_obj != NULL);
BUG_ON(bo->mem.mm_node != NULL);
BUG_ON(!list_empty(&bo->lru));
BUG_ON(!list_empty(&bo->ddestroy));
343,12 → 343,30
ww_mutex_unlock (&bo->resv->lock);
}
 
static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
{
struct reservation_object_list *fobj;
struct fence *fence;
int i;
 
fobj = reservation_object_get_list(bo->resv);
fence = reservation_object_get_excl(bo->resv);
if (fence && !fence->ops->signaled)
fence_enable_sw_signaling(fence);
 
for (i = 0; fobj && i < fobj->shared_count; ++i) {
fence = rcu_dereference_protected(fobj->shared[i],
reservation_object_held(bo->resv));
 
if (!fence->ops->signaled)
fence_enable_sw_signaling(fence);
}
}
 
static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
{
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_bo_global *glob = bo->glob;
struct ttm_bo_driver *driver = bdev->driver;
void *sync_obj = NULL;
int put_count;
int ret;
 
355,10 → 373,8
spin_lock(&glob->lru_lock);
ret = __ttm_bo_reserve(bo, false, true, false, NULL);
 
spin_lock(&bdev->fence_lock);
(void) ttm_bo_wait(bo, false, false, true);
if (!ret && !bo->sync_obj) {
spin_unlock(&bdev->fence_lock);
if (!ret) {
if (!ttm_bo_wait(bo, false, false, true)) {
put_count = ttm_bo_del_from_lru(bo);
 
spin_unlock(&glob->lru_lock);
367,13 → 383,9
ttm_bo_list_ref_sub(bo, put_count, true);
 
return;
}
if (bo->sync_obj)
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
} else
ttm_bo_flush_all_fences(bo);
 
if (!ret) {
 
/*
* Make NO_EVICT bos immediately available to
* shrinkers, now that they are queued for
391,10 → 403,6
list_add_tail(&bo->ddestroy, &bdev->ddestroy);
spin_unlock(&glob->lru_lock);
 
if (sync_obj) {
driver->sync_obj_flush(sync_obj);
driver->sync_obj_unref(&sync_obj);
}
// schedule_delayed_work(&bdev->wq,
// ((HZ / 100) < 1) ? 1 : HZ / 100);
}
415,44 → 423,26
bool interruptible,
bool no_wait_gpu)
{
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_bo_driver *driver = bdev->driver;
struct ttm_bo_global *glob = bo->glob;
int put_count;
int ret;
 
spin_lock(&bdev->fence_lock);
ret = ttm_bo_wait(bo, false, false, true);
 
if (ret && !no_wait_gpu) {
void *sync_obj;
 
/*
* Take a reference to the fence and unreserve,
* at this point the buffer should be dead, so
* no new sync objects can be attached.
*/
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
 
__ttm_bo_unreserve(bo);
long lret;
ww_mutex_unlock(&bo->resv->lock);
spin_unlock(&glob->lru_lock);
 
ret = driver->sync_obj_wait(sync_obj, false, interruptible);
driver->sync_obj_unref(&sync_obj);
if (ret)
return ret;
lret = reservation_object_wait_timeout_rcu(bo->resv,
true,
interruptible,
30 * HZ);
 
/*
* remove sync_obj with ttm_bo_wait, the wait should be
* finished, and no new wait object should have been added.
*/
spin_lock(&bdev->fence_lock);
ret = ttm_bo_wait(bo, false, false, true);
WARN_ON(ret);
spin_unlock(&bdev->fence_lock);
if (ret)
return ret;
if (lret < 0)
return lret;
else if (lret == 0)
return -EBUSY;
 
spin_lock(&glob->lru_lock);
ret = __ttm_bo_reserve(bo, false, true, false, NULL);
469,9 → 459,15
spin_unlock(&glob->lru_lock);
return 0;
}
} else
spin_unlock(&bdev->fence_lock);
 
/*
* remove sync_obj with ttm_bo_wait, the wait should be
* finished, and no new wait object should have been added.
*/
ret = ttm_bo_wait(bo, false, false, true);
WARN_ON(ret);
}
 
if (ret || unlikely(list_empty(&bo->ddestroy))) {
__ttm_bo_unreserve(bo);
spin_unlock(&glob->lru_lock);
601,7 → 597,7
*/
static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
uint32_t mem_type,
struct ttm_placement *placement,
const struct ttm_place *place,
struct ttm_mem_reg *mem,
bool interruptible,
bool no_wait_gpu)
611,7 → 607,7
int ret;
 
do {
ret = (*man->func->get_node)(man, bo, placement, 0, mem);
ret = (*man->func->get_node)(man, bo, place, mem);
if (unlikely(ret != 0))
return ret;
if (mem->mm_node)
654,18 → 650,18
 
static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
uint32_t mem_type,
uint32_t proposed_placement,
const struct ttm_place *place,
uint32_t *masked_placement)
{
uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 
if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
return false;
 
if ((proposed_placement & man->available_caching) == 0)
if ((place->flags & man->available_caching) == 0)
return false;
 
cur_flags |= (proposed_placement & man->available_caching);
cur_flags |= (place->flags & man->available_caching);
 
*masked_placement = cur_flags;
return true;
696,15 → 692,14
 
mem->mm_node = NULL;
for (i = 0; i < placement->num_placement; ++i) {
ret = ttm_mem_type_from_flags(placement->placement[i],
&mem_type);
const struct ttm_place *place = &placement->placement[i];
 
ret = ttm_mem_type_from_place(place, &mem_type);
if (ret)
return ret;
man = &bdev->man[mem_type];
 
type_ok = ttm_bo_mt_compatible(man,
mem_type,
placement->placement[i],
type_ok = ttm_bo_mt_compatible(man, mem_type, place,
&cur_flags);
 
if (!type_ok)
716,7 → 711,7
* Use the access and other non-mapping-related flag bits from
* the memory placement flags to the current flags
*/
ttm_flag_masked(&cur_flags, placement->placement[i],
ttm_flag_masked(&cur_flags, place->flags,
~TTM_PL_MASK_MEMTYPE);
 
if (mem_type == TTM_PL_SYSTEM)
724,8 → 719,7
 
if (man->has_type && man->use_type) {
type_found = true;
ret = (*man->func->get_node)(man, bo, placement,
cur_flags, mem);
ret = (*man->func->get_node)(man, bo, place, mem);
if (unlikely(ret))
return ret;
}
743,17 → 737,15
return -EINVAL;
 
for (i = 0; i < placement->num_busy_placement; ++i) {
ret = ttm_mem_type_from_flags(placement->busy_placement[i],
&mem_type);
const struct ttm_place *place = &placement->busy_placement[i];
 
ret = ttm_mem_type_from_place(place, &mem_type);
if (ret)
return ret;
man = &bdev->man[mem_type];
if (!man->has_type)
continue;
if (!ttm_bo_mt_compatible(man,
mem_type,
placement->busy_placement[i],
&cur_flags))
if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
continue;
 
cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
762,7 → 754,7
* Use the access and other non-mapping-related flag bits from
* the memory placement flags to the current flags
*/
ttm_flag_masked(&cur_flags, placement->busy_placement[i],
ttm_flag_masked(&cur_flags, place->flags,
~TTM_PL_MASK_MEMTYPE);
 
if (mem_type == TTM_PL_SYSTEM) {
772,7 → 764,7
return 0;
}
 
ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
ret = ttm_bo_mem_force_space(bo, mem_type, place, mem,
interruptible, no_wait_gpu);
if (ret == 0 && mem->mm_node) {
mem->placement = cur_flags;
793,7 → 785,6
{
int ret = 0;
struct ttm_mem_reg mem;
struct ttm_bo_device *bdev = bo->bdev;
 
lockdep_assert_held(&bo->resv->lock.base);
 
802,9 → 793,7
* Have the driver move function wait for idle when necessary,
* instead of doing it here.
*/
spin_lock(&bdev->fence_lock);
ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
spin_unlock(&bdev->fence_lock);
if (ret)
return ret;
mem.num_pages = bo->num_pages;
833,13 → 822,14
{
int i;
 
if (mem->mm_node && placement->lpfn != 0 &&
(mem->start < placement->fpfn ||
mem->start + mem->num_pages > placement->lpfn))
return false;
for (i = 0; i < placement->num_placement; i++) {
const struct ttm_place *heap = &placement->placement[i];
if (mem->mm_node &&
(mem->start < heap->fpfn ||
(heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
continue;
 
for (i = 0; i < placement->num_placement; i++) {
*new_flags = placement->placement[i];
*new_flags = heap->flags;
if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
(*new_flags & mem->placement & TTM_PL_MASK_MEM))
return true;
846,7 → 836,13
}
 
for (i = 0; i < placement->num_busy_placement; i++) {
*new_flags = placement->busy_placement[i];
const struct ttm_place *heap = &placement->busy_placement[i];
if (mem->mm_node &&
(mem->start < heap->fpfn ||
(heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
continue;
 
*new_flags = heap->flags;
if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
(*new_flags & mem->placement & TTM_PL_MASK_MEM))
return true;
864,11 → 860,6
uint32_t new_flags;
 
lockdep_assert_held(&bo->resv->lock.base);
/* Check that range is valid */
if (placement->lpfn || placement->fpfn)
if (placement->fpfn > placement->lpfn ||
(placement->lpfn - placement->fpfn) < bo->num_pages)
return -EINVAL;
/*
* Check whether we need to move buffer.
*/
897,15 → 888,6
}
EXPORT_SYMBOL(ttm_bo_validate);
 
int ttm_bo_check_placement(struct ttm_buffer_object *bo,
struct ttm_placement *placement)
{
BUG_ON((placement->fpfn || placement->lpfn) &&
(bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
 
return 0;
}
 
int ttm_bo_init(struct ttm_bo_device *bdev,
struct ttm_buffer_object *bo,
unsigned long size,
916,6 → 898,7
struct file *persistent_swap_storage,
size_t acc_size,
struct sg_table *sg,
struct reservation_object *resv,
void (*destroy) (struct ttm_buffer_object *))
{
int ret = 0;
957,29 → 940,37
bo->persistent_swap_storage = persistent_swap_storage;
bo->acc_size = acc_size;
bo->sg = sg;
if (resv) {
bo->resv = resv;
lockdep_assert_held(&bo->resv->lock.base);
} else {
bo->resv = &bo->ttm_resv;
reservation_object_init(bo->resv);
reservation_object_init(&bo->ttm_resv);
}
atomic_inc(&bo->glob->bo_count);
drm_vma_node_reset(&bo->vma_node);
 
ret = ttm_bo_check_placement(bo, placement);
 
/*
* For ttm_bo_type_device buffers, allocate
* address space from the device.
*/
if (likely(!ret) &&
(bo->type == ttm_bo_type_device ||
bo->type == ttm_bo_type_sg))
if (bo->type == ttm_bo_type_device ||
bo->type == ttm_bo_type_sg)
ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
bo->mem.num_pages);
 
/* passed reservation objects should already be locked,
* since otherwise lockdep will be angered in radeon.
*/
if (!resv) {
locked = ww_mutex_trylock(&bo->resv->lock);
WARN_ON(!locked);
}
 
if (likely(!ret))
ret = ttm_bo_validate(bo, placement, interruptible, false);
 
if (!resv)
ttm_bo_unreserve(bo);
 
if (unlikely(ret))
1118,7 → 1109,6
bdev->glob = glob;
bdev->need_dma32 = need_dma32;
bdev->val_seq = 0;
spin_lock_init(&bdev->fence_lock);
mutex_lock(&glob->device_list_mutex);
list_add_tail(&bdev->device_list, &glob->device_list);
mutex_unlock(&glob->device_list_mutex);
1171,59 → 1161,52
 
EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 
 
int ttm_bo_wait(struct ttm_buffer_object *bo,
bool lazy, bool interruptible, bool no_wait)
{
struct ttm_bo_driver *driver = bo->bdev->driver;
struct ttm_bo_device *bdev = bo->bdev;
void *sync_obj;
int ret = 0;
struct reservation_object_list *fobj;
struct reservation_object *resv;
struct fence *excl;
long timeout = 15 * HZ;
int i;
 
if (likely(bo->sync_obj == NULL))
return 0;
resv = bo->resv;
fobj = reservation_object_get_list(resv);
excl = reservation_object_get_excl(resv);
if (excl) {
if (!fence_is_signaled(excl)) {
if (no_wait)
return -EBUSY;
 
while (bo->sync_obj) {
 
if (driver->sync_obj_signaled(bo->sync_obj)) {
void *tmp_obj = bo->sync_obj;
bo->sync_obj = NULL;
clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&tmp_obj);
spin_lock(&bdev->fence_lock);
continue;
timeout = fence_wait_timeout(excl,
interruptible, timeout);
}
}
 
for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) {
struct fence *fence;
fence = rcu_dereference_protected(fobj->shared[i],
reservation_object_held(resv));
 
if (!fence_is_signaled(fence)) {
if (no_wait)
return -EBUSY;
 
sync_obj = driver->sync_obj_ref(bo->sync_obj);
spin_unlock(&bdev->fence_lock);
ret = driver->sync_obj_wait(sync_obj,
lazy, interruptible);
if (unlikely(ret != 0)) {
driver->sync_obj_unref(&sync_obj);
spin_lock(&bdev->fence_lock);
return ret;
timeout = fence_wait_timeout(fence,
interruptible, timeout);
}
spin_lock(&bdev->fence_lock);
if (likely(bo->sync_obj == sync_obj)) {
void *tmp_obj = bo->sync_obj;
bo->sync_obj = NULL;
clear_bit(TTM_BO_PRIV_FLAG_MOVING,
&bo->priv_flags);
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&sync_obj);
driver->sync_obj_unref(&tmp_obj);
spin_lock(&bdev->fence_lock);
} else {
spin_unlock(&bdev->fence_lock);
driver->sync_obj_unref(&sync_obj);
spin_lock(&bdev->fence_lock);
}
}
 
if (timeout < 0)
return timeout;
 
if (timeout == 0)
return -EBUSY;
 
reservation_object_add_excl_fence(resv, NULL);
clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
return 0;
}
EXPORT_SYMBOL(ttm_bo_wait);