Subversion Repositories Kolibri OS

Rev

Rev 1120 | Go to most recent revision | Blame | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright 2009 Jerome Glisse.
  3.  * All Rights Reserved.
  4.  *
  5.  * Permission is hereby granted, free of charge, to any person obtaining a
  6.  * copy of this software and associated documentation files (the
  7.  * "Software"), to deal in the Software without restriction, including
  8.  * without limitation the rights to use, copy, modify, merge, publish,
  9.  * distribute, sub license, and/or sell copies of the Software, and to
  10.  * permit persons to whom the Software is furnished to do so, subject to
  11.  * the following conditions:
  12.  *
  13.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15.  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16.  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17.  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18.  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19.  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20.  *
  21.  * The above copyright notice and this permission notice (including the
  22.  * next paragraph) shall be included in all copies or substantial portions
  23.  * of the Software.
  24.  *
  25.  */
  26. /*
  27.  * Authors:
  28.  *    Jerome Glisse <glisse@freedesktop.org>
  29.  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30.  *    Dave Airlie
  31.  */
  32. #include <list.h>
  33. #include <drmP.h>
  34.  
  35. #include "radeon_drm.h"
  36. #include "radeon.h"
  37. #include <drm_mm.h>
  38.  
  39. int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
  40.              int pages, u32_t *pagelist);
  41.  
  42.  
  43. #define TTM_PL_SYSTEM           0
  44. #define TTM_PL_TT               1
  45. #define TTM_PL_VRAM             2
  46. #define TTM_PL_PRIV0            3
  47. #define TTM_PL_PRIV1            4
  48. #define TTM_PL_PRIV2            5
  49. #define TTM_PL_PRIV3            6
  50. #define TTM_PL_PRIV4            7
  51. #define TTM_PL_PRIV5            8
  52. #define TTM_PL_SWAPPED          15
  53.  
  54. #define TTM_PL_FLAG_SYSTEM      (1 << TTM_PL_SYSTEM)
  55. #define TTM_PL_FLAG_TT          (1 << TTM_PL_TT)
  56. #define TTM_PL_FLAG_VRAM        (1 << TTM_PL_VRAM)
  57. #define TTM_PL_FLAG_PRIV0       (1 << TTM_PL_PRIV0)
  58. #define TTM_PL_FLAG_PRIV1       (1 << TTM_PL_PRIV1)
  59. #define TTM_PL_FLAG_PRIV2       (1 << TTM_PL_PRIV2)
  60. #define TTM_PL_FLAG_PRIV3       (1 << TTM_PL_PRIV3)
  61. #define TTM_PL_FLAG_PRIV4       (1 << TTM_PL_PRIV4)
  62. #define TTM_PL_FLAG_PRIV5       (1 << TTM_PL_PRIV5)
  63. #define TTM_PL_FLAG_SWAPPED     (1 << TTM_PL_SWAPPED)
  64. #define TTM_PL_MASK_MEM         0x0000FFFF
  65.  
  66.  
  67. struct ttm_mem_type_manager {
  68.  
  69.     /*
  70.      * No protection. Constant from start.
  71.      */
  72.  
  73.     bool            has_type;
  74.     bool            use_type;
  75.     uint32_t        flags;
  76.     unsigned long   gpu_offset;
  77.     unsigned long   io_offset;
  78.     unsigned long   io_size;
  79.     void            *io_addr;
  80.     uint64_t        size;
  81.     uint32_t        available_caching;
  82.     uint32_t        default_caching;
  83.  
  84.     /*
  85.      * Protected by the bdev->lru_lock.
  86.      * TODO: Consider one lru_lock per ttm_mem_type_manager.
  87.      * Plays ill with list removal, though.
  88.      */
  89.  
  90.     struct drm_mm manager;
  91.     struct list_head lru;
  92. };
  93.  
  94. struct ttm_bo_driver {
  95.     const uint32_t      *mem_type_prio;
  96.     const uint32_t      *mem_busy_prio;
  97.     uint32_t             num_mem_type_prio;
  98.     uint32_t             num_mem_busy_prio;
  99.  
  100.     /**
  101.      * struct ttm_bo_driver member create_ttm_backend_entry
  102.      *
  103.      * @bdev: The buffer object device.
  104.      *
  105.      * Create a driver specific struct ttm_backend.
  106.      */
  107.  
  108. //    struct ttm_backend *(*create_ttm_backend_entry)(struct ttm_bo_device *bdev);
  109.  
  110.     /**
  111.      * struct ttm_bo_driver member invalidate_caches
  112.      *
  113.      * @bdev: the buffer object device.
  114.      * @flags: new placement of the rebound buffer object.
  115.      *
  116.      * A previosly evicted buffer has been rebound in a
  117.      * potentially new location. Tell the driver that it might
  118.      * consider invalidating read (texture) caches on the next command
  119.      * submission as a consequence.
  120.      */
  121.  
  122. //    int (*invalidate_caches) (struct ttm_bo_device *bdev, uint32_t flags);
  123. //    int (*init_mem_type) (struct ttm_bo_device *bdev, uint32_t type,
  124. //                  struct ttm_mem_type_manager *man);
  125.     /**
  126.      * struct ttm_bo_driver member evict_flags:
  127.      *
  128.      * @bo: the buffer object to be evicted
  129.      *
  130.      * Return the bo flags for a buffer which is not mapped to the hardware.
  131.      * These will be placed in proposed_flags so that when the move is
  132.      * finished, they'll end up in bo->mem.flags
  133.      */
  134.  
  135. //     uint32_t(*evict_flags) (struct ttm_buffer_object *bo);
  136.     /**
  137.      * struct ttm_bo_driver member move:
  138.      *
  139.      * @bo: the buffer to move
  140.      * @evict: whether this motion is evicting the buffer from
  141.      * the graphics address space
  142.      * @interruptible: Use interruptible sleeps if possible when sleeping.
  143.      * @no_wait: whether this should give up and return -EBUSY
  144.      * if this move would require sleeping
  145.      * @new_mem: the new memory region receiving the buffer
  146.      *
  147.      * Move a buffer between two memory regions.
  148.      */
  149. //    int (*move) (struct ttm_buffer_object *bo,
  150. //             bool evict, bool interruptible,
  151. //             bool no_wait, struct ttm_mem_reg *new_mem);
  152.  
  153.     /**
  154.      * struct ttm_bo_driver_member verify_access
  155.      *
  156.      * @bo: Pointer to a buffer object.
  157.      * @filp: Pointer to a struct file trying to access the object.
  158.      *
  159.      * Called from the map / write / read methods to verify that the
  160.      * caller is permitted to access the buffer object.
  161.      * This member may be set to NULL, which will refuse this kind of
  162.      * access for all buffer objects.
  163.      * This function should return 0 if access is granted, -EPERM otherwise.
  164.      */
  165. //    int (*verify_access) (struct ttm_buffer_object *bo,
  166. //                  struct file *filp);
  167.  
  168.     /**
  169.      * In case a driver writer dislikes the TTM fence objects,
  170.      * the driver writer can replace those with sync objects of
  171.      * his / her own. If it turns out that no driver writer is
  172.      * using these. I suggest we remove these hooks and plug in
  173.      * fences directly. The bo driver needs the following functionality:
  174.      * See the corresponding functions in the fence object API
  175.      * documentation.
  176.      */
  177.  
  178. //    bool (*sync_obj_signaled) (void *sync_obj, void *sync_arg);
  179. //    int (*sync_obj_wait) (void *sync_obj, void *sync_arg,
  180. //                  bool lazy, bool interruptible);
  181. //    int (*sync_obj_flush) (void *sync_obj, void *sync_arg);
  182. //    void (*sync_obj_unref) (void **sync_obj);
  183. //    void *(*sync_obj_ref) (void *sync_obj);
  184. };
  185.  
  186. #define TTM_NUM_MEM_TYPES 8
  187.  
  188.  
  189. struct ttm_bo_device {
  190.  
  191.     /*
  192.      * Constant after bo device init / atomic.
  193.      */
  194.  
  195. //    struct ttm_mem_global *mem_glob;
  196.     struct ttm_bo_driver *driver;
  197. //    struct page *dummy_read_page;
  198. //    struct ttm_mem_shrink shrink;
  199.  
  200.     size_t      ttm_bo_extra_size;
  201.     size_t      ttm_bo_size;
  202.  
  203. //   rwlock_t vm_lock;
  204.     /*
  205.      * Protected by the vm lock.
  206.      */
  207.     struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES];
  208. //   struct rb_root addr_space_rb;
  209.     struct drm_mm       addr_space_mm;
  210.  
  211.     /*
  212.      * Might want to change this to one lock per manager.
  213.      */
  214. //   spinlock_t lru_lock;
  215.     /*
  216.      * Protected by the lru lock.
  217.      */
  218.     struct list_head ddestroy;
  219.     struct list_head swap_lru;
  220.  
  221.     /*
  222.      * Protected by load / firstopen / lastclose /unload sync.
  223.      */
  224.  
  225.     bool nice_mode;
  226. //   struct address_space *dev_mapping;
  227.  
  228.     /*
  229.      * Internal protection.
  230.      */
  231.  
  232. //   struct delayed_work wq;
  233. };
  234.  
  235. struct ttm_mem_reg {
  236.     struct drm_mm_node *mm_node;
  237.     unsigned long       size;
  238.     unsigned long       num_pages;
  239.     uint32_t            page_alignment;
  240.     uint32_t            mem_type;
  241.     uint32_t            placement;
  242. };
  243.  
  244. enum ttm_bo_type {
  245.     ttm_bo_type_device,
  246.     ttm_bo_type_user,
  247.     ttm_bo_type_kernel
  248. };
  249.  
  250. struct ttm_buffer_object {
  251.     /**
  252.      * Members constant at init.
  253.      */
  254.  
  255.     struct ttm_bo_device   *bdev;
  256.     unsigned long           buffer_start;
  257.     enum ttm_bo_type        type;
  258.     void (*destroy) (struct ttm_buffer_object *);
  259.     unsigned long           num_pages;
  260.     uint64_t                addr_space_offset;
  261.     size_t                  acc_size;
  262.  
  263.     /**
  264.     * Members not needing protection.
  265.     */
  266.  
  267. //    struct kref kref;
  268. //    struct kref list_kref;
  269. //    wait_queue_head_t event_queue;
  270. //    spinlock_t lock;
  271.  
  272.     /**
  273.      * Members protected by the bo::reserved lock.
  274.      */
  275.  
  276.     uint32_t                proposed_placement;
  277.     struct ttm_mem_reg      mem;
  278. //    struct file *persistant_swap_storage;
  279. //    struct ttm_tt *ttm;
  280.     bool evicted;
  281.  
  282.     /**
  283.      * Members protected by the bo::reserved lock only when written to.
  284.      */
  285.  
  286. //    atomic_t cpu_writers;
  287.  
  288.     /**
  289.      * Members protected by the bdev::lru_lock.
  290.      */
  291.  
  292.     struct list_head lru;
  293.     struct list_head ddestroy;
  294.     struct list_head swap;
  295.     uint32_t val_seq;
  296.     bool seq_valid;
  297.  
  298.     /**
  299.      * Members protected by the bdev::lru_lock
  300.      * only when written to.
  301.      */
  302.  
  303. //    atomic_t reserved;
  304.  
  305.  
  306.     /**
  307.      * Members protected by the bo::lock
  308.      */
  309.  
  310.     void *sync_obj_arg;
  311.     void *sync_obj;
  312.     unsigned long priv_flags;
  313.  
  314.     /**
  315.      * Members protected by the bdev::vm_lock
  316.      */
  317.  
  318. //    struct rb_node vm_rb;
  319.     struct drm_mm_node *vm_node;
  320.  
  321.  
  322.     /**
  323.      * Special members that are protected by the reserve lock
  324.      * and the bo::lock when written to. Can be read with
  325.      * either of these locks held.
  326.      */
  327.  
  328.     unsigned long offset;
  329.     uint32_t cur_placement;
  330. };
  331.  
  332. struct radeon_object
  333. {
  334.     struct ttm_buffer_object     tobj;
  335.     struct list_head            list;
  336.         struct radeon_device            *rdev;
  337. //   struct drm_gem_object       *gobj;
  338. //   struct ttm_bo_kmap_obj      kmap;
  339.  
  340.         unsigned                        pin_count;
  341.         uint64_t                        gpu_addr;
  342.         void                            *kptr;
  343.         bool                            is_iomem;
  344.  
  345.     struct drm_mm_node  *mm_node;
  346.     u32_t                vm_addr;
  347.     u32_t                cpu_addr;
  348.     u32_t                flags;
  349. };
  350.  
  351.  
  352.  
  353.  
  354. static struct drm_mm   mm_gtt;
  355. static struct drm_mm   mm_vram;
  356.  
  357.  
  358. int radeon_object_init(struct radeon_device *rdev)
  359. {
  360.     int r = 0;
  361.  
  362.     dbgprintf("%s\n",__FUNCTION__);
  363.  
  364.     r = drm_mm_init(&mm_vram, 0x800000 >> PAGE_SHIFT,
  365.                ((rdev->mc.aper_size - 0x800000) >> PAGE_SHIFT));
  366.     if (r) {
  367.         DRM_ERROR("Failed initializing VRAM heap.\n");
  368.         return r;
  369.     };
  370.  
  371.     r = drm_mm_init(&mm_gtt, 0, ((rdev->mc.gtt_size) >> PAGE_SHIFT));
  372.     if (r) {
  373.         DRM_ERROR("Failed initializing GTT heap.\n");
  374.         return r;
  375.     }
  376.  
  377.     return r;
  378.  //   return radeon_ttm_init(rdev);
  379. }
  380.  
  381. static inline uint32_t radeon_object_flags_from_domain(uint32_t domain)
  382. {
  383.     uint32_t flags = 0;
  384.     if (domain & RADEON_GEM_DOMAIN_VRAM) {
  385.         flags |= TTM_PL_FLAG_VRAM;
  386.     }
  387.     if (domain & RADEON_GEM_DOMAIN_GTT) {
  388.         flags |= TTM_PL_FLAG_TT;
  389.     }
  390.     if (domain & RADEON_GEM_DOMAIN_CPU) {
  391.         flags |= TTM_PL_FLAG_SYSTEM;
  392.     }
  393.     if (!flags) {
  394.         flags |= TTM_PL_FLAG_SYSTEM;
  395.     }
  396.     return flags;
  397. }
  398.  
  399.  
  400. int radeon_object_create(struct radeon_device *rdev,
  401.              struct drm_gem_object *gobj,
  402.              unsigned long size,
  403.              bool kernel,
  404.              uint32_t domain,
  405.              bool interruptible,
  406.              struct radeon_object **robj_ptr)
  407. {
  408.     struct radeon_object *robj;
  409.     enum ttm_bo_type type;
  410.     uint32_t flags;
  411.     int r;
  412.  
  413.     dbgprintf("%s\n",__FUNCTION__);
  414.  
  415.     if (kernel) {
  416.         type = ttm_bo_type_kernel;
  417.     } else {
  418.         type = ttm_bo_type_device;
  419.     }
  420.     *robj_ptr = NULL;
  421.     robj = kzalloc(sizeof(struct radeon_object), GFP_KERNEL);
  422.     if (robj == NULL) {
  423.         return -ENOMEM;
  424.     }
  425.     robj->rdev = rdev;
  426. //    robj->gobj = gobj;
  427.     INIT_LIST_HEAD(&robj->list);
  428.  
  429.     flags = radeon_object_flags_from_domain(domain);
  430.  
  431.     robj->flags = flags;
  432.  
  433.     dbgprintf("robj flags %x\n", robj->flags);
  434.  
  435.     if( flags & TTM_PL_FLAG_VRAM)
  436.     {
  437.         size_t num_pages;
  438.  
  439.         struct drm_mm_node *vm_node;
  440.  
  441.         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  442.  
  443.         if (num_pages == 0) {
  444.             printk("Illegal buffer object size.\n");
  445.             return -EINVAL;
  446.         }
  447. retry_pre_get:
  448.         r = drm_mm_pre_get(&mm_vram);
  449.  
  450.         if (unlikely(r != 0))
  451.             return r;
  452.  
  453.         vm_node = drm_mm_search_free(&mm_vram, num_pages, 0, 0);
  454.  
  455.         if (unlikely(vm_node == NULL)) {
  456.             r = -ENOMEM;
  457.             return r;
  458.         }
  459.  
  460.         robj->mm_node =  drm_mm_get_block_atomic(vm_node, num_pages, 0);
  461.  
  462.         if (unlikely(robj->mm_node == NULL)) {
  463.             goto retry_pre_get;
  464.         }
  465.  
  466.         robj->vm_addr = ((uint32_t)robj->mm_node->start);
  467.  
  468.         dbgprintf("alloc vram: base %x size %x\n",
  469.                    robj->vm_addr << PAGE_SHIFT, num_pages  << PAGE_SHIFT);
  470.  
  471.     };
  472.  
  473.     if( flags & TTM_PL_FLAG_TT)
  474.     {
  475.         size_t num_pages;
  476.  
  477.         struct drm_mm_node *vm_node;
  478.  
  479.         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  480.  
  481.         if (num_pages == 0) {
  482.             printk("Illegal buffer object size.\n");
  483.             return -EINVAL;
  484.         }
  485. retry_pre_get1:
  486.         r = drm_mm_pre_get(&mm_gtt);
  487.  
  488.         if (unlikely(r != 0))
  489.             return r;
  490.  
  491.         vm_node = drm_mm_search_free(&mm_gtt, num_pages, 0, 0);
  492.  
  493.         if (unlikely(vm_node == NULL)) {
  494.             r = -ENOMEM;
  495.             return r;
  496.         }
  497.  
  498.         robj->mm_node =  drm_mm_get_block_atomic(vm_node, num_pages, 0);
  499.  
  500.         if (unlikely(robj->mm_node == NULL)) {
  501.             goto retry_pre_get1;
  502.         }
  503.  
  504.         robj->vm_addr = ((uint32_t)robj->mm_node->start) ;
  505.  
  506.         dbgprintf("alloc gtt: base %x size %x\n",
  507.                    robj->vm_addr << PAGE_SHIFT, num_pages  << PAGE_SHIFT);
  508.     };
  509.  
  510. //   r = ttm_buffer_object_init(&rdev->mman.bdev, &robj->tobj, size, type, flags,
  511. //                  0, 0, false, NULL, size,
  512. //                  &radeon_ttm_object_object_destroy);
  513.     if (unlikely(r != 0)) {
  514.         /* ttm call radeon_ttm_object_object_destroy if error happen */
  515.         DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n",
  516.               size, flags, 0);
  517.         return r;
  518.     }
  519.     *robj_ptr = robj;
  520. //   if (gobj) {
  521. //       list_add_tail(&robj->list, &rdev->gem.objects);
  522. //   }
  523.     return 0;
  524. }
  525.  
  526. #define page_tabs  0xFDC00000
  527.  
  528. int radeon_object_pin(struct radeon_object *robj, uint32_t domain,
  529.               uint64_t *gpu_addr)
  530. {
  531.     uint32_t flags;
  532.     uint32_t tmp;
  533.     int r = 0;
  534.  
  535.     dbgprintf("%s\n",__FUNCTION__);
  536.  
  537. //    flags = radeon_object_flags_from_domain(domain);
  538. //   spin_lock(&robj->tobj.lock);
  539.     if (robj->pin_count) {
  540.         robj->pin_count++;
  541.         if (gpu_addr != NULL) {
  542.             *gpu_addr = robj->gpu_addr;
  543.         }
  544. //       spin_unlock(&robj->tobj.lock);
  545.         return 0;
  546.     }
  547. //   spin_unlock(&robj->tobj.lock);
  548. //    r = radeon_object_reserve(robj, false);
  549. //    if (unlikely(r != 0)) {
  550. //        DRM_ERROR("radeon: failed to reserve object for pinning it.\n");
  551. //        return r;
  552. //    }
  553. //    tmp = robj->tobj.mem.placement;
  554. //    ttm_flag_masked(&tmp, flags, TTM_PL_MASK_MEM);
  555. //    robj->tobj.proposed_placement = tmp | TTM_PL_FLAG_NO_EVICT | TTM_PL_MASK_CACHING;
  556. //    r = ttm_buffer_object_validate(&robj->tobj,
  557. //                       robj->tobj.proposed_placement,
  558. //                       false, false);
  559.  
  560.     robj->gpu_addr = ((u64)robj->vm_addr) << PAGE_SHIFT;
  561.  
  562.     if(robj->flags & TTM_PL_FLAG_VRAM)
  563.         robj->gpu_addr += (u64)robj->rdev->mc.vram_location;
  564.     else if (robj->flags & TTM_PL_FLAG_TT)
  565.     {
  566.         u32_t *pagelist;
  567.         robj->kptr  = KernelAlloc( robj->mm_node->size << PAGE_SHIFT );
  568.         dbgprintf("kernel alloc %x\n", robj->kptr );
  569.  
  570.         pagelist =  &((u32_t*)page_tabs)[(u32_t)robj->kptr >> 12];
  571.         dbgprintf("pagelist %x\n", pagelist);
  572.         radeon_gart_bind(robj->rdev, robj->gpu_addr,
  573.                          robj->mm_node->size,  pagelist);
  574.         robj->gpu_addr += (u64)robj->rdev->mc.gtt_location;
  575.     }
  576.     else
  577.     {
  578.         DRM_ERROR("Unknown placement %d\n", robj->flags);
  579.         robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL;
  580.         r = -1;
  581.     };
  582.  
  583. //    flags & TTM_PL_FLAG_VRAM
  584.     if (gpu_addr != NULL) {
  585.         *gpu_addr = robj->gpu_addr;
  586.     }
  587.     robj->pin_count = 1;
  588.     if (unlikely(r != 0)) {
  589.         DRM_ERROR("radeon: failed to pin object.\n");
  590.     }
  591.  
  592.     dbgprintf("done %s\n",__FUNCTION__);
  593.  
  594.     return r;
  595. }
  596.  
  597. int radeon_object_kmap(struct radeon_object *robj, void **ptr)
  598. {
  599.     int r = 0;
  600.  
  601.     dbgprintf("%s\n",__FUNCTION__);
  602.  
  603. //   spin_lock(&robj->tobj.lock);
  604.     if (robj->kptr) {
  605.         if (ptr) {
  606.             *ptr = robj->kptr;
  607.         }
  608. //       spin_unlock(&robj->tobj.lock);
  609.         return 0;
  610.     }
  611. //   spin_unlock(&robj->tobj.lock);
  612.  
  613.     if(robj->flags & TTM_PL_FLAG_VRAM)
  614.     {
  615.         robj->cpu_addr = robj->rdev->mc.aper_base +
  616.                          (robj->vm_addr << PAGE_SHIFT);
  617.         robj->kptr = (void*)MapIoMem(robj->cpu_addr,
  618.                            robj->mm_node->size << 12, PG_SW);
  619.         dbgprintf("map io mem %x at %x\n", robj->cpu_addr, robj->kptr);
  620.  
  621.     }
  622.     else
  623.     {
  624.         return -1;
  625.     }
  626.  
  627.     if (ptr) {
  628.         *ptr = robj->kptr;
  629.     }
  630.  
  631.     dbgprintf("done %s\n",__FUNCTION__);
  632.  
  633.     return 0;
  634. }
  635.  
  636.  
  637. #if 0
  638.  
  639. void radeon_object_unpin(struct radeon_object *robj)
  640. {
  641.     uint32_t flags;
  642.     int r;
  643.  
  644. //   spin_lock(&robj->tobj.lock);
  645.     if (!robj->pin_count) {
  646. //       spin_unlock(&robj->tobj.lock);
  647.         printk(KERN_WARNING "Unpin not necessary for %p !\n", robj);
  648.         return;
  649.     }
  650.     robj->pin_count--;
  651.     if (robj->pin_count) {
  652. //       spin_unlock(&robj->tobj.lock);
  653.         return;
  654.     }
  655. //   spin_unlock(&robj->tobj.lock);
  656.     r = radeon_object_reserve(robj, false);
  657.     if (unlikely(r != 0)) {
  658.         DRM_ERROR("radeon: failed to reserve object for unpinning it.\n");
  659.         return;
  660.     }
  661.     flags = robj->tobj.mem.placement;
  662.     robj->tobj.proposed_placement = flags & ~TTM_PL_FLAG_NO_EVICT;
  663.     r = ttm_buffer_object_validate(&robj->tobj,
  664.                        robj->tobj.proposed_placement,
  665.                        false, false);
  666.     if (unlikely(r != 0)) {
  667.         DRM_ERROR("radeon: failed to unpin buffer.\n");
  668.     }
  669.     radeon_object_unreserve(robj);
  670. }
  671.  
  672.  
  673.  
  674.  
  675.  
  676. /*
  677.  * To exclude mutual BO access we rely on bo_reserve exclusion, as all
  678.  * function are calling it.
  679.  */
  680.  
  681. static int radeon_object_reserve(struct radeon_object *robj, bool interruptible)
  682. {
  683.         return ttm_bo_reserve(&robj->tobj, interruptible, false, false, 0);
  684. }
  685.  
  686. static void radeon_object_unreserve(struct radeon_object *robj)
  687. {
  688.         ttm_bo_unreserve(&robj->tobj);
  689. }
  690.  
  691. static void radeon_ttm_object_object_destroy(struct ttm_buffer_object *tobj)
  692. {
  693.         struct radeon_object *robj;
  694.  
  695.         robj = container_of(tobj, struct radeon_object, tobj);
  696. //   list_del_init(&robj->list);
  697.         kfree(robj);
  698. }
  699.  
  700. static inline void radeon_object_gpu_addr(struct radeon_object *robj)
  701. {
  702.         /* Default gpu address */
  703.         robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL;
  704.         if (robj->tobj.mem.mm_node == NULL) {
  705.                 return;
  706.         }
  707.         robj->gpu_addr = ((u64)robj->tobj.mem.mm_node->start) << PAGE_SHIFT;
  708.         switch (robj->tobj.mem.mem_type) {
  709.         case TTM_PL_VRAM:
  710.                 robj->gpu_addr += (u64)robj->rdev->mc.vram_location;
  711.                 break;
  712.         case TTM_PL_TT:
  713.                 robj->gpu_addr += (u64)robj->rdev->mc.gtt_location;
  714.                 break;
  715.         default:
  716.                 DRM_ERROR("Unknown placement %d\n", robj->tobj.mem.mem_type);
  717.                 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL;
  718.                 return;
  719.         }
  720. }
  721.  
  722.  
  723. int radeon_object_create(struct radeon_device *rdev,
  724.                          struct drm_gem_object *gobj,
  725.                          unsigned long size,
  726.                          bool kernel,
  727.                          uint32_t domain,
  728.                          bool interruptible,
  729.                          struct radeon_object **robj_ptr)
  730. {
  731.         struct radeon_object *robj;
  732.         enum ttm_bo_type type;
  733.         uint32_t flags;
  734.         int r;
  735.  
  736. //   if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) {
  737. //       rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
  738. //   }
  739.         if (kernel) {
  740.                 type = ttm_bo_type_kernel;
  741.         } else {
  742.                 type = ttm_bo_type_device;
  743.         }
  744.         *robj_ptr = NULL;
  745.         robj = kzalloc(sizeof(struct radeon_object), GFP_KERNEL);
  746.         if (robj == NULL) {
  747.                 return -ENOMEM;
  748.         }
  749.         robj->rdev = rdev;
  750.         robj->gobj = gobj;
  751. //   INIT_LIST_HEAD(&robj->list);
  752.  
  753.         flags = radeon_object_flags_from_domain(domain);
  754. //   r = ttm_buffer_object_init(&rdev->mman.bdev, &robj->tobj, size, type, flags,
  755. //                  0, 0, false, NULL, size,
  756. //                  &radeon_ttm_object_object_destroy);
  757.         if (unlikely(r != 0)) {
  758.                 /* ttm call radeon_ttm_object_object_destroy if error happen */
  759.                 DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n",
  760.                           size, flags, 0);
  761.                 return r;
  762.         }
  763.         *robj_ptr = robj;
  764. //   if (gobj) {
  765. //       list_add_tail(&robj->list, &rdev->gem.objects);
  766. //   }
  767.         return 0;
  768. }
  769.  
  770. int radeon_object_kmap(struct radeon_object *robj, void **ptr)
  771. {
  772.         int r;
  773.  
  774. //   spin_lock(&robj->tobj.lock);
  775.         if (robj->kptr) {
  776.                 if (ptr) {
  777.                         *ptr = robj->kptr;
  778.                 }
  779. //       spin_unlock(&robj->tobj.lock);
  780.                 return 0;
  781.         }
  782. //   spin_unlock(&robj->tobj.lock);
  783.         r = ttm_bo_kmap(&robj->tobj, 0, robj->tobj.num_pages, &robj->kmap);
  784.         if (r) {
  785.                 return r;
  786.         }
  787. //   spin_lock(&robj->tobj.lock);
  788.         robj->kptr = ttm_kmap_obj_virtual(&robj->kmap, &robj->is_iomem);
  789. //   spin_unlock(&robj->tobj.lock);
  790.         if (ptr) {
  791.                 *ptr = robj->kptr;
  792.         }
  793.         return 0;
  794. }
  795.  
  796. void radeon_object_kunmap(struct radeon_object *robj)
  797. {
  798. //   spin_lock(&robj->tobj.lock);
  799.         if (robj->kptr == NULL) {
  800. //       spin_unlock(&robj->tobj.lock);
  801.                 return;
  802.         }
  803.         robj->kptr = NULL;
  804. //   spin_unlock(&robj->tobj.lock);
  805.         ttm_bo_kunmap(&robj->kmap);
  806. }
  807.  
  808. void radeon_object_unref(struct radeon_object **robj)
  809. {
  810.         struct ttm_buffer_object *tobj;
  811.  
  812.         if ((*robj) == NULL) {
  813.                 return;
  814.         }
  815.         tobj = &((*robj)->tobj);
  816.         ttm_bo_unref(&tobj);
  817.         if (tobj == NULL) {
  818.                 *robj = NULL;
  819.         }
  820. }
  821.  
  822. int radeon_object_mmap(struct radeon_object *robj, uint64_t *offset)
  823. {
  824.         *offset = robj->tobj.addr_space_offset;
  825.         return 0;
  826. }
  827.  
  828. int radeon_object_pin(struct radeon_object *robj, uint32_t domain,
  829.                       uint64_t *gpu_addr)
  830. {
  831.         uint32_t flags;
  832.         uint32_t tmp;
  833.         int r;
  834.  
  835.         flags = radeon_object_flags_from_domain(domain);
  836. //   spin_lock(&robj->tobj.lock);
  837.         if (robj->pin_count) {
  838.                 robj->pin_count++;
  839.                 if (gpu_addr != NULL) {
  840.                         *gpu_addr = robj->gpu_addr;
  841.                 }
  842. //       spin_unlock(&robj->tobj.lock);
  843.                 return 0;
  844.         }
  845. //   spin_unlock(&robj->tobj.lock);
  846.         r = radeon_object_reserve(robj, false);
  847.         if (unlikely(r != 0)) {
  848.                 DRM_ERROR("radeon: failed to reserve object for pinning it.\n");
  849.                 return r;
  850.         }
  851.         tmp = robj->tobj.mem.placement;
  852.         ttm_flag_masked(&tmp, flags, TTM_PL_MASK_MEM);
  853.         robj->tobj.proposed_placement = tmp | TTM_PL_FLAG_NO_EVICT | TTM_PL_MASK_CACHING;
  854.         r = ttm_buffer_object_validate(&robj->tobj,
  855.                                        robj->tobj.proposed_placement,
  856.                                        false, false);
  857.         radeon_object_gpu_addr(robj);
  858.         if (gpu_addr != NULL) {
  859.                 *gpu_addr = robj->gpu_addr;
  860.         }
  861.         robj->pin_count = 1;
  862.         if (unlikely(r != 0)) {
  863.                 DRM_ERROR("radeon: failed to pin object.\n");
  864.         }
  865.         radeon_object_unreserve(robj);
  866.         return r;
  867. }
  868.  
  869. void radeon_object_unpin(struct radeon_object *robj)
  870. {
  871.         uint32_t flags;
  872.         int r;
  873.  
  874. //   spin_lock(&robj->tobj.lock);
  875.         if (!robj->pin_count) {
  876. //       spin_unlock(&robj->tobj.lock);
  877.                 printk(KERN_WARNING "Unpin not necessary for %p !\n", robj);
  878.                 return;
  879.         }
  880.         robj->pin_count--;
  881.         if (robj->pin_count) {
  882. //       spin_unlock(&robj->tobj.lock);
  883.                 return;
  884.         }
  885. //   spin_unlock(&robj->tobj.lock);
  886.         r = radeon_object_reserve(robj, false);
  887.         if (unlikely(r != 0)) {
  888.                 DRM_ERROR("radeon: failed to reserve object for unpinning it.\n");
  889.                 return;
  890.         }
  891.         flags = robj->tobj.mem.placement;
  892.         robj->tobj.proposed_placement = flags & ~TTM_PL_FLAG_NO_EVICT;
  893.         r = ttm_buffer_object_validate(&robj->tobj,
  894.                                        robj->tobj.proposed_placement,
  895.                                        false, false);
  896.         if (unlikely(r != 0)) {
  897.                 DRM_ERROR("radeon: failed to unpin buffer.\n");
  898.         }
  899.         radeon_object_unreserve(robj);
  900. }
  901.  
  902. int radeon_object_wait(struct radeon_object *robj)
  903. {
  904.         int r = 0;
  905.  
  906.         /* FIXME: should use block reservation instead */
  907.         r = radeon_object_reserve(robj, true);
  908.         if (unlikely(r != 0)) {
  909.                 DRM_ERROR("radeon: failed to reserve object for waiting.\n");
  910.                 return r;
  911.         }
  912. //   spin_lock(&robj->tobj.lock);
  913.         if (robj->tobj.sync_obj) {
  914.                 r = ttm_bo_wait(&robj->tobj, true, false, false);
  915.         }
  916. //   spin_unlock(&robj->tobj.lock);
  917.         radeon_object_unreserve(robj);
  918.         return r;
  919. }
  920.  
  921. int radeon_object_evict_vram(struct radeon_device *rdev)
  922. {
  923.         if (rdev->flags & RADEON_IS_IGP) {
  924.                 /* Useless to evict on IGP chips */
  925.                 return 0;
  926.         }
  927.         return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM);
  928. }
  929.  
  930. void radeon_object_force_delete(struct radeon_device *rdev)
  931. {
  932.         struct radeon_object *robj, *n;
  933.         struct drm_gem_object *gobj;
  934.  
  935.         if (list_empty(&rdev->gem.objects)) {
  936.                 return;
  937.         }
  938.         DRM_ERROR("Userspace still has active objects !\n");
  939.         list_for_each_entry_safe(robj, n, &rdev->gem.objects, list) {
  940.                 mutex_lock(&rdev->ddev->struct_mutex);
  941.                 gobj = robj->gobj;
  942.                 DRM_ERROR("Force free for (%p,%p,%lu,%lu)\n",
  943.                           gobj, robj, (unsigned long)gobj->size,
  944.                           *((unsigned long *)&gobj->refcount));
  945.                 list_del_init(&robj->list);
  946.                 radeon_object_unref(&robj);
  947.                 gobj->driver_private = NULL;
  948.                 drm_gem_object_unreference(gobj);
  949.                 mutex_unlock(&rdev->ddev->struct_mutex);
  950.         }
  951. }
  952.  
  953. void radeon_object_fini(struct radeon_device *rdev)
  954. {
  955.         radeon_ttm_fini(rdev);
  956. }
  957.  
  958. void radeon_object_list_add_object(struct radeon_object_list *lobj,
  959.                                    struct list_head *head)
  960. {
  961.         if (lobj->wdomain) {
  962.                 list_add(&lobj->list, head);
  963.         } else {
  964.                 list_add_tail(&lobj->list, head);
  965.         }
  966. }
  967.  
  968. int radeon_object_list_reserve(struct list_head *head)
  969. {
  970.         struct radeon_object_list *lobj;
  971.         struct list_head *i;
  972.         int r;
  973.  
  974.         list_for_each(i, head) {
  975.                 lobj = list_entry(i, struct radeon_object_list, list);
  976.                 if (!lobj->robj->pin_count) {
  977.                         r = radeon_object_reserve(lobj->robj, true);
  978.                         if (unlikely(r != 0)) {
  979.                                 DRM_ERROR("radeon: failed to reserve object.\n");
  980.                                 return r;
  981.                         }
  982.                 } else {
  983.                 }
  984.         }
  985.         return 0;
  986. }
  987.  
  988. void radeon_object_list_unreserve(struct list_head *head)
  989. {
  990.         struct radeon_object_list *lobj;
  991.         struct list_head *i;
  992.  
  993.         list_for_each(i, head) {
  994.                 lobj = list_entry(i, struct radeon_object_list, list);
  995.                 if (!lobj->robj->pin_count) {
  996.                         radeon_object_unreserve(lobj->robj);
  997.                 } else {
  998.                 }
  999.         }
  1000. }
  1001.  
  1002. int radeon_object_list_validate(struct list_head *head, void *fence)
  1003. {
  1004.         struct radeon_object_list *lobj;
  1005.         struct radeon_object *robj;
  1006.         struct radeon_fence *old_fence = NULL;
  1007.         struct list_head *i;
  1008.         uint32_t flags;
  1009.         int r;
  1010.  
  1011.         r = radeon_object_list_reserve(head);
  1012.         if (unlikely(r != 0)) {
  1013.                 radeon_object_list_unreserve(head);
  1014.                 return r;
  1015.         }
  1016.         list_for_each(i, head) {
  1017.                 lobj = list_entry(i, struct radeon_object_list, list);
  1018.                 robj = lobj->robj;
  1019.                 if (lobj->wdomain) {
  1020.                         flags = radeon_object_flags_from_domain(lobj->wdomain);
  1021.                         flags |= TTM_PL_FLAG_TT;
  1022.                 } else {
  1023.                         flags = radeon_object_flags_from_domain(lobj->rdomain);
  1024.                         flags |= TTM_PL_FLAG_TT;
  1025.                         flags |= TTM_PL_FLAG_VRAM;
  1026.                 }
  1027.                 if (!robj->pin_count) {
  1028.                         robj->tobj.proposed_placement = flags | TTM_PL_MASK_CACHING;
  1029.                         r = ttm_buffer_object_validate(&robj->tobj,
  1030.                                                        robj->tobj.proposed_placement,
  1031.                                                        true, false);
  1032.                         if (unlikely(r)) {
  1033.                                 radeon_object_list_unreserve(head);
  1034.                                 DRM_ERROR("radeon: failed to validate.\n");
  1035.                                 return r;
  1036.                         }
  1037.                         radeon_object_gpu_addr(robj);
  1038.                 }
  1039.                 lobj->gpu_offset = robj->gpu_addr;
  1040.                 if (fence) {
  1041.                         old_fence = (struct radeon_fence *)robj->tobj.sync_obj;
  1042.                         robj->tobj.sync_obj = radeon_fence_ref(fence);
  1043.                         robj->tobj.sync_obj_arg = NULL;
  1044.                 }
  1045.                 if (old_fence) {
  1046.                         radeon_fence_unref(&old_fence);
  1047.                 }
  1048.         }
  1049.         return 0;
  1050. }
  1051.  
  1052. void radeon_object_list_unvalidate(struct list_head *head)
  1053. {
  1054.         struct radeon_object_list *lobj;
  1055.         struct radeon_fence *old_fence = NULL;
  1056.         struct list_head *i;
  1057.  
  1058.         list_for_each(i, head) {
  1059.                 lobj = list_entry(i, struct radeon_object_list, list);
  1060.                 old_fence = (struct radeon_fence *)lobj->robj->tobj.sync_obj;
  1061.                 lobj->robj->tobj.sync_obj = NULL;
  1062.                 if (old_fence) {
  1063.                         radeon_fence_unref(&old_fence);
  1064.                 }
  1065.         }
  1066.         radeon_object_list_unreserve(head);
  1067. }
  1068.  
  1069. void radeon_object_list_clean(struct list_head *head)
  1070. {
  1071.         radeon_object_list_unreserve(head);
  1072. }
  1073.  
  1074. int radeon_object_fbdev_mmap(struct radeon_object *robj,
  1075.                              struct vm_area_struct *vma)
  1076. {
  1077.         return ttm_fbdev_mmap(vma, &robj->tobj);
  1078. }
  1079.  
  1080. unsigned long radeon_object_size(struct radeon_object *robj)
  1081. {
  1082.         return robj->tobj.num_pages << PAGE_SHIFT;
  1083. }
  1084.  
  1085.  
  1086. #endif
  1087.