Subversion Repositories Kolibri OS

Rev

Blame | Last modification | View Log | RSS feed

  1.  
  2. #include "util/u_inlines.h"
  3. #include "util/u_memory.h"
  4. #include "util/u_math.h"
  5. #include "util/u_surface.h"
  6.  
  7. #include "nouveau_screen.h"
  8. #include "nouveau_context.h"
  9. #include "nouveau_winsys.h"
  10. #include "nouveau_fence.h"
  11. #include "nouveau_buffer.h"
  12. #include "nouveau_mm.h"
  13.  
  14. #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
  15.  
  16. struct nouveau_transfer {
  17.    struct pipe_transfer base;
  18.  
  19.    uint8_t *map;
  20.    struct nouveau_bo *bo;
  21.    struct nouveau_mm_allocation *mm;
  22.    uint32_t offset;
  23. };
  24.  
  25. static INLINE struct nouveau_transfer *
  26. nouveau_transfer(struct pipe_transfer *transfer)
  27. {
  28.    return (struct nouveau_transfer *)transfer;
  29. }
  30.  
  31. static INLINE boolean
  32. nouveau_buffer_malloc(struct nv04_resource *buf)
  33. {
  34.    if (!buf->data)
  35.       buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  36.    return !!buf->data;
  37. }
  38.  
  39. static INLINE boolean
  40. nouveau_buffer_allocate(struct nouveau_screen *screen,
  41.                         struct nv04_resource *buf, unsigned domain)
  42. {
  43.    uint32_t size = buf->base.width0;
  44.  
  45.    if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
  46.                          PIPE_BIND_COMPUTE_RESOURCE |
  47.                          PIPE_BIND_SHADER_RESOURCE))
  48.       size = align(size, 0x100);
  49.  
  50.    if (domain == NOUVEAU_BO_VRAM) {
  51.       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  52.                                     &buf->bo, &buf->offset);
  53.       if (!buf->bo)
  54.          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  55.       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
  56.    } else
  57.    if (domain == NOUVEAU_BO_GART) {
  58.       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  59.                                     &buf->bo, &buf->offset);
  60.       if (!buf->bo)
  61.          return FALSE;
  62.       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
  63.    } else {
  64.       assert(domain == 0);
  65.       if (!nouveau_buffer_malloc(buf))
  66.          return FALSE;
  67.    }
  68.    buf->domain = domain;
  69.    if (buf->bo)
  70.       buf->address = buf->bo->offset + buf->offset;
  71.  
  72.    util_range_set_empty(&buf->valid_buffer_range);
  73.  
  74.    return TRUE;
  75. }
  76.  
  77. static INLINE void
  78. release_allocation(struct nouveau_mm_allocation **mm,
  79.                    struct nouveau_fence *fence)
  80. {
  81.    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  82.    (*mm) = NULL;
  83. }
  84.  
  85. INLINE void
  86. nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  87. {
  88.    nouveau_bo_ref(NULL, &buf->bo);
  89.  
  90.    if (buf->mm)
  91.       release_allocation(&buf->mm, buf->fence);
  92.  
  93.    if (buf->domain == NOUVEAU_BO_VRAM)
  94.       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
  95.    if (buf->domain == NOUVEAU_BO_GART)
  96.       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
  97.  
  98.    buf->domain = 0;
  99. }
  100.  
  101. static INLINE boolean
  102. nouveau_buffer_reallocate(struct nouveau_screen *screen,
  103.                           struct nv04_resource *buf, unsigned domain)
  104. {
  105.    nouveau_buffer_release_gpu_storage(buf);
  106.  
  107.    nouveau_fence_ref(NULL, &buf->fence);
  108.    nouveau_fence_ref(NULL, &buf->fence_wr);
  109.  
  110.    buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  111.  
  112.    return nouveau_buffer_allocate(screen, buf, domain);
  113. }
  114.  
  115. static void
  116. nouveau_buffer_destroy(struct pipe_screen *pscreen,
  117.                        struct pipe_resource *presource)
  118. {
  119.    struct nv04_resource *res = nv04_resource(presource);
  120.  
  121.    nouveau_buffer_release_gpu_storage(res);
  122.  
  123.    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  124.       align_free(res->data);
  125.  
  126.    nouveau_fence_ref(NULL, &res->fence);
  127.    nouveau_fence_ref(NULL, &res->fence_wr);
  128.  
  129.    util_range_destroy(&res->valid_buffer_range);
  130.  
  131.    FREE(res);
  132.  
  133.    NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
  134. }
  135.  
  136. /* Set up a staging area for the transfer. This is either done in "regular"
  137.  * system memory if the driver supports push_data (nv50+) and the data is
  138.  * small enough (and permit_pb == true), or in GART memory.
  139.  */
  140. static uint8_t *
  141. nouveau_transfer_staging(struct nouveau_context *nv,
  142.                          struct nouveau_transfer *tx, boolean permit_pb)
  143. {
  144.    const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
  145.    const unsigned size = align(tx->base.box.width, 4) + adj;
  146.  
  147.    if (!nv->push_data)
  148.       permit_pb = FALSE;
  149.  
  150.    if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
  151.       tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  152.       if (tx->map)
  153.          tx->map += adj;
  154.    } else {
  155.       tx->mm =
  156.          nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
  157.       if (tx->bo) {
  158.          tx->offset += adj;
  159.          if (!nouveau_bo_map(tx->bo, 0, NULL))
  160.             tx->map = (uint8_t *)tx->bo->map + tx->offset;
  161.       }
  162.    }
  163.    return tx->map;
  164. }
  165.  
  166. /* Copies data from the resource into the the transfer's temporary GART
  167.  * buffer. Also updates buf->data if present.
  168.  *
  169.  * Maybe just migrate to GART right away if we actually need to do this. */
  170. static boolean
  171. nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
  172. {
  173.    struct nv04_resource *buf = nv04_resource(tx->base.resource);
  174.    const unsigned base = tx->base.box.x;
  175.    const unsigned size = tx->base.box.width;
  176.  
  177.    NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
  178.  
  179.    nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
  180.                  buf->bo, buf->offset + base, buf->domain, size);
  181.  
  182.    if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
  183.       return FALSE;
  184.  
  185.    if (buf->data)
  186.       memcpy(buf->data + base, tx->map, size);
  187.  
  188.    return TRUE;
  189. }
  190.  
  191. static void
  192. nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
  193.                        unsigned offset, unsigned size)
  194. {
  195.    struct nv04_resource *buf = nv04_resource(tx->base.resource);
  196.    uint8_t *data = tx->map + offset;
  197.    const unsigned base = tx->base.box.x + offset;
  198.    const boolean can_cb = !((base | size) & 3);
  199.  
  200.    if (buf->data)
  201.       memcpy(data, buf->data + base, size);
  202.    else
  203.       buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
  204.  
  205.    if (buf->domain == NOUVEAU_BO_VRAM)
  206.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
  207.    if (buf->domain == NOUVEAU_BO_GART)
  208.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
  209.  
  210.    if (tx->bo)
  211.       nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
  212.                     tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
  213.    else
  214.    if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
  215.       nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
  216.                   base, size / 4, (const uint32_t *)data);
  217.    else
  218.       nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
  219.  
  220.    nouveau_fence_ref(nv->screen->fence.current, &buf->fence);
  221.    nouveau_fence_ref(nv->screen->fence.current, &buf->fence_wr);
  222. }
  223.  
  224. /* Does a CPU wait for the buffer's backing data to become reliably accessible
  225.  * for write/read by waiting on the buffer's relevant fences.
  226.  */
  227. static INLINE boolean
  228. nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
  229. {
  230.    if (rw == PIPE_TRANSFER_READ) {
  231.       if (!buf->fence_wr)
  232.          return TRUE;
  233.       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
  234.                            !nouveau_fence_signalled(buf->fence_wr));
  235.       if (!nouveau_fence_wait(buf->fence_wr))
  236.          return FALSE;
  237.    } else {
  238.       if (!buf->fence)
  239.          return TRUE;
  240.       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
  241.                            !nouveau_fence_signalled(buf->fence));
  242.       if (!nouveau_fence_wait(buf->fence))
  243.          return FALSE;
  244.  
  245.       nouveau_fence_ref(NULL, &buf->fence);
  246.    }
  247.    nouveau_fence_ref(NULL, &buf->fence_wr);
  248.  
  249.    return TRUE;
  250. }
  251.  
  252. static INLINE boolean
  253. nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
  254. {
  255.    if (rw == PIPE_TRANSFER_READ)
  256.       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
  257.    else
  258.       return (buf->fence && !nouveau_fence_signalled(buf->fence));
  259. }
  260.  
  261. static INLINE void
  262. nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
  263.                              struct pipe_resource *resource,
  264.                              const struct pipe_box *box,
  265.                              unsigned usage)
  266. {
  267.    tx->base.resource = resource;
  268.    tx->base.level = 0;
  269.    tx->base.usage = usage;
  270.    tx->base.box.x = box->x;
  271.    tx->base.box.y = 0;
  272.    tx->base.box.z = 0;
  273.    tx->base.box.width = box->width;
  274.    tx->base.box.height = 1;
  275.    tx->base.box.depth = 1;
  276.    tx->base.stride = 0;
  277.    tx->base.layer_stride = 0;
  278.  
  279.    tx->bo = NULL;
  280.    tx->map = NULL;
  281. }
  282.  
  283. static INLINE void
  284. nouveau_buffer_transfer_del(struct nouveau_context *nv,
  285.                             struct nouveau_transfer *tx)
  286. {
  287.    if (tx->map) {
  288.       if (likely(tx->bo)) {
  289.          nouveau_bo_ref(NULL, &tx->bo);
  290.          if (tx->mm)
  291.             release_allocation(&tx->mm, nv->screen->fence.current);
  292.       } else {
  293.          align_free(tx->map -
  294.                     (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
  295.       }
  296.    }
  297. }
  298.  
  299. /* Creates a cache in system memory of the buffer data. */
  300. static boolean
  301. nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
  302. {
  303.    struct nouveau_transfer tx;
  304.    boolean ret;
  305.    tx.base.resource = &buf->base;
  306.    tx.base.box.x = 0;
  307.    tx.base.box.width = buf->base.width0;
  308.    tx.bo = NULL;
  309.    tx.map = NULL;
  310.  
  311.    if (!buf->data)
  312.       if (!nouveau_buffer_malloc(buf))
  313.          return FALSE;
  314.    if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
  315.       return TRUE;
  316.    nv->stats.buf_cache_count++;
  317.  
  318.    if (!nouveau_transfer_staging(nv, &tx, FALSE))
  319.       return FALSE;
  320.  
  321.    ret = nouveau_transfer_read(nv, &tx);
  322.    if (ret) {
  323.       buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
  324.       memcpy(buf->data, tx.map, buf->base.width0);
  325.    }
  326.    nouveau_buffer_transfer_del(nv, &tx);
  327.    return ret;
  328. }
  329.  
  330.  
  331. #define NOUVEAU_TRANSFER_DISCARD \
  332.    (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
  333.  
  334. /* Checks whether it is possible to completely discard the memory backing this
  335.  * resource. This can be useful if we would otherwise have to wait for a read
  336.  * operation to complete on this data.
  337.  */
  338. static INLINE boolean
  339. nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
  340. {
  341.    if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
  342.       return FALSE;
  343.    if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
  344.       return FALSE;
  345.    if (unlikely(usage & PIPE_TRANSFER_PERSISTENT))
  346.       return FALSE;
  347.    return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
  348. }
  349.  
  350. /* Returns a pointer to a memory area representing a window into the
  351.  * resource's data.
  352.  *
  353.  * This may or may not be the _actual_ memory area of the resource. However
  354.  * when calling nouveau_buffer_transfer_unmap, if it wasn't the actual memory
  355.  * area, the contents of the returned map are copied over to the resource.
  356.  *
  357.  * The usage indicates what the caller plans to do with the map:
  358.  *
  359.  *   WRITE means that the user plans to write to it
  360.  *
  361.  *   READ means that the user plans on reading from it
  362.  *
  363.  *   DISCARD_WHOLE_RESOURCE means that the whole resource is going to be
  364.  *   potentially overwritten, and even if it isn't, the bits that aren't don't
  365.  *   need to be maintained.
  366.  *
  367.  *   DISCARD_RANGE means that all the data in the specified range is going to
  368.  *   be overwritten.
  369.  *
  370.  * The strategy for determining what kind of memory area to return is complex,
  371.  * see comments inside of the function.
  372.  */
  373. static void *
  374. nouveau_buffer_transfer_map(struct pipe_context *pipe,
  375.                             struct pipe_resource *resource,
  376.                             unsigned level, unsigned usage,
  377.                             const struct pipe_box *box,
  378.                             struct pipe_transfer **ptransfer)
  379. {
  380.    struct nouveau_context *nv = nouveau_context(pipe);
  381.    struct nv04_resource *buf = nv04_resource(resource);
  382.    struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
  383.    uint8_t *map;
  384.    int ret;
  385.  
  386.    if (!tx)
  387.       return NULL;
  388.    nouveau_buffer_transfer_init(tx, resource, box, usage);
  389.    *ptransfer = &tx->base;
  390.  
  391.    if (usage & PIPE_TRANSFER_READ)
  392.       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
  393.    if (usage & PIPE_TRANSFER_WRITE)
  394.       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
  395.  
  396.    /* If we are trying to write to an uninitialized range, the user shouldn't
  397.     * care what was there before. So we can treat the write as if the target
  398.     * range were being discarded. Furthermore, since we know that even if this
  399.     * buffer is busy due to GPU activity, because the contents were
  400.     * uninitialized, the GPU can't care what was there, and so we can treat
  401.     * the write as being unsynchronized.
  402.     */
  403.    if ((usage & PIPE_TRANSFER_WRITE) &&
  404.        !util_ranges_intersect(&buf->valid_buffer_range, box->x, box->x + box->width))
  405.       usage |= PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_UNSYNCHRONIZED;
  406.  
  407.    if (usage & PIPE_TRANSFER_PERSISTENT)
  408.       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
  409.  
  410.    if (buf->domain == NOUVEAU_BO_VRAM) {
  411.       if (usage & NOUVEAU_TRANSFER_DISCARD) {
  412.          /* Set up a staging area for the user to write to. It will be copied
  413.           * back into VRAM on unmap. */
  414.          if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
  415.             buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  416.          nouveau_transfer_staging(nv, tx, TRUE);
  417.       } else {
  418.          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
  419.             /* The GPU is currently writing to this buffer. Copy its current
  420.              * contents to a staging area in the GART. This is necessary since
  421.              * not the whole area being mapped is being discarded.
  422.              */
  423.             if (buf->data) {
  424.                align_free(buf->data);
  425.                buf->data = NULL;
  426.             }
  427.             nouveau_transfer_staging(nv, tx, FALSE);
  428.             nouveau_transfer_read(nv, tx);
  429.          } else {
  430.             /* The buffer is currently idle. Create a staging area for writes,
  431.              * and make sure that the cached data is up-to-date. */
  432.             if (usage & PIPE_TRANSFER_WRITE)
  433.                nouveau_transfer_staging(nv, tx, TRUE);
  434.             if (!buf->data)
  435.                nouveau_buffer_cache(nv, buf);
  436.          }
  437.       }
  438.       return buf->data ? (buf->data + box->x) : tx->map;
  439.    } else
  440.    if (unlikely(buf->domain == 0)) {
  441.       return buf->data + box->x;
  442.    }
  443.  
  444.    /* At this point, buf->domain == GART */
  445.  
  446.    if (nouveau_buffer_should_discard(buf, usage)) {
  447.       int ref = buf->base.reference.count - 1;
  448.       nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
  449.       if (ref > 0) /* any references inside context possible ? */
  450.          nv->invalidate_resource_storage(nv, &buf->base, ref);
  451.    }
  452.  
  453.    /* Note that nouveau_bo_map ends up doing a nouveau_bo_wait with the
  454.     * relevant flags. If buf->mm is set, that means this resource is part of a
  455.     * larger slab bo that holds multiple resources. So in that case, don't
  456.     * wait on the whole slab and instead use the logic below to return a
  457.     * reasonable buffer for that case.
  458.     */
  459.    ret = nouveau_bo_map(buf->bo,
  460.                         buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
  461.                         nv->client);
  462.    if (ret) {
  463.       FREE(tx);
  464.       return NULL;
  465.    }
  466.    map = (uint8_t *)buf->bo->map + buf->offset + box->x;
  467.  
  468.    /* using kernel fences only if !buf->mm */
  469.    if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
  470.       return map;
  471.  
  472.    /* If the GPU is currently reading/writing this buffer, we shouldn't
  473.     * interfere with its progress. So instead we either wait for the GPU to
  474.     * complete its operation, or set up a staging area to perform our work in.
  475.     */
  476.    if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
  477.       if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
  478.          /* Discarding was not possible, must sync because
  479.           * subsequent transfers might use UNSYNCHRONIZED. */
  480.          nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
  481.       } else
  482.       if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
  483.          /* The whole range is being discarded, so it doesn't matter what was
  484.           * there before. No need to copy anything over. */
  485.          nouveau_transfer_staging(nv, tx, TRUE);
  486.          map = tx->map;
  487.       } else
  488.       if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
  489.          if (usage & PIPE_TRANSFER_DONTBLOCK)
  490.             map = NULL;
  491.          else
  492.             nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
  493.       } else {
  494.          /* It is expected that the returned buffer be a representation of the
  495.           * data in question, so we must copy it over from the buffer. */
  496.          nouveau_transfer_staging(nv, tx, TRUE);
  497.          if (tx->map)
  498.             memcpy(tx->map, map, box->width);
  499.          map = tx->map;
  500.       }
  501.    }
  502.    if (!map)
  503.       FREE(tx);
  504.    return map;
  505. }
  506.  
  507.  
  508.  
  509. static void
  510. nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
  511.                                      struct pipe_transfer *transfer,
  512.                                      const struct pipe_box *box)
  513. {
  514.    struct nouveau_transfer *tx = nouveau_transfer(transfer);
  515.    struct nv04_resource *buf = nv04_resource(transfer->resource);
  516.  
  517.    if (tx->map)
  518.       nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
  519.  
  520.    util_range_add(&buf->valid_buffer_range,
  521.                   tx->base.box.x + box->x,
  522.                   tx->base.box.x + box->x + box->width);
  523. }
  524.  
  525. /* Unmap stage of the transfer. If it was a WRITE transfer and the map that
  526.  * was returned was not the real resource's data, this needs to transfer the
  527.  * data back to the resource.
  528.  *
  529.  * Also marks vbo dirty based on the buffer's binding
  530.  */
  531. static void
  532. nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
  533.                               struct pipe_transfer *transfer)
  534. {
  535.    struct nouveau_context *nv = nouveau_context(pipe);
  536.    struct nouveau_transfer *tx = nouveau_transfer(transfer);
  537.    struct nv04_resource *buf = nv04_resource(transfer->resource);
  538.  
  539.    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
  540.       if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
  541.          nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
  542.  
  543.       if (likely(buf->domain)) {
  544.          const uint8_t bind = buf->base.bind;
  545.          /* make sure we invalidate dedicated caches */
  546.          if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
  547.             nv->vbo_dirty = TRUE;
  548.       }
  549.  
  550.       util_range_add(&buf->valid_buffer_range,
  551.                      tx->base.box.x, tx->base.box.x + tx->base.box.width);
  552.    }
  553.  
  554.    if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
  555.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
  556.  
  557.    nouveau_buffer_transfer_del(nv, tx);
  558.    FREE(tx);
  559. }
  560.  
  561.  
  562. void
  563. nouveau_copy_buffer(struct nouveau_context *nv,
  564.                     struct nv04_resource *dst, unsigned dstx,
  565.                     struct nv04_resource *src, unsigned srcx, unsigned size)
  566. {
  567.    assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
  568.  
  569.    if (likely(dst->domain) && likely(src->domain)) {
  570.       nv->copy_data(nv,
  571.                     dst->bo, dst->offset + dstx, dst->domain,
  572.                     src->bo, src->offset + srcx, src->domain, size);
  573.  
  574.       dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
  575.       nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
  576.       nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
  577.  
  578.       src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
  579.       nouveau_fence_ref(nv->screen->fence.current, &src->fence);
  580.    } else {
  581.       struct pipe_box src_box;
  582.       src_box.x = srcx;
  583.       src_box.y = 0;
  584.       src_box.z = 0;
  585.       src_box.width = size;
  586.       src_box.height = 1;
  587.       src_box.depth = 1;
  588.       util_resource_copy_region(&nv->pipe,
  589.                                 &dst->base, 0, dstx, 0, 0,
  590.                                 &src->base, 0, &src_box);
  591.    }
  592.  
  593.    util_range_add(&dst->valid_buffer_range, dstx, dstx + size);
  594. }
  595.  
  596.  
  597. void *
  598. nouveau_resource_map_offset(struct nouveau_context *nv,
  599.                             struct nv04_resource *res, uint32_t offset,
  600.                             uint32_t flags)
  601. {
  602.    if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  603.       return res->data + offset;
  604.  
  605.    if (res->domain == NOUVEAU_BO_VRAM) {
  606.       if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
  607.          nouveau_buffer_cache(nv, res);
  608.    }
  609.    if (res->domain != NOUVEAU_BO_GART)
  610.       return res->data + offset;
  611.  
  612.    if (res->mm) {
  613.       unsigned rw;
  614.       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
  615.       nouveau_buffer_sync(res, rw);
  616.       if (nouveau_bo_map(res->bo, 0, NULL))
  617.          return NULL;
  618.    } else {
  619.       if (nouveau_bo_map(res->bo, flags, nv->client))
  620.          return NULL;
  621.    }
  622.    return (uint8_t *)res->bo->map + res->offset + offset;
  623. }
  624.  
  625.  
  626. const struct u_resource_vtbl nouveau_buffer_vtbl =
  627. {
  628.    u_default_resource_get_handle,     /* get_handle */
  629.    nouveau_buffer_destroy,               /* resource_destroy */
  630.    nouveau_buffer_transfer_map,          /* transfer_map */
  631.    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
  632.    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
  633.    u_default_transfer_inline_write    /* transfer_inline_write */
  634. };
  635.  
  636. struct pipe_resource *
  637. nouveau_buffer_create(struct pipe_screen *pscreen,
  638.                       const struct pipe_resource *templ)
  639. {
  640.    struct nouveau_screen *screen = nouveau_screen(pscreen);
  641.    struct nv04_resource *buffer;
  642.    boolean ret;
  643.  
  644.    buffer = CALLOC_STRUCT(nv04_resource);
  645.    if (!buffer)
  646.       return NULL;
  647.  
  648.    buffer->base = *templ;
  649.    buffer->vtbl = &nouveau_buffer_vtbl;
  650.    pipe_reference_init(&buffer->base.reference, 1);
  651.    buffer->base.screen = pscreen;
  652.  
  653.    if (buffer->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
  654.                              PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
  655.       buffer->domain = NOUVEAU_BO_GART;
  656.    } else if (buffer->base.bind &
  657.               (screen->vidmem_bindings & screen->sysmem_bindings)) {
  658.       switch (buffer->base.usage) {
  659.       case PIPE_USAGE_DEFAULT:
  660.       case PIPE_USAGE_IMMUTABLE:
  661.          buffer->domain = NOUVEAU_BO_VRAM;
  662.          break;
  663.       case PIPE_USAGE_DYNAMIC:
  664.          /* For most apps, we'd have to do staging transfers to avoid sync
  665.           * with this usage, and GART -> GART copies would be suboptimal.
  666.           */
  667.          buffer->domain = NOUVEAU_BO_VRAM;
  668.          break;
  669.       case PIPE_USAGE_STAGING:
  670.       case PIPE_USAGE_STREAM:
  671.          buffer->domain = NOUVEAU_BO_GART;
  672.          break;
  673.       default:
  674.          assert(0);
  675.          break;
  676.       }
  677.    } else {
  678.       if (buffer->base.bind & screen->vidmem_bindings)
  679.          buffer->domain = NOUVEAU_BO_VRAM;
  680.       else
  681.       if (buffer->base.bind & screen->sysmem_bindings)
  682.          buffer->domain = NOUVEAU_BO_GART;
  683.    }
  684.    ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
  685.  
  686.    if (ret == FALSE)
  687.       goto fail;
  688.  
  689.    if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
  690.       nouveau_buffer_cache(NULL, buffer);
  691.  
  692.    NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
  693.  
  694.    util_range_init(&buffer->valid_buffer_range);
  695.  
  696.    return &buffer->base;
  697.  
  698. fail:
  699.    FREE(buffer);
  700.    return NULL;
  701. }
  702.  
  703.  
  704. struct pipe_resource *
  705. nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
  706.                            unsigned bytes, unsigned bind)
  707. {
  708.    struct nv04_resource *buffer;
  709.  
  710.    buffer = CALLOC_STRUCT(nv04_resource);
  711.    if (!buffer)
  712.       return NULL;
  713.  
  714.    pipe_reference_init(&buffer->base.reference, 1);
  715.    buffer->vtbl = &nouveau_buffer_vtbl;
  716.    buffer->base.screen = pscreen;
  717.    buffer->base.format = PIPE_FORMAT_R8_UNORM;
  718.    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
  719.    buffer->base.bind = bind;
  720.    buffer->base.width0 = bytes;
  721.    buffer->base.height0 = 1;
  722.    buffer->base.depth0 = 1;
  723.  
  724.    buffer->data = ptr;
  725.    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
  726.  
  727.    util_range_init(&buffer->valid_buffer_range);
  728.    util_range_add(&buffer->valid_buffer_range, 0, bytes);
  729.  
  730.    return &buffer->base;
  731. }
  732.  
  733. static INLINE boolean
  734. nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
  735.                           struct nouveau_bo *bo, unsigned offset, unsigned size)
  736. {
  737.    if (!nouveau_buffer_malloc(buf))
  738.       return FALSE;
  739.    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
  740.       return FALSE;
  741.    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
  742.    return TRUE;
  743. }
  744.  
  745. /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
  746. boolean
  747. nouveau_buffer_migrate(struct nouveau_context *nv,
  748.                        struct nv04_resource *buf, const unsigned new_domain)
  749. {
  750.    struct nouveau_screen *screen = nv->screen;
  751.    struct nouveau_bo *bo;
  752.    const unsigned old_domain = buf->domain;
  753.    unsigned size = buf->base.width0;
  754.    unsigned offset;
  755.    int ret;
  756.  
  757.    assert(new_domain != old_domain);
  758.  
  759.    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
  760.       if (!nouveau_buffer_allocate(screen, buf, new_domain))
  761.          return FALSE;
  762.       ret = nouveau_bo_map(buf->bo, 0, nv->client);
  763.       if (ret)
  764.          return ret;
  765.       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
  766.       align_free(buf->data);
  767.    } else
  768.    if (old_domain != 0 && new_domain != 0) {
  769.       struct nouveau_mm_allocation *mm = buf->mm;
  770.  
  771.       if (new_domain == NOUVEAU_BO_VRAM) {
  772.          /* keep a system memory copy of our data in case we hit a fallback */
  773.          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
  774.             return FALSE;
  775.          if (nouveau_mesa_debug)
  776.             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
  777.       }
  778.  
  779.       offset = buf->offset;
  780.       bo = buf->bo;
  781.       buf->bo = NULL;
  782.       buf->mm = NULL;
  783.       nouveau_buffer_allocate(screen, buf, new_domain);
  784.  
  785.       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
  786.                     bo, offset, old_domain, buf->base.width0);
  787.  
  788.       nouveau_bo_ref(NULL, &bo);
  789.       if (mm)
  790.          release_allocation(&mm, screen->fence.current);
  791.    } else
  792.    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
  793.       struct nouveau_transfer tx;
  794.       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
  795.          return FALSE;
  796.       tx.base.resource = &buf->base;
  797.       tx.base.box.x = 0;
  798.       tx.base.box.width = buf->base.width0;
  799.       tx.bo = NULL;
  800.       tx.map = NULL;
  801.       if (!nouveau_transfer_staging(nv, &tx, FALSE))
  802.          return FALSE;
  803.       nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
  804.       nouveau_buffer_transfer_del(nv, &tx);
  805.    } else
  806.       return FALSE;
  807.  
  808.    assert(buf->domain == new_domain);
  809.    return TRUE;
  810. }
  811.  
  812. /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
  813.  * We'd like to only allocate @size bytes here, but then we'd have to rebase
  814.  * the vertex indices ...
  815.  */
  816. boolean
  817. nouveau_user_buffer_upload(struct nouveau_context *nv,
  818.                            struct nv04_resource *buf,
  819.                            unsigned base, unsigned size)
  820. {
  821.    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
  822.    int ret;
  823.  
  824.    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
  825.  
  826.    buf->base.width0 = base + size;
  827.    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
  828.       return FALSE;
  829.  
  830.    ret = nouveau_bo_map(buf->bo, 0, nv->client);
  831.    if (ret)
  832.       return FALSE;
  833.    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
  834.  
  835.    return TRUE;
  836. }
  837.  
  838.  
  839. /* Scratch data allocation. */
  840.  
  841. static INLINE int
  842. nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
  843.                          unsigned size)
  844. {
  845.    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
  846.                          4096, size, NULL, pbo);
  847. }
  848.  
  849. static void
  850. nouveau_scratch_unref_bos(void *d)
  851. {
  852.    struct runout *b = d;
  853.    int i;
  854.  
  855.    for (i = 0; i < b->nr; ++i)
  856.       nouveau_bo_ref(NULL, &b->bo[i]);
  857.  
  858.    FREE(b);
  859. }
  860.  
  861. void
  862. nouveau_scratch_runout_release(struct nouveau_context *nv)
  863. {
  864.    if (!nv->scratch.runout)
  865.       return;
  866.  
  867.    if (!nouveau_fence_work(nv->screen->fence.current, nouveau_scratch_unref_bos,
  868.          nv->scratch.runout))
  869.       return;
  870.  
  871.    nv->scratch.end = 0;
  872.    nv->scratch.runout = NULL;
  873. }
  874.  
  875. /* Allocate an extra bo if we can't fit everything we need simultaneously.
  876.  * (Could happen for very large user arrays.)
  877.  */
  878. static INLINE boolean
  879. nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
  880. {
  881.    int ret;
  882.    unsigned n;
  883.  
  884.    if (nv->scratch.runout)
  885.       n = nv->scratch.runout->nr;
  886.    else
  887.       n = 0;
  888.    nv->scratch.runout = REALLOC(nv->scratch.runout, n == 0 ? 0 :
  889.                                 (sizeof(*nv->scratch.runout) + (n + 0) * sizeof(void *)),
  890.                                  sizeof(*nv->scratch.runout) + (n + 1) * sizeof(void *));
  891.    nv->scratch.runout->nr = n + 1;
  892.    nv->scratch.runout->bo[n] = NULL;
  893.  
  894.    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout->bo[n], size);
  895.    if (!ret) {
  896.       ret = nouveau_bo_map(nv->scratch.runout->bo[n], 0, NULL);
  897.       if (ret)
  898.          nouveau_bo_ref(NULL, &nv->scratch.runout->bo[--nv->scratch.runout->nr]);
  899.    }
  900.    if (!ret) {
  901.       nv->scratch.current = nv->scratch.runout->bo[n];
  902.       nv->scratch.offset = 0;
  903.       nv->scratch.end = size;
  904.       nv->scratch.map = nv->scratch.current->map;
  905.    }
  906.    return !ret;
  907. }
  908.  
  909. /* Continue to next scratch buffer, if available (no wrapping, large enough).
  910.  * Allocate it if it has not yet been created.
  911.  */
  912. static INLINE boolean
  913. nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
  914. {
  915.    struct nouveau_bo *bo;
  916.    int ret;
  917.    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
  918.  
  919.    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
  920.       return FALSE;
  921.    nv->scratch.id = i;
  922.  
  923.    bo = nv->scratch.bo[i];
  924.    if (!bo) {
  925.       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
  926.       if (ret)
  927.          return FALSE;
  928.       nv->scratch.bo[i] = bo;
  929.    }
  930.    nv->scratch.current = bo;
  931.    nv->scratch.offset = 0;
  932.    nv->scratch.end = nv->scratch.bo_size;
  933.  
  934.    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
  935.    if (!ret)
  936.       nv->scratch.map = bo->map;
  937.    return !ret;
  938. }
  939.  
  940. static boolean
  941. nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
  942. {
  943.    boolean ret;
  944.  
  945.    ret = nouveau_scratch_next(nv, min_size);
  946.    if (!ret)
  947.       ret = nouveau_scratch_runout(nv, min_size);
  948.    return ret;
  949. }
  950.  
  951.  
  952. /* Copy data to a scratch buffer and return address & bo the data resides in. */
  953. uint64_t
  954. nouveau_scratch_data(struct nouveau_context *nv,
  955.                      const void *data, unsigned base, unsigned size,
  956.                      struct nouveau_bo **bo)
  957. {
  958.    unsigned bgn = MAX2(base, nv->scratch.offset);
  959.    unsigned end = bgn + size;
  960.  
  961.    if (end >= nv->scratch.end) {
  962.       end = base + size;
  963.       if (!nouveau_scratch_more(nv, end))
  964.          return 0;
  965.       bgn = base;
  966.    }
  967.    nv->scratch.offset = align(end, 4);
  968.  
  969.    memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
  970.  
  971.    *bo = nv->scratch.current;
  972.    return (*bo)->offset + (bgn - base);
  973. }
  974.  
  975. void *
  976. nouveau_scratch_get(struct nouveau_context *nv,
  977.                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
  978. {
  979.    unsigned bgn = nv->scratch.offset;
  980.    unsigned end = nv->scratch.offset + size;
  981.  
  982.    if (end >= nv->scratch.end) {
  983.       end = size;
  984.       if (!nouveau_scratch_more(nv, end))
  985.          return NULL;
  986.       bgn = 0;
  987.    }
  988.    nv->scratch.offset = align(end, 4);
  989.  
  990.    *pbo = nv->scratch.current;
  991.    *gpu_addr = nv->scratch.current->offset + bgn;
  992.    return nv->scratch.map + bgn;
  993. }
  994.