Subversion Repositories Kolibri OS

Rev

Go to most recent revision | Blame | Last modification | View Log | RSS feed

  1.  
  2. #include "util/u_inlines.h"
  3. #include "util/u_memory.h"
  4. #include "util/u_math.h"
  5. #include "util/u_surface.h"
  6.  
  7. #include "nouveau_screen.h"
  8. #include "nouveau_context.h"
  9. #include "nouveau_winsys.h"
  10. #include "nouveau_fence.h"
  11. #include "nouveau_buffer.h"
  12. #include "nouveau_mm.h"
  13.  
  14. #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
  15.  
  16. struct nouveau_transfer {
  17.    struct pipe_transfer base;
  18.  
  19.    uint8_t *map;
  20.    struct nouveau_bo *bo;
  21.    struct nouveau_mm_allocation *mm;
  22.    uint32_t offset;
  23. };
  24.  
  25. static INLINE struct nouveau_transfer *
  26. nouveau_transfer(struct pipe_transfer *transfer)
  27. {
  28.    return (struct nouveau_transfer *)transfer;
  29. }
  30.  
  31. static INLINE boolean
  32. nouveau_buffer_malloc(struct nv04_resource *buf)
  33. {
  34.    if (!buf->data)
  35.       buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  36.    return !!buf->data;
  37. }
  38.  
  39. static INLINE boolean
  40. nouveau_buffer_allocate(struct nouveau_screen *screen,
  41.                         struct nv04_resource *buf, unsigned domain)
  42. {
  43.    uint32_t size = buf->base.width0;
  44.  
  45.    if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
  46.                          PIPE_BIND_COMPUTE_RESOURCE |
  47.                          PIPE_BIND_SHADER_RESOURCE))
  48.       size = align(size, 0x100);
  49.  
  50.    if (domain == NOUVEAU_BO_VRAM) {
  51.       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  52.                                     &buf->bo, &buf->offset);
  53.       if (!buf->bo)
  54.          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  55.       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
  56.    } else
  57.    if (domain == NOUVEAU_BO_GART) {
  58.       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  59.                                     &buf->bo, &buf->offset);
  60.       if (!buf->bo)
  61.          return FALSE;
  62.       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
  63.    } else {
  64.       assert(domain == 0);
  65.       if (!nouveau_buffer_malloc(buf))
  66.          return FALSE;
  67.    }
  68.    buf->domain = domain;
  69.    if (buf->bo)
  70.       buf->address = buf->bo->offset + buf->offset;
  71.  
  72.    return TRUE;
  73. }
  74.  
  75. static INLINE void
  76. release_allocation(struct nouveau_mm_allocation **mm,
  77.                    struct nouveau_fence *fence)
  78. {
  79.    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  80.    (*mm) = NULL;
  81. }
  82.  
  83. INLINE void
  84. nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  85. {
  86.    nouveau_bo_ref(NULL, &buf->bo);
  87.  
  88.    if (buf->mm)
  89.       release_allocation(&buf->mm, buf->fence);
  90.  
  91.    if (buf->domain == NOUVEAU_BO_VRAM)
  92.       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
  93.    if (buf->domain == NOUVEAU_BO_GART)
  94.       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
  95.  
  96.    buf->domain = 0;
  97. }
  98.  
  99. static INLINE boolean
  100. nouveau_buffer_reallocate(struct nouveau_screen *screen,
  101.                           struct nv04_resource *buf, unsigned domain)
  102. {
  103.    nouveau_buffer_release_gpu_storage(buf);
  104.  
  105.    nouveau_fence_ref(NULL, &buf->fence);
  106.    nouveau_fence_ref(NULL, &buf->fence_wr);
  107.  
  108.    buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  109.  
  110.    return nouveau_buffer_allocate(screen, buf, domain);
  111. }
  112.  
  113. static void
  114. nouveau_buffer_destroy(struct pipe_screen *pscreen,
  115.                        struct pipe_resource *presource)
  116. {
  117.    struct nv04_resource *res = nv04_resource(presource);
  118.  
  119.    nouveau_buffer_release_gpu_storage(res);
  120.  
  121.    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  122.       align_free(res->data);
  123.  
  124.    nouveau_fence_ref(NULL, &res->fence);
  125.    nouveau_fence_ref(NULL, &res->fence_wr);
  126.  
  127.    FREE(res);
  128.  
  129.    NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
  130. }
  131.  
  132. static uint8_t *
  133. nouveau_transfer_staging(struct nouveau_context *nv,
  134.                          struct nouveau_transfer *tx, boolean permit_pb)
  135. {
  136.    const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
  137.    const unsigned size = align(tx->base.box.width, 4) + adj;
  138.  
  139.    if (!nv->push_data)
  140.       permit_pb = FALSE;
  141.  
  142.    if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
  143.       tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  144.       if (tx->map)
  145.          tx->map += adj;
  146.    } else {
  147.       tx->mm =
  148.          nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
  149.       if (tx->bo) {
  150.          tx->offset += adj;
  151.          if (!nouveau_bo_map(tx->bo, 0, NULL))
  152.             tx->map = (uint8_t *)tx->bo->map + tx->offset;
  153.       }
  154.    }
  155.    return tx->map;
  156. }
  157.  
  158. /* Maybe just migrate to GART right away if we actually need to do this. */
  159. static boolean
  160. nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
  161. {
  162.    struct nv04_resource *buf = nv04_resource(tx->base.resource);
  163.    const unsigned base = tx->base.box.x;
  164.    const unsigned size = tx->base.box.width;
  165.  
  166.    NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
  167.  
  168.    nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
  169.                  buf->bo, buf->offset + base, buf->domain, size);
  170.  
  171.    if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
  172.       return FALSE;
  173.  
  174.    if (buf->data)
  175.       memcpy(buf->data + base, tx->map, size);
  176.  
  177.    return TRUE;
  178. }
  179.  
  180. static void
  181. nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
  182.                        unsigned offset, unsigned size)
  183. {
  184.    struct nv04_resource *buf = nv04_resource(tx->base.resource);
  185.    uint8_t *data = tx->map + offset;
  186.    const unsigned base = tx->base.box.x + offset;
  187.    const boolean can_cb = !((base | size) & 3);
  188.  
  189.    if (buf->data)
  190.       memcpy(data, buf->data + base, size);
  191.    else
  192.       buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
  193.  
  194.    if (buf->domain == NOUVEAU_BO_VRAM)
  195.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
  196.    if (buf->domain == NOUVEAU_BO_GART)
  197.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
  198.  
  199.    if (tx->bo)
  200.       nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
  201.                     tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
  202.    else
  203.    if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
  204.       nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
  205.                   base, size / 4, (const uint32_t *)data);
  206.    else
  207.       nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
  208. }
  209.  
  210.  
  211. static INLINE boolean
  212. nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
  213. {
  214.    if (rw == PIPE_TRANSFER_READ) {
  215.       if (!buf->fence_wr)
  216.          return TRUE;
  217.       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
  218.                            !nouveau_fence_signalled(buf->fence_wr));
  219.       if (!nouveau_fence_wait(buf->fence_wr))
  220.          return FALSE;
  221.    } else {
  222.       if (!buf->fence)
  223.          return TRUE;
  224.       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
  225.                            !nouveau_fence_signalled(buf->fence));
  226.       if (!nouveau_fence_wait(buf->fence))
  227.          return FALSE;
  228.  
  229.       nouveau_fence_ref(NULL, &buf->fence);
  230.    }
  231.    nouveau_fence_ref(NULL, &buf->fence_wr);
  232.  
  233.    return TRUE;
  234. }
  235.  
  236. static INLINE boolean
  237. nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
  238. {
  239.    if (rw == PIPE_TRANSFER_READ)
  240.       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
  241.    else
  242.       return (buf->fence && !nouveau_fence_signalled(buf->fence));
  243. }
  244.  
  245. static INLINE void
  246. nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
  247.                              struct pipe_resource *resource,
  248.                              const struct pipe_box *box,
  249.                              unsigned usage)
  250. {
  251.    tx->base.resource = resource;
  252.    tx->base.level = 0;
  253.    tx->base.usage = usage;
  254.    tx->base.box.x = box->x;
  255.    tx->base.box.y = 0;
  256.    tx->base.box.z = 0;
  257.    tx->base.box.width = box->width;
  258.    tx->base.box.height = 1;
  259.    tx->base.box.depth = 1;
  260.    tx->base.stride = 0;
  261.    tx->base.layer_stride = 0;
  262.  
  263.    tx->bo = NULL;
  264.    tx->map = NULL;
  265. }
  266.  
  267. static INLINE void
  268. nouveau_buffer_transfer_del(struct nouveau_context *nv,
  269.                             struct nouveau_transfer *tx)
  270. {
  271.    if (tx->map) {
  272.       if (likely(tx->bo)) {
  273.          nouveau_bo_ref(NULL, &tx->bo);
  274.          if (tx->mm)
  275.             release_allocation(&tx->mm, nv->screen->fence.current);
  276.       } else {
  277.          align_free(tx->map -
  278.                     (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
  279.       }
  280.    }
  281. }
  282.  
  283. static boolean
  284. nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
  285. {
  286.    struct nouveau_transfer tx;
  287.    boolean ret;
  288.    tx.base.resource = &buf->base;
  289.    tx.base.box.x = 0;
  290.    tx.base.box.width = buf->base.width0;
  291.    tx.bo = NULL;
  292.    tx.map = NULL;
  293.  
  294.    if (!buf->data)
  295.       if (!nouveau_buffer_malloc(buf))
  296.          return FALSE;
  297.    if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
  298.       return TRUE;
  299.    nv->stats.buf_cache_count++;
  300.  
  301.    if (!nouveau_transfer_staging(nv, &tx, FALSE))
  302.       return FALSE;
  303.  
  304.    ret = nouveau_transfer_read(nv, &tx);
  305.    if (ret) {
  306.       buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
  307.       memcpy(buf->data, tx.map, buf->base.width0);
  308.    }
  309.    nouveau_buffer_transfer_del(nv, &tx);
  310.    return ret;
  311. }
  312.  
  313.  
  314. #define NOUVEAU_TRANSFER_DISCARD \
  315.    (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
  316.  
  317. static INLINE boolean
  318. nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
  319. {
  320.    if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
  321.       return FALSE;
  322.    if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
  323.       return FALSE;
  324.    return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
  325. }
  326.  
  327. static void *
  328. nouveau_buffer_transfer_map(struct pipe_context *pipe,
  329.                             struct pipe_resource *resource,
  330.                             unsigned level, unsigned usage,
  331.                             const struct pipe_box *box,
  332.                             struct pipe_transfer **ptransfer)
  333. {
  334.    struct nouveau_context *nv = nouveau_context(pipe);
  335.    struct nv04_resource *buf = nv04_resource(resource);
  336.    struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
  337.    uint8_t *map;
  338.    int ret;
  339.  
  340.    if (!tx)
  341.       return NULL;
  342.    nouveau_buffer_transfer_init(tx, resource, box, usage);
  343.    *ptransfer = &tx->base;
  344.  
  345.    if (usage & PIPE_TRANSFER_READ)
  346.       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
  347.    if (usage & PIPE_TRANSFER_WRITE)
  348.       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
  349.  
  350.    if (buf->domain == NOUVEAU_BO_VRAM) {
  351.       if (usage & NOUVEAU_TRANSFER_DISCARD) {
  352.          if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
  353.             buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  354.          nouveau_transfer_staging(nv, tx, TRUE);
  355.       } else {
  356.          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
  357.             if (buf->data) {
  358.                align_free(buf->data);
  359.                buf->data = NULL;
  360.             }
  361.             nouveau_transfer_staging(nv, tx, FALSE);
  362.             nouveau_transfer_read(nv, tx);
  363.          } else {
  364.             if (usage & PIPE_TRANSFER_WRITE)
  365.                nouveau_transfer_staging(nv, tx, TRUE);
  366.             if (!buf->data)
  367.                nouveau_buffer_cache(nv, buf);
  368.          }
  369.       }
  370.       return buf->data ? (buf->data + box->x) : tx->map;
  371.    } else
  372.    if (unlikely(buf->domain == 0)) {
  373.       return buf->data + box->x;
  374.    }
  375.  
  376.    if (nouveau_buffer_should_discard(buf, usage)) {
  377.       int ref = buf->base.reference.count - 1;
  378.       nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
  379.       if (ref > 0) /* any references inside context possible ? */
  380.          nv->invalidate_resource_storage(nv, &buf->base, ref);
  381.    }
  382.  
  383.    ret = nouveau_bo_map(buf->bo,
  384.                         buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
  385.                         nv->client);
  386.    if (ret) {
  387.       FREE(tx);
  388.       return NULL;
  389.    }
  390.    map = (uint8_t *)buf->bo->map + buf->offset + box->x;
  391.  
  392.    /* using kernel fences only if !buf->mm */
  393.    if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
  394.       return map;
  395.  
  396.    if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
  397.       if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
  398.          /* Discarding was not possible, must sync because
  399.           * subsequent transfers might use UNSYNCHRONIZED. */
  400.          nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
  401.       } else
  402.       if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
  403.          nouveau_transfer_staging(nv, tx, TRUE);
  404.          map = tx->map;
  405.       } else
  406.       if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
  407.          if (usage & PIPE_TRANSFER_DONTBLOCK)
  408.             map = NULL;
  409.          else
  410.             nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
  411.       } else {
  412.          nouveau_transfer_staging(nv, tx, TRUE);
  413.          if (tx->map)
  414.             memcpy(tx->map, map, box->width);
  415.          map = tx->map;
  416.       }
  417.    }
  418.    if (!map)
  419.       FREE(tx);
  420.    return map;
  421. }
  422.  
  423.  
  424.  
  425. static void
  426. nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
  427.                                      struct pipe_transfer *transfer,
  428.                                      const struct pipe_box *box)
  429. {
  430.    struct nouveau_transfer *tx = nouveau_transfer(transfer);
  431.    if (tx->map)
  432.       nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
  433. }
  434.  
  435. static void
  436. nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
  437.                               struct pipe_transfer *transfer)
  438. {
  439.    struct nouveau_context *nv = nouveau_context(pipe);
  440.    struct nouveau_transfer *tx = nouveau_transfer(transfer);
  441.    struct nv04_resource *buf = nv04_resource(transfer->resource);
  442.  
  443.    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
  444.       if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
  445.          nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
  446.  
  447.       if (likely(buf->domain)) {
  448.          const uint8_t bind = buf->base.bind;
  449.          /* make sure we invalidate dedicated caches */
  450.          if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
  451.             nv->vbo_dirty = TRUE;
  452.          if (bind & (PIPE_BIND_CONSTANT_BUFFER))
  453.             nv->cb_dirty = TRUE;
  454.       }
  455.    }
  456.  
  457.    if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
  458.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
  459.  
  460.    nouveau_buffer_transfer_del(nv, tx);
  461.    FREE(tx);
  462. }
  463.  
  464.  
  465. void
  466. nouveau_copy_buffer(struct nouveau_context *nv,
  467.                     struct nv04_resource *dst, unsigned dstx,
  468.                     struct nv04_resource *src, unsigned srcx, unsigned size)
  469. {
  470.    assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
  471.  
  472.    if (likely(dst->domain) && likely(src->domain)) {
  473.       nv->copy_data(nv,
  474.                     dst->bo, dst->offset + dstx, dst->domain,
  475.                     src->bo, src->offset + srcx, src->domain, size);
  476.  
  477.       dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
  478.       nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
  479.       nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
  480.  
  481.       src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
  482.       nouveau_fence_ref(nv->screen->fence.current, &src->fence);
  483.    } else {
  484.       struct pipe_box src_box;
  485.       src_box.x = srcx;
  486.       src_box.y = 0;
  487.       src_box.z = 0;
  488.       src_box.width = size;
  489.       src_box.height = 1;
  490.       src_box.depth = 1;
  491.       util_resource_copy_region(&nv->pipe,
  492.                                 &dst->base, 0, dstx, 0, 0,
  493.                                 &src->base, 0, &src_box);
  494.    }
  495. }
  496.  
  497.  
  498. void *
  499. nouveau_resource_map_offset(struct nouveau_context *nv,
  500.                             struct nv04_resource *res, uint32_t offset,
  501.                             uint32_t flags)
  502. {
  503.    if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  504.       return res->data + offset;
  505.  
  506.    if (res->domain == NOUVEAU_BO_VRAM) {
  507.       if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
  508.          nouveau_buffer_cache(nv, res);
  509.    }
  510.    if (res->domain != NOUVEAU_BO_GART)
  511.       return res->data + offset;
  512.  
  513.    if (res->mm) {
  514.       unsigned rw;
  515.       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
  516.       nouveau_buffer_sync(res, rw);
  517.       if (nouveau_bo_map(res->bo, 0, NULL))
  518.          return NULL;
  519.    } else {
  520.       if (nouveau_bo_map(res->bo, flags, nv->client))
  521.          return NULL;
  522.    }
  523.    return (uint8_t *)res->bo->map + res->offset + offset;
  524. }
  525.  
  526.  
  527. const struct u_resource_vtbl nouveau_buffer_vtbl =
  528. {
  529.    u_default_resource_get_handle,     /* get_handle */
  530.    nouveau_buffer_destroy,               /* resource_destroy */
  531.    nouveau_buffer_transfer_map,          /* transfer_map */
  532.    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
  533.    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
  534.    u_default_transfer_inline_write    /* transfer_inline_write */
  535. };
  536.  
  537. struct pipe_resource *
  538. nouveau_buffer_create(struct pipe_screen *pscreen,
  539.                       const struct pipe_resource *templ)
  540. {
  541.    struct nouveau_screen *screen = nouveau_screen(pscreen);
  542.    struct nv04_resource *buffer;
  543.    boolean ret;
  544.  
  545.    buffer = CALLOC_STRUCT(nv04_resource);
  546.    if (!buffer)
  547.       return NULL;
  548.  
  549.    buffer->base = *templ;
  550.    buffer->vtbl = &nouveau_buffer_vtbl;
  551.    pipe_reference_init(&buffer->base.reference, 1);
  552.    buffer->base.screen = pscreen;
  553.  
  554.    if (buffer->base.bind &
  555.        (screen->vidmem_bindings & screen->sysmem_bindings)) {
  556.       switch (buffer->base.usage) {
  557.       case PIPE_USAGE_DEFAULT:
  558.       case PIPE_USAGE_IMMUTABLE:
  559.       case PIPE_USAGE_STATIC:
  560.          buffer->domain = NOUVEAU_BO_VRAM;
  561.          break;
  562.       case PIPE_USAGE_DYNAMIC:
  563.          /* For most apps, we'd have to do staging transfers to avoid sync
  564.           * with this usage, and GART -> GART copies would be suboptimal.
  565.           */
  566.          buffer->domain = NOUVEAU_BO_VRAM;
  567.          break;
  568.       case PIPE_USAGE_STAGING:
  569.       case PIPE_USAGE_STREAM:
  570.          buffer->domain = NOUVEAU_BO_GART;
  571.          break;
  572.       default:
  573.          assert(0);
  574.          break;
  575.       }
  576.    } else {
  577.       if (buffer->base.bind & screen->vidmem_bindings)
  578.          buffer->domain = NOUVEAU_BO_VRAM;
  579.       else
  580.       if (buffer->base.bind & screen->sysmem_bindings)
  581.          buffer->domain = NOUVEAU_BO_GART;
  582.    }
  583.    ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
  584.  
  585.    if (ret == FALSE)
  586.       goto fail;
  587.  
  588.    if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
  589.       nouveau_buffer_cache(NULL, buffer);
  590.  
  591.    NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
  592.  
  593.    return &buffer->base;
  594.  
  595. fail:
  596.    FREE(buffer);
  597.    return NULL;
  598. }
  599.  
  600.  
  601. struct pipe_resource *
  602. nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
  603.                            unsigned bytes, unsigned bind)
  604. {
  605.    struct nv04_resource *buffer;
  606.  
  607.    buffer = CALLOC_STRUCT(nv04_resource);
  608.    if (!buffer)
  609.       return NULL;
  610.  
  611.    pipe_reference_init(&buffer->base.reference, 1);
  612.    buffer->vtbl = &nouveau_buffer_vtbl;
  613.    buffer->base.screen = pscreen;
  614.    buffer->base.format = PIPE_FORMAT_R8_UNORM;
  615.    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
  616.    buffer->base.bind = bind;
  617.    buffer->base.width0 = bytes;
  618.    buffer->base.height0 = 1;
  619.    buffer->base.depth0 = 1;
  620.  
  621.    buffer->data = ptr;
  622.    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
  623.  
  624.    return &buffer->base;
  625. }
  626.  
  627. static INLINE boolean
  628. nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
  629.                           struct nouveau_bo *bo, unsigned offset, unsigned size)
  630. {
  631.    if (!nouveau_buffer_malloc(buf))
  632.       return FALSE;
  633.    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
  634.       return FALSE;
  635.    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
  636.    return TRUE;
  637. }
  638.  
  639. /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
  640. boolean
  641. nouveau_buffer_migrate(struct nouveau_context *nv,
  642.                        struct nv04_resource *buf, const unsigned new_domain)
  643. {
  644.    struct nouveau_screen *screen = nv->screen;
  645.    struct nouveau_bo *bo;
  646.    const unsigned old_domain = buf->domain;
  647.    unsigned size = buf->base.width0;
  648.    unsigned offset;
  649.    int ret;
  650.  
  651.    assert(new_domain != old_domain);
  652.  
  653.    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
  654.       if (!nouveau_buffer_allocate(screen, buf, new_domain))
  655.          return FALSE;
  656.       ret = nouveau_bo_map(buf->bo, 0, nv->client);
  657.       if (ret)
  658.          return ret;
  659.       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
  660.       align_free(buf->data);
  661.    } else
  662.    if (old_domain != 0 && new_domain != 0) {
  663.       struct nouveau_mm_allocation *mm = buf->mm;
  664.  
  665.       if (new_domain == NOUVEAU_BO_VRAM) {
  666.          /* keep a system memory copy of our data in case we hit a fallback */
  667.          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
  668.             return FALSE;
  669.          if (nouveau_mesa_debug)
  670.             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
  671.       }
  672.  
  673.       offset = buf->offset;
  674.       bo = buf->bo;
  675.       buf->bo = NULL;
  676.       buf->mm = NULL;
  677.       nouveau_buffer_allocate(screen, buf, new_domain);
  678.  
  679.       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
  680.                     bo, offset, old_domain, buf->base.width0);
  681.  
  682.       nouveau_bo_ref(NULL, &bo);
  683.       if (mm)
  684.          release_allocation(&mm, screen->fence.current);
  685.    } else
  686.    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
  687.       struct nouveau_transfer tx;
  688.       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
  689.          return FALSE;
  690.       tx.base.resource = &buf->base;
  691.       tx.base.box.x = 0;
  692.       tx.base.box.width = buf->base.width0;
  693.       tx.bo = NULL;
  694.       tx.map = NULL;
  695.       if (!nouveau_transfer_staging(nv, &tx, FALSE))
  696.          return FALSE;
  697.       nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
  698.       nouveau_buffer_transfer_del(nv, &tx);
  699.    } else
  700.       return FALSE;
  701.  
  702.    assert(buf->domain == new_domain);
  703.    return TRUE;
  704. }
  705.  
  706. /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
  707.  * We'd like to only allocate @size bytes here, but then we'd have to rebase
  708.  * the vertex indices ...
  709.  */
  710. boolean
  711. nouveau_user_buffer_upload(struct nouveau_context *nv,
  712.                            struct nv04_resource *buf,
  713.                            unsigned base, unsigned size)
  714. {
  715.    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
  716.    int ret;
  717.  
  718.    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
  719.  
  720.    buf->base.width0 = base + size;
  721.    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
  722.       return FALSE;
  723.  
  724.    ret = nouveau_bo_map(buf->bo, 0, nv->client);
  725.    if (ret)
  726.       return FALSE;
  727.    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
  728.  
  729.    return TRUE;
  730. }
  731.  
  732.  
  733. /* Scratch data allocation. */
  734.  
  735. static INLINE int
  736. nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
  737.                          unsigned size)
  738. {
  739.    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
  740.                          4096, size, NULL, pbo);
  741. }
  742.  
  743. void
  744. nouveau_scratch_runout_release(struct nouveau_context *nv)
  745. {
  746.    if (!nv->scratch.nr_runout)
  747.       return;
  748.    do {
  749.       --nv->scratch.nr_runout;
  750.       nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
  751.    } while (nv->scratch.nr_runout);
  752.  
  753.    FREE(nv->scratch.runout);
  754.    nv->scratch.end = 0;
  755.    nv->scratch.runout = NULL;
  756. }
  757.  
  758. /* Allocate an extra bo if we can't fit everything we need simultaneously.
  759.  * (Could happen for very large user arrays.)
  760.  */
  761. static INLINE boolean
  762. nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
  763. {
  764.    int ret;
  765.    const unsigned n = nv->scratch.nr_runout++;
  766.  
  767.    nv->scratch.runout = REALLOC(nv->scratch.runout,
  768.                                 (n + 0) * sizeof(*nv->scratch.runout),
  769.                                 (n + 1) * sizeof(*nv->scratch.runout));
  770.    nv->scratch.runout[n] = NULL;
  771.  
  772.    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
  773.    if (!ret) {
  774.       ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
  775.       if (ret)
  776.          nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
  777.    }
  778.    if (!ret) {
  779.       nv->scratch.current = nv->scratch.runout[n];
  780.       nv->scratch.offset = 0;
  781.       nv->scratch.end = size;
  782.       nv->scratch.map = nv->scratch.current->map;
  783.    }
  784.    return !ret;
  785. }
  786.  
  787. /* Continue to next scratch buffer, if available (no wrapping, large enough).
  788.  * Allocate it if it has not yet been created.
  789.  */
  790. static INLINE boolean
  791. nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
  792. {
  793.    struct nouveau_bo *bo;
  794.    int ret;
  795.    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
  796.  
  797.    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
  798.       return FALSE;
  799.    nv->scratch.id = i;
  800.  
  801.    bo = nv->scratch.bo[i];
  802.    if (!bo) {
  803.       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
  804.       if (ret)
  805.          return FALSE;
  806.       nv->scratch.bo[i] = bo;
  807.    }
  808.    nv->scratch.current = bo;
  809.    nv->scratch.offset = 0;
  810.    nv->scratch.end = nv->scratch.bo_size;
  811.  
  812.    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
  813.    if (!ret)
  814.       nv->scratch.map = bo->map;
  815.    return !ret;
  816. }
  817.  
  818. static boolean
  819. nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
  820. {
  821.    boolean ret;
  822.  
  823.    ret = nouveau_scratch_next(nv, min_size);
  824.    if (!ret)
  825.       ret = nouveau_scratch_runout(nv, min_size);
  826.    return ret;
  827. }
  828.  
  829.  
  830. /* Copy data to a scratch buffer and return address & bo the data resides in. */
  831. uint64_t
  832. nouveau_scratch_data(struct nouveau_context *nv,
  833.                      const void *data, unsigned base, unsigned size,
  834.                      struct nouveau_bo **bo)
  835. {
  836.    unsigned bgn = MAX2(base, nv->scratch.offset);
  837.    unsigned end = bgn + size;
  838.  
  839.    if (end >= nv->scratch.end) {
  840.       end = base + size;
  841.       if (!nouveau_scratch_more(nv, end))
  842.          return 0;
  843.       bgn = base;
  844.    }
  845.    nv->scratch.offset = align(end, 4);
  846.  
  847.    memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
  848.  
  849.    *bo = nv->scratch.current;
  850.    return (*bo)->offset + (bgn - base);
  851. }
  852.  
  853. void *
  854. nouveau_scratch_get(struct nouveau_context *nv,
  855.                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
  856. {
  857.    unsigned bgn = nv->scratch.offset;
  858.    unsigned end = nv->scratch.offset + size;
  859.  
  860.    if (end >= nv->scratch.end) {
  861.       end = size;
  862.       if (!nouveau_scratch_more(nv, end))
  863.          return NULL;
  864.       bgn = 0;
  865.    }
  866.    nv->scratch.offset = align(end, 4);
  867.  
  868.    *pbo = nv->scratch.current;
  869.    *gpu_addr = nv->scratch.current->offset + bgn;
  870.    return nv->scratch.map + bgn;
  871. }
  872.