Subversion Repositories Kolibri OS

Rev

Rev 4358 | Go to most recent revision | Blame | Last modification | View Log | RSS feed

  1.  
  2. #include "util/u_inlines.h"
  3. #include "util/u_memory.h"
  4. #include "util/u_math.h"
  5. #include "util/u_surface.h"
  6.  
  7. #include "nouveau_screen.h"
  8. #include "nouveau_context.h"
  9. #include "nouveau_winsys.h"
  10. #include "nouveau_fence.h"
  11. #include "nouveau_buffer.h"
  12. #include "nouveau_mm.h"
  13.  
  14. #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
  15.  
  16. struct nouveau_transfer {
  17.    struct pipe_transfer base;
  18.  
  19.    uint8_t *map;
  20.    struct nouveau_bo *bo;
  21.    struct nouveau_mm_allocation *mm;
  22.    uint32_t offset;
  23. };
  24.  
  25. static INLINE struct nouveau_transfer *
  26. nouveau_transfer(struct pipe_transfer *transfer)
  27. {
  28.    return (struct nouveau_transfer *)transfer;
  29. }
  30.  
  31. static INLINE boolean
  32. nouveau_buffer_malloc(struct nv04_resource *buf)
  33. {
  34.    if (!buf->data)
  35.       buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  36.    return !!buf->data;
  37. }
  38.  
  39. static INLINE boolean
  40. nouveau_buffer_allocate(struct nouveau_screen *screen,
  41.                         struct nv04_resource *buf, unsigned domain)
  42. {
  43.    uint32_t size = buf->base.width0;
  44.  
  45.    if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
  46.                          PIPE_BIND_COMPUTE_RESOURCE |
  47.                          PIPE_BIND_SHADER_RESOURCE))
  48.       size = align(size, 0x100);
  49.  
  50.    if (domain == NOUVEAU_BO_VRAM) {
  51.       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  52.                                     &buf->bo, &buf->offset);
  53.       if (!buf->bo)
  54.          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  55.       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
  56.    } else
  57.    if (domain == NOUVEAU_BO_GART) {
  58.       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  59.                                     &buf->bo, &buf->offset);
  60.       if (!buf->bo)
  61.          return FALSE;
  62.       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
  63.    } else {
  64.       assert(domain == 0);
  65.       if (!nouveau_buffer_malloc(buf))
  66.          return FALSE;
  67.    }
  68.    buf->domain = domain;
  69.    if (buf->bo)
  70.       buf->address = buf->bo->offset + buf->offset;
  71.  
  72.    return TRUE;
  73. }
  74.  
  75. static INLINE void
  76. release_allocation(struct nouveau_mm_allocation **mm,
  77.                    struct nouveau_fence *fence)
  78. {
  79.    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  80.    (*mm) = NULL;
  81. }
  82.  
  83. INLINE void
  84. nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  85. {
  86.    nouveau_bo_ref(NULL, &buf->bo);
  87.  
  88.    if (buf->mm)
  89.       release_allocation(&buf->mm, buf->fence);
  90.  
  91.    if (buf->domain == NOUVEAU_BO_VRAM)
  92.       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
  93.    if (buf->domain == NOUVEAU_BO_GART)
  94.       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
  95.  
  96.    buf->domain = 0;
  97. }
  98.  
  99. static INLINE boolean
  100. nouveau_buffer_reallocate(struct nouveau_screen *screen,
  101.                           struct nv04_resource *buf, unsigned domain)
  102. {
  103.    nouveau_buffer_release_gpu_storage(buf);
  104.  
  105.    nouveau_fence_ref(NULL, &buf->fence);
  106.    nouveau_fence_ref(NULL, &buf->fence_wr);
  107.  
  108.    buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  109.  
  110.    return nouveau_buffer_allocate(screen, buf, domain);
  111. }
  112.  
  113. static void
  114. nouveau_buffer_destroy(struct pipe_screen *pscreen,
  115.                        struct pipe_resource *presource)
  116. {
  117.    struct nv04_resource *res = nv04_resource(presource);
  118.  
  119.    nouveau_buffer_release_gpu_storage(res);
  120.  
  121.    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  122.       align_free(res->data);
  123.  
  124.    nouveau_fence_ref(NULL, &res->fence);
  125.    nouveau_fence_ref(NULL, &res->fence_wr);
  126.  
  127.    FREE(res);
  128.  
  129.    NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
  130. }
  131.  
  132. static uint8_t *
  133. nouveau_transfer_staging(struct nouveau_context *nv,
  134.                          struct nouveau_transfer *tx, boolean permit_pb)
  135. {
  136.    const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
  137.    const unsigned size = align(tx->base.box.width, 4) + adj;
  138.  
  139.    if (!nv->push_data)
  140.       permit_pb = FALSE;
  141.  
  142.    if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
  143.       tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  144.       if (tx->map)
  145.          tx->map += adj;
  146.    } else {
  147.       tx->mm =
  148.          nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
  149.       if (tx->bo) {
  150.          tx->offset += adj;
  151.          if (!nouveau_bo_map(tx->bo, 0, NULL))
  152.             tx->map = (uint8_t *)tx->bo->map + tx->offset;
  153.       }
  154.    }
  155.    return tx->map;
  156. }
  157.  
  158. /* Maybe just migrate to GART right away if we actually need to do this. */
  159. static boolean
  160. nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
  161. {
  162.    struct nv04_resource *buf = nv04_resource(tx->base.resource);
  163.    const unsigned base = tx->base.box.x;
  164.    const unsigned size = tx->base.box.width;
  165.  
  166.    NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
  167.  
  168.    nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
  169.                  buf->bo, buf->offset + base, buf->domain, size);
  170.  
  171.    if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
  172.       return FALSE;
  173.  
  174.    if (buf->data)
  175.       memcpy(buf->data + base, tx->map, size);
  176.  
  177.    return TRUE;
  178. }
  179.  
  180. static void
  181. nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
  182.                        unsigned offset, unsigned size)
  183. {
  184.    struct nv04_resource *buf = nv04_resource(tx->base.resource);
  185.    uint8_t *data = tx->map + offset;
  186.    const unsigned base = tx->base.box.x + offset;
  187.    const boolean can_cb = !((base | size) & 3);
  188.  
  189.    if (buf->data)
  190.       memcpy(data, buf->data + base, size);
  191.    else
  192.       buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
  193.  
  194.    if (buf->domain == NOUVEAU_BO_VRAM)
  195.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
  196.    if (buf->domain == NOUVEAU_BO_GART)
  197.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
  198.  
  199.    if (tx->bo)
  200.       nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
  201.                     tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
  202.    else
  203.    if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
  204.       nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
  205.                   base, size / 4, (const uint32_t *)data);
  206.    else
  207.       nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
  208.  
  209.    nouveau_fence_ref(nv->screen->fence.current, &buf->fence);
  210.    nouveau_fence_ref(nv->screen->fence.current, &buf->fence_wr);
  211. }
  212.  
  213.  
  214. static INLINE boolean
  215. nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
  216. {
  217.    if (rw == PIPE_TRANSFER_READ) {
  218.       if (!buf->fence_wr)
  219.          return TRUE;
  220.       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
  221.                            !nouveau_fence_signalled(buf->fence_wr));
  222.       if (!nouveau_fence_wait(buf->fence_wr))
  223.          return FALSE;
  224.    } else {
  225.       if (!buf->fence)
  226.          return TRUE;
  227.       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
  228.                            !nouveau_fence_signalled(buf->fence));
  229.       if (!nouveau_fence_wait(buf->fence))
  230.          return FALSE;
  231.  
  232.       nouveau_fence_ref(NULL, &buf->fence);
  233.    }
  234.    nouveau_fence_ref(NULL, &buf->fence_wr);
  235.  
  236.    return TRUE;
  237. }
  238.  
  239. static INLINE boolean
  240. nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
  241. {
  242.    if (rw == PIPE_TRANSFER_READ)
  243.       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
  244.    else
  245.       return (buf->fence && !nouveau_fence_signalled(buf->fence));
  246. }
  247.  
  248. static INLINE void
  249. nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
  250.                              struct pipe_resource *resource,
  251.                              const struct pipe_box *box,
  252.                              unsigned usage)
  253. {
  254.    tx->base.resource = resource;
  255.    tx->base.level = 0;
  256.    tx->base.usage = usage;
  257.    tx->base.box.x = box->x;
  258.    tx->base.box.y = 0;
  259.    tx->base.box.z = 0;
  260.    tx->base.box.width = box->width;
  261.    tx->base.box.height = 1;
  262.    tx->base.box.depth = 1;
  263.    tx->base.stride = 0;
  264.    tx->base.layer_stride = 0;
  265.  
  266.    tx->bo = NULL;
  267.    tx->map = NULL;
  268. }
  269.  
  270. static INLINE void
  271. nouveau_buffer_transfer_del(struct nouveau_context *nv,
  272.                             struct nouveau_transfer *tx)
  273. {
  274.    if (tx->map) {
  275.       if (likely(tx->bo)) {
  276.          nouveau_bo_ref(NULL, &tx->bo);
  277.          if (tx->mm)
  278.             release_allocation(&tx->mm, nv->screen->fence.current);
  279.       } else {
  280.          align_free(tx->map -
  281.                     (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
  282.       }
  283.    }
  284. }
  285.  
  286. static boolean
  287. nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
  288. {
  289.    struct nouveau_transfer tx;
  290.    boolean ret;
  291.    tx.base.resource = &buf->base;
  292.    tx.base.box.x = 0;
  293.    tx.base.box.width = buf->base.width0;
  294.    tx.bo = NULL;
  295.    tx.map = NULL;
  296.  
  297.    if (!buf->data)
  298.       if (!nouveau_buffer_malloc(buf))
  299.          return FALSE;
  300.    if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
  301.       return TRUE;
  302.    nv->stats.buf_cache_count++;
  303.  
  304.    if (!nouveau_transfer_staging(nv, &tx, FALSE))
  305.       return FALSE;
  306.  
  307.    ret = nouveau_transfer_read(nv, &tx);
  308.    if (ret) {
  309.       buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
  310.       memcpy(buf->data, tx.map, buf->base.width0);
  311.    }
  312.    nouveau_buffer_transfer_del(nv, &tx);
  313.    return ret;
  314. }
  315.  
  316.  
  317. #define NOUVEAU_TRANSFER_DISCARD \
  318.    (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
  319.  
  320. static INLINE boolean
  321. nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
  322. {
  323.    if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
  324.       return FALSE;
  325.    if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
  326.       return FALSE;
  327.    return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
  328. }
  329.  
  330. static void *
  331. nouveau_buffer_transfer_map(struct pipe_context *pipe,
  332.                             struct pipe_resource *resource,
  333.                             unsigned level, unsigned usage,
  334.                             const struct pipe_box *box,
  335.                             struct pipe_transfer **ptransfer)
  336. {
  337.    struct nouveau_context *nv = nouveau_context(pipe);
  338.    struct nv04_resource *buf = nv04_resource(resource);
  339.    struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
  340.    uint8_t *map;
  341.    int ret;
  342.  
  343.    if (!tx)
  344.       return NULL;
  345.    nouveau_buffer_transfer_init(tx, resource, box, usage);
  346.    *ptransfer = &tx->base;
  347.  
  348.    if (usage & PIPE_TRANSFER_READ)
  349.       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
  350.    if (usage & PIPE_TRANSFER_WRITE)
  351.       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
  352.  
  353.    if (buf->domain == NOUVEAU_BO_VRAM) {
  354.       if (usage & NOUVEAU_TRANSFER_DISCARD) {
  355.          if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
  356.             buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  357.          nouveau_transfer_staging(nv, tx, TRUE);
  358.       } else {
  359.          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
  360.             if (buf->data) {
  361.                align_free(buf->data);
  362.                buf->data = NULL;
  363.             }
  364.             nouveau_transfer_staging(nv, tx, FALSE);
  365.             nouveau_transfer_read(nv, tx);
  366.          } else {
  367.             if (usage & PIPE_TRANSFER_WRITE)
  368.                nouveau_transfer_staging(nv, tx, TRUE);
  369.             if (!buf->data)
  370.                nouveau_buffer_cache(nv, buf);
  371.          }
  372.       }
  373.       return buf->data ? (buf->data + box->x) : tx->map;
  374.    } else
  375.    if (unlikely(buf->domain == 0)) {
  376.       return buf->data + box->x;
  377.    }
  378.  
  379.    if (nouveau_buffer_should_discard(buf, usage)) {
  380.       int ref = buf->base.reference.count - 1;
  381.       nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
  382.       if (ref > 0) /* any references inside context possible ? */
  383.          nv->invalidate_resource_storage(nv, &buf->base, ref);
  384.    }
  385.  
  386.    ret = nouveau_bo_map(buf->bo,
  387.                         buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
  388.                         nv->client);
  389.    if (ret) {
  390.       FREE(tx);
  391.       return NULL;
  392.    }
  393.    map = (uint8_t *)buf->bo->map + buf->offset + box->x;
  394.  
  395.    /* using kernel fences only if !buf->mm */
  396.    if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
  397.       return map;
  398.  
  399.    if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
  400.       if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
  401.          /* Discarding was not possible, must sync because
  402.           * subsequent transfers might use UNSYNCHRONIZED. */
  403.          nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
  404.       } else
  405.       if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
  406.          nouveau_transfer_staging(nv, tx, TRUE);
  407.          map = tx->map;
  408.       } else
  409.       if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
  410.          if (usage & PIPE_TRANSFER_DONTBLOCK)
  411.             map = NULL;
  412.          else
  413.             nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
  414.       } else {
  415.          nouveau_transfer_staging(nv, tx, TRUE);
  416.          if (tx->map)
  417.             memcpy(tx->map, map, box->width);
  418.          map = tx->map;
  419.       }
  420.    }
  421.    if (!map)
  422.       FREE(tx);
  423.    return map;
  424. }
  425.  
  426.  
  427.  
  428. static void
  429. nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
  430.                                      struct pipe_transfer *transfer,
  431.                                      const struct pipe_box *box)
  432. {
  433.    struct nouveau_transfer *tx = nouveau_transfer(transfer);
  434.    if (tx->map)
  435.       nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
  436. }
  437.  
  438. static void
  439. nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
  440.                               struct pipe_transfer *transfer)
  441. {
  442.    struct nouveau_context *nv = nouveau_context(pipe);
  443.    struct nouveau_transfer *tx = nouveau_transfer(transfer);
  444.    struct nv04_resource *buf = nv04_resource(transfer->resource);
  445.  
  446.    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
  447.       if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
  448.          nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
  449.  
  450.       if (likely(buf->domain)) {
  451.          const uint8_t bind = buf->base.bind;
  452.          /* make sure we invalidate dedicated caches */
  453.          if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
  454.             nv->vbo_dirty = TRUE;
  455.          if (bind & (PIPE_BIND_CONSTANT_BUFFER))
  456.             nv->cb_dirty = TRUE;
  457.       }
  458.    }
  459.  
  460.    if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
  461.       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
  462.  
  463.    nouveau_buffer_transfer_del(nv, tx);
  464.    FREE(tx);
  465. }
  466.  
  467.  
  468. void
  469. nouveau_copy_buffer(struct nouveau_context *nv,
  470.                     struct nv04_resource *dst, unsigned dstx,
  471.                     struct nv04_resource *src, unsigned srcx, unsigned size)
  472. {
  473.    assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
  474.  
  475.    if (likely(dst->domain) && likely(src->domain)) {
  476.       nv->copy_data(nv,
  477.                     dst->bo, dst->offset + dstx, dst->domain,
  478.                     src->bo, src->offset + srcx, src->domain, size);
  479.  
  480.       dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
  481.       nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
  482.       nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
  483.  
  484.       src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
  485.       nouveau_fence_ref(nv->screen->fence.current, &src->fence);
  486.    } else {
  487.       struct pipe_box src_box;
  488.       src_box.x = srcx;
  489.       src_box.y = 0;
  490.       src_box.z = 0;
  491.       src_box.width = size;
  492.       src_box.height = 1;
  493.       src_box.depth = 1;
  494.       util_resource_copy_region(&nv->pipe,
  495.                                 &dst->base, 0, dstx, 0, 0,
  496.                                 &src->base, 0, &src_box);
  497.    }
  498. }
  499.  
  500.  
  501. void *
  502. nouveau_resource_map_offset(struct nouveau_context *nv,
  503.                             struct nv04_resource *res, uint32_t offset,
  504.                             uint32_t flags)
  505. {
  506.    if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  507.       return res->data + offset;
  508.  
  509.    if (res->domain == NOUVEAU_BO_VRAM) {
  510.       if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
  511.          nouveau_buffer_cache(nv, res);
  512.    }
  513.    if (res->domain != NOUVEAU_BO_GART)
  514.       return res->data + offset;
  515.  
  516.    if (res->mm) {
  517.       unsigned rw;
  518.       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
  519.       nouveau_buffer_sync(res, rw);
  520.       if (nouveau_bo_map(res->bo, 0, NULL))
  521.          return NULL;
  522.    } else {
  523.       if (nouveau_bo_map(res->bo, flags, nv->client))
  524.          return NULL;
  525.    }
  526.    return (uint8_t *)res->bo->map + res->offset + offset;
  527. }
  528.  
  529.  
  530. const struct u_resource_vtbl nouveau_buffer_vtbl =
  531. {
  532.    u_default_resource_get_handle,     /* get_handle */
  533.    nouveau_buffer_destroy,               /* resource_destroy */
  534.    nouveau_buffer_transfer_map,          /* transfer_map */
  535.    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
  536.    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
  537.    u_default_transfer_inline_write    /* transfer_inline_write */
  538. };
  539.  
  540. struct pipe_resource *
  541. nouveau_buffer_create(struct pipe_screen *pscreen,
  542.                       const struct pipe_resource *templ)
  543. {
  544.    struct nouveau_screen *screen = nouveau_screen(pscreen);
  545.    struct nv04_resource *buffer;
  546.    boolean ret;
  547.  
  548.    buffer = CALLOC_STRUCT(nv04_resource);
  549.    if (!buffer)
  550.       return NULL;
  551.  
  552.    buffer->base = *templ;
  553.    buffer->vtbl = &nouveau_buffer_vtbl;
  554.    pipe_reference_init(&buffer->base.reference, 1);
  555.    buffer->base.screen = pscreen;
  556.  
  557.    if (buffer->base.bind &
  558.        (screen->vidmem_bindings & screen->sysmem_bindings)) {
  559.       switch (buffer->base.usage) {
  560.       case PIPE_USAGE_DEFAULT:
  561.       case PIPE_USAGE_IMMUTABLE:
  562.       case PIPE_USAGE_STATIC:
  563.          buffer->domain = NOUVEAU_BO_VRAM;
  564.          break;
  565.       case PIPE_USAGE_DYNAMIC:
  566.          /* For most apps, we'd have to do staging transfers to avoid sync
  567.           * with this usage, and GART -> GART copies would be suboptimal.
  568.           */
  569.          buffer->domain = NOUVEAU_BO_VRAM;
  570.          break;
  571.       case PIPE_USAGE_STAGING:
  572.       case PIPE_USAGE_STREAM:
  573.          buffer->domain = NOUVEAU_BO_GART;
  574.          break;
  575.       default:
  576.          assert(0);
  577.          break;
  578.       }
  579.    } else {
  580.       if (buffer->base.bind & screen->vidmem_bindings)
  581.          buffer->domain = NOUVEAU_BO_VRAM;
  582.       else
  583.       if (buffer->base.bind & screen->sysmem_bindings)
  584.          buffer->domain = NOUVEAU_BO_GART;
  585.    }
  586.    ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
  587.  
  588.    if (ret == FALSE)
  589.       goto fail;
  590.  
  591.    if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
  592.       nouveau_buffer_cache(NULL, buffer);
  593.  
  594.    NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
  595.  
  596.    return &buffer->base;
  597.  
  598. fail:
  599.    FREE(buffer);
  600.    return NULL;
  601. }
  602.  
  603.  
  604. struct pipe_resource *
  605. nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
  606.                            unsigned bytes, unsigned bind)
  607. {
  608.    struct nv04_resource *buffer;
  609.  
  610.    buffer = CALLOC_STRUCT(nv04_resource);
  611.    if (!buffer)
  612.       return NULL;
  613.  
  614.    pipe_reference_init(&buffer->base.reference, 1);
  615.    buffer->vtbl = &nouveau_buffer_vtbl;
  616.    buffer->base.screen = pscreen;
  617.    buffer->base.format = PIPE_FORMAT_R8_UNORM;
  618.    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
  619.    buffer->base.bind = bind;
  620.    buffer->base.width0 = bytes;
  621.    buffer->base.height0 = 1;
  622.    buffer->base.depth0 = 1;
  623.  
  624.    buffer->data = ptr;
  625.    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
  626.  
  627.    return &buffer->base;
  628. }
  629.  
  630. static INLINE boolean
  631. nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
  632.                           struct nouveau_bo *bo, unsigned offset, unsigned size)
  633. {
  634.    if (!nouveau_buffer_malloc(buf))
  635.       return FALSE;
  636.    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
  637.       return FALSE;
  638.    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
  639.    return TRUE;
  640. }
  641.  
  642. /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
  643. boolean
  644. nouveau_buffer_migrate(struct nouveau_context *nv,
  645.                        struct nv04_resource *buf, const unsigned new_domain)
  646. {
  647.    struct nouveau_screen *screen = nv->screen;
  648.    struct nouveau_bo *bo;
  649.    const unsigned old_domain = buf->domain;
  650.    unsigned size = buf->base.width0;
  651.    unsigned offset;
  652.    int ret;
  653.  
  654.    assert(new_domain != old_domain);
  655.  
  656.    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
  657.       if (!nouveau_buffer_allocate(screen, buf, new_domain))
  658.          return FALSE;
  659.       ret = nouveau_bo_map(buf->bo, 0, nv->client);
  660.       if (ret)
  661.          return ret;
  662.       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
  663.       align_free(buf->data);
  664.    } else
  665.    if (old_domain != 0 && new_domain != 0) {
  666.       struct nouveau_mm_allocation *mm = buf->mm;
  667.  
  668.       if (new_domain == NOUVEAU_BO_VRAM) {
  669.          /* keep a system memory copy of our data in case we hit a fallback */
  670.          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
  671.             return FALSE;
  672.          if (nouveau_mesa_debug)
  673.             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
  674.       }
  675.  
  676.       offset = buf->offset;
  677.       bo = buf->bo;
  678.       buf->bo = NULL;
  679.       buf->mm = NULL;
  680.       nouveau_buffer_allocate(screen, buf, new_domain);
  681.  
  682.       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
  683.                     bo, offset, old_domain, buf->base.width0);
  684.  
  685.       nouveau_bo_ref(NULL, &bo);
  686.       if (mm)
  687.          release_allocation(&mm, screen->fence.current);
  688.    } else
  689.    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
  690.       struct nouveau_transfer tx;
  691.       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
  692.          return FALSE;
  693.       tx.base.resource = &buf->base;
  694.       tx.base.box.x = 0;
  695.       tx.base.box.width = buf->base.width0;
  696.       tx.bo = NULL;
  697.       tx.map = NULL;
  698.       if (!nouveau_transfer_staging(nv, &tx, FALSE))
  699.          return FALSE;
  700.       nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
  701.       nouveau_buffer_transfer_del(nv, &tx);
  702.    } else
  703.       return FALSE;
  704.  
  705.    assert(buf->domain == new_domain);
  706.    return TRUE;
  707. }
  708.  
  709. /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
  710.  * We'd like to only allocate @size bytes here, but then we'd have to rebase
  711.  * the vertex indices ...
  712.  */
  713. boolean
  714. nouveau_user_buffer_upload(struct nouveau_context *nv,
  715.                            struct nv04_resource *buf,
  716.                            unsigned base, unsigned size)
  717. {
  718.    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
  719.    int ret;
  720.  
  721.    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
  722.  
  723.    buf->base.width0 = base + size;
  724.    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
  725.       return FALSE;
  726.  
  727.    ret = nouveau_bo_map(buf->bo, 0, nv->client);
  728.    if (ret)
  729.       return FALSE;
  730.    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
  731.  
  732.    return TRUE;
  733. }
  734.  
  735.  
  736. /* Scratch data allocation. */
  737.  
  738. static INLINE int
  739. nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
  740.                          unsigned size)
  741. {
  742.    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
  743.                          4096, size, NULL, pbo);
  744. }
  745.  
  746. void
  747. nouveau_scratch_runout_release(struct nouveau_context *nv)
  748. {
  749.    if (!nv->scratch.nr_runout)
  750.       return;
  751.    do {
  752.       --nv->scratch.nr_runout;
  753.       nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
  754.    } while (nv->scratch.nr_runout);
  755.  
  756.    FREE(nv->scratch.runout);
  757.    nv->scratch.end = 0;
  758.    nv->scratch.runout = NULL;
  759. }
  760.  
  761. /* Allocate an extra bo if we can't fit everything we need simultaneously.
  762.  * (Could happen for very large user arrays.)
  763.  */
  764. static INLINE boolean
  765. nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
  766. {
  767.    int ret;
  768.    const unsigned n = nv->scratch.nr_runout++;
  769.  
  770.    nv->scratch.runout = REALLOC(nv->scratch.runout,
  771.                                 (n + 0) * sizeof(*nv->scratch.runout),
  772.                                 (n + 1) * sizeof(*nv->scratch.runout));
  773.    nv->scratch.runout[n] = NULL;
  774.  
  775.    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
  776.    if (!ret) {
  777.       ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
  778.       if (ret)
  779.          nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
  780.    }
  781.    if (!ret) {
  782.       nv->scratch.current = nv->scratch.runout[n];
  783.       nv->scratch.offset = 0;
  784.       nv->scratch.end = size;
  785.       nv->scratch.map = nv->scratch.current->map;
  786.    }
  787.    return !ret;
  788. }
  789.  
  790. /* Continue to next scratch buffer, if available (no wrapping, large enough).
  791.  * Allocate it if it has not yet been created.
  792.  */
  793. static INLINE boolean
  794. nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
  795. {
  796.    struct nouveau_bo *bo;
  797.    int ret;
  798.    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
  799.  
  800.    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
  801.       return FALSE;
  802.    nv->scratch.id = i;
  803.  
  804.    bo = nv->scratch.bo[i];
  805.    if (!bo) {
  806.       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
  807.       if (ret)
  808.          return FALSE;
  809.       nv->scratch.bo[i] = bo;
  810.    }
  811.    nv->scratch.current = bo;
  812.    nv->scratch.offset = 0;
  813.    nv->scratch.end = nv->scratch.bo_size;
  814.  
  815.    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
  816.    if (!ret)
  817.       nv->scratch.map = bo->map;
  818.    return !ret;
  819. }
  820.  
  821. static boolean
  822. nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
  823. {
  824.    boolean ret;
  825.  
  826.    ret = nouveau_scratch_next(nv, min_size);
  827.    if (!ret)
  828.       ret = nouveau_scratch_runout(nv, min_size);
  829.    return ret;
  830. }
  831.  
  832.  
  833. /* Copy data to a scratch buffer and return address & bo the data resides in. */
  834. uint64_t
  835. nouveau_scratch_data(struct nouveau_context *nv,
  836.                      const void *data, unsigned base, unsigned size,
  837.                      struct nouveau_bo **bo)
  838. {
  839.    unsigned bgn = MAX2(base, nv->scratch.offset);
  840.    unsigned end = bgn + size;
  841.  
  842.    if (end >= nv->scratch.end) {
  843.       end = base + size;
  844.       if (!nouveau_scratch_more(nv, end))
  845.          return 0;
  846.       bgn = base;
  847.    }
  848.    nv->scratch.offset = align(end, 4);
  849.  
  850.    memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
  851.  
  852.    *bo = nv->scratch.current;
  853.    return (*bo)->offset + (bgn - base);
  854. }
  855.  
  856. void *
  857. nouveau_scratch_get(struct nouveau_context *nv,
  858.                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
  859. {
  860.    unsigned bgn = nv->scratch.offset;
  861.    unsigned end = nv->scratch.offset + size;
  862.  
  863.    if (end >= nv->scratch.end) {
  864.       end = size;
  865.       if (!nouveau_scratch_more(nv, end))
  866.          return NULL;
  867.       bgn = 0;
  868.    }
  869.    nv->scratch.offset = align(end, 4);
  870.  
  871.    *pbo = nv->scratch.current;
  872.    *gpu_addr = nv->scratch.current->offset + bgn;
  873.    return nv->scratch.map + bgn;
  874. }
  875.