Subversion Repositories Kolibri OS

Rev

Go to most recent revision | Blame | Last modification | View Log | RSS feed

  1.  
  2. #include "util/u_format.h"
  3.  
  4. #include "nvc0_context.h"
  5.  
  6. #include "nv50/nv50_defs.xml.h"
  7.  
  8. struct nvc0_transfer {
  9.    struct pipe_transfer base;
  10.    struct nv50_m2mf_rect rect[2];
  11.    uint32_t nblocksx;
  12.    uint16_t nblocksy;
  13.    uint16_t nlayers;
  14. };
  15.  
  16. static void
  17. nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
  18.                         const struct nv50_m2mf_rect *dst,
  19.                         const struct nv50_m2mf_rect *src,
  20.                         uint32_t nblocksx, uint32_t nblocksy)
  21. {
  22.    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
  23.    struct nouveau_bufctx *bctx = nvc0->bufctx;
  24.    const int cpp = dst->cpp;
  25.    uint32_t src_ofst = src->base;
  26.    uint32_t dst_ofst = dst->base;
  27.    uint32_t height = nblocksy;
  28.    uint32_t sy = src->y;
  29.    uint32_t dy = dst->y;
  30.    uint32_t exec = (1 << 20);
  31.  
  32.    assert(dst->cpp == src->cpp);
  33.  
  34.    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
  35.    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
  36.    nouveau_pushbuf_bufctx(push, bctx);
  37.    nouveau_pushbuf_validate(push);
  38.  
  39.    if (nouveau_bo_memtype(src->bo)) {
  40.       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
  41.       PUSH_DATA (push, src->tile_mode);
  42.       PUSH_DATA (push, src->width * cpp);
  43.       PUSH_DATA (push, src->height);
  44.       PUSH_DATA (push, src->depth);
  45.       PUSH_DATA (push, src->z);
  46.    } else {
  47.       src_ofst += src->y * src->pitch + src->x * cpp;
  48.  
  49.       BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
  50.       PUSH_DATA (push, src->width * cpp);
  51.  
  52.       exec |= NVC0_M2MF_EXEC_LINEAR_IN;
  53.    }
  54.  
  55.    if (nouveau_bo_memtype(dst->bo)) {
  56.       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
  57.       PUSH_DATA (push, dst->tile_mode);
  58.       PUSH_DATA (push, dst->width * cpp);
  59.       PUSH_DATA (push, dst->height);
  60.       PUSH_DATA (push, dst->depth);
  61.       PUSH_DATA (push, dst->z);
  62.    } else {
  63.       dst_ofst += dst->y * dst->pitch + dst->x * cpp;
  64.  
  65.       BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
  66.       PUSH_DATA (push, dst->width * cpp);
  67.  
  68.       exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
  69.    }
  70.  
  71.    while (height) {
  72.       int line_count = height > 2047 ? 2047 : height;
  73.  
  74.       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
  75.       PUSH_DATAh(push, src->bo->offset + src_ofst);
  76.       PUSH_DATA (push, src->bo->offset + src_ofst);
  77.  
  78.       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
  79.       PUSH_DATAh(push, dst->bo->offset + dst_ofst);
  80.       PUSH_DATA (push, dst->bo->offset + dst_ofst);
  81.  
  82.       if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
  83.          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
  84.          PUSH_DATA (push, src->x * cpp);
  85.          PUSH_DATA (push, sy);
  86.       } else {
  87.          src_ofst += line_count * src->pitch;
  88.       }
  89.       if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
  90.          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
  91.          PUSH_DATA (push, dst->x * cpp);
  92.          PUSH_DATA (push, dy);
  93.       } else {
  94.          dst_ofst += line_count * dst->pitch;
  95.       }
  96.  
  97.       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
  98.       PUSH_DATA (push, nblocksx * cpp);
  99.       PUSH_DATA (push, line_count);
  100.       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
  101.       PUSH_DATA (push, exec);
  102.  
  103.       height -= line_count;
  104.       sy += line_count;
  105.       dy += line_count;
  106.    }
  107.  
  108.    nouveau_bufctx_reset(bctx, 0);
  109. }
  110.  
  111. static void
  112. nve4_m2mf_transfer_rect(struct nvc0_context *nvc0,
  113.                         const struct nv50_m2mf_rect *dst,
  114.                         const struct nv50_m2mf_rect *src,
  115.                         uint32_t nblocksx, uint32_t nblocksy)
  116. {
  117.    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
  118.    struct nouveau_bufctx *bctx = nvc0->bufctx;
  119.    uint32_t exec;
  120.    uint32_t src_base = src->base;
  121.    uint32_t dst_base = dst->base;
  122.    const int cpp = dst->cpp;
  123.  
  124.    assert(dst->cpp == src->cpp);
  125.  
  126.    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
  127.    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
  128.    nouveau_pushbuf_bufctx(push, bctx);
  129.    nouveau_pushbuf_validate(push);
  130.  
  131.    exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */;
  132.  
  133.    if (!nouveau_bo_memtype(dst->bo)) {
  134.       assert(!dst->z);
  135.       dst_base += dst->y * dst->pitch + dst->x * cpp;
  136.       exec |= 0x100; /* DST_MODE_2D_LINEAR */
  137.    }
  138.    if (!nouveau_bo_memtype(src->bo)) {
  139.       assert(!src->z);
  140.       src_base += src->y * src->pitch + src->x * cpp;
  141.       exec |= 0x080; /* SRC_MODE_2D_LINEAR */
  142.    }
  143.  
  144.    BEGIN_NVC0(push, SUBC_COPY(0x070c), 6);
  145.    PUSH_DATA (push, 0x1000 | dst->tile_mode);
  146.    PUSH_DATA (push, dst->pitch);
  147.    PUSH_DATA (push, dst->height);
  148.    PUSH_DATA (push, dst->depth);
  149.    PUSH_DATA (push, dst->z);
  150.    PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp));
  151.  
  152.    BEGIN_NVC0(push, SUBC_COPY(0x0728), 6);
  153.    PUSH_DATA (push, 0x1000 | src->tile_mode);
  154.    PUSH_DATA (push, src->pitch);
  155.    PUSH_DATA (push, src->height);
  156.    PUSH_DATA (push, src->depth);
  157.    PUSH_DATA (push, src->z);
  158.    PUSH_DATA (push, (src->y << 16) | (src->x * cpp));
  159.  
  160.    BEGIN_NVC0(push, SUBC_COPY(0x0400), 8);
  161.    PUSH_DATAh(push, src->bo->offset + src_base);
  162.    PUSH_DATA (push, src->bo->offset + src_base);
  163.    PUSH_DATAh(push, dst->bo->offset + dst_base);
  164.    PUSH_DATA (push, dst->bo->offset + dst_base);
  165.    PUSH_DATA (push, src->pitch);
  166.    PUSH_DATA (push, dst->pitch);
  167.    PUSH_DATA (push, nblocksx * cpp);
  168.    PUSH_DATA (push, nblocksy);
  169.  
  170.    BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
  171.    PUSH_DATA (push, exec);
  172.  
  173.    nouveau_bufctx_reset(bctx, 0);
  174. }
  175.  
  176. void
  177. nvc0_m2mf_push_linear(struct nouveau_context *nv,
  178.                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
  179.                       unsigned size, const void *data)
  180. {
  181.    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
  182.    struct nouveau_pushbuf *push = nv->pushbuf;
  183.    uint32_t *src = (uint32_t *)data;
  184.    unsigned count = (size + 3) / 4;
  185.  
  186.    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
  187.    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
  188.    nouveau_pushbuf_validate(push);
  189.  
  190.    while (count) {
  191.       unsigned nr;
  192.  
  193.       if (!PUSH_SPACE(push, 16))
  194.          break;
  195.       nr = PUSH_AVAIL(push);
  196.       assert(nr >= 16);
  197.       nr = MIN2(count, nr - 9);
  198.       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
  199.  
  200.       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
  201.       PUSH_DATAh(push, dst->offset + offset);
  202.       PUSH_DATA (push, dst->offset + offset);
  203.       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
  204.       PUSH_DATA (push, MIN2(size, nr * 4));
  205.       PUSH_DATA (push, 1);
  206.       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
  207.       PUSH_DATA (push, 0x100111);
  208.  
  209.       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
  210.       BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
  211.       PUSH_DATAp(push, src, nr);
  212.  
  213.       count -= nr;
  214.       src += nr;
  215.       offset += nr * 4;
  216.       size -= nr * 4;
  217.    }
  218.  
  219.    nouveau_bufctx_reset(nvc0->bufctx, 0);
  220. }
  221.  
  222. void
  223. nve4_p2mf_push_linear(struct nouveau_context *nv,
  224.                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
  225.                       unsigned size, const void *data)
  226. {
  227.    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
  228.    struct nouveau_pushbuf *push = nv->pushbuf;
  229.    uint32_t *src = (uint32_t *)data;
  230.    unsigned count = (size + 3) / 4;
  231.  
  232.    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
  233.    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
  234.    nouveau_pushbuf_validate(push);
  235.  
  236.    while (count) {
  237.       unsigned nr;
  238.  
  239.       if (!PUSH_SPACE(push, 16))
  240.          break;
  241.       nr = PUSH_AVAIL(push);
  242.       assert(nr >= 16);
  243.       nr = MIN2(count, nr - 8);
  244.       nr = MIN2(nr, (NV04_PFIFO_MAX_PACKET_LEN - 1));
  245.  
  246.       BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
  247.       PUSH_DATAh(push, dst->offset + offset);
  248.       PUSH_DATA (push, dst->offset + offset);
  249.       BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
  250.       PUSH_DATA (push, MIN2(size, nr * 4));
  251.       PUSH_DATA (push, 1);
  252.       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
  253.       BEGIN_1IC0(push, NVE4_P2MF(EXEC), nr + 1);
  254.       PUSH_DATA (push, 0x1001);
  255.       PUSH_DATAp(push, src, nr);
  256.  
  257.       count -= nr;
  258.       src += nr;
  259.       offset += nr * 4;
  260.       size -= nr * 4;
  261.    }
  262.  
  263.    nouveau_bufctx_reset(nvc0->bufctx, 0);
  264. }
  265.  
  266. static void
  267. nvc0_m2mf_copy_linear(struct nouveau_context *nv,
  268.                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
  269.                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
  270.                       unsigned size)
  271. {
  272.    struct nouveau_pushbuf *push = nv->pushbuf;
  273.    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
  274.  
  275.    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
  276.    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
  277.    nouveau_pushbuf_bufctx(push, bctx);
  278.    nouveau_pushbuf_validate(push);
  279.  
  280.    while (size) {
  281.       unsigned bytes = MIN2(size, 1 << 17);
  282.  
  283.       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
  284.       PUSH_DATAh(push, dst->offset + dstoff);
  285.       PUSH_DATA (push, dst->offset + dstoff);
  286.       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
  287.       PUSH_DATAh(push, src->offset + srcoff);
  288.       PUSH_DATA (push, src->offset + srcoff);
  289.       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
  290.       PUSH_DATA (push, bytes);
  291.       PUSH_DATA (push, 1);
  292.       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
  293.       PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
  294.                  NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
  295.  
  296.       srcoff += bytes;
  297.       dstoff += bytes;
  298.       size -= bytes;
  299.    }
  300.  
  301.    nouveau_bufctx_reset(bctx, 0);
  302. }
  303.  
  304. static void
  305. nve4_m2mf_copy_linear(struct nouveau_context *nv,
  306.                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
  307.                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
  308.                       unsigned size)
  309. {
  310.    struct nouveau_pushbuf *push = nv->pushbuf;
  311.    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
  312.  
  313.    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
  314.    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
  315.    nouveau_pushbuf_bufctx(push, bctx);
  316.    nouveau_pushbuf_validate(push);
  317.  
  318.    BEGIN_NVC0(push, SUBC_COPY(0x0400), 4);
  319.    PUSH_DATAh(push, src->offset + srcoff);
  320.    PUSH_DATA (push, src->offset + srcoff);
  321.    PUSH_DATAh(push, dst->offset + dstoff);
  322.    PUSH_DATA (push, dst->offset + dstoff);
  323.    BEGIN_NVC0(push, SUBC_COPY(0x0418), 1);
  324.    PUSH_DATA (push, size);
  325.    BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
  326.    PUSH_DATA (push, 0x186);
  327.  
  328.    nouveau_bufctx_reset(bctx, 0);
  329. }
  330.  
  331.  
  332. static INLINE boolean
  333. nvc0_mt_transfer_can_map_directly(struct nv50_miptree *mt)
  334. {
  335.    if (mt->base.domain == NOUVEAU_BO_VRAM)
  336.       return FALSE;
  337.    if (mt->base.base.usage != PIPE_USAGE_STAGING)
  338.       return FALSE;
  339.    return !nouveau_bo_memtype(mt->base.bo);
  340. }
  341.  
  342. static INLINE boolean
  343. nvc0_mt_sync(struct nvc0_context *nvc0, struct nv50_miptree *mt, unsigned usage)
  344. {
  345.    if (!mt->base.mm) {
  346.       uint32_t access = (usage & PIPE_TRANSFER_WRITE) ?
  347.          NOUVEAU_BO_WR : NOUVEAU_BO_RD;
  348.       return !nouveau_bo_wait(mt->base.bo, access, nvc0->base.client);
  349.    }
  350.    if (usage & PIPE_TRANSFER_WRITE)
  351.       return !mt->base.fence || nouveau_fence_wait(mt->base.fence);
  352.    return !mt->base.fence_wr || nouveau_fence_wait(mt->base.fence_wr);
  353. }
  354.  
  355. void *
  356. nvc0_miptree_transfer_map(struct pipe_context *pctx,
  357.                           struct pipe_resource *res,
  358.                           unsigned level,
  359.                           unsigned usage,
  360.                           const struct pipe_box *box,
  361.                           struct pipe_transfer **ptransfer)
  362. {
  363.    struct nvc0_context *nvc0 = nvc0_context(pctx);
  364.    struct nouveau_device *dev = nvc0->screen->base.device;
  365.    struct nv50_miptree *mt = nv50_miptree(res);
  366.    struct nvc0_transfer *tx;
  367.    uint32_t size;
  368.    int ret;
  369.    unsigned flags = 0;
  370.  
  371.    if (nvc0_mt_transfer_can_map_directly(mt)) {
  372.       ret = !nvc0_mt_sync(nvc0, mt, usage);
  373.       if (!ret)
  374.          ret = nouveau_bo_map(mt->base.bo, 0, NULL);
  375.       if (ret &&
  376.           (usage & PIPE_TRANSFER_MAP_DIRECTLY))
  377.          return NULL;
  378.       if (!ret)
  379.          usage |= PIPE_TRANSFER_MAP_DIRECTLY;
  380.    } else
  381.    if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
  382.       return NULL;
  383.  
  384.    tx = CALLOC_STRUCT(nvc0_transfer);
  385.    if (!tx)
  386.       return NULL;
  387.  
  388.    pipe_resource_reference(&tx->base.resource, res);
  389.  
  390.    tx->base.level = level;
  391.    tx->base.usage = usage;
  392.    tx->base.box = *box;
  393.  
  394.    if (util_format_is_plain(res->format)) {
  395.       tx->nblocksx = box->width << mt->ms_x;
  396.       tx->nblocksy = box->height << mt->ms_y;
  397.    } else {
  398.       tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
  399.       tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
  400.    }
  401.    tx->nlayers = box->depth;
  402.  
  403.    tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
  404.    tx->base.layer_stride = tx->nblocksy * tx->base.stride;
  405.  
  406.    if (usage & PIPE_TRANSFER_MAP_DIRECTLY) {
  407.       tx->base.stride = align(tx->base.stride, 128);
  408.       *ptransfer = &tx->base;
  409.       return mt->base.bo->map + mt->base.offset;
  410.    }
  411.  
  412.    nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
  413.  
  414.    size = tx->base.layer_stride;
  415.  
  416.    ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
  417.                         size * tx->nlayers, NULL, &tx->rect[1].bo);
  418.    if (ret) {
  419.       pipe_resource_reference(&tx->base.resource, NULL);
  420.       FREE(tx);
  421.       return NULL;
  422.    }
  423.  
  424.    tx->rect[1].cpp = tx->rect[0].cpp;
  425.    tx->rect[1].width = tx->nblocksx;
  426.    tx->rect[1].height = tx->nblocksy;
  427.    tx->rect[1].depth = 1;
  428.    tx->rect[1].pitch = tx->base.stride;
  429.    tx->rect[1].domain = NOUVEAU_BO_GART;
  430.  
  431.    if (usage & PIPE_TRANSFER_READ) {
  432.       unsigned base = tx->rect[0].base;
  433.       unsigned z = tx->rect[0].z;
  434.       unsigned i;
  435.       for (i = 0; i < tx->nlayers; ++i) {
  436.          nvc0->m2mf_copy_rect(nvc0, &tx->rect[1], &tx->rect[0],
  437.                               tx->nblocksx, tx->nblocksy);
  438.          if (mt->layout_3d)
  439.             tx->rect[0].z++;
  440.          else
  441.             tx->rect[0].base += mt->layer_stride;
  442.          tx->rect[1].base += size;
  443.       }
  444.       tx->rect[0].z = z;
  445.       tx->rect[0].base = base;
  446.       tx->rect[1].base = 0;
  447.    }
  448.  
  449.    if (tx->rect[1].bo->map) {
  450.       *ptransfer = &tx->base;
  451.       return tx->rect[1].bo->map;
  452.    }
  453.  
  454.    if (usage & PIPE_TRANSFER_READ)
  455.       flags = NOUVEAU_BO_RD;
  456.    if (usage & PIPE_TRANSFER_WRITE)
  457.       flags |= NOUVEAU_BO_WR;
  458.  
  459.    ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client);
  460.    if (ret) {
  461.       pipe_resource_reference(&tx->base.resource, NULL);
  462.       nouveau_bo_ref(NULL, &tx->rect[1].bo);
  463.       FREE(tx);
  464.       return NULL;
  465.    }
  466.  
  467.    *ptransfer = &tx->base;
  468.    return tx->rect[1].bo->map;
  469. }
  470.  
  471. void
  472. nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
  473.                             struct pipe_transfer *transfer)
  474. {
  475.    struct nvc0_context *nvc0 = nvc0_context(pctx);
  476.    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
  477.    struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
  478.    unsigned i;
  479.  
  480.    if (tx->base.usage & PIPE_TRANSFER_MAP_DIRECTLY) {
  481.       pipe_resource_reference(&transfer->resource, NULL);
  482.  
  483.       FREE(tx);
  484.       return;
  485.    }
  486.  
  487.    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
  488.       for (i = 0; i < tx->nlayers; ++i) {
  489.          nvc0->m2mf_copy_rect(nvc0, &tx->rect[0], &tx->rect[1],
  490.                               tx->nblocksx, tx->nblocksy);
  491.          if (mt->layout_3d)
  492.             tx->rect[0].z++;
  493.          else
  494.             tx->rect[0].base += mt->layer_stride;
  495.          tx->rect[1].base += tx->nblocksy * tx->base.stride;
  496.       }
  497.       NOUVEAU_DRV_STAT(&nvc0->screen->base, tex_transfers_wr, 1);
  498.    }
  499.    if (tx->base.usage & PIPE_TRANSFER_READ)
  500.       NOUVEAU_DRV_STAT(&nvc0->screen->base, tex_transfers_rd, 1);
  501.  
  502.    nouveau_bo_ref(NULL, &tx->rect[1].bo);
  503.    pipe_resource_reference(&transfer->resource, NULL);
  504.  
  505.    FREE(tx);
  506. }
  507.  
  508. void
  509. nvc0_cb_push(struct nouveau_context *nv,
  510.              struct nouveau_bo *bo, unsigned domain,
  511.              unsigned base, unsigned size,
  512.              unsigned offset, unsigned words, const uint32_t *data)
  513. {
  514.    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
  515.    struct nouveau_pushbuf *push = nv->pushbuf;
  516.  
  517.    NOUVEAU_DRV_STAT(nv->screen, constbuf_upload_count, 1);
  518.    NOUVEAU_DRV_STAT(nv->screen, constbuf_upload_bytes, words * 4);
  519.  
  520.    assert(!(offset & 3));
  521.    size = align(size, 0x100);
  522.  
  523.    nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
  524.    nouveau_pushbuf_bufctx(push, bctx);
  525.    nouveau_pushbuf_validate(push);
  526.  
  527.    BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
  528.    PUSH_DATA (push, size);
  529.    PUSH_DATAh(push, bo->offset + base);
  530.    PUSH_DATA (push, bo->offset + base);
  531.  
  532.    while (words) {
  533.       unsigned nr = PUSH_AVAIL(push);
  534.       nr = MIN2(nr, words);
  535.       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
  536.  
  537.       BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
  538.       PUSH_DATA (push, offset);
  539.       PUSH_DATAp(push, data, nr);
  540.  
  541.       words -= nr;
  542.       data += nr;
  543.       offset += nr * 4;
  544.    }
  545.  
  546.    nouveau_bufctx_reset(bctx, 0);
  547. }
  548.  
  549. void
  550. nvc0_init_transfer_functions(struct nvc0_context *nvc0)
  551. {
  552.    if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) {
  553.       nvc0->m2mf_copy_rect = nve4_m2mf_transfer_rect;
  554.       nvc0->base.copy_data = nve4_m2mf_copy_linear;
  555.       nvc0->base.push_data = nve4_p2mf_push_linear;
  556.    } else {
  557.       nvc0->m2mf_copy_rect = nvc0_m2mf_transfer_rect;
  558.       nvc0->base.copy_data = nvc0_m2mf_copy_linear;
  559.       nvc0->base.push_data = nvc0_m2mf_push_linear;
  560.    }
  561.    nvc0->base.push_cb = nvc0_cb_push;
  562. }
  563.