Subversion Repositories Kolibri OS

Rev

Go to most recent revision | Blame | Last modification | View Log | RSS feed

  1. /**************************************************************************
  2.  *
  3.  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
  4.  * All Rights Reserved.
  5.  *
  6.  * Permission is hereby granted, free of charge, to any person obtaining a
  7.  * copy of this software and associated documentation files (the
  8.  * "Software"), to deal in the Software without restriction, including
  9.  * without limitation the rights to use, copy, modify, merge, publish,
  10.  * distribute, sub license, and/or sell copies of the Software, and to
  11.  * permit persons to whom the Software is furnished to do so, subject to
  12.  * the following conditions:
  13.  *
  14.  * The above copyright notice and this permission notice (including the
  15.  * next paragraph) shall be included in all copies or substantial portions
  16.  * of the Software.
  17.  *
  18.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19.  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20.  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21.  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22.  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23.  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24.  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25.  *
  26.  **************************************************************************/
  27.  
  28. #include <GL/gl.h>
  29. #include <GL/internal/dri_interface.h>
  30.  
  31. #include "intel_batchbuffer.h"
  32. #include "intel_chipset.h"
  33. #include "intel_mipmap_tree.h"
  34. #include "intel_regions.h"
  35. #include "intel_resolve_map.h"
  36. #include "intel_tex.h"
  37. #include "intel_blit.h"
  38.  
  39. #include "brw_blorp.h"
  40. #include "brw_context.h"
  41.  
  42. #include "main/enums.h"
  43. #include "main/formats.h"
  44. #include "main/glformats.h"
  45. #include "main/texcompress_etc.h"
  46. #include "main/teximage.h"
  47.  
  48. #define FILE_DEBUG_FLAG DEBUG_MIPTREE
  49.  
  50. static GLenum
  51. target_to_target(GLenum target)
  52. {
  53.    switch (target) {
  54.    case GL_TEXTURE_CUBE_MAP_POSITIVE_X_ARB:
  55.    case GL_TEXTURE_CUBE_MAP_NEGATIVE_X_ARB:
  56.    case GL_TEXTURE_CUBE_MAP_POSITIVE_Y_ARB:
  57.    case GL_TEXTURE_CUBE_MAP_NEGATIVE_Y_ARB:
  58.    case GL_TEXTURE_CUBE_MAP_POSITIVE_Z_ARB:
  59.    case GL_TEXTURE_CUBE_MAP_NEGATIVE_Z_ARB:
  60.       return GL_TEXTURE_CUBE_MAP_ARB;
  61.    default:
  62.       return target;
  63.    }
  64. }
  65.  
  66.  
  67. /**
  68.  * Determine which MSAA layout should be used by the MSAA surface being
  69.  * created, based on the chip generation and the surface type.
  70.  */
  71. static enum intel_msaa_layout
  72. compute_msaa_layout(struct brw_context *brw, gl_format format, GLenum target)
  73. {
  74.    /* Prior to Gen7, all MSAA surfaces used IMS layout. */
  75.    if (brw->gen < 7)
  76.       return INTEL_MSAA_LAYOUT_IMS;
  77.  
  78.    /* In Gen7, IMS layout is only used for depth and stencil buffers. */
  79.    switch (_mesa_get_format_base_format(format)) {
  80.    case GL_DEPTH_COMPONENT:
  81.    case GL_STENCIL_INDEX:
  82.    case GL_DEPTH_STENCIL:
  83.       return INTEL_MSAA_LAYOUT_IMS;
  84.    default:
  85.       /* From the Ivy Bridge PRM, Vol4 Part1 p77 ("MCS Enable"):
  86.        *
  87.        *   This field must be set to 0 for all SINT MSRTs when all RT channels
  88.        *   are not written
  89.        *
  90.        * In practice this means that we have to disable MCS for all signed
  91.        * integer MSAA buffers.  The alternative, to disable MCS only when one
  92.        * of the render target channels is disabled, is impractical because it
  93.        * would require converting between CMS and UMS MSAA layouts on the fly,
  94.        * which is expensive.
  95.        */
  96.       if (_mesa_get_format_datatype(format) == GL_INT) {
  97.          /* TODO: is this workaround needed for future chipsets? */
  98.          assert(brw->gen == 7);
  99.          return INTEL_MSAA_LAYOUT_UMS;
  100.       } else {
  101.          /* For now, if we're going to be texturing from this surface,
  102.           * force UMS, so that the shader doesn't have to do different things
  103.           * based on whether there's a multisample control surface needing sampled first.
  104.           * We can't just blindly read the MCS surface in all cases because:
  105.           *
  106.           * From the Ivy Bridge PRM, Vol4 Part1 p77 ("MCS Enable"):
  107.           *
  108.           *    If this field is disabled and the sampling engine <ld_mcs> message
  109.           *    is issued on this surface, the MCS surface may be accessed. Software
  110.           *    must ensure that the surface is defined to avoid GTT errors.
  111.           */
  112.          if (target == GL_TEXTURE_2D_MULTISAMPLE ||
  113.              target == GL_TEXTURE_2D_MULTISAMPLE_ARRAY) {
  114.             return INTEL_MSAA_LAYOUT_UMS;
  115.          } else {
  116.             return INTEL_MSAA_LAYOUT_CMS;
  117.          }
  118.       }
  119.    }
  120. }
  121.  
  122.  
  123. /**
  124.  * For single-sampled render targets ("non-MSRT"), the MCS buffer is a
  125.  * scaled-down bitfield representation of the color buffer which is capable of
  126.  * recording when blocks of the color buffer are equal to the clear value.
  127.  * This function returns the block size that will be used by the MCS buffer
  128.  * corresponding to a certain color miptree.
  129.  *
  130.  * From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render Target(s)",
  131.  * beneath the "Fast Color Clear" bullet (p327):
  132.  *
  133.  *     The following table describes the RT alignment
  134.  *
  135.  *                       Pixels  Lines
  136.  *         TiledY RT CL
  137.  *             bpp
  138.  *              32          8      4
  139.  *              64          4      4
  140.  *             128          2      4
  141.  *         TiledX RT CL
  142.  *             bpp
  143.  *              32         16      2
  144.  *              64          8      2
  145.  *             128          4      2
  146.  *
  147.  * This alignment has the following uses:
  148.  *
  149.  * - For figuring out the size of the MCS buffer.  Each 4k tile in the MCS
  150.  *   buffer contains 128 blocks horizontally and 256 blocks vertically.
  151.  *
  152.  * - For figuring out alignment restrictions for a fast clear operation.  Fast
  153.  *   clear operations must always clear aligned multiples of 16 blocks
  154.  *   horizontally and 32 blocks vertically.
  155.  *
  156.  * - For scaling down the coordinates sent through the render pipeline during
  157.  *   a fast clear.  X coordinates must be scaled down by 8 times the block
  158.  *   width, and Y coordinates by 16 times the block height.
  159.  *
  160.  * - For scaling down the coordinates sent through the render pipeline during
  161.  *   a "Render Target Resolve" operation.  X coordinates must be scaled down
  162.  *   by half the block width, and Y coordinates by half the block height.
  163.  */
  164. void
  165. intel_get_non_msrt_mcs_alignment(struct brw_context *brw,
  166.                                  struct intel_mipmap_tree *mt,
  167.                                  unsigned *width_px, unsigned *height)
  168. {
  169.    switch (mt->region->tiling) {
  170.    default:
  171.       assert(!"Non-MSRT MCS requires X or Y tiling");
  172.       /* In release builds, fall through */
  173.    case I915_TILING_Y:
  174.       *width_px = 32 / mt->cpp;
  175.       *height = 4;
  176.       break;
  177.    case I915_TILING_X:
  178.       *width_px = 64 / mt->cpp;
  179.       *height = 2;
  180.    }
  181. }
  182.  
  183.  
  184. /**
  185.  * For a single-sampled render target ("non-MSRT"), determine if an MCS buffer
  186.  * can be used.
  187.  *
  188.  * From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render Target(s)",
  189.  * beneath the "Fast Color Clear" bullet (p326):
  190.  *
  191.  *     - Support is limited to tiled render targets.
  192.  *     - Support is for non-mip-mapped and non-array surface types only.
  193.  *
  194.  * And then later, on p327:
  195.  *
  196.  *     - MCS buffer for non-MSRT is supported only for RT formats 32bpp,
  197.  *       64bpp, and 128bpp.
  198.  */
  199. bool
  200. intel_is_non_msrt_mcs_buffer_supported(struct brw_context *brw,
  201.                                        struct intel_mipmap_tree *mt)
  202. {
  203.    /* MCS support does not exist prior to Gen7 */
  204.    if (brw->gen < 7)
  205.       return false;
  206.  
  207.    /* MCS is only supported for color buffers */
  208.    switch (_mesa_get_format_base_format(mt->format)) {
  209.    case GL_DEPTH_COMPONENT:
  210.    case GL_DEPTH_STENCIL:
  211.    case GL_STENCIL_INDEX:
  212.       return false;
  213.    }
  214.  
  215.    if (mt->region->tiling != I915_TILING_X &&
  216.        mt->region->tiling != I915_TILING_Y)
  217.       return false;
  218.    if (mt->cpp != 4 && mt->cpp != 8 && mt->cpp != 16)
  219.       return false;
  220.    if (mt->first_level != 0 || mt->last_level != 0)
  221.       return false;
  222.    if (mt->physical_depth0 != 1)
  223.       return false;
  224.  
  225.    /* There's no point in using an MCS buffer if the surface isn't in a
  226.     * renderable format.
  227.     */
  228.    if (!brw->format_supported_as_render_target[mt->format])
  229.       return false;
  230.  
  231.    return true;
  232. }
  233.  
  234.  
  235. /**
  236.  * @param for_bo Indicates that the caller is
  237.  *        intel_miptree_create_for_bo(). If true, then do not create
  238.  *        \c stencil_mt.
  239.  */
  240. struct intel_mipmap_tree *
  241. intel_miptree_create_layout(struct brw_context *brw,
  242.                             GLenum target,
  243.                             gl_format format,
  244.                             GLuint first_level,
  245.                             GLuint last_level,
  246.                             GLuint width0,
  247.                             GLuint height0,
  248.                             GLuint depth0,
  249.                             bool for_bo,
  250.                             GLuint num_samples)
  251. {
  252.    struct intel_mipmap_tree *mt = calloc(sizeof(*mt), 1);
  253.    if (!mt)
  254.       return NULL;
  255.  
  256.    DBG("%s target %s format %s level %d..%d <-- %p\n", __FUNCTION__,
  257.        _mesa_lookup_enum_by_nr(target),
  258.        _mesa_get_format_name(format),
  259.        first_level, last_level, mt);
  260.  
  261.    mt->target = target_to_target(target);
  262.    mt->format = format;
  263.    mt->first_level = first_level;
  264.    mt->last_level = last_level;
  265.    mt->logical_width0 = width0;
  266.    mt->logical_height0 = height0;
  267.    mt->logical_depth0 = depth0;
  268.    mt->mcs_state = INTEL_MCS_STATE_NONE;
  269.  
  270.    /* The cpp is bytes per (1, blockheight)-sized block for compressed
  271.     * textures.  This is why you'll see divides by blockheight all over
  272.     */
  273.    unsigned bw, bh;
  274.    _mesa_get_format_block_size(format, &bw, &bh);
  275.    assert(_mesa_get_format_bytes(mt->format) % bw == 0);
  276.    mt->cpp = _mesa_get_format_bytes(mt->format) / bw;
  277.  
  278.    mt->num_samples = num_samples;
  279.    mt->compressed = _mesa_is_format_compressed(format);
  280.    mt->msaa_layout = INTEL_MSAA_LAYOUT_NONE;
  281.    mt->refcount = 1;
  282.  
  283.    if (num_samples > 1) {
  284.       /* Adjust width/height/depth for MSAA */
  285.       mt->msaa_layout = compute_msaa_layout(brw, format, mt->target);
  286.       if (mt->msaa_layout == INTEL_MSAA_LAYOUT_IMS) {
  287.          /* In the Sandy Bridge PRM, volume 4, part 1, page 31, it says:
  288.           *
  289.           *     "Any of the other messages (sample*, LOD, load4) used with a
  290.           *      (4x) multisampled surface will in-effect sample a surface with
  291.           *      double the height and width as that indicated in the surface
  292.           *      state. Each pixel position on the original-sized surface is
  293.           *      replaced with a 2x2 of samples with the following arrangement:
  294.           *
  295.           *         sample 0 sample 2
  296.           *         sample 1 sample 3"
  297.           *
  298.           * Thus, when sampling from a multisampled texture, it behaves as
  299.           * though the layout in memory for (x,y,sample) is:
  300.           *
  301.           *      (0,0,0) (0,0,2)   (1,0,0) (1,0,2)
  302.           *      (0,0,1) (0,0,3)   (1,0,1) (1,0,3)
  303.           *
  304.           *      (0,1,0) (0,1,2)   (1,1,0) (1,1,2)
  305.           *      (0,1,1) (0,1,3)   (1,1,1) (1,1,3)
  306.           *
  307.           * However, the actual layout of multisampled data in memory is:
  308.           *
  309.           *      (0,0,0) (1,0,0)   (0,0,1) (1,0,1)
  310.           *      (0,1,0) (1,1,0)   (0,1,1) (1,1,1)
  311.           *
  312.           *      (0,0,2) (1,0,2)   (0,0,3) (1,0,3)
  313.           *      (0,1,2) (1,1,2)   (0,1,3) (1,1,3)
  314.           *
  315.           * This pattern repeats for each 2x2 pixel block.
  316.           *
  317.           * As a result, when calculating the size of our 4-sample buffer for
  318.           * an odd width or height, we have to align before scaling up because
  319.           * sample 3 is in that bottom right 2x2 block.
  320.           */
  321.          switch (num_samples) {
  322.          case 4:
  323.             width0 = ALIGN(width0, 2) * 2;
  324.             height0 = ALIGN(height0, 2) * 2;
  325.             break;
  326.          case 8:
  327.             width0 = ALIGN(width0, 2) * 4;
  328.             height0 = ALIGN(height0, 2) * 2;
  329.             break;
  330.          default:
  331.             /* num_samples should already have been quantized to 0, 1, 4, or
  332.              * 8.
  333.              */
  334.             assert(false);
  335.          }
  336.       } else {
  337.          /* Non-interleaved */
  338.          depth0 *= num_samples;
  339.       }
  340.    }
  341.  
  342.    /* array_spacing_lod0 is only used for non-IMS MSAA surfaces.  TODO: can we
  343.     * use it elsewhere?
  344.     */
  345.    switch (mt->msaa_layout) {
  346.    case INTEL_MSAA_LAYOUT_NONE:
  347.    case INTEL_MSAA_LAYOUT_IMS:
  348.       mt->array_spacing_lod0 = false;
  349.       break;
  350.    case INTEL_MSAA_LAYOUT_UMS:
  351.    case INTEL_MSAA_LAYOUT_CMS:
  352.       mt->array_spacing_lod0 = true;
  353.       break;
  354.    }
  355.  
  356.    if (target == GL_TEXTURE_CUBE_MAP) {
  357.       assert(depth0 == 1);
  358.       depth0 = 6;
  359.    }
  360.  
  361.    mt->physical_width0 = width0;
  362.    mt->physical_height0 = height0;
  363.    mt->physical_depth0 = depth0;
  364.  
  365.    if (!for_bo &&
  366.        _mesa_get_format_base_format(format) == GL_DEPTH_STENCIL &&
  367.        (brw->must_use_separate_stencil ||
  368.         (brw->has_separate_stencil && brw_is_hiz_depth_format(brw, format)))) {
  369.       mt->stencil_mt = intel_miptree_create(brw,
  370.                                             mt->target,
  371.                                             MESA_FORMAT_S8,
  372.                                             mt->first_level,
  373.                                             mt->last_level,
  374.                                             mt->logical_width0,
  375.                                             mt->logical_height0,
  376.                                             mt->logical_depth0,
  377.                                             true,
  378.                                             num_samples,
  379.                                             INTEL_MIPTREE_TILING_ANY);
  380.       if (!mt->stencil_mt) {
  381.          intel_miptree_release(&mt);
  382.          return NULL;
  383.       }
  384.  
  385.       /* Fix up the Z miptree format for how we're splitting out separate
  386.        * stencil.  Gen7 expects there to be no stencil bits in its depth buffer.
  387.        */
  388.       if (mt->format == MESA_FORMAT_S8_Z24) {
  389.          mt->format = MESA_FORMAT_X8_Z24;
  390.       } else if (mt->format == MESA_FORMAT_Z32_FLOAT_X24S8) {
  391.          mt->format = MESA_FORMAT_Z32_FLOAT;
  392.          mt->cpp = 4;
  393.       } else {
  394.          _mesa_problem(NULL, "Unknown format %s in separate stencil mt\n",
  395.                        _mesa_get_format_name(mt->format));
  396.       }
  397.    }
  398.  
  399.    brw_miptree_layout(brw, mt);
  400.  
  401.    return mt;
  402. }
  403.  
  404. /**
  405.  * \brief Helper function for intel_miptree_create().
  406.  */
  407. static uint32_t
  408. intel_miptree_choose_tiling(struct brw_context *brw,
  409.                             gl_format format,
  410.                             uint32_t width0,
  411.                             uint32_t num_samples,
  412.                             enum intel_miptree_tiling_mode requested,
  413.                             struct intel_mipmap_tree *mt)
  414. {
  415.    if (format == MESA_FORMAT_S8) {
  416.       /* The stencil buffer is W tiled. However, we request from the kernel a
  417.        * non-tiled buffer because the GTT is incapable of W fencing.
  418.        */
  419.       return I915_TILING_NONE;
  420.    }
  421.  
  422.    /* Some usages may want only one type of tiling, like depth miptrees (Y
  423.     * tiled), or temporary BOs for uploading data once (linear).
  424.     */
  425.    switch (requested) {
  426.    case INTEL_MIPTREE_TILING_ANY:
  427.       break;
  428.    case INTEL_MIPTREE_TILING_Y:
  429.       return I915_TILING_Y;
  430.    case INTEL_MIPTREE_TILING_NONE:
  431.       return I915_TILING_NONE;
  432.    }
  433.  
  434.    if (num_samples > 1) {
  435.       /* From p82 of the Sandy Bridge PRM, dw3[1] of SURFACE_STATE ("Tiled
  436.        * Surface"):
  437.        *
  438.        *   [DevSNB+]: For multi-sample render targets, this field must be
  439.        *   1. MSRTs can only be tiled.
  440.        *
  441.        * Our usual reason for preferring X tiling (fast blits using the
  442.        * blitting engine) doesn't apply to MSAA, since we'll generally be
  443.        * downsampling or upsampling when blitting between the MSAA buffer
  444.        * and another buffer, and the blitting engine doesn't support that.
  445.        * So use Y tiling, since it makes better use of the cache.
  446.        */
  447.       return I915_TILING_Y;
  448.    }
  449.  
  450.    GLenum base_format = _mesa_get_format_base_format(format);
  451.    if (base_format == GL_DEPTH_COMPONENT ||
  452.        base_format == GL_DEPTH_STENCIL_EXT)
  453.       return I915_TILING_Y;
  454.  
  455.    int minimum_pitch = mt->total_width * mt->cpp;
  456.  
  457.    /* If the width is much smaller than a tile, don't bother tiling. */
  458.    if (minimum_pitch < 64)
  459.       return I915_TILING_NONE;
  460.  
  461.    if (ALIGN(minimum_pitch, 512) >= 32768) {
  462.       perf_debug("%dx%d miptree too large to blit, falling back to untiled",
  463.                  mt->total_width, mt->total_height);
  464.       return I915_TILING_NONE;
  465.    }
  466.  
  467.    /* Pre-gen6 doesn't have BLORP to handle Y-tiling, so use X-tiling. */
  468.    if (brw->gen < 6)
  469.       return I915_TILING_X;
  470.  
  471.    /* From the Sandybridge PRM, Volume 1, Part 2, page 32:
  472.     * "NOTE: 128BPE Format Color Buffer ( render target ) MUST be either TileX
  473.     *  or Linear."
  474.     * 128 bits per pixel translates to 16 bytes per pixel.  This is necessary
  475.     * all the way back to 965, but is explicitly permitted on Gen7.
  476.     */
  477.    if (brw->gen != 7 && mt->cpp >= 16)
  478.       return I915_TILING_X;
  479.  
  480.    return I915_TILING_Y | I915_TILING_X;
  481. }
  482.  
  483. struct intel_mipmap_tree *
  484. intel_miptree_create(struct brw_context *brw,
  485.                      GLenum target,
  486.                      gl_format format,
  487.                      GLuint first_level,
  488.                      GLuint last_level,
  489.                      GLuint width0,
  490.                      GLuint height0,
  491.                      GLuint depth0,
  492.                      bool expect_accelerated_upload,
  493.                      GLuint num_samples,
  494.                      enum intel_miptree_tiling_mode requested_tiling)
  495. {
  496.    struct intel_mipmap_tree *mt;
  497.    gl_format tex_format = format;
  498.    gl_format etc_format = MESA_FORMAT_NONE;
  499.    GLuint total_width, total_height;
  500.  
  501.    if (!brw->is_baytrail) {
  502.       switch (format) {
  503.       case MESA_FORMAT_ETC1_RGB8:
  504.          format = MESA_FORMAT_RGBX8888_REV;
  505.          break;
  506.       case MESA_FORMAT_ETC2_RGB8:
  507.          format = MESA_FORMAT_RGBX8888_REV;
  508.          break;
  509.       case MESA_FORMAT_ETC2_SRGB8:
  510.       case MESA_FORMAT_ETC2_SRGB8_ALPHA8_EAC:
  511.       case MESA_FORMAT_ETC2_SRGB8_PUNCHTHROUGH_ALPHA1:
  512.          format = MESA_FORMAT_SARGB8;
  513.          break;
  514.       case MESA_FORMAT_ETC2_RGBA8_EAC:
  515.       case MESA_FORMAT_ETC2_RGB8_PUNCHTHROUGH_ALPHA1:
  516.          format = MESA_FORMAT_RGBA8888_REV;
  517.          break;
  518.       case MESA_FORMAT_ETC2_R11_EAC:
  519.          format = MESA_FORMAT_R16;
  520.          break;
  521.       case MESA_FORMAT_ETC2_SIGNED_R11_EAC:
  522.          format = MESA_FORMAT_SIGNED_R16;
  523.          break;
  524.       case MESA_FORMAT_ETC2_RG11_EAC:
  525.          format = MESA_FORMAT_GR1616;
  526.          break;
  527.       case MESA_FORMAT_ETC2_SIGNED_RG11_EAC:
  528.          format = MESA_FORMAT_SIGNED_GR1616;
  529.          break;
  530.       default:
  531.          /* Non ETC1 / ETC2 format */
  532.          break;
  533.       }
  534.    }
  535.  
  536.    etc_format = (format != tex_format) ? tex_format : MESA_FORMAT_NONE;
  537.  
  538.    mt = intel_miptree_create_layout(brw, target, format,
  539.                                       first_level, last_level, width0,
  540.                                       height0, depth0,
  541.                                       false, num_samples);
  542.    /*
  543.     * pitch == 0 || height == 0  indicates the null texture
  544.     */
  545.    if (!mt || !mt->total_width || !mt->total_height) {
  546.       intel_miptree_release(&mt);
  547.       return NULL;
  548.    }
  549.  
  550.    total_width = mt->total_width;
  551.    total_height = mt->total_height;
  552.  
  553.    if (format == MESA_FORMAT_S8) {
  554.       /* Align to size of W tile, 64x64. */
  555.       total_width = ALIGN(total_width, 64);
  556.       total_height = ALIGN(total_height, 64);
  557.    }
  558.  
  559.    uint32_t tiling = intel_miptree_choose_tiling(brw, format, width0,
  560.                                                  num_samples, requested_tiling,
  561.                                                  mt);
  562.    bool y_or_x = tiling == (I915_TILING_Y | I915_TILING_X);
  563.  
  564.    mt->etc_format = etc_format;
  565.    mt->region = intel_region_alloc(brw->intelScreen,
  566.                                    y_or_x ? I915_TILING_Y : tiling,
  567.                                    mt->cpp,
  568.                                    total_width,
  569.                                    total_height,
  570.                                    expect_accelerated_upload);
  571.  
  572.    /* If the region is too large to fit in the aperture, we need to use the
  573.     * BLT engine to support it.  The BLT paths can't currently handle Y-tiling,
  574.     * so we need to fall back to X.
  575.     */
  576.    if (y_or_x && mt->region->bo->size >= brw->max_gtt_map_object_size) {
  577.       perf_debug("%dx%d miptree larger than aperture; falling back to X-tiled\n",
  578.                  mt->total_width, mt->total_height);
  579.       intel_region_release(&mt->region);
  580.  
  581.       mt->region = intel_region_alloc(brw->intelScreen,
  582.                                       I915_TILING_X,
  583.                                       mt->cpp,
  584.                                       total_width,
  585.                                       total_height,
  586.                                       expect_accelerated_upload);
  587.    }
  588.  
  589.    mt->offset = 0;
  590.  
  591.    if (!mt->region) {
  592.        intel_miptree_release(&mt);
  593.        return NULL;
  594.    }
  595.  
  596.    /* If this miptree is capable of supporting fast color clears, set
  597.     * mcs_state appropriately to ensure that fast clears will occur.
  598.     * Allocation of the MCS miptree will be deferred until the first fast
  599.     * clear actually occurs.
  600.     */
  601.    if (intel_is_non_msrt_mcs_buffer_supported(brw, mt))
  602.       mt->mcs_state = INTEL_MCS_STATE_RESOLVED;
  603.  
  604.    return mt;
  605. }
  606.  
  607. struct intel_mipmap_tree *
  608. intel_miptree_create_for_bo(struct brw_context *brw,
  609.                             drm_intel_bo *bo,
  610.                             gl_format format,
  611.                             uint32_t offset,
  612.                             uint32_t width,
  613.                             uint32_t height,
  614.                             int pitch,
  615.                             uint32_t tiling)
  616. {
  617.    struct intel_mipmap_tree *mt;
  618.  
  619.    struct intel_region *region = calloc(1, sizeof(*region));
  620.    if (!region)
  621.       return NULL;
  622.  
  623.    /* Nothing will be able to use this miptree with the BO if the offset isn't
  624.     * aligned.
  625.     */
  626.    if (tiling != I915_TILING_NONE)
  627.       assert(offset % 4096 == 0);
  628.  
  629.    /* miptrees can't handle negative pitch.  If you need flipping of images,
  630.     * that's outside of the scope of the mt.
  631.     */
  632.    assert(pitch >= 0);
  633.  
  634.    mt = intel_miptree_create_layout(brw, GL_TEXTURE_2D, format,
  635.                                     0, 0,
  636.                                     width, height, 1,
  637.                                     true, 0 /* num_samples */);
  638.    if (!mt)
  639.       return mt;
  640.  
  641.    region->cpp = mt->cpp;
  642.    region->width = width;
  643.    region->height = height;
  644.    region->pitch = pitch;
  645.    region->refcount = 1;
  646.    drm_intel_bo_reference(bo);
  647.    region->bo = bo;
  648.    region->tiling = tiling;
  649.  
  650.    mt->region = region;
  651.    mt->offset = offset;
  652.  
  653.    return mt;
  654. }
  655.  
  656.  
  657. /**
  658.  * For a singlesample DRI2 buffer, this simply wraps the given region with a miptree.
  659.  *
  660.  * For a multisample DRI2 buffer, this wraps the given region with
  661.  * a singlesample miptree, then creates a multisample miptree into which the
  662.  * singlesample miptree is embedded as a child.
  663.  */
  664. struct intel_mipmap_tree*
  665. intel_miptree_create_for_dri2_buffer(struct brw_context *brw,
  666.                                      unsigned dri_attachment,
  667.                                      gl_format format,
  668.                                      uint32_t num_samples,
  669.                                      struct intel_region *region)
  670. {
  671.    struct intel_mipmap_tree *singlesample_mt = NULL;
  672.    struct intel_mipmap_tree *multisample_mt = NULL;
  673.  
  674.    /* Only the front and back buffers, which are color buffers, are shared
  675.     * through DRI2.
  676.     */
  677.    assert(dri_attachment == __DRI_BUFFER_BACK_LEFT ||
  678.           dri_attachment == __DRI_BUFFER_FRONT_LEFT ||
  679.           dri_attachment == __DRI_BUFFER_FAKE_FRONT_LEFT);
  680.    assert(_mesa_get_format_base_format(format) == GL_RGB ||
  681.           _mesa_get_format_base_format(format) == GL_RGBA);
  682.  
  683.    singlesample_mt = intel_miptree_create_for_bo(brw,
  684.                                                  region->bo,
  685.                                                  format,
  686.                                                  0,
  687.                                                  region->width,
  688.                                                  region->height,
  689.                                                  region->pitch,
  690.                                                  region->tiling);
  691.    if (!singlesample_mt)
  692.       return NULL;
  693.    singlesample_mt->region->name = region->name;
  694.  
  695.    /* If this miptree is capable of supporting fast color clears, set
  696.     * mcs_state appropriately to ensure that fast clears will occur.
  697.     * Allocation of the MCS miptree will be deferred until the first fast
  698.     * clear actually occurs.
  699.     */
  700.    if (intel_is_non_msrt_mcs_buffer_supported(brw, singlesample_mt))
  701.       singlesample_mt->mcs_state = INTEL_MCS_STATE_RESOLVED;
  702.  
  703.    if (num_samples == 0)
  704.       return singlesample_mt;
  705.  
  706.    multisample_mt = intel_miptree_create_for_renderbuffer(brw,
  707.                                                           format,
  708.                                                           region->width,
  709.                                                           region->height,
  710.                                                           num_samples);
  711.    if (!multisample_mt) {
  712.       intel_miptree_release(&singlesample_mt);
  713.       return NULL;
  714.    }
  715.  
  716.    multisample_mt->singlesample_mt = singlesample_mt;
  717.    multisample_mt->need_downsample = false;
  718.  
  719.    if (brw->is_front_buffer_rendering &&
  720.        (dri_attachment == __DRI_BUFFER_FRONT_LEFT ||
  721.         dri_attachment == __DRI_BUFFER_FAKE_FRONT_LEFT)) {
  722.       intel_miptree_upsample(brw, multisample_mt);
  723.    }
  724.  
  725.    return multisample_mt;
  726. }
  727.  
  728. struct intel_mipmap_tree*
  729. intel_miptree_create_for_renderbuffer(struct brw_context *brw,
  730.                                       gl_format format,
  731.                                       uint32_t width,
  732.                                       uint32_t height,
  733.                                       uint32_t num_samples)
  734. {
  735.    struct intel_mipmap_tree *mt;
  736.    uint32_t depth = 1;
  737.    bool ok;
  738.  
  739.    mt = intel_miptree_create(brw, GL_TEXTURE_2D, format, 0, 0,
  740.                              width, height, depth, true, num_samples,
  741.                              INTEL_MIPTREE_TILING_ANY);
  742.    if (!mt)
  743.       goto fail;
  744.  
  745.    if (brw_is_hiz_depth_format(brw, format)) {
  746.       ok = intel_miptree_alloc_hiz(brw, mt);
  747.       if (!ok)
  748.          goto fail;
  749.    }
  750.  
  751.    if (mt->msaa_layout == INTEL_MSAA_LAYOUT_CMS) {
  752.       ok = intel_miptree_alloc_mcs(brw, mt, num_samples);
  753.       if (!ok)
  754.          goto fail;
  755.    }
  756.  
  757.    return mt;
  758.  
  759. fail:
  760.    intel_miptree_release(&mt);
  761.    return NULL;
  762. }
  763.  
  764. void
  765. intel_miptree_reference(struct intel_mipmap_tree **dst,
  766.                         struct intel_mipmap_tree *src)
  767. {
  768.    if (*dst == src)
  769.       return;
  770.  
  771.    intel_miptree_release(dst);
  772.  
  773.    if (src) {
  774.       src->refcount++;
  775.       DBG("%s %p refcount now %d\n", __FUNCTION__, src, src->refcount);
  776.    }
  777.  
  778.    *dst = src;
  779. }
  780.  
  781.  
  782. void
  783. intel_miptree_release(struct intel_mipmap_tree **mt)
  784. {
  785.    if (!*mt)
  786.       return;
  787.  
  788.    DBG("%s %p refcount will be %d\n", __FUNCTION__, *mt, (*mt)->refcount - 1);
  789.    if (--(*mt)->refcount <= 0) {
  790.       GLuint i;
  791.  
  792.       DBG("%s deleting %p\n", __FUNCTION__, *mt);
  793.  
  794.       intel_region_release(&((*mt)->region));
  795.       intel_miptree_release(&(*mt)->stencil_mt);
  796.       intel_miptree_release(&(*mt)->hiz_mt);
  797.       intel_miptree_release(&(*mt)->mcs_mt);
  798.       intel_miptree_release(&(*mt)->singlesample_mt);
  799.       intel_resolve_map_clear(&(*mt)->hiz_map);
  800.  
  801.       for (i = 0; i < MAX_TEXTURE_LEVELS; i++) {
  802.          free((*mt)->level[i].slice);
  803.       }
  804.  
  805.       free(*mt);
  806.    }
  807.    *mt = NULL;
  808. }
  809.  
  810. void
  811. intel_miptree_get_dimensions_for_image(struct gl_texture_image *image,
  812.                                        int *width, int *height, int *depth)
  813. {
  814.    switch (image->TexObject->Target) {
  815.    case GL_TEXTURE_1D_ARRAY:
  816.       *width = image->Width;
  817.       *height = 1;
  818.       *depth = image->Height;
  819.       break;
  820.    default:
  821.       *width = image->Width;
  822.       *height = image->Height;
  823.       *depth = image->Depth;
  824.       break;
  825.    }
  826. }
  827.  
  828. /**
  829.  * Can the image be pulled into a unified mipmap tree?  This mirrors
  830.  * the completeness test in a lot of ways.
  831.  *
  832.  * Not sure whether I want to pass gl_texture_image here.
  833.  */
  834. bool
  835. intel_miptree_match_image(struct intel_mipmap_tree *mt,
  836.                           struct gl_texture_image *image)
  837. {
  838.    struct intel_texture_image *intelImage = intel_texture_image(image);
  839.    GLuint level = intelImage->base.Base.Level;
  840.    int width, height, depth;
  841.  
  842.    /* glTexImage* choose the texture object based on the target passed in, and
  843.     * objects can't change targets over their lifetimes, so this should be
  844.     * true.
  845.     */
  846.    assert(target_to_target(image->TexObject->Target) == mt->target);
  847.  
  848.    gl_format mt_format = mt->format;
  849.    if (mt->format == MESA_FORMAT_X8_Z24 && mt->stencil_mt)
  850.       mt_format = MESA_FORMAT_S8_Z24;
  851.    if (mt->format == MESA_FORMAT_Z32_FLOAT && mt->stencil_mt)
  852.       mt_format = MESA_FORMAT_Z32_FLOAT_X24S8;
  853.    if (mt->etc_format != MESA_FORMAT_NONE)
  854.       mt_format = mt->etc_format;
  855.  
  856.    if (image->TexFormat != mt_format)
  857.       return false;
  858.  
  859.    intel_miptree_get_dimensions_for_image(image, &width, &height, &depth);
  860.  
  861.    if (mt->target == GL_TEXTURE_CUBE_MAP)
  862.       depth = 6;
  863.  
  864.    /* Test image dimensions against the base level image adjusted for
  865.     * minification.  This will also catch images not present in the
  866.     * tree, changed targets, etc.
  867.     */
  868.    if (mt->target == GL_TEXTURE_2D_MULTISAMPLE ||
  869.          mt->target == GL_TEXTURE_2D_MULTISAMPLE_ARRAY) {
  870.       /* nonzero level here is always bogus */
  871.       assert(level == 0);
  872.  
  873.       if (width != mt->logical_width0 ||
  874.             height != mt->logical_height0 ||
  875.             depth != mt->logical_depth0) {
  876.          return false;
  877.       }
  878.    }
  879.    else {
  880.       /* all normal textures, renderbuffers, etc */
  881.       if (width != mt->level[level].width ||
  882.           height != mt->level[level].height ||
  883.           depth != mt->level[level].depth) {
  884.          return false;
  885.       }
  886.    }
  887.  
  888.    if (image->NumSamples != mt->num_samples)
  889.       return false;
  890.  
  891.    return true;
  892. }
  893.  
  894.  
  895. void
  896. intel_miptree_set_level_info(struct intel_mipmap_tree *mt,
  897.                              GLuint level,
  898.                              GLuint x, GLuint y,
  899.                              GLuint w, GLuint h, GLuint d)
  900. {
  901.    mt->level[level].width = w;
  902.    mt->level[level].height = h;
  903.    mt->level[level].depth = d;
  904.    mt->level[level].level_x = x;
  905.    mt->level[level].level_y = y;
  906.  
  907.    DBG("%s level %d size: %d,%d,%d offset %d,%d\n", __FUNCTION__,
  908.        level, w, h, d, x, y);
  909.  
  910.    assert(mt->level[level].slice == NULL);
  911.  
  912.    mt->level[level].slice = calloc(d, sizeof(*mt->level[0].slice));
  913.    mt->level[level].slice[0].x_offset = mt->level[level].level_x;
  914.    mt->level[level].slice[0].y_offset = mt->level[level].level_y;
  915. }
  916.  
  917.  
  918. void
  919. intel_miptree_set_image_offset(struct intel_mipmap_tree *mt,
  920.                                GLuint level, GLuint img,
  921.                                GLuint x, GLuint y)
  922. {
  923.    if (img == 0 && level == 0)
  924.       assert(x == 0 && y == 0);
  925.  
  926.    assert(img < mt->level[level].depth);
  927.  
  928.    mt->level[level].slice[img].x_offset = mt->level[level].level_x + x;
  929.    mt->level[level].slice[img].y_offset = mt->level[level].level_y + y;
  930.  
  931.    DBG("%s level %d img %d pos %d,%d\n",
  932.        __FUNCTION__, level, img,
  933.        mt->level[level].slice[img].x_offset,
  934.        mt->level[level].slice[img].y_offset);
  935. }
  936.  
  937. void
  938. intel_miptree_get_image_offset(struct intel_mipmap_tree *mt,
  939.                                GLuint level, GLuint slice,
  940.                                GLuint *x, GLuint *y)
  941. {
  942.    assert(slice < mt->level[level].depth);
  943.  
  944.    *x = mt->level[level].slice[slice].x_offset;
  945.    *y = mt->level[level].slice[slice].y_offset;
  946. }
  947.  
  948. /**
  949.  * Rendering with tiled buffers requires that the base address of the buffer
  950.  * be aligned to a page boundary.  For renderbuffers, and sometimes with
  951.  * textures, we may want the surface to point at a texture image level that
  952.  * isn't at a page boundary.
  953.  *
  954.  * This function returns an appropriately-aligned base offset
  955.  * according to the tiling restrictions, plus any required x/y offset
  956.  * from there.
  957.  */
  958. uint32_t
  959. intel_miptree_get_tile_offsets(struct intel_mipmap_tree *mt,
  960.                                GLuint level, GLuint slice,
  961.                                uint32_t *tile_x,
  962.                                uint32_t *tile_y)
  963. {
  964.    struct intel_region *region = mt->region;
  965.    uint32_t x, y;
  966.    uint32_t mask_x, mask_y;
  967.  
  968.    intel_region_get_tile_masks(region, &mask_x, &mask_y, false);
  969.    intel_miptree_get_image_offset(mt, level, slice, &x, &y);
  970.  
  971.    *tile_x = x & mask_x;
  972.    *tile_y = y & mask_y;
  973.  
  974.    return intel_region_get_aligned_offset(region, x & ~mask_x, y & ~mask_y,
  975.                                           false);
  976. }
  977.  
  978. static void
  979. intel_miptree_copy_slice_sw(struct brw_context *brw,
  980.                             struct intel_mipmap_tree *dst_mt,
  981.                             struct intel_mipmap_tree *src_mt,
  982.                             int level,
  983.                             int slice,
  984.                             int width,
  985.                             int height)
  986. {
  987.    void *src, *dst;
  988.    int src_stride, dst_stride;
  989.    int cpp = dst_mt->cpp;
  990.  
  991.    intel_miptree_map(brw, src_mt,
  992.                      level, slice,
  993.                      0, 0,
  994.                      width, height,
  995.                      GL_MAP_READ_BIT | BRW_MAP_DIRECT_BIT,
  996.                      &src, &src_stride);
  997.  
  998.    intel_miptree_map(brw, dst_mt,
  999.                      level, slice,
  1000.                      0, 0,
  1001.                      width, height,
  1002.                      GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT |
  1003.                      BRW_MAP_DIRECT_BIT,
  1004.                      &dst, &dst_stride);
  1005.  
  1006.    DBG("sw blit %s mt %p %p/%d -> %s mt %p %p/%d (%dx%d)\n",
  1007.        _mesa_get_format_name(src_mt->format),
  1008.        src_mt, src, src_stride,
  1009.        _mesa_get_format_name(dst_mt->format),
  1010.        dst_mt, dst, dst_stride,
  1011.        width, height);
  1012.  
  1013.    int row_size = cpp * width;
  1014.    if (src_stride == row_size &&
  1015.        dst_stride == row_size) {
  1016.       memcpy(dst, src, row_size * height);
  1017.    } else {
  1018.       for (int i = 0; i < height; i++) {
  1019.          memcpy(dst, src, row_size);
  1020.          dst += dst_stride;
  1021.          src += src_stride;
  1022.       }
  1023.    }
  1024.  
  1025.    intel_miptree_unmap(brw, dst_mt, level, slice);
  1026.    intel_miptree_unmap(brw, src_mt, level, slice);
  1027.  
  1028.    /* Don't forget to copy the stencil data over, too.  We could have skipped
  1029.     * passing BRW_MAP_DIRECT_BIT, but that would have meant intel_miptree_map
  1030.     * shuffling the two data sources in/out of temporary storage instead of
  1031.     * the direct mapping we get this way.
  1032.     */
  1033.    if (dst_mt->stencil_mt) {
  1034.       assert(src_mt->stencil_mt);
  1035.       intel_miptree_copy_slice_sw(brw, dst_mt->stencil_mt, src_mt->stencil_mt,
  1036.                                   level, slice, width, height);
  1037.    }
  1038. }
  1039.  
  1040. static void
  1041. intel_miptree_copy_slice(struct brw_context *brw,
  1042.                          struct intel_mipmap_tree *dst_mt,
  1043.                          struct intel_mipmap_tree *src_mt,
  1044.                          int level,
  1045.                          int face,
  1046.                          int depth)
  1047.  
  1048. {
  1049.    gl_format format = src_mt->format;
  1050.    uint32_t width = src_mt->level[level].width;
  1051.    uint32_t height = src_mt->level[level].height;
  1052.    int slice;
  1053.  
  1054.    if (face > 0)
  1055.       slice = face;
  1056.    else
  1057.       slice = depth;
  1058.  
  1059.    assert(depth < src_mt->level[level].depth);
  1060.    assert(src_mt->format == dst_mt->format);
  1061.  
  1062.    if (dst_mt->compressed) {
  1063.       height = ALIGN(height, dst_mt->align_h) / dst_mt->align_h;
  1064.       width = ALIGN(width, dst_mt->align_w);
  1065.    }
  1066.  
  1067.    /* If it's a packed depth/stencil buffer with separate stencil, the blit
  1068.     * below won't apply since we can't do the depth's Y tiling or the
  1069.     * stencil's W tiling in the blitter.
  1070.     */
  1071.    if (src_mt->stencil_mt) {
  1072.       intel_miptree_copy_slice_sw(brw,
  1073.                                   dst_mt, src_mt,
  1074.                                   level, slice,
  1075.                                   width, height);
  1076.       return;
  1077.    }
  1078.  
  1079.    uint32_t dst_x, dst_y, src_x, src_y;
  1080.    intel_miptree_get_image_offset(dst_mt, level, slice, &dst_x, &dst_y);
  1081.    intel_miptree_get_image_offset(src_mt, level, slice, &src_x, &src_y);
  1082.  
  1083.    DBG("validate blit mt %s %p %d,%d/%d -> mt %s %p %d,%d/%d (%dx%d)\n",
  1084.        _mesa_get_format_name(src_mt->format),
  1085.        src_mt, src_x, src_y, src_mt->region->pitch,
  1086.        _mesa_get_format_name(dst_mt->format),
  1087.        dst_mt, dst_x, dst_y, dst_mt->region->pitch,
  1088.        width, height);
  1089.  
  1090.    if (!intel_miptree_blit(brw,
  1091.                            src_mt, level, slice, 0, 0, false,
  1092.                            dst_mt, level, slice, 0, 0, false,
  1093.                            width, height, GL_COPY)) {
  1094.       perf_debug("miptree validate blit for %s failed\n",
  1095.                  _mesa_get_format_name(format));
  1096.  
  1097.       intel_miptree_copy_slice_sw(brw, dst_mt, src_mt, level, slice,
  1098.                                   width, height);
  1099.    }
  1100. }
  1101.  
  1102. /**
  1103.  * Copies the image's current data to the given miptree, and associates that
  1104.  * miptree with the image.
  1105.  *
  1106.  * If \c invalidate is true, then the actual image data does not need to be
  1107.  * copied, but the image still needs to be associated to the new miptree (this
  1108.  * is set to true if we're about to clear the image).
  1109.  */
  1110. void
  1111. intel_miptree_copy_teximage(struct brw_context *brw,
  1112.                             struct intel_texture_image *intelImage,
  1113.                             struct intel_mipmap_tree *dst_mt,
  1114.                             bool invalidate)
  1115. {
  1116.    struct intel_mipmap_tree *src_mt = intelImage->mt;
  1117.    struct intel_texture_object *intel_obj =
  1118.       intel_texture_object(intelImage->base.Base.TexObject);
  1119.    int level = intelImage->base.Base.Level;
  1120.    int face = intelImage->base.Base.Face;
  1121.    GLuint depth = intelImage->base.Base.Depth;
  1122.  
  1123.    if (!invalidate) {
  1124.       for (int slice = 0; slice < depth; slice++) {
  1125.          intel_miptree_copy_slice(brw, dst_mt, src_mt, level, face, slice);
  1126.       }
  1127.    }
  1128.  
  1129.    intel_miptree_reference(&intelImage->mt, dst_mt);
  1130.    intel_obj->needs_validate = true;
  1131. }
  1132.  
  1133. bool
  1134. intel_miptree_alloc_mcs(struct brw_context *brw,
  1135.                         struct intel_mipmap_tree *mt,
  1136.                         GLuint num_samples)
  1137. {
  1138.    assert(brw->gen >= 7); /* MCS only used on Gen7+ */
  1139.    assert(mt->mcs_mt == NULL);
  1140.  
  1141.    /* Choose the correct format for the MCS buffer.  All that really matters
  1142.     * is that we allocate the right buffer size, since we'll always be
  1143.     * accessing this miptree using MCS-specific hardware mechanisms, which
  1144.     * infer the correct format based on num_samples.
  1145.     */
  1146.    gl_format format;
  1147.    switch (num_samples) {
  1148.    case 4:
  1149.       /* 8 bits/pixel are required for MCS data when using 4x MSAA (2 bits for
  1150.        * each sample).
  1151.        */
  1152.       format = MESA_FORMAT_R8;
  1153.       break;
  1154.    case 8:
  1155.       /* 32 bits/pixel are required for MCS data when using 8x MSAA (3 bits
  1156.        * for each sample, plus 8 padding bits).
  1157.        */
  1158.       format = MESA_FORMAT_R_UINT32;
  1159.       break;
  1160.    default:
  1161.       assert(!"Unrecognized sample count in intel_miptree_alloc_mcs");
  1162.       return false;
  1163.    };
  1164.  
  1165.    /* From the Ivy Bridge PRM, Vol4 Part1 p76, "MCS Base Address":
  1166.     *
  1167.     *     "The MCS surface must be stored as Tile Y."
  1168.     */
  1169.    mt->mcs_state = INTEL_MCS_STATE_MSAA;
  1170.    mt->mcs_mt = intel_miptree_create(brw,
  1171.                                      mt->target,
  1172.                                      format,
  1173.                                      mt->first_level,
  1174.                                      mt->last_level,
  1175.                                      mt->logical_width0,
  1176.                                      mt->logical_height0,
  1177.                                      mt->logical_depth0,
  1178.                                      true,
  1179.                                      0 /* num_samples */,
  1180.                                      INTEL_MIPTREE_TILING_Y);
  1181.  
  1182.    /* From the Ivy Bridge PRM, Vol 2 Part 1 p326:
  1183.     *
  1184.     *     When MCS buffer is enabled and bound to MSRT, it is required that it
  1185.     *     is cleared prior to any rendering.
  1186.     *
  1187.     * Since we don't use the MCS buffer for any purpose other than rendering,
  1188.     * it makes sense to just clear it immediately upon allocation.
  1189.     *
  1190.     * Note: the clear value for MCS buffers is all 1's, so we memset to 0xff.
  1191.     */
  1192.    void *data = intel_miptree_map_raw(brw, mt->mcs_mt);
  1193.    memset(data, 0xff, mt->mcs_mt->region->bo->size);
  1194.    intel_miptree_unmap_raw(brw, mt->mcs_mt);
  1195.  
  1196.    return mt->mcs_mt;
  1197. }
  1198.  
  1199.  
  1200. bool
  1201. intel_miptree_alloc_non_msrt_mcs(struct brw_context *brw,
  1202.                                  struct intel_mipmap_tree *mt)
  1203. {
  1204.    assert(mt->mcs_mt == NULL);
  1205.  
  1206.    /* The format of the MCS buffer is opaque to the driver; all that matters
  1207.     * is that we get its size and pitch right.  We'll pretend that the format
  1208.     * is R32.  Since an MCS tile covers 128 blocks horizontally, and a Y-tiled
  1209.     * R32 buffer is 32 pixels across, we'll need to scale the width down by
  1210.     * the block width and then a further factor of 4.  Since an MCS tile
  1211.     * covers 256 blocks vertically, and a Y-tiled R32 buffer is 32 rows high,
  1212.     * we'll need to scale the height down by the block height and then a
  1213.     * further factor of 8.
  1214.     */
  1215.    const gl_format format = MESA_FORMAT_R_UINT32;
  1216.    unsigned block_width_px;
  1217.    unsigned block_height;
  1218.    intel_get_non_msrt_mcs_alignment(brw, mt, &block_width_px, &block_height);
  1219.    unsigned width_divisor = block_width_px * 4;
  1220.    unsigned height_divisor = block_height * 8;
  1221.    unsigned mcs_width =
  1222.       ALIGN(mt->logical_width0, width_divisor) / width_divisor;
  1223.    unsigned mcs_height =
  1224.       ALIGN(mt->logical_height0, height_divisor) / height_divisor;
  1225.    assert(mt->logical_depth0 == 1);
  1226.    mt->mcs_mt = intel_miptree_create(brw,
  1227.                                      mt->target,
  1228.                                      format,
  1229.                                      mt->first_level,
  1230.                                      mt->last_level,
  1231.                                      mcs_width,
  1232.                                      mcs_height,
  1233.                                      mt->logical_depth0,
  1234.                                      true,
  1235.                                      0 /* num_samples */,
  1236.                                      INTEL_MIPTREE_TILING_Y);
  1237.  
  1238.    return mt->mcs_mt;
  1239. }
  1240.  
  1241.  
  1242. /**
  1243.  * Helper for intel_miptree_alloc_hiz() that sets
  1244.  * \c mt->level[level].slice[layer].has_hiz. Return true if and only if
  1245.  * \c has_hiz was set.
  1246.  */
  1247. static bool
  1248. intel_miptree_slice_enable_hiz(struct brw_context *brw,
  1249.                                struct intel_mipmap_tree *mt,
  1250.                                uint32_t level,
  1251.                                uint32_t layer)
  1252. {
  1253.    assert(mt->hiz_mt);
  1254.  
  1255.    if (brw->is_haswell) {
  1256.       /* Disable HiZ for some slices to work around a hardware bug.
  1257.        *
  1258.        * Haswell hardware fails to respect
  1259.        * 3DSTATE_DEPTH_BUFFER.Depth_Coordinate_Offset_X/Y when during HiZ
  1260.        * ambiguate operations.  The failure is inconsistent and affected by
  1261.        * other GPU contexts. Running a heavy GPU workload in a separate
  1262.        * process causes the failure rate to drop to nearly 0.
  1263.        *
  1264.        * To workaround the bug, we enable HiZ only when we can guarantee that
  1265.        * the Depth Coordinate Offset fields will be set to 0. The function
  1266.        * brw_get_depthstencil_tile_masks() is used to calculate the fields,
  1267.        * and the function is sometimes called in such a way that the presence
  1268.        * of an attached stencil buffer changes the fuction's return value.
  1269.        *
  1270.        * The largest tile size considered by brw_get_depthstencil_tile_masks()
  1271.        * is that of the stencil buffer. Therefore, if this hiz slice's
  1272.        * corresponding depth slice has an offset that is aligned to the
  1273.        * stencil buffer tile size, 64x64 pixels, then
  1274.        * 3DSTATE_DEPTH_BUFFER.Depth_Coordinate_Offset_X/Y is set to 0.
  1275.        */
  1276.       uint32_t depth_x_offset = mt->level[level].slice[layer].x_offset;
  1277.       uint32_t depth_y_offset = mt->level[level].slice[layer].y_offset;
  1278.       if ((depth_x_offset & 63) || (depth_y_offset & 63)) {
  1279.          return false;
  1280.       }
  1281.    }
  1282.  
  1283.    mt->level[level].slice[layer].has_hiz = true;
  1284.    return true;
  1285. }
  1286.  
  1287.  
  1288.  
  1289. bool
  1290. intel_miptree_alloc_hiz(struct brw_context *brw,
  1291.                         struct intel_mipmap_tree *mt)
  1292. {
  1293.    assert(mt->hiz_mt == NULL);
  1294.    mt->hiz_mt = intel_miptree_create(brw,
  1295.                                      mt->target,
  1296.                                      mt->format,
  1297.                                      mt->first_level,
  1298.                                      mt->last_level,
  1299.                                      mt->logical_width0,
  1300.                                      mt->logical_height0,
  1301.                                      mt->logical_depth0,
  1302.                                      true,
  1303.                                      mt->num_samples,
  1304.                                      INTEL_MIPTREE_TILING_ANY);
  1305.  
  1306.    if (!mt->hiz_mt)
  1307.       return false;
  1308.  
  1309.    /* Mark that all slices need a HiZ resolve. */
  1310.    struct intel_resolve_map *head = &mt->hiz_map;
  1311.    for (int level = mt->first_level; level <= mt->last_level; ++level) {
  1312.       for (int layer = 0; layer < mt->level[level].depth; ++layer) {
  1313.          if (!intel_miptree_slice_enable_hiz(brw, mt, level, layer))
  1314.             continue;
  1315.  
  1316.          head->next = malloc(sizeof(*head->next));
  1317.          head->next->prev = head;
  1318.          head->next->next = NULL;
  1319.          head = head->next;
  1320.  
  1321.          head->level = level;
  1322.          head->layer = layer;
  1323.          head->need = GEN6_HIZ_OP_HIZ_RESOLVE;
  1324.       }
  1325.    }
  1326.  
  1327.    return true;
  1328. }
  1329.  
  1330. /**
  1331.  * Does the miptree slice have hiz enabled?
  1332.  */
  1333. bool
  1334. intel_miptree_slice_has_hiz(struct intel_mipmap_tree *mt,
  1335.                             uint32_t level,
  1336.                             uint32_t layer)
  1337. {
  1338.    intel_miptree_check_level_layer(mt, level, layer);
  1339.    return mt->level[level].slice[layer].has_hiz;
  1340. }
  1341.  
  1342. void
  1343. intel_miptree_slice_set_needs_hiz_resolve(struct intel_mipmap_tree *mt,
  1344.                                           uint32_t level,
  1345.                                           uint32_t layer)
  1346. {
  1347.    if (!intel_miptree_slice_has_hiz(mt, level, layer))
  1348.       return;
  1349.  
  1350.    intel_resolve_map_set(&mt->hiz_map,
  1351.                          level, layer, GEN6_HIZ_OP_HIZ_RESOLVE);
  1352. }
  1353.  
  1354.  
  1355. void
  1356. intel_miptree_slice_set_needs_depth_resolve(struct intel_mipmap_tree *mt,
  1357.                                             uint32_t level,
  1358.                                             uint32_t layer)
  1359. {
  1360.    if (!intel_miptree_slice_has_hiz(mt, level, layer))
  1361.       return;
  1362.  
  1363.    intel_resolve_map_set(&mt->hiz_map,
  1364.                          level, layer, GEN6_HIZ_OP_DEPTH_RESOLVE);
  1365. }
  1366.  
  1367. static bool
  1368. intel_miptree_slice_resolve(struct brw_context *brw,
  1369.                             struct intel_mipmap_tree *mt,
  1370.                             uint32_t level,
  1371.                             uint32_t layer,
  1372.                             enum gen6_hiz_op need)
  1373. {
  1374.    intel_miptree_check_level_layer(mt, level, layer);
  1375.  
  1376.    struct intel_resolve_map *item =
  1377.          intel_resolve_map_get(&mt->hiz_map, level, layer);
  1378.  
  1379.    if (!item || item->need != need)
  1380.       return false;
  1381.  
  1382.    intel_hiz_exec(brw, mt, level, layer, need);
  1383.    intel_resolve_map_remove(item);
  1384.    return true;
  1385. }
  1386.  
  1387. bool
  1388. intel_miptree_slice_resolve_hiz(struct brw_context *brw,
  1389.                                 struct intel_mipmap_tree *mt,
  1390.                                 uint32_t level,
  1391.                                 uint32_t layer)
  1392. {
  1393.    return intel_miptree_slice_resolve(brw, mt, level, layer,
  1394.                                       GEN6_HIZ_OP_HIZ_RESOLVE);
  1395. }
  1396.  
  1397. bool
  1398. intel_miptree_slice_resolve_depth(struct brw_context *brw,
  1399.                                   struct intel_mipmap_tree *mt,
  1400.                                   uint32_t level,
  1401.                                   uint32_t layer)
  1402. {
  1403.    return intel_miptree_slice_resolve(brw, mt, level, layer,
  1404.                                       GEN6_HIZ_OP_DEPTH_RESOLVE);
  1405. }
  1406.  
  1407. static bool
  1408. intel_miptree_all_slices_resolve(struct brw_context *brw,
  1409.                                  struct intel_mipmap_tree *mt,
  1410.                                  enum gen6_hiz_op need)
  1411. {
  1412.    bool did_resolve = false;
  1413.    struct intel_resolve_map *i, *next;
  1414.  
  1415.    for (i = mt->hiz_map.next; i; i = next) {
  1416.       next = i->next;
  1417.       if (i->need != need)
  1418.          continue;
  1419.  
  1420.       intel_hiz_exec(brw, mt, i->level, i->layer, need);
  1421.       intel_resolve_map_remove(i);
  1422.       did_resolve = true;
  1423.    }
  1424.  
  1425.    return did_resolve;
  1426. }
  1427.  
  1428. bool
  1429. intel_miptree_all_slices_resolve_hiz(struct brw_context *brw,
  1430.                                      struct intel_mipmap_tree *mt)
  1431. {
  1432.    return intel_miptree_all_slices_resolve(brw, mt,
  1433.                                            GEN6_HIZ_OP_HIZ_RESOLVE);
  1434. }
  1435.  
  1436. bool
  1437. intel_miptree_all_slices_resolve_depth(struct brw_context *brw,
  1438.                                        struct intel_mipmap_tree *mt)
  1439. {
  1440.    return intel_miptree_all_slices_resolve(brw, mt,
  1441.                                            GEN6_HIZ_OP_DEPTH_RESOLVE);
  1442. }
  1443.  
  1444.  
  1445. void
  1446. intel_miptree_resolve_color(struct brw_context *brw,
  1447.                             struct intel_mipmap_tree *mt)
  1448. {
  1449.    switch (mt->mcs_state) {
  1450.    case INTEL_MCS_STATE_NONE:
  1451.    case INTEL_MCS_STATE_MSAA:
  1452.    case INTEL_MCS_STATE_RESOLVED:
  1453.       /* No resolve needed */
  1454.       break;
  1455.    case INTEL_MCS_STATE_UNRESOLVED:
  1456.    case INTEL_MCS_STATE_CLEAR:
  1457.       brw_blorp_resolve_color(brw, mt);
  1458.       break;
  1459.    }
  1460. }
  1461.  
  1462.  
  1463. /**
  1464.  * Make it possible to share the region backing the given miptree with another
  1465.  * process or another miptree.
  1466.  *
  1467.  * Fast color clears are unsafe with shared buffers, so we need to resolve and
  1468.  * then discard the MCS buffer, if present.  We also set the mcs_state to
  1469.  * INTEL_MCS_STATE_NONE to ensure that no MCS buffer gets allocated in the
  1470.  * future.
  1471.  */
  1472. void
  1473. intel_miptree_make_shareable(struct brw_context *brw,
  1474.                              struct intel_mipmap_tree *mt)
  1475. {
  1476.    /* MCS buffers are also used for multisample buffers, but we can't resolve
  1477.     * away a multisample MCS buffer because it's an integral part of how the
  1478.     * pixel data is stored.  Fortunately this code path should never be
  1479.     * reached for multisample buffers.
  1480.     */
  1481.    assert(mt->msaa_layout == INTEL_MSAA_LAYOUT_NONE);
  1482.  
  1483.    if (mt->mcs_mt) {
  1484.       intel_miptree_resolve_color(brw, mt);
  1485.       intel_miptree_release(&mt->mcs_mt);
  1486.       mt->mcs_state = INTEL_MCS_STATE_NONE;
  1487.    }
  1488. }
  1489.  
  1490.  
  1491. /**
  1492.  * \brief Get pointer offset into stencil buffer.
  1493.  *
  1494.  * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
  1495.  * must decode the tile's layout in software.
  1496.  *
  1497.  * See
  1498.  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
  1499.  *     Format.
  1500.  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
  1501.  *
  1502.  * Even though the returned offset is always positive, the return type is
  1503.  * signed due to
  1504.  *    commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
  1505.  *    mesa: Fix return type of  _mesa_get_format_bytes() (#37351)
  1506.  */
  1507. static intptr_t
  1508. intel_offset_S8(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
  1509. {
  1510.    uint32_t tile_size = 4096;
  1511.    uint32_t tile_width = 64;
  1512.    uint32_t tile_height = 64;
  1513.    uint32_t row_size = 64 * stride;
  1514.  
  1515.    uint32_t tile_x = x / tile_width;
  1516.    uint32_t tile_y = y / tile_height;
  1517.  
  1518.    /* The byte's address relative to the tile's base addres. */
  1519.    uint32_t byte_x = x % tile_width;
  1520.    uint32_t byte_y = y % tile_height;
  1521.  
  1522.    uintptr_t u = tile_y * row_size
  1523.                + tile_x * tile_size
  1524.                + 512 * (byte_x / 8)
  1525.                +  64 * (byte_y / 8)
  1526.                +  32 * ((byte_y / 4) % 2)
  1527.                +  16 * ((byte_x / 4) % 2)
  1528.                +   8 * ((byte_y / 2) % 2)
  1529.                +   4 * ((byte_x / 2) % 2)
  1530.                +   2 * (byte_y % 2)
  1531.                +   1 * (byte_x % 2);
  1532.  
  1533.    if (swizzled) {
  1534.       /* adjust for bit6 swizzling */
  1535.       if (((byte_x / 8) % 2) == 1) {
  1536.          if (((byte_y / 8) % 2) == 0) {
  1537.             u += 64;
  1538.          } else {
  1539.             u -= 64;
  1540.          }
  1541.       }
  1542.    }
  1543.  
  1544.    return u;
  1545. }
  1546.  
  1547. static void
  1548. intel_miptree_updownsample(struct brw_context *brw,
  1549.                            struct intel_mipmap_tree *src,
  1550.                            struct intel_mipmap_tree *dst,
  1551.                            unsigned width,
  1552.                            unsigned height)
  1553. {
  1554.    int src_x0 = 0;
  1555.    int src_y0 = 0;
  1556.    int dst_x0 = 0;
  1557.    int dst_y0 = 0;
  1558.  
  1559.    brw_blorp_blit_miptrees(brw,
  1560.                            src, 0 /* level */, 0 /* layer */,
  1561.                            dst, 0 /* level */, 0 /* layer */,
  1562.                            src_x0, src_y0,
  1563.                            width, height,
  1564.                            dst_x0, dst_y0,
  1565.                            width, height,
  1566.                            false, false /*mirror x, y*/);
  1567.  
  1568.    if (src->stencil_mt) {
  1569.       brw_blorp_blit_miptrees(brw,
  1570.                               src->stencil_mt, 0 /* level */, 0 /* layer */,
  1571.                               dst->stencil_mt, 0 /* level */, 0 /* layer */,
  1572.                               src_x0, src_y0,
  1573.                               width, height,
  1574.                               dst_x0, dst_y0,
  1575.                               width, height,
  1576.                               false, false /*mirror x, y*/);
  1577.    }
  1578. }
  1579.  
  1580. static void
  1581. assert_is_flat(struct intel_mipmap_tree *mt)
  1582. {
  1583.    assert(mt->target == GL_TEXTURE_2D);
  1584.    assert(mt->first_level == 0);
  1585.    assert(mt->last_level == 0);
  1586. }
  1587.  
  1588. /**
  1589.  * \brief Downsample from mt to mt->singlesample_mt.
  1590.  *
  1591.  * If the miptree needs no downsample, then skip.
  1592.  */
  1593. void
  1594. intel_miptree_downsample(struct brw_context *brw,
  1595.                          struct intel_mipmap_tree *mt)
  1596. {
  1597.    /* Only flat, renderbuffer-like miptrees are supported. */
  1598.    assert_is_flat(mt);
  1599.  
  1600.    if (!mt->need_downsample)
  1601.       return;
  1602.    intel_miptree_updownsample(brw,
  1603.                               mt, mt->singlesample_mt,
  1604.                               mt->logical_width0,
  1605.                               mt->logical_height0);
  1606.    mt->need_downsample = false;
  1607. }
  1608.  
  1609. /**
  1610.  * \brief Upsample from mt->singlesample_mt to mt.
  1611.  *
  1612.  * The upsample is done unconditionally.
  1613.  */
  1614. void
  1615. intel_miptree_upsample(struct brw_context *brw,
  1616.                        struct intel_mipmap_tree *mt)
  1617. {
  1618.    /* Only flat, renderbuffer-like miptrees are supported. */
  1619.    assert_is_flat(mt);
  1620.    assert(!mt->need_downsample);
  1621.  
  1622.    intel_miptree_updownsample(brw,
  1623.                               mt->singlesample_mt, mt,
  1624.                               mt->logical_width0,
  1625.                               mt->logical_height0);
  1626. }
  1627.  
  1628. void *
  1629. intel_miptree_map_raw(struct brw_context *brw, struct intel_mipmap_tree *mt)
  1630. {
  1631.    struct gl_context *ctx = &brw->ctx;
  1632.    /* CPU accesses to color buffers don't understand fast color clears, so
  1633.     * resolve any pending fast color clears before we map.
  1634.     */
  1635.    intel_miptree_resolve_color(brw, mt);
  1636.  
  1637.    drm_intel_bo *bo = mt->region->bo;
  1638.  
  1639.    if (unlikely(INTEL_DEBUG & DEBUG_PERF)) {
  1640.       if (drm_intel_bo_busy(bo)) {
  1641.          perf_debug("Mapping a busy BO, causing a stall on the GPU.\n");
  1642.       }
  1643.    }
  1644.  
  1645.    intel_flush(ctx);
  1646.  
  1647.    if (mt->region->tiling != I915_TILING_NONE)
  1648.       drm_intel_gem_bo_map_gtt(bo);
  1649.    else
  1650.       drm_intel_bo_map(bo, true);
  1651.  
  1652.    return bo->virtual;
  1653. }
  1654.  
  1655. void
  1656. intel_miptree_unmap_raw(struct brw_context *brw,
  1657.                         struct intel_mipmap_tree *mt)
  1658. {
  1659.    drm_intel_bo_unmap(mt->region->bo);
  1660. }
  1661.  
  1662. static void
  1663. intel_miptree_map_gtt(struct brw_context *brw,
  1664.                       struct intel_mipmap_tree *mt,
  1665.                       struct intel_miptree_map *map,
  1666.                       unsigned int level, unsigned int slice)
  1667. {
  1668.    unsigned int bw, bh;
  1669.    void *base;
  1670.    unsigned int image_x, image_y;
  1671.    int x = map->x;
  1672.    int y = map->y;
  1673.  
  1674.    /* For compressed formats, the stride is the number of bytes per
  1675.     * row of blocks.  intel_miptree_get_image_offset() already does
  1676.     * the divide.
  1677.     */
  1678.    _mesa_get_format_block_size(mt->format, &bw, &bh);
  1679.    assert(y % bh == 0);
  1680.    y /= bh;
  1681.  
  1682.    base = intel_miptree_map_raw(brw, mt) + mt->offset;
  1683.  
  1684.    if (base == NULL)
  1685.       map->ptr = NULL;
  1686.    else {
  1687.       /* Note that in the case of cube maps, the caller must have passed the
  1688.        * slice number referencing the face.
  1689.       */
  1690.       intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
  1691.       x += image_x;
  1692.       y += image_y;
  1693.  
  1694.       map->stride = mt->region->pitch;
  1695.       map->ptr = base + y * map->stride + x * mt->cpp;
  1696.    }
  1697.  
  1698.    DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__,
  1699.        map->x, map->y, map->w, map->h,
  1700.        mt, _mesa_get_format_name(mt->format),
  1701.        x, y, map->ptr, map->stride);
  1702. }
  1703.  
  1704. static void
  1705. intel_miptree_unmap_gtt(struct brw_context *brw,
  1706.                         struct intel_mipmap_tree *mt,
  1707.                         struct intel_miptree_map *map,
  1708.                         unsigned int level,
  1709.                         unsigned int slice)
  1710. {
  1711.    intel_miptree_unmap_raw(brw, mt);
  1712. }
  1713.  
  1714. static void
  1715. intel_miptree_map_blit(struct brw_context *brw,
  1716.                        struct intel_mipmap_tree *mt,
  1717.                        struct intel_miptree_map *map,
  1718.                        unsigned int level, unsigned int slice)
  1719. {
  1720.    map->mt = intel_miptree_create(brw, GL_TEXTURE_2D, mt->format,
  1721.                                   0, 0,
  1722.                                   map->w, map->h, 1,
  1723.                                   false, 0,
  1724.                                   INTEL_MIPTREE_TILING_NONE);
  1725.    if (!map->mt) {
  1726.       fprintf(stderr, "Failed to allocate blit temporary\n");
  1727.       goto fail;
  1728.    }
  1729.    map->stride = map->mt->region->pitch;
  1730.  
  1731.    if (!intel_miptree_blit(brw,
  1732.                            mt, level, slice,
  1733.                            map->x, map->y, false,
  1734.                            map->mt, 0, 0,
  1735.                            0, 0, false,
  1736.                            map->w, map->h, GL_COPY)) {
  1737.       fprintf(stderr, "Failed to blit\n");
  1738.       goto fail;
  1739.    }
  1740.  
  1741.    intel_batchbuffer_flush(brw);
  1742.    map->ptr = intel_miptree_map_raw(brw, map->mt);
  1743.  
  1744.    DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __FUNCTION__,
  1745.        map->x, map->y, map->w, map->h,
  1746.        mt, _mesa_get_format_name(mt->format),
  1747.        level, slice, map->ptr, map->stride);
  1748.  
  1749.    return;
  1750.  
  1751. fail:
  1752.    intel_miptree_release(&map->mt);
  1753.    map->ptr = NULL;
  1754.    map->stride = 0;
  1755. }
  1756.  
  1757. static void
  1758. intel_miptree_unmap_blit(struct brw_context *brw,
  1759.                          struct intel_mipmap_tree *mt,
  1760.                          struct intel_miptree_map *map,
  1761.                          unsigned int level,
  1762.                          unsigned int slice)
  1763. {
  1764.    struct gl_context *ctx = &brw->ctx;
  1765.  
  1766.    intel_miptree_unmap_raw(brw, map->mt);
  1767.  
  1768.    if (map->mode & GL_MAP_WRITE_BIT) {
  1769.       bool ok = intel_miptree_blit(brw,
  1770.                                    map->mt, 0, 0,
  1771.                                    0, 0, false,
  1772.                                    mt, level, slice,
  1773.                                    map->x, map->y, false,
  1774.                                    map->w, map->h, GL_COPY);
  1775.       WARN_ONCE(!ok, "Failed to blit from linear temporary mapping");
  1776.    }
  1777.  
  1778.    intel_miptree_release(&map->mt);
  1779. }
  1780.  
  1781. static void
  1782. intel_miptree_map_s8(struct brw_context *brw,
  1783.                      struct intel_mipmap_tree *mt,
  1784.                      struct intel_miptree_map *map,
  1785.                      unsigned int level, unsigned int slice)
  1786. {
  1787.    map->stride = map->w;
  1788.    map->buffer = map->ptr = malloc(map->stride * map->h);
  1789.    if (!map->buffer)
  1790.       return;
  1791.  
  1792.    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
  1793.     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
  1794.     * invalidate is set, since we'll be writing the whole rectangle from our
  1795.     * temporary buffer back out.
  1796.     */
  1797.    if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
  1798.       uint8_t *untiled_s8_map = map->ptr;
  1799.       uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt);
  1800.       unsigned int image_x, image_y;
  1801.  
  1802.       intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
  1803.  
  1804.       for (uint32_t y = 0; y < map->h; y++) {
  1805.          for (uint32_t x = 0; x < map->w; x++) {
  1806.             ptrdiff_t offset = intel_offset_S8(mt->region->pitch,
  1807.                                                x + image_x + map->x,
  1808.                                                y + image_y + map->y,
  1809.                                                brw->has_swizzling);
  1810.             untiled_s8_map[y * map->w + x] = tiled_s8_map[offset];
  1811.          }
  1812.       }
  1813.  
  1814.       intel_miptree_unmap_raw(brw, mt);
  1815.  
  1816.       DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __FUNCTION__,
  1817.           map->x, map->y, map->w, map->h,
  1818.           mt, map->x + image_x, map->y + image_y, map->ptr, map->stride);
  1819.    } else {
  1820.       DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __FUNCTION__,
  1821.           map->x, map->y, map->w, map->h,
  1822.           mt, map->ptr, map->stride);
  1823.    }
  1824. }
  1825.  
  1826. static void
  1827. intel_miptree_unmap_s8(struct brw_context *brw,
  1828.                        struct intel_mipmap_tree *mt,
  1829.                        struct intel_miptree_map *map,
  1830.                        unsigned int level,
  1831.                        unsigned int slice)
  1832. {
  1833.    if (map->mode & GL_MAP_WRITE_BIT) {
  1834.       unsigned int image_x, image_y;
  1835.       uint8_t *untiled_s8_map = map->ptr;
  1836.       uint8_t *tiled_s8_map = intel_miptree_map_raw(brw, mt);
  1837.  
  1838.       intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
  1839.  
  1840.       for (uint32_t y = 0; y < map->h; y++) {
  1841.          for (uint32_t x = 0; x < map->w; x++) {
  1842.             ptrdiff_t offset = intel_offset_S8(mt->region->pitch,
  1843.                                                x + map->x,
  1844.                                                y + map->y,
  1845.                                                brw->has_swizzling);
  1846.             tiled_s8_map[offset] = untiled_s8_map[y * map->w + x];
  1847.          }
  1848.       }
  1849.  
  1850.       intel_miptree_unmap_raw(brw, mt);
  1851.    }
  1852.  
  1853.    free(map->buffer);
  1854. }
  1855.  
  1856. static void
  1857. intel_miptree_map_etc(struct brw_context *brw,
  1858.                       struct intel_mipmap_tree *mt,
  1859.                       struct intel_miptree_map *map,
  1860.                       unsigned int level,
  1861.                       unsigned int slice)
  1862. {
  1863.    assert(mt->etc_format != MESA_FORMAT_NONE);
  1864.    if (mt->etc_format == MESA_FORMAT_ETC1_RGB8) {
  1865.       assert(mt->format == MESA_FORMAT_RGBX8888_REV);
  1866.    }
  1867.  
  1868.    assert(map->mode & GL_MAP_WRITE_BIT);
  1869.    assert(map->mode & GL_MAP_INVALIDATE_RANGE_BIT);
  1870.  
  1871.    map->stride = _mesa_format_row_stride(mt->etc_format, map->w);
  1872.    map->buffer = malloc(_mesa_format_image_size(mt->etc_format,
  1873.                                                 map->w, map->h, 1));
  1874.    map->ptr = map->buffer;
  1875. }
  1876.  
  1877. static void
  1878. intel_miptree_unmap_etc(struct brw_context *brw,
  1879.                         struct intel_mipmap_tree *mt,
  1880.                         struct intel_miptree_map *map,
  1881.                         unsigned int level,
  1882.                         unsigned int slice)
  1883. {
  1884.    uint32_t image_x;
  1885.    uint32_t image_y;
  1886.    intel_miptree_get_image_offset(mt, level, slice, &image_x, &image_y);
  1887.  
  1888.    image_x += map->x;
  1889.    image_y += map->y;
  1890.  
  1891.    uint8_t *dst = intel_miptree_map_raw(brw, mt)
  1892.                 + image_y * mt->region->pitch
  1893.                 + image_x * mt->region->cpp;
  1894.  
  1895.    if (mt->etc_format == MESA_FORMAT_ETC1_RGB8)
  1896.       _mesa_etc1_unpack_rgba8888(dst, mt->region->pitch,
  1897.                                  map->ptr, map->stride,
  1898.                                  map->w, map->h);
  1899.    else
  1900.       _mesa_unpack_etc2_format(dst, mt->region->pitch,
  1901.                                map->ptr, map->stride,
  1902.                                map->w, map->h, mt->etc_format);
  1903.  
  1904.    intel_miptree_unmap_raw(brw, mt);
  1905.    free(map->buffer);
  1906. }
  1907.  
  1908. /**
  1909.  * Mapping function for packed depth/stencil miptrees backed by real separate
  1910.  * miptrees for depth and stencil.
  1911.  *
  1912.  * On gen7, and to support HiZ pre-gen7, we have to have the stencil buffer
  1913.  * separate from the depth buffer.  Yet at the GL API level, we have to expose
  1914.  * packed depth/stencil textures and FBO attachments, and Mesa core expects to
  1915.  * be able to map that memory for texture storage and glReadPixels-type
  1916.  * operations.  We give Mesa core that access by mallocing a temporary and
  1917.  * copying the data between the actual backing store and the temporary.
  1918.  */
  1919. static void
  1920. intel_miptree_map_depthstencil(struct brw_context *brw,
  1921.                                struct intel_mipmap_tree *mt,
  1922.                                struct intel_miptree_map *map,
  1923.                                unsigned int level, unsigned int slice)
  1924. {
  1925.    struct intel_mipmap_tree *z_mt = mt;
  1926.    struct intel_mipmap_tree *s_mt = mt->stencil_mt;
  1927.    bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z32_FLOAT;
  1928.    int packed_bpp = map_z32f_x24s8 ? 8 : 4;
  1929.  
  1930.    map->stride = map->w * packed_bpp;
  1931.    map->buffer = map->ptr = malloc(map->stride * map->h);
  1932.    if (!map->buffer)
  1933.       return;
  1934.  
  1935.    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
  1936.     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
  1937.     * invalidate is set, since we'll be writing the whole rectangle from our
  1938.     * temporary buffer back out.
  1939.     */
  1940.    if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) {
  1941.       uint32_t *packed_map = map->ptr;
  1942.       uint8_t *s_map = intel_miptree_map_raw(brw, s_mt);
  1943.       uint32_t *z_map = intel_miptree_map_raw(brw, z_mt);
  1944.       unsigned int s_image_x, s_image_y;
  1945.       unsigned int z_image_x, z_image_y;
  1946.  
  1947.       intel_miptree_get_image_offset(s_mt, level, slice,
  1948.                                      &s_image_x, &s_image_y);
  1949.       intel_miptree_get_image_offset(z_mt, level, slice,
  1950.                                      &z_image_x, &z_image_y);
  1951.  
  1952.       for (uint32_t y = 0; y < map->h; y++) {
  1953.          for (uint32_t x = 0; x < map->w; x++) {
  1954.             int map_x = map->x + x, map_y = map->y + y;
  1955.             ptrdiff_t s_offset = intel_offset_S8(s_mt->region->pitch,
  1956.                                                  map_x + s_image_x,
  1957.                                                  map_y + s_image_y,
  1958.                                                  brw->has_swizzling);
  1959.             ptrdiff_t z_offset = ((map_y + z_image_y) *
  1960.                                   (z_mt->region->pitch / 4) +
  1961.                                   (map_x + z_image_x));
  1962.             uint8_t s = s_map[s_offset];
  1963.             uint32_t z = z_map[z_offset];
  1964.  
  1965.             if (map_z32f_x24s8) {
  1966.                packed_map[(y * map->w + x) * 2 + 0] = z;
  1967.                packed_map[(y * map->w + x) * 2 + 1] = s;
  1968.             } else {
  1969.                packed_map[y * map->w + x] = (s << 24) | (z & 0x00ffffff);
  1970.             }
  1971.          }
  1972.       }
  1973.  
  1974.       intel_miptree_unmap_raw(brw, s_mt);
  1975.       intel_miptree_unmap_raw(brw, z_mt);
  1976.  
  1977.       DBG("%s: %d,%d %dx%d from z mt %p %d,%d, s mt %p %d,%d = %p/%d\n",
  1978.           __FUNCTION__,
  1979.           map->x, map->y, map->w, map->h,
  1980.           z_mt, map->x + z_image_x, map->y + z_image_y,
  1981.           s_mt, map->x + s_image_x, map->y + s_image_y,
  1982.           map->ptr, map->stride);
  1983.    } else {
  1984.       DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __FUNCTION__,
  1985.           map->x, map->y, map->w, map->h,
  1986.           mt, map->ptr, map->stride);
  1987.    }
  1988. }
  1989.  
  1990. static void
  1991. intel_miptree_unmap_depthstencil(struct brw_context *brw,
  1992.                                  struct intel_mipmap_tree *mt,
  1993.                                  struct intel_miptree_map *map,
  1994.                                  unsigned int level,
  1995.                                  unsigned int slice)
  1996. {
  1997.    struct intel_mipmap_tree *z_mt = mt;
  1998.    struct intel_mipmap_tree *s_mt = mt->stencil_mt;
  1999.    bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z32_FLOAT;
  2000.  
  2001.    if (map->mode & GL_MAP_WRITE_BIT) {
  2002.       uint32_t *packed_map = map->ptr;
  2003.       uint8_t *s_map = intel_miptree_map_raw(brw, s_mt);
  2004.       uint32_t *z_map = intel_miptree_map_raw(brw, z_mt);
  2005.       unsigned int s_image_x, s_image_y;
  2006.       unsigned int z_image_x, z_image_y;
  2007.  
  2008.       intel_miptree_get_image_offset(s_mt, level, slice,
  2009.                                      &s_image_x, &s_image_y);
  2010.       intel_miptree_get_image_offset(z_mt, level, slice,
  2011.                                      &z_image_x, &z_image_y);
  2012.  
  2013.       for (uint32_t y = 0; y < map->h; y++) {
  2014.          for (uint32_t x = 0; x < map->w; x++) {
  2015.             ptrdiff_t s_offset = intel_offset_S8(s_mt->region->pitch,
  2016.                                                  x + s_image_x + map->x,
  2017.                                                  y + s_image_y + map->y,
  2018.                                                  brw->has_swizzling);
  2019.             ptrdiff_t z_offset = ((y + z_image_y) *
  2020.                                   (z_mt->region->pitch / 4) +
  2021.                                   (x + z_image_x));
  2022.  
  2023.             if (map_z32f_x24s8) {
  2024.                z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0];
  2025.                s_map[s_offset] = packed_map[(y * map->w + x) * 2 + 1];
  2026.             } else {
  2027.                uint32_t packed = packed_map[y * map->w + x];
  2028.                s_map[s_offset] = packed >> 24;
  2029.                z_map[z_offset] = packed;
  2030.             }
  2031.          }
  2032.       }
  2033.  
  2034.       intel_miptree_unmap_raw(brw, s_mt);
  2035.       intel_miptree_unmap_raw(brw, z_mt);
  2036.  
  2037.       DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n",
  2038.           __FUNCTION__,
  2039.           map->x, map->y, map->w, map->h,
  2040.           z_mt, _mesa_get_format_name(z_mt->format),
  2041.           map->x + z_image_x, map->y + z_image_y,
  2042.           s_mt, map->x + s_image_x, map->y + s_image_y,
  2043.           map->ptr, map->stride);
  2044.    }
  2045.  
  2046.    free(map->buffer);
  2047. }
  2048.  
  2049. /**
  2050.  * Create and attach a map to the miptree at (level, slice). Return the
  2051.  * attached map.
  2052.  */
  2053. static struct intel_miptree_map*
  2054. intel_miptree_attach_map(struct intel_mipmap_tree *mt,
  2055.                          unsigned int level,
  2056.                          unsigned int slice,
  2057.                          unsigned int x,
  2058.                          unsigned int y,
  2059.                          unsigned int w,
  2060.                          unsigned int h,
  2061.                          GLbitfield mode)
  2062. {
  2063.    struct intel_miptree_map *map = calloc(1, sizeof(*map));
  2064.  
  2065.    if (!map)
  2066.       return NULL;
  2067.  
  2068.    assert(mt->level[level].slice[slice].map == NULL);
  2069.    mt->level[level].slice[slice].map = map;
  2070.  
  2071.    map->mode = mode;
  2072.    map->x = x;
  2073.    map->y = y;
  2074.    map->w = w;
  2075.    map->h = h;
  2076.  
  2077.    return map;
  2078. }
  2079.  
  2080. /**
  2081.  * Release the map at (level, slice).
  2082.  */
  2083. static void
  2084. intel_miptree_release_map(struct intel_mipmap_tree *mt,
  2085.                          unsigned int level,
  2086.                          unsigned int slice)
  2087. {
  2088.    struct intel_miptree_map **map;
  2089.  
  2090.    map = &mt->level[level].slice[slice].map;
  2091.    free(*map);
  2092.    *map = NULL;
  2093. }
  2094.  
  2095. static void
  2096. intel_miptree_map_singlesample(struct brw_context *brw,
  2097.                                struct intel_mipmap_tree *mt,
  2098.                                unsigned int level,
  2099.                                unsigned int slice,
  2100.                                unsigned int x,
  2101.                                unsigned int y,
  2102.                                unsigned int w,
  2103.                                unsigned int h,
  2104.                                GLbitfield mode,
  2105.                                void **out_ptr,
  2106.                                int *out_stride)
  2107. {
  2108.    struct intel_miptree_map *map;
  2109.  
  2110.    assert(mt->num_samples <= 1);
  2111.  
  2112.    map = intel_miptree_attach_map(mt, level, slice, x, y, w, h, mode);
  2113.    if (!map){
  2114.       *out_ptr = NULL;
  2115.       *out_stride = 0;
  2116.       return;
  2117.    }
  2118.  
  2119.    intel_miptree_slice_resolve_depth(brw, mt, level, slice);
  2120.    if (map->mode & GL_MAP_WRITE_BIT) {
  2121.       intel_miptree_slice_set_needs_hiz_resolve(mt, level, slice);
  2122.    }
  2123.  
  2124.    if (mt->format == MESA_FORMAT_S8) {
  2125.       intel_miptree_map_s8(brw, mt, map, level, slice);
  2126.    } else if (mt->etc_format != MESA_FORMAT_NONE &&
  2127.               !(mode & BRW_MAP_DIRECT_BIT)) {
  2128.       intel_miptree_map_etc(brw, mt, map, level, slice);
  2129.    } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) {
  2130.       intel_miptree_map_depthstencil(brw, mt, map, level, slice);
  2131.    }
  2132.    /* See intel_miptree_blit() for details on the 32k pitch limit. */
  2133.    else if (brw->has_llc &&
  2134.             !(mode & GL_MAP_WRITE_BIT) &&
  2135.             !mt->compressed &&
  2136.             (mt->region->tiling == I915_TILING_X ||
  2137.              (brw->gen >= 6 && mt->region->tiling == I915_TILING_Y)) &&
  2138.             mt->region->pitch < 32768) {
  2139.       intel_miptree_map_blit(brw, mt, map, level, slice);
  2140.    } else if (mt->region->tiling != I915_TILING_NONE &&
  2141.               mt->region->bo->size >= brw->max_gtt_map_object_size) {
  2142.       assert(mt->region->pitch < 32768);
  2143.       intel_miptree_map_blit(brw, mt, map, level, slice);
  2144.    } else {
  2145.       intel_miptree_map_gtt(brw, mt, map, level, slice);
  2146.    }
  2147.  
  2148.    *out_ptr = map->ptr;
  2149.    *out_stride = map->stride;
  2150.  
  2151.    if (map->ptr == NULL)
  2152.       intel_miptree_release_map(mt, level, slice);
  2153. }
  2154.  
  2155. static void
  2156. intel_miptree_unmap_singlesample(struct brw_context *brw,
  2157.                                  struct intel_mipmap_tree *mt,
  2158.                                  unsigned int level,
  2159.                                  unsigned int slice)
  2160. {
  2161.    struct intel_miptree_map *map = mt->level[level].slice[slice].map;
  2162.  
  2163.    assert(mt->num_samples <= 1);
  2164.  
  2165.    if (!map)
  2166.       return;
  2167.  
  2168.    DBG("%s: mt %p (%s) level %d slice %d\n", __FUNCTION__,
  2169.        mt, _mesa_get_format_name(mt->format), level, slice);
  2170.  
  2171.    if (mt->format == MESA_FORMAT_S8) {
  2172.       intel_miptree_unmap_s8(brw, mt, map, level, slice);
  2173.    } else if (mt->etc_format != MESA_FORMAT_NONE &&
  2174.               !(map->mode & BRW_MAP_DIRECT_BIT)) {
  2175.       intel_miptree_unmap_etc(brw, mt, map, level, slice);
  2176.    } else if (mt->stencil_mt && !(map->mode & BRW_MAP_DIRECT_BIT)) {
  2177.       intel_miptree_unmap_depthstencil(brw, mt, map, level, slice);
  2178.    } else if (map->mt) {
  2179.       intel_miptree_unmap_blit(brw, mt, map, level, slice);
  2180.    } else {
  2181.       intel_miptree_unmap_gtt(brw, mt, map, level, slice);
  2182.    }
  2183.  
  2184.    intel_miptree_release_map(mt, level, slice);
  2185. }
  2186.  
  2187. static void
  2188. intel_miptree_map_multisample(struct brw_context *brw,
  2189.                               struct intel_mipmap_tree *mt,
  2190.                               unsigned int level,
  2191.                               unsigned int slice,
  2192.                               unsigned int x,
  2193.                               unsigned int y,
  2194.                               unsigned int w,
  2195.                               unsigned int h,
  2196.                               GLbitfield mode,
  2197.                               void **out_ptr,
  2198.                               int *out_stride)
  2199. {
  2200.    struct gl_context *ctx = &brw->ctx;
  2201.    struct intel_miptree_map *map;
  2202.  
  2203.    assert(mt->num_samples > 1);
  2204.  
  2205.    /* Only flat, renderbuffer-like miptrees are supported. */
  2206.    if (mt->target != GL_TEXTURE_2D ||
  2207.        mt->first_level != 0 ||
  2208.        mt->last_level != 0) {
  2209.       _mesa_problem(ctx, "attempt to map a multisample miptree for "
  2210.                     "which (target, first_level, last_level != "
  2211.                     "(GL_TEXTURE_2D, 0, 0)");
  2212.       goto fail;
  2213.    }
  2214.  
  2215.    map = intel_miptree_attach_map(mt, level, slice, x, y, w, h, mode);
  2216.    if (!map)
  2217.       goto fail;
  2218.  
  2219.    if (!mt->singlesample_mt) {
  2220.       mt->singlesample_mt =
  2221.          intel_miptree_create_for_renderbuffer(brw,
  2222.                                                mt->format,
  2223.                                                mt->logical_width0,
  2224.                                                mt->logical_height0,
  2225.                                                0 /*num_samples*/);
  2226.       if (!mt->singlesample_mt)
  2227.          goto fail;
  2228.  
  2229.       map->singlesample_mt_is_tmp = true;
  2230.       mt->need_downsample = true;
  2231.    }
  2232.  
  2233.    intel_miptree_downsample(brw, mt);
  2234.    intel_miptree_map_singlesample(brw, mt->singlesample_mt,
  2235.                                   level, slice,
  2236.                                   x, y, w, h,
  2237.                                   mode,
  2238.                                   out_ptr, out_stride);
  2239.    return;
  2240.  
  2241. fail:
  2242.    intel_miptree_release_map(mt, level, slice);
  2243.    *out_ptr = NULL;
  2244.    *out_stride = 0;
  2245. }
  2246.  
  2247. static void
  2248. intel_miptree_unmap_multisample(struct brw_context *brw,
  2249.                                 struct intel_mipmap_tree *mt,
  2250.                                 unsigned int level,
  2251.                                 unsigned int slice)
  2252. {
  2253.    struct intel_miptree_map *map = mt->level[level].slice[slice].map;
  2254.  
  2255.    assert(mt->num_samples > 1);
  2256.  
  2257.    if (!map)
  2258.       return;
  2259.  
  2260.    intel_miptree_unmap_singlesample(brw, mt->singlesample_mt, level, slice);
  2261.  
  2262.    mt->need_downsample = false;
  2263.    if (map->mode & GL_MAP_WRITE_BIT)
  2264.       intel_miptree_upsample(brw, mt);
  2265.  
  2266.    if (map->singlesample_mt_is_tmp)
  2267.       intel_miptree_release(&mt->singlesample_mt);
  2268.  
  2269.    intel_miptree_release_map(mt, level, slice);
  2270. }
  2271.  
  2272. void
  2273. intel_miptree_map(struct brw_context *brw,
  2274.                   struct intel_mipmap_tree *mt,
  2275.                   unsigned int level,
  2276.                   unsigned int slice,
  2277.                   unsigned int x,
  2278.                   unsigned int y,
  2279.                   unsigned int w,
  2280.                   unsigned int h,
  2281.                   GLbitfield mode,
  2282.                   void **out_ptr,
  2283.                   int *out_stride)
  2284. {
  2285.    if (mt->num_samples <= 1)
  2286.       intel_miptree_map_singlesample(brw, mt,
  2287.                                      level, slice,
  2288.                                      x, y, w, h,
  2289.                                      mode,
  2290.                                      out_ptr, out_stride);
  2291.    else
  2292.       intel_miptree_map_multisample(brw, mt,
  2293.                                     level, slice,
  2294.                                     x, y, w, h,
  2295.                                     mode,
  2296.                                     out_ptr, out_stride);
  2297. }
  2298.  
  2299. void
  2300. intel_miptree_unmap(struct brw_context *brw,
  2301.                     struct intel_mipmap_tree *mt,
  2302.                     unsigned int level,
  2303.                     unsigned int slice)
  2304. {
  2305.    if (mt->num_samples <= 1)
  2306.       intel_miptree_unmap_singlesample(brw, mt, level, slice);
  2307.    else
  2308.       intel_miptree_unmap_multisample(brw, mt, level, slice);
  2309. }
  2310.