Subversion Repositories Kolibri OS

Rev

Blame | Last modification | View Log | RSS feed

  1. /* Copyright © 2011 Intel Corporation
  2.  *
  3.  * Permission is hereby granted, free of charge, to any person obtaining a
  4.  * copy of this software and associated documentation files (the "Software"),
  5.  * to deal in the Software without restriction, including without limitation
  6.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  7.  * and/or sell copies of the Software, and to permit persons to whom the
  8.  * Software is furnished to do so, subject to the following conditions:
  9.  *
  10.  * The above copyright notice and this permission notice (including the next
  11.  * paragraph) shall be included in all copies or substantial portions of the
  12.  * Software.
  13.  *
  14.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  20.  * IN THE SOFTWARE.
  21.  */
  22.  
  23. #include <ctype.h>
  24. #include "brw_vec4.h"
  25. #include "brw_cfg.h"
  26.  
  27. extern "C" {
  28. #include "brw_eu.h"
  29. #include "main/macros.h"
  30. #include "program/prog_print.h"
  31. #include "program/prog_parameter.h"
  32. };
  33.  
  34. namespace brw {
  35.  
  36. struct brw_reg
  37. vec4_instruction::get_dst(void)
  38. {
  39.    struct brw_reg brw_reg;
  40.  
  41.    switch (dst.file) {
  42.    case GRF:
  43.       brw_reg = brw_vec8_grf(dst.reg + dst.reg_offset, 0);
  44.       brw_reg = retype(brw_reg, dst.type);
  45.       brw_reg.dw1.bits.writemask = dst.writemask;
  46.       break;
  47.  
  48.    case MRF:
  49.       brw_reg = brw_message_reg(dst.reg + dst.reg_offset);
  50.       brw_reg = retype(brw_reg, dst.type);
  51.       brw_reg.dw1.bits.writemask = dst.writemask;
  52.       break;
  53.  
  54.    case HW_REG:
  55.       assert(dst.type == dst.fixed_hw_reg.type);
  56.       brw_reg = dst.fixed_hw_reg;
  57.       break;
  58.  
  59.    case BAD_FILE:
  60.       brw_reg = brw_null_reg();
  61.       break;
  62.  
  63.    default:
  64.       unreachable("not reached");
  65.    }
  66.    return brw_reg;
  67. }
  68.  
  69. struct brw_reg
  70. vec4_instruction::get_src(const struct brw_vue_prog_data *prog_data, int i)
  71. {
  72.    struct brw_reg brw_reg;
  73.  
  74.    switch (src[i].file) {
  75.    case GRF:
  76.       brw_reg = brw_vec8_grf(src[i].reg + src[i].reg_offset, 0);
  77.       brw_reg = retype(brw_reg, src[i].type);
  78.       brw_reg.dw1.bits.swizzle = src[i].swizzle;
  79.       if (src[i].abs)
  80.          brw_reg = brw_abs(brw_reg);
  81.       if (src[i].negate)
  82.          brw_reg = negate(brw_reg);
  83.       break;
  84.  
  85.    case IMM:
  86.       switch (src[i].type) {
  87.       case BRW_REGISTER_TYPE_F:
  88.          brw_reg = brw_imm_f(src[i].fixed_hw_reg.dw1.f);
  89.          break;
  90.       case BRW_REGISTER_TYPE_D:
  91.          brw_reg = brw_imm_d(src[i].fixed_hw_reg.dw1.d);
  92.          break;
  93.       case BRW_REGISTER_TYPE_UD:
  94.          brw_reg = brw_imm_ud(src[i].fixed_hw_reg.dw1.ud);
  95.          break;
  96.       case BRW_REGISTER_TYPE_VF:
  97.          brw_reg = brw_imm_vf(src[i].fixed_hw_reg.dw1.ud);
  98.          break;
  99.       default:
  100.          unreachable("not reached");
  101.       }
  102.       break;
  103.  
  104.    case UNIFORM:
  105.       brw_reg = stride(brw_vec4_grf(prog_data->base.dispatch_grf_start_reg +
  106.                                     (src[i].reg + src[i].reg_offset) / 2,
  107.                                     ((src[i].reg + src[i].reg_offset) % 2) * 4),
  108.                        0, 4, 1);
  109.       brw_reg = retype(brw_reg, src[i].type);
  110.       brw_reg.dw1.bits.swizzle = src[i].swizzle;
  111.       if (src[i].abs)
  112.          brw_reg = brw_abs(brw_reg);
  113.       if (src[i].negate)
  114.          brw_reg = negate(brw_reg);
  115.  
  116.       /* This should have been moved to pull constants. */
  117.       assert(!src[i].reladdr);
  118.       break;
  119.  
  120.    case HW_REG:
  121.       assert(src[i].type == src[i].fixed_hw_reg.type);
  122.       brw_reg = src[i].fixed_hw_reg;
  123.       break;
  124.  
  125.    case BAD_FILE:
  126.       /* Probably unused. */
  127.       brw_reg = brw_null_reg();
  128.       break;
  129.    case ATTR:
  130.    default:
  131.       unreachable("not reached");
  132.    }
  133.  
  134.    return brw_reg;
  135. }
  136.  
  137. vec4_generator::vec4_generator(struct brw_context *brw,
  138.                                struct gl_shader_program *shader_prog,
  139.                                struct gl_program *prog,
  140.                                struct brw_vue_prog_data *prog_data,
  141.                                void *mem_ctx,
  142.                                bool debug_flag,
  143.                                const char *stage_name,
  144.                                const char *stage_abbrev)
  145.    : brw(brw), devinfo(brw->intelScreen->devinfo),
  146.      shader_prog(shader_prog), prog(prog), prog_data(prog_data),
  147.      mem_ctx(mem_ctx), stage_name(stage_name), stage_abbrev(stage_abbrev),
  148.      debug_flag(debug_flag)
  149. {
  150.    p = rzalloc(mem_ctx, struct brw_codegen);
  151.    brw_init_codegen(brw->intelScreen->devinfo, p, mem_ctx);
  152. }
  153.  
  154. vec4_generator::~vec4_generator()
  155. {
  156. }
  157.  
  158. void
  159. vec4_generator::generate_math1_gen4(vec4_instruction *inst,
  160.                                     struct brw_reg dst,
  161.                                     struct brw_reg src)
  162. {
  163.    gen4_math(p,
  164.              dst,
  165.              brw_math_function(inst->opcode),
  166.              inst->base_mrf,
  167.              src,
  168.              BRW_MATH_PRECISION_FULL);
  169. }
  170.  
  171. static void
  172. check_gen6_math_src_arg(struct brw_reg src)
  173. {
  174.    /* Source swizzles are ignored. */
  175.    assert(!src.abs);
  176.    assert(!src.negate);
  177.    assert(src.dw1.bits.swizzle == BRW_SWIZZLE_XYZW);
  178. }
  179.  
  180. void
  181. vec4_generator::generate_math_gen6(vec4_instruction *inst,
  182.                                    struct brw_reg dst,
  183.                                    struct brw_reg src0,
  184.                                    struct brw_reg src1)
  185. {
  186.    /* Can't do writemask because math can't be align16. */
  187.    assert(dst.dw1.bits.writemask == WRITEMASK_XYZW);
  188.    /* Source swizzles are ignored. */
  189.    check_gen6_math_src_arg(src0);
  190.    if (src1.file == BRW_GENERAL_REGISTER_FILE)
  191.       check_gen6_math_src_arg(src1);
  192.  
  193.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  194.    gen6_math(p, dst, brw_math_function(inst->opcode), src0, src1);
  195.    brw_set_default_access_mode(p, BRW_ALIGN_16);
  196. }
  197.  
  198. void
  199. vec4_generator::generate_math2_gen4(vec4_instruction *inst,
  200.                                     struct brw_reg dst,
  201.                                     struct brw_reg src0,
  202.                                     struct brw_reg src1)
  203. {
  204.    /* From the Ironlake PRM, Volume 4, Part 1, Section 6.1.13
  205.     * "Message Payload":
  206.     *
  207.     * "Operand0[7].  For the INT DIV functions, this operand is the
  208.     *  denominator."
  209.     *  ...
  210.     * "Operand1[7].  For the INT DIV functions, this operand is the
  211.     *  numerator."
  212.     */
  213.    bool is_int_div = inst->opcode != SHADER_OPCODE_POW;
  214.    struct brw_reg &op0 = is_int_div ? src1 : src0;
  215.    struct brw_reg &op1 = is_int_div ? src0 : src1;
  216.  
  217.    brw_push_insn_state(p);
  218.    brw_set_default_saturate(p, false);
  219.    brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
  220.    brw_MOV(p, retype(brw_message_reg(inst->base_mrf + 1), op1.type), op1);
  221.    brw_pop_insn_state(p);
  222.  
  223.    gen4_math(p,
  224.              dst,
  225.              brw_math_function(inst->opcode),
  226.              inst->base_mrf,
  227.              op0,
  228.              BRW_MATH_PRECISION_FULL);
  229. }
  230.  
  231. void
  232. vec4_generator::generate_tex(vec4_instruction *inst,
  233.                              struct brw_reg dst,
  234.                              struct brw_reg src,
  235.                              struct brw_reg sampler_index)
  236. {
  237.    int msg_type = -1;
  238.  
  239.    if (devinfo->gen >= 5) {
  240.       switch (inst->opcode) {
  241.       case SHADER_OPCODE_TEX:
  242.       case SHADER_OPCODE_TXL:
  243.          if (inst->shadow_compare) {
  244.             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE;
  245.          } else {
  246.             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LOD;
  247.          }
  248.          break;
  249.       case SHADER_OPCODE_TXD:
  250.          if (inst->shadow_compare) {
  251.             /* Gen7.5+.  Otherwise, lowered by brw_lower_texture_gradients(). */
  252.             assert(devinfo->gen >= 8 || devinfo->is_haswell);
  253.             msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
  254.          } else {
  255.             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_DERIVS;
  256.          }
  257.          break;
  258.       case SHADER_OPCODE_TXF:
  259.          msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
  260.          break;
  261.       case SHADER_OPCODE_TXF_CMS:
  262.          if (devinfo->gen >= 7)
  263.             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD2DMS;
  264.          else
  265.             msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_LD;
  266.          break;
  267.       case SHADER_OPCODE_TXF_MCS:
  268.          assert(devinfo->gen >= 7);
  269.          msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_LD_MCS;
  270.          break;
  271.       case SHADER_OPCODE_TXS:
  272.          msg_type = GEN5_SAMPLER_MESSAGE_SAMPLE_RESINFO;
  273.          break;
  274.       case SHADER_OPCODE_TG4:
  275.          if (inst->shadow_compare) {
  276.             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C;
  277.          } else {
  278.             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4;
  279.          }
  280.          break;
  281.       case SHADER_OPCODE_TG4_OFFSET:
  282.          if (inst->shadow_compare) {
  283.             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C;
  284.          } else {
  285.             msg_type = GEN7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO;
  286.          }
  287.          break;
  288.       default:
  289.          unreachable("should not get here: invalid vec4 texture opcode");
  290.       }
  291.    } else {
  292.       switch (inst->opcode) {
  293.       case SHADER_OPCODE_TEX:
  294.       case SHADER_OPCODE_TXL:
  295.          if (inst->shadow_compare) {
  296.             msg_type = BRW_SAMPLER_MESSAGE_SIMD4X2_SAMPLE_LOD_COMPARE;
  297.             assert(inst->mlen == 3);
  298.          } else {
  299.             msg_type = BRW_SAMPLER_MESSAGE_SIMD4X2_SAMPLE_LOD;
  300.             assert(inst->mlen == 2);
  301.          }
  302.          break;
  303.       case SHADER_OPCODE_TXD:
  304.          /* There is no sample_d_c message; comparisons are done manually. */
  305.          msg_type = BRW_SAMPLER_MESSAGE_SIMD4X2_SAMPLE_GRADIENTS;
  306.          assert(inst->mlen == 4);
  307.          break;
  308.       case SHADER_OPCODE_TXF:
  309.          msg_type = BRW_SAMPLER_MESSAGE_SIMD4X2_LD;
  310.          assert(inst->mlen == 2);
  311.          break;
  312.       case SHADER_OPCODE_TXS:
  313.          msg_type = BRW_SAMPLER_MESSAGE_SIMD4X2_RESINFO;
  314.          assert(inst->mlen == 2);
  315.          break;
  316.       default:
  317.          unreachable("should not get here: invalid vec4 texture opcode");
  318.       }
  319.    }
  320.  
  321.    assert(msg_type != -1);
  322.  
  323.    assert(sampler_index.type == BRW_REGISTER_TYPE_UD);
  324.  
  325.    /* Load the message header if present.  If there's a texture offset, we need
  326.     * to set it up explicitly and load the offset bitfield.  Otherwise, we can
  327.     * use an implied move from g0 to the first message register.
  328.     */
  329.    if (inst->header_size != 0) {
  330.       if (devinfo->gen < 6 && !inst->offset) {
  331.          /* Set up an implied move from g0 to the MRF. */
  332.          src = brw_vec8_grf(0, 0);
  333.       } else {
  334.          struct brw_reg header =
  335.             retype(brw_message_reg(inst->base_mrf), BRW_REGISTER_TYPE_UD);
  336.          uint32_t dw2 = 0;
  337.  
  338.          /* Explicitly set up the message header by copying g0 to the MRF. */
  339.          brw_push_insn_state(p);
  340.          brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  341.          brw_MOV(p, header, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
  342.  
  343.          brw_set_default_access_mode(p, BRW_ALIGN_1);
  344.  
  345.          if (inst->offset)
  346.             /* Set the texel offset bits in DWord 2. */
  347.             dw2 = inst->offset;
  348.  
  349.          if (devinfo->gen >= 9)
  350.             /* SKL+ overloads BRW_SAMPLER_SIMD_MODE_SIMD4X2 to also do SIMD8D,
  351.              * based on bit 22 in the header.
  352.              */
  353.             dw2 |= GEN9_SAMPLER_SIMD_MODE_EXTENSION_SIMD4X2;
  354.  
  355.          if (dw2)
  356.             brw_MOV(p, get_element_ud(header, 2), brw_imm_ud(dw2));
  357.  
  358.          brw_adjust_sampler_state_pointer(p, header, sampler_index);
  359.          brw_pop_insn_state(p);
  360.       }
  361.    }
  362.  
  363.    uint32_t return_format;
  364.  
  365.    switch (dst.type) {
  366.    case BRW_REGISTER_TYPE_D:
  367.       return_format = BRW_SAMPLER_RETURN_FORMAT_SINT32;
  368.       break;
  369.    case BRW_REGISTER_TYPE_UD:
  370.       return_format = BRW_SAMPLER_RETURN_FORMAT_UINT32;
  371.       break;
  372.    default:
  373.       return_format = BRW_SAMPLER_RETURN_FORMAT_FLOAT32;
  374.       break;
  375.    }
  376.  
  377.    uint32_t base_binding_table_index = (inst->opcode == SHADER_OPCODE_TG4 ||
  378.          inst->opcode == SHADER_OPCODE_TG4_OFFSET)
  379.          ? prog_data->base.binding_table.gather_texture_start
  380.          : prog_data->base.binding_table.texture_start;
  381.  
  382.    if (sampler_index.file == BRW_IMMEDIATE_VALUE) {
  383.       uint32_t sampler = sampler_index.dw1.ud;
  384.  
  385.       brw_SAMPLE(p,
  386.                  dst,
  387.                  inst->base_mrf,
  388.                  src,
  389.                  sampler + base_binding_table_index,
  390.                  sampler % 16,
  391.                  msg_type,
  392.                  1, /* response length */
  393.                  inst->mlen,
  394.                  inst->header_size != 0,
  395.                  BRW_SAMPLER_SIMD_MODE_SIMD4X2,
  396.                  return_format);
  397.  
  398.       brw_mark_surface_used(&prog_data->base, sampler + base_binding_table_index);
  399.    } else {
  400.       /* Non-constant sampler index. */
  401.       /* Note: this clobbers `dst` as a temporary before emitting the send */
  402.  
  403.       struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
  404.       struct brw_reg temp = vec1(retype(dst, BRW_REGISTER_TYPE_UD));
  405.  
  406.       struct brw_reg sampler_reg = vec1(retype(sampler_index, BRW_REGISTER_TYPE_UD));
  407.  
  408.       brw_push_insn_state(p);
  409.       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  410.       brw_set_default_access_mode(p, BRW_ALIGN_1);
  411.  
  412.       /* Some care required: `sampler` and `temp` may alias:
  413.        *    addr = sampler & 0xff
  414.        *    temp = (sampler << 8) & 0xf00
  415.        *    addr = addr | temp
  416.        */
  417.       brw_ADD(p, addr, sampler_reg, brw_imm_ud(base_binding_table_index));
  418.       brw_SHL(p, temp, sampler_reg, brw_imm_ud(8u));
  419.       brw_AND(p, temp, temp, brw_imm_ud(0x0f00));
  420.       brw_AND(p, addr, addr, brw_imm_ud(0x0ff));
  421.       brw_OR(p, addr, addr, temp);
  422.  
  423.       brw_pop_insn_state(p);
  424.  
  425.       /* dst = send(offset, a0.0 | <descriptor>) */
  426.       brw_inst *insn = brw_send_indirect_message(
  427.          p, BRW_SFID_SAMPLER, dst, src, addr);
  428.       brw_set_sampler_message(p, insn,
  429.                               0 /* surface */,
  430.                               0 /* sampler */,
  431.                               msg_type,
  432.                               1 /* rlen */,
  433.                               inst->mlen /* mlen */,
  434.                               inst->header_size != 0 /* header */,
  435.                               BRW_SAMPLER_SIMD_MODE_SIMD4X2,
  436.                               return_format);
  437.  
  438.       /* visitor knows more than we do about the surface limit required,
  439.        * so has already done marking.
  440.        */
  441.    }
  442. }
  443.  
  444. void
  445. vec4_generator::generate_vs_urb_write(vec4_instruction *inst)
  446. {
  447.    brw_urb_WRITE(p,
  448.                  brw_null_reg(), /* dest */
  449.                  inst->base_mrf, /* starting mrf reg nr */
  450.                  brw_vec8_grf(0, 0), /* src */
  451.                  inst->urb_write_flags,
  452.                  inst->mlen,
  453.                  0,             /* response len */
  454.                  inst->offset,  /* urb destination offset */
  455.                  BRW_URB_SWIZZLE_INTERLEAVE);
  456. }
  457.  
  458. void
  459. vec4_generator::generate_gs_urb_write(vec4_instruction *inst)
  460. {
  461.    struct brw_reg src = brw_message_reg(inst->base_mrf);
  462.    brw_urb_WRITE(p,
  463.                  brw_null_reg(), /* dest */
  464.                  inst->base_mrf, /* starting mrf reg nr */
  465.                  src,
  466.                  inst->urb_write_flags,
  467.                  inst->mlen,
  468.                  0,             /* response len */
  469.                  inst->offset,  /* urb destination offset */
  470.                  BRW_URB_SWIZZLE_INTERLEAVE);
  471. }
  472.  
  473. void
  474. vec4_generator::generate_gs_urb_write_allocate(vec4_instruction *inst)
  475. {
  476.    struct brw_reg src = brw_message_reg(inst->base_mrf);
  477.  
  478.    /* We pass the temporary passed in src0 as the writeback register */
  479.    brw_urb_WRITE(p,
  480.                  inst->get_src(this->prog_data, 0), /* dest */
  481.                  inst->base_mrf, /* starting mrf reg nr */
  482.                  src,
  483.                  BRW_URB_WRITE_ALLOCATE_COMPLETE,
  484.                  inst->mlen,
  485.                  1, /* response len */
  486.                  inst->offset,  /* urb destination offset */
  487.                  BRW_URB_SWIZZLE_INTERLEAVE);
  488.  
  489.    /* Now put allocated urb handle in dst.0 */
  490.    brw_push_insn_state(p);
  491.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  492.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  493.    brw_MOV(p, get_element_ud(inst->get_dst(), 0),
  494.            get_element_ud(inst->get_src(this->prog_data, 0), 0));
  495.    brw_set_default_access_mode(p, BRW_ALIGN_16);
  496.    brw_pop_insn_state(p);
  497. }
  498.  
  499. void
  500. vec4_generator::generate_gs_thread_end(vec4_instruction *inst)
  501. {
  502.    struct brw_reg src = brw_message_reg(inst->base_mrf);
  503.    brw_urb_WRITE(p,
  504.                  brw_null_reg(), /* dest */
  505.                  inst->base_mrf, /* starting mrf reg nr */
  506.                  src,
  507.                  BRW_URB_WRITE_EOT | inst->urb_write_flags,
  508.                  devinfo->gen >= 8 ? 2 : 1,/* message len */
  509.                  0,              /* response len */
  510.                  0,              /* urb destination offset */
  511.                  BRW_URB_SWIZZLE_INTERLEAVE);
  512. }
  513.  
  514. void
  515. vec4_generator::generate_gs_set_write_offset(struct brw_reg dst,
  516.                                              struct brw_reg src0,
  517.                                              struct brw_reg src1)
  518. {
  519.    /* From p22 of volume 4 part 2 of the Ivy Bridge PRM (2.4.3.1 Message
  520.     * Header: M0.3):
  521.     *
  522.     *     Slot 0 Offset. This field, after adding to the Global Offset field
  523.     *     in the message descriptor, specifies the offset (in 256-bit units)
  524.     *     from the start of the URB entry, as referenced by URB Handle 0, at
  525.     *     which the data will be accessed.
  526.     *
  527.     * Similar text describes DWORD M0.4, which is slot 1 offset.
  528.     *
  529.     * Therefore, we want to multiply DWORDs 0 and 4 of src0 (the x components
  530.     * of the register for geometry shader invocations 0 and 1) by the
  531.     * immediate value in src1, and store the result in DWORDs 3 and 4 of dst.
  532.     *
  533.     * We can do this with the following EU instruction:
  534.     *
  535.     *     mul(2) dst.3<1>UD src0<8;2,4>UD src1<...>UW   { Align1 WE_all }
  536.     */
  537.    brw_push_insn_state(p);
  538.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  539.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  540.    assert(devinfo->gen >= 7 &&
  541.           src1.file == BRW_IMMEDIATE_VALUE &&
  542.           src1.type == BRW_REGISTER_TYPE_UD &&
  543.           src1.dw1.ud <= USHRT_MAX);
  544.    brw_MUL(p, suboffset(stride(dst, 2, 2, 1), 3), stride(src0, 8, 2, 4),
  545.            retype(src1, BRW_REGISTER_TYPE_UW));
  546.    brw_set_default_access_mode(p, BRW_ALIGN_16);
  547.    brw_pop_insn_state(p);
  548. }
  549.  
  550. void
  551. vec4_generator::generate_gs_set_vertex_count(struct brw_reg dst,
  552.                                              struct brw_reg src)
  553. {
  554.    brw_push_insn_state(p);
  555.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  556.  
  557.    if (devinfo->gen >= 8) {
  558.       /* Move the vertex count into the second MRF for the EOT write. */
  559.       brw_MOV(p, retype(brw_message_reg(dst.nr + 1), BRW_REGISTER_TYPE_UD),
  560.               src);
  561.    } else {
  562.       /* If we think of the src and dst registers as composed of 8 DWORDs each,
  563.        * we want to pick up the contents of DWORDs 0 and 4 from src, truncate
  564.        * them to WORDs, and then pack them into DWORD 2 of dst.
  565.        *
  566.        * It's easier to get the EU to do this if we think of the src and dst
  567.        * registers as composed of 16 WORDS each; then, we want to pick up the
  568.        * contents of WORDs 0 and 8 from src, and pack them into WORDs 4 and 5
  569.        * of dst.
  570.        *
  571.        * We can do that by the following EU instruction:
  572.        *
  573.        *     mov (2) dst.4<1>:uw src<8;1,0>:uw   { Align1, Q1, NoMask }
  574.        */
  575.       brw_set_default_access_mode(p, BRW_ALIGN_1);
  576.       brw_MOV(p,
  577.               suboffset(stride(retype(dst, BRW_REGISTER_TYPE_UW), 2, 2, 1), 4),
  578.               stride(retype(src, BRW_REGISTER_TYPE_UW), 8, 1, 0));
  579.       brw_set_default_access_mode(p, BRW_ALIGN_16);
  580.    }
  581.    brw_pop_insn_state(p);
  582. }
  583.  
  584. void
  585. vec4_generator::generate_gs_svb_write(vec4_instruction *inst,
  586.                                       struct brw_reg dst,
  587.                                       struct brw_reg src0,
  588.                                       struct brw_reg src1)
  589. {
  590.    int binding = inst->sol_binding;
  591.    bool final_write = inst->sol_final_write;
  592.  
  593.    brw_push_insn_state(p);
  594.    /* Copy Vertex data into M0.x */
  595.    brw_MOV(p, stride(dst, 4, 4, 1),
  596.            stride(retype(src0, BRW_REGISTER_TYPE_UD), 4, 4, 1));
  597.  
  598.    /* Send SVB Write */
  599.    brw_svb_write(p,
  600.                  final_write ? src1 : brw_null_reg(), /* dest == src1 */
  601.                  1, /* msg_reg_nr */
  602.                  dst, /* src0 == previous dst */
  603.                  SURF_INDEX_GEN6_SOL_BINDING(binding), /* binding_table_index */
  604.                  final_write); /* send_commit_msg */
  605.  
  606.    /* Finally, wait for the write commit to occur so that we can proceed to
  607.     * other things safely.
  608.     *
  609.     * From the Sandybridge PRM, Volume 4, Part 1, Section 3.3:
  610.     *
  611.     *   The write commit does not modify the destination register, but
  612.     *   merely clears the dependency associated with the destination
  613.     *   register. Thus, a simple “mov” instruction using the register as a
  614.     *   source is sufficient to wait for the write commit to occur.
  615.     */
  616.    if (final_write) {
  617.       brw_MOV(p, src1, src1);
  618.    }
  619.    brw_pop_insn_state(p);
  620. }
  621.  
  622. void
  623. vec4_generator::generate_gs_svb_set_destination_index(vec4_instruction *inst,
  624.                                                       struct brw_reg dst,
  625.                                                       struct brw_reg src)
  626. {
  627.  
  628.    int vertex = inst->sol_vertex;
  629.    brw_push_insn_state(p);
  630.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  631.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  632.    brw_MOV(p, get_element_ud(dst, 5), get_element_ud(src, vertex));
  633.    brw_pop_insn_state(p);
  634. }
  635.  
  636. void
  637. vec4_generator::generate_gs_set_dword_2(struct brw_reg dst, struct brw_reg src)
  638. {
  639.    brw_push_insn_state(p);
  640.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  641.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  642.    brw_MOV(p, suboffset(vec1(dst), 2), suboffset(vec1(src), 0));
  643.    brw_pop_insn_state(p);
  644. }
  645.  
  646. void
  647. vec4_generator::generate_gs_prepare_channel_masks(struct brw_reg dst)
  648. {
  649.    /* We want to left shift just DWORD 4 (the x component belonging to the
  650.     * second geometry shader invocation) by 4 bits.  So generate the
  651.     * instruction:
  652.     *
  653.     *     shl(1) dst.4<1>UD dst.4<0,1,0>UD 4UD { align1 WE_all }
  654.     */
  655.    dst = suboffset(vec1(dst), 4);
  656.    brw_push_insn_state(p);
  657.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  658.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  659.    brw_SHL(p, dst, dst, brw_imm_ud(4));
  660.    brw_pop_insn_state(p);
  661. }
  662.  
  663. void
  664. vec4_generator::generate_gs_set_channel_masks(struct brw_reg dst,
  665.                                               struct brw_reg src)
  666. {
  667.    /* From p21 of volume 4 part 2 of the Ivy Bridge PRM (2.4.3.1 Message
  668.     * Header: M0.5):
  669.     *
  670.     *     15 Vertex 1 DATA [3] / Vertex 0 DATA[7] Channel Mask
  671.     *
  672.     *        When Swizzle Control = URB_INTERLEAVED this bit controls Vertex 1
  673.     *        DATA[3], when Swizzle Control = URB_NOSWIZZLE this bit controls
  674.     *        Vertex 0 DATA[7].  This bit is ANDed with the corresponding
  675.     *        channel enable to determine the final channel enable.  For the
  676.     *        URB_READ_OWORD & URB_READ_HWORD messages, when final channel
  677.     *        enable is 1 it indicates that Vertex 1 DATA [3] will be included
  678.     *        in the writeback message.  For the URB_WRITE_OWORD &
  679.     *        URB_WRITE_HWORD messages, when final channel enable is 1 it
  680.     *        indicates that Vertex 1 DATA [3] will be written to the surface.
  681.     *
  682.     *        0: Vertex 1 DATA [3] / Vertex 0 DATA[7] channel not included
  683.     *        1: Vertex DATA [3] / Vertex 0 DATA[7] channel included
  684.     *
  685.     *     14 Vertex 1 DATA [2] Channel Mask
  686.     *     13 Vertex 1 DATA [1] Channel Mask
  687.     *     12 Vertex 1 DATA [0] Channel Mask
  688.     *     11 Vertex 0 DATA [3] Channel Mask
  689.     *     10 Vertex 0 DATA [2] Channel Mask
  690.     *      9 Vertex 0 DATA [1] Channel Mask
  691.     *      8 Vertex 0 DATA [0] Channel Mask
  692.     *
  693.     * (This is from a section of the PRM that is agnostic to the particular
  694.     * type of shader being executed, so "Vertex 0" and "Vertex 1" refer to
  695.     * geometry shader invocations 0 and 1, respectively).  Since we have the
  696.     * enable flags for geometry shader invocation 0 in bits 3:0 of DWORD 0,
  697.     * and the enable flags for geometry shader invocation 1 in bits 7:0 of
  698.     * DWORD 4, we just need to OR them together and store the result in bits
  699.     * 15:8 of DWORD 5.
  700.     *
  701.     * It's easier to get the EU to do this if we think of the src and dst
  702.     * registers as composed of 32 bytes each; then, we want to pick up the
  703.     * contents of bytes 0 and 16 from src, OR them together, and store them in
  704.     * byte 21.
  705.     *
  706.     * We can do that by the following EU instruction:
  707.     *
  708.     *     or(1) dst.21<1>UB src<0,1,0>UB src.16<0,1,0>UB { align1 WE_all }
  709.     *
  710.     * Note: this relies on the source register having zeros in (a) bits 7:4 of
  711.     * DWORD 0 and (b) bits 3:0 of DWORD 4.  We can rely on (b) because the
  712.     * source register was prepared by GS_OPCODE_PREPARE_CHANNEL_MASKS (which
  713.     * shifts DWORD 4 left by 4 bits), and we can rely on (a) because prior to
  714.     * the execution of GS_OPCODE_PREPARE_CHANNEL_MASKS, DWORDs 0 and 4 need to
  715.     * contain valid channel mask values (which are in the range 0x0-0xf).
  716.     */
  717.    dst = retype(dst, BRW_REGISTER_TYPE_UB);
  718.    src = retype(src, BRW_REGISTER_TYPE_UB);
  719.    brw_push_insn_state(p);
  720.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  721.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  722.    brw_OR(p, suboffset(vec1(dst), 21), vec1(src), suboffset(vec1(src), 16));
  723.    brw_pop_insn_state(p);
  724. }
  725.  
  726. void
  727. vec4_generator::generate_gs_get_instance_id(struct brw_reg dst)
  728. {
  729.    /* We want to right shift R0.0 & R0.1 by GEN7_GS_PAYLOAD_INSTANCE_ID_SHIFT
  730.     * and store into dst.0 & dst.4. So generate the instruction:
  731.     *
  732.     *     shr(8) dst<1> R0<1,4,0> GEN7_GS_PAYLOAD_INSTANCE_ID_SHIFT { align1 WE_normal 1Q }
  733.     */
  734.    brw_push_insn_state(p);
  735.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  736.    dst = retype(dst, BRW_REGISTER_TYPE_UD);
  737.    struct brw_reg r0(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
  738.    brw_SHR(p, dst, stride(r0, 1, 4, 0),
  739.            brw_imm_ud(GEN7_GS_PAYLOAD_INSTANCE_ID_SHIFT));
  740.    brw_pop_insn_state(p);
  741. }
  742.  
  743. void
  744. vec4_generator::generate_gs_ff_sync_set_primitives(struct brw_reg dst,
  745.                                                    struct brw_reg src0,
  746.                                                    struct brw_reg src1,
  747.                                                    struct brw_reg src2)
  748. {
  749.    brw_push_insn_state(p);
  750.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  751.    /* Save src0 data in 16:31 bits of dst.0 */
  752.    brw_AND(p, suboffset(vec1(dst), 0), suboffset(vec1(src0), 0),
  753.            brw_imm_ud(0xffffu));
  754.    brw_SHL(p, suboffset(vec1(dst), 0), suboffset(vec1(dst), 0), brw_imm_ud(16));
  755.    /* Save src1 data in 0:15 bits of dst.0 */
  756.    brw_AND(p, suboffset(vec1(src2), 0), suboffset(vec1(src1), 0),
  757.            brw_imm_ud(0xffffu));
  758.    brw_OR(p, suboffset(vec1(dst), 0),
  759.           suboffset(vec1(dst), 0),
  760.           suboffset(vec1(src2), 0));
  761.    brw_pop_insn_state(p);
  762. }
  763.  
  764. void
  765. vec4_generator::generate_gs_ff_sync(vec4_instruction *inst,
  766.                                     struct brw_reg dst,
  767.                                     struct brw_reg src0,
  768.                                     struct brw_reg src1)
  769. {
  770.    /* This opcode uses an implied MRF register for:
  771.     *  - the header of the ff_sync message. And as such it is expected to be
  772.     *    initialized to r0 before calling here.
  773.     *  - the destination where we will write the allocated URB handle.
  774.     */
  775.    struct brw_reg header =
  776.       retype(brw_message_reg(inst->base_mrf), BRW_REGISTER_TYPE_UD);
  777.  
  778.    /* Overwrite dword 0 of the header (SO vertices to write) and
  779.     * dword 1 (number of primitives written).
  780.     */
  781.    brw_push_insn_state(p);
  782.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  783.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  784.    brw_MOV(p, get_element_ud(header, 0), get_element_ud(src1, 0));
  785.    brw_MOV(p, get_element_ud(header, 1), get_element_ud(src0, 0));
  786.    brw_pop_insn_state(p);
  787.  
  788.    /* Allocate URB handle in dst */
  789.    brw_ff_sync(p,
  790.                dst,
  791.                0,
  792.                header,
  793.                1, /* allocate */
  794.                1, /* response length */
  795.                0 /* eot */);
  796.  
  797.    /* Now put allocated urb handle in header.0 */
  798.    brw_push_insn_state(p);
  799.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  800.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  801.    brw_MOV(p, get_element_ud(header, 0), get_element_ud(dst, 0));
  802.  
  803.    /* src1 is not an immediate when we use transform feedback */
  804.    if (src1.file != BRW_IMMEDIATE_VALUE)
  805.       brw_MOV(p, brw_vec4_grf(src1.nr, 0), brw_vec4_grf(dst.nr, 1));
  806.  
  807.    brw_pop_insn_state(p);
  808. }
  809.  
  810. void
  811. vec4_generator::generate_gs_set_primitive_id(struct brw_reg dst)
  812. {
  813.    /* In gen6, PrimitiveID is delivered in R0.1 of the payload */
  814.    struct brw_reg src = brw_vec8_grf(0, 0);
  815.    brw_push_insn_state(p);
  816.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  817.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  818.    brw_MOV(p, get_element_ud(dst, 0), get_element_ud(src, 1));
  819.    brw_pop_insn_state(p);
  820. }
  821.  
  822. void
  823. vec4_generator::generate_oword_dual_block_offsets(struct brw_reg m1,
  824.                                                   struct brw_reg index)
  825. {
  826.    int second_vertex_offset;
  827.  
  828.    if (devinfo->gen >= 6)
  829.       second_vertex_offset = 1;
  830.    else
  831.       second_vertex_offset = 16;
  832.  
  833.    m1 = retype(m1, BRW_REGISTER_TYPE_D);
  834.  
  835.    /* Set up M1 (message payload).  Only the block offsets in M1.0 and
  836.     * M1.4 are used, and the rest are ignored.
  837.     */
  838.    struct brw_reg m1_0 = suboffset(vec1(m1), 0);
  839.    struct brw_reg m1_4 = suboffset(vec1(m1), 4);
  840.    struct brw_reg index_0 = suboffset(vec1(index), 0);
  841.    struct brw_reg index_4 = suboffset(vec1(index), 4);
  842.  
  843.    brw_push_insn_state(p);
  844.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  845.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  846.  
  847.    brw_MOV(p, m1_0, index_0);
  848.  
  849.    if (index.file == BRW_IMMEDIATE_VALUE) {
  850.       index_4.dw1.ud += second_vertex_offset;
  851.       brw_MOV(p, m1_4, index_4);
  852.    } else {
  853.       brw_ADD(p, m1_4, index_4, brw_imm_d(second_vertex_offset));
  854.    }
  855.  
  856.    brw_pop_insn_state(p);
  857. }
  858.  
  859. void
  860. vec4_generator::generate_unpack_flags(struct brw_reg dst)
  861. {
  862.    brw_push_insn_state(p);
  863.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  864.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  865.  
  866.    struct brw_reg flags = brw_flag_reg(0, 0);
  867.    struct brw_reg dst_0 = suboffset(vec1(dst), 0);
  868.    struct brw_reg dst_4 = suboffset(vec1(dst), 4);
  869.  
  870.    brw_AND(p, dst_0, flags, brw_imm_ud(0x0f));
  871.    brw_AND(p, dst_4, flags, brw_imm_ud(0xf0));
  872.    brw_SHR(p, dst_4, dst_4, brw_imm_ud(4));
  873.  
  874.    brw_pop_insn_state(p);
  875. }
  876.  
  877. void
  878. vec4_generator::generate_scratch_read(vec4_instruction *inst,
  879.                                       struct brw_reg dst,
  880.                                       struct brw_reg index)
  881. {
  882.    struct brw_reg header = brw_vec8_grf(0, 0);
  883.  
  884.    gen6_resolve_implied_move(p, &header, inst->base_mrf);
  885.  
  886.    generate_oword_dual_block_offsets(brw_message_reg(inst->base_mrf + 1),
  887.                                      index);
  888.  
  889.    uint32_t msg_type;
  890.  
  891.    if (devinfo->gen >= 6)
  892.       msg_type = GEN6_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ;
  893.    else if (devinfo->gen == 5 || devinfo->is_g4x)
  894.       msg_type = G45_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ;
  895.    else
  896.       msg_type = BRW_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ;
  897.  
  898.    /* Each of the 8 channel enables is considered for whether each
  899.     * dword is written.
  900.     */
  901.    brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
  902.    brw_set_dest(p, send, dst);
  903.    brw_set_src0(p, send, header);
  904.    if (devinfo->gen < 6)
  905.       brw_inst_set_cond_modifier(p->devinfo, send, inst->base_mrf);
  906.    brw_set_dp_read_message(p, send,
  907.                            255, /* binding table index: stateless access */
  908.                            BRW_DATAPORT_OWORD_DUAL_BLOCK_1OWORD,
  909.                            msg_type,
  910.                            BRW_DATAPORT_READ_TARGET_RENDER_CACHE,
  911.                            2, /* mlen */
  912.                            true, /* header_present */
  913.                            1 /* rlen */);
  914. }
  915.  
  916. void
  917. vec4_generator::generate_scratch_write(vec4_instruction *inst,
  918.                                        struct brw_reg dst,
  919.                                        struct brw_reg src,
  920.                                        struct brw_reg index)
  921. {
  922.    struct brw_reg header = brw_vec8_grf(0, 0);
  923.    bool write_commit;
  924.  
  925.    /* If the instruction is predicated, we'll predicate the send, not
  926.     * the header setup.
  927.     */
  928.    brw_set_default_predicate_control(p, false);
  929.  
  930.    gen6_resolve_implied_move(p, &header, inst->base_mrf);
  931.  
  932.    generate_oword_dual_block_offsets(brw_message_reg(inst->base_mrf + 1),
  933.                                      index);
  934.  
  935.    brw_MOV(p,
  936.            retype(brw_message_reg(inst->base_mrf + 2), BRW_REGISTER_TYPE_D),
  937.            retype(src, BRW_REGISTER_TYPE_D));
  938.  
  939.    uint32_t msg_type;
  940.  
  941.    if (devinfo->gen >= 7)
  942.       msg_type = GEN7_DATAPORT_DC_OWORD_DUAL_BLOCK_WRITE;
  943.    else if (devinfo->gen == 6)
  944.       msg_type = GEN6_DATAPORT_WRITE_MESSAGE_OWORD_DUAL_BLOCK_WRITE;
  945.    else
  946.       msg_type = BRW_DATAPORT_WRITE_MESSAGE_OWORD_DUAL_BLOCK_WRITE;
  947.  
  948.    brw_set_default_predicate_control(p, inst->predicate);
  949.  
  950.    /* Pre-gen6, we have to specify write commits to ensure ordering
  951.     * between reads and writes within a thread.  Afterwards, that's
  952.     * guaranteed and write commits only matter for inter-thread
  953.     * synchronization.
  954.     */
  955.    if (devinfo->gen >= 6) {
  956.       write_commit = false;
  957.    } else {
  958.       /* The visitor set up our destination register to be g0.  This
  959.        * means that when the next read comes along, we will end up
  960.        * reading from g0 and causing a block on the write commit.  For
  961.        * write-after-read, we are relying on the value of the previous
  962.        * read being used (and thus blocking on completion) before our
  963.        * write is executed.  This means we have to be careful in
  964.        * instruction scheduling to not violate this assumption.
  965.        */
  966.       write_commit = true;
  967.    }
  968.  
  969.    /* Each of the 8 channel enables is considered for whether each
  970.     * dword is written.
  971.     */
  972.    brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
  973.    brw_set_dest(p, send, dst);
  974.    brw_set_src0(p, send, header);
  975.    if (devinfo->gen < 6)
  976.       brw_inst_set_cond_modifier(p->devinfo, send, inst->base_mrf);
  977.    brw_set_dp_write_message(p, send,
  978.                             255, /* binding table index: stateless access */
  979.                             BRW_DATAPORT_OWORD_DUAL_BLOCK_1OWORD,
  980.                             msg_type,
  981.                             3, /* mlen */
  982.                             true, /* header present */
  983.                             false, /* not a render target write */
  984.                             write_commit, /* rlen */
  985.                             false, /* eot */
  986.                             write_commit);
  987. }
  988.  
  989. void
  990. vec4_generator::generate_pull_constant_load(vec4_instruction *inst,
  991.                                             struct brw_reg dst,
  992.                                             struct brw_reg index,
  993.                                             struct brw_reg offset)
  994. {
  995.    assert(index.file == BRW_IMMEDIATE_VALUE &&
  996.           index.type == BRW_REGISTER_TYPE_UD);
  997.    uint32_t surf_index = index.dw1.ud;
  998.  
  999.    struct brw_reg header = brw_vec8_grf(0, 0);
  1000.  
  1001.    gen6_resolve_implied_move(p, &header, inst->base_mrf);
  1002.  
  1003.    brw_MOV(p, retype(brw_message_reg(inst->base_mrf + 1), BRW_REGISTER_TYPE_D),
  1004.            offset);
  1005.  
  1006.    uint32_t msg_type;
  1007.  
  1008.    if (devinfo->gen >= 6)
  1009.       msg_type = GEN6_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ;
  1010.    else if (devinfo->gen == 5 || devinfo->is_g4x)
  1011.       msg_type = G45_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ;
  1012.    else
  1013.       msg_type = BRW_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ;
  1014.  
  1015.    /* Each of the 8 channel enables is considered for whether each
  1016.     * dword is written.
  1017.     */
  1018.    brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
  1019.    brw_set_dest(p, send, dst);
  1020.    brw_set_src0(p, send, header);
  1021.    if (devinfo->gen < 6)
  1022.       brw_inst_set_cond_modifier(p->devinfo, send, inst->base_mrf);
  1023.    brw_set_dp_read_message(p, send,
  1024.                            surf_index,
  1025.                            BRW_DATAPORT_OWORD_DUAL_BLOCK_1OWORD,
  1026.                            msg_type,
  1027.                            BRW_DATAPORT_READ_TARGET_DATA_CACHE,
  1028.                            2, /* mlen */
  1029.                            true, /* header_present */
  1030.                            1 /* rlen */);
  1031.  
  1032.    brw_mark_surface_used(&prog_data->base, surf_index);
  1033. }
  1034.  
  1035. void
  1036. vec4_generator::generate_pull_constant_load_gen7(vec4_instruction *inst,
  1037.                                                  struct brw_reg dst,
  1038.                                                  struct brw_reg surf_index,
  1039.                                                  struct brw_reg offset)
  1040. {
  1041.    assert(surf_index.type == BRW_REGISTER_TYPE_UD);
  1042.  
  1043.    if (surf_index.file == BRW_IMMEDIATE_VALUE) {
  1044.  
  1045.       brw_inst *insn = brw_next_insn(p, BRW_OPCODE_SEND);
  1046.       brw_set_dest(p, insn, dst);
  1047.       brw_set_src0(p, insn, offset);
  1048.       brw_set_sampler_message(p, insn,
  1049.                               surf_index.dw1.ud,
  1050.                               0, /* LD message ignores sampler unit */
  1051.                               GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
  1052.                               1, /* rlen */
  1053.                               inst->mlen,
  1054.                               inst->header_size != 0,
  1055.                               BRW_SAMPLER_SIMD_MODE_SIMD4X2,
  1056.                               0);
  1057.  
  1058.       brw_mark_surface_used(&prog_data->base, surf_index.dw1.ud);
  1059.  
  1060.    } else {
  1061.  
  1062.       struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
  1063.  
  1064.       brw_push_insn_state(p);
  1065.       brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  1066.       brw_set_default_access_mode(p, BRW_ALIGN_1);
  1067.  
  1068.       /* a0.0 = surf_index & 0xff */
  1069.       brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
  1070.       brw_inst_set_exec_size(p->devinfo, insn_and, BRW_EXECUTE_1);
  1071.       brw_set_dest(p, insn_and, addr);
  1072.       brw_set_src0(p, insn_and, vec1(retype(surf_index, BRW_REGISTER_TYPE_UD)));
  1073.       brw_set_src1(p, insn_and, brw_imm_ud(0x0ff));
  1074.  
  1075.       brw_pop_insn_state(p);
  1076.  
  1077.       /* dst = send(offset, a0.0 | <descriptor>) */
  1078.       brw_inst *insn = brw_send_indirect_message(
  1079.          p, BRW_SFID_SAMPLER, dst, offset, addr);
  1080.       brw_set_sampler_message(p, insn,
  1081.                               0 /* surface */,
  1082.                               0 /* sampler */,
  1083.                               GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
  1084.                               1 /* rlen */,
  1085.                               inst->mlen,
  1086.                               inst->header_size != 0,
  1087.                               BRW_SAMPLER_SIMD_MODE_SIMD4X2,
  1088.                               0);
  1089.  
  1090.       /* visitor knows more than we do about the surface limit required,
  1091.        * so has already done marking.
  1092.        */
  1093.    }
  1094. }
  1095.  
  1096. void
  1097. vec4_generator::generate_set_simd4x2_header_gen9(vec4_instruction *inst,
  1098.                                                  struct brw_reg dst)
  1099. {
  1100.    brw_push_insn_state(p);
  1101.    brw_set_default_mask_control(p, BRW_MASK_DISABLE);
  1102.  
  1103.    brw_set_default_exec_size(p, BRW_EXECUTE_8);
  1104.    brw_MOV(p, vec8(dst), retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
  1105.  
  1106.    brw_set_default_access_mode(p, BRW_ALIGN_1);
  1107.    brw_MOV(p, get_element_ud(dst, 2),
  1108.            brw_imm_ud(GEN9_SAMPLER_SIMD_MODE_EXTENSION_SIMD4X2));
  1109.  
  1110.    brw_pop_insn_state(p);
  1111. }
  1112.  
  1113. void
  1114. vec4_generator::generate_code(const cfg_t *cfg)
  1115. {
  1116.    struct annotation_info annotation;
  1117.    memset(&annotation, 0, sizeof(annotation));
  1118.    int loop_count = 0;
  1119.  
  1120.    foreach_block_and_inst (block, vec4_instruction, inst, cfg) {
  1121.       struct brw_reg src[3], dst;
  1122.  
  1123.       if (unlikely(debug_flag))
  1124.          annotate(p->devinfo, &annotation, cfg, inst, p->next_insn_offset);
  1125.  
  1126.       for (unsigned int i = 0; i < 3; i++) {
  1127.          src[i] = inst->get_src(this->prog_data, i);
  1128.       }
  1129.       dst = inst->get_dst();
  1130.  
  1131.       brw_set_default_predicate_control(p, inst->predicate);
  1132.       brw_set_default_predicate_inverse(p, inst->predicate_inverse);
  1133.       brw_set_default_flag_reg(p, 0, inst->flag_subreg);
  1134.       brw_set_default_saturate(p, inst->saturate);
  1135.       brw_set_default_mask_control(p, inst->force_writemask_all);
  1136.       brw_set_default_acc_write_control(p, inst->writes_accumulator);
  1137.  
  1138.       unsigned pre_emit_nr_insn = p->nr_insn;
  1139.  
  1140.       if (dst.width == BRW_WIDTH_4) {
  1141.          /* This happens in attribute fixups for "dual instanced" geometry
  1142.           * shaders, since they use attributes that are vec4's.  Since the exec
  1143.           * width is only 4, it's essential that the caller set
  1144.           * force_writemask_all in order to make sure the instruction is executed
  1145.           * regardless of which channels are enabled.
  1146.           */
  1147.          assert(inst->force_writemask_all);
  1148.  
  1149.          /* Fix up any <8;8,1> or <0;4,1> source registers to <4;4,1> to satisfy
  1150.           * the following register region restrictions (from Graphics BSpec:
  1151.           * 3D-Media-GPGPU Engine > EU Overview > Registers and Register Regions
  1152.           * > Register Region Restrictions)
  1153.           *
  1154.           *     1. ExecSize must be greater than or equal to Width.
  1155.           *
  1156.           *     2. If ExecSize = Width and HorzStride != 0, VertStride must be set
  1157.           *        to Width * HorzStride."
  1158.           */
  1159.          for (int i = 0; i < 3; i++) {
  1160.             if (src[i].file == BRW_GENERAL_REGISTER_FILE)
  1161.                src[i] = stride(src[i], 4, 4, 1);
  1162.          }
  1163.       }
  1164.  
  1165.       switch (inst->opcode) {
  1166.       case VEC4_OPCODE_UNPACK_UNIFORM:
  1167.       case BRW_OPCODE_MOV:
  1168.          brw_MOV(p, dst, src[0]);
  1169.          break;
  1170.       case BRW_OPCODE_ADD:
  1171.          brw_ADD(p, dst, src[0], src[1]);
  1172.          break;
  1173.       case BRW_OPCODE_MUL:
  1174.          brw_MUL(p, dst, src[0], src[1]);
  1175.          break;
  1176.       case BRW_OPCODE_MACH:
  1177.          brw_MACH(p, dst, src[0], src[1]);
  1178.          break;
  1179.  
  1180.       case BRW_OPCODE_MAD:
  1181.          assert(devinfo->gen >= 6);
  1182.          brw_MAD(p, dst, src[0], src[1], src[2]);
  1183.          break;
  1184.  
  1185.       case BRW_OPCODE_FRC:
  1186.          brw_FRC(p, dst, src[0]);
  1187.          break;
  1188.       case BRW_OPCODE_RNDD:
  1189.          brw_RNDD(p, dst, src[0]);
  1190.          break;
  1191.       case BRW_OPCODE_RNDE:
  1192.          brw_RNDE(p, dst, src[0]);
  1193.          break;
  1194.       case BRW_OPCODE_RNDZ:
  1195.          brw_RNDZ(p, dst, src[0]);
  1196.          break;
  1197.  
  1198.       case BRW_OPCODE_AND:
  1199.          brw_AND(p, dst, src[0], src[1]);
  1200.          break;
  1201.       case BRW_OPCODE_OR:
  1202.          brw_OR(p, dst, src[0], src[1]);
  1203.          break;
  1204.       case BRW_OPCODE_XOR:
  1205.          brw_XOR(p, dst, src[0], src[1]);
  1206.          break;
  1207.       case BRW_OPCODE_NOT:
  1208.          brw_NOT(p, dst, src[0]);
  1209.          break;
  1210.       case BRW_OPCODE_ASR:
  1211.          brw_ASR(p, dst, src[0], src[1]);
  1212.          break;
  1213.       case BRW_OPCODE_SHR:
  1214.          brw_SHR(p, dst, src[0], src[1]);
  1215.          break;
  1216.       case BRW_OPCODE_SHL:
  1217.          brw_SHL(p, dst, src[0], src[1]);
  1218.          break;
  1219.  
  1220.       case BRW_OPCODE_CMP:
  1221.          brw_CMP(p, dst, inst->conditional_mod, src[0], src[1]);
  1222.          break;
  1223.       case BRW_OPCODE_SEL:
  1224.          brw_SEL(p, dst, src[0], src[1]);
  1225.          break;
  1226.  
  1227.       case BRW_OPCODE_DPH:
  1228.          brw_DPH(p, dst, src[0], src[1]);
  1229.          break;
  1230.  
  1231.       case BRW_OPCODE_DP4:
  1232.          brw_DP4(p, dst, src[0], src[1]);
  1233.          break;
  1234.  
  1235.       case BRW_OPCODE_DP3:
  1236.          brw_DP3(p, dst, src[0], src[1]);
  1237.          break;
  1238.  
  1239.       case BRW_OPCODE_DP2:
  1240.          brw_DP2(p, dst, src[0], src[1]);
  1241.          break;
  1242.  
  1243.       case BRW_OPCODE_F32TO16:
  1244.          assert(devinfo->gen >= 7);
  1245.          brw_F32TO16(p, dst, src[0]);
  1246.          break;
  1247.  
  1248.       case BRW_OPCODE_F16TO32:
  1249.          assert(devinfo->gen >= 7);
  1250.          brw_F16TO32(p, dst, src[0]);
  1251.          break;
  1252.  
  1253.       case BRW_OPCODE_LRP:
  1254.          assert(devinfo->gen >= 6);
  1255.          brw_LRP(p, dst, src[0], src[1], src[2]);
  1256.          break;
  1257.  
  1258.       case BRW_OPCODE_BFREV:
  1259.          assert(devinfo->gen >= 7);
  1260.          /* BFREV only supports UD type for src and dst. */
  1261.          brw_BFREV(p, retype(dst, BRW_REGISTER_TYPE_UD),
  1262.                    retype(src[0], BRW_REGISTER_TYPE_UD));
  1263.          break;
  1264.       case BRW_OPCODE_FBH:
  1265.          assert(devinfo->gen >= 7);
  1266.          /* FBH only supports UD type for dst. */
  1267.          brw_FBH(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
  1268.          break;
  1269.       case BRW_OPCODE_FBL:
  1270.          assert(devinfo->gen >= 7);
  1271.          /* FBL only supports UD type for dst. */
  1272.          brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
  1273.          break;
  1274.       case BRW_OPCODE_CBIT:
  1275.          assert(devinfo->gen >= 7);
  1276.          /* CBIT only supports UD type for dst. */
  1277.          brw_CBIT(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
  1278.          break;
  1279.       case BRW_OPCODE_ADDC:
  1280.          assert(devinfo->gen >= 7);
  1281.          brw_ADDC(p, dst, src[0], src[1]);
  1282.          break;
  1283.       case BRW_OPCODE_SUBB:
  1284.          assert(devinfo->gen >= 7);
  1285.          brw_SUBB(p, dst, src[0], src[1]);
  1286.          break;
  1287.       case BRW_OPCODE_MAC:
  1288.          brw_MAC(p, dst, src[0], src[1]);
  1289.          break;
  1290.  
  1291.       case BRW_OPCODE_BFE:
  1292.          assert(devinfo->gen >= 7);
  1293.          brw_BFE(p, dst, src[0], src[1], src[2]);
  1294.          break;
  1295.  
  1296.       case BRW_OPCODE_BFI1:
  1297.          assert(devinfo->gen >= 7);
  1298.          brw_BFI1(p, dst, src[0], src[1]);
  1299.          break;
  1300.       case BRW_OPCODE_BFI2:
  1301.          assert(devinfo->gen >= 7);
  1302.          brw_BFI2(p, dst, src[0], src[1], src[2]);
  1303.          break;
  1304.  
  1305.       case BRW_OPCODE_IF:
  1306.          if (inst->src[0].file != BAD_FILE) {
  1307.             /* The instruction has an embedded compare (only allowed on gen6) */
  1308.             assert(devinfo->gen == 6);
  1309.             gen6_IF(p, inst->conditional_mod, src[0], src[1]);
  1310.          } else {
  1311.             brw_inst *if_inst = brw_IF(p, BRW_EXECUTE_8);
  1312.             brw_inst_set_pred_control(p->devinfo, if_inst, inst->predicate);
  1313.          }
  1314.          break;
  1315.  
  1316.       case BRW_OPCODE_ELSE:
  1317.          brw_ELSE(p);
  1318.          break;
  1319.       case BRW_OPCODE_ENDIF:
  1320.          brw_ENDIF(p);
  1321.          break;
  1322.  
  1323.       case BRW_OPCODE_DO:
  1324.          brw_DO(p, BRW_EXECUTE_8);
  1325.          break;
  1326.  
  1327.       case BRW_OPCODE_BREAK:
  1328.          brw_BREAK(p);
  1329.          brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
  1330.          break;
  1331.       case BRW_OPCODE_CONTINUE:
  1332.          brw_CONT(p);
  1333.          brw_set_default_predicate_control(p, BRW_PREDICATE_NONE);
  1334.          break;
  1335.  
  1336.       case BRW_OPCODE_WHILE:
  1337.          brw_WHILE(p);
  1338.          loop_count++;
  1339.          break;
  1340.  
  1341.       case SHADER_OPCODE_RCP:
  1342.       case SHADER_OPCODE_RSQ:
  1343.       case SHADER_OPCODE_SQRT:
  1344.       case SHADER_OPCODE_EXP2:
  1345.       case SHADER_OPCODE_LOG2:
  1346.       case SHADER_OPCODE_SIN:
  1347.       case SHADER_OPCODE_COS:
  1348.          assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
  1349.          if (devinfo->gen >= 7) {
  1350.             gen6_math(p, dst, brw_math_function(inst->opcode), src[0],
  1351.                       brw_null_reg());
  1352.          } else if (devinfo->gen == 6) {
  1353.             generate_math_gen6(inst, dst, src[0], brw_null_reg());
  1354.          } else {
  1355.             generate_math1_gen4(inst, dst, src[0]);
  1356.          }
  1357.          break;
  1358.  
  1359.       case SHADER_OPCODE_POW:
  1360.       case SHADER_OPCODE_INT_QUOTIENT:
  1361.       case SHADER_OPCODE_INT_REMAINDER:
  1362.          assert(inst->conditional_mod == BRW_CONDITIONAL_NONE);
  1363.          if (devinfo->gen >= 7) {
  1364.             gen6_math(p, dst, brw_math_function(inst->opcode), src[0], src[1]);
  1365.          } else if (devinfo->gen == 6) {
  1366.             generate_math_gen6(inst, dst, src[0], src[1]);
  1367.          } else {
  1368.             generate_math2_gen4(inst, dst, src[0], src[1]);
  1369.          }
  1370.          break;
  1371.  
  1372.       case SHADER_OPCODE_TEX:
  1373.       case SHADER_OPCODE_TXD:
  1374.       case SHADER_OPCODE_TXF:
  1375.       case SHADER_OPCODE_TXF_CMS:
  1376.       case SHADER_OPCODE_TXF_MCS:
  1377.       case SHADER_OPCODE_TXL:
  1378.       case SHADER_OPCODE_TXS:
  1379.       case SHADER_OPCODE_TG4:
  1380.       case SHADER_OPCODE_TG4_OFFSET:
  1381.          generate_tex(inst, dst, src[0], src[1]);
  1382.          break;
  1383.  
  1384.       case VS_OPCODE_URB_WRITE:
  1385.          generate_vs_urb_write(inst);
  1386.          break;
  1387.  
  1388.       case SHADER_OPCODE_GEN4_SCRATCH_READ:
  1389.          generate_scratch_read(inst, dst, src[0]);
  1390.          break;
  1391.  
  1392.       case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
  1393.          generate_scratch_write(inst, dst, src[0], src[1]);
  1394.          break;
  1395.  
  1396.       case VS_OPCODE_PULL_CONSTANT_LOAD:
  1397.          generate_pull_constant_load(inst, dst, src[0], src[1]);
  1398.          break;
  1399.  
  1400.       case VS_OPCODE_PULL_CONSTANT_LOAD_GEN7:
  1401.          generate_pull_constant_load_gen7(inst, dst, src[0], src[1]);
  1402.          break;
  1403.  
  1404.       case VS_OPCODE_SET_SIMD4X2_HEADER_GEN9:
  1405.          generate_set_simd4x2_header_gen9(inst, dst);
  1406.          break;
  1407.  
  1408.       case GS_OPCODE_URB_WRITE:
  1409.          generate_gs_urb_write(inst);
  1410.          break;
  1411.  
  1412.       case GS_OPCODE_URB_WRITE_ALLOCATE:
  1413.          generate_gs_urb_write_allocate(inst);
  1414.          break;
  1415.  
  1416.       case GS_OPCODE_SVB_WRITE:
  1417.          generate_gs_svb_write(inst, dst, src[0], src[1]);
  1418.          break;
  1419.  
  1420.       case GS_OPCODE_SVB_SET_DST_INDEX:
  1421.          generate_gs_svb_set_destination_index(inst, dst, src[0]);
  1422.          break;
  1423.  
  1424.       case GS_OPCODE_THREAD_END:
  1425.          generate_gs_thread_end(inst);
  1426.          break;
  1427.  
  1428.       case GS_OPCODE_SET_WRITE_OFFSET:
  1429.          generate_gs_set_write_offset(dst, src[0], src[1]);
  1430.          break;
  1431.  
  1432.       case GS_OPCODE_SET_VERTEX_COUNT:
  1433.          generate_gs_set_vertex_count(dst, src[0]);
  1434.          break;
  1435.  
  1436.       case GS_OPCODE_FF_SYNC:
  1437.          generate_gs_ff_sync(inst, dst, src[0], src[1]);
  1438.          break;
  1439.  
  1440.       case GS_OPCODE_FF_SYNC_SET_PRIMITIVES:
  1441.          generate_gs_ff_sync_set_primitives(dst, src[0], src[1], src[2]);
  1442.          break;
  1443.  
  1444.       case GS_OPCODE_SET_PRIMITIVE_ID:
  1445.          generate_gs_set_primitive_id(dst);
  1446.          break;
  1447.  
  1448.       case GS_OPCODE_SET_DWORD_2:
  1449.          generate_gs_set_dword_2(dst, src[0]);
  1450.          break;
  1451.  
  1452.       case GS_OPCODE_PREPARE_CHANNEL_MASKS:
  1453.          generate_gs_prepare_channel_masks(dst);
  1454.          break;
  1455.  
  1456.       case GS_OPCODE_SET_CHANNEL_MASKS:
  1457.          generate_gs_set_channel_masks(dst, src[0]);
  1458.          break;
  1459.  
  1460.       case GS_OPCODE_GET_INSTANCE_ID:
  1461.          generate_gs_get_instance_id(dst);
  1462.          break;
  1463.  
  1464.       case SHADER_OPCODE_SHADER_TIME_ADD:
  1465.          brw_shader_time_add(p, src[0],
  1466.                              prog_data->base.binding_table.shader_time_start);
  1467.          brw_mark_surface_used(&prog_data->base,
  1468.                                prog_data->base.binding_table.shader_time_start);
  1469.          break;
  1470.  
  1471.       case SHADER_OPCODE_UNTYPED_ATOMIC:
  1472.          assert(src[1].file == BRW_IMMEDIATE_VALUE &&
  1473.                 src[2].file == BRW_IMMEDIATE_VALUE);
  1474.          brw_untyped_atomic(p, dst, src[0], src[1], src[2].dw1.ud, inst->mlen,
  1475.                             !inst->dst.is_null());
  1476.          brw_mark_surface_used(&prog_data->base, src[1].dw1.ud);
  1477.          break;
  1478.  
  1479.       case SHADER_OPCODE_UNTYPED_SURFACE_READ:
  1480.          assert(src[1].file == BRW_IMMEDIATE_VALUE &&
  1481.                 src[2].file == BRW_IMMEDIATE_VALUE);
  1482.          brw_untyped_surface_read(p, dst, src[0], src[1], inst->mlen,
  1483.                                   src[2].dw1.ud);
  1484.          brw_mark_surface_used(&prog_data->base, src[1].dw1.ud);
  1485.          break;
  1486.  
  1487.       case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
  1488.          assert(src[2].file == BRW_IMMEDIATE_VALUE);
  1489.          brw_untyped_surface_write(p, src[0], src[1], inst->mlen,
  1490.                                    src[2].dw1.ud);
  1491.          break;
  1492.  
  1493.       case SHADER_OPCODE_TYPED_ATOMIC:
  1494.          assert(src[2].file == BRW_IMMEDIATE_VALUE);
  1495.          brw_typed_atomic(p, dst, src[0], src[1], src[2].dw1.ud, inst->mlen,
  1496.                           !inst->dst.is_null());
  1497.          break;
  1498.  
  1499.       case SHADER_OPCODE_TYPED_SURFACE_READ:
  1500.          assert(src[2].file == BRW_IMMEDIATE_VALUE);
  1501.          brw_typed_surface_read(p, dst, src[0], src[1], inst->mlen,
  1502.                                 src[2].dw1.ud);
  1503.          break;
  1504.  
  1505.       case SHADER_OPCODE_TYPED_SURFACE_WRITE:
  1506.          assert(src[2].file == BRW_IMMEDIATE_VALUE);
  1507.          brw_typed_surface_write(p, src[0], src[1], inst->mlen,
  1508.                                  src[2].dw1.ud);
  1509.          break;
  1510.  
  1511.       case SHADER_OPCODE_MEMORY_FENCE:
  1512.          brw_memory_fence(p, dst);
  1513.          break;
  1514.  
  1515.       case SHADER_OPCODE_FIND_LIVE_CHANNEL:
  1516.          brw_find_live_channel(p, dst);
  1517.          break;
  1518.  
  1519.       case SHADER_OPCODE_BROADCAST:
  1520.          brw_broadcast(p, dst, src[0], src[1]);
  1521.          break;
  1522.  
  1523.       case VS_OPCODE_UNPACK_FLAGS_SIMD4X2:
  1524.          generate_unpack_flags(dst);
  1525.          break;
  1526.  
  1527.       case VEC4_OPCODE_MOV_BYTES: {
  1528.          /* Moves the low byte from each channel, using an Align1 access mode
  1529.           * and a <4,1,0> source region.
  1530.           */
  1531.          assert(src[0].type == BRW_REGISTER_TYPE_UB ||
  1532.                 src[0].type == BRW_REGISTER_TYPE_B);
  1533.  
  1534.          brw_set_default_access_mode(p, BRW_ALIGN_1);
  1535.          src[0].vstride = BRW_VERTICAL_STRIDE_4;
  1536.          src[0].width = BRW_WIDTH_1;
  1537.          src[0].hstride = BRW_HORIZONTAL_STRIDE_0;
  1538.          brw_MOV(p, dst, src[0]);
  1539.          brw_set_default_access_mode(p, BRW_ALIGN_16);
  1540.          break;
  1541.       }
  1542.  
  1543.       case VEC4_OPCODE_PACK_BYTES: {
  1544.          /* Is effectively:
  1545.           *
  1546.           *   mov(8) dst<16,4,1>:UB src<4,1,0>:UB
  1547.           *
  1548.           * but destinations' only regioning is horizontal stride, so instead we
  1549.           * have to use two instructions:
  1550.           *
  1551.           *   mov(4) dst<1>:UB     src<4,1,0>:UB
  1552.           *   mov(4) dst.16<1>:UB  src.16<4,1,0>:UB
  1553.           *
  1554.           * where they pack the four bytes from the low and high four DW.
  1555.           */
  1556.          assert(is_power_of_two(dst.dw1.bits.writemask) &&
  1557.                 dst.dw1.bits.writemask != 0);
  1558.          unsigned offset = __builtin_ctz(dst.dw1.bits.writemask);
  1559.  
  1560.          dst.type = BRW_REGISTER_TYPE_UB;
  1561.  
  1562.          brw_set_default_access_mode(p, BRW_ALIGN_1);
  1563.  
  1564.          src[0].type = BRW_REGISTER_TYPE_UB;
  1565.          src[0].vstride = BRW_VERTICAL_STRIDE_4;
  1566.          src[0].width = BRW_WIDTH_1;
  1567.          src[0].hstride = BRW_HORIZONTAL_STRIDE_0;
  1568.          dst.subnr = offset * 4;
  1569.          struct brw_inst *insn = brw_MOV(p, dst, src[0]);
  1570.          brw_inst_set_exec_size(p->devinfo, insn, BRW_EXECUTE_4);
  1571.          brw_inst_set_no_dd_clear(p->devinfo, insn, true);
  1572.          brw_inst_set_no_dd_check(p->devinfo, insn, inst->no_dd_check);
  1573.  
  1574.          src[0].subnr = 16;
  1575.          dst.subnr = 16 + offset * 4;
  1576.          insn = brw_MOV(p, dst, src[0]);
  1577.          brw_inst_set_exec_size(p->devinfo, insn, BRW_EXECUTE_4);
  1578.          brw_inst_set_no_dd_clear(p->devinfo, insn, inst->no_dd_clear);
  1579.          brw_inst_set_no_dd_check(p->devinfo, insn, true);
  1580.  
  1581.          brw_set_default_access_mode(p, BRW_ALIGN_16);
  1582.          break;
  1583.       }
  1584.  
  1585.       default:
  1586.          unreachable("Unsupported opcode");
  1587.       }
  1588.  
  1589.       if (inst->opcode == VEC4_OPCODE_PACK_BYTES) {
  1590.          /* Handled dependency hints in the generator. */
  1591.  
  1592.          assert(!inst->conditional_mod);
  1593.       } else if (inst->no_dd_clear || inst->no_dd_check || inst->conditional_mod) {
  1594.          assert(p->nr_insn == pre_emit_nr_insn + 1 ||
  1595.                 !"conditional_mod, no_dd_check, or no_dd_clear set for IR "
  1596.                  "emitting more than 1 instruction");
  1597.  
  1598.          brw_inst *last = &p->store[pre_emit_nr_insn];
  1599.  
  1600.          if (inst->conditional_mod)
  1601.             brw_inst_set_cond_modifier(p->devinfo, last, inst->conditional_mod);
  1602.          brw_inst_set_no_dd_clear(p->devinfo, last, inst->no_dd_clear);
  1603.          brw_inst_set_no_dd_check(p->devinfo, last, inst->no_dd_check);
  1604.       }
  1605.    }
  1606.  
  1607.    brw_set_uip_jip(p);
  1608.    annotation_finalize(&annotation, p->next_insn_offset);
  1609.  
  1610.    int before_size = p->next_insn_offset;
  1611.    brw_compact_instructions(p, 0, annotation.ann_count, annotation.ann);
  1612.    int after_size = p->next_insn_offset;
  1613.  
  1614.    if (unlikely(debug_flag)) {
  1615.       if (shader_prog) {
  1616.          fprintf(stderr, "Native code for %s %s shader %d:\n",
  1617.                  shader_prog->Label ? shader_prog->Label : "unnamed",
  1618.                  stage_name, shader_prog->Name);
  1619.       } else {
  1620.          fprintf(stderr, "Native code for %s program %d:\n", stage_name,
  1621.                  prog->Id);
  1622.       }
  1623.       fprintf(stderr, "%s vec4 shader: %d instructions. %d loops. Compacted %d to %d"
  1624.                       " bytes (%.0f%%)\n",
  1625.               stage_abbrev,
  1626.               before_size / 16, loop_count, before_size, after_size,
  1627.               100.0f * (before_size - after_size) / before_size);
  1628.  
  1629.       dump_assembly(p->store, annotation.ann_count, annotation.ann,
  1630.                     p->devinfo, prog);
  1631.       ralloc_free(annotation.ann);
  1632.    }
  1633.  
  1634.    static GLuint msg_id = 0;
  1635.    _mesa_gl_debug(&brw->ctx, &msg_id,
  1636.                   MESA_DEBUG_SOURCE_SHADER_COMPILER,
  1637.                   MESA_DEBUG_TYPE_OTHER,
  1638.                   MESA_DEBUG_SEVERITY_NOTIFICATION,
  1639.                   "%s vec4 shader: %d inst, %d loops, "
  1640.                   "compacted %d to %d bytes.\n",
  1641.                   stage_abbrev,
  1642.                   before_size / 16, loop_count,
  1643.                   before_size, after_size);
  1644. }
  1645.  
  1646. const unsigned *
  1647. vec4_generator::generate_assembly(const cfg_t *cfg,
  1648.                                   unsigned *assembly_size)
  1649. {
  1650.    brw_set_default_access_mode(p, BRW_ALIGN_16);
  1651.    generate_code(cfg);
  1652.  
  1653.    return brw_get_program(p, assembly_size);
  1654. }
  1655.  
  1656. } /* namespace brw */
  1657.