Subversion Repositories Kolibri OS

/**************************************************************************

 *

 * Copyright 2009 VMware, Inc.

 * Copyright 2007-2008 VMware, Inc.

 * All Rights Reserved.

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the

 * "Software"), to deal in the Software without restriction, including

 * without limitation the rights to use, copy, modify, merge, publish,

 * distribute, sub license, and/or sell copies of the Software, and to

 * permit persons to whom the Software is furnished to do so, subject to

 * the following conditions:

 *

 * The above copyright notice and this permission notice (including the

 * next paragraph) shall be included in all copies or substantial portions

 * of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS

 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.

 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR

 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,

 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE

 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

 *

 **************************************************************************/

/**

 * @file

 * TGSI to LLVM IR translation -- SoA.

 *

 * @author Jose Fonseca 

 *

 * Based on tgsi_sse2.c code written by Michal Krol, Keith Whitwell,

 * Brian Paul, and others.

 */

#include "pipe/p_config.h"

#include "pipe/p_shader_tokens.h"

#include "util/u_debug.h"

#include "util/u_math.h"

#include "util/u_memory.h"

#include "tgsi/tgsi_dump.h"

#include "tgsi/tgsi_exec.h"

#include "tgsi/tgsi_info.h"

#include "tgsi/tgsi_parse.h"

#include "tgsi/tgsi_util.h"

#include "tgsi/tgsi_scan.h"

#include "tgsi/tgsi_strings.h"

#include "lp_bld_tgsi_action.h"

#include "lp_bld_type.h"

#include "lp_bld_const.h"

#include "lp_bld_arit.h"

#include "lp_bld_bitarit.h"

#include "lp_bld_gather.h"

#include "lp_bld_init.h"

#include "lp_bld_logic.h"

#include "lp_bld_swizzle.h"

#include "lp_bld_flow.h"

#include "lp_bld_quad.h"

#include "lp_bld_tgsi.h"

#include "lp_bld_limits.h"

#include "lp_bld_debug.h"

#include "lp_bld_printf.h"

#include "lp_bld_sample.h"

#include "lp_bld_struct.h"

/* SM 4.0 says that subroutines can nest 32 deep and

 * we need one more for our main function */

#define LP_MAX_NUM_FUNCS 33

#define DUMP_GS_EMITS 0

/*

 * If non-zero, the generated LLVM IR will print intermediate results on every TGSI

 * instruction.

 *

 * TODO:

 * - take execution masks in consideration

 * - debug control-flow instructions

 */

#define DEBUG_EXECUTION 0

/*

 * Emit code to print a register value.

 */

static void

emit_dump_reg(struct gallivm_state *gallivm,

              unsigned file,

              unsigned index,

              unsigned chan,

              LLVMValueRef value)

{

   char buf[32];

   util_snprintf(buf, sizeof buf, "    %s[%u].%c = ",

                 tgsi_file_name(file),

                 index, "xyzw"[chan]);

   lp_build_print_value(gallivm, buf, value);

}

/*

 * Return the context for the current function.

 * (always 'main', if shader doesn't do any function calls)

 */

static INLINE struct function_ctx *

func_ctx(struct lp_exec_mask *mask)

{

   assert(mask->function_stack_size > 0);

   assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);

   return &mask->function_stack[mask->function_stack_size - 1];

}

/*

 * Returns true if we're in a loop.

 * It's global, meaning that it returns true even if there's

 * no loop inside the current function, but we were inside

 * a loop inside another function, from which this one was called.

 */

static INLINE boolean

mask_has_loop(struct lp_exec_mask *mask)

{

   int i;

   for (i = mask->function_stack_size - 1; i >= 0; --i) {

      const struct function_ctx *ctx = &mask->function_stack[i];

      if (ctx->loop_stack_size > 0)

         return TRUE;

   }

   return FALSE;

}

/*

 * Returns true if we're inside a switch statement.

 * It's global, meaning that it returns true even if there's

 * no switch in the current function, but we were inside

 * a switch inside another function, from which this one was called.

 */

static INLINE boolean

mask_has_switch(struct lp_exec_mask *mask)

{

   int i;

   for (i = mask->function_stack_size - 1; i >= 0; --i) {

      const struct function_ctx *ctx = &mask->function_stack[i];

      if (ctx->switch_stack_size > 0)

         return TRUE;

   }

   return FALSE;

}

/*

 * Returns true if we're inside a conditional.

 * It's global, meaning that it returns true even if there's

 * no conditional in the current function, but we were inside

 * a conditional inside another function, from which this one was called.

 */

static INLINE boolean

mask_has_cond(struct lp_exec_mask *mask)

{

   int i;

   for (i = mask->function_stack_size - 1; i >= 0; --i) {

      const struct function_ctx *ctx = &mask->function_stack[i];

      if (ctx->cond_stack_size > 0)

         return TRUE;

   }

   return FALSE;

}

/*

 * Initialize a function context at the specified index.

 */

static void

lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)

{

   LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx =  &mask->function_stack[function_idx];

   ctx->cond_stack_size = 0;

   ctx->loop_stack_size = 0;

   ctx->switch_stack_size = 0;

   if (function_idx == 0) {

      ctx->ret_mask = mask->ret_mask;

   }

   ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,

                                       int_type, "looplimiter");

   LLVMBuildStore(

      builder,

      LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),

      ctx->loop_limiter);

}

static void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)

{

   mask->bld = bld;

   mask->has_mask = FALSE;

   mask->ret_in_main = FALSE;

   /* For the main function */

   mask->function_stack_size = 1;

   mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);

   mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =

         mask->cond_mask = mask->switch_mask =

         LLVMConstAllOnes(mask->int_vec_type);

   mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,

                                 sizeof(mask->function_stack[0]));

   lp_exec_mask_function_init(mask, 0);

}

static void

lp_exec_mask_fini(struct lp_exec_mask *mask)

{

   FREE(mask->function_stack);

}

static void lp_exec_mask_update(struct lp_exec_mask *mask)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   boolean has_loop_mask = mask_has_loop(mask);

   boolean has_cond_mask = mask_has_cond(mask);

   boolean has_switch_mask = mask_has_switch(mask);

   boolean has_ret_mask = mask->function_stack_size > 1 ||

         mask->ret_in_main;

   if (has_loop_mask) {

      /*for loops we need to update the entire mask at runtime */

      LLVMValueRef tmp;

      assert(mask->break_mask);

      tmp = LLVMBuildAnd(builder,

                         mask->cont_mask,

                         mask->break_mask,

                         "maskcb");

      mask->exec_mask = LLVMBuildAnd(builder,

                                     mask->cond_mask,

                                     tmp,

                                     "maskfull");

   } else

      mask->exec_mask = mask->cond_mask;

   if (has_switch_mask) {

      mask->exec_mask = LLVMBuildAnd(builder,

                                     mask->exec_mask,

                                     mask->switch_mask,

                                     "switchmask");

   }

   if (has_ret_mask) {

      mask->exec_mask = LLVMBuildAnd(builder,

                                     mask->exec_mask,

                                     mask->ret_mask,

                                     "callmask");

   }

   mask->has_mask = (has_cond_mask ||

                     has_loop_mask ||

                     has_switch_mask ||

                     has_ret_mask);

}

static void lp_exec_mask_cond_push(struct lp_exec_mask *mask,

                                   LLVMValueRef val)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {

      ctx->cond_stack_size++;

      return;

   }

   if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {

      assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));

   }

   ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;

   assert(LLVMTypeOf(val) == mask->int_vec_type);

   mask->cond_mask = LLVMBuildAnd(builder,

                                  mask->cond_mask,

                                  val,

                                  "");

   lp_exec_mask_update(mask);

}

static void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   LLVMValueRef prev_mask;

   LLVMValueRef inv_mask;

   assert(ctx->cond_stack_size);

   if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)

      return;

   prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];

   if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {

      assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));

   }

   inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");

   mask->cond_mask = LLVMBuildAnd(builder,

                                  inv_mask,

                                  prev_mask, "");

   lp_exec_mask_update(mask);

}

static void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)

{

   struct function_ctx *ctx = func_ctx(mask);

   assert(ctx->cond_stack_size);

   --ctx->cond_stack_size;

   if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)

      return;

   mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];

   lp_exec_mask_update(mask);

}

static void lp_exec_bgnloop(struct lp_exec_mask *mask)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {

      ++ctx->loop_stack_size;

      return;

   }

   ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =

      ctx->break_type;

   ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;

   ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;

   ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;

   ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;

   ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;

   ++ctx->loop_stack_size;

   ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");

   LLVMBuildStore(builder, mask->break_mask, ctx->break_var);

   ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");

   LLVMBuildBr(builder, ctx->loop_block);

   LLVMPositionBuilderAtEnd(builder, ctx->loop_block);

   mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");

   lp_exec_mask_update(mask);

}

static void lp_exec_break(struct lp_exec_mask *mask,

                          struct lp_build_tgsi_context * bld_base)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {

      LLVMValueRef exec_mask = LLVMBuildNot(builder,

                                            mask->exec_mask,

                                            "break");

      mask->break_mask = LLVMBuildAnd(builder,

                                      mask->break_mask,

                                      exec_mask, "break_full");

   }

   else {

      unsigned opcode = bld_base->instructions[bld_base->pc + 1].Instruction.Opcode;

      boolean break_always = (opcode == TGSI_OPCODE_ENDSWITCH ||

                              opcode == TGSI_OPCODE_CASE);

      if (ctx->switch_in_default) {

         /*

          * stop default execution but only if this is an unconditional switch.

          * (The condition here is not perfect since dead code after break is

          * allowed but should be sufficient since false negatives are just

          * unoptimized - so we don't have to pre-evaluate that).

          */

         if(break_always && ctx->switch_pc) {

            bld_base->pc = ctx->switch_pc;

            return;

         }

      }

      if (break_always) {

         mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);

      }

      else {

         LLVMValueRef exec_mask = LLVMBuildNot(builder,

                                               mask->exec_mask,

                                               "break");

         mask->switch_mask = LLVMBuildAnd(builder,

                                          mask->switch_mask,

                                          exec_mask, "break_switch");

      }

   }

   lp_exec_mask_update(mask);

}

static void lp_exec_break_condition(struct lp_exec_mask *mask,

                                    LLVMValueRef cond)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   LLVMValueRef cond_mask = LLVMBuildAnd(builder,

                                         mask->exec_mask,

                                         cond, "cond_mask");

   cond_mask = LLVMBuildNot(builder, cond_mask, "break_cond");

   if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {

      mask->break_mask = LLVMBuildAnd(builder,

                                      mask->break_mask,

                                      cond_mask, "breakc_full");

   }

   else {

      mask->switch_mask = LLVMBuildAnd(builder,

                                       mask->switch_mask,

                                       cond_mask, "breakc_switch");

   }

   lp_exec_mask_update(mask);

}

static void lp_exec_continue(struct lp_exec_mask *mask)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   LLVMValueRef exec_mask = LLVMBuildNot(builder,

                                         mask->exec_mask,

                                         "");

   mask->cont_mask = LLVMBuildAnd(builder,

                                  mask->cont_mask,

                                  exec_mask, "");

   lp_exec_mask_update(mask);

}

static void lp_exec_endloop(struct gallivm_state *gallivm,

                            struct lp_exec_mask *mask)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   LLVMBasicBlockRef endloop;

   LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);

   LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,

                                               mask->bld->type.width *

                                               mask->bld->type.length);

   LLVMValueRef i1cond, i2cond, icond, limiter;

   assert(mask->break_mask);

   assert(ctx->loop_stack_size);

   if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {

      --ctx->loop_stack_size;

      return;

   }

   /*

    * Restore the cont_mask, but don't pop

    */

   mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;

   lp_exec_mask_update(mask);

   /*

    * Unlike the continue mask, the break_mask must be preserved across loop

    * iterations

    */

   LLVMBuildStore(builder, mask->break_mask, ctx->break_var);

   /* Decrement the loop limiter */

   limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");

   limiter = LLVMBuildSub(

      builder,

      limiter,

      LLVMConstInt(int_type, 1, false),

      "");

   LLVMBuildStore(builder, limiter, ctx->loop_limiter);

   /* i1cond = (mask != 0) */

   i1cond = LLVMBuildICmp(

      builder,

      LLVMIntNE,

      LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),

      LLVMConstNull(reg_type), "i1cond");

   /* i2cond = (looplimiter > 0) */

   i2cond = LLVMBuildICmp(

      builder,

      LLVMIntSGT,

      limiter,

      LLVMConstNull(int_type), "i2cond");

   /* if( i1cond && i2cond ) */

   icond = LLVMBuildAnd(builder, i1cond, i2cond, "");

   endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");

   LLVMBuildCondBr(builder,

                   icond, ctx->loop_block, endloop);

   LLVMPositionBuilderAtEnd(builder, endloop);

   assert(ctx->loop_stack_size);

   --ctx->loop_stack_size;

   mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;

   mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;

   ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;

   ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;

   ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +

         ctx->switch_stack_size];

   lp_exec_mask_update(mask);

}

static void lp_exec_switch(struct lp_exec_mask *mask,

                           LLVMValueRef switchval)

{

   struct function_ctx *ctx = func_ctx(mask);

   if (ctx->switch_stack_size >= LP_MAX_TGSI_NESTING ||

       ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {

      ctx->switch_stack_size++;

      return;

   }

   ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =

      ctx->break_type;

   ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_SWITCH;

   ctx->switch_stack[ctx->switch_stack_size].switch_mask = mask->switch_mask;

   ctx->switch_stack[ctx->switch_stack_size].switch_val = ctx->switch_val;

   ctx->switch_stack[ctx->switch_stack_size].switch_mask_default = ctx->switch_mask_default;

   ctx->switch_stack[ctx->switch_stack_size].switch_in_default = ctx->switch_in_default;

   ctx->switch_stack[ctx->switch_stack_size].switch_pc = ctx->switch_pc;

   ctx->switch_stack_size++;

   mask->switch_mask = LLVMConstNull(mask->int_vec_type);

   ctx->switch_val = switchval;

   ctx->switch_mask_default = LLVMConstNull(mask->int_vec_type);

   ctx->switch_in_default = false;

   ctx->switch_pc = 0;

   lp_exec_mask_update(mask);

}

static void lp_exec_endswitch(struct lp_exec_mask *mask,

                              struct lp_build_tgsi_context * bld_base)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {

      ctx->switch_stack_size--;

      return;

   }

   /* check if there's deferred default if so do it now */

   if (ctx->switch_pc && !ctx->switch_in_default) {

      LLVMValueRef prevmask, defaultmask;

      unsigned tmp_pc;

      prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;

      defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");

      mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");

      ctx->switch_in_default = true;

      lp_exec_mask_update(mask);

      assert(bld_base->instructions[ctx->switch_pc - 1].Instruction.Opcode ==

             TGSI_OPCODE_DEFAULT);

      tmp_pc = bld_base->pc;

      bld_base->pc = ctx->switch_pc;

      /*

       * re-purpose switch_pc to point to here again, since we stop execution of

       * the deferred default after next break.

       */

      ctx->switch_pc = tmp_pc - 1;

      return;

   }

   else if (ctx->switch_pc && ctx->switch_in_default) {

      assert(bld_base->pc == ctx->switch_pc + 1);

   }

   ctx->switch_stack_size--;

   mask->switch_mask = ctx->switch_stack[ctx->switch_stack_size].switch_mask;

   ctx->switch_val = ctx->switch_stack[ctx->switch_stack_size].switch_val;

   ctx->switch_mask_default = ctx->switch_stack[ctx->switch_stack_size].switch_mask_default;

   ctx->switch_in_default = ctx->switch_stack[ctx->switch_stack_size].switch_in_default;

   ctx->switch_pc = ctx->switch_stack[ctx->switch_stack_size].switch_pc;

   ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size];

   lp_exec_mask_update(mask);

}

static void lp_exec_case(struct lp_exec_mask *mask,

                         LLVMValueRef caseval)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   LLVMValueRef casemask, prevmask;

   if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {

      return;

   }

   /* skipping case mask evaluation here is NOT optional (not in all cases anyway). */

   if (!ctx->switch_in_default) {

      prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;

      casemask = lp_build_cmp(mask->bld, PIPE_FUNC_EQUAL, caseval, ctx->switch_val);

      ctx->switch_mask_default = LLVMBuildOr(builder, casemask,

                                             ctx->switch_mask_default, "sw_default_mask");

      casemask = LLVMBuildOr(builder, casemask, mask->switch_mask, "");

      mask->switch_mask = LLVMBuildAnd(builder, casemask, prevmask, "sw_mask");

      lp_exec_mask_update(mask);

   }

}

/*

 * Analyse default statement in a switch.

 * \return true if default is last statement, false otherwise

 * \param default_pc_start contains pc of instruction to jump to

 *                         if default wasn't last but there's no

 *                         fallthrough into default.

 */

static boolean default_analyse_is_last(struct lp_exec_mask *mask,

                                       struct lp_build_tgsi_context * bld_base,

                                       int *default_pc_start)

{

   unsigned pc = bld_base->pc;

   struct function_ctx *ctx = func_ctx(mask);

   unsigned curr_switch_stack = ctx->switch_stack_size;

   if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {

      return false;

   }

   /* skip over case statements which are together with default */

   while (bld_base->instructions[pc].Instruction.Opcode == TGSI_OPCODE_CASE) {

      pc++;

   }

   while (pc != -1 && pc < bld_base->num_instructions) {

      unsigned opcode = bld_base->instructions[pc].Instruction.Opcode;

      switch (opcode) {

      case TGSI_OPCODE_CASE:

         if (curr_switch_stack == ctx->switch_stack_size) {

            *default_pc_start = pc - 1;

            return false;

         }

         break;

      case TGSI_OPCODE_SWITCH:

         curr_switch_stack++;

         break;

      case TGSI_OPCODE_ENDSWITCH:

         if (curr_switch_stack == ctx->switch_stack_size) {

            *default_pc_start = pc - 1;

            return true;

         }

         curr_switch_stack--;

         break;

      }

      pc++;

   }

   /* should never arrive here */

   assert(0);

   return true;

}

static void lp_exec_default(struct lp_exec_mask *mask,

                            struct lp_build_tgsi_context * bld_base)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   int default_exec_pc;

   boolean default_is_last;

   if (ctx->switch_stack_size > LP_MAX_TGSI_NESTING) {

      return;

   }

   /*

    * This is a messy opcode, because it may not be always at the end and

    * there can be fallthrough in and out of it.

    */

   default_is_last = default_analyse_is_last(mask, bld_base, &default_exec_pc);

   /*

    * If it is last statement in switch (note that case statements appearing

    * "at the same time" as default don't change that) everything is just fine,

    * update switch mask and go on. This means we can handle default with

    * fallthrough INTO it without overhead, if it is last.

    */

   if (default_is_last) {

      LLVMValueRef prevmask, defaultmask;

      prevmask = ctx->switch_stack[ctx->switch_stack_size - 1].switch_mask;

      defaultmask = LLVMBuildNot(builder, ctx->switch_mask_default, "sw_default_mask");

      defaultmask = LLVMBuildOr(builder, defaultmask, mask->switch_mask, "");

      mask->switch_mask = LLVMBuildAnd(builder, prevmask, defaultmask, "sw_mask");

      ctx->switch_in_default = true;

      lp_exec_mask_update(mask);

   }

   else {

      /*

       * Technically, "case" immediately before default isn't really a

       * fallthrough, however we still have to count them as such as we

       * already have updated the masks.

       * If that happens in practice could add a switch optimizer pass

       * which just gets rid of all case statements appearing together with

       * default (or could do switch analysis at switch start time instead).

       */

      unsigned opcode = bld_base->instructions[bld_base->pc - 1].Instruction.Opcode;

      boolean ft_into = (opcode != TGSI_OPCODE_BRK &&

                         opcode != TGSI_OPCODE_SWITCH);

      /*

       * If it is not last statement and there was no fallthrough into it,

       * we record the PC and continue execution at next case (again, those

       * case encountered at the same time don't count). At endswitch

       * time, we update switchmask, and go back executing the code we skipped

       * until the next break (possibly re-executing some code with changed mask

       * if there was a fallthrough out of default).

       * Finally, if it is not last statement and there was a fallthrough into it,

       * do the same as with the former case, except instead of skipping the code

       * just execute it without updating the mask, then go back and re-execute.

       */

      ctx->switch_pc = bld_base->pc;

      if (!ft_into) {

         bld_base->pc = default_exec_pc;

      }

   }

}

/* stores val into an address pointed to by dst_ptr.

 * mask->exec_mask is used to figure out which bits of val

 * should be stored into the address

 * (0 means don't store this bit, 1 means do store).

 */

static void lp_exec_mask_store(struct lp_exec_mask *mask,

                               struct lp_build_context *bld_store,

                               LLVMValueRef pred,

                               LLVMValueRef val,

                               LLVMValueRef dst_ptr)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   assert(lp_check_value(bld_store->type, val));

   assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);

   assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val));

   /* Mix the predicate and execution mask */

   if (mask->has_mask) {

      if (pred) {

         pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");

      } else {

         pred = mask->exec_mask;

      }

   }

   if (pred) {

      LLVMValueRef res, dst;

      dst = LLVMBuildLoad(builder, dst_ptr, "");

      res = lp_build_select(bld_store, pred, val, dst);

      LLVMBuildStore(builder, res, dst_ptr);

   } else

      LLVMBuildStore(builder, val, dst_ptr);

}

static void lp_exec_mask_call(struct lp_exec_mask *mask,

                              int func,

                              int *pc)

{

   if (mask->function_stack_size >= LP_MAX_NUM_FUNCS) {

      return;

   }

   lp_exec_mask_function_init(mask, mask->function_stack_size);

   mask->function_stack[mask->function_stack_size].pc = *pc;

   mask->function_stack[mask->function_stack_size].ret_mask = mask->ret_mask;

   mask->function_stack_size++;

   *pc = func;

}

static void lp_exec_mask_ret(struct lp_exec_mask *mask, int *pc)

{

   LLVMBuilderRef builder = mask->bld->gallivm->builder;

   struct function_ctx *ctx = func_ctx(mask);

   LLVMValueRef exec_mask;

   if (ctx->cond_stack_size == 0 &&

       ctx->loop_stack_size == 0 &&

       ctx->switch_stack_size == 0 &&

       mask->function_stack_size == 1) {

      /* returning from main() */

      *pc = -1;

      return;

   }

   if (mask->function_stack_size == 1) {

      /*

       * This requires special handling since we need to ensure

       * we don't drop the mask even if we have no call stack

       * (e.g. after a ret in a if clause after the endif)

       */

      mask->ret_in_main = TRUE;

   }

   exec_mask = LLVMBuildNot(builder,

                            mask->exec_mask,

                            "ret");

   mask->ret_mask = LLVMBuildAnd(builder,

                                 mask->ret_mask,

                                 exec_mask, "ret_full");

   lp_exec_mask_update(mask);

}

static void lp_exec_mask_bgnsub(struct lp_exec_mask *mask)

{

}

static void lp_exec_mask_endsub(struct lp_exec_mask *mask, int *pc)

{

   struct function_ctx *ctx;

   assert(mask->function_stack_size > 1);

   assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);

   ctx = func_ctx(mask);

   mask->function_stack_size--;

   *pc = ctx->pc;

   mask->ret_mask = ctx->ret_mask;

   lp_exec_mask_update(mask);

}

static LLVMValueRef

get_file_ptr(struct lp_build_tgsi_soa_context *bld,

             unsigned file,

             unsigned index,

             unsigned chan)

{

   LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;

   LLVMValueRef (*array_of_vars)[TGSI_NUM_CHANNELS];

   LLVMValueRef var_of_array;

   switch (file) {

   case TGSI_FILE_TEMPORARY:

      array_of_vars = bld->temps;

      var_of_array = bld->temps_array;

      break;

   case TGSI_FILE_OUTPUT:

      array_of_vars = bld->outputs;

      var_of_array = bld->outputs_array;

      break;

   default:

      assert(0);

      return NULL;

   }

   assert(chan < 4);

   if (bld->indirect_files & (1 << file)) {

      LLVMValueRef lindex = lp_build_const_int32(bld->bld_base.base.gallivm, index * 4 + chan);

      return LLVMBuildGEP(builder, var_of_array, &lindex, 1, "");

   }

   else {

      assert(index <= bld->bld_base.info->file_max[file]);

      return array_of_vars[index][chan];

   }

}

/**

 * Return pointer to a temporary register channel (src or dest).

 * Note that indirect addressing cannot be handled here.

 * \param index  which temporary register

 * \param chan  which channel of the temp register.

 */

LLVMValueRef

lp_get_temp_ptr_soa(struct lp_build_tgsi_soa_context *bld,

             unsigned index,

             unsigned chan)

{

   return get_file_ptr(bld, TGSI_FILE_TEMPORARY, index, chan);

}

/**

 * Return pointer to a output register channel (src or dest).

 * Note that indirect addressing cannot be handled here.

 * \param index  which output register

 * \param chan  which channel of the output register.

 */

LLVMValueRef

lp_get_output_ptr(struct lp_build_tgsi_soa_context *bld,

               unsigned index,

               unsigned chan)

{

   return get_file_ptr(bld, TGSI_FILE_OUTPUT, index, chan);

}

/*

 * If we have indirect addressing in outputs copy our alloca array

 * to the outputs slots specified by the caller to make sure

 * our outputs are delivered consistently via the same interface.

 */

static void

gather_outputs(struct lp_build_tgsi_soa_context * bld)

{

   if ((bld->indirect_files & (1 << TGSI_FILE_OUTPUT))) {

      unsigned index, chan;

      assert(bld->bld_base.info->num_outputs <=

             bld->bld_base.info->file_max[TGSI_FILE_OUTPUT] + 1);

      for (index = 0; index < bld->bld_base.info->num_outputs; ++index) {

         for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {

            bld->outputs[index][chan] = lp_get_output_ptr(bld, index, chan);

         }

      }

   }

}

/**

 * Gather vector.

 * XXX the lp_build_gather() function should be capable of doing this

 * with a little work.

 */

static LLVMValueRef

build_gather(struct lp_build_tgsi_context *bld_base,

             LLVMValueRef base_ptr,

             LLVMValueRef indexes,

             LLVMValueRef overflow_mask)

{

   struct gallivm_state *gallivm = bld_base->base.gallivm;

   LLVMBuilderRef builder = gallivm->builder;

   struct lp_build_context *uint_bld = &bld_base->uint_bld;

   struct lp_build_context *bld = &bld_base->base;

   LLVMValueRef res = bld->undef;

   unsigned i;

   /*

    * overflow_mask is a vector telling us which channels

    * in the vector overflowed. We use the overflow behavior for

    * constant buffers which is defined as:

    * Out of bounds access to constant buffer returns 0 in all

    * components. Out of bounds behavior is always with respect

    * to the size of the buffer bound at that slot.

    */

   if (overflow_mask) {

      /*

       * We avoid per-element control flow here (also due to llvm going crazy,

       * though I suspect it's better anyway since overflow is likely rare).

       * Note that since we still fetch from buffers even if num_elements was

       * zero (in this case we'll fetch from index zero) the jit func callers

       * MUST provide valid fake constant buffers of size 4x32 (the values do

       * not matter), otherwise we'd still need (not per element though)

       * control flow.

       */

      indexes = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes);

   }

   /*

    * Loop over elements of index_vec, load scalar value, insert it into 'res'.

    */

   for (i = 0; i < bld->type.length; i++) {

      LLVMValueRef ii = lp_build_const_int32(bld->gallivm, i);

      LLVMValueRef index = LLVMBuildExtractElement(builder,

                                                   indexes, ii, "");

      LLVMValueRef scalar_ptr, scalar;

      scalar_ptr = LLVMBuildGEP(builder, base_ptr,

                                &index, 1, "gather_ptr");

      scalar = LLVMBuildLoad(builder, scalar_ptr, "");

      res = LLVMBuildInsertElement(builder, res, scalar, ii, "");

   }

   if (overflow_mask) {

      res = lp_build_select(bld, overflow_mask, bld->zero, res);

   }

   return res;

}

/**

 * Scatter/store vector.

 */

static void

emit_mask_scatter(struct lp_build_tgsi_soa_context *bld,

                  LLVMValueRef base_ptr,

                  LLVMValueRef indexes,

                  LLVMValueRef values,

                  struct lp_exec_mask *mask,

                  LLVMValueRef pred)

{

   struct gallivm_state *gallivm = bld->bld_base.base.gallivm;

   LLVMBuilderRef builder = gallivm->builder;

   unsigned i;

   /* Mix the predicate and execution mask */

   if (mask->has_mask) {

      if (pred) {

         pred = LLVMBuildAnd(builder, pred, mask->exec_mask, "");

      }

      else {

         pred = mask->exec_mask;

      }

   }

   /*

    * Loop over elements of index_vec, store scalar value.

    */

   for (i = 0; i < bld->bld_base.base.type.length; i++) {

      LLVMValueRef ii = lp_build_const_int32(gallivm, i);

      LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");

      LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");

      LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");

      LLVMValueRef scalar_pred = pred ?

         LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;

      if (0)

         lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",

                         ii, val, index, scalar_ptr);

      if (scalar_pred) {

         LLVMValueRef real_val, dst_val;

         dst_val = LLVMBuildLoad(builder, scalar_ptr, "");

         real_val = lp_build_select(&bld->elem_bld, scalar_pred, val, dst_val);

         LLVMBuildStore(builder, real_val, scalar_ptr);

      }

      else {

         LLVMBuildStore(builder, val, scalar_ptr);

      }

   }

}

/**

 * Read the current value of the ADDR register, convert the floats to

 * ints, add the base index and return the vector of offsets.

 * The offsets will be used to index into the constant buffer or

 * temporary register file.

 */

static LLVMValueRef

get_indirect_index(struct lp_build_tgsi_soa_context *bld,

                   unsigned reg_file, unsigned reg_index,

                   const struct tgsi_ind_register *indirect_reg)

{

   LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;

   struct lp_build_context *uint_bld = &bld->bld_base.uint_bld;

   /* always use X component of address register */

   unsigned swizzle = indirect_reg->Swizzle;

   LLVMValueRef base;

   LLVMValueRef rel;

   LLVMValueRef max_index;

   LLVMValueRef index;

   assert(bld->indirect_files & (1 << reg_file));

   base = lp_build_const_int_vec(bld->bld_base.base.gallivm, uint_bld->type, reg_index);

   assert(swizzle < 4);

   switch (indirect_reg->File) {

   case TGSI_FILE_ADDRESS:

      rel = LLVMBuildLoad(builder,

                          bld->addr[indirect_reg->Index][swizzle],

                          "load addr reg");

      /* ADDR LLVM values already have LLVM integer type. */

      break;

   case TGSI_FILE_TEMPORARY:

      rel = lp_get_temp_ptr_soa(bld, indirect_reg->Index, swizzle);

      rel = LLVMBuildLoad(builder, rel, "load temp reg");

      /* TEMP LLVM values always have LLVM float type, but for indirection, the

       * value actually stored is expected to be an integer */

      rel = LLVMBuildBitCast(builder, rel, uint_bld->vec_type, "");

      break;

   default:

      assert(0);

      rel = uint_bld->zero;

   }

   index = lp_build_add(uint_bld, base, rel);

   /*

    * emit_fetch_constant handles constant buffer overflow so this code

    * is pointless for them.

    * Furthermore the D3D10 spec in section 6.5 says:

    * If the constant buffer bound to a slot is larger than the size

    * declared in the shader for that slot, implementations are allowed

    * to return incorrect data (not necessarily 0) for indices that are

    * larger than the declared size but smaller than the buffer size.

    */

   if (reg_file != TGSI_FILE_CONSTANT) {

      max_index = lp_build_const_int_vec(bld->bld_base.base.gallivm,

                                         uint_bld->type,

                                         bld->bld_base.info->file_max[reg_file]);

      assert(!uint_bld->type.sign);

      index = lp_build_min(uint_bld, index, max_index);

   }

   return index;

}

static struct lp_build_context *

stype_to_fetch(struct lp_build_tgsi_context * bld_base,

	       enum tgsi_opcode_type stype)

{

   struct lp_build_context *bld_fetch;