Subversion Repositories Kolibri OS

/* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */

/*

 * Copyright (C) 2013 Rob Clark 

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the "Software"),

 * to deal in the Software without restriction, including without limitation

 * the rights to use, copy, modify, merge, publish, distribute, sublicense,

 * and/or sell copies of the Software, and to permit persons to whom the

 * Software is furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice (including the next

 * paragraph) shall be included in all copies or substantial portions of the

 * Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

 * SOFTWARE.

 *

 * Authors:

 *    Rob Clark 

 */

#include 

#include "pipe/p_state.h"

#include "util/u_string.h"

#include "util/u_memory.h"

#include "util/u_inlines.h"

#include "tgsi/tgsi_lowering.h"

#include "tgsi/tgsi_parse.h"

#include "tgsi/tgsi_ureg.h"

#include "tgsi/tgsi_info.h"

#include "tgsi/tgsi_strings.h"

#include "tgsi/tgsi_dump.h"

#include "tgsi/tgsi_scan.h"

#include "freedreno_util.h"

#include "ir3_compiler.h"

#include "ir3_shader.h"

#include "instr-a3xx.h"

#include "ir3.h"

struct ir3_compile_context {

	const struct tgsi_token *tokens;

	bool free_tokens;

	struct ir3 *ir;

	struct ir3_shader_variant *so;

	uint16_t integer_s;

	struct ir3_block *block;

	struct ir3_instruction *current_instr;

	/* we need to defer updates to block->outputs[] until the end

	 * of an instruction (so we don't see new value until *after*

	 * the src registers are processed)

	 */

	struct {

		struct ir3_instruction *instr, **instrp;

	} output_updates[64];

	unsigned num_output_updates;

	/* are we in a sequence of "atomic" instructions?

	 */

	bool atomic;

	/* For fragment shaders, from the hw perspective the only

	 * actual input is r0.xy position register passed to bary.f.

	 * But TGSI doesn't know that, it still declares things as

	 * IN[] registers.  So we do all the input tracking normally

	 * and fix things up after compile_instructions()

	 *

	 * NOTE that frag_pos is the hardware position (possibly it

	 * is actually an index or tag or some such.. it is *not*

	 * values that can be directly used for gl_FragCoord..)

	 */

	struct ir3_instruction *frag_pos, *frag_face, *frag_coord[4];

	/* For vertex shaders, keep track of the system values sources */

	struct ir3_instruction *vertex_id, *basevertex, *instance_id;

	struct tgsi_parse_context parser;

	unsigned type;

	struct tgsi_shader_info info;

	/* hmm, would be nice if tgsi_scan_shader figured this out

	 * for us:

	 */

	struct {

		unsigned first, last;

		struct ir3_instruction *fanin;

	} array[MAX_ARRAYS];

	uint32_t array_dirty;

	/* offset into array[], per file, of first array info */

	uint8_t array_offsets[TGSI_FILE_COUNT];

	/* for calculating input/output positions/linkages: */

	unsigned next_inloc;

	/* a4xx (at least patchlevel 0) cannot seem to flat-interpolate

	 * so we need to use ldlv.u32 to load the varying directly:

	 */

	bool flat_bypass;

	unsigned num_internal_temps;

	struct tgsi_src_register internal_temps[8];

	/* for looking up which system value is which */

	unsigned sysval_semantics[8];

	/* idx/slot for last compiler generated immediate */

	unsigned immediate_idx;

	/* stack of branch instructions that mark (potentially nested)

	 * branch if/else/loop/etc

	 */

	struct {

		struct ir3_instruction *instr, *cond;

		bool inv;   /* true iff in else leg of branch */

	} branch[16];

	unsigned int branch_count;

	/* list of kill instructions: */

	struct ir3_instruction *kill[16];

	unsigned int kill_count;

	/* used when dst is same as one of the src, to avoid overwriting a

	 * src element before the remaining scalar instructions that make

	 * up the vector operation

	 */

	struct tgsi_dst_register tmp_dst;

	struct tgsi_src_register *tmp_src;

	/* just for catching incorrect use of get_dst()/put_dst():

	 */

	bool using_tmp_dst;

};

static void vectorize(struct ir3_compile_context *ctx,

		struct ir3_instruction *instr, struct tgsi_dst_register *dst,

		int nsrcs, ...);

static void create_mov(struct ir3_compile_context *ctx,

		struct tgsi_dst_register *dst, struct tgsi_src_register *src);

static type_t get_ftype(struct ir3_compile_context *ctx);

static type_t get_utype(struct ir3_compile_context *ctx);

static unsigned setup_arrays(struct ir3_compile_context *ctx, unsigned file, unsigned i)

{

	/* ArrayID 0 for a given file is the legacy array spanning the entire file: */

	ctx->array[i].first = 0;

	ctx->array[i].last = ctx->info.file_max[file];

	ctx->array_offsets[file] = i;

	i += ctx->info.array_max[file] + 1;

	return i;

}

static unsigned

compile_init(struct ir3_compile_context *ctx, struct ir3_shader_variant *so,

		const struct tgsi_token *tokens)

{

	unsigned ret, i;

	struct tgsi_shader_info *info = &ctx->info;

	struct tgsi_lowering_config lconfig = {

			.color_two_side = so->key.color_two_side,

			.lower_DST  = true,

			.lower_XPD  = true,

			.lower_SCS  = true,

			.lower_LRP  = true,

			.lower_FRC  = true,

			.lower_POW  = true,

			.lower_LIT  = true,

			.lower_EXP  = true,

			.lower_LOG  = true,

			.lower_DP4  = true,

			.lower_DP3  = true,

			.lower_DPH  = true,

			.lower_DP2  = true,

			.lower_DP2A = true,

	};

	switch (so->type) {

	case SHADER_FRAGMENT:

	case SHADER_COMPUTE:

		lconfig.saturate_s = so->key.fsaturate_s;

		lconfig.saturate_t = so->key.fsaturate_t;

		lconfig.saturate_r = so->key.fsaturate_r;

		ctx->integer_s = so->key.finteger_s;

		break;

	case SHADER_VERTEX:

		lconfig.saturate_s = so->key.vsaturate_s;

		lconfig.saturate_t = so->key.vsaturate_t;

		lconfig.saturate_r = so->key.vsaturate_r;

		ctx->integer_s = so->key.vinteger_s;

		break;

	}

	if (!so->shader) {

		/* hack for standalone compiler which does not have

		 * screen/context:

		 */

	} else if (ir3_shader_gpuid(so->shader) >= 400) {

		/* a4xx seems to have *no* sam.p */

		lconfig.lower_TXP = ~0;  /* lower all txp */

		/* need special handling for "flat" */

		ctx->flat_bypass = true;

	} else {

		/* a3xx just needs to avoid sam.p for 3d tex */

		lconfig.lower_TXP = (1 << TGSI_TEXTURE_3D);

		/* no special handling for "flat" */

		ctx->flat_bypass = false;

	}

	ctx->tokens = tgsi_transform_lowering(&lconfig, tokens, &ctx->info);

	ctx->free_tokens = !!ctx->tokens;

	if (!ctx->tokens) {

		/* no lowering */

		ctx->tokens = tokens;

	}

	ctx->ir = so->ir;

	ctx->so = so;

	ctx->array_dirty = 0;

	ctx->next_inloc = 8;

	ctx->num_internal_temps = 0;

	ctx->branch_count = 0;

	ctx->kill_count = 0;

	ctx->block = NULL;

	ctx->current_instr = NULL;

	ctx->num_output_updates = 0;

	ctx->atomic = false;

	ctx->frag_pos = NULL;

	ctx->frag_face = NULL;

	ctx->vertex_id = NULL;

	ctx->instance_id = NULL;

	ctx->tmp_src = NULL;

	ctx->using_tmp_dst = false;

	memset(ctx->frag_coord, 0, sizeof(ctx->frag_coord));

	memset(ctx->array, 0, sizeof(ctx->array));

	memset(ctx->array_offsets, 0, sizeof(ctx->array_offsets));

#define FM(x) (1 << TGSI_FILE_##x)

	/* NOTE: if relative addressing is used, we set constlen in

	 * the compiler (to worst-case value) since we don't know in

	 * the assembler what the max addr reg value can be:

	 */

	if (info->indirect_files & FM(CONSTANT))

		so->constlen = MIN2(255, ctx->info.const_file_max[0] + 1);

	i = 0;

	i += setup_arrays(ctx, TGSI_FILE_INPUT, i);

	i += setup_arrays(ctx, TGSI_FILE_TEMPORARY, i);

	i += setup_arrays(ctx, TGSI_FILE_OUTPUT, i);

	/* any others? we don't track arrays for const..*/

	/* Immediates go after constants: */

	so->first_immediate = so->first_driver_param =

		info->const_file_max[0] + 1;

	/* 1 unit for the vertex id base */

	if (so->type == SHADER_VERTEX)

		so->first_immediate++;

	/* 4 (vec4) units for ubo base addresses */

	so->first_immediate += 4;

	ctx->immediate_idx = 4 * (ctx->info.file_max[TGSI_FILE_IMMEDIATE] + 1);

	ret = tgsi_parse_init(&ctx->parser, ctx->tokens);

	if (ret != TGSI_PARSE_OK)

		return ret;

	ctx->type = ctx->parser.FullHeader.Processor.Processor;

	return ret;

}

static void

compile_error(struct ir3_compile_context *ctx, const char *format, ...)

{

	va_list ap;

	va_start(ap, format);

	_debug_vprintf(format, ap);

	va_end(ap);

	tgsi_dump(ctx->tokens, 0);

	debug_assert(0);

}

#define compile_assert(ctx, cond) do { \

		if (!(cond)) compile_error((ctx), "failed assert: "#cond"\n"); \

	} while (0)

static void

compile_free(struct ir3_compile_context *ctx)

{

	if (ctx->free_tokens)

		free((void *)ctx->tokens);

	tgsi_parse_free(&ctx->parser);

}

struct instr_translater {

	void (*fxn)(const struct instr_translater *t,

			struct ir3_compile_context *ctx,

			struct tgsi_full_instruction *inst);

	unsigned tgsi_opc;

	opc_t opc;

	opc_t hopc;    /* opc to use for half_precision mode, if different */

	unsigned arg;

};

static void

instr_finish(struct ir3_compile_context *ctx)

{

	unsigned i;

	if (ctx->atomic)

		return;

	for (i = 0; i < ctx->num_output_updates; i++)

		*(ctx->output_updates[i].instrp) = ctx->output_updates[i].instr;

	ctx->num_output_updates = 0;

	while (ctx->array_dirty) {

		unsigned aid = ffs(ctx->array_dirty) - 1;

		ctx->array[aid].fanin = NULL;

		ctx->array_dirty &= ~(1 << aid);

	}

}

/* For "atomic" groups of instructions, for example the four scalar

 * instructions to perform a vec4 operation.  Basically this just

 * blocks out handling of output_updates so the next scalar instruction

 * still sees the result from before the start of the atomic group.

 *

 * NOTE: when used properly, this could probably replace get/put_dst()

 * stuff.

 */

static void

instr_atomic_start(struct ir3_compile_context *ctx)

{

	ctx->atomic = true;

}

static void

instr_atomic_end(struct ir3_compile_context *ctx)

{

	ctx->atomic = false;

	instr_finish(ctx);

}

static struct ir3_instruction *

instr_create(struct ir3_compile_context *ctx, int category, opc_t opc)

{

	instr_finish(ctx);

	return (ctx->current_instr = ir3_instr_create(ctx->block, category, opc));

}

static struct ir3_block *

push_block(struct ir3_compile_context *ctx)

{

	struct ir3_block *block;

	unsigned ntmp, nin, nout;

#define SCALAR_REGS(file) (4 * (ctx->info.file_max[TGSI_FILE_ ## file] + 1))

	/* hmm, give ourselves room to create 8 extra temporaries (vec4):

	 */

	ntmp = SCALAR_REGS(TEMPORARY);

	ntmp += 8 * 4;

	nout = SCALAR_REGS(OUTPUT);

	nin  = SCALAR_REGS(INPUT) + SCALAR_REGS(SYSTEM_VALUE);

	/* for outermost block, 'inputs' are the actual shader INPUT

	 * register file.  Reads from INPUT registers always go back to

	 * top block.  For nested blocks, 'inputs' is used to track any

	 * TEMPORARY file register from one of the enclosing blocks that

	 * is ready in this block.

	 */

	if (!ctx->block) {

		/* NOTE: fragment shaders actually have two inputs (r0.xy, the

		 * position)

		 */

		if (ctx->type == TGSI_PROCESSOR_FRAGMENT) {

			int n = 2;

			if (ctx->info.reads_position)

				n += 4;

			if (ctx->info.uses_frontface)

				n += 4;

			nin = MAX2(n, nin);

			nout += ARRAY_SIZE(ctx->kill);

		}

	} else {

		nin = ntmp;

	}

	block = ir3_block_create(ctx->ir, ntmp, nin, nout);

	if ((ctx->type == TGSI_PROCESSOR_FRAGMENT) && !ctx->block)

		block->noutputs -= ARRAY_SIZE(ctx->kill);

	block->parent = ctx->block;

	ctx->block = block;

	return block;

}

static void

pop_block(struct ir3_compile_context *ctx)

{

	ctx->block = ctx->block->parent;

	compile_assert(ctx, ctx->block);

}

static struct ir3_instruction *

create_output(struct ir3_block *block, struct ir3_instruction *instr,

		unsigned n)

{

	struct ir3_instruction *out;

	out = ir3_instr_create(block, -1, OPC_META_OUTPUT);

	out->inout.block = block;

	ir3_reg_create(out, n, 0);

	if (instr)

		ir3_reg_create(out, 0, IR3_REG_SSA)->instr = instr;

	return out;

}

static struct ir3_instruction *

create_input(struct ir3_block *block, struct ir3_instruction *instr,

		unsigned n)

{

	struct ir3_instruction *in;

	in = ir3_instr_create(block, -1, OPC_META_INPUT);

	in->inout.block = block;

	ir3_reg_create(in, n, 0);

	if (instr)

		ir3_reg_create(in, 0, IR3_REG_SSA)->instr = instr;

	return in;

}

static struct ir3_instruction *

block_input(struct ir3_block *block, unsigned n)

{

	/* references to INPUT register file always go back up to

	 * top level:

	 */

	if (block->parent)

		return block_input(block->parent, n);

	return block->inputs[n];

}

/* return temporary in scope, creating if needed meta-input node

 * to track block inputs

 */

static struct ir3_instruction *

block_temporary(struct ir3_block *block, unsigned n)

{

	/* references to TEMPORARY register file, find the nearest

	 * enclosing block which has already assigned this temporary,

	 * creating meta-input instructions along the way to keep

	 * track of block inputs

	 */

	if (block->parent && !block->temporaries[n]) {

		/* if already have input for this block, reuse: */

		if (!block->inputs[n])

			block->inputs[n] = block_temporary(block->parent, n);

		/* and create new input to return: */

		return create_input(block, block->inputs[n], n);

	}

	return block->temporaries[n];

}

static struct ir3_instruction *

create_immed(struct ir3_compile_context *ctx, float val)

{

	/* NOTE: *don't* use instr_create() here!

	 */

	struct ir3_instruction *instr;

	instr = ir3_instr_create(ctx->block, 1, 0);

	instr->cat1.src_type = get_ftype(ctx);

	instr->cat1.dst_type = get_ftype(ctx);

	ir3_reg_create(instr, 0, 0);

	ir3_reg_create(instr, 0, IR3_REG_IMMED)->fim_val = val;

	return instr;

}

static void

ssa_instr_set(struct ir3_compile_context *ctx, unsigned file, unsigned n,

		struct ir3_instruction *instr)

{

	struct ir3_block *block = ctx->block;

	unsigned idx = ctx->num_output_updates;

	compile_assert(ctx, idx < ARRAY_SIZE(ctx->output_updates));

	/* NOTE: defer update of temporaries[idx] or output[idx]

	 * until instr_finish(), so that if the current instruction

	 * reads the same TEMP/OUT[] it gets the old value:

	 *

	 * bleh.. this might be a bit easier to just figure out

	 * in instr_finish().  But at that point we've already

	 * lost information about OUTPUT vs TEMPORARY register

	 * file..

	 */

	switch (file) {

	case TGSI_FILE_OUTPUT:

		compile_assert(ctx, n < block->noutputs);

		ctx->output_updates[idx].instrp = &block->outputs[n];

		ctx->output_updates[idx].instr = instr;

		ctx->num_output_updates++;

		break;

	case TGSI_FILE_TEMPORARY:

		compile_assert(ctx, n < block->ntemporaries);

		ctx->output_updates[idx].instrp = &block->temporaries[n];

		ctx->output_updates[idx].instr = instr;

		ctx->num_output_updates++;

		break;

	case TGSI_FILE_ADDRESS:

		compile_assert(ctx, n < 1);

		ctx->output_updates[idx].instrp = &block->address;

		ctx->output_updates[idx].instr = instr;

		ctx->num_output_updates++;

		break;

	}

}

static struct ir3_instruction *

ssa_instr_get(struct ir3_compile_context *ctx, unsigned file, unsigned n)

{

	struct ir3_block *block = ctx->block;

	struct ir3_instruction *instr = NULL;

	switch (file) {

	case TGSI_FILE_INPUT:

		instr = block_input(ctx->block, n);

		break;

	case TGSI_FILE_OUTPUT:

		/* really this should just happen in case of 'MOV_SAT OUT[n], ..',

		 * for the following clamp instructions:

		 */

		instr = block->outputs[n];

		/* we don't have to worry about read from an OUTPUT that was

		 * assigned outside of the current block, because the _SAT

		 * clamp instructions will always be in the same block as

		 * the original instruction which wrote the OUTPUT

		 */

		compile_assert(ctx, instr);

		break;

	case TGSI_FILE_TEMPORARY:

		instr = block_temporary(ctx->block, n);

		if (!instr) {

			/* this can happen when registers (or components of a TGSI

			 * register) are used as src before they have been assigned

			 * (undefined contents).  To avoid confusing the rest of the

			 * compiler, and to generally keep things peachy, substitute

			 * an instruction that sets the src to 0.0.  Or to keep

			 * things undefined, I could plug in a random number? :-P

			 *

			 * NOTE: *don't* use instr_create() here!

			 */

			instr = create_immed(ctx, 0.0);

			/* no need to recreate the immed for every access: */

			block->temporaries[n] = instr;

		}

		break;

	case TGSI_FILE_SYSTEM_VALUE:

		switch (ctx->sysval_semantics[n >> 2]) {

		case TGSI_SEMANTIC_VERTEXID_NOBASE:

			instr = ctx->vertex_id;

			break;

		case TGSI_SEMANTIC_BASEVERTEX:

			instr = ctx->basevertex;

			break;

		case TGSI_SEMANTIC_INSTANCEID:

			instr = ctx->instance_id;

			break;

		}

		break;

	}

	return instr;

}

static int dst_array_id(struct ir3_compile_context *ctx,

		const struct tgsi_dst_register *dst)

{

	// XXX complete hack to recover tgsi_full_dst_register...

	// nothing that isn't wrapped in a tgsi_full_dst_register

	// should be indirect

	const struct tgsi_full_dst_register *fdst = (const void *)dst;

	return fdst->Indirect.ArrayID + ctx->array_offsets[dst->File];

}

static int src_array_id(struct ir3_compile_context *ctx,

		const struct tgsi_src_register *src)

{

	// XXX complete hack to recover tgsi_full_src_register...

	// nothing that isn't wrapped in a tgsi_full_src_register

	// should be indirect

	const struct tgsi_full_src_register *fsrc = (const void *)src;

	debug_assert(src->File != TGSI_FILE_CONSTANT);

	return fsrc->Indirect.ArrayID + ctx->array_offsets[src->File];

}

static struct ir3_instruction *

array_fanin(struct ir3_compile_context *ctx, unsigned aid, unsigned file)

{

	struct ir3_instruction *instr;

	if (ctx->array[aid].fanin) {

		instr = ctx->array[aid].fanin;

	} else {

		unsigned first = ctx->array[aid].first;

		unsigned last  = ctx->array[aid].last;

		unsigned i, j;

		instr = ir3_instr_create2(ctx->block, -1, OPC_META_FI,

				1 + (4 * (last + 1 - first)));

		ir3_reg_create(instr, 0, 0);

		for (i = first; i <= last; i++) {

			for (j = 0; j < 4; j++) {

				unsigned n = regid(i, j);

				ir3_reg_create(instr, 0, IR3_REG_SSA)->instr =

						ssa_instr_get(ctx, file, n);

			}

		}

		ctx->array[aid].fanin = instr;

		ctx->array_dirty |= (1 << aid);

	}

	return instr;

}

static void

ssa_dst(struct ir3_compile_context *ctx, struct ir3_instruction *instr,

		const struct tgsi_dst_register *dst, unsigned chan)

{

	if (dst->Indirect) {

		struct ir3_register *reg = instr->regs[0];

		unsigned i, aid = dst_array_id(ctx, dst);

		unsigned first = ctx->array[aid].first;

		unsigned last  = ctx->array[aid].last;

		unsigned off   = dst->Index - first; /* vec4 offset */

		reg->size = 4 * (1 + last - first);

		reg->offset = regid(off, chan);

		instr->fanin = array_fanin(ctx, aid, dst->File);

		/* annotate with the array-id, to help out the register-

		 * assignment stage.  At least for the case of indirect

		 * writes, we should capture enough dependencies to

		 * preserve the order of reads/writes of the array, so

		 * the multiple "names" for the array should end up all

		 * assigned to the same registers.

		 */

		instr->fanin->fi.aid = aid;

		/* Since we are scalarizing vec4 tgsi instructions/regs, we

		 * run into a slight complication here.  To do the naive thing

		 * and setup a fanout for each scalar array element would end

		 * up with the result that the instructions generated for each

		 * component of the vec4 would end up clobbering each other.

		 * So we take advantage here of knowing that the array index

		 * (after the shl.b) will be a multiple of four, and only set

		 * every fourth scalar component in the array.  See also

		 * fixup_ssa_dst_array()

		 */

		for (i = first; i <= last; i++) {

			struct ir3_instruction *split;

			unsigned n = regid(i, chan);

			int off = (4 * (i - first)) + chan;

			if (is_meta(instr) && (instr->opc == OPC_META_FO))

				off -= instr->fo.off;

			split = ir3_instr_create(ctx->block, -1, OPC_META_FO);

			split->fo.off = off;

			ir3_reg_create(split, 0, 0);

			ir3_reg_create(split, 0, IR3_REG_SSA)->instr = instr;

			ssa_instr_set(ctx, dst->File, n, split);

		}

	} else {

		/* normal case (not relative addressed GPR) */

		ssa_instr_set(ctx, dst->File, regid(dst->Index, chan), instr);

	}

}

static void

ssa_src(struct ir3_compile_context *ctx, struct ir3_register *reg,

		const struct tgsi_src_register *src, unsigned chan)

{

	struct ir3_instruction *instr;

	if (src->Indirect && (src->File != TGSI_FILE_CONSTANT)) {

		/* for relative addressing of gpr's (due to register assignment)

		 * we must generate a fanin instruction to collect all possible

		 * array elements that the instruction could address together:

		 */

		unsigned aid   = src_array_id(ctx, src);

		unsigned first = ctx->array[aid].first;

		unsigned last  = ctx->array[aid].last;

		unsigned off   = src->Index - first; /* vec4 offset */

		reg->size = 4 * (1 + last - first);

		reg->offset = regid(off, chan);

		instr = array_fanin(ctx, aid, src->File);

	} else if (src->File == TGSI_FILE_CONSTANT && src->Dimension) {

		const struct tgsi_full_src_register *fsrc = (const void *)src;

		struct ir3_instruction *temp = NULL;

		int ubo_regid = regid(ctx->so->first_driver_param, 0) +

			fsrc->Dimension.Index - 1;

		int offset = 0;

		/* We don't handle indirect UBO array accesses... yet. */

		compile_assert(ctx, !fsrc->Dimension.Indirect);

		/* UBOs start at index 1. */

		compile_assert(ctx, fsrc->Dimension.Index > 0);

		if (src->Indirect) {

			/* In case of an indirect index, it will have been loaded into an

			 * address register. There will be a sequence of

			 *

			 *   shl.b x, val, 2

			 *   mova a0, x

			 *

			 * We rely on this sequence to get the original val out and shift

			 * it by 4, since we're dealing in vec4 units.

			 */

			compile_assert(ctx, ctx->block->address);

			compile_assert(ctx, ctx->block->address->regs[1]->instr->opc ==

						   OPC_SHL_B);

			temp = instr = instr_create(ctx, 2, OPC_SHL_B);

			ir3_reg_create(instr, 0, 0);

			ir3_reg_create(instr, 0, IR3_REG_HALF | IR3_REG_SSA)->instr =

				ctx->block->address->regs[1]->instr->regs[1]->instr;

			ir3_reg_create(instr, 0, IR3_REG_IMMED)->iim_val = 4;

		} else if (src->Index >= 64) {

			/* Otherwise it's a plain index (in vec4 units). Move it into a

			 * register.

			 */

			temp = instr = instr_create(ctx, 1, 0);

			instr->cat1.src_type = get_utype(ctx);

			instr->cat1.dst_type = get_utype(ctx);

			ir3_reg_create(instr, 0, 0);

			ir3_reg_create(instr, 0, IR3_REG_IMMED)->iim_val = src->Index * 16;

		} else {

			/* The offset is small enough to fit into the ldg instruction

			 * directly.

			 */

			offset = src->Index * 16;

		}

		if (temp) {

			/* If there was an offset (most common), add it to the buffer

			 * address.

			 */

			instr = instr_create(ctx, 2, OPC_ADD_S);

			ir3_reg_create(instr, 0, 0);

			ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = temp;

			ir3_reg_create(instr, ubo_regid, IR3_REG_CONST);

		} else {

			/* Otherwise just load the buffer address directly */

			instr = instr_create(ctx, 1, 0);

			instr->cat1.src_type = get_utype(ctx);

			instr->cat1.dst_type = get_utype(ctx);

			ir3_reg_create(instr, 0, 0);

			ir3_reg_create(instr, ubo_regid, IR3_REG_CONST);

		}

		temp = instr;

		instr = instr_create(ctx, 6, OPC_LDG);

		instr->cat6.type = TYPE_U32;

		instr->cat6.offset = offset + chan * 4;

		ir3_reg_create(instr, 0, 0);

		ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = temp;

		ir3_reg_create(instr, 0, IR3_REG_IMMED)->iim_val = 1;

		reg->flags &= ~(IR3_REG_RELATIV | IR3_REG_CONST);

	} else {

		/* normal case (not relative addressed GPR) */

		instr = ssa_instr_get(ctx, src->File, regid(src->Index, chan));

	}

	if (instr) {

		reg->flags |= IR3_REG_SSA;

		reg->instr = instr;

	} else if (reg->flags & IR3_REG_SSA) {

		/* special hack for trans_samp() which calls ssa_src() directly

		 * to build up the collect (fanin) for const src.. (so SSA flag

		 * set but no src instr... it basically gets lucky because we

		 * default to 0.0 for "undefined" src instructions, which is

		 * what it wants.  We probably need to give it a better way to

		 * do this, but for now this hack:

		 */

		reg->instr = create_immed(ctx, 0.0);

	}

}

static struct ir3_register *

add_dst_reg_wrmask(struct ir3_compile_context *ctx,

		struct ir3_instruction *instr, const struct tgsi_dst_register *dst,

		unsigned chan, unsigned wrmask)

{

	unsigned flags = 0, num = 0;

	struct ir3_register *reg;

	switch (dst->File) {

	case TGSI_FILE_OUTPUT:

	case TGSI_FILE_TEMPORARY:

		/* uses SSA */

		break;

	case TGSI_FILE_ADDRESS:

		flags |= IR3_REG_ADDR;

		/* uses SSA */

		break;

	default:

		compile_error(ctx, "unsupported dst register file: %s\n",

			tgsi_file_name(dst->File));

		break;

	}

	if (dst->Indirect) {

		flags |= IR3_REG_RELATIV;

		/* shouldn't happen, and we can't cope with it below: */

		compile_assert(ctx, wrmask == 0x1);

		compile_assert(ctx, ctx->block->address);

		if (instr->address)

			compile_assert(ctx, ctx->block->address == instr->address);

		instr->address = ctx->block->address;

		array_insert(ctx->ir->indirects, instr);

	}

	reg = ir3_reg_create(instr, regid(num, chan), flags);

	reg->wrmask = wrmask;

	if (wrmask == 0x1) {

		/* normal case */

		ssa_dst(ctx, instr, dst, chan);

	} else if ((dst->File == TGSI_FILE_TEMPORARY) ||

			(dst->File == TGSI_FILE_OUTPUT) ||

			(dst->File == TGSI_FILE_ADDRESS)) {

		struct ir3_instruction *prev = NULL;

		unsigned i;

		compile_assert(ctx, !dst->Indirect);

		/* if instruction writes multiple, we need to create

		 * some place-holder collect the registers:

		 */

		for (i = 0; i < 4; i++) {

			/* NOTE: slightly ugly that we setup neighbor ptrs

			 * for FO here, but handle FI in CP pass.. we should

			 * probably just always setup neighbor ptrs in the

			 * frontend?

			 */

			struct ir3_instruction *split =

					ir3_instr_create(ctx->block, -1, OPC_META_FO);

			split->fo.off = i;

			/* unused dst reg: */

			/* NOTE: set SSA flag on dst here, because unused FO's

			 * which don't get scheduled will end up not in the

			 * instruction list when RA sets SSA flag on each dst.

			 * Slight hack.  We really should set SSA flag on

			 * every dst register in the frontend.

			 */

			ir3_reg_create(split, 0, IR3_REG_SSA);

			/* and src reg used to hold original instr */

			ir3_reg_create(split, 0, IR3_REG_SSA)->instr = instr;

			if (prev) {

				split->cp.left = prev;

				split->cp.left_cnt++;

				prev->cp.right = split;

				prev->cp.right_cnt++;

			}

			if ((wrmask & (1 << i)) && !ctx->atomic)

				ssa_dst(ctx, split, dst, chan+i);

			prev = split;

		}

	}

	return reg;

}

static struct ir3_register *

add_dst_reg(struct ir3_compile_context *ctx, struct ir3_instruction *instr,

		const struct tgsi_dst_register *dst, unsigned chan)

{

	return add_dst_reg_wrmask(ctx, instr, dst, chan, 0x1);

}

static struct ir3_register *

add_src_reg_wrmask(struct ir3_compile_context *ctx,

		struct ir3_instruction *instr, const struct tgsi_src_register *src,

		unsigned chan, unsigned wrmask)

{

	unsigned flags = 0, num = 0;

	struct ir3_register *reg;

	switch (src->File) {

	case TGSI_FILE_IMMEDIATE:

		/* TODO if possible, use actual immediate instead of const.. but

		 * TGSI has vec4 immediates, we can only embed scalar (of limited

		 * size, depending on instruction..)

		 */

		flags |= IR3_REG_CONST;

		num = src->Index + ctx->so->first_immediate;

		break;

	case TGSI_FILE_CONSTANT:

		flags |= IR3_REG_CONST;

		num = src->Index;

		break;

	case TGSI_FILE_OUTPUT:

		/* NOTE: we should only end up w/ OUTPUT file for things like

		 * clamp()'ing saturated dst instructions

		 */

	case TGSI_FILE_INPUT:

	case TGSI_FILE_TEMPORARY:

	case TGSI_FILE_SYSTEM_VALUE:

		/* uses SSA */

		break;

	default:

		compile_error(ctx, "unsupported src register file: %s\n",

			tgsi_file_name(src->File));

		break;

	}

	/* We seem to have 8 bits (6.2) for dst register always, so I think

	 * it is safe to assume GPR cannot be >=64

	 *

	 * cat3 instructions only have 8 bits for src2, but cannot take a

	 * const for src2

	 *

	 * cat5 and cat6 in some cases only has 8 bits, but cannot take a

	 * const for any src.

	 *

	 * Other than that we seem to have 12 bits to encode const src,

	 * except for cat1 which may only have 11 bits (but that seems like

	 * a bug)

	 */

	if (flags & IR3_REG_CONST)

		compile_assert(ctx, src->Index < (1 << 9));

	else

		compile_assert(ctx, src->Index < (1 << 6));

	/* NOTE: abs/neg modifiers in tgsi only apply to float */

	if (src->Absolute)

		flags |= IR3_REG_FABS;

	if (src->Negate)

		flags |= IR3_REG_FNEG;

	if (src->Indirect) {

		flags |= IR3_REG_RELATIV;

		/* shouldn't happen, and we can't cope with it below: */

		compile_assert(ctx, wrmask == 0x1);

		compile_assert(ctx, ctx->block->address);

		if (instr->address)

			compile_assert(ctx, ctx->block->address == instr->address);

		instr->address = ctx->block->address;

		array_insert(ctx->ir->indirects, instr);

	}

	reg = ir3_reg_create(instr, regid(num, chan), flags);

	reg->wrmask = wrmask;

	if (wrmask == 0x1) {

		/* normal case */

		ssa_src(ctx, reg, src, chan);

	} else if ((src->File == TGSI_FILE_TEMPORARY) ||

			(src->File == TGSI_FILE_OUTPUT) ||

			(src->File == TGSI_FILE_INPUT)) {

		struct ir3_instruction *collect;

		unsigned i;

		compile_assert(ctx, !src->Indirect);

		/* if instruction reads multiple, we need to create

		 * some place-holder collect the registers:

		 */

		collect = ir3_instr_create(ctx->block, -1, OPC_META_FI);

		ir3_reg_create(collect, 0, 0);   /* unused dst reg */

		for (i = 0; i < 4; i++) {

			if (wrmask & (1 << i)) {

				/* and src reg used point to the original instr */

				ssa_src(ctx, ir3_reg_create(collect, 0, IR3_REG_SSA),

						src, chan + i);

			} else if (wrmask & ~((i << i) - 1)) {

				/* if any remaining components, then dummy

				 * placeholder src reg to fill in the blanks:

				 */

				ir3_reg_create(collect, 0, 0);

			}

		}

		reg->flags |= IR3_REG_SSA;

		reg->instr = collect;

	}

	return reg;

}

static struct ir3_register *

add_src_reg(struct ir3_compile_context *ctx, struct ir3_instruction *instr,

		const struct tgsi_src_register *src, unsigned chan)

{

	return add_src_reg_wrmask(ctx, instr, src, chan, 0x1);

}

static void

src_from_dst(struct tgsi_src_register *src, struct tgsi_dst_register *dst)

{

	src->File      = dst->File;

	src->Indirect  = dst->Indirect;

	src->Dimension = dst->Dimension;

	src->Index     = dst->Index;

	src->Absolute  = 0;

	src->Negate    = 0;

	src->SwizzleX  = TGSI_SWIZZLE_X;

	src->SwizzleY  = TGSI_SWIZZLE_Y;

	src->SwizzleZ  = TGSI_SWIZZLE_Z;

	src->SwizzleW  = TGSI_SWIZZLE_W;

}

/* Get internal-temp src/dst to use for a sequence of instructions

 * generated by a single TGSI op.

 */

static struct tgsi_src_register *

get_internal_temp(struct ir3_compile_context *ctx,

		struct tgsi_dst_register *tmp_dst)

{

	struct tgsi_src_register *tmp_src;

	int n;

	tmp_dst->File      = TGSI_FILE_TEMPORARY;

	tmp_dst->WriteMask = TGSI_WRITEMASK_XYZW;

	tmp_dst->Indirect  = 0;

	tmp_dst->Dimension = 0;

	/* assign next temporary: */

	n = ctx->num_internal_temps++;

	compile_assert(ctx, n < ARRAY_SIZE(ctx->internal_temps));

	tmp_src = &ctx->internal_temps[n];

	tmp_dst->Index = ctx->info.file_max[TGSI_FILE_TEMPORARY] + n + 1;

	src_from_dst(tmp_src, tmp_dst);

	return tmp_src;

}

static inline bool

is_const(struct tgsi_src_register *src)

{

	return (src->File == TGSI_FILE_CONSTANT) ||

			(src->File == TGSI_FILE_IMMEDIATE);

}

static inline bool

is_relative(struct tgsi_src_register *src)

{

	return src->Indirect;

}

static inline bool

is_rel_or_const(struct tgsi_src_register *src)

{

	return is_relative(src) || is_const(src);

}

static type_t

get_ftype(struct ir3_compile_context *ctx)

{

	return TYPE_F32;

}

static type_t

get_utype(struct ir3_compile_context *ctx)

{

	return TYPE_U32;

}

static type_t

get_stype(struct ir3_compile_context *ctx)

{

	return TYPE_S32;

}

static unsigned

src_swiz(struct tgsi_src_register *src, int chan)

{

	switch (chan) {

	case 0: return src->SwizzleX;

	case 1: return src->SwizzleY;

	case 2: return src->SwizzleZ;

	case 3: return src->SwizzleW;

	}

	assert(0);

	return 0;

}

/* for instructions that cannot take a const register as src, if needed

 * generate a move to temporary gpr:

 */

static struct tgsi_src_register *

get_unconst(struct ir3_compile_context *ctx, struct tgsi_src_register *src)

{