Subversion Repositories Kolibri OS

Compare Revisions

Regard whitespace Rev 4357 → Rev 4358

/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir.cpp
0,0 → 1,1231
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_target.h"
#include "nv50_ir_driver.h"
 
extern "C" {
#include "nv50/nv50_program.h"
#include "nv50/nv50_debug.h"
}
 
namespace nv50_ir {
 
Modifier::Modifier(operation op)
{
switch (op) {
case OP_NEG: bits = NV50_IR_MOD_NEG; break;
case OP_ABS: bits = NV50_IR_MOD_ABS; break;
case OP_SAT: bits = NV50_IR_MOD_SAT; break;
case OP_NOT: bits = NV50_IR_MOD_NOT; break;
default:
bits = 0;
break;
}
}
 
Modifier Modifier::operator*(const Modifier m) const
{
unsigned int a, b, c;
 
b = m.bits;
if (this->bits & NV50_IR_MOD_ABS)
b &= ~NV50_IR_MOD_NEG;
 
a = (this->bits ^ b) & (NV50_IR_MOD_NOT | NV50_IR_MOD_NEG);
c = (this->bits | m.bits) & (NV50_IR_MOD_ABS | NV50_IR_MOD_SAT);
 
return Modifier(a | c);
}
 
ValueRef::ValueRef(Value *v) : value(NULL), insn(NULL)
{
indirect[0] = -1;
indirect[1] = -1;
usedAsPtr = false;
set(v);
}
 
ValueRef::ValueRef(const ValueRef& ref) : value(NULL), insn(ref.insn)
{
set(ref);
usedAsPtr = ref.usedAsPtr;
}
 
ValueRef::~ValueRef()
{
this->set(NULL);
}
 
bool ValueRef::getImmediate(ImmediateValue &imm) const
{
const ValueRef *src = this;
Modifier m;
DataType type = src->insn->sType;
 
while (src) {
if (src->mod) {
if (src->insn->sType != type)
break;
m *= src->mod;
}
if (src->getFile() == FILE_IMMEDIATE) {
imm = *(src->value->asImm());
// The immediate's type isn't required to match its use, it's
// more of a hint; applying a modifier makes use of that hint.
imm.reg.type = type;
m.applyTo(imm);
return true;
}
 
Instruction *insn = src->value->getUniqueInsn();
 
if (insn && insn->op == OP_MOV) {
src = &insn->src(0);
if (src->mod)
WARN("OP_MOV with modifier encountered !\n");
} else {
src = NULL;
}
}
return false;
}
 
ValueDef::ValueDef(Value *v) : value(NULL), insn(NULL)
{
set(v);
}
 
ValueDef::ValueDef(const ValueDef& def) : value(NULL), insn(NULL)
{
set(def.get());
}
 
ValueDef::~ValueDef()
{
this->set(NULL);
}
 
void
ValueRef::set(const ValueRef &ref)
{
this->set(ref.get());
mod = ref.mod;
indirect[0] = ref.indirect[0];
indirect[1] = ref.indirect[1];
}
 
void
ValueRef::set(Value *refVal)
{
if (value == refVal)
return;
if (value)
value->uses.remove(this);
if (refVal)
refVal->uses.push_back(this);
 
value = refVal;
}
 
void
ValueDef::set(Value *defVal)
{
if (value == defVal)
return;
if (value)
value->defs.remove(this);
if (defVal)
defVal->defs.push_back(this);
 
value = defVal;
}
 
// Check if we can replace this definition's value by the value in @rep,
// including the source modifiers, i.e. make sure that all uses support
// @rep.mod.
bool
ValueDef::mayReplace(const ValueRef &rep)
{
if (!rep.mod)
return true;
 
if (!insn || !insn->bb) // Unbound instruction ?
return false;
 
const Target *target = insn->bb->getProgram()->getTarget();
 
for (Value::UseIterator it = value->uses.begin(); it != value->uses.end();
++it) {
Instruction *insn = (*it)->getInsn();
int s = -1;
 
for (int i = 0; insn->srcExists(i); ++i) {
if (insn->src(i).get() == value) {
// If there are multiple references to us we'd have to check if the
// combination of mods is still supported, but just bail for now.
if (&insn->src(i) != (*it))
return false;
s = i;
}
}
assert(s >= 0); // integrity of uses list
 
if (!target->isModSupported(insn, s, rep.mod))
return false;
}
return true;
}
 
void
ValueDef::replace(const ValueRef &repVal, bool doSet)
{
assert(mayReplace(repVal));
 
if (value == repVal.get())
return;
 
while (!value->uses.empty()) {
ValueRef *ref = value->uses.front();
ref->set(repVal.get());
ref->mod *= repVal.mod;
}
 
if (doSet)
set(repVal.get());
}
 
Value::Value()
{
join = this;
memset(&reg, 0, sizeof(reg));
reg.size = 4;
}
 
LValue::LValue(Function *fn, DataFile file)
{
reg.file = file;
reg.size = (file != FILE_PREDICATE) ? 4 : 1;
reg.data.id = -1;
 
compMask = 0;
compound = 0;
ssa = 0;
fixedReg = 0;
noSpill = 0;
 
fn->add(this, this->id);
}
 
LValue::LValue(Function *fn, LValue *lval)
{
assert(lval);
 
reg.file = lval->reg.file;
reg.size = lval->reg.size;
reg.data.id = -1;
 
compMask = 0;
compound = 0;
ssa = 0;
fixedReg = 0;
noSpill = 0;
 
fn->add(this, this->id);
}
 
LValue *
LValue::clone(ClonePolicy<Function>& pol) const
{
LValue *that = new_LValue(pol.context(), reg.file);
 
pol.set<Value>(this, that);
 
that->reg.size = this->reg.size;
that->reg.type = this->reg.type;
that->reg.data = this->reg.data;
 
return that;
}
 
bool
LValue::isUniform() const
{
if (defs.size() > 1)
return false;
Instruction *insn = getInsn();
// let's not try too hard here for now ...
return !insn->srcExists(1) && insn->getSrc(0)->isUniform();
}
 
Symbol::Symbol(Program *prog, DataFile f, ubyte fidx)
{
baseSym = NULL;
 
reg.file = f;
reg.fileIndex = fidx;
reg.data.offset = 0;
 
prog->add(this, this->id);
}
 
Symbol *
Symbol::clone(ClonePolicy<Function>& pol) const
{
Program *prog = pol.context()->getProgram();
 
Symbol *that = new_Symbol(prog, reg.file, reg.fileIndex);
 
pol.set<Value>(this, that);
 
that->reg.size = this->reg.size;
that->reg.type = this->reg.type;
that->reg.data = this->reg.data;
 
that->baseSym = this->baseSym;
 
return that;
}
 
bool
Symbol::isUniform() const
{
return
reg.file != FILE_SYSTEM_VALUE &&
reg.file != FILE_MEMORY_LOCAL &&
reg.file != FILE_SHADER_INPUT;
}
 
ImmediateValue::ImmediateValue(Program *prog, uint32_t uval)
{
memset(&reg, 0, sizeof(reg));
 
reg.file = FILE_IMMEDIATE;
reg.size = 4;
reg.type = TYPE_U32;
 
reg.data.u32 = uval;
 
prog->add(this, this->id);
}
 
ImmediateValue::ImmediateValue(Program *prog, float fval)
{
memset(&reg, 0, sizeof(reg));
 
reg.file = FILE_IMMEDIATE;
reg.size = 4;
reg.type = TYPE_F32;
 
reg.data.f32 = fval;
 
prog->add(this, this->id);
}
 
ImmediateValue::ImmediateValue(Program *prog, double dval)
{
memset(&reg, 0, sizeof(reg));
 
reg.file = FILE_IMMEDIATE;
reg.size = 8;
reg.type = TYPE_F64;
 
reg.data.f64 = dval;
 
prog->add(this, this->id);
}
 
ImmediateValue::ImmediateValue(const ImmediateValue *proto, DataType ty)
{
reg = proto->reg;
 
reg.type = ty;
reg.size = typeSizeof(ty);
}
 
ImmediateValue *
ImmediateValue::clone(ClonePolicy<Function>& pol) const
{
Program *prog = pol.context()->getProgram();
ImmediateValue *that = new_ImmediateValue(prog, 0u);
 
pol.set<Value>(this, that);
 
that->reg.size = this->reg.size;
that->reg.type = this->reg.type;
that->reg.data = this->reg.data;
 
return that;
}
 
bool
ImmediateValue::isInteger(const int i) const
{
switch (reg.type) {
case TYPE_S8:
return reg.data.s8 == i;
case TYPE_U8:
return reg.data.u8 == i;
case TYPE_S16:
return reg.data.s16 == i;
case TYPE_U16:
return reg.data.u16 == i;
case TYPE_S32:
case TYPE_U32:
return reg.data.s32 == i; // as if ...
case TYPE_F32:
return reg.data.f32 == static_cast<float>(i);
case TYPE_F64:
return reg.data.f64 == static_cast<double>(i);
default:
return false;
}
}
 
bool
ImmediateValue::isNegative() const
{
switch (reg.type) {
case TYPE_S8: return reg.data.s8 < 0;
case TYPE_S16: return reg.data.s16 < 0;
case TYPE_S32:
case TYPE_U32: return reg.data.s32 < 0;
case TYPE_F32: return reg.data.u32 & (1 << 31);
case TYPE_F64: return reg.data.u64 & (1ULL << 63);
default:
return false;
}
}
 
bool
ImmediateValue::isPow2() const
{
switch (reg.type) {
case TYPE_U8:
case TYPE_U16:
case TYPE_U32: return util_is_power_of_two(reg.data.u32);
default:
return false;
}
}
 
void
ImmediateValue::applyLog2()
{
switch (reg.type) {
case TYPE_S8:
case TYPE_S16:
case TYPE_S32:
assert(!this->isNegative());
// fall through
case TYPE_U8:
case TYPE_U16:
case TYPE_U32:
reg.data.u32 = util_logbase2(reg.data.u32);
break;
case TYPE_F32:
reg.data.f32 = log2f(reg.data.f32);
break;
case TYPE_F64:
reg.data.f64 = log2(reg.data.f64);
break;
default:
assert(0);
break;
}
}
 
bool
ImmediateValue::compare(CondCode cc, float fval) const
{
if (reg.type != TYPE_F32)
ERROR("immediate value is not of type f32");
 
switch (static_cast<CondCode>(cc & 7)) {
case CC_TR: return true;
case CC_FL: return false;
case CC_LT: return reg.data.f32 < fval;
case CC_LE: return reg.data.f32 <= fval;
case CC_GT: return reg.data.f32 > fval;
case CC_GE: return reg.data.f32 >= fval;
case CC_EQ: return reg.data.f32 == fval;
case CC_NE: return reg.data.f32 != fval;
default:
assert(0);
return false;
}
}
 
ImmediateValue&
ImmediateValue::operator=(const ImmediateValue &that)
{
this->reg = that.reg;
return (*this);
}
 
bool
Value::interfers(const Value *that) const
{
uint32_t idA, idB;
 
if (that->reg.file != reg.file || that->reg.fileIndex != reg.fileIndex)
return false;
if (this->asImm())
return false;
 
if (this->asSym()) {
idA = this->join->reg.data.offset;
idB = that->join->reg.data.offset;
} else {
idA = this->join->reg.data.id * MIN2(this->reg.size, 4);
idB = that->join->reg.data.id * MIN2(that->reg.size, 4);
}
 
if (idA < idB)
return (idA + this->reg.size > idB);
else
if (idA > idB)
return (idB + that->reg.size > idA);
else
return (idA == idB);
}
 
bool
Value::equals(const Value *that, bool strict) const
{
if (strict)
return this == that;
 
if (that->reg.file != reg.file || that->reg.fileIndex != reg.fileIndex)
return false;
if (that->reg.size != this->reg.size)
return false;
 
if (that->reg.data.id != this->reg.data.id)
return false;
 
return true;
}
 
bool
ImmediateValue::equals(const Value *that, bool strict) const
{
const ImmediateValue *imm = that->asImm();
if (!imm)
return false;
return reg.data.u64 == imm->reg.data.u64;
}
 
bool
Symbol::equals(const Value *that, bool strict) const
{
if (reg.file != that->reg.file || reg.fileIndex != that->reg.fileIndex)
return false;
assert(that->asSym());
 
if (this->baseSym != that->asSym()->baseSym)
return false;
 
if (reg.file == FILE_SYSTEM_VALUE)
return (this->reg.data.sv.sv == that->reg.data.sv.sv &&
this->reg.data.sv.index == that->reg.data.sv.index);
return this->reg.data.offset == that->reg.data.offset;
}
 
void Instruction::init()
{
next = prev = 0;
 
cc = CC_ALWAYS;
rnd = ROUND_N;
cache = CACHE_CA;
subOp = 0;
 
saturate = 0;
join = 0;
exit = 0;
terminator = 0;
ftz = 0;
dnz = 0;
perPatch = 0;
fixed = 0;
encSize = 0;
ipa = 0;
mask = 0;
 
lanes = 0xf;
 
postFactor = 0;
 
predSrc = -1;
flagsDef = -1;
flagsSrc = -1;
}
 
Instruction::Instruction()
{
init();
 
op = OP_NOP;
dType = sType = TYPE_F32;
 
id = -1;
bb = 0;
}
 
Instruction::Instruction(Function *fn, operation opr, DataType ty)
{
init();
 
op = opr;
dType = sType = ty;
 
fn->add(this, id);
}
 
Instruction::~Instruction()
{
if (bb) {
Function *fn = bb->getFunction();
bb->remove(this);
fn->allInsns.remove(id);
}
 
for (int s = 0; srcExists(s); ++s)
setSrc(s, NULL);
// must unlink defs too since the list pointers will get deallocated
for (int d = 0; defExists(d); ++d)
setDef(d, NULL);
}
 
void
Instruction::setDef(int i, Value *val)
{
int size = defs.size();
if (i >= size) {
defs.resize(i + 1);
while (size <= i)
defs[size++].setInsn(this);
}
defs[i].set(val);
}
 
void
Instruction::setSrc(int s, Value *val)
{
int size = srcs.size();
if (s >= size) {
srcs.resize(s + 1);
while (size <= s)
srcs[size++].setInsn(this);
}
srcs[s].set(val);
}
 
void
Instruction::setSrc(int s, const ValueRef& ref)
{
setSrc(s, ref.get());
srcs[s].mod = ref.mod;
}
 
void
Instruction::swapSources(int a, int b)
{
Value *value = srcs[a].get();
Modifier m = srcs[a].mod;
 
setSrc(a, srcs[b]);
 
srcs[b].set(value);
srcs[b].mod = m;
}
 
static inline void moveSourcesAdjustIndex(int8_t &index, int s, int delta)
{
if (index >= s)
index += delta;
else
if ((delta < 0) && (index >= (s + delta)))
index = -1;
}
 
// Moves sources [@s,last_source] by @delta.
// If @delta < 0, sources [@s - abs(@delta), @s) are erased.
void
Instruction::moveSources(const int s, const int delta)
{
if (delta == 0)
return;
assert(s + delta >= 0);
 
int k;
 
for (k = 0; srcExists(k); ++k) {
for (int i = 0; i < 2; ++i)
moveSourcesAdjustIndex(src(k).indirect[i], s, delta);
}
moveSourcesAdjustIndex(predSrc, s, delta);
moveSourcesAdjustIndex(flagsSrc, s, delta);
if (asTex()) {
TexInstruction *tex = asTex();
moveSourcesAdjustIndex(tex->tex.rIndirectSrc, s, delta);
moveSourcesAdjustIndex(tex->tex.sIndirectSrc, s, delta);
}
 
if (delta > 0) {
--k;
for (int p = k + delta; k >= s; --k, --p)
setSrc(p, src(k));
} else {
int p;
for (p = s; p < k; ++p)
setSrc(p + delta, src(p));
for (; (p + delta) < k; ++p)
setSrc(p + delta, NULL);
}
}
 
void
Instruction::takeExtraSources(int s, Value *values[3])
{
values[0] = getIndirect(s, 0);
if (values[0])
setIndirect(s, 0, NULL);
 
values[1] = getIndirect(s, 1);
if (values[1])
setIndirect(s, 1, NULL);
 
values[2] = getPredicate();
if (values[2])
setPredicate(cc, NULL);
}
 
void
Instruction::putExtraSources(int s, Value *values[3])
{
if (values[0])
setIndirect(s, 0, values[0]);
if (values[1])
setIndirect(s, 1, values[1]);
if (values[2])
setPredicate(cc, values[2]);
}
 
Instruction *
Instruction::clone(ClonePolicy<Function>& pol, Instruction *i) const
{
if (!i)
i = new_Instruction(pol.context(), op, dType);
#ifndef NDEBUG // non-conformant assert, so this is required
assert(typeid(*i) == typeid(*this));
#endif
 
pol.set<Instruction>(this, i);
 
i->sType = sType;
 
i->rnd = rnd;
i->cache = cache;
i->subOp = subOp;
 
i->saturate = saturate;
i->join = join;
i->exit = exit;
i->mask = mask;
i->ftz = ftz;
i->dnz = dnz;
i->ipa = ipa;
i->lanes = lanes;
i->perPatch = perPatch;
 
i->postFactor = postFactor;
 
for (int d = 0; defExists(d); ++d)
i->setDef(d, pol.get(getDef(d)));
 
for (int s = 0; srcExists(s); ++s) {
i->setSrc(s, pol.get(getSrc(s)));
i->src(s).mod = src(s).mod;
}
 
i->cc = cc;
i->predSrc = predSrc;
i->flagsDef = flagsDef;
i->flagsSrc = flagsSrc;
 
return i;
}
 
unsigned int
Instruction::defCount(unsigned int mask, bool singleFile) const
{
unsigned int i, n;
 
if (singleFile) {
unsigned int d = ffs(mask);
if (!d)
return 0;
for (i = d--; defExists(i); ++i)
if (getDef(i)->reg.file != getDef(d)->reg.file)
mask &= ~(1 << i);
}
 
for (n = 0, i = 0; this->defExists(i); ++i, mask >>= 1)
n += mask & 1;
return n;
}
 
unsigned int
Instruction::srcCount(unsigned int mask, bool singleFile) const
{
unsigned int i, n;
 
if (singleFile) {
unsigned int s = ffs(mask);
if (!s)
return 0;
for (i = s--; srcExists(i); ++i)
if (getSrc(i)->reg.file != getSrc(s)->reg.file)
mask &= ~(1 << i);
}
 
for (n = 0, i = 0; this->srcExists(i); ++i, mask >>= 1)
n += mask & 1;
return n;
}
 
bool
Instruction::setIndirect(int s, int dim, Value *value)
{
assert(this->srcExists(s));
 
int p = srcs[s].indirect[dim];
if (p < 0) {
if (!value)
return true;
p = srcs.size();
while (p > 0 && !srcExists(p - 1))
--p;
}
setSrc(p, value);
srcs[p].usedAsPtr = (value != 0);
srcs[s].indirect[dim] = value ? p : -1;
return true;
}
 
bool
Instruction::setPredicate(CondCode ccode, Value *value)
{
cc = ccode;
 
if (!value) {
if (predSrc >= 0) {
srcs[predSrc].set(NULL);
predSrc = -1;
}
return true;
}
 
if (predSrc < 0) {
predSrc = srcs.size();
while (predSrc > 0 && !srcExists(predSrc - 1))
--predSrc;
}
 
setSrc(predSrc, value);
return true;
}
 
bool
Instruction::writesPredicate() const
{
for (int d = 0; defExists(d); ++d)
if (getDef(d)->inFile(FILE_PREDICATE) || getDef(d)->inFile(FILE_FLAGS))
return true;
return false;
}
 
static bool
insnCheckCommutationDefSrc(const Instruction *a, const Instruction *b)
{
for (int d = 0; a->defExists(d); ++d)
for (int s = 0; b->srcExists(s); ++s)
if (a->getDef(d)->interfers(b->getSrc(s)))
return false;
return true;
}
 
static bool
insnCheckCommutationDefDef(const Instruction *a, const Instruction *b)
{
for (int d = 0; a->defExists(d); ++d)
for (int c = 0; b->defExists(c); ++c)
if (a->getDef(d)->interfers(b->getDef(c)))
return false;
return true;
}
 
bool
Instruction::isCommutationLegal(const Instruction *i) const
{
bool ret = insnCheckCommutationDefDef(this, i);
ret = ret && insnCheckCommutationDefSrc(this, i);
ret = ret && insnCheckCommutationDefSrc(i, this);
return ret;
}
 
TexInstruction::TexInstruction(Function *fn, operation op)
: Instruction(fn, op, TYPE_F32)
{
memset(&tex, 0, sizeof(tex));
 
tex.rIndirectSrc = -1;
tex.sIndirectSrc = -1;
}
 
TexInstruction::~TexInstruction()
{
for (int c = 0; c < 3; ++c) {
dPdx[c].set(NULL);
dPdy[c].set(NULL);
}
}
 
TexInstruction *
TexInstruction::clone(ClonePolicy<Function>& pol, Instruction *i) const
{
TexInstruction *tex = (i ? static_cast<TexInstruction *>(i) :
new_TexInstruction(pol.context(), op));
 
Instruction::clone(pol, tex);
 
tex->tex = this->tex;
 
if (op == OP_TXD) {
for (unsigned int c = 0; c < tex->tex.target.getDim(); ++c) {
tex->dPdx[c].set(dPdx[c]);
tex->dPdy[c].set(dPdy[c]);
}
}
 
return tex;
}
 
const struct TexInstruction::Target::Desc TexInstruction::Target::descTable[] =
{
{ "1D", 1, 1, false, false, false },
{ "2D", 2, 2, false, false, false },
{ "2D_MS", 2, 3, false, false, false },
{ "3D", 3, 3, false, false, false },
{ "CUBE", 2, 3, false, true, false },
{ "1D_SHADOW", 1, 1, false, false, true },
{ "2D_SHADOW", 2, 2, false, false, true },
{ "CUBE_SHADOW", 2, 3, false, true, true },
{ "1D_ARRAY", 1, 2, true, false, false },
{ "2D_ARRAY", 2, 3, true, false, false },
{ "2D_MS_ARRAY", 2, 4, true, false, false },
{ "CUBE_ARRAY", 2, 4, true, true, false },
{ "1D_ARRAY_SHADOW", 1, 2, true, false, true },
{ "2D_ARRAY_SHADOW", 2, 3, true, false, true },
{ "RECT", 2, 2, false, false, false },
{ "RECT_SHADOW", 2, 2, false, false, true },
{ "CUBE_ARRAY_SHADOW", 2, 4, true, true, true },
{ "BUFFER", 1, 1, false, false, false },
};
 
void
TexInstruction::setIndirectR(Value *v)
{
int p = ((tex.rIndirectSrc < 0) && v) ? srcs.size() : tex.rIndirectSrc;
if (p >= 0) {
tex.rIndirectSrc = p;
setSrc(p, v);
srcs[p].usedAsPtr = !!v;
}
}
 
void
TexInstruction::setIndirectS(Value *v)
{
int p = ((tex.sIndirectSrc < 0) && v) ? srcs.size() : tex.sIndirectSrc;
if (p >= 0) {
tex.sIndirectSrc = p;
setSrc(p, v);
srcs[p].usedAsPtr = !!v;
}
}
 
CmpInstruction::CmpInstruction(Function *fn, operation op)
: Instruction(fn, op, TYPE_F32)
{
setCond = CC_ALWAYS;
}
 
CmpInstruction *
CmpInstruction::clone(ClonePolicy<Function>& pol, Instruction *i) const
{
CmpInstruction *cmp = (i ? static_cast<CmpInstruction *>(i) :
new_CmpInstruction(pol.context(), op));
cmp->dType = dType;
Instruction::clone(pol, cmp);
cmp->setCond = setCond;
return cmp;
}
 
FlowInstruction::FlowInstruction(Function *fn, operation op, void *targ)
: Instruction(fn, op, TYPE_NONE)
{
if (op == OP_CALL)
target.fn = reinterpret_cast<Function *>(targ);
else
target.bb = reinterpret_cast<BasicBlock *>(targ);
 
if (op == OP_BRA ||
op == OP_CONT || op == OP_BREAK ||
op == OP_RET || op == OP_EXIT)
terminator = 1;
else
if (op == OP_JOIN)
terminator = targ ? 1 : 0;
 
allWarp = absolute = limit = builtin = indirect = 0;
}
 
FlowInstruction *
FlowInstruction::clone(ClonePolicy<Function>& pol, Instruction *i) const
{
FlowInstruction *flow = (i ? static_cast<FlowInstruction *>(i) :
new_FlowInstruction(pol.context(), op, NULL));
 
Instruction::clone(pol, flow);
flow->allWarp = allWarp;
flow->absolute = absolute;
flow->limit = limit;
flow->builtin = builtin;
 
if (builtin)
flow->target.builtin = target.builtin;
else
if (op == OP_CALL)
flow->target.fn = target.fn;
else
if (target.bb)
flow->target.bb = pol.get<BasicBlock>(target.bb);
 
return flow;
}
 
Program::Program(Type type, Target *arch)
: progType(type),
target(arch),
mem_Instruction(sizeof(Instruction), 6),
mem_CmpInstruction(sizeof(CmpInstruction), 4),
mem_TexInstruction(sizeof(TexInstruction), 4),
mem_FlowInstruction(sizeof(FlowInstruction), 4),
mem_LValue(sizeof(LValue), 8),
mem_Symbol(sizeof(Symbol), 7),
mem_ImmediateValue(sizeof(ImmediateValue), 7)
{
code = NULL;
binSize = 0;
 
maxGPR = -1;
 
main = new Function(this, "MAIN", ~0);
calls.insert(&main->call);
 
dbgFlags = 0;
optLevel = 0;
 
targetPriv = NULL;
}
 
Program::~Program()
{
for (ArrayList::Iterator it = allFuncs.iterator(); !it.end(); it.next())
delete reinterpret_cast<Function *>(it.get());
 
for (ArrayList::Iterator it = allRValues.iterator(); !it.end(); it.next())
releaseValue(reinterpret_cast<Value *>(it.get()));
}
 
void Program::releaseInstruction(Instruction *insn)
{
// TODO: make this not suck so much
 
insn->~Instruction();
 
if (insn->asCmp())
mem_CmpInstruction.release(insn);
else
if (insn->asTex())
mem_TexInstruction.release(insn);
else
if (insn->asFlow())
mem_FlowInstruction.release(insn);
else
mem_Instruction.release(insn);
}
 
void Program::releaseValue(Value *value)
{
value->~Value();
 
if (value->asLValue())
mem_LValue.release(value);
else
if (value->asImm())
mem_ImmediateValue.release(value);
else
if (value->asSym())
mem_Symbol.release(value);
}
 
 
} // namespace nv50_ir
 
extern "C" {
 
static void
nv50_ir_init_prog_info(struct nv50_ir_prog_info *info)
{
#if defined(PIPE_SHADER_HULL) && defined(PIPE_SHADER_DOMAIN)
if (info->type == PIPE_SHADER_HULL || info->type == PIPE_SHADER_DOMAIN) {
info->prop.tp.domain = PIPE_PRIM_MAX;
info->prop.tp.outputPrim = PIPE_PRIM_MAX;
}
#endif
if (info->type == PIPE_SHADER_GEOMETRY) {
info->prop.gp.instanceCount = 1;
info->prop.gp.maxVertices = 1;
}
info->io.clipDistance = 0xff;
info->io.pointSize = 0xff;
info->io.instanceId = 0xff;
info->io.vertexId = 0xff;
info->io.edgeFlagIn = 0xff;
info->io.edgeFlagOut = 0xff;
info->io.fragDepth = 0xff;
info->io.sampleMask = 0xff;
info->io.backFaceColor[0] = info->io.backFaceColor[1] = 0xff;
}
int
nv50_ir_generate_code(struct nv50_ir_prog_info *info)
{
int ret = 0;
 
nv50_ir::Program::Type type;
 
nv50_ir_init_prog_info(info);
 
#define PROG_TYPE_CASE(a, b) \
case PIPE_SHADER_##a: type = nv50_ir::Program::TYPE_##b; break
 
switch (info->type) {
PROG_TYPE_CASE(VERTEX, VERTEX);
// PROG_TYPE_CASE(HULL, TESSELLATION_CONTROL);
// PROG_TYPE_CASE(DOMAIN, TESSELLATION_EVAL);
PROG_TYPE_CASE(GEOMETRY, GEOMETRY);
PROG_TYPE_CASE(FRAGMENT, FRAGMENT);
PROG_TYPE_CASE(COMPUTE, COMPUTE);
default:
type = nv50_ir::Program::TYPE_COMPUTE;
break;
}
INFO_DBG(info->dbgFlags, VERBOSE, "translating program of type %u\n", type);
 
nv50_ir::Target *targ = nv50_ir::Target::create(info->target);
if (!targ)
return -1;
 
nv50_ir::Program *prog = new nv50_ir::Program(type, targ);
if (!prog)
return -1;
prog->driver = info;
prog->dbgFlags = info->dbgFlags;
prog->optLevel = info->optLevel;
 
switch (info->bin.sourceRep) {
#if 0
case PIPE_IR_LLVM:
case PIPE_IR_GLSL:
return -1;
case PIPE_IR_SM4:
ret = prog->makeFromSM4(info) ? 0 : -2;
break;
case PIPE_IR_TGSI:
#endif
default:
ret = prog->makeFromTGSI(info) ? 0 : -2;
break;
}
if (ret < 0)
goto out;
if (prog->dbgFlags & NV50_IR_DEBUG_VERBOSE)
prog->print();
 
targ->parseDriverInfo(info);
prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_PRE_SSA);
 
prog->convertToSSA();
 
if (prog->dbgFlags & NV50_IR_DEBUG_VERBOSE)
prog->print();
 
prog->optimizeSSA(info->optLevel);
prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_SSA);
 
if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
prog->print();
 
if (!prog->registerAllocation()) {
ret = -4;
goto out;
}
prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_POST_RA);
 
prog->optimizePostRA(info->optLevel);
 
if (!prog->emitBinary(info)) {
ret = -5;
goto out;
}
 
out:
INFO_DBG(prog->dbgFlags, VERBOSE, "nv50_ir_generate_code: ret = %i\n", ret);
 
info->bin.maxGPR = prog->maxGPR;
info->bin.code = prog->code;
info->bin.codeSize = prog->binSize;
info->bin.tlsSpace = prog->tlsSize;
 
delete prog;
nv50_ir::Target::destroy(targ);
 
return ret;
}
 
} // extern "C"
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir.h
0,0 → 1,1197
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_H__
#define __NV50_IR_H__
 
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <deque>
#include <list>
#include <vector>
 
#include "nv50_ir_util.h"
#include "nv50_ir_graph.h"
 
#include "nv50_ir_driver.h"
 
namespace nv50_ir {
 
enum operation
{
OP_NOP = 0,
OP_PHI,
OP_UNION, // unify a new definition and several source values
OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
OP_CONSTRAINT, // copy values into consecutive registers
OP_MOV, // simple copy, no modifiers allowed
OP_LOAD,
OP_STORE,
OP_ADD, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds
OP_SUB,
OP_MUL,
OP_DIV,
OP_MOD,
OP_MAD,
OP_FMA,
OP_SAD, // abs(src0 - src1) + src2
OP_ABS,
OP_NEG,
OP_NOT,
OP_AND,
OP_OR,
OP_XOR,
OP_SHL,
OP_SHR,
OP_MAX,
OP_MIN,
OP_SAT, // CLAMP(f32, 0.0, 1.0)
OP_CEIL,
OP_FLOOR,
OP_TRUNC,
OP_CVT,
OP_SET_AND, // dst = (src0 CMP src1) & src2
OP_SET_OR,
OP_SET_XOR,
OP_SET,
OP_SELP, // dst = src2 ? src0 : src1
OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
OP_RCP,
OP_RSQ,
OP_LG2,
OP_SIN,
OP_COS,
OP_EX2,
OP_EXP, // exponential (base M_E)
OP_LOG, // natural logarithm
OP_PRESIN,
OP_PREEX2,
OP_SQRT,
OP_POW,
OP_BRA,
OP_CALL,
OP_RET,
OP_CONT,
OP_BREAK,
OP_PRERET,
OP_PRECONT,
OP_PREBREAK,
OP_BRKPT, // breakpoint (not related to loops)
OP_JOINAT, // push control flow convergence point
OP_JOIN, // converge
OP_DISCARD,
OP_EXIT,
OP_MEMBAR, // memory barrier (mfence, lfence, sfence)
OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
OP_EXPORT,
OP_LINTERP,
OP_PINTERP,
OP_EMIT, // emit vertex
OP_RESTART, // restart primitive
OP_TEX,
OP_TXB, // texture bias
OP_TXL, // texure lod
OP_TXF, // texel fetch
OP_TXQ, // texture size query
OP_TXD, // texture derivatives
OP_TXG, // texture gather
OP_TEXCSAA, // texture op for coverage sampling
OP_TEXPREP, // turn cube map array into 2d array coordinates
OP_SULDB, // surface load (raw)
OP_SULDP, // surface load (formatted)
OP_SUSTB, // surface store (raw)
OP_SUSTP, // surface store (formatted)
OP_SUREDB,
OP_SUREDP, // surface reduction (atomic op)
OP_SULEA, // surface load effective address
OP_SUBFM, // surface bitfield manipulation
OP_SUCLAMP, // clamp surface coordinates
OP_SUEAU, // surface effective address
OP_MADSP, // special integer multiply-add
OP_TEXBAR, // texture dependency barrier
OP_DFDX,
OP_DFDY,
OP_RDSV, // read system value
OP_WRSV, // write system value
OP_QUADOP,
OP_QUADON,
OP_QUADPOP,
OP_POPCNT, // bitcount(src0 & src1)
OP_INSBF, // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
OP_EXTBF, // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK
OP_PERMT, // dst = bytes from src2,src0 selected by src1 (nvc0's src order)
OP_ATOM,
OP_BAR, // execution barrier, sources = { id, thread count, predicate }
OP_VADD, // byte/word vector operations
OP_VAVG,
OP_VMIN,
OP_VMAX,
OP_VSAD,
OP_VSET,
OP_VSHR,
OP_VSHL,
OP_VSEL,
OP_CCTL, // cache control
OP_LAST
};
 
// various instruction-specific modifier definitions Instruction::subOp
// MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
#define NV50_IR_SUBOP_MUL_HIGH 1
#define NV50_IR_SUBOP_EMIT_RESTART 1
#define NV50_IR_SUBOP_LDC_IL 1
#define NV50_IR_SUBOP_LDC_IS 2
#define NV50_IR_SUBOP_LDC_ISL 3
#define NV50_IR_SUBOP_SHIFT_WRAP 1
#define NV50_IR_SUBOP_EMU_PRERET 1
#define NV50_IR_SUBOP_TEXBAR(n) n
#define NV50_IR_SUBOP_MOV_FINAL 1
#define NV50_IR_SUBOP_EXTBF_REV 1
#define NV50_IR_SUBOP_PERMT_F4E 1
#define NV50_IR_SUBOP_PERMT_B4E 2
#define NV50_IR_SUBOP_PERMT_RC8 3
#define NV50_IR_SUBOP_PERMT_ECL 4
#define NV50_IR_SUBOP_PERMT_ECR 5
#define NV50_IR_SUBOP_PERMT_RC16 6
#define NV50_IR_SUBOP_BAR_SYNC 0
#define NV50_IR_SUBOP_BAR_ARRIVE 1
#define NV50_IR_SUBOP_BAR_RED_AND 2
#define NV50_IR_SUBOP_BAR_RED_OR 3
#define NV50_IR_SUBOP_BAR_RED_POPC 4
#define NV50_IR_SUBOP_MEMBAR_L 1
#define NV50_IR_SUBOP_MEMBAR_S 2
#define NV50_IR_SUBOP_MEMBAR_M 3
#define NV50_IR_SUBOP_MEMBAR_CTA (0 << 2)
#define NV50_IR_SUBOP_MEMBAR_GL (1 << 2)
#define NV50_IR_SUBOP_MEMBAR_SYS (2 << 2)
#define NV50_IR_SUBOP_MEMBAR_DIR(m) ((m) & 0x3)
#define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3)
#define NV50_IR_SUBOP_MEMBAR(d,s) \
(NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s)
#define NV50_IR_SUBOP_ATOM_ADD 0
#define NV50_IR_SUBOP_ATOM_MIN 1
#define NV50_IR_SUBOP_ATOM_MAX 2
#define NV50_IR_SUBOP_ATOM_INC 3
#define NV50_IR_SUBOP_ATOM_DEC 4
#define NV50_IR_SUBOP_ATOM_AND 5
#define NV50_IR_SUBOP_ATOM_OR 6
#define NV50_IR_SUBOP_ATOM_XOR 7
#define NV50_IR_SUBOP_ATOM_CAS 8
#define NV50_IR_SUBOP_ATOM_EXCH 9
#define NV50_IR_SUBOP_CCTL_IV 5
#define NV50_IR_SUBOP_CCTL_IVALL 6
#define NV50_IR_SUBOP_SUST_IGN 0
#define NV50_IR_SUBOP_SUST_TRAP 1
#define NV50_IR_SUBOP_SUST_SDCL 3
#define NV50_IR_SUBOP_SULD_ZERO 0
#define NV50_IR_SUBOP_SULD_TRAP 1
#define NV50_IR_SUBOP_SULD_SDCL 3
#define NV50_IR_SUBOP_SUBFM_3D 1
#define NV50_IR_SUBOP_SUCLAMP_2D 0x10
#define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0))
#define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0))
#define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0))
#define NV50_IR_SUBOP_MADSP_SD 0xffff
// Yes, we could represent those with DataType.
// Or put the type into operation and have a couple 1000 values in that enum.
// This will have to do for now.
// The bitfields are supposed to correspond to nve4 ISA.
#define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a))
#define NV50_IR_SUBOP_V1(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x0000)
#define NV50_IR_SUBOP_V2(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x4000)
#define NV50_IR_SUBOP_V4(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x8000)
#define NV50_IR_SUBOP_Vn(n) ((n) >> 14)
 
enum DataType
{
TYPE_NONE,
TYPE_U8,
TYPE_S8,
TYPE_U16,
TYPE_S16,
TYPE_U32,
TYPE_S32,
TYPE_U64, // 64 bit operations are only lowered after register allocation
TYPE_S64,
TYPE_F16,
TYPE_F32,
TYPE_F64,
TYPE_B96,
TYPE_B128
};
 
enum CondCode
{
CC_FL = 0,
CC_NEVER = CC_FL, // when used with FILE_FLAGS
CC_LT = 1,
CC_EQ = 2,
CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
CC_LE = 3,
CC_GT = 4,
CC_NE = 5,
CC_P = CC_NE,
CC_GE = 6,
CC_TR = 7,
CC_ALWAYS = CC_TR,
CC_U = 8,
CC_LTU = 9,
CC_EQU = 10,
CC_LEU = 11,
CC_GTU = 12,
CC_NEU = 13,
CC_GEU = 14,
CC_NO = 0x10,
CC_NC = 0x11,
CC_NS = 0x12,
CC_NA = 0x13,
CC_A = 0x14,
CC_S = 0x15,
CC_C = 0x16,
CC_O = 0x17
};
 
enum RoundMode
{
ROUND_N, // nearest
ROUND_M, // towards -inf
ROUND_Z, // towards 0
ROUND_P, // towards +inf
ROUND_NI, // nearest integer
ROUND_MI, // to integer towards -inf
ROUND_ZI, // to integer towards 0
ROUND_PI, // to integer towards +inf
};
 
enum CacheMode
{
CACHE_CA, // cache at all levels
CACHE_WB = CACHE_CA, // cache write back
CACHE_CG, // cache at global level
CACHE_CS, // cache streaming
CACHE_CV, // cache as volatile
CACHE_WT = CACHE_CV // cache write-through
};
 
enum DataFile
{
FILE_NULL = 0,
FILE_GPR,
FILE_PREDICATE, // boolean predicate
FILE_FLAGS, // zero/sign/carry/overflow bits
FILE_ADDRESS,
LAST_REGISTER_FILE = FILE_ADDRESS,
FILE_IMMEDIATE,
FILE_MEMORY_CONST,
FILE_SHADER_INPUT,
FILE_SHADER_OUTPUT,
FILE_MEMORY_GLOBAL,
FILE_MEMORY_SHARED,
FILE_MEMORY_LOCAL,
FILE_SYSTEM_VALUE,
DATA_FILE_COUNT
};
 
enum TexTarget
{
TEX_TARGET_1D,
TEX_TARGET_2D,
TEX_TARGET_2D_MS,
TEX_TARGET_3D,
TEX_TARGET_CUBE,
TEX_TARGET_1D_SHADOW,
TEX_TARGET_2D_SHADOW,
TEX_TARGET_CUBE_SHADOW,
TEX_TARGET_1D_ARRAY,
TEX_TARGET_2D_ARRAY,
TEX_TARGET_2D_MS_ARRAY,
TEX_TARGET_CUBE_ARRAY,
TEX_TARGET_1D_ARRAY_SHADOW,
TEX_TARGET_2D_ARRAY_SHADOW,
TEX_TARGET_RECT,
TEX_TARGET_RECT_SHADOW,
TEX_TARGET_CUBE_ARRAY_SHADOW,
TEX_TARGET_BUFFER,
TEX_TARGET_COUNT
};
 
enum SVSemantic
{
SV_POSITION, // WPOS
SV_VERTEX_ID,
SV_INSTANCE_ID,
SV_INVOCATION_ID,
SV_PRIMITIVE_ID,
SV_VERTEX_COUNT, // gl_PatchVerticesIn
SV_LAYER,
SV_VIEWPORT_INDEX,
SV_YDIR,
SV_FACE,
SV_POINT_SIZE,
SV_POINT_COORD,
SV_CLIP_DISTANCE,
SV_SAMPLE_INDEX,
SV_TESS_FACTOR,
SV_TESS_COORD,
SV_TID,
SV_CTAID,
SV_NTID,
SV_GRIDID,
SV_NCTAID,
SV_LANEID,
SV_PHYSID,
SV_NPHYSID,
SV_CLOCK,
SV_LBASE,
SV_SBASE,
SV_UNDEFINED,
SV_LAST
};
 
class Program;
class Function;
class BasicBlock;
 
class Target;
 
class Instruction;
class CmpInstruction;
class TexInstruction;
class FlowInstruction;
 
class Value;
class LValue;
class Symbol;
class ImmediateValue;
 
struct Storage
{
DataFile file;
int8_t fileIndex; // signed, may be indirect for CONST[]
uint8_t size; // this should match the Instruction type's size
DataType type; // mainly for pretty printing
union {
uint64_t u64; // immediate values
uint32_t u32;
uint16_t u16;
uint8_t u8;
int64_t s64;
int32_t s32;
int16_t s16;
int8_t s8;
float f32;
double f64;
int32_t offset; // offset from 0 (base of address space)
int32_t id; // register id (< 0 if virtual/unassigned, in units <= 4)
struct {
SVSemantic sv;
int index;
} sv;
} data;
};
 
// precedence: NOT after SAT after NEG after ABS
#define NV50_IR_MOD_ABS (1 << 0)
#define NV50_IR_MOD_NEG (1 << 1)
#define NV50_IR_MOD_SAT (1 << 2)
#define NV50_IR_MOD_NOT (1 << 3)
#define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
 
#define NV50_IR_INTERP_MODE_MASK 0x3
#define NV50_IR_INTERP_LINEAR (0 << 0)
#define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
#define NV50_IR_INTERP_FLAT (2 << 0)
#define NV50_IR_INTERP_SC (3 << 0) // what exactly is that ?
#define NV50_IR_INTERP_SAMPLE_MASK 0xc
#define NV50_IR_INTERP_DEFAULT (0 << 2)
#define NV50_IR_INTERP_CENTROID (1 << 2)
#define NV50_IR_INTERP_OFFSET (2 << 2)
#define NV50_IR_INTERP_SAMPLEID (3 << 2)
 
// do we really want this to be a class ?
class Modifier
{
public:
Modifier() : bits(0) { }
Modifier(unsigned int m) : bits(m) { }
Modifier(operation op);
 
// @return new Modifier applying a after b (asserts if unrepresentable)
Modifier operator*(const Modifier) const;
Modifier operator*=(const Modifier m) { *this = *this * m; return *this; }
Modifier operator==(const Modifier m) const { return m.bits == bits; }
Modifier operator!=(const Modifier m) const { return m.bits != bits; }
 
inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
 
operation getOp() const;
 
inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
 
inline operator bool() const { return bits ? true : false; }
 
void applyTo(ImmediateValue &imm) const;
 
int print(char *buf, size_t size) const;
 
private:
uint8_t bits;
};
 
class ValueRef
{
public:
ValueRef(Value * = NULL);
ValueRef(const ValueRef&);
~ValueRef();
 
inline bool exists() const { return value != NULL; }
 
void set(Value *);
void set(const ValueRef&);
inline Value *get() const { return value; }
inline Value *rep() const;
 
inline Instruction *getInsn() const { return insn; }
inline void setInsn(Instruction *inst) { insn = inst; }
 
inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
inline const ValueRef *getIndirect(int dim) const;
 
inline DataFile getFile() const;
inline unsigned getSize() const;
 
// SSA: return eventual (traverse MOVs) literal value, if it exists
bool getImmediate(ImmediateValue&) const;
 
public:
Modifier mod;
int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
uint8_t swizzle;
 
bool usedAsPtr; // for printing
 
private:
Value *value;
Instruction *insn;
};
 
class ValueDef
{
public:
ValueDef(Value * = NULL);
ValueDef(const ValueDef&);
~ValueDef();
 
inline bool exists() const { return value != NULL; }
 
inline Value *get() const { return value; }
inline Value *rep() const;
void set(Value *);
bool mayReplace(const ValueRef &);
void replace(const ValueRef &, bool doSet); // replace all uses of the old value
 
inline Instruction *getInsn() const { return insn; }
inline void setInsn(Instruction *inst) { insn = inst; }
 
inline DataFile getFile() const;
inline unsigned getSize() const;
 
inline void setSSA(LValue *);
inline const LValue *preSSA() const;
 
private:
Value *value; // should make this LValue * ...
LValue *origin; // pre SSA value
Instruction *insn;
};
 
class Value
{
public:
Value();
virtual ~Value() { }
 
virtual Value *clone(ClonePolicy<Function>&) const = 0;
 
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
 
virtual bool equals(const Value *, bool strict = false) const;
virtual bool interfers(const Value *) const;
virtual bool isUniform() const { return true; }
 
inline Value *rep() const { return join; }
 
inline Instruction *getUniqueInsn() const;
inline Instruction *getInsn() const; // use when uniqueness is certain
 
inline int refCount() { return uses.size(); }
 
inline LValue *asLValue();
inline Symbol *asSym();
inline ImmediateValue *asImm();
inline const Symbol *asSym() const;
inline const ImmediateValue *asImm() const;
 
inline bool inFile(DataFile f) { return reg.file == f; }
 
static inline Value *get(Iterator&);
 
std::list<ValueRef *> uses;
std::list<ValueDef *> defs;
typedef std::list<ValueRef *>::iterator UseIterator;
typedef std::list<ValueRef *>::const_iterator UseCIterator;
typedef std::list<ValueDef *>::iterator DefIterator;
typedef std::list<ValueDef *>::const_iterator DefCIterator;
 
int id;
Storage reg;
 
// TODO: these should be in LValue:
Interval livei;
Value *join;
};
 
class LValue : public Value
{
public:
LValue(Function *, DataFile file);
LValue(Function *, LValue *);
~LValue() { }
 
virtual bool isUniform() const;
 
virtual LValue *clone(ClonePolicy<Function>&) const;
 
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 
public:
unsigned compMask : 8; // compound/component mask
unsigned compound : 1; // used by RA, value involved in split/merge
unsigned ssa : 1;
unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0)
unsigned noSpill : 1; // do not spill (e.g. if spill temporary already)
};
 
class Symbol : public Value
{
public:
Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
~Symbol() { }
 
virtual Symbol *clone(ClonePolicy<Function>&) const;
 
virtual bool equals(const Value *that, bool strict) const;
 
virtual bool isUniform() const;
 
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
 
// print with indirect values
int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
 
inline void setFile(DataFile file, ubyte fileIndex = 0)
{
reg.file = file;
reg.fileIndex = fileIndex;
}
 
inline void setOffset(int32_t offset);
inline void setAddress(Symbol *base, int32_t offset);
inline void setSV(SVSemantic sv, uint32_t idx = 0);
 
inline const Symbol *getBase() const { return baseSym; }
 
private:
Symbol *baseSym; // array base for Symbols representing array elements
};
 
class ImmediateValue : public Value
{
public:
ImmediateValue() { }
ImmediateValue(Program *, uint32_t);
ImmediateValue(Program *, float);
ImmediateValue(Program *, double);
// NOTE: not added to program with
ImmediateValue(const ImmediateValue *, DataType ty);
~ImmediateValue() { };
 
virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
 
virtual bool equals(const Value *that, bool strict) const;
 
// these only work if 'type' is valid (we mostly use untyped literals):
bool isInteger(const int ival) const; // ival is cast to this' type
bool isNegative() const;
bool isPow2() const;
 
void applyLog2();
 
// for constant folding:
ImmediateValue operator+(const ImmediateValue&) const;
ImmediateValue operator-(const ImmediateValue&) const;
ImmediateValue operator*(const ImmediateValue&) const;
ImmediateValue operator/(const ImmediateValue&) const;
 
ImmediateValue& operator=(const ImmediateValue&); // only sets value !
 
bool compare(CondCode cc, float fval) const;
 
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
};
 
class Instruction
{
public:
Instruction();
Instruction(Function *, operation, DataType);
virtual ~Instruction();
 
virtual Instruction *clone(ClonePolicy<Function>&,
Instruction * = NULL) const;
 
void setDef(int i, Value *);
void setSrc(int s, Value *);
void setSrc(int s, const ValueRef&);
void swapSources(int a, int b);
void moveSources(int s, int delta);
bool setIndirect(int s, int dim, Value *);
 
inline ValueRef& src(int s) { return srcs[s]; }
inline ValueDef& def(int s) { return defs[s]; }
inline const ValueRef& src(int s) const { return srcs[s]; }
inline const ValueDef& def(int s) const { return defs[s]; }
 
inline Value *getDef(int d) const { return defs[d].get(); }
inline Value *getSrc(int s) const { return srcs[s].get(); }
inline Value *getIndirect(int s, int dim) const;
 
inline bool defExists(unsigned d) const
{
return d < defs.size() && defs[d].exists();
}
inline bool srcExists(unsigned s) const
{
return s < srcs.size() && srcs[s].exists();
}
 
inline bool constrainedDefs() const;
 
bool setPredicate(CondCode ccode, Value *);
inline Value *getPredicate() const;
bool writesPredicate() const;
inline bool isPredicated() const { return predSrc >= 0; }
 
inline void setFlagsSrc(int s, Value *);
inline void setFlagsDef(int d, Value *);
inline bool usesFlags() const { return flagsSrc >= 0; }
 
unsigned int defCount() const { return defs.size(); };
unsigned int defCount(unsigned int mask, bool singleFile = false) const;
unsigned int srcCount() const { return srcs.size(); };
unsigned int srcCount(unsigned int mask, bool singleFile = false) const;
 
// save & remove / set indirect[0,1] and predicate source
void takeExtraSources(int s, Value *[3]);
void putExtraSources(int s, Value *[3]);
 
inline void setType(DataType type) { dType = sType = type; }
 
inline void setType(DataType dtype, DataType stype)
{
dType = dtype;
sType = stype;
}
 
inline bool isPseudo() const { return op < OP_MOV; }
bool isDead() const;
bool isNop() const;
bool isCommutationLegal(const Instruction *) const; // must be adjacent !
bool isActionEqual(const Instruction *) const;
bool isResultEqual(const Instruction *) const;
 
void print() const;
 
inline CmpInstruction *asCmp();
inline TexInstruction *asTex();
inline FlowInstruction *asFlow();
inline const TexInstruction *asTex() const;
inline const CmpInstruction *asCmp() const;
inline const FlowInstruction *asFlow() const;
 
public:
Instruction *next;
Instruction *prev;
int id;
int serial; // CFG order
 
operation op;
DataType dType; // destination or defining type
DataType sType; // source or secondary type
CondCode cc;
RoundMode rnd;
CacheMode cache;
 
uint16_t subOp; // quadop, 1 for mul-high, etc.
 
unsigned encSize : 4; // encoding size in bytes
unsigned saturate : 1; // to [0.0f, 1.0f]
unsigned join : 1; // converge control flow (use OP_JOIN until end)
unsigned fixed : 1; // prevent dead code elimination
unsigned terminator : 1; // end of basic block
unsigned ftz : 1; // flush denormal to zero
unsigned dnz : 1; // denormals, NaN are zero
unsigned ipa : 4; // interpolation mode
unsigned lanes : 4;
unsigned perPatch : 1;
unsigned exit : 1; // terminate program after insn
unsigned mask : 4; // for vector ops
 
int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
 
int8_t predSrc;
int8_t flagsDef;
int8_t flagsSrc;
 
uint8_t sched; // scheduling data (NOTE: maybe move to separate storage)
 
BasicBlock *bb;
 
protected:
std::deque<ValueDef> defs; // no gaps !
std::deque<ValueRef> srcs; // no gaps !
 
// instruction specific methods:
// (don't want to subclass, would need more constructors and memory pools)
public:
inline void setInterpolate(unsigned int mode) { ipa = mode; }
 
unsigned int getInterpMode() const { return ipa & 0x3; }
unsigned int getSampleMode() const { return ipa & 0xc; }
 
private:
void init();
};
 
enum TexQuery
{
TXQ_DIMS,
TXQ_TYPE,
TXQ_SAMPLE_POSITION,
TXQ_FILTER,
TXQ_LOD,
TXQ_WRAP,
TXQ_BORDER_COLOUR
};
 
class TexInstruction : public Instruction
{
public:
class Target
{
public:
Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
 
const char *getName() const { return descTable[target].name; }
unsigned int getArgCount() const { return descTable[target].argc; }
unsigned int getDim() const { return descTable[target].dim; }
int isArray() const { return descTable[target].array ? 1 : 0; }
int isCube() const { return descTable[target].cube ? 1 : 0; }
int isShadow() const { return descTable[target].shadow ? 1 : 0; }
int isMS() const {
return target == TEX_TARGET_2D_MS || target == TEX_TARGET_2D_MS_ARRAY; }
 
Target& operator=(TexTarget targ)
{
assert(targ < TEX_TARGET_COUNT);
target = targ;
return *this;
}
 
inline bool operator==(TexTarget targ) const { return target == targ; }
inline bool operator!=(TexTarget targ) const { return target != targ; }
 
enum TexTarget getEnum() const { return target; }
 
private:
struct Desc
{
char name[19];
uint8_t dim;
uint8_t argc;
bool array;
bool cube;
bool shadow;
};
 
static const struct Desc descTable[TEX_TARGET_COUNT];
 
private:
enum TexTarget target;
};
 
public:
TexInstruction(Function *, operation);
virtual ~TexInstruction();
 
virtual TexInstruction *clone(ClonePolicy<Function>&,
Instruction * = NULL) const;
 
inline void setTexture(Target targ, uint8_t r, uint8_t s)
{
tex.r = r;
tex.s = s;
tex.target = targ;
}
 
void setIndirectR(Value *);
void setIndirectS(Value *);
inline Value *getIndirectR() const;
inline Value *getIndirectS() const;
 
public:
struct {
Target target;
 
uint16_t r;
uint16_t s;
int8_t rIndirectSrc;
int8_t sIndirectSrc;
 
uint8_t mask;
uint8_t gatherComp;
 
bool liveOnly; // only execute on live pixels of a quad (optimization)
bool levelZero;
bool derivAll;
 
int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
int8_t offset[4][3];
 
enum TexQuery query;
} tex;
 
ValueRef dPdx[3];
ValueRef dPdy[3];
};
 
class CmpInstruction : public Instruction
{
public:
CmpInstruction(Function *, operation);
 
virtual CmpInstruction *clone(ClonePolicy<Function>&,
Instruction * = NULL) const;
 
void setCondition(CondCode cond) { setCond = cond; }
CondCode getCondition() const { return setCond; }
 
public:
CondCode setCond;
};
 
class FlowInstruction : public Instruction
{
public:
FlowInstruction(Function *, operation, void *target);
 
virtual FlowInstruction *clone(ClonePolicy<Function>&,
Instruction * = NULL) const;
 
public:
unsigned allWarp : 1;
unsigned absolute : 1;
unsigned limit : 1;
unsigned builtin : 1; // true for calls to emulation code
unsigned indirect : 1; // target in src(0)
 
union {
BasicBlock *bb;
int builtin;
Function *fn;
} target;
};
 
class BasicBlock
{
public:
BasicBlock(Function *);
~BasicBlock();
 
BasicBlock *clone(ClonePolicy<Function>&) const;
 
inline int getId() const { return id; }
inline unsigned int getInsnCount() const { return numInsns; }
inline bool isTerminated() const { return exit && exit->terminator; }
 
bool dominatedBy(BasicBlock *bb);
inline bool reachableBy(const BasicBlock *by, const BasicBlock *term);
 
// returns mask of conditional out blocks
// e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
unsigned int initiatesSimpleConditional() const;
 
public:
Function *getFunction() const { return func; }
Program *getProgram() const { return program; }
 
Instruction *getEntry() const { return entry; } // first non-phi instruction
Instruction *getPhi() const { return phi; }
Instruction *getFirst() const { return phi ? phi : entry; }
Instruction *getExit() const { return exit; }
 
void insertHead(Instruction *);
void insertTail(Instruction *);
void insertBefore(Instruction *, Instruction *);
void insertAfter(Instruction *, Instruction *);
void remove(Instruction *);
void permuteAdjacent(Instruction *, Instruction *);
 
BasicBlock *idom() const;
 
// NOTE: currently does not rebuild the dominator tree
BasicBlock *splitBefore(Instruction *, bool attach = true);
BasicBlock *splitAfter(Instruction *, bool attach = true);
 
DLList& getDF() { return df; }
DLList::Iterator iterDF() { return df.iterator(); }
 
static inline BasicBlock *get(Iterator&);
static inline BasicBlock *get(Graph::Node *);
 
public:
Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
Graph::Node dom;
 
BitSet liveSet;
BitSet defSet;
 
uint32_t binPos;
uint32_t binSize;
 
Instruction *joinAt; // for quick reference
 
bool explicitCont; // loop headers: true if loop contains continue stmts
 
private:
int id;
DLList df;
 
Instruction *phi;
Instruction *entry;
Instruction *exit;
 
unsigned int numInsns;
 
private:
Function *func;
Program *program;
 
void splitCommon(Instruction *, BasicBlock *, bool attach);
};
 
class Function
{
public:
Function(Program *, const char *name, uint32_t label);
~Function();
 
static inline Function *get(Graph::Node *node);
 
inline Program *getProgram() const { return prog; }
inline const char *getName() const { return name; }
inline int getId() const { return id; }
inline uint32_t getLabel() const { return label; }
 
void print();
void printLiveIntervals() const;
void printCFGraph(const char *filePath);
 
bool setEntry(BasicBlock *);
bool setExit(BasicBlock *);
 
unsigned int orderInstructions(ArrayList&);
 
inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
 
inline LValue *getLValue(int id);
 
void buildLiveSets();
void buildDefSets();
bool convertToSSA();
 
public:
std::deque<ValueDef> ins;
std::deque<ValueRef> outs;
std::deque<Value *> clobbers;
 
Graph cfg;
Graph::Node *cfgExit;
Graph *domTree;
Graph::Node call; // node in the call graph
 
BasicBlock **bbArray; // BBs in emission order
int bbCount;
 
unsigned int loopNestingBound;
int regClobberMax;
 
uint32_t binPos;
uint32_t binSize;
 
Value *stackPtr;
 
uint32_t tlsBase; // base address for l[] space (if no stack pointer is used)
uint32_t tlsSize;
 
ArrayList allBBlocks;
ArrayList allInsns;
ArrayList allLValues;
 
private:
void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
void buildDefSetsPreSSA(BasicBlock *bb, const int seq);
 
private:
uint32_t label;
int id;
const char *const name;
Program *prog;
};
 
enum CGStage
{
CG_STAGE_PRE_SSA,
CG_STAGE_SSA, // expected directly before register allocation
CG_STAGE_POST_RA
};
 
class Program
{
public:
enum Type
{
TYPE_VERTEX,
TYPE_TESSELLATION_CONTROL,
TYPE_TESSELLATION_EVAL,
TYPE_GEOMETRY,
TYPE_FRAGMENT,
TYPE_COMPUTE
};
 
Program(Type type, Target *targ);
~Program();
 
void print();
 
Type getType() const { return progType; }
 
inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
inline void del(Function *fn, int& id) { allFuncs.remove(id); }
inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
 
bool makeFromTGSI(struct nv50_ir_prog_info *);
bool makeFromSM4(struct nv50_ir_prog_info *);
bool convertToSSA();
bool optimizeSSA(int level);
bool optimizePostRA(int level);
bool registerAllocation();
bool emitBinary(struct nv50_ir_prog_info *);
 
const Target *getTarget() const { return target; }
 
private:
void emitSymbolTable(struct nv50_ir_prog_info *);
 
Type progType;
Target *target;
 
public:
Function *main;
Graph calls;
 
ArrayList allFuncs;
ArrayList allRValues;
 
uint32_t *code;
uint32_t binSize;
uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL
 
int maxGPR;
 
MemoryPool mem_Instruction;
MemoryPool mem_CmpInstruction;
MemoryPool mem_TexInstruction;
MemoryPool mem_FlowInstruction;
MemoryPool mem_LValue;
MemoryPool mem_Symbol;
MemoryPool mem_ImmediateValue;
 
uint32_t dbgFlags;
uint8_t optLevel;
 
void *targetPriv; // e.g. to carry information between passes
 
const struct nv50_ir_prog_info *driver; // for driver configuration
 
void releaseInstruction(Instruction *);
void releaseValue(Value *);
};
 
// TODO: add const version
class Pass
{
public:
bool run(Program *, bool ordered = false, bool skipPhi = false);
bool run(Function *, bool ordered = false, bool skipPhi = false);
 
private:
// return false to continue with next entity on next higher level
virtual bool visit(Function *) { return true; }
virtual bool visit(BasicBlock *) { return true; }
virtual bool visit(Instruction *) { return false; }
 
bool doRun(Program *, bool ordered, bool skipPhi);
bool doRun(Function *, bool ordered, bool skipPhi);
 
protected:
bool err;
Function *func;
Program *prog;
};
 
// =============================================================================
 
#include "nv50_ir_inlines.h"
 
} // namespace nv50_ir
 
#endif // __NV50_IR_H__
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_bb.cpp
0,0 → 1,550
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
 
namespace nv50_ir {
 
Function::Function(Program *p, const char *fnName, uint32_t label)
: call(this),
label(label),
name(fnName),
prog(p)
{
cfgExit = NULL;
domTree = NULL;
 
bbArray = NULL;
bbCount = 0;
loopNestingBound = 0;
regClobberMax = 0;
 
binPos = 0;
binSize = 0;
 
stackPtr = NULL;
tlsBase = 0;
tlsSize = 0;
 
prog->add(this, id);
}
 
Function::~Function()
{
prog->del(this, id);
 
if (domTree)
delete domTree;
if (bbArray)
delete[] bbArray;
 
// clear value refs and defs
ins.clear();
outs.clear();
 
for (ArrayList::Iterator it = allInsns.iterator(); !it.end(); it.next())
delete_Instruction(prog, reinterpret_cast<Instruction *>(it.get()));
 
for (ArrayList::Iterator it = allLValues.iterator(); !it.end(); it.next())
delete_Value(prog, reinterpret_cast<LValue *>(it.get()));
 
for (ArrayList::Iterator BBs = allBBlocks.iterator(); !BBs.end(); BBs.next())
delete reinterpret_cast<BasicBlock *>(BBs.get());
}
 
BasicBlock::BasicBlock(Function *fn) : cfg(this), dom(this), func(fn)
{
program = func->getProgram();
 
joinAt = phi = entry = exit = NULL;
 
numInsns = 0;
binPos = 0;
binSize = 0;
 
explicitCont = false;
 
func->add(this, this->id);
}
 
BasicBlock::~BasicBlock()
{
// nothing yet
}
 
BasicBlock *
BasicBlock::clone(ClonePolicy<Function>& pol) const
{
BasicBlock *bb = new BasicBlock(pol.context());
 
pol.set(this, bb);
 
for (Instruction *i = getFirst(); i; i = i->next)
bb->insertTail(i->clone(pol));
 
pol.context()->cfg.insert(&bb->cfg);
 
for (Graph::EdgeIterator it = cfg.outgoing(); !it.end(); it.next()) {
BasicBlock *obb = BasicBlock::get(it.getNode());
bb->cfg.attach(&pol.get(obb)->cfg, it.getType());
}
 
return bb;
}
 
BasicBlock *
BasicBlock::idom() const
{
Graph::Node *dn = dom.parent();
return dn ? BasicBlock::get(dn) : NULL;
}
 
void
BasicBlock::insertHead(Instruction *inst)
{
assert(inst->next == 0 && inst->prev == 0);
 
if (inst->op == OP_PHI) {
if (phi) {
insertBefore(phi, inst);
} else {
if (entry) {
insertBefore(entry, inst);
} else {
assert(!exit);
phi = exit = inst;
inst->bb = this;
++numInsns;
}
}
} else {
if (entry) {
insertBefore(entry, inst);
} else {
if (phi) {
insertAfter(exit, inst); // after last phi
} else {
assert(!exit);
entry = exit = inst;
inst->bb = this;
++numInsns;
}
}
}
}
 
void
BasicBlock::insertTail(Instruction *inst)
{
assert(inst->next == 0 && inst->prev == 0);
 
if (inst->op == OP_PHI) {
if (entry) {
insertBefore(entry, inst);
} else
if (exit) {
assert(phi);
insertAfter(exit, inst);
} else {
assert(!phi);
phi = exit = inst;
inst->bb = this;
++numInsns;
}
} else {
if (exit) {
insertAfter(exit, inst);
} else {
assert(!phi);
entry = exit = inst;
inst->bb = this;
++numInsns;
}
}
}
 
void
BasicBlock::insertBefore(Instruction *q, Instruction *p)
{
assert(p && q);
 
assert(p->next == 0 && p->prev == 0);
 
if (q == entry) {
if (p->op == OP_PHI) {
if (!phi)
phi = p;
} else {
entry = p;
}
} else
if (q == phi) {
assert(p->op == OP_PHI);
phi = p;
}
 
p->next = q;
p->prev = q->prev;
if (p->prev)
p->prev->next = p;
q->prev = p;
 
p->bb = this;
++numInsns;
}
 
void
BasicBlock::insertAfter(Instruction *p, Instruction *q)
{
assert(p && q);
assert(q->op != OP_PHI || p->op == OP_PHI);
 
assert(q->next == 0 && q->prev == 0);
 
if (p == exit)
exit = q;
if (p->op == OP_PHI && q->op != OP_PHI)
entry = q;
 
q->prev = p;
q->next = p->next;
if (q->next)
q->next->prev = q;
p->next = q;
 
q->bb = this;
++numInsns;
}
 
void
BasicBlock::remove(Instruction *insn)
{
assert(insn->bb == this);
 
if (insn->prev)
insn->prev->next = insn->next;
 
if (insn->next)
insn->next->prev = insn->prev;
else
exit = insn->prev;
 
if (insn == entry) {
if (insn->next)
entry = insn->next;
else
if (insn->prev && insn->prev->op != OP_PHI)
entry = insn->prev;
else
entry = NULL;
}
 
if (insn == phi)
phi = (insn->next && insn->next->op == OP_PHI) ? insn->next : 0;
 
--numInsns;
insn->bb = NULL;
insn->next =
insn->prev = NULL;
}
 
void BasicBlock::permuteAdjacent(Instruction *a, Instruction *b)
{
assert(a->bb == b->bb);
 
if (a->next != b) {
Instruction *i = a;
a = b;
b = i;
}
assert(a->next == b);
assert(a->op != OP_PHI && b->op != OP_PHI);
 
if (b == exit)
exit = a;
if (a == entry)
entry = b;
 
b->prev = a->prev;
a->next = b->next;
b->next = a;
a->prev = b;
 
if (b->prev)
b->prev->next = b;
if (a->prev)
a->next->prev = a;
}
 
void
BasicBlock::splitCommon(Instruction *insn, BasicBlock *bb, bool attach)
{
bb->entry = insn;
 
if (insn) {
exit = insn->prev;
insn->prev = NULL;
}
 
if (exit)
exit->next = NULL;
else
entry = NULL;
 
while (!cfg.outgoing(true).end()) {
Graph::Edge *e = cfg.outgoing(true).getEdge();
bb->cfg.attach(e->getTarget(), e->getType());
this->cfg.detach(e->getTarget());
}
 
for (; insn; insn = insn->next) {
this->numInsns--;
bb->numInsns++;
insn->bb = bb;
bb->exit = insn;
}
if (attach)
this->cfg.attach(&bb->cfg, Graph::Edge::TREE);
}
 
BasicBlock *
BasicBlock::splitBefore(Instruction *insn, bool attach)
{
BasicBlock *bb = new BasicBlock(func);
assert(!insn || insn->op != OP_PHI);
 
splitCommon(insn, bb, attach);
return bb;
}
 
BasicBlock *
BasicBlock::splitAfter(Instruction *insn, bool attach)
{
BasicBlock *bb = new BasicBlock(func);
assert(!insn || insn->op != OP_PHI);
 
bb->joinAt = joinAt;
joinAt = NULL;
 
splitCommon(insn ? insn->next : NULL, bb, attach);
return bb;
}
 
bool
BasicBlock::dominatedBy(BasicBlock *that)
{
Graph::Node *bn = &that->dom;
Graph::Node *dn = &this->dom;
 
while (dn && dn != bn)
dn = dn->parent();
 
return dn != NULL;
}
 
unsigned int
BasicBlock::initiatesSimpleConditional() const
{
Graph::Node *out[2];
int n;
Graph::Edge::Type eR;
 
if (cfg.outgoingCount() != 2) // -> if and -> else/endif
return false;
 
n = 0;
for (Graph::EdgeIterator ei = cfg.outgoing(); !ei.end(); ei.next())
out[n++] = ei.getNode();
eR = out[1]->outgoing().getType();
 
// IF block is out edge to the right
if (eR == Graph::Edge::CROSS || eR == Graph::Edge::BACK)
return 0x2;
 
if (out[1]->outgoingCount() != 1) // 0 is IF { RET; }, >1 is more divergence
return 0x0;
// do they reconverge immediately ?
if (out[1]->outgoing().getNode() == out[0])
return 0x1;
if (out[0]->outgoingCount() == 1)
if (out[0]->outgoing().getNode() == out[1]->outgoing().getNode())
return 0x3;
 
return 0x0;
}
 
bool
Function::setEntry(BasicBlock *bb)
{
if (cfg.getRoot())
return false;
cfg.insert(&bb->cfg);
return true;
}
 
bool
Function::setExit(BasicBlock *bb)
{
if (cfgExit)
return false;
cfgExit = &bb->cfg;
return true;
}
 
unsigned int
Function::orderInstructions(ArrayList &result)
{
result.clear();
 
for (IteratorRef it = cfg.iteratorCFG(); !it->end(); it->next()) {
BasicBlock *bb =
BasicBlock::get(reinterpret_cast<Graph::Node *>(it->get()));
 
for (Instruction *insn = bb->getFirst(); insn; insn = insn->next)
result.insert(insn, insn->serial);
}
 
return result.getSize();
}
 
void
Function::buildLiveSets()
{
for (unsigned i = 0; i <= loopNestingBound; ++i)
buildLiveSetsPreSSA(BasicBlock::get(cfg.getRoot()), cfg.nextSequence());
 
for (ArrayList::Iterator bi = allBBlocks.iterator(); !bi.end(); bi.next())
BasicBlock::get(bi)->liveSet.marker = false;
}
 
void
Function::buildDefSets()
{
for (unsigned i = 0; i <= loopNestingBound; ++i)
buildDefSetsPreSSA(BasicBlock::get(cfgExit), cfg.nextSequence());
 
for (ArrayList::Iterator bi = allBBlocks.iterator(); !bi.end(); bi.next())
BasicBlock::get(bi)->liveSet.marker = false;
}
 
bool
Pass::run(Program *prog, bool ordered, bool skipPhi)
{
this->prog = prog;
err = false;
return doRun(prog, ordered, skipPhi);
}
 
bool
Pass::doRun(Program *prog, bool ordered, bool skipPhi)
{
for (IteratorRef it = prog->calls.iteratorDFS(false);
!it->end(); it->next()) {
Graph::Node *n = reinterpret_cast<Graph::Node *>(it->get());
if (!doRun(Function::get(n), ordered, skipPhi))
return false;
}
return !err;
}
 
bool
Pass::run(Function *func, bool ordered, bool skipPhi)
{
prog = func->getProgram();
err = false;
return doRun(func, ordered, skipPhi);
}
 
bool
Pass::doRun(Function *func, bool ordered, bool skipPhi)
{
IteratorRef bbIter;
BasicBlock *bb;
Instruction *insn, *next;
 
this->func = func;
if (!visit(func))
return false;
 
bbIter = ordered ? func->cfg.iteratorCFG() : func->cfg.iteratorDFS();
 
for (; !bbIter->end(); bbIter->next()) {
bb = BasicBlock::get(reinterpret_cast<Graph::Node *>(bbIter->get()));
if (!visit(bb))
break;
for (insn = skipPhi ? bb->getEntry() : bb->getFirst(); insn != NULL;
insn = next) {
next = insn->next;
if (!visit(insn))
break;
}
}
 
return !err;
}
 
void
Function::printCFGraph(const char *filePath)
{
FILE *out = fopen(filePath, "a");
if (!out) {
ERROR("failed to open file: %s\n", filePath);
return;
}
INFO("printing control flow graph to: %s\n", filePath);
 
fprintf(out, "digraph G {\n");
 
for (IteratorRef it = cfg.iteratorDFS(); !it->end(); it->next()) {
BasicBlock *bb = BasicBlock::get(
reinterpret_cast<Graph::Node *>(it->get()));
int idA = bb->getId();
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
int idB = BasicBlock::get(ei.getNode())->getId();
switch (ei.getType()) {
case Graph::Edge::TREE:
fprintf(out, "\t%i -> %i;\n", idA, idB);
break;
case Graph::Edge::FORWARD:
fprintf(out, "\t%i -> %i [color=green];\n", idA, idB);
break;
case Graph::Edge::CROSS:
fprintf(out, "\t%i -> %i [color=red];\n", idA, idB);
break;
case Graph::Edge::BACK:
fprintf(out, "\t%i -> %i;\n", idA, idB);
break;
case Graph::Edge::DUMMY:
fprintf(out, "\t%i -> %i [style=dotted];\n", idA, idB);
break;
default:
assert(0);
break;
}
}
}
 
fprintf(out, "}\n");
fclose(out);
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.cpp
0,0 → 1,614
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_build_util.h"
 
namespace nv50_ir {
 
BuildUtil::BuildUtil()
{
init(NULL);
}
 
BuildUtil::BuildUtil(Program *prog)
{
init(prog);
}
 
void
BuildUtil::init(Program *prog)
{
this->prog = prog;
 
func = NULL;
bb = NULL;
pos = NULL;
 
memset(imms, 0, sizeof(imms));
immCount = 0;
}
 
void
BuildUtil::addImmediate(ImmediateValue *imm)
{
if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
return;
 
unsigned int pos = u32Hash(imm->reg.data.u32);
 
while (imms[pos])
pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
imms[pos] = imm;
immCount++;
}
 
Instruction *
BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
{
Instruction *insn = new_Instruction(func, op, ty);
 
insn->setDef(0, dst);
insn->setSrc(0, src);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
Value *src0, Value *src1)
{
Instruction *insn = new_Instruction(func, op, ty);
 
insn->setDef(0, dst);
insn->setSrc(0, src0);
insn->setSrc(1, src1);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
Value *src0, Value *src1, Value *src2)
{
Instruction *insn = new_Instruction(func, op, ty);
 
insn->setDef(0, dst);
insn->setSrc(0, src0);
insn->setSrc(1, src1);
insn->setSrc(2, src2);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr)
{
Instruction *insn = new_Instruction(func, OP_LOAD, ty);
 
insn->setDef(0, dst);
insn->setSrc(0, mem);
if (ptr)
insn->setIndirect(0, 0, ptr);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
Value *stVal)
{
Instruction *insn = new_Instruction(func, op, ty);
 
insn->setSrc(0, mem);
insn->setSrc(1, stVal);
if (ptr)
insn->setIndirect(0, 0, ptr);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
Value *attrRel, Value *primRel)
{
Symbol *sym = mkSymbol(file, 0, ty, offset);
 
Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
 
insn->setIndirect(0, 0, attrRel);
insn->setIndirect(0, 1, primRel);
 
// already inserted
return insn;
}
 
Instruction *
BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel)
{
operation op = OP_LINTERP;
DataType ty = TYPE_F32;
 
if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT)
ty = TYPE_U32;
else
if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE)
op = OP_PINTERP;
 
Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset);
 
Instruction *insn = mkOp1(op, ty, dst, sym);
insn->setIndirect(0, 0, rel);
return insn;
}
 
Instruction *
BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
{
Instruction *insn = new_Instruction(func, OP_MOV, ty);
 
insn->setDef(0, dst);
insn->setSrc(0, src);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkMovToReg(int id, Value *src)
{
Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
 
insn->setDef(0, new_LValue(func, FILE_GPR));
insn->getDef(0)->reg.data.id = id;
insn->setSrc(0, src);
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkMovFromReg(Value *dst, int id)
{
Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
 
insn->setDef(0, dst);
insn->setSrc(0, new_LValue(func, FILE_GPR));
insn->getSrc(0)->reg.data.id = id;
 
insert(insn);
return insn;
}
 
Instruction *
BuildUtil::mkCvt(operation op,
DataType dstTy, Value *dst, DataType srcTy, Value *src)
{
Instruction *insn = new_Instruction(func, op, dstTy);
 
insn->setType(dstTy, srcTy);
insn->setDef(0, dst);
insn->setSrc(0, src);
 
insert(insn);
return insn;
}
 
CmpInstruction *
BuildUtil::mkCmp(operation op, CondCode cc, DataType ty, Value *dst,
Value *src0, Value *src1, Value *src2)
{
CmpInstruction *insn = new_CmpInstruction(func, op);
 
insn->setType((dst->reg.file == FILE_PREDICATE ||
dst->reg.file == FILE_FLAGS) ? TYPE_U8 : ty, ty);
insn->setCondition(cc);
insn->setDef(0, dst);
insn->setSrc(0, src0);
insn->setSrc(1, src1);
if (src2)
insn->setSrc(2, src2);
 
if (dst->reg.file == FILE_FLAGS)
insn->flagsDef = 0;
 
insert(insn);
return insn;
}
 
TexInstruction *
BuildUtil::mkTex(operation op, TexTarget targ,
uint16_t tic, uint16_t tsc,
const std::vector<Value *> &def,
const std::vector<Value *> &src)
{
TexInstruction *tex = new_TexInstruction(func, op);
 
for (size_t d = 0; d < def.size() && def[d]; ++d)
tex->setDef(d, def[d]);
for (size_t s = 0; s < src.size() && src[s]; ++s)
tex->setSrc(s, src[s]);
 
tex->setTexture(targ, tic, tsc);
 
insert(tex);
return tex;
}
 
Instruction *
BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
{
Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
quadop->subOp = q;
quadop->lanes = l;
return quadop;
}
 
Instruction *
BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
{
LValue *def0 = getSSA();
LValue *def1 = getSSA();
 
mkMov(def0, trSrc)->setPredicate(CC_P, pred);
mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
 
return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
}
 
Instruction *
BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val)
{
Instruction *insn = NULL;
 
const DataType fTy = typeOfSize(halfSize * 2);
 
if (val->reg.file == FILE_IMMEDIATE)
val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
 
if (isMemoryFile(val->reg.file)) {
h[0] = cloneShallow(getFunction(), val);
h[1] = cloneShallow(getFunction(), val);
h[0]->reg.size = halfSize;
h[1]->reg.size = halfSize;
h[1]->reg.data.offset += halfSize;
} else {
h[0] = getSSA(halfSize, val->reg.file);
h[1] = getSSA(halfSize, val->reg.file);
insn = mkOp1(OP_SPLIT, fTy, h[0], val);
insn->setDef(1, h[1]);
}
return insn;
}
 
FlowInstruction *
BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred)
{
FlowInstruction *insn = new_FlowInstruction(func, op, targ);
 
if (pred)
insn->setPredicate(cc, pred);
 
insert(insn);
return insn;
}
 
void
BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
{
static const uint16_t baseSize2[16] =
{
0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040,
};
 
int base = 0;
 
for (; rMask; rMask >>= 4, base += 4) {
const uint32_t mask = rMask & 0xf;
if (!mask)
continue;
int base1 = (baseSize2[mask] >> 0) & 0xf;
int size1 = (baseSize2[mask] >> 4) & 0xf;
int base2 = (baseSize2[mask] >> 8) & 0xf;
int size2 = (baseSize2[mask] >> 12) & 0xf;
Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
if (1) { // size1 can't be 0
LValue *reg = new_LValue(func, f);
reg->reg.size = size1 << unit;
reg->reg.data.id = base + base1;
insn->setDef(0, reg);
}
if (size2) {
LValue *reg = new_LValue(func, f);
reg->reg.size = size2 << unit;
reg->reg.data.id = base + base2;
insn->setDef(1, reg);
}
}
}
 
ImmediateValue *
BuildUtil::mkImm(uint32_t u)
{
unsigned int pos = u32Hash(u);
 
while (imms[pos] && imms[pos]->reg.data.u32 != u)
pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
 
ImmediateValue *imm = imms[pos];
if (!imm) {
imm = new_ImmediateValue(prog, u);
addImmediate(imm);
}
return imm;
}
 
ImmediateValue *
BuildUtil::mkImm(uint64_t u)
{
ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
 
imm->reg.size = 8;
imm->reg.type = TYPE_U64;
imm->reg.data.u64 = u;
 
return imm;
}
 
ImmediateValue *
BuildUtil::mkImm(float f)
{
union {
float f32;
uint32_t u32;
} u;
u.f32 = f;
return mkImm(u.u32);
}
 
Value *
BuildUtil::loadImm(Value *dst, float f)
{
return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
}
 
Value *
BuildUtil::loadImm(Value *dst, uint32_t u)
{
return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
}
 
Value *
BuildUtil::loadImm(Value *dst, uint64_t u)
{
return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
}
 
Symbol *
BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
uint32_t baseAddr)
{
Symbol *sym = new_Symbol(prog, file, fileIndex);
 
sym->setOffset(baseAddr);
sym->reg.type = ty;
sym->reg.size = typeSizeof(ty);
 
return sym;
}
 
Symbol *
BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
{
Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
 
assert(svIndex < 4 ||
(svName == SV_CLIP_DISTANCE || svName == SV_TESS_FACTOR));
 
switch (svName) {
case SV_POSITION:
case SV_FACE:
case SV_YDIR:
case SV_POINT_SIZE:
case SV_POINT_COORD:
case SV_CLIP_DISTANCE:
case SV_TESS_FACTOR:
sym->reg.type = TYPE_F32;
break;
default:
sym->reg.type = TYPE_U32;
break;
}
sym->reg.size = typeSizeof(sym->reg.type);
 
sym->reg.data.sv.sv = svName;
sym->reg.data.sv.index = svIndex;
 
return sym;
}
 
void
BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx,
uint32_t base, int len, int vecDim, int eltSize,
DataFile file, int8_t fileIdx)
{
this->array = array;
this->arrayIdx = arrayIdx;
this->baseAddr = base;
this->arrayLen = len;
this->vecDim = vecDim;
this->eltSize = eltSize;
this->file = file;
this->regOnly = !isMemoryFile(file);
 
if (!regOnly) {
baseSym = new_Symbol(up->getProgram(), file, fileIdx);
baseSym->setOffset(baseAddr);
baseSym->reg.size = eltSize;
} else {
baseSym = NULL;
}
}
 
Value *
BuildUtil::DataArray::acquire(ValueMap &m, int i, int c)
{
if (regOnly) {
Value *v = lookup(m, i, c);
if (!v)
v = insert(m, i, c, new_LValue(up->getFunction(), file));
 
return v;
} else {
return up->getScratch();
}
}
 
Value *
BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr)
{
if (regOnly) {
Value *v = lookup(m, i, c);
if (!v)
v = insert(m, i, c, new_LValue(up->getFunction(), file));
 
return v;
} else {
Value *sym = lookup(m, i, c);
if (!sym)
sym = insert(m, i, c, mkSymbol(i, c));
 
return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr);
}
}
 
void
BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value)
{
if (regOnly) {
assert(!ptr);
if (!lookup(m, i, c))
insert(m, i, c, value);
 
assert(lookup(m, i, c) == value);
} else {
Value *sym = lookup(m, i, c);
if (!sym)
sym = insert(m, i, c, mkSymbol(i, c));
 
const DataType stTy = typeOfSize(value->reg.size);
 
up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value);
}
}
 
Symbol *
BuildUtil::DataArray::mkSymbol(int i, int c)
{
const unsigned int idx = i * vecDim + c;
Symbol *sym = new_Symbol(up->getProgram(), file, 0);
 
assert(baseSym || (idx < arrayLen && c < vecDim));
 
sym->reg.size = eltSize;
sym->reg.type = typeOfSize(eltSize);
sym->setAddress(baseSym, baseAddr + idx * eltSize);
return sym;
}
 
 
Instruction *
BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i,
Value *zero,
Value *carry)
{
DataType hTy;
int srcNr;
 
switch (i->dType) {
case TYPE_U64: hTy = TYPE_U32; break;
case TYPE_S64: hTy = TYPE_S32; break;
default:
return NULL;
}
 
switch (i->op) {
case OP_MOV: srcNr = 1; break;
case OP_ADD:
case OP_SUB:
if (!carry)
return NULL;
srcNr = 2;
break;
default:
// TODO when needed
return NULL;
}
 
i->setType(hTy);
i->setDef(0, cloneShallow(fn, i->getDef(0)));
i->getDef(0)->reg.size = 4;
Instruction *lo = i;
Instruction *hi = cloneForward(fn, i);
lo->bb->insertAfter(lo, hi);
 
hi->getDef(0)->reg.data.id++;
 
for (int s = 0; s < srcNr; ++s) {
if (lo->getSrc(s)->reg.size < 8) {
hi->setSrc(s, zero);
} else {
if (lo->getSrc(s)->refCount() > 1)
lo->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
lo->getSrc(s)->reg.size /= 2;
hi->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
 
switch (hi->src(s).getFile()) {
case FILE_IMMEDIATE:
hi->getSrc(s)->reg.data.u64 >>= 32;
break;
case FILE_MEMORY_CONST:
case FILE_MEMORY_SHARED:
case FILE_SHADER_INPUT:
hi->getSrc(s)->reg.data.offset += 4;
break;
default:
assert(hi->src(s).getFile() == FILE_GPR);
hi->getSrc(s)->reg.data.id++;
break;
}
}
}
if (srcNr == 2) {
lo->setDef(1, carry);
hi->setFlagsSrc(hi->srcCount(), carry);
}
return hi;
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.h
0,0 → 1,324
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_BUILD_UTIL__
#define __NV50_IR_BUILD_UTIL__
 
namespace nv50_ir {
 
class BuildUtil
{
public:
BuildUtil();
BuildUtil(Program *);
 
inline void setProgram(Program *);
inline Program *getProgram() const { return prog; }
inline Function *getFunction() const { return func; }
 
// keeps inserting at head/tail of block
inline void setPosition(BasicBlock *, bool tail);
// position advances only if @after is true
inline void setPosition(Instruction *, bool after);
 
inline BasicBlock *getBB() { return bb; }
 
inline void insert(Instruction *);
inline void remove(Instruction *i) { assert(i->bb == bb); bb->remove(i); }
 
inline LValue *getScratch(int size = 4, DataFile = FILE_GPR);
// scratch value for a single assignment:
inline LValue *getSSA(int size = 4, DataFile = FILE_GPR);
 
inline Instruction *mkOp(operation, DataType, Value *);
Instruction *mkOp1(operation, DataType, Value *, Value *);
Instruction *mkOp2(operation, DataType, Value *, Value *, Value *);
Instruction *mkOp3(operation, DataType, Value *, Value *, Value *, Value *);
 
LValue *mkOp1v(operation, DataType, Value *, Value *);
LValue *mkOp2v(operation, DataType, Value *, Value *, Value *);
LValue *mkOp3v(operation, DataType, Value *, Value *, Value *, Value *);
 
Instruction *mkLoad(DataType, Value *dst, Symbol *, Value *ptr);
Instruction *mkStore(operation, DataType, Symbol *, Value *ptr, Value *val);
 
LValue *mkLoadv(DataType, Symbol *, Value *ptr);
 
Instruction *mkMov(Value *, Value *, DataType = TYPE_U32);
Instruction *mkMovToReg(int id, Value *);
Instruction *mkMovFromReg(Value *, int id);
 
Instruction *mkInterp(unsigned mode, Value *, int32_t offset, Value *rel);
Instruction *mkFetch(Value *, DataType, DataFile, int32_t offset,
Value *attrRel, Value *primRel);
 
Instruction *mkCvt(operation, DataType, Value *, DataType, Value *);
CmpInstruction *mkCmp(operation, CondCode, DataType,
Value *,
Value *, Value *, Value * = NULL);
TexInstruction *mkTex(operation, TexTarget,
uint16_t tic, uint16_t tsc,
const std::vector<Value *> &def,
const std::vector<Value *> &src);
Instruction *mkQuadop(uint8_t qop, Value *, uint8_t l, Value *, Value *);
 
FlowInstruction *mkFlow(operation, void *target, CondCode, Value *pred);
 
Instruction *mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc);
 
Instruction *mkSplit(Value *half[2], uint8_t halfSize, Value *);
 
void mkClobber(DataFile file, uint32_t regMask, int regUnitLog2);
 
ImmediateValue *mkImm(float);
ImmediateValue *mkImm(uint32_t);
ImmediateValue *mkImm(uint64_t);
 
ImmediateValue *mkImm(int i) { return mkImm((uint32_t)i); }
 
Value *loadImm(Value *dst, float);
Value *loadImm(Value *dst, uint32_t);
Value *loadImm(Value *dst, uint64_t);
 
Value *loadImm(Value *dst, int i) { return loadImm(dst, (uint32_t)i); }
 
// returns high part of the operation
static Instruction *split64BitOpPostRA(Function *, Instruction *,
Value *zero, Value *carry);
 
struct Location
{
Location(unsigned array, unsigned arrayIdx, unsigned i, unsigned c)
: array(array), arrayIdx(arrayIdx), i(i), c(c) { }
Location(const Location &l)
: array(l.array), arrayIdx(l.arrayIdx), i(l.i), c(l.c) { }
 
bool operator==(const Location &l) const
{
return
array == l.array && arrayIdx == l.arrayIdx && i == l.i && c == l.c;
}
 
bool operator<(const Location &l) const
{
return array != l.array ? array < l.array :
arrayIdx != l.arrayIdx ? arrayIdx < l.arrayIdx :
i != l.i ? i < l.i :
c != l.c ? c < l.c :
false;
}
 
unsigned array, arrayIdx, i, c;
};
 
typedef bimap<Location, Value *> ValueMap;
 
class DataArray
{
public:
DataArray(BuildUtil *bld) : up(bld) { }
 
void setup(unsigned array, unsigned arrayIdx,
uint32_t base, int len, int vecDim, int eltSize,
DataFile file, int8_t fileIdx);
 
inline bool exists(ValueMap&, unsigned int i, unsigned int c);
 
Value *load(ValueMap&, int i, int c, Value *ptr);
void store(ValueMap&, int i, int c, Value *ptr, Value *value);
Value *acquire(ValueMap&, int i, int c);
 
private:
inline Value *lookup(ValueMap&, unsigned i, unsigned c);
inline Value *insert(ValueMap&, unsigned i, unsigned c, Value *v);
 
Symbol *mkSymbol(int i, int c);
 
private:
BuildUtil *up;
unsigned array, arrayIdx;
 
uint32_t baseAddr;
uint32_t arrayLen;
Symbol *baseSym;
 
uint8_t vecDim;
uint8_t eltSize; // in bytes
 
DataFile file;
bool regOnly;
};
 
Symbol *mkSymbol(DataFile file, int8_t fileIndex,
DataType ty, uint32_t baseAddress);
 
Symbol *mkSysVal(SVSemantic svName, uint32_t svIndex);
 
private:
void init(Program *);
void addImmediate(ImmediateValue *);
inline unsigned int u32Hash(uint32_t);
 
protected:
Program *prog;
Function *func;
Instruction *pos;
BasicBlock *bb;
bool tail;
 
#define NV50_IR_BUILD_IMM_HT_SIZE 256
 
ImmediateValue *imms[NV50_IR_BUILD_IMM_HT_SIZE];
unsigned int immCount;
};
 
unsigned int BuildUtil::u32Hash(uint32_t u)
{
return (u % 273) % NV50_IR_BUILD_IMM_HT_SIZE;
}
 
void BuildUtil::setProgram(Program *program)
{
prog = program;
}
 
void
BuildUtil::setPosition(BasicBlock *block, bool atTail)
{
bb = block;
prog = bb->getProgram();
func = bb->getFunction();
pos = NULL;
tail = atTail;
}
 
void
BuildUtil::setPosition(Instruction *i, bool after)
{
bb = i->bb;
prog = bb->getProgram();
func = bb->getFunction();
pos = i;
tail = after;
assert(bb);
}
 
LValue *
BuildUtil::getScratch(int size, DataFile f)
{
LValue *lval = new_LValue(func, f);
lval->reg.size = size;
return lval;
}
 
LValue *
BuildUtil::getSSA(int size, DataFile f)
{
LValue *lval = new_LValue(func, f);
lval->ssa = 1;
lval->reg.size = size;
return lval;
}
 
void BuildUtil::insert(Instruction *i)
{
if (!pos) {
tail ? bb->insertTail(i) : bb->insertHead(i);
} else {
if (tail) {
bb->insertAfter(pos, i);
pos = i;
} else {
bb->insertBefore(pos, i);
}
}
}
 
Instruction *
BuildUtil::mkOp(operation op, DataType ty, Value *dst)
{
Instruction *insn = new_Instruction(func, op, ty);
insn->setDef(0, dst);
insert(insn);
if (op == OP_DISCARD || op == OP_EXIT ||
op == OP_JOIN ||
op == OP_QUADON || op == OP_QUADPOP ||
op == OP_EMIT || op == OP_RESTART)
insn->fixed = 1;
return insn;
}
 
inline LValue *
BuildUtil::mkOp1v(operation op, DataType ty, Value *dst, Value *src)
{
mkOp1(op, ty, dst, src);
return dst->asLValue();
}
 
inline LValue *
BuildUtil::mkOp2v(operation op, DataType ty, Value *dst,
Value *src0, Value *src1)
{
mkOp2(op, ty, dst, src0, src1);
return dst->asLValue();
}
 
inline LValue *
BuildUtil::mkOp3v(operation op, DataType ty, Value *dst,
Value *src0, Value *src1, Value *src2)
{
mkOp3(op, ty, dst, src0, src1, src2);
return dst->asLValue();
}
 
inline LValue *
BuildUtil::mkLoadv(DataType ty, Symbol *mem, Value *ptr)
{
LValue *dst = getScratch();
mkLoad(ty, dst, mem, ptr);
return dst;
}
 
bool
BuildUtil::DataArray::exists(ValueMap &m, unsigned int i, unsigned int c)
{
assert(i < arrayLen && c < vecDim);
return !regOnly || m.r.count(Location(array, arrayIdx, i, c));
}
 
Value *
BuildUtil::DataArray::lookup(ValueMap &m, unsigned i, unsigned c)
{
ValueMap::r_iterator it = m.r.find(Location(array, arrayIdx, i, c));
return it != m.r.end() ? it->second : NULL;
}
 
Value *
BuildUtil::DataArray::insert(ValueMap &m, unsigned i, unsigned c, Value *v)
{
m.insert(Location(array, arrayIdx, i, c), v);
return v;
}
 
} // namespace nv50_ir
 
#endif // __NV50_IR_BUILD_UTIL_H__
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_driver.h
0,0 → 1,220
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_DRIVER_H__
#define __NV50_IR_DRIVER_H__
 
#include "pipe/p_shader_tokens.h"
 
#include "tgsi/tgsi_util.h"
#include "tgsi/tgsi_parse.h"
#include "tgsi/tgsi_scan.h"
 
/*
* This struct constitutes linkage information in TGSI terminology.
*
* It is created by the code generator and handed to the pipe driver
* for input/output slot assignment.
*/
struct nv50_ir_varying
{
uint8_t slot[4]; /* native slots for xyzw (addresses in 32-bit words) */
 
unsigned mask : 4; /* vec4 mask */
unsigned linear : 1; /* linearly interpolated if true (and not flat) */
unsigned flat : 1;
unsigned sc : 1; /* special colour interpolation mode (SHADE_MODEL) */
unsigned centroid : 1;
unsigned patch : 1; /* patch constant value */
unsigned regular : 1; /* driver-specific meaning (e.g. input in sreg) */
unsigned input : 1; /* indicates direction of system values */
unsigned oread : 1; /* true if output is read from parallel TCP */
 
ubyte id; /* TGSI register index */
ubyte sn; /* TGSI semantic name */
ubyte si; /* TGSI semantic index */
};
 
#define NV50_PROGRAM_IR_TGSI 0
#define NV50_PROGRAM_IR_SM4 1
#define NV50_PROGRAM_IR_GLSL 2
#define NV50_PROGRAM_IR_LLVM 3
 
#ifdef DEBUG
# define NV50_IR_DEBUG_BASIC (1 << 0)
# define NV50_IR_DEBUG_VERBOSE (2 << 0)
# define NV50_IR_DEBUG_REG_ALLOC (1 << 2)
#else
# define NV50_IR_DEBUG_BASIC 0
# define NV50_IR_DEBUG_VERBOSE 0
# define NV50_IR_DEBUG_REG_ALLOC 0
#endif
 
#define NV50_SEMANTIC_CLIPDISTANCE (TGSI_SEMANTIC_COUNT + 0)
#define NV50_SEMANTIC_VIEWPORTINDEX (TGSI_SEMANTIC_COUNT + 4)
#define NV50_SEMANTIC_LAYER (TGSI_SEMANTIC_COUNT + 5)
#define NV50_SEMANTIC_INVOCATIONID (TGSI_SEMANTIC_COUNT + 6)
#define NV50_SEMANTIC_TESSFACTOR (TGSI_SEMANTIC_COUNT + 7)
#define NV50_SEMANTIC_TESSCOORD (TGSI_SEMANTIC_COUNT + 8)
#define NV50_SEMANTIC_SAMPLEMASK (TGSI_SEMANTIC_COUNT + 9)
#define NV50_SEMANTIC_COUNT (TGSI_SEMANTIC_COUNT + 10)
 
#define NV50_TESS_PART_FRACT_ODD 0
#define NV50_TESS_PART_FRACT_EVEN 1
#define NV50_TESS_PART_POW2 2
#define NV50_TESS_PART_INTEGER 3
 
#define NV50_PRIM_PATCHES PIPE_PRIM_MAX
 
struct nv50_ir_prog_symbol
{
uint32_t label;
uint32_t offset;
};
 
#define NVISA_GF100_CHIPSET_C0 0xc0
#define NVISA_GF100_CHIPSET_D0 0xd0
#define NVISA_GK104_CHIPSET 0xe0
#define NVISA_GK110_CHIPSET 0xf0
 
struct nv50_ir_prog_info
{
uint16_t target; /* chipset (0x50, 0x84, 0xc0, ...) */
 
uint8_t type; /* PIPE_SHADER */
 
uint8_t optLevel; /* optimization level (0 to 3) */
uint8_t dbgFlags;
 
struct {
int16_t maxGPR; /* may be -1 if none used */
int16_t maxOutput;
uint32_t tlsSpace; /* required local memory per thread */
uint32_t *code;
uint32_t codeSize;
uint8_t sourceRep; /* NV50_PROGRAM_IR */
const void *source;
void *relocData;
struct nv50_ir_prog_symbol *syms;
uint16_t numSyms;
} bin;
 
struct nv50_ir_varying sv[PIPE_MAX_SHADER_INPUTS];
struct nv50_ir_varying in[PIPE_MAX_SHADER_INPUTS];
struct nv50_ir_varying out[PIPE_MAX_SHADER_OUTPUTS];
uint8_t numInputs;
uint8_t numOutputs;
uint8_t numPatchConstants; /* also included in numInputs/numOutputs */
uint8_t numSysVals;
 
struct {
uint32_t *buf; /* for IMMEDIATE_ARRAY */
uint16_t bufSize; /* size of immediate array */
uint16_t count; /* count of inline immediates */
uint32_t *data; /* inline immediate data */
uint8_t *type; /* for each vec4 (128 bit) */
} immd;
 
union {
struct {
uint32_t inputMask[4]; /* mask of attributes read (1 bit per scalar) */
} vp;
struct {
uint8_t inputPatchSize;
uint8_t outputPatchSize;
uint8_t partitioning; /* PIPE_TESS_PART */
int8_t winding; /* +1 (clockwise) / -1 (counter-clockwise) */
uint8_t domain; /* PIPE_PRIM_{QUADS,TRIANGLES,LINES} */
uint8_t outputPrim; /* PIPE_PRIM_{TRIANGLES,LINES,POINTS} */
} tp;
struct {
uint8_t inputPrim;
uint8_t outputPrim;
unsigned instanceCount;
unsigned maxVertices;
} gp;
struct {
unsigned numColourResults;
boolean writesDepth;
boolean earlyFragTests;
boolean separateFragData;
boolean usesDiscard;
} fp;
struct {
uint32_t inputOffset; /* base address for user args */
uint32_t sharedOffset; /* reserved space in s[] */
uint32_t gridInfoBase; /* base address for NTID,NCTAID */
} cp;
} prop;
 
uint8_t numBarriers;
 
struct {
uint8_t clipDistance; /* index of first clip distance output */
uint8_t clipDistanceMask; /* mask of clip distances defined */
uint8_t cullDistanceMask; /* clip distance mode (1 bit per output) */
int8_t genUserClip; /* request user clip planes for ClipVertex */
uint16_t ucpBase; /* base address for UCPs */
uint8_t ucpCBSlot; /* constant buffer index of UCP data */
uint8_t pointSize; /* output index for PointSize */
uint8_t instanceId; /* system value index of InstanceID */
uint8_t vertexId; /* system value index of VertexID */
uint8_t edgeFlagIn;
uint8_t edgeFlagOut;
uint8_t fragDepth; /* output index of FragDepth */
uint8_t sampleMask; /* output index of SampleMask */
uint8_t backFaceColor[2]; /* input/output indices of back face colour */
uint8_t globalAccess; /* 1 for read, 2 for wr, 3 for rw */
boolean nv50styleSurfaces; /* generate gX[] access for raw buffers */
uint8_t resInfoCBSlot; /* cX[] used for tex handles, surface info */
uint16_t texBindBase; /* base address for tex handles (nve4) */
uint16_t suInfoBase; /* base address for surface info (nve4) */
uint8_t msInfoCBSlot; /* cX[] used for multisample info */
uint16_t msInfoBase; /* base address for multisample info */
} io;
 
/* driver callback to assign input/output locations */
int (*assignSlots)(struct nv50_ir_prog_info *);
 
void *driverPriv;
};
 
#ifdef __cplusplus
extern "C" {
#endif
 
extern int nv50_ir_generate_code(struct nv50_ir_prog_info *);
 
extern void nv50_ir_relocate_code(void *relocData, uint32_t *code,
uint32_t codePos,
uint32_t libPos,
uint32_t dataPos);
 
/* obtain code that will be shared among programs */
extern void nv50_ir_get_target_library(uint32_t chipset,
const uint32_t **code, uint32_t *size);
 
#ifdef __cplusplus
}
#endif
 
#endif // __NV50_IR_DRIVER_H__
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp
0,0 → 1,1962
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_target_nv50.h"
 
namespace nv50_ir {
 
#define NV50_OP_ENC_LONG 0
#define NV50_OP_ENC_SHORT 1
#define NV50_OP_ENC_IMM 2
#define NV50_OP_ENC_LONG_ALT 3
 
class CodeEmitterNV50 : public CodeEmitter
{
public:
CodeEmitterNV50(const TargetNV50 *);
 
virtual bool emitInstruction(Instruction *);
 
virtual uint32_t getMinEncodingSize(const Instruction *) const;
 
inline void setProgramType(Program::Type pType) { progType = pType; }
 
virtual void prepareEmission(Function *);
 
private:
Program::Type progType;
 
const TargetNV50 *targNV50;
 
private:
inline void defId(const ValueDef&, const int pos);
inline void srcId(const ValueRef&, const int pos);
inline void srcId(const ValueRef *, const int pos);
 
inline void srcAddr16(const ValueRef&, bool adj, const int pos);
inline void srcAddr8(const ValueRef&, const int pos);
 
void emitFlagsRd(const Instruction *);
void emitFlagsWr(const Instruction *);
 
void emitCondCode(CondCode cc, DataType ty, int pos);
 
inline void setARegBits(unsigned int);
 
void setAReg16(const Instruction *, int s);
void setImmediate(const Instruction *, int s);
 
void setDst(const Value *);
void setDst(const Instruction *, int d);
void setSrcFileBits(const Instruction *, int enc);
void setSrc(const Instruction *, unsigned int s, int slot);
 
void emitForm_MAD(const Instruction *);
void emitForm_ADD(const Instruction *);
void emitForm_MUL(const Instruction *);
void emitForm_IMM(const Instruction *);
 
void emitLoadStoreSizeLG(DataType ty, int pos);
void emitLoadStoreSizeCS(DataType ty);
 
void roundMode_MAD(const Instruction *);
void roundMode_CVT(RoundMode);
 
void emitMNeg12(const Instruction *);
 
void emitLOAD(const Instruction *);
void emitSTORE(const Instruction *);
void emitMOV(const Instruction *);
void emitNOP();
void emitINTERP(const Instruction *);
void emitPFETCH(const Instruction *);
void emitOUT(const Instruction *);
 
void emitUADD(const Instruction *);
void emitAADD(const Instruction *);
void emitFADD(const Instruction *);
void emitIMUL(const Instruction *);
void emitFMUL(const Instruction *);
void emitFMAD(const Instruction *);
void emitIMAD(const Instruction *);
void emitISAD(const Instruction *);
 
void emitMINMAX(const Instruction *);
 
void emitPreOp(const Instruction *);
void emitSFnOp(const Instruction *, uint8_t subOp);
 
void emitShift(const Instruction *);
void emitARL(const Instruction *, unsigned int shl);
void emitLogicOp(const Instruction *);
void emitNOT(const Instruction *);
 
void emitCVT(const Instruction *);
void emitSET(const Instruction *);
 
void emitTEX(const TexInstruction *);
void emitTXQ(const TexInstruction *);
void emitTEXPREP(const TexInstruction *);
 
void emitQUADOP(const Instruction *, uint8_t lane, uint8_t quOp);
 
void emitFlow(const Instruction *, uint8_t flowOp);
void emitPRERETEmu(const FlowInstruction *);
void emitBAR(const Instruction *);
 
void emitATOM(const Instruction *);
};
 
#define SDATA(a) ((a).rep()->reg.data)
#define DDATA(a) ((a).rep()->reg.data)
 
void CodeEmitterNV50::srcId(const ValueRef& src, const int pos)
{
assert(src.get());
code[pos / 32] |= SDATA(src).id << (pos % 32);
}
 
void CodeEmitterNV50::srcId(const ValueRef *src, const int pos)
{
assert(src->get());
code[pos / 32] |= SDATA(*src).id << (pos % 32);
}
 
void CodeEmitterNV50::srcAddr16(const ValueRef& src, bool adj, const int pos)
{
assert(src.get());
 
int32_t offset = SDATA(src).offset;
 
assert(!adj || src.get()->reg.size <= 4);
if (adj)
offset /= src.get()->reg.size;
 
assert(offset <= 0x7fff && offset >= (int32_t)-0x8000 && (pos % 32) <= 16);
 
if (offset < 0)
offset &= adj ? (0xffff >> (src.get()->reg.size >> 1)) : 0xffff;
 
code[pos / 32] |= offset << (pos % 32);
}
 
void CodeEmitterNV50::srcAddr8(const ValueRef& src, const int pos)
{
assert(src.get());
 
uint32_t offset = SDATA(src).offset;
 
assert((offset <= 0x1fc || offset == 0x3fc) && !(offset & 0x3));
 
code[pos / 32] |= (offset >> 2) << (pos % 32);
}
 
void CodeEmitterNV50::defId(const ValueDef& def, const int pos)
{
assert(def.get() && def.getFile() != FILE_SHADER_OUTPUT);
 
code[pos / 32] |= DDATA(def).id << (pos % 32);
}
 
void
CodeEmitterNV50::roundMode_MAD(const Instruction *insn)
{
switch (insn->rnd) {
case ROUND_M: code[1] |= 1 << 22; break;
case ROUND_P: code[1] |= 2 << 22; break;
case ROUND_Z: code[1] |= 3 << 22; break;
default:
assert(insn->rnd == ROUND_N);
break;
}
}
 
void
CodeEmitterNV50::emitMNeg12(const Instruction *i)
{
code[1] |= i->src(0).mod.neg() << 26;
code[1] |= i->src(1).mod.neg() << 27;
}
 
void CodeEmitterNV50::emitCondCode(CondCode cc, DataType ty, int pos)
{
uint8_t enc;
 
assert(pos >= 32 || pos <= 27);
 
switch (cc) {
case CC_LT: enc = 0x1; break;
case CC_LTU: enc = 0x9; break;
case CC_EQ: enc = 0x2; break;
case CC_EQU: enc = 0xa; break;
case CC_LE: enc = 0x3; break;
case CC_LEU: enc = 0xb; break;
case CC_GT: enc = 0x4; break;
case CC_GTU: enc = 0xc; break;
case CC_NE: enc = 0x5; break;
case CC_NEU: enc = 0xd; break;
case CC_GE: enc = 0x6; break;
case CC_GEU: enc = 0xe; break;
case CC_TR: enc = 0xf; break;
case CC_FL: enc = 0x0; break;
 
case CC_O: enc = 0x10; break;
case CC_C: enc = 0x11; break;
case CC_A: enc = 0x12; break;
case CC_S: enc = 0x13; break;
case CC_NS: enc = 0x1c; break;
case CC_NA: enc = 0x1d; break;
case CC_NC: enc = 0x1e; break;
case CC_NO: enc = 0x1f; break;
 
default:
enc = 0;
assert(!"invalid condition code");
break;
}
if (ty != TYPE_NONE && !isFloatType(ty))
enc &= ~0x8; // unordered only exists for float types
 
code[pos / 32] |= enc << (pos % 32);
}
 
void
CodeEmitterNV50::emitFlagsRd(const Instruction *i)
{
int s = (i->flagsSrc >= 0) ? i->flagsSrc : i->predSrc;
 
assert(!(code[1] & 0x00003f80));
 
if (s >= 0) {
assert(i->getSrc(s)->reg.file == FILE_FLAGS);
emitCondCode(i->cc, TYPE_NONE, 32 + 7);
srcId(i->src(s), 32 + 12);
} else {
code[1] |= 0x0780;
}
}
 
void
CodeEmitterNV50::emitFlagsWr(const Instruction *i)
{
assert(!(code[1] & 0x70));
 
int flagsDef = i->flagsDef;
 
// find flags definition and check that it is the last def
if (flagsDef < 0) {
for (int d = 0; i->defExists(d); ++d)
if (i->def(d).getFile() == FILE_FLAGS)
flagsDef = d;
if (flagsDef >= 0 && 0) // TODO: enforce use of flagsDef at some point
WARN("Instruction::flagsDef was not set properly\n");
}
if (flagsDef == 0 && i->defExists(1))
WARN("flags def should not be the primary definition\n");
 
if (flagsDef >= 0)
code[1] |= (DDATA(i->def(flagsDef)).id << 4) | 0x40;
 
}
 
void
CodeEmitterNV50::setARegBits(unsigned int u)
{
code[0] |= (u & 3) << 26;
code[1] |= (u & 4);
}
 
void
CodeEmitterNV50::setAReg16(const Instruction *i, int s)
{
if (i->srcExists(s)) {
s = i->src(s).indirect[0];
if (s >= 0)
setARegBits(SDATA(i->src(s)).id + 1);
}
}
 
void
CodeEmitterNV50::setImmediate(const Instruction *i, int s)
{
const ImmediateValue *imm = i->src(s).get()->asImm();
assert(imm);
 
uint32_t u = imm->reg.data.u32;
 
if (i->src(s).mod & Modifier(NV50_IR_MOD_NOT))
u = ~u;
 
code[1] |= 3;
code[0] |= (u & 0x3f) << 16;
code[1] |= (u >> 6) << 2;
}
 
void
CodeEmitterNV50::setDst(const Value *dst)
{
const Storage *reg = &dst->join->reg;
 
assert(reg->file != FILE_ADDRESS);
 
if (reg->data.id < 0 || reg->file == FILE_FLAGS) {
code[0] |= (127 << 2) | 1;
code[1] |= 8;
} else {
int id;
if (reg->file == FILE_SHADER_OUTPUT) {
code[1] |= 8;
id = reg->data.offset / 4;
} else {
id = reg->data.id;
}
code[0] |= id << 2;
}
}
 
void
CodeEmitterNV50::setDst(const Instruction *i, int d)
{
if (i->defExists(d)) {
setDst(i->getDef(d));
} else
if (!d) {
code[0] |= 0x01fc; // bit bucket
code[1] |= 0x0008;
}
}
 
// 3 * 2 bits:
// 0: r
// 1: a/s
// 2: c
// 3: i
void
CodeEmitterNV50::setSrcFileBits(const Instruction *i, int enc)
{
uint8_t mode = 0;
 
for (unsigned int s = 0; s < Target::operationSrcNr[i->op]; ++s) {
switch (i->src(s).getFile()) {
case FILE_GPR:
break;
case FILE_MEMORY_SHARED:
case FILE_SHADER_INPUT:
mode |= 1 << (s * 2);
break;
case FILE_MEMORY_CONST:
mode |= 2 << (s * 2);
break;
case FILE_IMMEDIATE:
mode |= 3 << (s * 2);
break;
default:
ERROR("invalid file on source %i: %u\n", s, i->src(s).getFile());
assert(0);
break;
}
}
switch (mode) {
case 0x00: // rrr
break;
case 0x01: // arr/grr
if (progType == Program::TYPE_GEOMETRY) {
code[0] |= 0x01800000;
if (enc == NV50_OP_ENC_LONG || enc == NV50_OP_ENC_LONG_ALT)
code[1] |= 0x00200000;
} else {
if (enc == NV50_OP_ENC_SHORT)
code[0] |= 0x01000000;
else
code[1] |= 0x00200000;
}
break;
case 0x03: // irr
assert(i->op == OP_MOV);
return;
case 0x0c: // rir
break;
case 0x0d: // gir
code[0] |= 0x01000000;
assert(progType == Program::TYPE_GEOMETRY ||
progType == Program::TYPE_COMPUTE);
break;
case 0x08: // rcr
code[0] |= (enc == NV50_OP_ENC_LONG_ALT) ? 0x01000000 : 0x00800000;
code[1] |= (i->getSrc(1)->reg.fileIndex << 22);
break;
case 0x09: // acr/gcr
if (progType == Program::TYPE_GEOMETRY) {
code[0] |= 0x01800000;
} else {
code[0] |= (enc == NV50_OP_ENC_LONG_ALT) ? 0x01000000 : 0x00800000;
code[1] |= 0x00200000;
}
code[1] |= (i->getSrc(1)->reg.fileIndex << 22);
break;
case 0x20: // rrc
code[0] |= 0x01000000;
code[1] |= (i->getSrc(2)->reg.fileIndex << 22);
break;
case 0x21: // arc
code[0] |= 0x01000000;
code[1] |= 0x00200000 | (i->getSrc(2)->reg.fileIndex << 22);
assert(progType != Program::TYPE_GEOMETRY);
break;
default:
ERROR("not encodable: %x\n", mode);
assert(0);
break;
}
if (progType != Program::TYPE_COMPUTE)
return;
 
if ((mode & 3) == 1) {
const int pos = i->src(1).getFile() == FILE_IMMEDIATE ? 13 : 14;
 
switch (i->getSrc(0)->reg.type) {
case TYPE_U8:
break;
case TYPE_U16:
code[0] |= 1 << pos;
break;
case TYPE_S16:
code[0] |= 2 << pos;
break;
default:
code[0] |= 3 << pos;
assert(i->getSrc(0)->reg.size == 4);
break;
}
}
}
 
void
CodeEmitterNV50::setSrc(const Instruction *i, unsigned int s, int slot)
{
if (Target::operationSrcNr[i->op] <= s)
return;
const Storage *reg = &i->src(s).rep()->reg;
 
unsigned int id = (reg->file == FILE_GPR) ?
reg->data.id :
reg->data.offset >> (reg->size >> 1); // no > 4 byte sources here
 
switch (slot) {
case 0: code[0] |= id << 9; break;
case 1: code[0] |= id << 16; break;
case 2: code[1] |= id << 14; break;
default:
assert(0);
break;
}
}
 
// the default form:
// - long instruction
// - 1 to 3 sources in slots 0, 1, 2 (rrr, arr, rcr, acr, rrc, arc, gcr, grr)
// - address & flags
void
CodeEmitterNV50::emitForm_MAD(const Instruction *i)
{
assert(i->encSize == 8);
code[0] |= 1;
 
emitFlagsRd(i);
emitFlagsWr(i);
 
setDst(i, 0);
 
setSrcFileBits(i, NV50_OP_ENC_LONG);
setSrc(i, 0, 0);
setSrc(i, 1, 1);
setSrc(i, 2, 2);
 
setAReg16(i, 1);
}
 
// like default form, but 2nd source in slot 2, and no 3rd source
void
CodeEmitterNV50::emitForm_ADD(const Instruction *i)
{
assert(i->encSize == 8);
code[0] |= 1;
 
emitFlagsRd(i);
emitFlagsWr(i);
 
setDst(i, 0);
 
setSrcFileBits(i, NV50_OP_ENC_LONG_ALT);
setSrc(i, 0, 0);
setSrc(i, 1, 2);
 
setAReg16(i, 1);
}
 
// default short form (rr, ar, rc, gr)
void
CodeEmitterNV50::emitForm_MUL(const Instruction *i)
{
assert(i->encSize == 4 && !(code[0] & 1));
assert(i->defExists(0));
assert(!i->getPredicate());
 
setDst(i, 0);
 
setSrcFileBits(i, NV50_OP_ENC_SHORT);
setSrc(i, 0, 0);
setSrc(i, 1, 1);
}
 
// usual immediate form
// - 1 to 3 sources where last is immediate (rir, gir)
// - no address or predicate possible
void
CodeEmitterNV50::emitForm_IMM(const Instruction *i)
{
assert(i->encSize == 8);
code[0] |= 1;
 
assert(i->defExists(0) && i->srcExists(0));
 
setDst(i, 0);
 
setSrcFileBits(i, NV50_OP_ENC_IMM);
if (Target::operationSrcNr[i->op] > 1) {
setSrc(i, 0, 0);
setImmediate(i, 1);
setSrc(i, 2, 1);
} else {
setImmediate(i, 0);
}
}
 
void
CodeEmitterNV50::emitLoadStoreSizeLG(DataType ty, int pos)
{
uint8_t enc;
 
switch (ty) {
case TYPE_F32: // fall through
case TYPE_S32: // fall through
case TYPE_U32: enc = 0x6; break;
case TYPE_B128: enc = 0x5; break;
case TYPE_F64: // fall through
case TYPE_S64: // fall through
case TYPE_U64: enc = 0x4; break;
case TYPE_S16: enc = 0x3; break;
case TYPE_U16: enc = 0x2; break;
case TYPE_S8: enc = 0x1; break;
case TYPE_U8: enc = 0x0; break;
default:
enc = 0;
assert(!"invalid load/store type");
break;
}
code[pos / 32] |= enc << (pos % 32);
}
 
void
CodeEmitterNV50::emitLoadStoreSizeCS(DataType ty)
{
switch (ty) {
case TYPE_U8: break;
case TYPE_U16: code[1] |= 0x4000; break;
case TYPE_S16: code[1] |= 0x8000; break;
case TYPE_F32:
case TYPE_S32:
case TYPE_U32: code[1] |= 0xc000; break;
default:
assert(0);
break;
}
}
 
void
CodeEmitterNV50::emitLOAD(const Instruction *i)
{
DataFile sf = i->src(0).getFile();
int32_t offset = i->getSrc(0)->reg.data.offset;
 
switch (sf) {
case FILE_SHADER_INPUT:
// use 'mov' where we can
code[0] = i->src(0).isIndirect(0) ? 0x00000001 : 0x10000001;
code[1] = 0x00200000 | (i->lanes << 14);
if (typeSizeof(i->dType) == 4)
code[1] |= 0x04000000;
break;
case FILE_MEMORY_SHARED:
if (targ->getChipset() >= 0x84) {
assert(offset <= (int32_t)(0x3fff * typeSizeof(i->sType)));
code[0] = 0x10000001;
code[1] = 0x40000000;
 
if (typeSizeof(i->dType) == 4)
code[1] |= 0x04000000;
 
emitLoadStoreSizeCS(i->sType);
} else {
assert(offset <= (int32_t)(0x1f * typeSizeof(i->sType)));
code[0] = 0x10000001;
code[1] = 0x00200000 | (i->lanes << 14);
emitLoadStoreSizeCS(i->sType);
}
break;
case FILE_MEMORY_CONST:
code[0] = 0x10000001;
code[1] = 0x20000000 | (i->getSrc(0)->reg.fileIndex << 22);
if (typeSizeof(i->dType) == 4)
code[1] |= 0x04000000;
emitLoadStoreSizeCS(i->sType);
break;
case FILE_MEMORY_LOCAL:
code[0] = 0xd0000001;
code[1] = 0x40000000;
break;
case FILE_MEMORY_GLOBAL:
code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16);
code[1] = 0x80000000;
break;
default:
assert(!"invalid load source file");
break;
}
if (sf == FILE_MEMORY_LOCAL ||
sf == FILE_MEMORY_GLOBAL)
emitLoadStoreSizeLG(i->sType, 21 + 32);
 
setDst(i, 0);
 
emitFlagsRd(i);
emitFlagsWr(i);
 
if (i->src(0).getFile() == FILE_MEMORY_GLOBAL) {
srcId(*i->src(0).getIndirect(0), 9);
} else {
setAReg16(i, 0);
srcAddr16(i->src(0), i->src(0).getFile() != FILE_MEMORY_LOCAL, 9);
}
}
 
void
CodeEmitterNV50::emitSTORE(const Instruction *i)
{
DataFile f = i->getSrc(0)->reg.file;
int32_t offset = i->getSrc(0)->reg.data.offset;
 
switch (f) {
case FILE_SHADER_OUTPUT:
code[0] = 0x00000001 | ((offset >> 2) << 9);
code[1] = 0x80c00000;
srcId(i->src(1), 32 + 14);
break;
case FILE_MEMORY_GLOBAL:
code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16);
code[1] = 0xa0000000;
emitLoadStoreSizeLG(i->dType, 21 + 32);
srcId(i->src(1), 2);
break;
case FILE_MEMORY_LOCAL:
code[0] = 0xd0000001;
code[1] = 0x60000000;
emitLoadStoreSizeLG(i->dType, 21 + 32);
srcId(i->src(1), 2);
break;
case FILE_MEMORY_SHARED:
code[0] = 0x00000001;
code[1] = 0xe0000000;
switch (typeSizeof(i->dType)) {
case 1:
code[0] |= offset << 9;
code[1] |= 0x00400000;
break;
case 2:
code[0] |= (offset >> 1) << 9;
break;
case 4:
code[0] |= (offset >> 2) << 9;
code[1] |= 0x04200000;
break;
default:
assert(0);
break;
}
srcId(i->src(1), 32 + 14);
break;
default:
assert(!"invalid store destination file");
break;
}
 
if (f == FILE_MEMORY_GLOBAL)
srcId(*i->src(0).getIndirect(0), 9);
else
setAReg16(i, 0);
 
if (f == FILE_MEMORY_LOCAL)
srcAddr16(i->src(0), false, 9);
 
emitFlagsRd(i);
}
 
void
CodeEmitterNV50::emitMOV(const Instruction *i)
{
DataFile sf = i->getSrc(0)->reg.file;
DataFile df = i->getDef(0)->reg.file;
 
assert(sf == FILE_GPR || df == FILE_GPR);
 
if (sf == FILE_FLAGS) {
code[0] = 0x00000001;
code[1] = 0x20000000;
defId(i->def(0), 2);
srcId(i->src(0), 12);
emitFlagsRd(i);
} else
if (sf == FILE_ADDRESS) {
code[0] = 0x00000001;
code[1] = 0x40000000;
defId(i->def(0), 2);
setARegBits(SDATA(i->src(0)).id + 1);
emitFlagsRd(i);
} else
if (df == FILE_FLAGS) {
code[0] = 0x00000001;
code[1] = 0xa0000000;
defId(i->def(0), 4);
srcId(i->src(0), 9);
emitFlagsRd(i);
} else
if (sf == FILE_IMMEDIATE) {
code[0] = 0x10008001;
code[1] = 0x00000003;
emitForm_IMM(i);
} else {
if (i->encSize == 4) {
code[0] = 0x10008000;
} else {
code[0] = 0x10000001;
code[1] = (typeSizeof(i->dType) == 2) ? 0 : 0x04000000;
code[1] |= (i->lanes << 14);
emitFlagsRd(i);
}
defId(i->def(0), 2);
srcId(i->src(0), 9);
}
if (df == FILE_SHADER_OUTPUT) {
assert(i->encSize == 8);
code[1] |= 0x8;
}
}
 
void
CodeEmitterNV50::emitNOP()
{
code[0] = 0xf0000001;
code[1] = 0xe0000000;
}
 
void
CodeEmitterNV50::emitQUADOP(const Instruction *i, uint8_t lane, uint8_t quOp)
{
code[0] = 0xc0000000 | (lane << 16);
code[1] = 0x80000000;
 
code[0] |= (quOp & 0x03) << 20;
code[1] |= (quOp & 0xfc) << 20;
 
emitForm_ADD(i);
 
if (!i->srcExists(1))
srcId(i->src(0), 32 + 14);
}
 
void
CodeEmitterNV50::emitPFETCH(const Instruction *i)
{
code[0] = 0x11800001;
code[1] = 0x04200000 | (0xf << 14);
 
defId(i->def(0), 2);
srcAddr8(i->src(0), 9);
setAReg16(i, 0);
}
 
void
CodeEmitterNV50::emitINTERP(const Instruction *i)
{
code[0] = 0x80000000;
 
defId(i->def(0), 2);
srcAddr8(i->src(0), 16);
 
if (i->getInterpMode() == NV50_IR_INTERP_FLAT) {
code[0] |= 1 << 8;
} else {
if (i->op == OP_PINTERP) {
code[0] |= 1 << 25;
srcId(i->src(1), 9);
}
if (i->getSampleMode() == NV50_IR_INTERP_CENTROID)
code[0] |= 1 << 24;
}
 
if (i->encSize == 8) {
code[1] =
(code[0] & (3 << 24)) >> (24 - 16) |
(code[0] & (1 << 8)) << (18 - 8);
code[0] &= ~0x03000100;
code[0] |= 1;
emitFlagsRd(i);
}
}
 
void
CodeEmitterNV50::emitMINMAX(const Instruction *i)
{
if (i->dType == TYPE_F64) {
code[0] = 0xe0000000;
code[1] = (i->op == OP_MIN) ? 0xa0000000 : 0xc0000000;
} else {
code[0] = 0x30000000;
code[1] = 0x80000000;
if (i->op == OP_MIN)
code[1] |= 0x20000000;
 
switch (i->dType) {
case TYPE_F32: code[0] |= 0x80000000; break;
case TYPE_S32: code[1] |= 0x8c000000; break;
case TYPE_U32: code[1] |= 0x84000000; break;
case TYPE_S16: code[1] |= 0x80000000; break;
case TYPE_U16: break;
default:
assert(0);
break;
}
code[1] |= i->src(0).mod.abs() << 20;
code[1] |= i->src(1).mod.abs() << 19;
}
emitForm_MAD(i);
}
 
void
CodeEmitterNV50::emitFMAD(const Instruction *i)
{
const int neg_mul = i->src(0).mod.neg() ^ i->src(1).mod.neg();
const int neg_add = i->src(2).mod.neg();
 
code[0] = 0xe0000000;
 
if (i->encSize == 4) {
emitForm_MUL(i);
assert(!neg_mul && !neg_add);
} else {
code[1] = neg_mul << 26;
code[1] |= neg_add << 27;
if (i->saturate)
code[1] |= 1 << 29;
emitForm_MAD(i);
}
}
 
void
CodeEmitterNV50::emitFADD(const Instruction *i)
{
const int neg0 = i->src(0).mod.neg();
const int neg1 = i->src(1).mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0);
 
code[0] = 0xb0000000;
 
assert(!(i->src(0).mod | i->src(1).mod).abs());
 
if (i->src(1).getFile() == FILE_IMMEDIATE) {
code[1] = 0;
emitForm_IMM(i);
code[0] |= neg0 << 15;
code[0] |= neg1 << 22;
if (i->saturate)
code[0] |= 1 << 8;
} else
if (i->encSize == 8) {
code[1] = 0;
emitForm_ADD(i);
code[1] |= neg0 << 26;
code[1] |= neg1 << 27;
if (i->saturate)
code[1] |= 1 << 29;
} else {
emitForm_MUL(i);
code[0] |= neg0 << 15;
code[0] |= neg1 << 22;
if (i->saturate)
code[0] |= 1 << 8;
}
}
 
void
CodeEmitterNV50::emitUADD(const Instruction *i)
{
const int neg0 = i->src(0).mod.neg();
const int neg1 = i->src(1).mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0);
 
code[0] = 0x20008000;
 
if (i->src(1).getFile() == FILE_IMMEDIATE) {
code[1] = 0;
emitForm_IMM(i);
} else
if (i->encSize == 8) {
code[0] = 0x20000000;
code[1] = (typeSizeof(i->dType) == 2) ? 0 : 0x04000000;
emitForm_ADD(i);
} else {
emitForm_MUL(i);
}
assert(!(neg0 && neg1));
code[0] |= neg0 << 28;
code[0] |= neg1 << 22;
 
if (i->flagsSrc >= 0) {
// addc == sub | subr
assert(!(code[0] & 0x10400000) && !i->getPredicate());
code[0] |= 0x10400000;
srcId(i->src(i->flagsSrc), 32 + 12);
}
}
 
void
CodeEmitterNV50::emitAADD(const Instruction *i)
{
const int s = (i->op == OP_MOV) ? 0 : 1;
 
code[0] = 0xd0000001 | (i->getSrc(s)->reg.data.u16 << 9);
code[1] = 0x20000000;
 
code[0] |= (DDATA(i->def(0)).id + 1) << 2;
 
emitFlagsRd(i);
 
if (s && i->srcExists(0))
setARegBits(SDATA(i->src(0)).id + 1);
}
 
void
CodeEmitterNV50::emitIMUL(const Instruction *i)
{
code[0] = 0x40000000;
 
if (i->encSize == 8) {
code[1] = (i->sType == TYPE_S16) ? (0x8000 | 0x4000) : 0x0000;
emitForm_MAD(i);
} else {
if (i->sType == TYPE_S16)
code[0] |= 0x8100;
emitForm_MUL(i);
}
}
 
void
CodeEmitterNV50::emitFMUL(const Instruction *i)
{
const int neg = (i->src(0).mod ^ i->src(1).mod).neg();
 
code[0] = 0xc0000000;
 
if (i->src(1).getFile() == FILE_IMMEDIATE) {
code[1] = 0;
emitForm_IMM(i);
if (neg)
code[0] |= 0x8000;
} else
if (i->encSize == 8) {
code[1] = i->rnd == ROUND_Z ? 0x0000c000 : 0;
if (neg)
code[1] |= 0x08000000;
emitForm_MAD(i);
} else {
emitForm_MUL(i);
if (neg)
code[0] |= 0x8000;
}
}
 
void
CodeEmitterNV50::emitIMAD(const Instruction *i)
{
code[0] = 0x60000000;
if (isSignedType(i->sType))
code[1] = i->saturate ? 0x40000000 : 0x20000000;
else
code[1] = 0x00000000;
 
int neg1 = i->src(0).mod.neg() ^ i->src(1).mod.neg();
int neg2 = i->src(2).mod.neg();
 
assert(!(neg1 & neg2));
code[1] |= neg1 << 27;
code[1] |= neg2 << 26;
 
emitForm_MAD(i);
 
if (i->flagsSrc >= 0) {
// add with carry from $cX
assert(!(code[1] & 0x0c000000) && !i->getPredicate());
code[1] |= 0xc << 24;
srcId(i->src(i->flagsSrc), 32 + 12);
}
}
 
void
CodeEmitterNV50::emitISAD(const Instruction *i)
{
if (i->encSize == 8) {
code[0] = 0x50000000;
switch (i->sType) {
case TYPE_U32: code[1] = 0x04000000; break;
case TYPE_S32: code[1] = 0x0c000000; break;
case TYPE_U16: code[1] = 0x00000000; break;
case TYPE_S16: code[1] = 0x08000000; break;
default:
assert(0);
break;
}
emitForm_MAD(i);
} else {
switch (i->sType) {
case TYPE_U32: code[0] = 0x50008000; break;
case TYPE_S32: code[0] = 0x50008100; break;
case TYPE_U16: code[0] = 0x50000000; break;
case TYPE_S16: code[0] = 0x50000100; break;
default:
assert(0);
break;
}
emitForm_MUL(i);
}
}
 
void
CodeEmitterNV50::emitSET(const Instruction *i)
{
code[0] = 0x30000000;
code[1] = 0x60000000;
 
emitCondCode(i->asCmp()->setCond, i->sType, 32 + 14);
 
switch (i->sType) {
case TYPE_F32: code[0] |= 0x80000000; break;
case TYPE_S32: code[1] |= 0x0c000000; break;
case TYPE_U32: code[1] |= 0x04000000; break;
case TYPE_S16: code[1] |= 0x08000000; break;
case TYPE_U16: break;
default:
assert(0);
break;
}
if (i->src(0).mod.neg()) code[1] |= 0x04000000;
if (i->src(1).mod.neg()) code[1] |= 0x08000000;
if (i->src(0).mod.abs()) code[1] |= 0x00100000;
if (i->src(1).mod.abs()) code[1] |= 0x00080000;
 
emitForm_MAD(i);
}
 
void
CodeEmitterNV50::roundMode_CVT(RoundMode rnd)
{
switch (rnd) {
case ROUND_NI: code[1] |= 0x08000000; break;
case ROUND_M: code[1] |= 0x00020000; break;
case ROUND_MI: code[1] |= 0x08020000; break;
case ROUND_P: code[1] |= 0x00040000; break;
case ROUND_PI: code[1] |= 0x08040000; break;
case ROUND_Z: code[1] |= 0x00060000; break;
case ROUND_ZI: code[1] |= 0x08060000; break;
default:
assert(rnd == ROUND_N);
break;
}
}
 
void
CodeEmitterNV50::emitCVT(const Instruction *i)
{
const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
RoundMode rnd;
 
switch (i->op) {
case OP_CEIL: rnd = f2f ? ROUND_PI : ROUND_P; break;
case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;
case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;
default:
rnd = i->rnd;
break;
}
 
code[0] = 0xa0000000;
 
switch (i->dType) {
case TYPE_F64:
switch (i->sType) {
case TYPE_F64: code[1] = 0xc4404000; break;
case TYPE_S64: code[1] = 0x44414000; break;
case TYPE_U64: code[1] = 0x44404000; break;
case TYPE_F32: code[1] = 0xc4400000; break;
case TYPE_S32: code[1] = 0x44410000; break;
case TYPE_U32: code[1] = 0x44400000; break;
default:
assert(0);
break;
}
break;
case TYPE_S64:
switch (i->sType) {
case TYPE_F64: code[1] = 0x8c404000; break;
case TYPE_F32: code[1] = 0x8c400000; break;
default:
assert(0);
break;
}
break;
case TYPE_U64:
switch (i->sType) {
case TYPE_F64: code[1] = 0x84404000; break;
case TYPE_F32: code[1] = 0x84400000; break;
default:
assert(0);
break;
}
break;
case TYPE_F32:
switch (i->sType) {
case TYPE_F64: code[1] = 0xc0404000; break;
case TYPE_S64: code[1] = 0x40414000; break;
case TYPE_U64: code[1] = 0x40404000; break;
case TYPE_F32: code[1] = 0xc4004000; break;
case TYPE_S32: code[1] = 0x44014000; break;
case TYPE_U32: code[1] = 0x44004000; break;
case TYPE_F16: code[1] = 0xc4000000; break;
default:
assert(0);
break;
}
break;
case TYPE_S32:
switch (i->sType) {
case TYPE_F64: code[1] = 0x88404000; break;
case TYPE_F32: code[1] = 0x8c004000; break;
case TYPE_S32: code[1] = 0x0c014000; break;
case TYPE_U32: code[1] = 0x0c004000; break;
case TYPE_F16: code[1] = 0x8c000000; break;
case TYPE_S16: code[1] = 0x0c010000; break;
case TYPE_U16: code[1] = 0x0c000000; break;
case TYPE_S8: code[1] = 0x0c018000; break;
case TYPE_U8: code[1] = 0x0c008000; break;
default:
assert(0);
break;
}
break;
case TYPE_U32:
switch (i->sType) {
case TYPE_F64: code[1] = 0x80404000; break;
case TYPE_F32: code[1] = 0x84004000; break;
case TYPE_S32: code[1] = 0x04014000; break;
case TYPE_U32: code[1] = 0x04004000; break;
case TYPE_F16: code[1] = 0x84000000; break;
case TYPE_S16: code[1] = 0x04010000; break;
case TYPE_U16: code[1] = 0x04000000; break;
case TYPE_S8: code[1] = 0x04018000; break;
case TYPE_U8: code[1] = 0x04008000; break;
default:
assert(0);
break;
}
break;
case TYPE_S16:
case TYPE_U16:
case TYPE_S8:
case TYPE_U8:
default:
assert(0);
break;
}
if (typeSizeof(i->sType) == 1 && i->getSrc(0)->reg.size == 4)
code[1] |= 0x00004000;
 
roundMode_CVT(rnd);
 
switch (i->op) {
case OP_ABS: code[1] |= 1 << 20; break;
case OP_SAT: code[1] |= 1 << 19; break;
case OP_NEG: code[1] |= 1 << 29; break;
default:
break;
}
code[1] ^= i->src(0).mod.neg() << 29;
code[1] |= i->src(0).mod.abs() << 20;
if (i->saturate)
code[1] |= 1 << 19;
 
assert(i->op != OP_ABS || !i->src(0).mod.neg());
 
emitForm_MAD(i);
}
 
void
CodeEmitterNV50::emitPreOp(const Instruction *i)
{
code[0] = 0xb0000000;
code[1] = (i->op == OP_PREEX2) ? 0xc0004000 : 0xc0000000;
 
code[1] |= i->src(0).mod.abs() << 20;
code[1] |= i->src(0).mod.neg() << 26;
 
emitForm_MAD(i);
}
 
void
CodeEmitterNV50::emitSFnOp(const Instruction *i, uint8_t subOp)
{
code[0] = 0x90000000;
 
if (i->encSize == 4) {
assert(i->op == OP_RCP);
code[0] |= i->src(0).mod.abs() << 15;
code[0] |= i->src(0).mod.neg() << 22;
emitForm_MUL(i);
} else {
code[1] = subOp << 29;
code[1] |= i->src(0).mod.abs() << 20;
code[1] |= i->src(0).mod.neg() << 26;
emitForm_MAD(i);
}
}
 
void
CodeEmitterNV50::emitNOT(const Instruction *i)
{
code[0] = 0xd0000000;
code[1] = 0x0002c000;
 
switch (i->sType) {
case TYPE_U32:
case TYPE_S32:
code[1] |= 0x04000000;
break;
default:
break;
}
emitForm_MAD(i);
setSrc(i, 0, 1);
}
 
void
CodeEmitterNV50::emitLogicOp(const Instruction *i)
{
code[0] = 0xd0000000;
code[1] = 0;
 
if (i->src(1).getFile() == FILE_IMMEDIATE) {
switch (i->op) {
case OP_OR: code[0] |= 0x0100; break;
case OP_XOR: code[0] |= 0x8000; break;
default:
assert(i->op == OP_AND);
break;
}
if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT))
code[0] |= 1 << 22;
 
emitForm_IMM(i);
} else {
switch (i->op) {
case OP_AND: code[1] = 0x04000000; break;
case OP_OR: code[1] = 0x04004000; break;
case OP_XOR: code[1] = 0x04008000; break;
default:
assert(0);
break;
}
if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT))
code[1] |= 1 << 16;
if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT))
code[1] |= 1 << 17;
 
emitForm_MAD(i);
}
}
 
void
CodeEmitterNV50::emitARL(const Instruction *i, unsigned int shl)
{
code[0] = 0x00000001 | (shl << 16);
code[1] = 0xc0000000;
 
code[0] |= (DDATA(i->def(0)).id + 1) << 2;
 
setSrcFileBits(i, NV50_OP_ENC_IMM);
setSrc(i, 0, 0);
emitFlagsRd(i);
}
 
void
CodeEmitterNV50::emitShift(const Instruction *i)
{
if (i->def(0).getFile() == FILE_ADDRESS) {
assert(i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE);
emitARL(i, i->getSrc(1)->reg.data.u32 & 0x3f);
} else {
code[0] = 0x30000001;
code[1] = (i->op == OP_SHR) ? 0xe4000000 : 0xc4000000;
if (i->op == OP_SHR && isSignedType(i->sType))
code[1] |= 1 << 27;
 
if (i->src(1).getFile() == FILE_IMMEDIATE) {
code[1] |= 1 << 20;
code[0] |= (i->getSrc(1)->reg.data.u32 & 0x7f) << 16;
defId(i->def(0), 2);
srcId(i->src(0), 9);
emitFlagsRd(i);
} else {
emitForm_MAD(i);
}
}
}
 
void
CodeEmitterNV50::emitOUT(const Instruction *i)
{
code[0] = (i->op == OP_EMIT) ? 0xf0000200 : 0xf0000400;
code[1] = 0xc0000001;
 
emitFlagsRd(i);
}
 
void
CodeEmitterNV50::emitTEX(const TexInstruction *i)
{
code[0] = 0xf0000001;
code[1] = 0x00000000;
 
switch (i->op) {
case OP_TXB:
code[1] = 0x20000000;
break;
case OP_TXL:
code[1] = 0x40000000;
break;
case OP_TXF:
code[0] |= 0x01000000;
break;
case OP_TXG:
code[0] = 0x01000000;
code[1] = 0x80000000;
break;
default:
assert(i->op == OP_TEX);
break;
}
 
code[0] |= i->tex.r << 9;
code[0] |= i->tex.s << 17;
 
int argc = i->tex.target.getArgCount();
 
if (i->op == OP_TXB || i->op == OP_TXL || i->op == OP_TXF)
argc += 1;
if (i->tex.target.isShadow())
argc += 1;
assert(argc <= 4);
 
code[0] |= (argc - 1) << 22;
 
if (i->tex.target.isCube()) {
code[0] |= 0x08000000;
} else
if (i->tex.useOffsets) {
code[1] |= (i->tex.offset[0][0] & 0xf) << 24;
code[1] |= (i->tex.offset[0][1] & 0xf) << 20;
code[1] |= (i->tex.offset[0][2] & 0xf) << 16;
}
 
code[0] |= (i->tex.mask & 0x3) << 25;
code[1] |= (i->tex.mask & 0xc) << 12;
 
if (i->tex.liveOnly)
code[1] |= 4;
 
defId(i->def(0), 2);
 
emitFlagsRd(i);
}
 
void
CodeEmitterNV50::emitTXQ(const TexInstruction *i)
{
assert(i->tex.query == TXQ_DIMS);
 
code[0] = 0xf0000001;
code[1] = 0x60000000;
 
code[0] |= i->tex.r << 9;
code[0] |= i->tex.s << 17;
 
code[0] |= (i->tex.mask & 0x3) << 25;
code[1] |= (i->tex.mask & 0xc) << 12;
 
defId(i->def(0), 2);
 
emitFlagsRd(i);
}
 
void
CodeEmitterNV50::emitTEXPREP(const TexInstruction *i)
{
code[0] = 0xf8000001 | (3 << 22) | (i->tex.s << 17) | (i->tex.r << 9);
code[1] = 0x60010000;
 
code[0] |= (i->tex.mask & 0x3) << 25;
code[1] |= (i->tex.mask & 0xc) << 12;
defId(i->def(0), 2);
 
emitFlagsRd(i);
}
 
void
CodeEmitterNV50::emitPRERETEmu(const FlowInstruction *i)
{
uint32_t pos = i->target.bb->binPos + 8; // +8 to skip an op */
 
code[0] = 0x10000003; // bra
code[1] = 0x00000780; // always
 
switch (i->subOp) {
case NV50_IR_SUBOP_EMU_PRERET + 0: // bra to the call
break;
case NV50_IR_SUBOP_EMU_PRERET + 1: // bra to skip the call
pos += 8;
break;
default:
assert(i->subOp == (NV50_IR_SUBOP_EMU_PRERET + 2));
code[0] = 0x20000003; // call
code[1] = 0x00000000; // no predicate
break;
}
addReloc(RelocEntry::TYPE_CODE, 0, pos, 0x07fff800, 9);
addReloc(RelocEntry::TYPE_CODE, 1, pos, 0x000fc000, -4);
}
 
void
CodeEmitterNV50::emitFlow(const Instruction *i, uint8_t flowOp)
{
const FlowInstruction *f = i->asFlow();
bool hasPred = false;
bool hasTarg = false;
 
code[0] = 0x00000003 | (flowOp << 28);
code[1] = 0x00000000;
 
switch (i->op) {
case OP_BRA:
hasPred = true;
hasTarg = true;
break;
case OP_BREAK:
case OP_BRKPT:
case OP_DISCARD:
case OP_RET:
hasPred = true;
break;
case OP_CALL:
case OP_PREBREAK:
case OP_JOINAT:
hasTarg = true;
break;
case OP_PRERET:
hasTarg = true;
if (i->subOp >= NV50_IR_SUBOP_EMU_PRERET) {
emitPRERETEmu(f);
return;
}
break;
default:
break;
}
 
if (hasPred)
emitFlagsRd(i);
 
if (hasTarg && f) {
uint32_t pos;
 
if (f->op == OP_CALL) {
if (f->builtin) {
pos = targNV50->getBuiltinOffset(f->target.builtin);
} else {
pos = f->target.fn->binPos;
}
} else {
pos = f->target.bb->binPos;
}
 
code[0] |= ((pos >> 2) & 0xffff) << 11;
code[1] |= ((pos >> 18) & 0x003f) << 14;
 
RelocEntry::Type relocTy;
 
relocTy = f->builtin ? RelocEntry::TYPE_BUILTIN : RelocEntry::TYPE_CODE;
 
addReloc(relocTy, 0, pos, 0x07fff800, 9);
addReloc(relocTy, 1, pos, 0x000fc000, -4);
}
}
 
void
CodeEmitterNV50::emitBAR(const Instruction *i)
{
ImmediateValue *barId = i->getSrc(0)->asImm();
assert(barId);
 
code[0] = 0x82000003 | (barId->reg.data.u32 << 21);
code[1] = 0x00004000;
 
if (i->subOp == NV50_IR_SUBOP_BAR_SYNC)
code[0] |= 1 << 26;
}
 
void
CodeEmitterNV50::emitATOM(const Instruction *i)
{
uint8_t subOp;
switch (i->subOp) {
case NV50_IR_SUBOP_ATOM_ADD: subOp = 0x0; break;
case NV50_IR_SUBOP_ATOM_MIN: subOp = 0x7; break;
case NV50_IR_SUBOP_ATOM_MAX: subOp = 0x6; break;
case NV50_IR_SUBOP_ATOM_INC: subOp = 0x4; break;
case NV50_IR_SUBOP_ATOM_DEC: subOp = 0x5; break;
case NV50_IR_SUBOP_ATOM_AND: subOp = 0xa; break;
case NV50_IR_SUBOP_ATOM_OR: subOp = 0xb; break;
case NV50_IR_SUBOP_ATOM_XOR: subOp = 0xc; break;
case NV50_IR_SUBOP_ATOM_CAS: subOp = 0x2; break;
case NV50_IR_SUBOP_ATOM_EXCH: subOp = 0x1; break;
default:
assert(!"invalid subop");
return;
}
code[0] = 0xd0000001;
code[1] = 0xe0c00000 | (subOp << 2);
if (isSignedType(i->dType))
code[1] |= 1 << 21;
 
// args
emitFlagsRd(i);
setDst(i, 0);
setSrc(i, 1, 1);
if (i->subOp == NV50_IR_SUBOP_ATOM_CAS)
setSrc(i, 2, 2);
 
// g[] pointer
code[0] |= i->getSrc(0)->reg.fileIndex << 23;
srcId(i->getIndirect(0, 0), 9);
}
 
bool
CodeEmitterNV50::emitInstruction(Instruction *insn)
{
if (!insn->encSize) {
ERROR("skipping unencodable instruction: "); insn->print();
return false;
} else
if (codeSize + insn->encSize > codeSizeLimit) {
ERROR("code emitter output buffer too small\n");
return false;
}
 
if (insn->bb->getProgram()->dbgFlags & NV50_IR_DEBUG_BASIC) {
INFO("EMIT: "); insn->print();
}
 
switch (insn->op) {
case OP_MOV:
emitMOV(insn);
break;
case OP_EXIT:
case OP_NOP:
case OP_JOIN:
emitNOP();
break;
case OP_VFETCH:
case OP_LOAD:
emitLOAD(insn);
break;
case OP_EXPORT:
case OP_STORE:
emitSTORE(insn);
break;
case OP_PFETCH:
emitPFETCH(insn);
break;
case OP_LINTERP:
case OP_PINTERP:
emitINTERP(insn);
break;
case OP_ADD:
case OP_SUB:
if (isFloatType(insn->dType))
emitFADD(insn);
else if (insn->getDef(0)->reg.file == FILE_ADDRESS)
emitAADD(insn);
else
emitUADD(insn);
break;
case OP_MUL:
if (isFloatType(insn->dType))
emitFMUL(insn);
else
emitIMUL(insn);
break;
case OP_MAD:
case OP_FMA:
if (isFloatType(insn->dType))
emitFMAD(insn);
else
emitIMAD(insn);
break;
case OP_SAD:
emitISAD(insn);
break;
case OP_NOT:
emitNOT(insn);
break;
case OP_AND:
case OP_OR:
case OP_XOR:
emitLogicOp(insn);
break;
case OP_SHL:
case OP_SHR:
emitShift(insn);
break;
case OP_SET:
emitSET(insn);
break;
case OP_MIN:
case OP_MAX:
emitMINMAX(insn);
break;
case OP_CEIL:
case OP_FLOOR:
case OP_TRUNC:
case OP_ABS:
case OP_NEG:
case OP_SAT:
emitCVT(insn);
break;
case OP_CVT:
if (insn->def(0).getFile() == FILE_ADDRESS)
emitARL(insn, 0);
else
if (insn->def(0).getFile() == FILE_FLAGS ||
insn->src(0).getFile() == FILE_FLAGS ||
insn->src(0).getFile() == FILE_ADDRESS)
emitMOV(insn);
else
emitCVT(insn);
break;
case OP_RCP:
emitSFnOp(insn, 0);
break;
case OP_RSQ:
emitSFnOp(insn, 2);
break;
case OP_LG2:
emitSFnOp(insn, 3);
break;
case OP_SIN:
emitSFnOp(insn, 4);
break;
case OP_COS:
emitSFnOp(insn, 5);
break;
case OP_EX2:
emitSFnOp(insn, 6);
break;
case OP_PRESIN:
case OP_PREEX2:
emitPreOp(insn);
break;
case OP_TEX:
case OP_TXB:
case OP_TXL:
case OP_TXF:
emitTEX(insn->asTex());
break;
case OP_TXQ:
emitTXQ(insn->asTex());
break;
case OP_TEXPREP:
emitTEXPREP(insn->asTex());
break;
case OP_EMIT:
case OP_RESTART:
emitOUT(insn);
break;
case OP_DISCARD:
emitFlow(insn, 0x0);
break;
case OP_BRA:
emitFlow(insn, 0x1);
break;
case OP_CALL:
emitFlow(insn, 0x2);
break;
case OP_RET:
emitFlow(insn, 0x3);
break;
case OP_PREBREAK:
emitFlow(insn, 0x4);
break;
case OP_BREAK:
emitFlow(insn, 0x5);
break;
case OP_QUADON:
emitFlow(insn, 0x6);
break;
case OP_QUADPOP:
emitFlow(insn, 0x7);
break;
case OP_JOINAT:
emitFlow(insn, 0xa);
break;
case OP_PRERET:
emitFlow(insn, 0xd);
break;
case OP_QUADOP:
emitQUADOP(insn, insn->lanes, insn->subOp);
break;
case OP_DFDX:
emitQUADOP(insn, 4, insn->src(0).mod.neg() ? 0x66 : 0x99);
break;
case OP_DFDY:
emitQUADOP(insn, 5, insn->src(0).mod.neg() ? 0x5a : 0xa5);
break;
case OP_ATOM:
emitATOM(insn);
break;
case OP_BAR:
emitBAR(insn);
break;
case OP_PHI:
case OP_UNION:
case OP_CONSTRAINT:
ERROR("operation should have been eliminated\n");
return false;
case OP_EXP:
case OP_LOG:
case OP_SQRT:
case OP_POW:
case OP_SELP:
case OP_SLCT:
case OP_TXD:
case OP_PRECONT:
case OP_CONT:
case OP_POPCNT:
case OP_INSBF:
case OP_EXTBF:
ERROR("operation should have been lowered\n");
return false;
default:
ERROR("unknown op: %u\n", insn->op);
return false;
}
if (insn->join || insn->op == OP_JOIN)
code[1] |= 0x2;
else
if (insn->exit || insn->op == OP_EXIT)
code[1] |= 0x1;
 
assert((insn->encSize == 8) == (code[0] & 1));
 
code += insn->encSize / 4;
codeSize += insn->encSize;
return true;
}
 
uint32_t
CodeEmitterNV50::getMinEncodingSize(const Instruction *i) const
{
const Target::OpInfo &info = targ->getOpInfo(i);
 
if (info.minEncSize > 4)
return 8;
 
// check constraints on dst and src operands
for (int d = 0; i->defExists(d); ++d) {
if (i->def(d).rep()->reg.data.id > 63 ||
i->def(d).rep()->reg.file != FILE_GPR)
return 8;
}
 
for (int s = 0; i->srcExists(s); ++s) {
DataFile sf = i->src(s).getFile();
if (sf != FILE_GPR)
if (sf != FILE_SHADER_INPUT || progType != Program::TYPE_FRAGMENT)
return 8;
if (i->src(s).rep()->reg.data.id > 63)
return 8;
}
 
// check modifiers & rounding
if (i->join || i->lanes != 0xf || i->exit)
return 8;
if (i->op == OP_MUL && i->rnd != ROUND_N)
return 8;
 
if (i->asTex())
return 8; // TODO: short tex encoding
 
// check constraints on short MAD
if (info.srcNr >= 2 && i->srcExists(2)) {
if (i->saturate || i->src(2).mod)
return 8;
if ((i->src(0).mod ^ i->src(1).mod) ||
(i->src(0).mod | i->src(1).mod).abs())
return 8;
if (!i->defExists(0) ||
i->def(0).rep()->reg.data.id != i->src(2).rep()->reg.data.id)
return 8;
}
 
return info.minEncSize;
}
 
// Change the encoding size of an instruction after BBs have been scheduled.
static void
makeInstructionLong(Instruction *insn)
{
if (insn->encSize == 8)
return;
Function *fn = insn->bb->getFunction();
int n = 0;
int adj = 4;
 
for (Instruction *i = insn->next; i && i->encSize == 4; ++n, i = i->next);
 
if (n & 1) {
adj = 8;
insn->next->encSize = 8;
} else
if (insn->prev && insn->prev->encSize == 4) {
adj = 8;
insn->prev->encSize = 8;
}
insn->encSize = 8;
 
for (int i = fn->bbCount - 1; i >= 0 && fn->bbArray[i] != insn->bb; --i) {
fn->bbArray[i]->binPos += 4;
}
fn->binSize += adj;
insn->bb->binSize += adj;
}
 
static bool
trySetExitModifier(Instruction *insn)
{
if (insn->op == OP_DISCARD ||
insn->op == OP_QUADON ||
insn->op == OP_QUADPOP)
return false;
for (int s = 0; insn->srcExists(s); ++s)
if (insn->src(s).getFile() == FILE_IMMEDIATE)
return false;
if (insn->asFlow()) {
if (insn->op == OP_CALL) // side effects !
return false;
if (insn->getPredicate()) // cannot do conditional exit (or can we ?)
return false;
insn->op = OP_EXIT;
}
insn->exit = 1;
makeInstructionLong(insn);
return true;
}
 
static void
replaceExitWithModifier(Function *func)
{
BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
 
if (!epilogue->getExit() ||
epilogue->getExit()->op != OP_EXIT) // only main will use OP_EXIT
return;
 
if (epilogue->getEntry()->op != OP_EXIT) {
Instruction *insn = epilogue->getExit()->prev;
if (!insn || !trySetExitModifier(insn))
return;
insn->exit = 1;
} else {
for (Graph::EdgeIterator ei = func->cfgExit->incident();
!ei.end(); ei.next()) {
BasicBlock *bb = BasicBlock::get(ei.getNode());
Instruction *i = bb->getExit();
 
if (!i || !trySetExitModifier(i))
return;
}
}
epilogue->binSize -= 8;
func->binSize -= 8;
delete_Instruction(func->getProgram(), epilogue->getExit());
}
 
void
CodeEmitterNV50::prepareEmission(Function *func)
{
CodeEmitter::prepareEmission(func);
 
replaceExitWithModifier(func);
}
 
CodeEmitterNV50::CodeEmitterNV50(const TargetNV50 *target) :
CodeEmitter(target), targNV50(target)
{
targ = target; // specialized
code = NULL;
codeSize = codeSizeLimit = 0;
relocInfo = NULL;
}
 
CodeEmitter *
TargetNV50::getCodeEmitter(Program::Type type)
{
CodeEmitterNV50 *emit = new CodeEmitterNV50(this);
emit->setProgramType(type);
return emit;
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_from_tgsi.cpp
0,0 → 1,2835
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
extern "C" {
#include "tgsi/tgsi_dump.h"
#include "tgsi/tgsi_scan.h"
#include "tgsi/tgsi_util.h"
}
 
#include <set>
 
#include "nv50_ir.h"
#include "nv50_ir_util.h"
#include "nv50_ir_build_util.h"
 
namespace tgsi {
 
class Source;
 
static nv50_ir::operation translateOpcode(uint opcode);
static nv50_ir::DataFile translateFile(uint file);
static nv50_ir::TexTarget translateTexture(uint texTarg);
static nv50_ir::SVSemantic translateSysVal(uint sysval);
 
class Instruction
{
public:
Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
 
class SrcRegister
{
public:
SrcRegister(const struct tgsi_full_src_register *src)
: reg(src->Register),
fsr(src)
{ }
 
SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
 
SrcRegister(const struct tgsi_ind_register& ind)
: reg(tgsi_util_get_src_from_ind(&ind)),
fsr(NULL)
{ }
 
struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
{
struct tgsi_src_register reg;
memset(&reg, 0, sizeof(reg));
reg.Index = off.Index;
reg.File = off.File;
reg.SwizzleX = off.SwizzleX;
reg.SwizzleY = off.SwizzleY;
reg.SwizzleZ = off.SwizzleZ;
return reg;
}
 
SrcRegister(const struct tgsi_texture_offset& off) :
reg(offsetToSrc(off)),
fsr(NULL)
{ }
 
uint getFile() const { return reg.File; }
 
bool is2D() const { return reg.Dimension; }
 
bool isIndirect(int dim) const
{
return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
}
 
int getIndex(int dim) const
{
return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
}
 
int getSwizzle(int chan) const
{
return tgsi_util_get_src_register_swizzle(&reg, chan);
}
 
nv50_ir::Modifier getMod(int chan) const;
 
SrcRegister getIndirect(int dim) const
{
assert(fsr && isIndirect(dim));
if (dim)
return SrcRegister(fsr->DimIndirect);
return SrcRegister(fsr->Indirect);
}
 
uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
{
assert(reg.File == TGSI_FILE_IMMEDIATE);
assert(!reg.Absolute);
assert(!reg.Negate);
return info->immd.data[reg.Index * 4 + getSwizzle(c)];
}
 
private:
const struct tgsi_src_register reg;
const struct tgsi_full_src_register *fsr;
};
 
class DstRegister
{
public:
DstRegister(const struct tgsi_full_dst_register *dst)
: reg(dst->Register),
fdr(dst)
{ }
 
DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
 
uint getFile() const { return reg.File; }
 
bool is2D() const { return reg.Dimension; }
 
bool isIndirect(int dim) const
{
return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
}
 
int getIndex(int dim) const
{
return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
}
 
unsigned int getMask() const { return reg.WriteMask; }
 
bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
 
SrcRegister getIndirect(int dim) const
{
assert(fdr && isIndirect(dim));
if (dim)
return SrcRegister(fdr->DimIndirect);
return SrcRegister(fdr->Indirect);
}
 
private:
const struct tgsi_dst_register reg;
const struct tgsi_full_dst_register *fdr;
};
 
inline uint getOpcode() const { return insn->Instruction.Opcode; }
 
unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
 
// mask of used components of source s
unsigned int srcMask(unsigned int s) const;
 
SrcRegister getSrc(unsigned int s) const
{
assert(s < srcCount());
return SrcRegister(&insn->Src[s]);
}
 
DstRegister getDst(unsigned int d) const
{
assert(d < dstCount());
return DstRegister(&insn->Dst[d]);
}
 
SrcRegister getTexOffset(unsigned int i) const
{
assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
return SrcRegister(insn->TexOffsets[i]);
}
 
unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
 
bool checkDstSrcAliasing() const;
 
inline nv50_ir::operation getOP() const {
return translateOpcode(getOpcode()); }
 
nv50_ir::DataType inferSrcType() const;
nv50_ir::DataType inferDstType() const;
 
nv50_ir::CondCode getSetCond() const;
 
nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
 
inline uint getLabel() { return insn->Label.Label; }
 
unsigned getSaturate() const { return insn->Instruction.Saturate; }
 
void print() const
{
tgsi_dump_instruction(insn, 1);
}
 
private:
const struct tgsi_full_instruction *insn;
};
 
unsigned int Instruction::srcMask(unsigned int s) const
{
unsigned int mask = insn->Dst[0].Register.WriteMask;
 
switch (insn->Instruction.Opcode) {
case TGSI_OPCODE_COS:
case TGSI_OPCODE_SIN:
return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
case TGSI_OPCODE_DP2:
return 0x3;
case TGSI_OPCODE_DP3:
return 0x7;
case TGSI_OPCODE_DP4:
case TGSI_OPCODE_DPH:
case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
return 0xf;
case TGSI_OPCODE_DST:
return mask & (s ? 0xa : 0x6);
case TGSI_OPCODE_EX2:
case TGSI_OPCODE_EXP:
case TGSI_OPCODE_LG2:
case TGSI_OPCODE_LOG:
case TGSI_OPCODE_POW:
case TGSI_OPCODE_RCP:
case TGSI_OPCODE_RSQ:
case TGSI_OPCODE_SCS:
return 0x1;
case TGSI_OPCODE_IF:
case TGSI_OPCODE_UIF:
return 0x1;
case TGSI_OPCODE_LIT:
return 0xb;
case TGSI_OPCODE_TEX2:
case TGSI_OPCODE_TXB2:
case TGSI_OPCODE_TXL2:
return (s == 0) ? 0xf : 0x3;
case TGSI_OPCODE_TEX:
case TGSI_OPCODE_TXB:
case TGSI_OPCODE_TXD:
case TGSI_OPCODE_TXL:
case TGSI_OPCODE_TXP:
{
const struct tgsi_instruction_texture *tex = &insn->Texture;
 
assert(insn->Instruction.Texture);
 
mask = 0x7;
if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
insn->Instruction.Opcode != TGSI_OPCODE_TXD)
mask |= 0x8; /* bias, lod or proj */
 
switch (tex->Texture) {
case TGSI_TEXTURE_1D:
mask &= 0x9;
break;
case TGSI_TEXTURE_SHADOW1D:
mask &= 0xd;
break;
case TGSI_TEXTURE_1D_ARRAY:
case TGSI_TEXTURE_2D:
case TGSI_TEXTURE_RECT:
mask &= 0xb;
break;
case TGSI_TEXTURE_CUBE_ARRAY:
case TGSI_TEXTURE_SHADOW2D_ARRAY:
case TGSI_TEXTURE_SHADOWCUBE:
case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
mask |= 0x8;
break;
default:
break;
}
}
return mask;
case TGSI_OPCODE_XPD:
{
unsigned int x = 0;
if (mask & 1) x |= 0x6;
if (mask & 2) x |= 0x5;
if (mask & 4) x |= 0x3;
return x;
}
default:
break;
}
 
return mask;
}
 
nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
{
nv50_ir::Modifier m(0);
 
if (reg.Absolute)
m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
if (reg.Negate)
m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
return m;
}
 
static nv50_ir::DataFile translateFile(uint file)
{
switch (file) {
case TGSI_FILE_CONSTANT: return nv50_ir::FILE_MEMORY_CONST;
case TGSI_FILE_INPUT: return nv50_ir::FILE_SHADER_INPUT;
case TGSI_FILE_OUTPUT: return nv50_ir::FILE_SHADER_OUTPUT;
case TGSI_FILE_TEMPORARY: return nv50_ir::FILE_GPR;
case TGSI_FILE_ADDRESS: return nv50_ir::FILE_ADDRESS;
case TGSI_FILE_PREDICATE: return nv50_ir::FILE_PREDICATE;
case TGSI_FILE_IMMEDIATE: return nv50_ir::FILE_IMMEDIATE;
case TGSI_FILE_SYSTEM_VALUE: return nv50_ir::FILE_SYSTEM_VALUE;
case TGSI_FILE_RESOURCE: return nv50_ir::FILE_MEMORY_GLOBAL;
case TGSI_FILE_SAMPLER:
case TGSI_FILE_NULL:
default:
return nv50_ir::FILE_NULL;
}
}
 
static nv50_ir::SVSemantic translateSysVal(uint sysval)
{
switch (sysval) {
case TGSI_SEMANTIC_FACE: return nv50_ir::SV_FACE;
case TGSI_SEMANTIC_PSIZE: return nv50_ir::SV_POINT_SIZE;
case TGSI_SEMANTIC_PRIMID: return nv50_ir::SV_PRIMITIVE_ID;
case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
case TGSI_SEMANTIC_VERTEXID: return nv50_ir::SV_VERTEX_ID;
case TGSI_SEMANTIC_GRID_SIZE: return nv50_ir::SV_NCTAID;
case TGSI_SEMANTIC_BLOCK_ID: return nv50_ir::SV_CTAID;
case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
case TGSI_SEMANTIC_THREAD_ID: return nv50_ir::SV_TID;
default:
assert(0);
return nv50_ir::SV_CLOCK;
}
}
 
#define NV50_IR_TEX_TARG_CASE(a, b) \
case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
 
static nv50_ir::TexTarget translateTexture(uint tex)
{
switch (tex) {
NV50_IR_TEX_TARG_CASE(1D, 1D);
NV50_IR_TEX_TARG_CASE(2D, 2D);
NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
NV50_IR_TEX_TARG_CASE(3D, 3D);
NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
NV50_IR_TEX_TARG_CASE(RECT, RECT);
NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
 
case TGSI_TEXTURE_UNKNOWN:
default:
assert(!"invalid texture target");
return nv50_ir::TEX_TARGET_2D;
}
}
nv50_ir::DataType Instruction::inferSrcType() const
{
switch (getOpcode()) {
case TGSI_OPCODE_UIF:
case TGSI_OPCODE_AND:
case TGSI_OPCODE_OR:
case TGSI_OPCODE_XOR:
case TGSI_OPCODE_NOT:
case TGSI_OPCODE_U2F:
case TGSI_OPCODE_UADD:
case TGSI_OPCODE_UDIV:
case TGSI_OPCODE_UMOD:
case TGSI_OPCODE_UMAD:
case TGSI_OPCODE_UMUL:
case TGSI_OPCODE_UMAX:
case TGSI_OPCODE_UMIN:
case TGSI_OPCODE_USEQ:
case TGSI_OPCODE_USGE:
case TGSI_OPCODE_USLT:
case TGSI_OPCODE_USNE:
case TGSI_OPCODE_USHR:
case TGSI_OPCODE_UCMP:
case TGSI_OPCODE_ATOMUADD:
case TGSI_OPCODE_ATOMXCHG:
case TGSI_OPCODE_ATOMCAS:
case TGSI_OPCODE_ATOMAND:
case TGSI_OPCODE_ATOMOR:
case TGSI_OPCODE_ATOMXOR:
case TGSI_OPCODE_ATOMUMIN:
case TGSI_OPCODE_ATOMUMAX:
return nv50_ir::TYPE_U32;
case TGSI_OPCODE_I2F:
case TGSI_OPCODE_IDIV:
case TGSI_OPCODE_IMAX:
case TGSI_OPCODE_IMIN:
case TGSI_OPCODE_IABS:
case TGSI_OPCODE_INEG:
case TGSI_OPCODE_ISGE:
case TGSI_OPCODE_ISHR:
case TGSI_OPCODE_ISLT:
case TGSI_OPCODE_ISSG:
case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
case TGSI_OPCODE_MOD:
case TGSI_OPCODE_UARL:
case TGSI_OPCODE_ATOMIMIN:
case TGSI_OPCODE_ATOMIMAX:
return nv50_ir::TYPE_S32;
default:
return nv50_ir::TYPE_F32;
}
}
 
nv50_ir::DataType Instruction::inferDstType() const
{
switch (getOpcode()) {
case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
case TGSI_OPCODE_I2F:
case TGSI_OPCODE_U2F:
return nv50_ir::TYPE_F32;
default:
return inferSrcType();
}
}
 
nv50_ir::CondCode Instruction::getSetCond() const
{
using namespace nv50_ir;
 
switch (getOpcode()) {
case TGSI_OPCODE_SLT:
case TGSI_OPCODE_ISLT:
case TGSI_OPCODE_USLT:
return CC_LT;
case TGSI_OPCODE_SLE:
return CC_LE;
case TGSI_OPCODE_SGE:
case TGSI_OPCODE_ISGE:
case TGSI_OPCODE_USGE:
return CC_GE;
case TGSI_OPCODE_SGT:
return CC_GT;
case TGSI_OPCODE_SEQ:
case TGSI_OPCODE_USEQ:
return CC_EQ;
case TGSI_OPCODE_SNE:
return CC_NEU;
case TGSI_OPCODE_USNE:
return CC_NE;
case TGSI_OPCODE_SFL:
return CC_NEVER;
case TGSI_OPCODE_STR:
default:
return CC_ALWAYS;
}
}
 
#define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
static nv50_ir::operation translateOpcode(uint opcode)
{
switch (opcode) {
NV50_IR_OPCODE_CASE(ARL, SHL);
NV50_IR_OPCODE_CASE(MOV, MOV);
 
NV50_IR_OPCODE_CASE(RCP, RCP);
NV50_IR_OPCODE_CASE(RSQ, RSQ);
 
NV50_IR_OPCODE_CASE(MUL, MUL);
NV50_IR_OPCODE_CASE(ADD, ADD);
 
NV50_IR_OPCODE_CASE(MIN, MIN);
NV50_IR_OPCODE_CASE(MAX, MAX);
NV50_IR_OPCODE_CASE(SLT, SET);
NV50_IR_OPCODE_CASE(SGE, SET);
NV50_IR_OPCODE_CASE(MAD, MAD);
NV50_IR_OPCODE_CASE(SUB, SUB);
 
NV50_IR_OPCODE_CASE(FLR, FLOOR);
NV50_IR_OPCODE_CASE(ROUND, CVT);
NV50_IR_OPCODE_CASE(EX2, EX2);
NV50_IR_OPCODE_CASE(LG2, LG2);
NV50_IR_OPCODE_CASE(POW, POW);
 
NV50_IR_OPCODE_CASE(ABS, ABS);
 
NV50_IR_OPCODE_CASE(COS, COS);
NV50_IR_OPCODE_CASE(DDX, DFDX);
NV50_IR_OPCODE_CASE(DDY, DFDY);
NV50_IR_OPCODE_CASE(KILL, DISCARD);
 
NV50_IR_OPCODE_CASE(SEQ, SET);
NV50_IR_OPCODE_CASE(SFL, SET);
NV50_IR_OPCODE_CASE(SGT, SET);
NV50_IR_OPCODE_CASE(SIN, SIN);
NV50_IR_OPCODE_CASE(SLE, SET);
NV50_IR_OPCODE_CASE(SNE, SET);
NV50_IR_OPCODE_CASE(STR, SET);
NV50_IR_OPCODE_CASE(TEX, TEX);
NV50_IR_OPCODE_CASE(TXD, TXD);
NV50_IR_OPCODE_CASE(TXP, TEX);
 
NV50_IR_OPCODE_CASE(BRA, BRA);
NV50_IR_OPCODE_CASE(CAL, CALL);
NV50_IR_OPCODE_CASE(RET, RET);
NV50_IR_OPCODE_CASE(CMP, SLCT);
 
NV50_IR_OPCODE_CASE(TXB, TXB);
 
NV50_IR_OPCODE_CASE(DIV, DIV);
 
NV50_IR_OPCODE_CASE(TXL, TXL);
 
NV50_IR_OPCODE_CASE(CEIL, CEIL);
NV50_IR_OPCODE_CASE(I2F, CVT);
NV50_IR_OPCODE_CASE(NOT, NOT);
NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
NV50_IR_OPCODE_CASE(SHL, SHL);
 
NV50_IR_OPCODE_CASE(AND, AND);
NV50_IR_OPCODE_CASE(OR, OR);
NV50_IR_OPCODE_CASE(MOD, MOD);
NV50_IR_OPCODE_CASE(XOR, XOR);
NV50_IR_OPCODE_CASE(SAD, SAD);
NV50_IR_OPCODE_CASE(TXF, TXF);
NV50_IR_OPCODE_CASE(TXQ, TXQ);
 
NV50_IR_OPCODE_CASE(EMIT, EMIT);
NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
 
NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
 
NV50_IR_OPCODE_CASE(F2I, CVT);
NV50_IR_OPCODE_CASE(IDIV, DIV);
NV50_IR_OPCODE_CASE(IMAX, MAX);
NV50_IR_OPCODE_CASE(IMIN, MIN);
NV50_IR_OPCODE_CASE(IABS, ABS);
NV50_IR_OPCODE_CASE(INEG, NEG);
NV50_IR_OPCODE_CASE(ISGE, SET);
NV50_IR_OPCODE_CASE(ISHR, SHR);
NV50_IR_OPCODE_CASE(ISLT, SET);
NV50_IR_OPCODE_CASE(F2U, CVT);
NV50_IR_OPCODE_CASE(U2F, CVT);
NV50_IR_OPCODE_CASE(UADD, ADD);
NV50_IR_OPCODE_CASE(UDIV, DIV);
NV50_IR_OPCODE_CASE(UMAD, MAD);
NV50_IR_OPCODE_CASE(UMAX, MAX);
NV50_IR_OPCODE_CASE(UMIN, MIN);
NV50_IR_OPCODE_CASE(UMOD, MOD);
NV50_IR_OPCODE_CASE(UMUL, MUL);
NV50_IR_OPCODE_CASE(USEQ, SET);
NV50_IR_OPCODE_CASE(USGE, SET);
NV50_IR_OPCODE_CASE(USHR, SHR);
NV50_IR_OPCODE_CASE(USLT, SET);
NV50_IR_OPCODE_CASE(USNE, SET);
 
NV50_IR_OPCODE_CASE(SAMPLE, TEX);
NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
NV50_IR_OPCODE_CASE(GATHER4, TXG);
NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
 
NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
 
NV50_IR_OPCODE_CASE(TEX2, TEX);
NV50_IR_OPCODE_CASE(TXB2, TXB);
NV50_IR_OPCODE_CASE(TXL2, TXL);
 
NV50_IR_OPCODE_CASE(END, EXIT);
 
default:
return nv50_ir::OP_NOP;
}
}
 
static uint16_t opcodeToSubOp(uint opcode)
{
switch (opcode) {
case TGSI_OPCODE_LFENCE: return NV50_IR_SUBOP_MEMBAR(L, GL);
case TGSI_OPCODE_SFENCE: return NV50_IR_SUBOP_MEMBAR(S, GL);
case TGSI_OPCODE_MFENCE: return NV50_IR_SUBOP_MEMBAR(M, GL);
case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
case TGSI_OPCODE_ATOMCAS: return NV50_IR_SUBOP_ATOM_CAS;
case TGSI_OPCODE_ATOMAND: return NV50_IR_SUBOP_ATOM_AND;
case TGSI_OPCODE_ATOMOR: return NV50_IR_SUBOP_ATOM_OR;
case TGSI_OPCODE_ATOMXOR: return NV50_IR_SUBOP_ATOM_XOR;
case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
default:
return 0;
}
}
 
bool Instruction::checkDstSrcAliasing() const
{
if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
return false;
 
for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
if (insn->Src[s].Register.File == TGSI_FILE_NULL)
break;
if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
return true;
}
return false;
}
 
class Source
{
public:
Source(struct nv50_ir_prog_info *);
~Source();
 
public:
bool scanSource();
unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
 
public:
struct tgsi_shader_info scan;
struct tgsi_full_instruction *insns;
const struct tgsi_token *tokens;
struct nv50_ir_prog_info *info;
 
nv50_ir::DynArray tempArrays;
nv50_ir::DynArray immdArrays;
 
typedef nv50_ir::BuildUtil::Location Location;
// these registers are per-subroutine, cannot be used for parameter passing
std::set<Location> locals;
 
bool mainTempsInLMem;
 
int clipVertexOutput;
 
struct TextureView {
uint8_t target; // TGSI_TEXTURE_*
};
std::vector<TextureView> textureViews;
 
struct Resource {
uint8_t target; // TGSI_TEXTURE_*
bool raw;
uint8_t slot; // $surface index
};
std::vector<Resource> resources;
 
private:
int inferSysValDirection(unsigned sn) const;
bool scanDeclaration(const struct tgsi_full_declaration *);
bool scanInstruction(const struct tgsi_full_instruction *);
void scanProperty(const struct tgsi_full_property *);
void scanImmediate(const struct tgsi_full_immediate *);
 
inline bool isEdgeFlagPassthrough(const Instruction&) const;
};
 
Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
{
tokens = (const struct tgsi_token *)info->bin.source;
 
if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
tgsi_dump(tokens, 0);
 
mainTempsInLMem = FALSE;
}
 
Source::~Source()
{
if (insns)
FREE(insns);
 
if (info->immd.data)
FREE(info->immd.data);
if (info->immd.type)
FREE(info->immd.type);
}
 
bool Source::scanSource()
{
unsigned insnCount = 0;
struct tgsi_parse_context parse;
 
tgsi_scan_shader(tokens, &scan);
 
insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
sizeof(insns[0]));
if (!insns)
return false;
 
clipVertexOutput = -1;
 
textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
 
info->immd.bufSize = 0;
 
info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
 
if (info->type == PIPE_SHADER_FRAGMENT) {
info->prop.fp.writesDepth = scan.writes_z;
info->prop.fp.usesDiscard = scan.uses_kill;
} else
if (info->type == PIPE_SHADER_GEOMETRY) {
info->prop.gp.instanceCount = 1; // default value
}
 
info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
 
tgsi_parse_init(&parse, tokens);
while (!tgsi_parse_end_of_tokens(&parse)) {
tgsi_parse_token(&parse);
 
switch (parse.FullToken.Token.Type) {
case TGSI_TOKEN_TYPE_IMMEDIATE:
scanImmediate(&parse.FullToken.FullImmediate);
break;
case TGSI_TOKEN_TYPE_DECLARATION:
scanDeclaration(&parse.FullToken.FullDeclaration);
break;
case TGSI_TOKEN_TYPE_INSTRUCTION:
insns[insnCount++] = parse.FullToken.FullInstruction;
scanInstruction(&parse.FullToken.FullInstruction);
break;
case TGSI_TOKEN_TYPE_PROPERTY:
scanProperty(&parse.FullToken.FullProperty);
break;
default:
INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
break;
}
}
tgsi_parse_free(&parse);
 
if (mainTempsInLMem)
info->bin.tlsSpace += (scan.file_max[TGSI_FILE_TEMPORARY] + 1) * 16;
 
if (info->io.genUserClip > 0) {
info->io.clipDistanceMask = (1 << info->io.genUserClip) - 1;
 
const unsigned int nOut = (info->io.genUserClip + 3) / 4;
 
for (unsigned int n = 0; n < nOut; ++n) {
unsigned int i = info->numOutputs++;
info->out[i].id = i;
info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
info->out[i].si = n;
info->out[i].mask = info->io.clipDistanceMask >> (n * 4);
}
}
 
return info->assignSlots(info) == 0;
}
 
void Source::scanProperty(const struct tgsi_full_property *prop)
{
switch (prop->Property.PropertyName) {
case TGSI_PROPERTY_GS_OUTPUT_PRIM:
info->prop.gp.outputPrim = prop->u[0].Data;
break;
case TGSI_PROPERTY_GS_INPUT_PRIM:
info->prop.gp.inputPrim = prop->u[0].Data;
break;
case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
info->prop.gp.maxVertices = prop->u[0].Data;
break;
#if 0
case TGSI_PROPERTY_GS_INSTANCE_COUNT:
info->prop.gp.instanceCount = prop->u[0].Data;
break;
#endif
case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
info->prop.fp.separateFragData = TRUE;
break;
case TGSI_PROPERTY_FS_COORD_ORIGIN:
case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
// we don't care
break;
case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
info->io.genUserClip = -1;
break;
default:
INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
break;
}
}
 
void Source::scanImmediate(const struct tgsi_full_immediate *imm)
{
const unsigned n = info->immd.count++;
 
assert(n < scan.immediate_count);
 
for (int c = 0; c < 4; ++c)
info->immd.data[n * 4 + c] = imm->u[c].Uint;
 
info->immd.type[n] = imm->Immediate.DataType;
}
 
int Source::inferSysValDirection(unsigned sn) const
{
switch (sn) {
case TGSI_SEMANTIC_INSTANCEID:
case TGSI_SEMANTIC_VERTEXID:
return 1;
#if 0
case TGSI_SEMANTIC_LAYER:
case TGSI_SEMANTIC_VIEWPORTINDEX:
return 0;
#endif
case TGSI_SEMANTIC_PRIMID:
return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
default:
return 0;
}
}
 
bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
{
unsigned i, c;
unsigned sn = TGSI_SEMANTIC_GENERIC;
unsigned si = 0;
const unsigned first = decl->Range.First, last = decl->Range.Last;
 
if (decl->Declaration.Semantic) {
sn = decl->Semantic.Name;
si = decl->Semantic.Index;
}
 
if (decl->Declaration.Local) {
for (i = first; i <= last; ++i) {
for (c = 0; c < 4; ++c) {
locals.insert(
Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
}
}
}
 
switch (decl->Declaration.File) {
case TGSI_FILE_INPUT:
if (info->type == PIPE_SHADER_VERTEX) {
// all vertex attributes are equal
for (i = first; i <= last; ++i) {
info->in[i].sn = TGSI_SEMANTIC_GENERIC;
info->in[i].si = i;
}
} else {
for (i = first; i <= last; ++i, ++si) {
info->in[i].id = i;
info->in[i].sn = sn;
info->in[i].si = si;
if (info->type == PIPE_SHADER_FRAGMENT) {
// translate interpolation mode
switch (decl->Interp.Interpolate) {
case TGSI_INTERPOLATE_CONSTANT:
info->in[i].flat = 1;
break;
case TGSI_INTERPOLATE_COLOR:
info->in[i].sc = 1;
break;
case TGSI_INTERPOLATE_LINEAR:
info->in[i].linear = 1;
break;
default:
break;
}
if (decl->Interp.Centroid)
info->in[i].centroid = 1;
}
}
}
break;
case TGSI_FILE_OUTPUT:
for (i = first; i <= last; ++i, ++si) {
switch (sn) {
case TGSI_SEMANTIC_POSITION:
if (info->type == PIPE_SHADER_FRAGMENT)
info->io.fragDepth = i;
else
if (clipVertexOutput < 0)
clipVertexOutput = i;
break;
case TGSI_SEMANTIC_COLOR:
if (info->type == PIPE_SHADER_FRAGMENT)
info->prop.fp.numColourResults++;
break;
case TGSI_SEMANTIC_EDGEFLAG:
info->io.edgeFlagOut = i;
break;
case TGSI_SEMANTIC_CLIPVERTEX:
clipVertexOutput = i;
break;
case TGSI_SEMANTIC_CLIPDIST:
info->io.clipDistanceMask |=
decl->Declaration.UsageMask << (si * 4);
info->io.genUserClip = -1;
break;
default:
break;
}
info->out[i].id = i;
info->out[i].sn = sn;
info->out[i].si = si;
}
break;
case TGSI_FILE_SYSTEM_VALUE:
switch (sn) {
case TGSI_SEMANTIC_INSTANCEID:
info->io.instanceId = first;
break;
case TGSI_SEMANTIC_VERTEXID:
info->io.vertexId = first;
break;
default:
break;
}
for (i = first; i <= last; ++i, ++si) {
info->sv[i].sn = sn;
info->sv[i].si = si;
info->sv[i].input = inferSysValDirection(sn);
}
break;
case TGSI_FILE_RESOURCE:
for (i = first; i <= last; ++i) {
resources[i].target = decl->Resource.Resource;
resources[i].raw = decl->Resource.Raw;
resources[i].slot = i;
}
break;
case TGSI_FILE_SAMPLER_VIEW:
for (i = first; i <= last; ++i)
textureViews[i].target = decl->SamplerView.Resource;
break;
case TGSI_FILE_NULL:
case TGSI_FILE_TEMPORARY:
case TGSI_FILE_ADDRESS:
case TGSI_FILE_CONSTANT:
case TGSI_FILE_IMMEDIATE:
case TGSI_FILE_PREDICATE:
case TGSI_FILE_SAMPLER:
break;
default:
ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
return false;
}
return true;
}
 
inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
{
return insn.getOpcode() == TGSI_OPCODE_MOV &&
insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
}
 
bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
{
Instruction insn(inst);
 
if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
info->numBarriers = 1;
 
if (insn.dstCount()) {
if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
Instruction::DstRegister dst = insn.getDst(0);
 
if (dst.isIndirect(0))
for (unsigned i = 0; i < info->numOutputs; ++i)
info->out[i].mask = 0xf;
else
info->out[dst.getIndex(0)].mask |= dst.getMask();
 
if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
info->out[dst.getIndex(0)].mask &= 1;
 
if (isEdgeFlagPassthrough(insn))
info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
} else
if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
if (insn.getDst(0).isIndirect(0))
mainTempsInLMem = TRUE;
}
}
 
for (unsigned s = 0; s < insn.srcCount(); ++s) {
Instruction::SrcRegister src = insn.getSrc(s);
if (src.getFile() == TGSI_FILE_TEMPORARY) {
if (src.isIndirect(0))
mainTempsInLMem = TRUE;
} else
if (src.getFile() == TGSI_FILE_RESOURCE) {
if (src.getIndex(0) == TGSI_RESOURCE_GLOBAL)
info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
0x1 : 0x2;
}
if (src.getFile() != TGSI_FILE_INPUT)
continue;
unsigned mask = insn.srcMask(s);
 
if (src.isIndirect(0)) {
for (unsigned i = 0; i < info->numInputs; ++i)
info->in[i].mask = 0xf;
} else {
const int i = src.getIndex(0);
for (unsigned c = 0; c < 4; ++c) {
if (!(mask & (1 << c)))
continue;
int k = src.getSwizzle(c);
if (k <= TGSI_SWIZZLE_W)
info->in[i].mask |= 1 << k;
}
switch (info->in[i].sn) {
case TGSI_SEMANTIC_PSIZE:
case TGSI_SEMANTIC_PRIMID:
case TGSI_SEMANTIC_FOG:
info->in[i].mask &= 0x1;
break;
case TGSI_SEMANTIC_PCOORD:
info->in[i].mask &= 0x3;
break;
default:
break;
}
}
}
return true;
}
 
nv50_ir::TexInstruction::Target
Instruction::getTexture(const tgsi::Source *code, int s) const
{
// XXX: indirect access
unsigned int r;
 
switch (getSrc(s).getFile()) {
case TGSI_FILE_RESOURCE:
r = getSrc(s).getIndex(0);
return translateTexture(code->resources.at(r).target);
case TGSI_FILE_SAMPLER_VIEW:
r = getSrc(s).getIndex(0);
return translateTexture(code->textureViews.at(r).target);
default:
return translateTexture(insn->Texture.Texture);
}
}
 
} // namespace tgsi
 
namespace {
 
using namespace nv50_ir;
 
class Converter : public BuildUtil
{
public:
Converter(Program *, const tgsi::Source *);
~Converter();
 
bool run();
 
private:
struct Subroutine
{
Subroutine(Function *f) : f(f) { }
Function *f;
ValueMap values;
};
 
Value *getVertexBase(int s);
DataArray *getArrayForFile(unsigned file, int idx);
Value *fetchSrc(int s, int c);
Value *acquireDst(int d, int c);
void storeDst(int d, int c, Value *);
 
Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
void storeDst(const tgsi::Instruction::DstRegister dst, int c,
Value *val, Value *ptr);
 
Value *applySrcMod(Value *, int s, int c);
 
Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
 
bool handleInstruction(const struct tgsi_full_instruction *);
void exportOutputs();
inline Subroutine *getSubroutine(unsigned ip);
inline Subroutine *getSubroutine(Function *);
inline bool isEndOfSubroutine(uint ip);
 
void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
 
// R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
void handleTXF(Value *dst0[4], int R, int L_M);
void handleTXQ(Value *dst0[4], enum TexQuery);
void handleLIT(Value *dst0[4]);
void handleUserClipPlanes();
 
Symbol *getResourceBase(int r);
void getResourceCoords(std::vector<Value *>&, int r, int s);
 
void handleLOAD(Value *dst0[4]);
void handleSTORE();
void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
 
Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
 
void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
 
Value *buildDot(int dim);
 
class BindArgumentsPass : public Pass {
public:
BindArgumentsPass(Converter &conv) : conv(conv) { }
 
private:
Converter &conv;
Subroutine *sub;
 
inline const Location *getValueLocation(Subroutine *, Value *);
 
template<typename T> inline void
updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
T (Function::*proto));
 
template<typename T> inline void
updatePrototype(BitSet *set, void (Function::*updateSet)(),
T (Function::*proto));
 
protected:
bool visit(Function *);
bool visit(BasicBlock *bb) { return false; }
};
 
private:
const struct tgsi::Source *code;
const struct nv50_ir_prog_info *info;
 
struct {
std::map<unsigned, Subroutine> map;
Subroutine *cur;
} sub;
 
uint ip; // instruction pointer
 
tgsi::Instruction tgsi;
 
DataType dstTy;
DataType srcTy;
 
DataArray tData; // TGSI_FILE_TEMPORARY
DataArray aData; // TGSI_FILE_ADDRESS
DataArray pData; // TGSI_FILE_PREDICATE
DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
 
Value *zero;
Value *fragCoord[4];
Value *clipVtx[4];
 
Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
uint8_t vtxBaseValid;
 
Stack condBBs; // fork BB, then else clause BB
Stack joinBBs; // fork BB, for inserting join ops on ENDIF
Stack loopBBs; // loop headers
Stack breakBBs; // end of / after loop
};
 
Symbol *
Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
{
const int swz = src.getSwizzle(c);
 
return makeSym(src.getFile(),
src.is2D() ? src.getIndex(1) : 0,
src.isIndirect(0) ? -1 : src.getIndex(0), swz,
src.getIndex(0) * 16 + swz * 4);
}
 
Symbol *
Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
{
return makeSym(dst.getFile(),
dst.is2D() ? dst.getIndex(1) : 0,
dst.isIndirect(0) ? -1 : dst.getIndex(0), c,
dst.getIndex(0) * 16 + c * 4);
}
 
Symbol *
Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
{
Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
 
sym->reg.fileIndex = fileIdx;
 
if (idx >= 0) {
if (sym->reg.file == FILE_SHADER_INPUT)
sym->setOffset(info->in[idx].slot[c] * 4);
else
if (sym->reg.file == FILE_SHADER_OUTPUT)
sym->setOffset(info->out[idx].slot[c] * 4);
else
if (sym->reg.file == FILE_SYSTEM_VALUE)
sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
else
sym->setOffset(address);
} else {
sym->setOffset(address);
}
return sym;
}
 
static inline uint8_t
translateInterpMode(const struct nv50_ir_varying *var, operation& op)
{
uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
 
if (var->flat)
mode = NV50_IR_INTERP_FLAT;
else
if (var->linear)
mode = NV50_IR_INTERP_LINEAR;
else
if (var->sc)
mode = NV50_IR_INTERP_SC;
 
op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
? OP_PINTERP : OP_LINTERP;
 
if (var->centroid)
mode |= NV50_IR_INTERP_CENTROID;
 
return mode;
}
 
Value *
Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
{
operation op;
 
// XXX: no way to know interpolation mode if we don't know what's accessed
const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
src.getIndex(0)], op);
 
Instruction *insn = new_Instruction(func, op, TYPE_F32);
 
insn->setDef(0, getScratch());
insn->setSrc(0, srcToSym(src, c));
if (op == OP_PINTERP)
insn->setSrc(1, fragCoord[3]);
if (ptr)
insn->setIndirect(0, 0, ptr);
 
insn->setInterpolate(mode);
 
bb->insertTail(insn);
return insn->getDef(0);
}
 
Value *
Converter::applySrcMod(Value *val, int s, int c)
{
Modifier m = tgsi.getSrc(s).getMod(c);
DataType ty = tgsi.inferSrcType();
 
if (m & Modifier(NV50_IR_MOD_ABS))
val = mkOp1v(OP_ABS, ty, getScratch(), val);
 
if (m & Modifier(NV50_IR_MOD_NEG))
val = mkOp1v(OP_NEG, ty, getScratch(), val);
 
return val;
}
 
Value *
Converter::getVertexBase(int s)
{
assert(s < 5);
if (!(vtxBaseValid & (1 << s))) {
const int index = tgsi.getSrc(s).getIndex(1);
Value *rel = NULL;
if (tgsi.getSrc(s).isIndirect(1))
rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
vtxBaseValid |= 1 << s;
vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(), mkImm(index), rel);
}
return vtxBase[s];
}
 
Value *
Converter::fetchSrc(int s, int c)
{
Value *res;
Value *ptr = NULL, *dimRel = NULL;
 
tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
 
if (src.isIndirect(0))
ptr = fetchSrc(src.getIndirect(0), 0, NULL);
 
if (src.is2D()) {
switch (src.getFile()) {
case TGSI_FILE_INPUT:
dimRel = getVertexBase(s);
break;
case TGSI_FILE_CONSTANT:
// on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
if (src.isIndirect(1))
dimRel = fetchSrc(src.getIndirect(1), 0, 0);
break;
default:
break;
}
}
 
res = fetchSrc(src, c, ptr);
 
if (dimRel)
res->getInsn()->setIndirect(0, 1, dimRel);
 
return applySrcMod(res, s, c);
}
 
Converter::DataArray *
Converter::getArrayForFile(unsigned file, int idx)
{
switch (file) {
case TGSI_FILE_TEMPORARY:
return &tData;
case TGSI_FILE_PREDICATE:
return &pData;
case TGSI_FILE_ADDRESS:
return &aData;
case TGSI_FILE_OUTPUT:
assert(prog->getType() == Program::TYPE_FRAGMENT);
return &oData;
default:
assert(!"invalid/unhandled TGSI source file");
return NULL;
}
}
 
Value *
Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
{
const int idx2d = src.is2D() ? src.getIndex(1) : 0;
const int idx = src.getIndex(0);
const int swz = src.getSwizzle(c);
 
switch (src.getFile()) {
case TGSI_FILE_IMMEDIATE:
assert(!ptr);
return loadImm(NULL, info->immd.data[idx * 4 + swz]);
case TGSI_FILE_CONSTANT:
return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
case TGSI_FILE_INPUT:
if (prog->getType() == Program::TYPE_FRAGMENT) {
// don't load masked inputs, won't be assigned a slot
if (!ptr && !(info->in[idx].mask & (1 << swz)))
return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_FACE)
return mkOp1v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_FACE, 0));
return interpolate(src, c, ptr);
}
return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
case TGSI_FILE_OUTPUT:
assert(!"load from output file");
return NULL;
case TGSI_FILE_SYSTEM_VALUE:
assert(!ptr);
return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
default:
return getArrayForFile(src.getFile(), idx2d)->load(
sub.cur->values, idx, swz, ptr);
}
}
 
Value *
Converter::acquireDst(int d, int c)
{
const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
const unsigned f = dst.getFile();
const int idx = dst.getIndex(0);
const int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
 
if (dst.isMasked(c) || f == TGSI_FILE_RESOURCE)
return NULL;
 
if (dst.isIndirect(0) ||
f == TGSI_FILE_SYSTEM_VALUE ||
(f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
return getScratch();
 
return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
}
 
void
Converter::storeDst(int d, int c, Value *val)
{
const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
 
switch (tgsi.getSaturate()) {
case TGSI_SAT_NONE:
break;
case TGSI_SAT_ZERO_ONE:
mkOp1(OP_SAT, dstTy, val, val);
break;
case TGSI_SAT_MINUS_PLUS_ONE:
mkOp2(OP_MAX, dstTy, val, val, mkImm(-1.0f));
mkOp2(OP_MIN, dstTy, val, val, mkImm(+1.0f));
break;
default:
assert(!"invalid saturation mode");
break;
}
 
Value *ptr = dst.isIndirect(0) ?
fetchSrc(dst.getIndirect(0), 0, NULL) : NULL;
 
if (info->io.genUserClip > 0 &&
dst.getFile() == TGSI_FILE_OUTPUT &&
!dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
mkMov(clipVtx[c], val);
val = clipVtx[c];
}
 
storeDst(dst, c, val, ptr);
}
 
void
Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
Value *val, Value *ptr)
{
const unsigned f = dst.getFile();
const int idx = dst.getIndex(0);
const int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
 
if (f == TGSI_FILE_SYSTEM_VALUE) {
assert(!ptr);
mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
} else
if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
if (ptr || (info->out[idx].mask & (1 << c)))
mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val);
} else
if (f == TGSI_FILE_TEMPORARY ||
f == TGSI_FILE_PREDICATE ||
f == TGSI_FILE_ADDRESS ||
f == TGSI_FILE_OUTPUT) {
getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
} else {
assert(!"invalid dst file");
}
}
 
#define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
for (chan = 0; chan < 4; ++chan) \
if (!inst.getDst(d).isMasked(chan))
 
Value *
Converter::buildDot(int dim)
{
assert(dim > 0);
 
Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
Value *dotp = getScratch();
 
mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
 
for (int c = 1; c < dim; ++c) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
}
return dotp;
}
 
void
Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
{
FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
join->fixed = 1;
conv->insertHead(join);
 
fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
fork->insertBefore(fork->getExit(), fork->joinAt);
}
 
void
Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
{
unsigned rIdx = 0, sIdx = 0;
 
if (R >= 0)
rIdx = tgsi.getSrc(R).getIndex(0);
if (S >= 0)
sIdx = tgsi.getSrc(S).getIndex(0);
 
tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
 
if (tgsi.getSrc(R).isIndirect(0)) {
tex->tex.rIndirectSrc = s;
tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
}
if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
tex->tex.sIndirectSrc = s;
tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
}
}
 
void
Converter::handleTXQ(Value *dst0[4], enum TexQuery query)
{
TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
tex->tex.query = query;
unsigned int c, d;
 
for (d = 0, c = 0; c < 4; ++c) {
if (!dst0[c])
continue;
tex->tex.mask |= 1 << c;
tex->setDef(d++, dst0[c]);
}
tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
 
setTexRS(tex, c, 1, -1);
 
bb->insertTail(tex);
}
 
void
Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
{
Value *proj = fetchSrc(0, 3);
Instruction *insn = proj->getUniqueInsn();
int c;
 
if (insn->op == OP_PINTERP) {
bb->insertTail(insn = cloneForward(func, insn));
insn->op = OP_LINTERP;
insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
insn->setSrc(1, NULL);
proj = insn->getDef(0);
}
proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
 
for (c = 0; c < 4; ++c) {
if (!(mask & (1 << c)))
continue;
if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
continue;
mask &= ~(1 << c);
 
bb->insertTail(insn = cloneForward(func, insn));
insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
insn->setSrc(1, proj);
dst[c] = insn->getDef(0);
}
if (!mask)
return;
 
proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
 
for (c = 0; c < 4; ++c)
if (mask & (1 << c))
dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
}
 
// order of nv50 ir sources: x y z layer lod/bias shadow
// order of TGSI TEX sources: x y z layer shadow lod/bias
// lowering will finally set the hw specific order (like array first on nvc0)
void
Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
{
Value *val;
Value *arg[4], *src[8];
Value *lod = NULL, *shd = NULL;
unsigned int s, c, d;
TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
 
TexInstruction::Target tgt = tgsi.getTexture(code, R);
 
for (s = 0; s < tgt.getArgCount(); ++s)
arg[s] = src[s] = fetchSrc(0, s);
 
if (texi->op == OP_TXL || texi->op == OP_TXB)
lod = fetchSrc(L >> 4, L & 3);
 
if (C == 0x0f)
C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
 
if (tgt.isShadow())
shd = fetchSrc(C >> 4, C & 3);
 
if (texi->op == OP_TXD) {
for (c = 0; c < tgt.getDim(); ++c) {
texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
}
}
 
// cube textures don't care about projection value, it's divided out
if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
unsigned int n = tgt.getDim();
if (shd) {
arg[n] = shd;
++n;
assert(tgt.getDim() == tgt.getArgCount());
}
loadProjTexCoords(src, arg, (1 << n) - 1);
if (shd)
shd = src[n - 1];
}
 
if (tgt.isCube()) {
for (c = 0; c < 3; ++c)
src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]);
val = getScratch();
mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
mkOp1(OP_RCP, TYPE_F32, val, val);
for (c = 0; c < 3; ++c)
src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val);
}
 
for (c = 0, d = 0; c < 4; ++c) {
if (dst[c]) {
texi->setDef(d++, dst[c]);
texi->tex.mask |= 1 << c;
} else {
// NOTE: maybe hook up def too, for CSE
}
}
for (s = 0; s < tgt.getArgCount(); ++s)
texi->setSrc(s, src[s]);
if (lod)
texi->setSrc(s++, lod);
if (shd)
texi->setSrc(s++, shd);
 
setTexRS(texi, s, R, S);
 
if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
texi->tex.levelZero = true;
 
bb->insertTail(texi);
}
 
// 1st source: xyz = coordinates, w = lod/sample
// 2nd source: offset
void
Converter::handleTXF(Value *dst[4], int R, int L_M)
{
TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
int ms;
unsigned int c, d, s;
 
texi->tex.target = tgsi.getTexture(code, R);
 
ms = texi->tex.target.isMS() ? 1 : 0;
texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
 
for (c = 0, d = 0; c < 4; ++c) {
if (dst[c]) {
texi->setDef(d++, dst[c]);
texi->tex.mask |= 1 << c;
}
}
for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
texi->setSrc(c, fetchSrc(0, c));
texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
 
setTexRS(texi, c, R, -1);
 
for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
for (c = 0; c < 3; ++c) {
texi->tex.offset[s][c] = tgsi.getTexOffset(s).getValueU32(c, info);
if (texi->tex.offset[s][c])
texi->tex.useOffsets = s + 1;
}
}
 
bb->insertTail(texi);
}
 
void
Converter::handleLIT(Value *dst0[4])
{
Value *val0 = NULL;
unsigned int mask = tgsi.getDst(0).getMask();
 
if (mask & (1 << 0))
loadImm(dst0[0], 1.0f);
 
if (mask & (1 << 3))
loadImm(dst0[3], 1.0f);
 
if (mask & (3 << 1)) {
val0 = getScratch();
mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
if (mask & (1 << 1))
mkMov(dst0[1], val0);
}
 
if (mask & (1 << 2)) {
Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
Value *val1 = getScratch(), *val3 = getScratch();
 
Value *pos128 = loadImm(NULL, +127.999999f);
Value *neg128 = loadImm(NULL, -127.999999f);
 
mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
 
mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], val3, zero, val0);
}
}
 
static inline bool
isResourceSpecial(const int r)
{
return (r == TGSI_RESOURCE_GLOBAL ||
r == TGSI_RESOURCE_LOCAL ||
r == TGSI_RESOURCE_PRIVATE ||
r == TGSI_RESOURCE_INPUT);
}
 
static inline bool
isResourceRaw(const struct tgsi::Source *code, const int r)
{
return isResourceSpecial(r) || code->resources[r].raw;
}
 
static inline nv50_ir::TexTarget
getResourceTarget(const struct tgsi::Source *code, int r)
{
if (isResourceSpecial(r))
return nv50_ir::TEX_TARGET_BUFFER;
return tgsi::translateTexture(code->resources.at(r).target);
}
 
Symbol *
Converter::getResourceBase(const int r)
{
Symbol *sym = NULL;
 
switch (r) {
case TGSI_RESOURCE_GLOBAL:
sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15);
break;
case TGSI_RESOURCE_LOCAL:
assert(prog->getType() == Program::TYPE_COMPUTE);
sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
info->prop.cp.sharedOffset);
break;
case TGSI_RESOURCE_PRIVATE:
sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
info->bin.tlsSpace);
break;
case TGSI_RESOURCE_INPUT:
assert(prog->getType() == Program::TYPE_COMPUTE);
sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
info->prop.cp.inputOffset);
break;
default:
sym = new_Symbol(prog,
nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
break;
}
return sym;
}
 
void
Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
{
const int arg =
TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
 
for (int c = 0; c < arg; ++c)
coords.push_back(fetchSrc(s, c));
 
// NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
if (r == TGSI_RESOURCE_LOCAL ||
r == TGSI_RESOURCE_PRIVATE ||
r == TGSI_RESOURCE_INPUT)
coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
coords[0]);
}
 
static inline int
partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
{
int n = 0;
 
while (mask) {
if (mask & 1) {
size[n]++;
} else {
if (size[n])
comp[n = 1] = size[0] + 1;
else
comp[n]++;
}
mask >>= 1;
}
if (size[0] == 3) {
n = 1;
size[0] = (comp[0] == 1) ? 1 : 2;
size[1] = 3 - size[0];
comp[1] = comp[0] + size[0];
}
return n + 1;
}
 
// For raw loads, granularity is 4 byte.
// Usage of the texture read mask on OP_SULDP is not allowed.
void
Converter::handleLOAD(Value *dst0[4])
{
const int r = tgsi.getSrc(0).getIndex(0);
int c;
std::vector<Value *> off, src, ldv, def;
 
getResourceCoords(off, r, 1);
 
if (isResourceRaw(code, r)) {
uint8_t mask = 0;
uint8_t comp[2] = { 0, 0 };
uint8_t size[2] = { 0, 0 };
 
Symbol *base = getResourceBase(r);
 
// determine the base and size of the at most 2 load ops
for (c = 0; c < 4; ++c)
if (!tgsi.getDst(0).isMasked(c))
mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
 
int n = partitionLoadStore(comp, size, mask);
 
src = off;
 
def.resize(4); // index by component, the ones we need will be non-NULL
for (c = 0; c < 4; ++c) {
if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
def[c] = dst0[c];
else
if (mask & (1 << c))
def[c] = getScratch();
}
 
const bool useLd = isResourceSpecial(r) ||
(info->io.nv50styleSurfaces &&
code->resources[r].target == TGSI_TEXTURE_BUFFER);
 
for (int i = 0; i < n; ++i) {
ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
 
if (comp[i]) // adjust x component of source address if necessary
src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
off[0], mkImm(comp[i] * 4));
else
src[0] = off[0];
 
if (useLd) {
Instruction *ld =
mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
for (size_t c = 1; c < ldv.size(); ++c)
ld->setDef(c, ldv[c]);
} else {
mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
0, ldv, src)->dType = typeOfSize(size[i] * 4);
}
}
} else {
def.resize(4);
for (c = 0; c < 4; ++c) {
if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
def[c] = getScratch();
else
def[c] = dst0[c];
}
 
mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
def, off);
}
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
if (dst0[c] != def[c])
mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
}
 
// For formatted stores, the write mask on OP_SUSTP can be used.
// Raw stores have to be split.
void
Converter::handleSTORE()
{
const int r = tgsi.getDst(0).getIndex(0);
int c;
std::vector<Value *> off, src, dummy;
 
getResourceCoords(off, r, 0);
src = off;
const int s = src.size();
 
if (isResourceRaw(code, r)) {
uint8_t comp[2] = { 0, 0 };
uint8_t size[2] = { 0, 0 };
 
int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
 
Symbol *base = getResourceBase(r);
 
const bool useSt = isResourceSpecial(r) ||
(info->io.nv50styleSurfaces &&
code->resources[r].target == TGSI_TEXTURE_BUFFER);
 
for (int i = 0; i < n; ++i) {
if (comp[i]) // adjust x component of source address if necessary
src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
off[0], mkImm(comp[i] * 4));
else
src[0] = off[0];
 
const DataType stTy = typeOfSize(size[i] * 4);
 
if (useSt) {
Instruction *st =
mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
for (c = 1; c < size[i]; ++c)
st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
st->setIndirect(0, 0, src[0]);
} else {
// attach values to be stored
src.resize(s + size[i]);
for (c = 0; c < size[i]; ++c)
src[s + c] = fetchSrc(1, comp[i] + c);
mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
0, dummy, src)->setType(stTy);
}
}
} else {
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
src.push_back(fetchSrc(1, c));
 
mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
dummy, src)->tex.mask = tgsi.getDst(0).getMask();
}
}
 
// XXX: These only work on resources with the single-component u32/s32 formats.
// Therefore the result is replicated. This might not be intended by TGSI, but
// operating on more than 1 component would produce undefined results because
// they do not exist.
void
Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
{
const int r = tgsi.getSrc(0).getIndex(0);
std::vector<Value *> srcv;
std::vector<Value *> defv;
LValue *dst = getScratch();
 
getResourceCoords(srcv, r, 1);
 
if (isResourceSpecial(r)) {
assert(r != TGSI_RESOURCE_INPUT);
Instruction *insn;
insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
insn->subOp = subOp;
if (subOp == NV50_IR_SUBOP_ATOM_CAS)
insn->setSrc(2, fetchSrc(3, 0));
insn->setIndirect(0, 0, srcv.at(0));
} else {
operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
TexTarget targ = getResourceTarget(code, r);
int idx = code->resources[r].slot;
defv.push_back(dst);
srcv.push_back(fetchSrc(2, 0));
if (subOp == NV50_IR_SUBOP_ATOM_CAS)
srcv.push_back(fetchSrc(3, 0));
TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
tex->subOp = subOp;
tex->tex.mask = 1;
tex->setType(ty);
}
 
for (int c = 0; c < 4; ++c)
if (dst0[c])
dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
}
 
Converter::Subroutine *
Converter::getSubroutine(unsigned ip)
{
std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
 
if (it == sub.map.end())
it = sub.map.insert(std::make_pair(
ip, Subroutine(new Function(prog, "SUB", ip)))).first;
 
return &it->second;
}
 
Converter::Subroutine *
Converter::getSubroutine(Function *f)
{
unsigned ip = f->getLabel();
std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
 
if (it == sub.map.end())
it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
 
return &it->second;
}
 
bool
Converter::isEndOfSubroutine(uint ip)
{
assert(ip < code->scan.num_instructions);
tgsi::Instruction insn(&code->insns[ip]);
return (insn.getOpcode() == TGSI_OPCODE_END ||
insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
// does END occur at end of main or the very end ?
insn.getOpcode() == TGSI_OPCODE_BGNSUB);
}
 
bool
Converter::handleInstruction(const struct tgsi_full_instruction *insn)
{
Instruction *geni;
 
Value *dst0[4], *rDst0[4];
Value *src0, *src1, *src2;
Value *val0, *val1;
int c;
 
tgsi = tgsi::Instruction(insn);
 
bool useScratchDst = tgsi.checkDstSrcAliasing();
 
operation op = tgsi.getOP();
dstTy = tgsi.inferDstType();
srcTy = tgsi.inferSrcType();
 
unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
 
if (tgsi.dstCount()) {
for (c = 0; c < 4; ++c) {
rDst0[c] = acquireDst(0, c);
dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
}
}
 
switch (tgsi.getOpcode()) {
case TGSI_OPCODE_ADD:
case TGSI_OPCODE_UADD:
case TGSI_OPCODE_AND:
case TGSI_OPCODE_DIV:
case TGSI_OPCODE_IDIV:
case TGSI_OPCODE_UDIV:
case TGSI_OPCODE_MAX:
case TGSI_OPCODE_MIN:
case TGSI_OPCODE_IMAX:
case TGSI_OPCODE_IMIN:
case TGSI_OPCODE_UMAX:
case TGSI_OPCODE_UMIN:
case TGSI_OPCODE_MOD:
case TGSI_OPCODE_UMOD:
case TGSI_OPCODE_MUL:
case TGSI_OPCODE_UMUL:
case TGSI_OPCODE_OR:
case TGSI_OPCODE_POW:
case TGSI_OPCODE_SHL:
case TGSI_OPCODE_ISHR:
case TGSI_OPCODE_USHR:
case TGSI_OPCODE_SUB:
case TGSI_OPCODE_XOR:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
mkOp2(op, dstTy, dst0[c], src0, src1);
}
break;
case TGSI_OPCODE_MAD:
case TGSI_OPCODE_UMAD:
case TGSI_OPCODE_SAD:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
src2 = fetchSrc(2, c);
mkOp3(op, dstTy, dst0[c], src0, src1, src2);
}
break;
case TGSI_OPCODE_MOV:
case TGSI_OPCODE_ABS:
case TGSI_OPCODE_CEIL:
case TGSI_OPCODE_FLR:
case TGSI_OPCODE_TRUNC:
case TGSI_OPCODE_RCP:
case TGSI_OPCODE_IABS:
case TGSI_OPCODE_INEG:
case TGSI_OPCODE_NOT:
case TGSI_OPCODE_DDX:
case TGSI_OPCODE_DDY:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
break;
case TGSI_OPCODE_RSQ:
src0 = fetchSrc(0, 0);
val0 = getScratch();
mkOp1(OP_ABS, TYPE_F32, val0, src0);
mkOp1(OP_RSQ, TYPE_F32, val0, val0);
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkMov(dst0[c], val0);
break;
case TGSI_OPCODE_ARL:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = ROUND_M;
mkOp2(OP_SHL, TYPE_U32, dst0[c], dst0[c], mkImm(4));
}
break;
case TGSI_OPCODE_UARL:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkOp2(OP_SHL, TYPE_U32, dst0[c], fetchSrc(0, c), mkImm(4));
break;
case TGSI_OPCODE_EX2:
case TGSI_OPCODE_LG2:
val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
break;
case TGSI_OPCODE_COS:
case TGSI_OPCODE_SIN:
val0 = getScratch();
if (mask & 7) {
mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
mkOp1(op, TYPE_F32, val0, val0);
for (c = 0; c < 3; ++c)
if (dst0[c])
mkMov(dst0[c], val0);
}
if (dst0[3]) {
mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
mkOp1(op, TYPE_F32, dst0[3], val0);
}
break;
case TGSI_OPCODE_SCS:
if (mask & 3) {
val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
if (dst0[0])
mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
if (dst0[1])
mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
}
if (dst0[2])
loadImm(dst0[2], 0.0f);
if (dst0[3])
loadImm(dst0[3], 1.0f);
break;
case TGSI_OPCODE_EXP:
src0 = fetchSrc(0, 0);
val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
if (dst0[1])
mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
if (dst0[0])
mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
if (dst0[2])
mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
if (dst0[3])
loadImm(dst0[3], 1.0f);
break;
case TGSI_OPCODE_LOG:
src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
if (dst0[0] || dst0[1])
val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
if (dst0[1]) {
mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
}
if (dst0[3])
loadImm(dst0[3], 1.0f);
break;
case TGSI_OPCODE_DP2:
val0 = buildDot(2);
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkMov(dst0[c], val0);
break;
case TGSI_OPCODE_DP3:
val0 = buildDot(3);
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkMov(dst0[c], val0);
break;
case TGSI_OPCODE_DP4:
val0 = buildDot(4);
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkMov(dst0[c], val0);
break;
case TGSI_OPCODE_DPH:
val0 = buildDot(3);
src1 = fetchSrc(1, 3);
mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkMov(dst0[c], val0);
break;
case TGSI_OPCODE_DST:
if (dst0[0])
loadImm(dst0[0], 1.0f);
if (dst0[1]) {
src0 = fetchSrc(0, 1);
src1 = fetchSrc(1, 1);
mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
}
if (dst0[2])
mkMov(dst0[2], fetchSrc(0, 2));
if (dst0[3])
mkMov(dst0[3], fetchSrc(1, 3));
break;
case TGSI_OPCODE_LRP:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
src2 = fetchSrc(2, c);
mkOp3(OP_MAD, TYPE_F32, dst0[c],
mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
}
break;
case TGSI_OPCODE_LIT:
handleLIT(dst0);
break;
case TGSI_OPCODE_XPD:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
if (c < 3) {
val0 = getSSA();
src0 = fetchSrc(1, (c + 1) % 3);
src1 = fetchSrc(0, (c + 2) % 3);
mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
mkOp1(OP_NEG, TYPE_F32, val0, val0);
 
src0 = fetchSrc(0, (c + 1) % 3);
src1 = fetchSrc(1, (c + 2) % 3);
mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
} else {
loadImm(dst0[c], 1.0f);
}
}
break;
case TGSI_OPCODE_ISSG:
case TGSI_OPCODE_SSG:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
val0 = getScratch();
val1 = getScratch();
mkCmp(OP_SET, CC_GT, srcTy, val0, src0, zero);
mkCmp(OP_SET, CC_LT, srcTy, val1, src0, zero);
if (srcTy == TYPE_F32)
mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
else
mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
}
break;
case TGSI_OPCODE_UCMP:
case TGSI_OPCODE_CMP:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
src2 = fetchSrc(2, c);
if (src1 == src2)
mkMov(dst0[c], src1);
else
mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
srcTy, dst0[c], src1, src2, src0);
}
break;
case TGSI_OPCODE_FRC:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
val0 = getScratch();
mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
}
break;
case TGSI_OPCODE_ROUND:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
->rnd = ROUND_NI;
break;
case TGSI_OPCODE_CLAMP:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
src2 = fetchSrc(2, c);
val0 = getScratch();
mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
}
break;
case TGSI_OPCODE_SLT:
case TGSI_OPCODE_SGE:
case TGSI_OPCODE_SEQ:
case TGSI_OPCODE_SFL:
case TGSI_OPCODE_SGT:
case TGSI_OPCODE_SLE:
case TGSI_OPCODE_SNE:
case TGSI_OPCODE_STR:
case TGSI_OPCODE_ISGE:
case TGSI_OPCODE_ISLT:
case TGSI_OPCODE_USEQ:
case TGSI_OPCODE_USGE:
case TGSI_OPCODE_USLT:
case TGSI_OPCODE_USNE:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
src0 = fetchSrc(0, c);
src1 = fetchSrc(1, c);
mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], src0, src1);
}
break;
case TGSI_OPCODE_KILL_IF:
val0 = new_LValue(func, FILE_PREDICATE);
for (c = 0; c < 4; ++c) {
mkCmp(OP_SET, CC_LT, TYPE_F32, val0, fetchSrc(0, c), zero);
mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
}
break;
case TGSI_OPCODE_KILL:
mkOp(OP_DISCARD, TYPE_NONE, NULL);
break;
case TGSI_OPCODE_TEX:
case TGSI_OPCODE_TXB:
case TGSI_OPCODE_TXL:
case TGSI_OPCODE_TXP:
// R S L C Dx Dy
handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
break;
case TGSI_OPCODE_TXD:
handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
break;
case TGSI_OPCODE_TEX2:
handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
break;
case TGSI_OPCODE_TXB2:
case TGSI_OPCODE_TXL2:
handleTEX(dst0, 2, 2, 0x10, 0x11, 0x00, 0x00);
break;
case TGSI_OPCODE_SAMPLE:
case TGSI_OPCODE_SAMPLE_B:
case TGSI_OPCODE_SAMPLE_D:
case TGSI_OPCODE_SAMPLE_L:
case TGSI_OPCODE_SAMPLE_C:
case TGSI_OPCODE_SAMPLE_C_LZ:
handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
break;
case TGSI_OPCODE_TXF:
handleTXF(dst0, 1, 0x03);
break;
case TGSI_OPCODE_SAMPLE_I:
handleTXF(dst0, 1, 0x03);
break;
case TGSI_OPCODE_SAMPLE_I_MS:
handleTXF(dst0, 1, 0x20);
break;
case TGSI_OPCODE_TXQ:
case TGSI_OPCODE_SVIEWINFO:
handleTXQ(dst0, TXQ_DIMS);
break;
case TGSI_OPCODE_F2I:
case TGSI_OPCODE_F2U:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
break;
case TGSI_OPCODE_I2F:
case TGSI_OPCODE_U2F:
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
break;
case TGSI_OPCODE_EMIT:
case TGSI_OPCODE_ENDPRIM:
// get vertex stream if specified (must be immediate)
src0 = tgsi.srcCount() ?
mkImm(tgsi.getSrc(0).getValueU32(0, info)) : zero;
mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
break;
case TGSI_OPCODE_IF:
case TGSI_OPCODE_UIF:
{
BasicBlock *ifBB = new BasicBlock(func);
 
bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
condBBs.push(bb);
joinBBs.push(bb);
 
mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
 
setPosition(ifBB, true);
}
break;
case TGSI_OPCODE_ELSE:
{
BasicBlock *elseBB = new BasicBlock(func);
BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
 
forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
condBBs.push(bb);
 
forkBB->getExit()->asFlow()->target.bb = elseBB;
if (!bb->isTerminated())
mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
 
setPosition(elseBB, true);
}
break;
case TGSI_OPCODE_ENDIF:
{
BasicBlock *convBB = new BasicBlock(func);
BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
 
if (!bb->isTerminated()) {
// we only want join if none of the clauses ended with CONT/BREAK/RET
if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
insertConvergenceOps(convBB, forkBB);
mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
}
 
if (prevBB->getExit()->op == OP_BRA) {
prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
prevBB->getExit()->asFlow()->target.bb = convBB;
}
setPosition(convBB, true);
}
break;
case TGSI_OPCODE_BGNLOOP:
{
BasicBlock *lbgnBB = new BasicBlock(func);
BasicBlock *lbrkBB = new BasicBlock(func);
 
loopBBs.push(lbgnBB);
breakBBs.push(lbrkBB);
if (loopBBs.getSize() > func->loopNestingBound)
func->loopNestingBound++;
 
mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
 
bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
setPosition(lbgnBB, true);
mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
}
break;
case TGSI_OPCODE_ENDLOOP:
{
BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
 
if (!bb->isTerminated()) {
mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
}
setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
}
break;
case TGSI_OPCODE_BRK:
{
if (bb->isTerminated())
break;
BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
}
break;
case TGSI_OPCODE_CONT:
{
if (bb->isTerminated())
break;
BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
contBB->explicitCont = true;
bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
}
break;
case TGSI_OPCODE_BGNSUB:
{
Subroutine *s = getSubroutine(ip);
BasicBlock *entry = new BasicBlock(s->f);
BasicBlock *leave = new BasicBlock(s->f);
 
// multiple entrypoints possible, keep the graph connected
if (prog->getType() == Program::TYPE_COMPUTE)
prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
 
sub.cur = s;
s->f->setEntry(entry);
s->f->setExit(leave);
setPosition(entry, true);
return true;
}
case TGSI_OPCODE_ENDSUB:
{
sub.cur = getSubroutine(prog->main);
setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
return true;
}
case TGSI_OPCODE_CAL:
{
Subroutine *s = getSubroutine(tgsi.getLabel());
mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
func->call.attach(&s->f->call, Graph::Edge::TREE);
return true;
}
case TGSI_OPCODE_RET:
{
if (bb->isTerminated())
return true;
BasicBlock *leave = BasicBlock::get(func->cfgExit);
 
if (!isEndOfSubroutine(ip + 1)) {
// insert a PRERET at the entry if this is an early return
// (only needed for sharing code in the epilogue)
BasicBlock *pos = getBB();
setPosition(BasicBlock::get(func->cfg.getRoot()), false);
mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
setPosition(pos, true);
}
mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
}
break;
case TGSI_OPCODE_END:
{
// attach and generate epilogue code
BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
setPosition(epilogue, true);
if (prog->getType() == Program::TYPE_FRAGMENT)
exportOutputs();
if (info->io.genUserClip > 0)
handleUserClipPlanes();
mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
}
break;
case TGSI_OPCODE_SWITCH:
case TGSI_OPCODE_CASE:
ERROR("switch/case opcode encountered, should have been lowered\n");
abort();
break;
case TGSI_OPCODE_LOAD:
handleLOAD(dst0);
break;
case TGSI_OPCODE_STORE:
handleSTORE();
break;
case TGSI_OPCODE_BARRIER:
geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
geni->fixed = 1;
geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
break;
case TGSI_OPCODE_MFENCE:
case TGSI_OPCODE_LFENCE:
case TGSI_OPCODE_SFENCE:
geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
geni->fixed = 1;
geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
break;
case TGSI_OPCODE_ATOMUADD:
case TGSI_OPCODE_ATOMXCHG:
case TGSI_OPCODE_ATOMCAS:
case TGSI_OPCODE_ATOMAND:
case TGSI_OPCODE_ATOMOR:
case TGSI_OPCODE_ATOMXOR:
case TGSI_OPCODE_ATOMUMIN:
case TGSI_OPCODE_ATOMIMIN:
case TGSI_OPCODE_ATOMUMAX:
case TGSI_OPCODE_ATOMIMAX:
handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
break;
default:
ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
assert(0);
break;
}
 
if (tgsi.dstCount()) {
for (c = 0; c < 4; ++c) {
if (!dst0[c])
continue;
if (dst0[c] != rDst0[c])
mkMov(rDst0[c], dst0[c]);
storeDst(0, c, rDst0[c]);
}
}
vtxBaseValid = 0;
 
return true;
}
 
void
Converter::handleUserClipPlanes()
{
Value *res[8];
int n, i, c;
 
for (c = 0; c < 4; ++c) {
for (i = 0; i < info->io.genUserClip; ++i) {
Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.ucpCBSlot,
TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
if (c == 0)
res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
else
mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
}
}
 
const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
 
for (i = 0; i < info->io.genUserClip; ++i) {
n = i / 4 + first;
c = i % 4;
Symbol *sym =
mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
}
}
 
void
Converter::exportOutputs()
{
for (unsigned int i = 0; i < info->numOutputs; ++i) {
for (unsigned int c = 0; c < 4; ++c) {
if (!oData.exists(sub.cur->values, i, c))
continue;
Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
info->out[i].slot[c] * 4);
Value *val = oData.load(sub.cur->values, i, c, NULL);
if (val)
mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
}
}
}
 
Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
code(code),
tgsi(NULL),
tData(this), aData(this), pData(this), oData(this)
{
info = code->info;
 
const DataFile tFile = code->mainTempsInLMem ? FILE_MEMORY_LOCAL : FILE_GPR;
 
const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
 
tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, tFile, 0);
pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_ADDRESS, 0);
oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
 
zero = mkImm((uint32_t)0);
 
vtxBaseValid = 0;
}
 
Converter::~Converter()
{
}
 
inline const Converter::Location *
Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
{
ValueMap::l_iterator it = s->values.l.find(v);
return it == s->values.l.end() ? NULL : &it->second;
}
 
template<typename T> inline void
Converter::BindArgumentsPass::updateCallArgs(
Instruction *i, void (Instruction::*setArg)(int, Value *),
T (Function::*proto))
{
Function *g = i->asFlow()->target.fn;
Subroutine *subg = conv.getSubroutine(g);
 
for (unsigned a = 0; a < (g->*proto).size(); ++a) {
Value *v = (g->*proto)[a].get();
const Converter::Location &l = *getValueLocation(subg, v);
Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
 
(i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
}
}
 
template<typename T> inline void
Converter::BindArgumentsPass::updatePrototype(
BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
{
(func->*updateSet)();
 
for (unsigned i = 0; i < set->getSize(); ++i) {
Value *v = func->getLValue(i);
const Converter::Location *l = getValueLocation(sub, v);
 
// only include values with a matching TGSI register
if (set->test(i) && l && !conv.code->locals.count(*l))
(func->*proto).push_back(v);
}
}
 
bool
Converter::BindArgumentsPass::visit(Function *f)
{
sub = conv.getSubroutine(f);
 
for (ArrayList::Iterator bi = f->allBBlocks.iterator();
!bi.end(); bi.next()) {
for (Instruction *i = BasicBlock::get(bi)->getFirst();
i; i = i->next) {
if (i->op == OP_CALL && !i->asFlow()->builtin) {
updateCallArgs(i, &Instruction::setSrc, &Function::ins);
updateCallArgs(i, &Instruction::setDef, &Function::outs);
}
}
}
 
if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
return true;
updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
&Function::buildLiveSets, &Function::ins);
updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
&Function::buildDefSets, &Function::outs);
 
return true;
}
 
bool
Converter::run()
{
BasicBlock *entry = new BasicBlock(prog->main);
BasicBlock *leave = new BasicBlock(prog->main);
 
prog->main->setEntry(entry);
prog->main->setExit(leave);
 
setPosition(entry, true);
sub.cur = getSubroutine(prog->main);
 
if (info->io.genUserClip > 0) {
for (int c = 0; c < 4; ++c)
clipVtx[c] = getScratch();
}
 
if (prog->getType() == Program::TYPE_FRAGMENT) {
Symbol *sv = mkSysVal(SV_POSITION, 3);
fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
}
 
for (ip = 0; ip < code->scan.num_instructions; ++ip) {
if (!handleInstruction(&code->insns[ip]))
return false;
}
 
if (!BindArgumentsPass(*this).run(prog))
return false;
 
return true;
}
 
} // unnamed namespace
 
namespace nv50_ir {
 
bool
Program::makeFromTGSI(struct nv50_ir_prog_info *info)
{
tgsi::Source src(info);
if (!src.scanSource())
return false;
tlsSize = info->bin.tlsSpace;
 
Converter builder(this, &src);
return builder.run();
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_graph.cpp
0,0 → 1,436
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir_graph.h"
#include <limits>
#include <list>
#include <stack>
#include "nv50_ir.h"
 
namespace nv50_ir {
 
Graph::Graph()
{
root = NULL;
size = 0;
sequence = 0;
}
 
Graph::~Graph()
{
for (IteratorRef it = safeIteratorDFS(); !it->end(); it->next())
reinterpret_cast<Node *>(it->get())->cut();
}
 
void Graph::insert(Node *node)
{
if (!root)
root = node;
 
node->graph = this;
size++;
}
 
void Graph::Edge::unlink()
{
if (origin) {
prev[0]->next[0] = next[0];
next[0]->prev[0] = prev[0];
if (origin->out == this)
origin->out = (next[0] == this) ? NULL : next[0];
 
--origin->outCount;
}
if (target) {
prev[1]->next[1] = next[1];
next[1]->prev[1] = prev[1];
if (target->in == this)
target->in = (next[1] == this) ? NULL : next[1];
 
--target->inCount;
}
}
 
const char *Graph::Edge::typeStr() const
{
switch (type) {
case TREE: return "tree";
case FORWARD: return "forward";
case BACK: return "back";
case CROSS: return "cross";
case DUMMY: return "dummy";
case UNKNOWN:
default:
return "unk";
}
}
 
Graph::Node::Node(void *priv) : data(priv),
in(0), out(0), graph(0),
visited(0),
inCount(0), outCount(0)
{
// nothing to do
}
 
void Graph::Node::attach(Node *node, Edge::Type kind)
{
Edge *edge = new Edge(this, node, kind);
 
// insert head
if (this->out) {
edge->next[0] = this->out;
edge->prev[0] = this->out->prev[0];
edge->prev[0]->next[0] = edge;
this->out->prev[0] = edge;
}
this->out = edge;
 
if (node->in) {
edge->next[1] = node->in;
edge->prev[1] = node->in->prev[1];
edge->prev[1]->next[1] = edge;
node->in->prev[1] = edge;
}
node->in = edge;
 
++this->outCount;
++node->inCount;
 
assert(graph || node->graph);
if (!node->graph)
graph->insert(node);
if (!graph)
node->graph->insert(this);
 
if (kind == Edge::UNKNOWN)
graph->classifyEdges();
}
 
bool Graph::Node::detach(Graph::Node *node)
{
EdgeIterator ei = this->outgoing();
for (; !ei.end(); ei.next())
if (ei.getNode() == node)
break;
if (ei.end()) {
ERROR("no such node attached\n");
return false;
}
delete ei.getEdge();
return true;
}
 
// Cut a node from the graph, deleting all attached edges.
void Graph::Node::cut()
{
while (out)
delete out;
while (in)
delete in;
 
if (graph) {
if (graph->root == this)
graph->root = NULL;
graph = NULL;
}
}
 
Graph::Edge::Edge(Node *org, Node *tgt, Type kind)
{
target = tgt;
origin = org;
type = kind;
 
next[0] = next[1] = this;
prev[0] = prev[1] = this;
}
 
bool
Graph::Node::reachableBy(const Node *node, const Node *term) const
{
std::stack<const Node *> stack;
const Node *pos = NULL;
const int seq = graph->nextSequence();
 
stack.push(node);
 
while (!stack.empty()) {
pos = stack.top();
stack.pop();
 
if (pos == this)
return true;
if (pos == term)
continue;
 
for (EdgeIterator ei = pos->outgoing(); !ei.end(); ei.next()) {
if (ei.getType() == Edge::BACK || ei.getType() == Edge::DUMMY)
continue;
if (ei.getNode()->visit(seq))
stack.push(ei.getNode());
}
}
return pos == this;
}
 
class DFSIterator : public Iterator
{
public:
DFSIterator(Graph *graph, const bool preorder)
{
unsigned int seq = graph->nextSequence();
 
nodes = new Graph::Node * [graph->getSize() + 1];
count = 0;
pos = 0;
nodes[graph->getSize()] = 0;
 
if (graph->getRoot()) {
graph->getRoot()->visit(seq);
search(graph->getRoot(), preorder, seq);
}
}
 
~DFSIterator()
{
if (nodes)
delete[] nodes;
}
 
void search(Graph::Node *node, const bool preorder, const int sequence)
{
if (preorder)
nodes[count++] = node;
 
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next())
if (ei.getNode()->visit(sequence))
search(ei.getNode(), preorder, sequence);
 
if (!preorder)
nodes[count++] = node;
}
 
virtual bool end() const { return pos >= count; }
virtual void next() { if (pos < count) ++pos; }
virtual void *get() const { return nodes[pos]; }
virtual void reset() { pos = 0; }
 
protected:
Graph::Node **nodes;
int count;
int pos;
};
 
IteratorRef Graph::iteratorDFS(bool preorder)
{
return IteratorRef(new DFSIterator(this, preorder));
}
 
IteratorRef Graph::safeIteratorDFS(bool preorder)
{
return this->iteratorDFS(preorder);
}
 
class CFGIterator : public Iterator
{
public:
CFGIterator(Graph *graph)
{
nodes = new Graph::Node * [graph->getSize() + 1];
count = 0;
pos = 0;
nodes[graph->getSize()] = 0;
 
// TODO: argh, use graph->sequence instead of tag and just raise it by > 1
for (IteratorRef it = graph->iteratorDFS(); !it->end(); it->next())
reinterpret_cast<Graph::Node *>(it->get())->tag = 0;
 
if (graph->getRoot())
search(graph->getRoot(), graph->nextSequence());
}
 
~CFGIterator()
{
if (nodes)
delete[] nodes;
}
 
virtual void *get() const { return nodes[pos]; }
virtual bool end() const { return pos >= count; }
virtual void next() { if (pos < count) ++pos; }
virtual void reset() { pos = 0; }
 
private:
void search(Graph::Node *node, const int sequence)
{
Stack bb, cross;
 
bb.push(node);
 
while (bb.getSize()) {
node = reinterpret_cast<Graph::Node *>(bb.pop().u.p);
assert(node);
if (!node->visit(sequence))
continue;
node->tag = 0;
 
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) {
switch (ei.getType()) {
case Graph::Edge::TREE:
case Graph::Edge::FORWARD:
case Graph::Edge::DUMMY:
if (++(ei.getNode()->tag) == ei.getNode()->incidentCountFwd())
bb.push(ei.getNode());
break;
case Graph::Edge::BACK:
continue;
case Graph::Edge::CROSS:
if (++(ei.getNode()->tag) == 1)
cross.push(ei.getNode());
break;
default:
assert(!"unknown edge kind in CFG");
break;
}
}
nodes[count++] = node;
 
if (bb.getSize() == 0)
cross.moveTo(bb);
}
}
 
private:
Graph::Node **nodes;
int count;
int pos;
};
 
IteratorRef Graph::iteratorCFG()
{
return IteratorRef(new CFGIterator(this));
}
 
IteratorRef Graph::safeIteratorCFG()
{
return this->iteratorCFG();
}
 
void Graph::classifyEdges()
{
int seq;
 
for (IteratorRef it = iteratorDFS(true); !it->end(); it->next()) {
Node *node = reinterpret_cast<Node *>(it->get());
node->visit(0);
node->tag = 0;
}
 
classifyDFS(root, (seq = 0));
 
sequence = seq;
}
 
void Graph::classifyDFS(Node *curr, int& seq)
{
Graph::Edge *edge;
Graph::Node *node;
 
curr->visit(++seq);
curr->tag = 1;
 
for (edge = curr->out; edge; edge = edge->next[0]) {
node = edge->target;
if (edge->type == Edge::DUMMY)
continue;
 
if (node->getSequence() == 0) {
edge->type = Edge::TREE;
classifyDFS(node, seq);
} else
if (node->getSequence() > curr->getSequence()) {
edge->type = Edge::FORWARD;
} else {
edge->type = node->tag ? Edge::BACK : Edge::CROSS;
}
}
 
for (edge = curr->in; edge; edge = edge->next[1]) {
node = edge->origin;
if (edge->type == Edge::DUMMY)
continue;
 
if (node->getSequence() == 0) {
edge->type = Edge::TREE;
classifyDFS(node, seq);
} else
if (node->getSequence() > curr->getSequence()) {
edge->type = Edge::FORWARD;
} else {
edge->type = node->tag ? Edge::BACK : Edge::CROSS;
}
}
 
curr->tag = 0;
}
 
// @dist is indexed by Node::tag, returns -1 if no path found
int
Graph::findLightestPathWeight(Node *a, Node *b, const std::vector<int> &weight)
{
std::vector<int> path(weight.size(), std::numeric_limits<int>::max());
std::list<Node *> nodeList;
const int seq = nextSequence();
 
path[a->tag] = 0;
for (Node *c = a; c && c != b;) {
const int p = path[c->tag] + weight[c->tag];
for (EdgeIterator ei = c->outgoing(); !ei.end(); ei.next()) {
Node *t = ei.getNode();
if (t->getSequence() < seq) {
if (path[t->tag] == std::numeric_limits<int>::max())
nodeList.push_front(t);
if (p < path[t->tag])
path[t->tag] = p;
}
}
c->visit(seq);
Node *next = NULL;
for (std::list<Node *>::iterator n = nodeList.begin();
n != nodeList.end(); ++n) {
if (!next || path[(*n)->tag] < path[next->tag])
next = *n;
if ((*n) == c) {
// erase visited
n = nodeList.erase(n);
--n;
}
}
c = next;
}
if (path[b->tag] == std::numeric_limits<int>::max())
return -1;
return path[b->tag];
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_graph.h
0,0 → 1,228
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_GRAPH_H__
#define __NV50_IR_GRAPH_H__
 
#include "nv50_ir_util.h"
#include <vector>
 
namespace nv50_ir {
 
#define ITER_NODE(x) reinterpret_cast<Graph::Node *>((x).get())
#define ITER_EDGE(x) reinterpret_cast<Graph::Edge *>((x).get())
 
// A connected graph.
class Graph
{
public:
class Node;
 
class Edge
{
public:
enum Type
{
UNKNOWN,
TREE,
FORWARD,
BACK,
CROSS, // e.g. loop break
DUMMY
};
 
Edge(Node *dst, Node *src, Type kind);
~Edge() { unlink(); }
 
inline Node *getOrigin() const { return origin; }
inline Node *getTarget() const { return target; }
 
inline Type getType() const { return type; }
const char *typeStr() const;
 
private:
Node *origin;
Node *target;
 
Type type;
Edge *next[2]; // next edge outgoing/incident from/to origin/target
Edge *prev[2];
 
void unlink();
 
friend class Graph;
};
 
class EdgeIterator : public Iterator
{
public:
EdgeIterator() : e(0), t(0), d(0), rev(false) { }
EdgeIterator(Graph::Edge *first, int dir, bool reverse)
: d(dir), rev(reverse)
{
t = e = ((rev && first) ? first->prev[d] : first);
}
 
virtual void next()
{
Graph::Edge *n = (rev ? e->prev[d] : e->next[d]);
e = (n == t ? NULL : n);
}
virtual bool end() const { return !e; }
virtual void *get() const { return e; }
 
inline Node *getNode() const { assert(e); return d ?
e->origin : e->target; }
inline Edge *getEdge() const { return e; }
inline Edge::Type getType() { return e ? e->getType() : Edge::UNKNOWN; }
 
private:
Graph::Edge *e;
Graph::Edge *t;
int d;
bool rev;
};
 
class Node
{
public:
Node(void *);
~Node() { cut(); }
 
void attach(Node *, Edge::Type);
bool detach(Node *);
void cut();
 
inline EdgeIterator outgoing(bool reverse = false) const;
inline EdgeIterator incident(bool reverse = false) const;
 
inline Node *parent() const; // returns NULL if count(incident edges) != 1
 
bool reachableBy(const Node *node, const Node *term) const;
 
inline bool visit(int);
inline int getSequence() const;
 
inline int incidentCountFwd() const; // count of incident non-back edges
inline int incidentCount() const { return inCount; }
inline int outgoingCount() const { return outCount; }
 
Graph *getGraph() const { return graph; }
 
void *data;
 
private:
Edge *in;
Edge *out;
Graph *graph;
 
int visited;
 
int16_t inCount;
int16_t outCount;
public:
int tag; // for temporary use
 
friend class Graph;
};
 
public:
Graph();
~Graph(); // does *not* free the nodes (make it an option ?)
 
inline Node *getRoot() const { return root; }
 
inline unsigned int getSize() const { return size; }
 
inline int nextSequence();
 
void insert(Node *node); // attach to or set as root
 
IteratorRef iteratorDFS(bool preorder = true);
IteratorRef iteratorCFG();
 
// safe iterators are unaffected by changes to the *edges* of the graph
IteratorRef safeIteratorDFS(bool preorder = true);
IteratorRef safeIteratorCFG();
 
void classifyEdges();
 
// @weights: indexed by Node::tag
int findLightestPathWeight(Node *, Node *, const std::vector<int>& weights);
 
private:
void classifyDFS(Node *, int&);
 
private:
Node *root;
unsigned int size;
int sequence;
};
 
int Graph::nextSequence()
{
return ++sequence;
}
 
Graph::Node *Graph::Node::parent() const
{
if (inCount != 1)
return NULL;
assert(in);
return in->origin;
}
 
bool Graph::Node::visit(int v)
{
if (visited == v)
return false;
visited = v;
return true;
}
 
int Graph::Node::getSequence() const
{
return visited;
}
 
Graph::EdgeIterator Graph::Node::outgoing(bool reverse) const
{
return EdgeIterator(out, 0, reverse);
}
 
Graph::EdgeIterator Graph::Node::incident(bool reverse) const
{
return EdgeIterator(in, 1, reverse);
}
 
int Graph::Node::incidentCountFwd() const
{
int n = 0;
for (EdgeIterator ei = incident(); !ei.end(); ei.next())
if (ei.getType() != Edge::BACK)
++n;
return n;
}
 
} // namespace nv50_ir
 
#endif // __NV50_IR_GRAPH_H__
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_inlines.h
0,0 → 1,420
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_INLINES_H__
#define __NV50_IR_INLINES_H__
 
static inline CondCode reverseCondCode(CondCode cc)
{
static const uint8_t ccRev[8] = { 0, 4, 2, 6, 1, 5, 3, 7 };
 
return static_cast<CondCode>(ccRev[cc & 7] | (cc & ~7));
}
 
static inline CondCode inverseCondCode(CondCode cc)
{
return static_cast<CondCode>(cc ^ 7);
}
 
static inline bool isMemoryFile(DataFile f)
{
return (f >= FILE_MEMORY_CONST && f <= FILE_MEMORY_LOCAL);
}
 
// contrary to asTex(), this will never include SULD/SUST
static inline bool isTextureOp(operation op)
{
return (op >= OP_TEX && op <= OP_TEXPREP);
}
 
static inline bool isSurfaceOp(operation op)
{
return (op >= OP_SULDB && op <= OP_SULEA);
}
 
static inline unsigned int typeSizeof(DataType ty)
{
switch (ty) {
case TYPE_U8:
case TYPE_S8:
return 1;
case TYPE_F16:
case TYPE_U16:
case TYPE_S16:
return 2;
case TYPE_F32:
case TYPE_U32:
case TYPE_S32:
return 4;
case TYPE_F64:
case TYPE_U64:
case TYPE_S64:
return 8;
case TYPE_B96:
return 12;
case TYPE_B128:
return 16;
default:
return 0;
}
}
 
static inline unsigned int typeSizeofLog2(DataType ty)
{
switch (ty) {
case TYPE_F16:
case TYPE_U16:
case TYPE_S16:
return 1;
case TYPE_F32:
case TYPE_U32:
case TYPE_S32:
return 2;
case TYPE_F64:
case TYPE_U64:
case TYPE_S64:
return 3;
case TYPE_B96:
case TYPE_B128:
return 4;
case TYPE_U8:
case TYPE_S8:
default:
return 0;
}
}
 
static inline DataType typeOfSize(unsigned int size,
bool flt = false, bool sgn = false)
{
switch (size) {
case 1: return sgn ? TYPE_S8 : TYPE_U8;
case 2: return flt ? TYPE_F16 : (sgn ? TYPE_S16 : TYPE_U16);
case 8: return flt ? TYPE_F64 : (sgn ? TYPE_S64 : TYPE_U64);
case 12: return TYPE_B96;
case 16: return TYPE_B128;
case 4:
return flt ? TYPE_F32 : (sgn ? TYPE_S32 : TYPE_U32);
default:
return TYPE_NONE;
}
}
 
static inline bool isFloatType(DataType ty)
{
return (ty >= TYPE_F16 && ty <= TYPE_F64);
}
 
static inline bool isSignedIntType(DataType ty)
{
return (ty == TYPE_S8 || ty == TYPE_S16 || ty == TYPE_S32);
}
 
static inline bool isSignedType(DataType ty)
{
switch (ty) {
case TYPE_NONE:
case TYPE_U8:
case TYPE_U16:
case TYPE_U32:
case TYPE_B96:
case TYPE_B128:
return false;
default:
return true;
}
}
 
static inline DataType intTypeToSigned(DataType ty)
{
switch (ty) {
case TYPE_U32: return TYPE_S32;
case TYPE_U16: return TYPE_S16;
case TYPE_U8: return TYPE_S8;
default:
return ty;
}
}
 
const ValueRef *ValueRef::getIndirect(int dim) const
{
return isIndirect(dim) ? &insn->src(indirect[dim]) : NULL;
}
 
DataFile ValueRef::getFile() const
{
return value ? value->reg.file : FILE_NULL;
}
 
unsigned int ValueRef::getSize() const
{
return value ? value->reg.size : 0;
}
 
Value *ValueRef::rep() const
{
assert(value);
return value->join;
}
 
Value *ValueDef::rep() const
{
assert(value);
return value->join;
}
 
DataFile ValueDef::getFile() const
{
return value ? value->reg.file : FILE_NULL;
}
 
unsigned int ValueDef::getSize() const
{
return value ? value->reg.size : 0;
}
 
void ValueDef::setSSA(LValue *lval)
{
origin = value->asLValue();
set(lval);
}
 
const LValue *ValueDef::preSSA() const
{
return origin;
}
 
Instruction *Value::getInsn() const
{
return defs.empty() ? NULL : defs.front()->getInsn();
}
 
Instruction *Value::getUniqueInsn() const
{
if (defs.empty())
return NULL;
 
// after regalloc, the definitions of coalesced values are linked
if (join != this) {
for (DefCIterator it = defs.begin(); it != defs.end(); ++it)
if ((*it)->get() == this)
return (*it)->getInsn();
// should be unreachable and trigger assertion at the end
}
#ifdef DEBUG
if (reg.data.id < 0) {
int n = 0;
for (DefCIterator it = defs.begin(); n < 2 && it != defs.end(); ++it)
if ((*it)->get() == this) // don't count joined values
++n;
if (n > 1)
WARN("value %%%i not uniquely defined\n", id); // return NULL ?
}
#endif
assert(defs.front()->get() == this);
return defs.front()->getInsn();
}
 
inline bool Instruction::constrainedDefs() const
{
return defExists(1) || op == OP_UNION;
}
 
Value *Instruction::getIndirect(int s, int dim) const
{
return srcs[s].isIndirect(dim) ? getSrc(srcs[s].indirect[dim]) : NULL;
}
 
Value *Instruction::getPredicate() const
{
return (predSrc >= 0) ? getSrc(predSrc) : NULL;
}
 
void Instruction::setFlagsDef(int d, Value *val)
{
if (val) {
if (flagsDef < 0)
flagsDef = d;
setDef(flagsDef, val);
} else {
if (flagsDef >= 0) {
setDef(flagsDef, NULL);
flagsDef = -1;
}
}
}
 
void Instruction::setFlagsSrc(int s, Value *val)
{
flagsSrc = s;
setSrc(flagsSrc, val);
}
 
Value *TexInstruction::getIndirectR() const
{
return tex.rIndirectSrc >= 0 ? getSrc(tex.rIndirectSrc) : NULL;
}
 
Value *TexInstruction::getIndirectS() const
{
return tex.rIndirectSrc >= 0 ? getSrc(tex.rIndirectSrc) : NULL;
}
 
CmpInstruction *Instruction::asCmp()
{
if (op >= OP_SET_AND && op <= OP_SLCT && op != OP_SELP)
return static_cast<CmpInstruction *>(this);
return NULL;
}
 
const CmpInstruction *Instruction::asCmp() const
{
if (op >= OP_SET_AND && op <= OP_SLCT && op != OP_SELP)
return static_cast<const CmpInstruction *>(this);
return NULL;
}
 
FlowInstruction *Instruction::asFlow()
{
if (op >= OP_BRA && op <= OP_JOIN)
return static_cast<FlowInstruction *>(this);
return NULL;
}
 
const FlowInstruction *Instruction::asFlow() const
{
if (op >= OP_BRA && op <= OP_JOINAT)
return static_cast<const FlowInstruction *>(this);
return NULL;
}
 
TexInstruction *Instruction::asTex()
{
if (op >= OP_TEX && op <= OP_SULEA)
return static_cast<TexInstruction *>(this);
return NULL;
}
 
const TexInstruction *Instruction::asTex() const
{
if (op >= OP_TEX && op <= OP_SULEA)
return static_cast<const TexInstruction *>(this);
return NULL;
}
 
static inline Instruction *cloneForward(Function *ctx, Instruction *obj)
{
DeepClonePolicy<Function> pol(ctx);
 
for (int i = 0; obj->srcExists(i); ++i)
pol.set(obj->getSrc(i), obj->getSrc(i));
 
return obj->clone(pol);
}
 
// XXX: use a virtual function so we're really really safe ?
LValue *Value::asLValue()
{
if (reg.file >= FILE_GPR && reg.file <= FILE_ADDRESS)
return static_cast<LValue *>(this);
return NULL;
}
 
Symbol *Value::asSym()
{
if (reg.file >= FILE_MEMORY_CONST)
return static_cast<Symbol *>(this);
return NULL;
}
 
const Symbol *Value::asSym() const
{
if (reg.file >= FILE_MEMORY_CONST)
return static_cast<const Symbol *>(this);
return NULL;
}
 
void Symbol::setOffset(int32_t offset)
{
reg.data.offset = offset;
}
 
void Symbol::setAddress(Symbol *base, int32_t offset)
{
baseSym = base;
reg.data.offset = offset;
}
 
void Symbol::setSV(SVSemantic sv, uint32_t index)
{
reg.data.sv.sv = sv;
reg.data.sv.index = index;
}
 
ImmediateValue *Value::asImm()
{
if (reg.file == FILE_IMMEDIATE)
return static_cast<ImmediateValue *>(this);
return NULL;
}
 
const ImmediateValue *Value::asImm() const
{
if (reg.file == FILE_IMMEDIATE)
return static_cast<const ImmediateValue *>(this);
return NULL;
}
 
Value *Value::get(Iterator &it)
{
return reinterpret_cast<Value *>(it.get());
}
 
bool BasicBlock::reachableBy(const BasicBlock *by, const BasicBlock *term)
{
return cfg.reachableBy(&by->cfg, &term->cfg);
}
 
BasicBlock *BasicBlock::get(Iterator &iter)
{
return reinterpret_cast<BasicBlock *>(iter.get());
}
 
BasicBlock *BasicBlock::get(Graph::Node *node)
{
assert(node);
return reinterpret_cast<BasicBlock *>(node->data);
}
 
Function *Function::get(Graph::Node *node)
{
assert(node);
return reinterpret_cast<Function *>(node->data);
}
 
LValue *Function::getLValue(int id)
{
assert((unsigned int)id < (unsigned int)allLValues.getSize());
return reinterpret_cast<LValue *>(allLValues.get(id));
}
 
#endif // __NV50_IR_INLINES_H__
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_lowering_nv50.cpp
0,0 → 1,1101
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50/codegen/nv50_ir.h"
#include "nv50/codegen/nv50_ir_build_util.h"
 
#include "nv50_ir_target_nv50.h"
 
namespace nv50_ir {
 
// nv50 doesn't support 32 bit integer multiplication
//
// ah al * bh bl = LO32: (al * bh + ah * bl) << 16 + (al * bl)
// -------------------
// al*bh 00 HI32: (al * bh + ah * bl) >> 16 + (ah * bh) +
// ah*bh 00 00 ( carry1) << 16 + ( carry2)
// al*bl
// ah*bl 00
//
// fffe0001 + fffe0001
static bool
expandIntegerMUL(BuildUtil *bld, Instruction *mul)
{
const bool highResult = mul->subOp == NV50_IR_SUBOP_MUL_HIGH;
 
DataType fTy = mul->sType; // full type
DataType hTy;
switch (fTy) {
case TYPE_S32: hTy = TYPE_S16; break;
case TYPE_U32: hTy = TYPE_U16; break;
case TYPE_U64: hTy = TYPE_U32; break;
case TYPE_S64: hTy = TYPE_S32; break;
default:
return false;
}
unsigned int fullSize = typeSizeof(fTy);
unsigned int halfSize = typeSizeof(hTy);
 
Instruction *i[9];
 
bld->setPosition(mul, true);
 
Value *a[2], *b[2];
Value *c[2];
Value *t[4];
for (int j = 0; j < 4; ++j)
t[j] = bld->getSSA(fullSize);
 
// split sources into halves
i[0] = bld->mkSplit(a, halfSize, mul->getSrc(0));
i[1] = bld->mkSplit(b, halfSize, mul->getSrc(1));
 
i[2] = bld->mkOp2(OP_MUL, fTy, t[0], a[0], b[1]);
i[3] = bld->mkOp3(OP_MAD, fTy, t[1], a[1], b[0], t[0]);
i[7] = bld->mkOp2(OP_SHL, fTy, t[2], t[1], bld->mkImm(halfSize * 8));
i[4] = bld->mkOp3(OP_MAD, fTy, t[3], a[0], b[0], t[2]);
 
if (highResult) {
Value *r[3];
Value *imm = bld->loadImm(NULL, 1 << (halfSize * 8));
c[0] = bld->getSSA(1, FILE_FLAGS);
c[1] = bld->getSSA(1, FILE_FLAGS);
for (int j = 0; j < 3; ++j)
r[j] = bld->getSSA(fullSize);
 
i[8] = bld->mkOp2(OP_SHR, fTy, r[0], t[1], bld->mkImm(halfSize * 8));
i[6] = bld->mkOp2(OP_ADD, fTy, r[1], r[0], imm);
bld->mkOp2(OP_UNION, TYPE_U32, r[2], r[1], r[0]);
i[5] = bld->mkOp3(OP_MAD, fTy, mul->getDef(0), a[1], b[1], r[2]);
 
// set carry defs / sources
i[3]->setFlagsDef(1, c[0]);
i[4]->setFlagsDef(0, c[1]); // actual result not required, just the carry
i[6]->setPredicate(CC_C, c[0]);
i[5]->setFlagsSrc(3, c[1]);
} else {
bld->mkMov(mul->getDef(0), t[3]);
}
delete_Instruction(bld->getProgram(), mul);
 
for (int j = 2; j <= (highResult ? 5 : 4); ++j)
if (i[j])
i[j]->sType = hTy;
 
return true;
}
 
#define QOP_ADD 0
#define QOP_SUBR 1
#define QOP_SUB 2
#define QOP_MOV2 3
 
// UL UR LL LR
#define QUADOP(q, r, s, t) \
((QOP_##q << 6) | (QOP_##r << 4) | \
(QOP_##s << 2) | (QOP_##t << 0))
 
class NV50LegalizePostRA : public Pass
{
private:
virtual bool visit(Function *);
virtual bool visit(BasicBlock *);
 
void handlePRERET(FlowInstruction *);
void replaceZero(Instruction *);
 
LValue *r63;
};
 
bool
NV50LegalizePostRA::visit(Function *fn)
{
Program *prog = fn->getProgram();
 
r63 = new_LValue(fn, FILE_GPR);
r63->reg.data.id = 63;
 
// this is actually per-program, but we can do it all on visiting main()
std::list<Instruction *> *outWrites =
reinterpret_cast<std::list<Instruction *> *>(prog->targetPriv);
 
if (outWrites) {
for (std::list<Instruction *>::iterator it = outWrites->begin();
it != outWrites->end(); ++it)
(*it)->getSrc(1)->defs.front()->getInsn()->setDef(0, (*it)->getSrc(0));
// instructions will be deleted on exit
outWrites->clear();
}
 
return true;
}
 
void
NV50LegalizePostRA::replaceZero(Instruction *i)
{
for (int s = 0; i->srcExists(s); ++s) {
ImmediateValue *imm = i->getSrc(s)->asImm();
if (imm && imm->reg.data.u64 == 0)
i->setSrc(s, r63);
}
}
 
// Emulate PRERET: jump to the target and call to the origin from there
//
// WARNING: atm only works if BBs are affected by at most a single PRERET
//
// BB:0
// preret BB:3
// (...)
// BB:3
// (...)
// --->
// BB:0
// bra BB:3 + n0 (directly to the call; move to beginning of BB and fixate)
// (...)
// BB:3
// bra BB:3 + n1 (skip the call)
// call BB:0 + n2 (skip bra at beginning of BB:0)
// (...)
void
NV50LegalizePostRA::handlePRERET(FlowInstruction *pre)
{
BasicBlock *bbE = pre->bb;
BasicBlock *bbT = pre->target.bb;
 
pre->subOp = NV50_IR_SUBOP_EMU_PRERET + 0;
bbE->remove(pre);
bbE->insertHead(pre);
 
Instruction *skip = new_FlowInstruction(func, OP_PRERET, bbT);
Instruction *call = new_FlowInstruction(func, OP_PRERET, bbE);
 
bbT->insertHead(call);
bbT->insertHead(skip);
 
// NOTE: maybe split blocks to prevent the instructions from moving ?
 
skip->subOp = NV50_IR_SUBOP_EMU_PRERET + 1;
call->subOp = NV50_IR_SUBOP_EMU_PRERET + 2;
}
 
bool
NV50LegalizePostRA::visit(BasicBlock *bb)
{
Instruction *i, *next;
 
// remove pseudo operations and non-fixed no-ops, split 64 bit operations
for (i = bb->getFirst(); i; i = next) {
next = i->next;
if (i->isNop()) {
bb->remove(i);
} else
if (i->op == OP_PRERET && prog->getTarget()->getChipset() < 0xa0) {
handlePRERET(i->asFlow());
} else {
// TODO: We will want to do this before register allocation,
// since have to use a $c register for the carry flag.
if (typeSizeof(i->dType) == 8) {
Instruction *hi = BuildUtil::split64BitOpPostRA(func, i, r63, NULL);
if (hi)
next = hi;
}
 
if (i->op != OP_MOV && i->op != OP_PFETCH &&
i->op != OP_BAR &&
(!i->defExists(0) || i->def(0).getFile() != FILE_ADDRESS))
replaceZero(i);
}
}
if (!bb->getEntry())
return true;
 
return true;
}
 
class NV50LegalizeSSA : public Pass
{
public:
NV50LegalizeSSA(Program *);
 
virtual bool visit(BasicBlock *bb);
 
private:
void propagateWriteToOutput(Instruction *);
void handleDIV(Instruction *);
void handleMOD(Instruction *);
void handleMUL(Instruction *);
void handleAddrDef(Instruction *);
 
inline bool isARL(const Instruction *) const;
 
BuildUtil bld;
 
std::list<Instruction *> *outWrites;
};
 
NV50LegalizeSSA::NV50LegalizeSSA(Program *prog)
{
bld.setProgram(prog);
 
if (prog->optLevel >= 2 &&
(prog->getType() == Program::TYPE_GEOMETRY ||
prog->getType() == Program::TYPE_VERTEX))
outWrites =
reinterpret_cast<std::list<Instruction *> *>(prog->targetPriv);
else
outWrites = NULL;
}
 
void
NV50LegalizeSSA::propagateWriteToOutput(Instruction *st)
{
if (st->src(0).isIndirect(0) || st->getSrc(1)->refCount() != 1)
return;
 
// check def instruction can store
Instruction *di = st->getSrc(1)->defs.front()->getInsn();
 
// TODO: move exports (if beneficial) in common opt pass
if (di->isPseudo() || isTextureOp(di->op) || di->defCount(0xff, true) > 1)
return;
for (int s = 0; di->srcExists(s); ++s)
if (di->src(s).getFile() == FILE_IMMEDIATE)
return;
 
// We cannot set defs to non-lvalues before register allocation, so
// save & remove (to save registers) the exports and replace later.
outWrites->push_back(st);
st->bb->remove(st);
}
 
bool
NV50LegalizeSSA::isARL(const Instruction *i) const
{
ImmediateValue imm;
 
if (i->op != OP_SHL || i->src(0).getFile() != FILE_GPR)
return false;
if (!i->src(1).getImmediate(imm))
return false;
return imm.isInteger(0);
}
 
void
NV50LegalizeSSA::handleAddrDef(Instruction *i)
{
Instruction *arl;
 
i->getDef(0)->reg.size = 2; // $aX are only 16 bit
 
// only ADDR <- SHL(GPR, IMM) and ADDR <- ADD(ADDR, IMM) are valid
if (i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE) {
if (i->op == OP_SHL && i->src(0).getFile() == FILE_GPR)
return;
if (i->op == OP_ADD && i->src(0).getFile() == FILE_ADDRESS)
return;
}
 
// turn $a sources into $r sources (can't operate on $a)
for (int s = 0; i->srcExists(s); ++s) {
Value *a = i->getSrc(s);
Value *r;
if (a->reg.file == FILE_ADDRESS) {
if (a->getInsn() && isARL(a->getInsn())) {
i->setSrc(s, a->getInsn()->getSrc(0));
} else {
bld.setPosition(i, false);
r = bld.getSSA();
bld.mkMov(r, a);
i->setSrc(s, r);
}
}
}
if (i->op == OP_SHL && i->src(1).getFile() == FILE_IMMEDIATE)
return;
 
// turn result back into $a
bld.setPosition(i, true);
arl = bld.mkOp2(OP_SHL, TYPE_U32, i->getDef(0), bld.getSSA(), bld.mkImm(0));
i->setDef(0, arl->getSrc(0));
}
 
void
NV50LegalizeSSA::handleMUL(Instruction *mul)
{
if (isFloatType(mul->sType) || typeSizeof(mul->sType) <= 2)
return;
Value *def = mul->getDef(0);
Value *pred = mul->getPredicate();
CondCode cc = mul->cc;
if (pred)
mul->setPredicate(CC_ALWAYS, NULL);
 
if (mul->op == OP_MAD) {
Instruction *add = mul;
bld.setPosition(add, false);
Value *res = cloneShallow(func, mul->getDef(0));
mul = bld.mkOp2(OP_MUL, add->sType, res, add->getSrc(0), add->getSrc(1));
add->op = OP_ADD;
add->setSrc(0, mul->getDef(0));
add->setSrc(1, add->getSrc(2));
for (int s = 2; add->srcExists(s); ++s)
add->setSrc(s, NULL);
mul->subOp = add->subOp;
add->subOp = 0;
}
expandIntegerMUL(&bld, mul);
if (pred)
def->getInsn()->setPredicate(cc, pred);
}
 
// Use f32 division: first compute an approximate result, use it to reduce
// the dividend, which should then be representable as f32, divide the reduced
// dividend, and add the quotients.
void
NV50LegalizeSSA::handleDIV(Instruction *div)
{
const DataType ty = div->sType;
 
if (ty != TYPE_U32 && ty != TYPE_S32)
return;
 
Value *q, *q0, *qf, *aR, *aRf, *qRf, *qR, *t, *s, *m, *cond;
 
bld.setPosition(div, false);
 
Value *a, *af = bld.getSSA();
Value *b, *bf = bld.getSSA();
 
bld.mkCvt(OP_CVT, TYPE_F32, af, ty, div->getSrc(0));
bld.mkCvt(OP_CVT, TYPE_F32, bf, ty, div->getSrc(1));
 
if (isSignedType(ty)) {
af->getInsn()->src(0).mod = Modifier(NV50_IR_MOD_ABS);
bf->getInsn()->src(0).mod = Modifier(NV50_IR_MOD_ABS);
a = bld.getSSA();
b = bld.getSSA();
bld.mkOp1(OP_ABS, ty, a, div->getSrc(0));
bld.mkOp1(OP_ABS, ty, b, div->getSrc(1));
} else {
a = div->getSrc(0);
b = div->getSrc(1);
}
 
bf = bld.mkOp1v(OP_RCP, TYPE_F32, bld.getSSA(), bf);
bf = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), bf, bld.mkImm(-2));
 
bld.mkOp2(OP_MUL, TYPE_F32, (qf = bld.getSSA()), af, bf)->rnd = ROUND_Z;
bld.mkCvt(OP_CVT, ty, (q0 = bld.getSSA()), TYPE_F32, qf)->rnd = ROUND_Z;
 
// get error of 1st result
expandIntegerMUL(&bld,
bld.mkOp2(OP_MUL, TYPE_U32, (t = bld.getSSA()), q0, b));
bld.mkOp2(OP_SUB, TYPE_U32, (aRf = bld.getSSA()), a, t);
 
bld.mkCvt(OP_CVT, TYPE_F32, (aR = bld.getSSA()), TYPE_U32, aRf);
 
bld.mkOp2(OP_MUL, TYPE_F32, (qRf = bld.getSSA()), aR, bf)->rnd = ROUND_Z;
bld.mkCvt(OP_CVT, TYPE_U32, (qR = bld.getSSA()), TYPE_F32, qRf)
->rnd = ROUND_Z;
bld.mkOp2(OP_ADD, ty, (q = bld.getSSA()), q0, qR); // add quotients
 
// correction: if modulus >= divisor, add 1
expandIntegerMUL(&bld,
bld.mkOp2(OP_MUL, TYPE_U32, (t = bld.getSSA()), q, b));
bld.mkOp2(OP_SUB, TYPE_U32, (m = bld.getSSA()), a, t);
bld.mkCmp(OP_SET, CC_GE, TYPE_U32, (s = bld.getSSA()), m, b);
if (!isSignedType(ty)) {
div->op = OP_SUB;
div->setSrc(0, q);
div->setSrc(1, s);
} else {
t = q;
bld.mkOp2(OP_SUB, TYPE_U32, (q = bld.getSSA()), t, s);
s = bld.getSSA();
t = bld.getSSA();
// fix the sign
bld.mkOp2(OP_XOR, TYPE_U32, NULL, div->getSrc(0), div->getSrc(1))
->setFlagsDef(0, (cond = bld.getSSA(1, FILE_FLAGS)));
bld.mkOp1(OP_NEG, ty, s, q)->setPredicate(CC_S, cond);
bld.mkOp1(OP_MOV, ty, t, q)->setPredicate(CC_NS, cond);
 
div->op = OP_UNION;
div->setSrc(0, s);
div->setSrc(1, t);
}
}
 
void
NV50LegalizeSSA::handleMOD(Instruction *mod)
{
if (mod->dType != TYPE_U32 && mod->dType != TYPE_S32)
return;
bld.setPosition(mod, false);
 
Value *q = bld.getSSA();
Value *m = bld.getSSA();
 
bld.mkOp2(OP_DIV, mod->dType, q, mod->getSrc(0), mod->getSrc(1));
handleDIV(q->getInsn());
 
bld.setPosition(mod, false);
expandIntegerMUL(&bld, bld.mkOp2(OP_MUL, TYPE_U32, m, q, mod->getSrc(1)));
 
mod->op = OP_SUB;
mod->setSrc(1, m);
}
 
bool
NV50LegalizeSSA::visit(BasicBlock *bb)
{
Instruction *insn, *next;
// skipping PHIs (don't pass them to handleAddrDef) !
for (insn = bb->getEntry(); insn; insn = next) {
next = insn->next;
 
switch (insn->op) {
case OP_EXPORT:
if (outWrites)
propagateWriteToOutput(insn);
break;
case OP_DIV:
handleDIV(insn);
break;
case OP_MOD:
handleMOD(insn);
break;
case OP_MAD:
case OP_MUL:
handleMUL(insn);
break;
default:
break;
}
 
if (insn->defExists(0) && insn->getDef(0)->reg.file == FILE_ADDRESS)
handleAddrDef(insn);
}
return true;
}
 
class NV50LoweringPreSSA : public Pass
{
public:
NV50LoweringPreSSA(Program *);
 
private:
virtual bool visit(Instruction *);
virtual bool visit(Function *);
 
bool handleRDSV(Instruction *);
bool handleWRSV(Instruction *);
 
bool handleEXPORT(Instruction *);
 
bool handleDIV(Instruction *);
bool handleSQRT(Instruction *);
bool handlePOW(Instruction *);
 
bool handleSET(Instruction *);
bool handleSLCT(CmpInstruction *);
bool handleSELP(Instruction *);
 
bool handleTEX(TexInstruction *);
bool handleTXB(TexInstruction *); // I really
bool handleTXL(TexInstruction *); // hate
bool handleTXD(TexInstruction *); // these 3
 
bool handleCALL(Instruction *);
bool handlePRECONT(Instruction *);
bool handleCONT(Instruction *);
 
void checkPredicate(Instruction *);
 
private:
const Target *const targ;
 
BuildUtil bld;
 
Value *tid;
};
 
NV50LoweringPreSSA::NV50LoweringPreSSA(Program *prog) :
targ(prog->getTarget()), tid(NULL)
{
bld.setProgram(prog);
}
 
bool
NV50LoweringPreSSA::visit(Function *f)
{
BasicBlock *root = BasicBlock::get(func->cfg.getRoot());
 
if (prog->getType() == Program::TYPE_COMPUTE) {
// Add implicit "thread id" argument in $r0 to the function
Value *arg = new_LValue(func, FILE_GPR);
arg->reg.data.id = 0;
f->ins.push_back(arg);
 
bld.setPosition(root, false);
tid = bld.mkMov(bld.getScratch(), arg, TYPE_U32)->getDef(0);
}
 
return true;
}
 
bool
NV50LoweringPreSSA::handleTEX(TexInstruction *i)
{
const int arg = i->tex.target.getArgCount();
const int dref = arg;
const int lod = i->tex.target.isShadow() ? (arg + 1) : arg;
 
// dref comes before bias/lod
if (i->tex.target.isShadow())
if (i->op == OP_TXB || i->op == OP_TXL)
i->swapSources(dref, lod);
 
// array index must be converted to u32
if (i->tex.target.isArray()) {
Value *layer = i->getSrc(arg - 1);
LValue *src = new_LValue(func, FILE_GPR);
bld.mkCvt(OP_CVT, TYPE_U32, src, TYPE_F32, layer);
bld.mkOp2(OP_MIN, TYPE_U32, src, src, bld.loadImm(NULL, 511));
i->setSrc(arg - 1, src);
 
if (i->tex.target.isCube()) {
std::vector<Value *> acube, a2d;
int c;
 
acube.resize(4);
for (c = 0; c < 4; ++c)
acube[c] = i->getSrc(c);
a2d.resize(4);
for (c = 0; c < 3; ++c)
a2d[c] = new_LValue(func, FILE_GPR);
a2d[3] = NULL;
 
bld.mkTex(OP_TEXPREP, TEX_TARGET_CUBE_ARRAY, i->tex.r, i->tex.s,
a2d, acube)->asTex()->tex.mask = 0x7;
 
for (c = 0; c < 3; ++c)
i->setSrc(c, a2d[c]);
i->setSrc(c, NULL);
for (; i->srcExists(c + 1); ++c)
i->setSrc(c, i->getSrc(c + 1));
 
i->tex.target = i->tex.target.isShadow() ?
TEX_TARGET_2D_ARRAY_SHADOW : TEX_TARGET_2D_ARRAY;
}
}
 
// texel offsets are 3 immediate fields in the instruction,
// nv50 cannot do textureGatherOffsets
assert(i->tex.useOffsets <= 1);
 
return true;
}
 
// Bias must be equal for all threads of a quad or lod calculation will fail.
//
// The lanes of a quad are grouped by the bit in the condition register they
// have set, which is selected by differing bias values.
// Move the input values for TEX into a new register set for each group and
// execute TEX only for a specific group.
// We always need to use 4 new registers for the inputs/outputs because the
// implicitly calculated derivatives must be correct.
//
// TODO: move to SSA phase so we can easily determine whether bias is constant
bool
NV50LoweringPreSSA::handleTXB(TexInstruction *i)
{
const CondCode cc[4] = { CC_EQU, CC_S, CC_C, CC_O };
int l, d;
 
handleTEX(i);
Value *bias = i->getSrc(i->tex.target.getArgCount());
if (bias->isUniform())
return true;
 
Instruction *cond = bld.mkOp1(OP_UNION, TYPE_U32, bld.getScratch(),
bld.loadImm(NULL, 1));
bld.setPosition(cond, false);
 
for (l = 1; l < 4; ++l) {
const uint8_t qop = QUADOP(SUBR, SUBR, SUBR, SUBR);
Value *bit = bld.getSSA();
Value *pred = bld.getScratch(1, FILE_FLAGS);
Value *imm = bld.loadImm(NULL, (1 << l));
bld.mkQuadop(qop, pred, l, bias, bias)->flagsDef = 0;
bld.mkMov(bit, imm)->setPredicate(CC_EQ, pred);
cond->setSrc(l, bit);
}
Value *flags = bld.getScratch(1, FILE_FLAGS);
bld.setPosition(cond, true);
bld.mkCvt(OP_CVT, TYPE_U8, flags, TYPE_U32, cond->getDef(0));
 
Instruction *tex[4];
for (l = 0; l < 4; ++l) {
(tex[l] = cloneForward(func, i))->setPredicate(cc[l], flags);
bld.insert(tex[l]);
}
 
Value *res[4][4];
for (d = 0; i->defExists(d); ++d)
res[0][d] = tex[0]->getDef(d);
for (l = 1; l < 4; ++l) {
for (d = 0; tex[l]->defExists(d); ++d) {
res[l][d] = cloneShallow(func, res[0][d]);
bld.mkMov(res[l][d], tex[l]->getDef(d))->setPredicate(cc[l], flags);
}
}
 
for (d = 0; i->defExists(d); ++d) {
Instruction *dst = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(d));
for (l = 0; l < 4; ++l)
dst->setSrc(l, res[l][d]);
}
delete_Instruction(prog, i);
return true;
}
 
// LOD must be equal for all threads of a quad.
// Unlike with TXB, here we can just diverge since there's no LOD calculation
// that would require all 4 threads' sources to be set up properly.
bool
NV50LoweringPreSSA::handleTXL(TexInstruction *i)
{
handleTEX(i);
Value *lod = i->getSrc(i->tex.target.getArgCount());
if (lod->isUniform())
return true;
 
BasicBlock *currBB = i->bb;
BasicBlock *texiBB = i->bb->splitBefore(i, false);
BasicBlock *joinBB = i->bb->splitAfter(i);
 
bld.setPosition(currBB, true);
currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
 
for (int l = 0; l <= 3; ++l) {
const uint8_t qop = QUADOP(SUBR, SUBR, SUBR, SUBR);
Value *pred = bld.getScratch(1, FILE_FLAGS);
bld.setPosition(currBB, true);
bld.mkQuadop(qop, pred, l, lod, lod)->flagsDef = 0;
bld.mkFlow(OP_BRA, texiBB, CC_EQ, pred)->fixed = 1;
currBB->cfg.attach(&texiBB->cfg, Graph::Edge::FORWARD);
if (l <= 2) {
BasicBlock *laneBB = new BasicBlock(func);
currBB->cfg.attach(&laneBB->cfg, Graph::Edge::TREE);
currBB = laneBB;
}
}
bld.setPosition(joinBB, false);
bld.mkOp(OP_JOIN, TYPE_NONE, NULL);
return true;
}
 
bool
NV50LoweringPreSSA::handleTXD(TexInstruction *i)
{
static const uint8_t qOps[4][2] =
{
{ QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }, // l0
{ QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD, ADD) }, // l1
{ QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2
{ QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3
};
Value *def[4][4];
Value *crd[3];
Instruction *tex;
Value *zero = bld.loadImm(bld.getSSA(), 0);
int l, c;
const int dim = i->tex.target.getDim();
 
handleTEX(i);
i->op = OP_TEX; // no need to clone dPdx/dPdy later
 
for (c = 0; c < dim; ++c)
crd[c] = bld.getScratch();
 
bld.mkOp(OP_QUADON, TYPE_NONE, NULL);
for (l = 0; l < 4; ++l) {
// mov coordinates from lane l to all lanes
for (c = 0; c < dim; ++c)
bld.mkQuadop(0x00, crd[c], l, i->getSrc(c), zero);
// add dPdx from lane l to lanes dx
for (c = 0; c < dim; ++c)
bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]);
// add dPdy from lane l to lanes dy
for (c = 0; c < dim; ++c)
bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]);
// texture
bld.insert(tex = cloneForward(func, i));
for (c = 0; c < dim; ++c)
tex->setSrc(c, crd[c]);
// save results
for (c = 0; i->defExists(c); ++c) {
Instruction *mov;
def[c][l] = bld.getSSA();
mov = bld.mkMov(def[c][l], tex->getDef(c));
mov->fixed = 1;
mov->lanes = 1 << l;
}
}
bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);
 
for (c = 0; i->defExists(c); ++c) {
Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));
for (l = 0; l < 4; ++l)
u->setSrc(l, def[c][l]);
}
 
i->bb->remove(i);
return true;
}
 
bool
NV50LoweringPreSSA::handleSET(Instruction *i)
{
if (i->dType == TYPE_F32) {
bld.setPosition(i, true);
i->dType = TYPE_U32;
bld.mkOp1(OP_ABS, TYPE_S32, i->getDef(0), i->getDef(0));
bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(0), TYPE_S32, i->getDef(0));
}
return true;
}
 
bool
NV50LoweringPreSSA::handleSLCT(CmpInstruction *i)
{
Value *src0 = bld.getSSA();
Value *src1 = bld.getSSA();
Value *pred = bld.getScratch(1, FILE_FLAGS);
 
Value *v0 = i->getSrc(0);
Value *v1 = i->getSrc(1);
// XXX: these probably shouldn't be immediates in the first place ...
if (v0->asImm())
v0 = bld.mkMov(bld.getSSA(), v0)->getDef(0);
if (v1->asImm())
v1 = bld.mkMov(bld.getSSA(), v1)->getDef(0);
 
bld.setPosition(i, true);
bld.mkMov(src0, v0)->setPredicate(CC_NE, pred);
bld.mkMov(src1, v1)->setPredicate(CC_EQ, pred);
bld.mkOp2(OP_UNION, i->dType, i->getDef(0), src0, src1);
 
bld.setPosition(i, false);
i->op = OP_SET;
i->setFlagsDef(0, pred);
i->dType = TYPE_U8;
i->setSrc(0, i->getSrc(2));
i->setSrc(2, NULL);
i->setSrc(1, bld.loadImm(NULL, 0));
 
return true;
}
 
bool
NV50LoweringPreSSA::handleSELP(Instruction *i)
{
Value *src0 = bld.getSSA();
Value *src1 = bld.getSSA();
 
Value *v0 = i->getSrc(0);
Value *v1 = i->getSrc(1);
if (v0->asImm())
v0 = bld.mkMov(bld.getSSA(), v0)->getDef(0);
if (v1->asImm())
v1 = bld.mkMov(bld.getSSA(), v1)->getDef(0);
 
bld.mkMov(src0, v0)->setPredicate(CC_NE, i->getSrc(2));
bld.mkMov(src1, v1)->setPredicate(CC_EQ, i->getSrc(2));
bld.mkOp2(OP_UNION, i->dType, i->getDef(0), src0, src1);
delete_Instruction(prog, i);
return true;
}
 
bool
NV50LoweringPreSSA::handleWRSV(Instruction *i)
{
Symbol *sym = i->getSrc(0)->asSym();
 
// these are all shader outputs, $sreg are not writeable
uint32_t addr = targ->getSVAddress(FILE_SHADER_OUTPUT, sym);
if (addr >= 0x400)
return false;
sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr);
 
bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0), i->getSrc(1));
 
bld.getBB()->remove(i);
return true;
}
 
bool
NV50LoweringPreSSA::handleCALL(Instruction *i)
{
if (prog->getType() == Program::TYPE_COMPUTE) {
// Add implicit "thread id" argument in $r0 to the function
i->setSrc(i->srcCount(), tid);
}
return true;
}
 
bool
NV50LoweringPreSSA::handlePRECONT(Instruction *i)
{
delete_Instruction(prog, i);
return true;
}
 
bool
NV50LoweringPreSSA::handleCONT(Instruction *i)
{
i->op = OP_BRA;
return true;
}
 
bool
NV50LoweringPreSSA::handleRDSV(Instruction *i)
{
Symbol *sym = i->getSrc(0)->asSym();
uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym);
Value *def = i->getDef(0);
SVSemantic sv = sym->reg.data.sv.sv;
int idx = sym->reg.data.sv.index;
 
if (addr >= 0x400) // mov $sreg
return true;
 
switch (sv) {
case SV_POSITION:
assert(prog->getType() == Program::TYPE_FRAGMENT);
bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL);
break;
case SV_FACE:
bld.mkInterp(NV50_IR_INTERP_FLAT, def, addr, NULL);
if (i->dType == TYPE_F32) {
bld.mkOp2(OP_AND, TYPE_U32, def, def, bld.mkImm(0x80000000));
bld.mkOp2(OP_XOR, TYPE_U32, def, def, bld.mkImm(0xbf800000));
}
break;
case SV_NCTAID:
case SV_CTAID:
case SV_NTID:
if ((sv == SV_NCTAID && idx >= 2) ||
(sv == SV_NTID && idx >= 3)) {
bld.mkMov(def, bld.mkImm(1));
} else if (sv == SV_CTAID && idx >= 2) {
bld.mkMov(def, bld.mkImm(0));
} else {
Value *x = bld.getSSA(2);
bld.mkOp1(OP_LOAD, TYPE_U16, x,
bld.mkSymbol(FILE_MEMORY_SHARED, 0, TYPE_U16, addr));
bld.mkCvt(OP_CVT, TYPE_U32, def, TYPE_U16, x);
}
break;
case SV_TID:
if (idx == 0) {
bld.mkOp2(OP_AND, TYPE_U32, def, tid, bld.mkImm(0x0000ffff));
} else if (idx == 1) {
bld.mkOp2(OP_AND, TYPE_U32, def, tid, bld.mkImm(0x03ff0000));
bld.mkOp2(OP_SHR, TYPE_U32, def, def, bld.mkImm(16));
} else if (idx == 2) {
bld.mkOp2(OP_SHR, TYPE_U32, def, tid, bld.mkImm(26));
} else {
bld.mkMov(def, bld.mkImm(0));
}
break;
default:
bld.mkFetch(i->getDef(0), i->dType,
FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), NULL);
break;
}
bld.getBB()->remove(i);
return true;
}
 
bool
NV50LoweringPreSSA::handleDIV(Instruction *i)
{
if (!isFloatType(i->dType))
return true;
bld.setPosition(i, false);
Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(), i->getSrc(1));
i->op = OP_MUL;
i->setSrc(1, rcp->getDef(0));
return true;
}
 
bool
NV50LoweringPreSSA::handleSQRT(Instruction *i)
{
Instruction *rsq = bld.mkOp1(OP_RSQ, TYPE_F32,
bld.getSSA(), i->getSrc(0));
i->op = OP_MUL;
i->setSrc(1, rsq->getDef(0));
 
return true;
}
 
bool
NV50LoweringPreSSA::handlePOW(Instruction *i)
{
LValue *val = bld.getScratch();
 
bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0));
bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1;
bld.mkOp1(OP_PREEX2, TYPE_F32, val, val);
 
i->op = OP_EX2;
i->setSrc(0, val);
i->setSrc(1, NULL);
 
return true;
}
 
bool
NV50LoweringPreSSA::handleEXPORT(Instruction *i)
{
if (prog->getType() == Program::TYPE_FRAGMENT) {
if (i->getIndirect(0, 0)) {
// TODO: redirect to l[] here, load to GPRs at exit
return false;
} else {
int id = i->getSrc(0)->reg.data.offset / 4; // in 32 bit reg units
 
i->op = OP_MOV;
i->subOp = NV50_IR_SUBOP_MOV_FINAL;
i->src(0).set(i->src(1));
i->setSrc(1, NULL);
i->setDef(0, new_LValue(func, FILE_GPR));
i->getDef(0)->reg.data.id = id;
 
prog->maxGPR = MAX2(prog->maxGPR, id);
}
}
return true;
}
 
// Set flags according to predicate and make the instruction read $cX.
void
NV50LoweringPreSSA::checkPredicate(Instruction *insn)
{
Value *pred = insn->getPredicate();
Value *cdst;
 
if (!pred || pred->reg.file == FILE_FLAGS)
return;
cdst = bld.getSSA(1, FILE_FLAGS);
 
bld.mkCmp(OP_SET, CC_NEU, insn->dType, cdst, bld.loadImm(NULL, 0), pred);
 
insn->setPredicate(insn->cc, cdst);
}
 
//
// - add quadop dance for texturing
// - put FP outputs in GPRs
// - convert instruction sequences
//
bool
NV50LoweringPreSSA::visit(Instruction *i)
{
bld.setPosition(i, false);
 
if (i->cc != CC_ALWAYS)
checkPredicate(i);
 
switch (i->op) {
case OP_TEX:
case OP_TXF:
case OP_TXG:
return handleTEX(i->asTex());
case OP_TXB:
return handleTXB(i->asTex());
case OP_TXL:
return handleTXL(i->asTex());
case OP_TXD:
return handleTXD(i->asTex());
case OP_EX2:
bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0));
i->setSrc(0, i->getDef(0));
break;
case OP_SET:
return handleSET(i);
case OP_SLCT:
return handleSLCT(i->asCmp());
case OP_SELP:
return handleSELP(i);
case OP_POW:
return handlePOW(i);
case OP_DIV:
return handleDIV(i);
case OP_SQRT:
return handleSQRT(i);
case OP_EXPORT:
return handleEXPORT(i);
case OP_RDSV:
return handleRDSV(i);
case OP_WRSV:
return handleWRSV(i);
case OP_CALL:
return handleCALL(i);
case OP_PRECONT:
return handlePRECONT(i);
case OP_CONT:
return handleCONT(i);
default:
break;
}
return true;
}
 
bool
TargetNV50::runLegalizePass(Program *prog, CGStage stage) const
{
bool ret = false;
 
if (stage == CG_STAGE_PRE_SSA) {
NV50LoweringPreSSA pass(prog);
ret = pass.run(prog, false, true);
} else
if (stage == CG_STAGE_SSA) {
if (!prog->targetPriv)
prog->targetPriv = new std::list<Instruction *>();
NV50LegalizeSSA pass(prog);
ret = pass.run(prog, false, true);
} else
if (stage == CG_STAGE_POST_RA) {
NV50LegalizePostRA pass;
ret = pass.run(prog, false, true);
if (prog->targetPriv)
delete reinterpret_cast<std::list<Instruction *> *>(prog->targetPriv);
}
return ret;
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_peephole.cpp
0,0 → 1,2464
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_target.h"
#include "nv50_ir_build_util.h"
 
extern "C" {
#include "util/u_math.h"
}
 
namespace nv50_ir {
 
bool
Instruction::isNop() const
{
if (op == OP_PHI || op == OP_SPLIT || op == OP_MERGE || op == OP_CONSTRAINT)
return true;
if (terminator || join) // XXX: should terminator imply flow ?
return false;
if (op == OP_ATOM)
return false;
if (!fixed && op == OP_NOP)
return true;
 
if (defExists(0) && def(0).rep()->reg.data.id < 0) {
for (int d = 1; defExists(d); ++d)
if (def(d).rep()->reg.data.id >= 0)
WARN("part of vector result is unused !\n");
return true;
}
 
if (op == OP_MOV || op == OP_UNION) {
if (!getDef(0)->equals(getSrc(0)))
return false;
if (op == OP_UNION)
if (!def(0).rep()->equals(getSrc(1)))
return false;
return true;
}
 
return false;
}
 
bool Instruction::isDead() const
{
if (op == OP_STORE ||
op == OP_EXPORT ||
op == OP_ATOM ||
op == OP_SUSTB || op == OP_SUSTP || op == OP_SUREDP || op == OP_SUREDB ||
op == OP_WRSV)
return false;
 
for (int d = 0; defExists(d); ++d)
if (getDef(d)->refCount() || getDef(d)->reg.data.id >= 0)
return false;
 
if (terminator || asFlow())
return false;
if (fixed)
return false;
 
return true;
};
 
// =============================================================================
 
class CopyPropagation : public Pass
{
private:
virtual bool visit(BasicBlock *);
};
 
// Propagate all MOVs forward to make subsequent optimization easier, except if
// the sources stem from a phi, in which case we don't want to mess up potential
// swaps $rX <-> $rY, i.e. do not create live range overlaps of phi src and def.
bool
CopyPropagation::visit(BasicBlock *bb)
{
Instruction *mov, *si, *next;
 
for (mov = bb->getEntry(); mov; mov = next) {
next = mov->next;
if (mov->op != OP_MOV || mov->fixed || !mov->getSrc(0)->asLValue())
continue;
if (mov->getPredicate())
continue;
if (mov->def(0).getFile() != mov->src(0).getFile())
continue;
si = mov->getSrc(0)->getInsn();
if (mov->getDef(0)->reg.data.id < 0 && si && si->op != OP_PHI) {
// propagate
mov->def(0).replace(mov->getSrc(0), false);
delete_Instruction(prog, mov);
}
}
return true;
}
 
// =============================================================================
 
class LoadPropagation : public Pass
{
private:
virtual bool visit(BasicBlock *);
 
void checkSwapSrc01(Instruction *);
 
bool isCSpaceLoad(Instruction *);
bool isImmd32Load(Instruction *);
bool isAttribOrSharedLoad(Instruction *);
};
 
bool
LoadPropagation::isCSpaceLoad(Instruction *ld)
{
return ld && ld->op == OP_LOAD && ld->src(0).getFile() == FILE_MEMORY_CONST;
}
 
bool
LoadPropagation::isImmd32Load(Instruction *ld)
{
if (!ld || (ld->op != OP_MOV) || (typeSizeof(ld->dType) != 4))
return false;
return ld->src(0).getFile() == FILE_IMMEDIATE;
}
 
bool
LoadPropagation::isAttribOrSharedLoad(Instruction *ld)
{
return ld &&
(ld->op == OP_VFETCH ||
(ld->op == OP_LOAD &&
(ld->src(0).getFile() == FILE_SHADER_INPUT ||
ld->src(0).getFile() == FILE_MEMORY_SHARED)));
}
 
void
LoadPropagation::checkSwapSrc01(Instruction *insn)
{
if (!prog->getTarget()->getOpInfo(insn).commutative)
if (insn->op != OP_SET && insn->op != OP_SLCT)
return;
if (insn->src(1).getFile() != FILE_GPR)
return;
 
Instruction *i0 = insn->getSrc(0)->getInsn();
Instruction *i1 = insn->getSrc(1)->getInsn();
 
if (isCSpaceLoad(i0)) {
if (!isCSpaceLoad(i1))
insn->swapSources(0, 1);
else
return;
} else
if (isImmd32Load(i0)) {
if (!isCSpaceLoad(i1) && !isImmd32Load(i1))
insn->swapSources(0, 1);
else
return;
} else
if (isAttribOrSharedLoad(i1)) {
if (!isAttribOrSharedLoad(i0))
insn->swapSources(0, 1);
else
return;
} else {
return;
}
 
if (insn->op == OP_SET)
insn->asCmp()->setCond = reverseCondCode(insn->asCmp()->setCond);
else
if (insn->op == OP_SLCT)
insn->asCmp()->setCond = inverseCondCode(insn->asCmp()->setCond);
}
 
bool
LoadPropagation::visit(BasicBlock *bb)
{
const Target *targ = prog->getTarget();
Instruction *next;
 
for (Instruction *i = bb->getEntry(); i; i = next) {
next = i->next;
 
if (i->op == OP_CALL) // calls have args as sources, they must be in regs
continue;
 
if (i->srcExists(1))
checkSwapSrc01(i);
 
for (int s = 0; i->srcExists(s); ++s) {
Instruction *ld = i->getSrc(s)->getInsn();
 
if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV))
continue;
if (!targ->insnCanLoad(i, s, ld))
continue;
 
// propagate !
i->setSrc(s, ld->getSrc(0));
if (ld->src(0).isIndirect(0))
i->setIndirect(s, 0, ld->getIndirect(0, 0));
 
if (ld->getDef(0)->refCount() == 0)
delete_Instruction(prog, ld);
}
}
return true;
}
 
// =============================================================================
 
// Evaluate constant expressions.
class ConstantFolding : public Pass
{
public:
bool foldAll(Program *);
 
private:
virtual bool visit(BasicBlock *);
 
void expr(Instruction *, ImmediateValue&, ImmediateValue&);
void opnd(Instruction *, ImmediateValue&, int s);
 
void unary(Instruction *, const ImmediateValue&);
 
void tryCollapseChainedMULs(Instruction *, const int s, ImmediateValue&);
 
// TGSI 'true' is converted to -1 by F2I(NEG(SET)), track back to SET
CmpInstruction *findOriginForTestWithZero(Value *);
 
unsigned int foldCount;
 
BuildUtil bld;
};
 
// TODO: remember generated immediates and only revisit these
bool
ConstantFolding::foldAll(Program *prog)
{
unsigned int iterCount = 0;
do {
foldCount = 0;
if (!run(prog))
return false;
} while (foldCount && ++iterCount < 2);
return true;
}
 
bool
ConstantFolding::visit(BasicBlock *bb)
{
Instruction *i, *next;
 
for (i = bb->getEntry(); i; i = next) {
next = i->next;
if (i->op == OP_MOV || i->op == OP_CALL)
continue;
 
ImmediateValue src0, src1;
 
if (i->srcExists(1) &&
i->src(0).getImmediate(src0) && i->src(1).getImmediate(src1))
expr(i, src0, src1);
else
if (i->srcExists(0) && i->src(0).getImmediate(src0))
opnd(i, src0, 0);
else
if (i->srcExists(1) && i->src(1).getImmediate(src1))
opnd(i, src1, 1);
}
return true;
}
 
CmpInstruction *
ConstantFolding::findOriginForTestWithZero(Value *value)
{
if (!value)
return NULL;
Instruction *insn = value->getInsn();
 
while (insn && insn->op != OP_SET) {
Instruction *next = NULL;
switch (insn->op) {
case OP_NEG:
case OP_ABS:
case OP_CVT:
next = insn->getSrc(0)->getInsn();
if (insn->sType != next->dType)
return NULL;
break;
case OP_MOV:
next = insn->getSrc(0)->getInsn();
break;
default:
return NULL;
}
insn = next;
}
return insn ? insn->asCmp() : NULL;
}
 
void
Modifier::applyTo(ImmediateValue& imm) const
{
if (!bits) // avoid failure if imm.reg.type is unhandled (e.g. b128)
return;
switch (imm.reg.type) {
case TYPE_F32:
if (bits & NV50_IR_MOD_ABS)
imm.reg.data.f32 = fabsf(imm.reg.data.f32);
if (bits & NV50_IR_MOD_NEG)
imm.reg.data.f32 = -imm.reg.data.f32;
if (bits & NV50_IR_MOD_SAT) {
if (imm.reg.data.f32 < 0.0f)
imm.reg.data.f32 = 0.0f;
else
if (imm.reg.data.f32 > 1.0f)
imm.reg.data.f32 = 1.0f;
}
assert(!(bits & NV50_IR_MOD_NOT));
break;
 
case TYPE_S8: // NOTE: will be extended
case TYPE_S16:
case TYPE_S32:
case TYPE_U8: // NOTE: treated as signed
case TYPE_U16:
case TYPE_U32:
if (bits & NV50_IR_MOD_ABS)
imm.reg.data.s32 = (imm.reg.data.s32 >= 0) ?
imm.reg.data.s32 : -imm.reg.data.s32;
if (bits & NV50_IR_MOD_NEG)
imm.reg.data.s32 = -imm.reg.data.s32;
if (bits & NV50_IR_MOD_NOT)
imm.reg.data.s32 = ~imm.reg.data.s32;
break;
 
case TYPE_F64:
if (bits & NV50_IR_MOD_ABS)
imm.reg.data.f64 = fabs(imm.reg.data.f64);
if (bits & NV50_IR_MOD_NEG)
imm.reg.data.f64 = -imm.reg.data.f64;
if (bits & NV50_IR_MOD_SAT) {
if (imm.reg.data.f64 < 0.0)
imm.reg.data.f64 = 0.0;
else
if (imm.reg.data.f64 > 1.0)
imm.reg.data.f64 = 1.0;
}
assert(!(bits & NV50_IR_MOD_NOT));
break;
 
default:
assert(!"invalid/unhandled type");
imm.reg.data.u64 = 0;
break;
}
}
 
operation
Modifier::getOp() const
{
switch (bits) {
case NV50_IR_MOD_ABS: return OP_ABS;
case NV50_IR_MOD_NEG: return OP_NEG;
case NV50_IR_MOD_SAT: return OP_SAT;
case NV50_IR_MOD_NOT: return OP_NOT;
case 0:
return OP_MOV;
default:
return OP_CVT;
}
}
 
void
ConstantFolding::expr(Instruction *i,
ImmediateValue &imm0, ImmediateValue &imm1)
{
struct Storage *const a = &imm0.reg, *const b = &imm1.reg;
struct Storage res;
 
memset(&res.data, 0, sizeof(res.data));
 
switch (i->op) {
case OP_MAD:
case OP_FMA:
case OP_MUL:
if (i->dnz && i->dType == TYPE_F32) {
if (!isfinite(a->data.f32))
a->data.f32 = 0.0f;
if (!isfinite(b->data.f32))
b->data.f32 = 0.0f;
}
switch (i->dType) {
case TYPE_F32: res.data.f32 = a->data.f32 * b->data.f32; break;
case TYPE_F64: res.data.f64 = a->data.f64 * b->data.f64; break;
case TYPE_S32:
case TYPE_U32: res.data.u32 = a->data.u32 * b->data.u32; break;
default:
return;
}
break;
case OP_DIV:
if (b->data.u32 == 0)
break;
switch (i->dType) {
case TYPE_F32: res.data.f32 = a->data.f32 / b->data.f32; break;
case TYPE_F64: res.data.f64 = a->data.f64 / b->data.f64; break;
case TYPE_S32: res.data.s32 = a->data.s32 / b->data.s32; break;
case TYPE_U32: res.data.u32 = a->data.u32 / b->data.u32; break;
default:
return;
}
break;
case OP_ADD:
switch (i->dType) {
case TYPE_F32: res.data.f32 = a->data.f32 + b->data.f32; break;
case TYPE_F64: res.data.f64 = a->data.f64 + b->data.f64; break;
case TYPE_S32:
case TYPE_U32: res.data.u32 = a->data.u32 + b->data.u32; break;
default:
return;
}
break;
case OP_POW:
switch (i->dType) {
case TYPE_F32: res.data.f32 = pow(a->data.f32, b->data.f32); break;
case TYPE_F64: res.data.f64 = pow(a->data.f64, b->data.f64); break;
default:
return;
}
break;
case OP_MAX:
switch (i->dType) {
case TYPE_F32: res.data.f32 = MAX2(a->data.f32, b->data.f32); break;
case TYPE_F64: res.data.f64 = MAX2(a->data.f64, b->data.f64); break;
case TYPE_S32: res.data.s32 = MAX2(a->data.s32, b->data.s32); break;
case TYPE_U32: res.data.u32 = MAX2(a->data.u32, b->data.u32); break;
default:
return;
}
break;
case OP_MIN:
switch (i->dType) {
case TYPE_F32: res.data.f32 = MIN2(a->data.f32, b->data.f32); break;
case TYPE_F64: res.data.f64 = MIN2(a->data.f64, b->data.f64); break;
case TYPE_S32: res.data.s32 = MIN2(a->data.s32, b->data.s32); break;
case TYPE_U32: res.data.u32 = MIN2(a->data.u32, b->data.u32); break;
default:
return;
}
break;
case OP_AND:
res.data.u64 = a->data.u64 & b->data.u64;
break;
case OP_OR:
res.data.u64 = a->data.u64 | b->data.u64;
break;
case OP_XOR:
res.data.u64 = a->data.u64 ^ b->data.u64;
break;
case OP_SHL:
res.data.u32 = a->data.u32 << b->data.u32;
break;
case OP_SHR:
switch (i->dType) {
case TYPE_S32: res.data.s32 = a->data.s32 >> b->data.u32; break;
case TYPE_U32: res.data.u32 = a->data.u32 >> b->data.u32; break;
default:
return;
}
break;
case OP_SLCT:
if (a->data.u32 != b->data.u32)
return;
res.data.u32 = a->data.u32;
break;
default:
return;
}
++foldCount;
 
i->src(0).mod = Modifier(0);
i->src(1).mod = Modifier(0);
 
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
i->setSrc(1, NULL);
 
i->getSrc(0)->reg.data = res.data;
 
if (i->op == OP_MAD || i->op == OP_FMA) {
i->op = OP_ADD;
 
i->setSrc(1, i->getSrc(0));
i->src(1).mod = i->src(2).mod;
i->setSrc(0, i->getSrc(2));
i->setSrc(2, NULL);
 
ImmediateValue src0;
if (i->src(0).getImmediate(src0))
expr(i, src0, *i->getSrc(1)->asImm());
} else {
i->op = OP_MOV;
}
}
 
void
ConstantFolding::unary(Instruction *i, const ImmediateValue &imm)
{
Storage res;
 
if (i->dType != TYPE_F32)
return;
switch (i->op) {
case OP_NEG: res.data.f32 = -imm.reg.data.f32; break;
case OP_ABS: res.data.f32 = fabsf(imm.reg.data.f32); break;
case OP_RCP: res.data.f32 = 1.0f / imm.reg.data.f32; break;
case OP_RSQ: res.data.f32 = 1.0f / sqrtf(imm.reg.data.f32); break;
case OP_LG2: res.data.f32 = log2f(imm.reg.data.f32); break;
case OP_EX2: res.data.f32 = exp2f(imm.reg.data.f32); break;
case OP_SIN: res.data.f32 = sinf(imm.reg.data.f32); break;
case OP_COS: res.data.f32 = cosf(imm.reg.data.f32); break;
case OP_SQRT: res.data.f32 = sqrtf(imm.reg.data.f32); break;
case OP_PRESIN:
case OP_PREEX2:
// these should be handled in subsequent OP_SIN/COS/EX2
res.data.f32 = imm.reg.data.f32;
break;
default:
return;
}
i->op = OP_MOV;
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.f32));
i->src(0).mod = Modifier(0);
}
 
void
ConstantFolding::tryCollapseChainedMULs(Instruction *mul2,
const int s, ImmediateValue& imm2)
{
const int t = s ? 0 : 1;
Instruction *insn;
Instruction *mul1 = NULL; // mul1 before mul2
int e = 0;
float f = imm2.reg.data.f32;
ImmediateValue imm1;
 
assert(mul2->op == OP_MUL && mul2->dType == TYPE_F32);
 
if (mul2->getSrc(t)->refCount() == 1) {
insn = mul2->getSrc(t)->getInsn();
if (!mul2->src(t).mod && insn->op == OP_MUL && insn->dType == TYPE_F32)
mul1 = insn;
if (mul1 && !mul1->saturate) {
int s1;
 
if (mul1->src(s1 = 0).getImmediate(imm1) ||
mul1->src(s1 = 1).getImmediate(imm1)) {
bld.setPosition(mul1, false);
// a = mul r, imm1
// d = mul a, imm2 -> d = mul r, (imm1 * imm2)
mul1->setSrc(s1, bld.loadImm(NULL, f * imm1.reg.data.f32));
mul1->src(s1).mod = Modifier(0);
mul2->def(0).replace(mul1->getDef(0), false);
} else
if (prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
// c = mul a, b
// d = mul c, imm -> d = mul_x_imm a, b
mul1->postFactor = e;
mul2->def(0).replace(mul1->getDef(0), false);
if (f < 0)
mul1->src(0).mod *= Modifier(NV50_IR_MOD_NEG);
}
mul1->saturate = mul2->saturate;
return;
}
}
if (mul2->getDef(0)->refCount() == 1 && !mul2->saturate) {
// b = mul a, imm
// d = mul b, c -> d = mul_x_imm a, c
int s2, t2;
insn = mul2->getDef(0)->uses.front()->getInsn();
if (!insn)
return;
mul1 = mul2;
mul2 = NULL;
s2 = insn->getSrc(0) == mul1->getDef(0) ? 0 : 1;
t2 = s2 ? 0 : 1;
if (insn->op == OP_MUL && insn->dType == TYPE_F32)
if (!insn->src(s2).mod && !insn->src(t2).getImmediate(imm1))
mul2 = insn;
if (mul2 && prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
mul2->postFactor = e;
mul2->setSrc(s2, mul1->src(t));
if (f < 0)
mul2->src(s2).mod *= Modifier(NV50_IR_MOD_NEG);
}
}
}
 
void
ConstantFolding::opnd(Instruction *i, ImmediateValue &imm0, int s)
{
const int t = !s;
const operation op = i->op;
 
switch (i->op) {
case OP_MUL:
if (i->dType == TYPE_F32)
tryCollapseChainedMULs(i, s, imm0);
 
if (imm0.isInteger(0)) {
i->op = OP_MOV;
i->setSrc(0, new_ImmediateValue(prog, 0u));
i->src(0).mod = Modifier(0);
i->setSrc(1, NULL);
} else
if (imm0.isInteger(1) || imm0.isInteger(-1)) {
if (imm0.isNegative())
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
i->op = i->src(t).mod.getOp();
if (s == 0) {
i->setSrc(0, i->getSrc(1));
i->src(0).mod = i->src(1).mod;
i->src(1).mod = 0;
}
if (i->op != OP_CVT)
i->src(0).mod = 0;
i->setSrc(1, NULL);
} else
if (imm0.isInteger(2) || imm0.isInteger(-2)) {
if (imm0.isNegative())
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
i->op = OP_ADD;
i->setSrc(s, i->getSrc(t));
i->src(s).mod = i->src(t).mod;
} else
if (!isFloatType(i->sType) && !imm0.isNegative() && imm0.isPow2()) {
i->op = OP_SHL;
imm0.applyLog2();
i->setSrc(0, i->getSrc(t));
i->src(0).mod = i->src(t).mod;
i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
i->src(1).mod = 0;
}
break;
case OP_ADD:
if (i->usesFlags())
break;
if (imm0.isInteger(0)) {
if (s == 0) {
i->setSrc(0, i->getSrc(1));
i->src(0).mod = i->src(1).mod;
}
i->setSrc(1, NULL);
i->op = i->src(0).mod.getOp();
if (i->op != OP_CVT)
i->src(0).mod = Modifier(0);
}
break;
 
case OP_DIV:
if (s != 1 || (i->dType != TYPE_S32 && i->dType != TYPE_U32))
break;
bld.setPosition(i, false);
if (imm0.reg.data.u32 == 0) {
break;
} else
if (imm0.reg.data.u32 == 1) {
i->op = OP_MOV;
i->setSrc(1, NULL);
} else
if (i->dType == TYPE_U32 && imm0.isPow2()) {
i->op = OP_SHR;
i->setSrc(1, bld.mkImm(util_logbase2(imm0.reg.data.u32)));
} else
if (i->dType == TYPE_U32) {
Instruction *mul;
Value *tA, *tB;
const uint32_t d = imm0.reg.data.u32;
uint32_t m;
int r, s;
uint32_t l = util_logbase2(d);
if (((uint32_t)1 << l) < d)
++l;
m = (((uint64_t)1 << 32) * (((uint64_t)1 << l) - d)) / d + 1;
r = l ? 1 : 0;
s = l ? (l - 1) : 0;
 
tA = bld.getSSA();
tB = bld.getSSA();
mul = bld.mkOp2(OP_MUL, TYPE_U32, tA, i->getSrc(0),
bld.loadImm(NULL, m));
mul->subOp = NV50_IR_SUBOP_MUL_HIGH;
bld.mkOp2(OP_SUB, TYPE_U32, tB, i->getSrc(0), tA);
tA = bld.getSSA();
if (r)
bld.mkOp2(OP_SHR, TYPE_U32, tA, tB, bld.mkImm(r));
else
tA = tB;
tB = s ? bld.getSSA() : i->getDef(0);
bld.mkOp2(OP_ADD, TYPE_U32, tB, mul->getDef(0), tA);
if (s)
bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), tB, bld.mkImm(s));
 
delete_Instruction(prog, i);
} else
if (imm0.reg.data.s32 == -1) {
i->op = OP_NEG;
i->setSrc(1, NULL);
} else {
LValue *tA, *tB;
LValue *tD;
const int32_t d = imm0.reg.data.s32;
int32_t m;
int32_t l = util_logbase2(static_cast<unsigned>(abs(d)));
if ((1 << l) < abs(d))
++l;
if (!l)
l = 1;
m = ((uint64_t)1 << (32 + l - 1)) / abs(d) + 1 - ((uint64_t)1 << 32);
 
tA = bld.getSSA();
tB = bld.getSSA();
bld.mkOp3(OP_MAD, TYPE_S32, tA, i->getSrc(0), bld.loadImm(NULL, m),
i->getSrc(0))->subOp = NV50_IR_SUBOP_MUL_HIGH;
if (l > 1)
bld.mkOp2(OP_SHR, TYPE_S32, tB, tA, bld.mkImm(l - 1));
else
tB = tA;
tA = bld.getSSA();
bld.mkCmp(OP_SET, CC_LT, TYPE_S32, tA, i->getSrc(0), bld.mkImm(0));
tD = (d < 0) ? bld.getSSA() : i->getDef(0)->asLValue();
bld.mkOp2(OP_SUB, TYPE_U32, tD, tB, tA);
if (d < 0)
bld.mkOp1(OP_NEG, TYPE_S32, i->getDef(0), tB);
 
delete_Instruction(prog, i);
}
break;
 
case OP_MOD:
if (i->sType == TYPE_U32 && imm0.isPow2()) {
bld.setPosition(i, false);
i->op = OP_AND;
i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 - 1));
}
break;
 
case OP_SET: // TODO: SET_AND,OR,XOR
{
CmpInstruction *si = findOriginForTestWithZero(i->getSrc(t));
CondCode cc, ccZ;
if (i->src(t).mod != Modifier(0))
return;
if (imm0.reg.data.u32 != 0 || !si || si->op != OP_SET)
return;
cc = si->setCond;
ccZ = (CondCode)((unsigned int)i->asCmp()->setCond & ~CC_U);
if (s == 0)
ccZ = reverseCondCode(ccZ);
switch (ccZ) {
case CC_LT: cc = CC_FL; break;
case CC_GE: cc = CC_TR; break;
case CC_EQ: cc = inverseCondCode(cc); break;
case CC_LE: cc = inverseCondCode(cc); break;
case CC_GT: break;
case CC_NE: break;
default:
return;
}
i->asCmp()->setCond = cc;
i->setSrc(0, si->src(0));
i->setSrc(1, si->src(1));
i->sType = si->sType;
}
break;
 
case OP_SHL:
{
if (s != 1 || i->src(0).mod != Modifier(0))
break;
// try to concatenate shifts
Instruction *si = i->getSrc(0)->getInsn();
if (!si || si->op != OP_SHL)
break;
ImmediateValue imm1;
if (si->src(1).getImmediate(imm1)) {
bld.setPosition(i, false);
i->setSrc(0, si->getSrc(0));
i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 + imm1.reg.data.u32));
}
}
break;
 
case OP_ABS:
case OP_NEG:
case OP_LG2:
case OP_RCP:
case OP_SQRT:
case OP_RSQ:
case OP_PRESIN:
case OP_SIN:
case OP_COS:
case OP_PREEX2:
case OP_EX2:
unary(i, imm0);
break;
default:
return;
}
if (i->op != op)
foldCount++;
}
 
// =============================================================================
 
// Merge modifier operations (ABS, NEG, NOT) into ValueRefs where allowed.
class ModifierFolding : public Pass
{
private:
virtual bool visit(BasicBlock *);
};
 
bool
ModifierFolding::visit(BasicBlock *bb)
{
const Target *target = prog->getTarget();
 
Instruction *i, *next, *mi;
Modifier mod;
 
for (i = bb->getEntry(); i; i = next) {
next = i->next;
 
if (0 && i->op == OP_SUB) {
// turn "sub" into "add neg" (do we really want this ?)
i->op = OP_ADD;
i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
}
 
for (int s = 0; s < 3 && i->srcExists(s); ++s) {
mi = i->getSrc(s)->getInsn();
if (!mi ||
mi->predSrc >= 0 || mi->getDef(0)->refCount() > 8)
continue;
if (i->sType == TYPE_U32 && mi->dType == TYPE_S32) {
if ((i->op != OP_ADD &&
i->op != OP_MUL) ||
(mi->op != OP_ABS &&
mi->op != OP_NEG))
continue;
} else
if (i->sType != mi->dType) {
continue;
}
if ((mod = Modifier(mi->op)) == Modifier(0))
continue;
mod *= mi->src(0).mod;
 
if ((i->op == OP_ABS) || i->src(s).mod.abs()) {
// abs neg [abs] = abs
mod = mod & Modifier(~(NV50_IR_MOD_NEG | NV50_IR_MOD_ABS));
} else
if ((i->op == OP_NEG) && mod.neg()) {
assert(s == 0);
// neg as both opcode and modifier on same insn is prohibited
// neg neg abs = abs, neg neg = identity
mod = mod & Modifier(~NV50_IR_MOD_NEG);
i->op = mod.getOp();
mod = mod & Modifier(~NV50_IR_MOD_ABS);
if (mod == Modifier(0))
i->op = OP_MOV;
}
 
if (target->isModSupported(i, s, mod)) {
i->setSrc(s, mi->getSrc(0));
i->src(s).mod *= mod;
}
}
 
if (i->op == OP_SAT) {
mi = i->getSrc(0)->getInsn();
if (mi &&
mi->getDef(0)->refCount() <= 1 && target->isSatSupported(mi)) {
mi->saturate = 1;
mi->setDef(0, i->getDef(0));
delete_Instruction(prog, i);
}
}
}
 
return true;
}
 
// =============================================================================
 
// MUL + ADD -> MAD/FMA
// MIN/MAX(a, a) -> a, etc.
// SLCT(a, b, const) -> cc(const) ? a : b
// RCP(RCP(a)) -> a
// MUL(MUL(a, b), const) -> MUL_Xconst(a, b)
class AlgebraicOpt : public Pass
{
private:
virtual bool visit(BasicBlock *);
 
void handleABS(Instruction *);
bool handleADD(Instruction *);
bool tryADDToMADOrSAD(Instruction *, operation toOp);
void handleMINMAX(Instruction *);
void handleRCP(Instruction *);
void handleSLCT(Instruction *);
void handleLOGOP(Instruction *);
void handleCVT(Instruction *);
void handleSUCLAMP(Instruction *);
 
BuildUtil bld;
};
 
void
AlgebraicOpt::handleABS(Instruction *abs)
{
Instruction *sub = abs->getSrc(0)->getInsn();
DataType ty;
if (!sub ||
!prog->getTarget()->isOpSupported(OP_SAD, abs->dType))
return;
// expect not to have mods yet, if we do, bail
if (sub->src(0).mod || sub->src(1).mod)
return;
// hidden conversion ?
ty = intTypeToSigned(sub->dType);
if (abs->dType != abs->sType || ty != abs->sType)
return;
 
if ((sub->op != OP_ADD && sub->op != OP_SUB) ||
sub->src(0).getFile() != FILE_GPR || sub->src(0).mod ||
sub->src(1).getFile() != FILE_GPR || sub->src(1).mod)
return;
 
Value *src0 = sub->getSrc(0);
Value *src1 = sub->getSrc(1);
 
if (sub->op == OP_ADD) {
Instruction *neg = sub->getSrc(1)->getInsn();
if (neg && neg->op != OP_NEG) {
neg = sub->getSrc(0)->getInsn();
src0 = sub->getSrc(1);
}
if (!neg || neg->op != OP_NEG ||
neg->dType != neg->sType || neg->sType != ty)
return;
src1 = neg->getSrc(0);
}
 
// found ABS(SUB))
abs->moveSources(1, 2); // move sources >=1 up by 2
abs->op = OP_SAD;
abs->setType(sub->dType);
abs->setSrc(0, src0);
abs->setSrc(1, src1);
bld.setPosition(abs, false);
abs->setSrc(2, bld.loadImm(bld.getSSA(typeSizeof(ty)), 0));
}
 
bool
AlgebraicOpt::handleADD(Instruction *add)
{
Value *src0 = add->getSrc(0);
Value *src1 = add->getSrc(1);
 
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
return false;
 
bool changed = false;
if (!changed && prog->getTarget()->isOpSupported(OP_MAD, add->dType))
changed = tryADDToMADOrSAD(add, OP_MAD);
if (!changed && prog->getTarget()->isOpSupported(OP_SAD, add->dType))
changed = tryADDToMADOrSAD(add, OP_SAD);
return changed;
}
 
// ADD(SAD(a,b,0), c) -> SAD(a,b,c)
// ADD(MUL(a,b), c) -> MAD(a,b,c)
bool
AlgebraicOpt::tryADDToMADOrSAD(Instruction *add, operation toOp)
{
Value *src0 = add->getSrc(0);
Value *src1 = add->getSrc(1);
Value *src;
int s;
const operation srcOp = toOp == OP_SAD ? OP_SAD : OP_MUL;
const Modifier modBad = Modifier(~((toOp == OP_MAD) ? NV50_IR_MOD_NEG : 0));
Modifier mod[4];
 
if (src0->refCount() == 1 &&
src0->getUniqueInsn() && src0->getUniqueInsn()->op == srcOp)
s = 0;
else
if (src1->refCount() == 1 &&
src1->getUniqueInsn() && src1->getUniqueInsn()->op == srcOp)
s = 1;
else
return false;
 
if ((src0->getUniqueInsn() && src0->getUniqueInsn()->bb != add->bb) ||
(src1->getUniqueInsn() && src1->getUniqueInsn()->bb != add->bb))
return false;
 
src = add->getSrc(s);
 
if (src->getInsn()->postFactor)
return false;
if (toOp == OP_SAD) {
ImmediateValue imm;
if (!src->getInsn()->src(2).getImmediate(imm))
return false;
if (!imm.isInteger(0))
return false;
}
 
mod[0] = add->src(0).mod;
mod[1] = add->src(1).mod;
mod[2] = src->getUniqueInsn()->src(0).mod;
mod[3] = src->getUniqueInsn()->src(1).mod;
 
if (((mod[0] | mod[1]) | (mod[2] | mod[3])) & modBad)
return false;
 
add->op = toOp;
add->subOp = src->getInsn()->subOp; // potentially mul-high
 
add->setSrc(2, add->src(s ? 0 : 1));
 
add->setSrc(0, src->getInsn()->getSrc(0));
add->src(0).mod = mod[2] ^ mod[s];
add->setSrc(1, src->getInsn()->getSrc(1));
add->src(1).mod = mod[3];
 
return true;
}
 
void
AlgebraicOpt::handleMINMAX(Instruction *minmax)
{
Value *src0 = minmax->getSrc(0);
Value *src1 = minmax->getSrc(1);
 
if (src0 != src1 || src0->reg.file != FILE_GPR)
return;
if (minmax->src(0).mod == minmax->src(1).mod) {
if (minmax->def(0).mayReplace(minmax->src(0))) {
minmax->def(0).replace(minmax->src(0), false);
minmax->bb->remove(minmax);
} else {
minmax->op = OP_CVT;
minmax->setSrc(1, NULL);
}
} else {
// TODO:
// min(x, -x) = -abs(x)
// min(x, -abs(x)) = -abs(x)
// min(x, abs(x)) = x
// max(x, -abs(x)) = x
// max(x, abs(x)) = abs(x)
// max(x, -x) = abs(x)
}
}
 
void
AlgebraicOpt::handleRCP(Instruction *rcp)
{
Instruction *si = rcp->getSrc(0)->getUniqueInsn();
 
if (si && si->op == OP_RCP) {
Modifier mod = rcp->src(0).mod * si->src(0).mod;
rcp->op = mod.getOp();
rcp->setSrc(0, si->getSrc(0));
}
}
 
void
AlgebraicOpt::handleSLCT(Instruction *slct)
{
if (slct->getSrc(2)->reg.file == FILE_IMMEDIATE) {
if (slct->getSrc(2)->asImm()->compare(slct->asCmp()->setCond, 0.0f))
slct->setSrc(0, slct->getSrc(1));
} else
if (slct->getSrc(0) != slct->getSrc(1)) {
return;
}
slct->op = OP_MOV;
slct->setSrc(1, NULL);
slct->setSrc(2, NULL);
}
 
void
AlgebraicOpt::handleLOGOP(Instruction *logop)
{
Value *src0 = logop->getSrc(0);
Value *src1 = logop->getSrc(1);
 
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
return;
 
if (src0 == src1) {
if ((logop->op == OP_AND || logop->op == OP_OR) &&
logop->def(0).mayReplace(logop->src(0))) {
logop->def(0).replace(logop->src(0), false);
delete_Instruction(prog, logop);
}
} else {
// try AND(SET, SET) -> SET_AND(SET)
Instruction *set0 = src0->getInsn();
Instruction *set1 = src1->getInsn();
 
if (!set0 || set0->fixed || !set1 || set1->fixed)
return;
if (set1->op != OP_SET) {
Instruction *xchg = set0;
set0 = set1;
set1 = xchg;
if (set1->op != OP_SET)
return;
}
operation redOp = (logop->op == OP_AND ? OP_SET_AND :
logop->op == OP_XOR ? OP_SET_XOR : OP_SET_OR);
if (!prog->getTarget()->isOpSupported(redOp, set1->sType))
return;
if (set0->op != OP_SET &&
set0->op != OP_SET_AND &&
set0->op != OP_SET_OR &&
set0->op != OP_SET_XOR)
return;
if (set0->getDef(0)->refCount() > 1 &&
set1->getDef(0)->refCount() > 1)
return;
if (set0->getPredicate() || set1->getPredicate())
return;
// check that they don't source each other
for (int s = 0; s < 2; ++s)
if (set0->getSrc(s) == set1->getDef(0) ||
set1->getSrc(s) == set0->getDef(0))
return;
 
set0 = cloneForward(func, set0);
set1 = cloneShallow(func, set1);
logop->bb->insertAfter(logop, set1);
logop->bb->insertAfter(logop, set0);
 
set0->dType = TYPE_U8;
set0->getDef(0)->reg.file = FILE_PREDICATE;
set0->getDef(0)->reg.size = 1;
set1->setSrc(2, set0->getDef(0));
set1->op = redOp;
set1->setDef(0, logop->getDef(0));
delete_Instruction(prog, logop);
}
}
 
// F2I(NEG(SET with result 1.0f/0.0f)) -> SET with result -1/0
// nv50:
// F2I(NEG(I2F(ABS(SET))))
void
AlgebraicOpt::handleCVT(Instruction *cvt)
{
if (cvt->sType != TYPE_F32 ||
cvt->dType != TYPE_S32 || cvt->src(0).mod != Modifier(0))
return;
Instruction *insn = cvt->getSrc(0)->getInsn();
if (!insn || insn->op != OP_NEG || insn->dType != TYPE_F32)
return;
if (insn->src(0).mod != Modifier(0))
return;
insn = insn->getSrc(0)->getInsn();
 
// check for nv50 SET(-1,0) -> SET(1.0f/0.0f) chain and nvc0's f32 SET
if (insn && insn->op == OP_CVT &&
insn->dType == TYPE_F32 &&
insn->sType == TYPE_S32) {
insn = insn->getSrc(0)->getInsn();
if (!insn || insn->op != OP_ABS || insn->sType != TYPE_S32 ||
insn->src(0).mod)
return;
insn = insn->getSrc(0)->getInsn();
if (!insn || insn->op != OP_SET || insn->dType != TYPE_U32)
return;
} else
if (!insn || insn->op != OP_SET || insn->dType != TYPE_F32) {
return;
}
 
Instruction *bset = cloneShallow(func, insn);
bset->dType = TYPE_U32;
bset->setDef(0, cvt->getDef(0));
cvt->bb->insertAfter(cvt, bset);
delete_Instruction(prog, cvt);
}
 
// SUCLAMP dst, (ADD b imm), k, 0 -> SUCLAMP dst, b, k, imm (if imm fits s6)
void
AlgebraicOpt::handleSUCLAMP(Instruction *insn)
{
ImmediateValue imm;
int32_t val = insn->getSrc(2)->asImm()->reg.data.s32;
int s;
Instruction *add;
 
assert(insn->srcExists(0) && insn->src(0).getFile() == FILE_GPR);
 
// look for ADD (TODO: only count references by non-SUCLAMP)
if (insn->getSrc(0)->refCount() > 1)
return;
add = insn->getSrc(0)->getInsn();
if (!add || add->op != OP_ADD ||
(add->dType != TYPE_U32 &&
add->dType != TYPE_S32))
return;
 
// look for immediate
for (s = 0; s < 2; ++s)
if (add->src(s).getImmediate(imm))
break;
if (s >= 2)
return;
s = s ? 0 : 1;
// determine if immediate fits
val += imm.reg.data.s32;
if (val > 31 || val < -32)
return;
// determine if other addend fits
if (add->src(s).getFile() != FILE_GPR || add->src(s).mod != Modifier(0))
return;
 
bld.setPosition(insn, false); // make sure bld is init'ed
// replace sources
insn->setSrc(2, bld.mkImm(val));
insn->setSrc(0, add->getSrc(s));
}
 
bool
AlgebraicOpt::visit(BasicBlock *bb)
{
Instruction *next;
for (Instruction *i = bb->getEntry(); i; i = next) {
next = i->next;
switch (i->op) {
case OP_ABS:
handleABS(i);
break;
case OP_ADD:
handleADD(i);
break;
case OP_RCP:
handleRCP(i);
break;
case OP_MIN:
case OP_MAX:
handleMINMAX(i);
break;
case OP_SLCT:
handleSLCT(i);
break;
case OP_AND:
case OP_OR:
case OP_XOR:
handleLOGOP(i);
break;
case OP_CVT:
handleCVT(i);
break;
case OP_SUCLAMP:
handleSUCLAMP(i);
break;
default:
break;
}
}
 
return true;
}
 
// =============================================================================
 
static inline void
updateLdStOffset(Instruction *ldst, int32_t offset, Function *fn)
{
if (offset != ldst->getSrc(0)->reg.data.offset) {
if (ldst->getSrc(0)->refCount() > 1)
ldst->setSrc(0, cloneShallow(fn, ldst->getSrc(0)));
ldst->getSrc(0)->reg.data.offset = offset;
}
}
 
// Combine loads and stores, forward stores to loads where possible.
class MemoryOpt : public Pass
{
private:
class Record
{
public:
Record *next;
Instruction *insn;
const Value *rel[2];
const Value *base;
int32_t offset;
int8_t fileIndex;
uint8_t size;
bool locked;
Record *prev;
 
bool overlaps(const Instruction *ldst) const;
 
inline void link(Record **);
inline void unlink(Record **);
inline void set(const Instruction *ldst);
};
 
public:
MemoryOpt();
 
Record *loads[DATA_FILE_COUNT];
Record *stores[DATA_FILE_COUNT];
 
MemoryPool recordPool;
 
private:
virtual bool visit(BasicBlock *);
bool runOpt(BasicBlock *);
 
Record **getList(const Instruction *);
 
Record *findRecord(const Instruction *, bool load, bool& isAdjacent) const;
 
// merge @insn into load/store instruction from @rec
bool combineLd(Record *rec, Instruction *ld);
bool combineSt(Record *rec, Instruction *st);
 
bool replaceLdFromLd(Instruction *ld, Record *ldRec);
bool replaceLdFromSt(Instruction *ld, Record *stRec);
bool replaceStFromSt(Instruction *restrict st, Record *stRec);
 
void addRecord(Instruction *ldst);
void purgeRecords(Instruction *const st, DataFile);
void lockStores(Instruction *const ld);
void reset();
 
private:
Record *prevRecord;
};
 
MemoryOpt::MemoryOpt() : recordPool(sizeof(MemoryOpt::Record), 6)
{
for (int i = 0; i < DATA_FILE_COUNT; ++i) {
loads[i] = NULL;
stores[i] = NULL;
}
prevRecord = NULL;
}
 
void
MemoryOpt::reset()
{
for (unsigned int i = 0; i < DATA_FILE_COUNT; ++i) {
Record *it, *next;
for (it = loads[i]; it; it = next) {
next = it->next;
recordPool.release(it);
}
loads[i] = NULL;
for (it = stores[i]; it; it = next) {
next = it->next;
recordPool.release(it);
}
stores[i] = NULL;
}
}
 
bool
MemoryOpt::combineLd(Record *rec, Instruction *ld)
{
int32_t offRc = rec->offset;
int32_t offLd = ld->getSrc(0)->reg.data.offset;
int sizeRc = rec->size;
int sizeLd = typeSizeof(ld->dType);
int size = sizeRc + sizeLd;
int d, j;
 
if (!prog->getTarget()->
isAccessSupported(ld->getSrc(0)->reg.file, typeOfSize(size)))
return false;
// no unaligned loads
if (((size == 0x8) && (MIN2(offLd, offRc) & 0x7)) ||
((size == 0xc) && (MIN2(offLd, offRc) & 0xf)))
return false;
 
assert(sizeRc + sizeLd <= 16 && offRc != offLd);
 
for (j = 0; sizeRc; sizeRc -= rec->insn->getDef(j)->reg.size, ++j);
 
if (offLd < offRc) {
int sz;
for (sz = 0, d = 0; sz < sizeLd; sz += ld->getDef(d)->reg.size, ++d);
// d: nr of definitions in ld
// j: nr of definitions in rec->insn, move:
for (d = d + j - 1; j > 0; --j, --d)
rec->insn->setDef(d, rec->insn->getDef(j - 1));
 
if (rec->insn->getSrc(0)->refCount() > 1)
rec->insn->setSrc(0, cloneShallow(func, rec->insn->getSrc(0)));
rec->offset = rec->insn->getSrc(0)->reg.data.offset = offLd;
 
d = 0;
} else {
d = j;
}
// move definitions of @ld to @rec->insn
for (j = 0; sizeLd; ++j, ++d) {
sizeLd -= ld->getDef(j)->reg.size;
rec->insn->setDef(d, ld->getDef(j));
}
 
rec->size = size;
rec->insn->getSrc(0)->reg.size = size;
rec->insn->setType(typeOfSize(size));
 
delete_Instruction(prog, ld);
 
return true;
}
 
bool
MemoryOpt::combineSt(Record *rec, Instruction *st)
{
int32_t offRc = rec->offset;
int32_t offSt = st->getSrc(0)->reg.data.offset;
int sizeRc = rec->size;
int sizeSt = typeSizeof(st->dType);
int s = sizeSt / 4;
int size = sizeRc + sizeSt;
int j, k;
Value *src[4]; // no modifiers in ValueRef allowed for st
Value *extra[3];
 
if (!prog->getTarget()->
isAccessSupported(st->getSrc(0)->reg.file, typeOfSize(size)))
return false;
if (size == 8 && MIN2(offRc, offSt) & 0x7)
return false;
 
st->takeExtraSources(0, extra); // save predicate and indirect address
 
if (offRc < offSt) {
// save values from @st
for (s = 0; sizeSt; ++s) {
sizeSt -= st->getSrc(s + 1)->reg.size;
src[s] = st->getSrc(s + 1);
}
// set record's values as low sources of @st
for (j = 1; sizeRc; ++j) {
sizeRc -= rec->insn->getSrc(j)->reg.size;
st->setSrc(j, rec->insn->getSrc(j));
}
// set saved values as high sources of @st
for (k = j, j = 0; j < s; ++j)
st->setSrc(k++, src[j]);
 
updateLdStOffset(st, offRc, func);
} else {
for (j = 1; sizeSt; ++j)
sizeSt -= st->getSrc(j)->reg.size;
for (s = 1; sizeRc; ++j, ++s) {
sizeRc -= rec->insn->getSrc(s)->reg.size;
st->setSrc(j, rec->insn->getSrc(s));
}
rec->offset = offSt;
}
st->putExtraSources(0, extra); // restore pointer and predicate
 
delete_Instruction(prog, rec->insn);
rec->insn = st;
rec->size = size;
rec->insn->getSrc(0)->reg.size = size;
rec->insn->setType(typeOfSize(size));
return true;
}
 
void
MemoryOpt::Record::set(const Instruction *ldst)
{
const Symbol *mem = ldst->getSrc(0)->asSym();
fileIndex = mem->reg.fileIndex;
rel[0] = ldst->getIndirect(0, 0);
rel[1] = ldst->getIndirect(0, 1);
offset = mem->reg.data.offset;
base = mem->getBase();
size = typeSizeof(ldst->sType);
}
 
void
MemoryOpt::Record::link(Record **list)
{
next = *list;
if (next)
next->prev = this;
prev = NULL;
*list = this;
}
 
void
MemoryOpt::Record::unlink(Record **list)
{
if (next)
next->prev = prev;
if (prev)
prev->next = next;
else
*list = next;
}
 
MemoryOpt::Record **
MemoryOpt::getList(const Instruction *insn)
{
if (insn->op == OP_LOAD || insn->op == OP_VFETCH)
return &loads[insn->src(0).getFile()];
return &stores[insn->src(0).getFile()];
}
 
void
MemoryOpt::addRecord(Instruction *i)
{
Record **list = getList(i);
Record *it = reinterpret_cast<Record *>(recordPool.allocate());
 
it->link(list);
it->set(i);
it->insn = i;
it->locked = false;
}
 
MemoryOpt::Record *
MemoryOpt::findRecord(const Instruction *insn, bool load, bool& isAdj) const
{
const Symbol *sym = insn->getSrc(0)->asSym();
const int size = typeSizeof(insn->sType);
Record *rec = NULL;
Record *it = load ? loads[sym->reg.file] : stores[sym->reg.file];
 
for (; it; it = it->next) {
if (it->locked && insn->op != OP_LOAD)
continue;
if ((it->offset >> 4) != (sym->reg.data.offset >> 4) ||
it->rel[0] != insn->getIndirect(0, 0) ||
it->fileIndex != sym->reg.fileIndex ||
it->rel[1] != insn->getIndirect(0, 1))
continue;
 
if (it->offset < sym->reg.data.offset) {
if (it->offset + it->size >= sym->reg.data.offset) {
isAdj = (it->offset + it->size == sym->reg.data.offset);
if (!isAdj)
return it;
if (!(it->offset & 0x7))
rec = it;
}
} else {
isAdj = it->offset != sym->reg.data.offset;
if (size <= it->size && !isAdj)
return it;
else
if (!(sym->reg.data.offset & 0x7))
if (it->offset - size <= sym->reg.data.offset)
rec = it;
}
}
return rec;
}
 
bool
MemoryOpt::replaceLdFromSt(Instruction *ld, Record *rec)
{
Instruction *st = rec->insn;
int32_t offSt = rec->offset;
int32_t offLd = ld->getSrc(0)->reg.data.offset;
int d, s;
 
for (s = 1; offSt != offLd && st->srcExists(s); ++s)
offSt += st->getSrc(s)->reg.size;
if (offSt != offLd)
return false;
 
for (d = 0; ld->defExists(d) && st->srcExists(s); ++d, ++s) {
if (ld->getDef(d)->reg.size != st->getSrc(s)->reg.size)
return false;
if (st->getSrc(s)->reg.file != FILE_GPR)
return false;
ld->def(d).replace(st->src(s), false);
}
ld->bb->remove(ld);
return true;
}
 
bool
MemoryOpt::replaceLdFromLd(Instruction *ldE, Record *rec)
{
Instruction *ldR = rec->insn;
int32_t offR = rec->offset;
int32_t offE = ldE->getSrc(0)->reg.data.offset;
int dR, dE;
 
assert(offR <= offE);
for (dR = 0; offR < offE && ldR->defExists(dR); ++dR)
offR += ldR->getDef(dR)->reg.size;
if (offR != offE)
return false;
 
for (dE = 0; ldE->defExists(dE) && ldR->defExists(dR); ++dE, ++dR) {
if (ldE->getDef(dE)->reg.size != ldR->getDef(dR)->reg.size)
return false;
ldE->def(dE).replace(ldR->getDef(dR), false);
}
 
delete_Instruction(prog, ldE);
return true;
}
 
bool
MemoryOpt::replaceStFromSt(Instruction *restrict st, Record *rec)
{
const Instruction *const ri = rec->insn;
Value *extra[3];
 
int32_t offS = st->getSrc(0)->reg.data.offset;
int32_t offR = rec->offset;
int32_t endS = offS + typeSizeof(st->dType);
int32_t endR = offR + typeSizeof(ri->dType);
 
rec->size = MAX2(endS, endR) - MIN2(offS, offR);
 
st->takeExtraSources(0, extra);
 
if (offR < offS) {
Value *vals[10];
int s, n;
int k = 0;
// get non-replaced sources of ri
for (s = 1; offR < offS; offR += ri->getSrc(s)->reg.size, ++s)
vals[k++] = ri->getSrc(s);
n = s;
// get replaced sources of st
for (s = 1; st->srcExists(s); offS += st->getSrc(s)->reg.size, ++s)
vals[k++] = st->getSrc(s);
// skip replaced sources of ri
for (s = n; offR < endS; offR += ri->getSrc(s)->reg.size, ++s);
// get non-replaced sources after values covered by st
for (; offR < endR; offR += ri->getSrc(s)->reg.size, ++s)
vals[k++] = ri->getSrc(s);
assert((unsigned int)k <= Elements(vals));
for (s = 0; s < k; ++s)
st->setSrc(s + 1, vals[s]);
st->setSrc(0, ri->getSrc(0));
} else
if (endR > endS) {
int j, s;
for (j = 1; offR < endS; offR += ri->getSrc(j++)->reg.size);
for (s = 1; offS < endS; offS += st->getSrc(s++)->reg.size);
for (; offR < endR; offR += ri->getSrc(j++)->reg.size)
st->setSrc(s++, ri->getSrc(j));
}
st->putExtraSources(0, extra);
 
delete_Instruction(prog, rec->insn);
 
rec->insn = st;
rec->offset = st->getSrc(0)->reg.data.offset;
 
st->setType(typeOfSize(rec->size));
 
return true;
}
 
bool
MemoryOpt::Record::overlaps(const Instruction *ldst) const
{
Record that;
that.set(ldst);
 
if (this->fileIndex != that.fileIndex)
return false;
 
if (this->rel[0] || that.rel[0])
return this->base == that.base;
return
(this->offset < that.offset + that.size) &&
(this->offset + this->size > that.offset);
}
 
// We must not eliminate stores that affect the result of @ld if
// we find later stores to the same location, and we may no longer
// merge them with later stores.
// The stored value can, however, still be used to determine the value
// returned by future loads.
void
MemoryOpt::lockStores(Instruction *const ld)
{
for (Record *r = stores[ld->src(0).getFile()]; r; r = r->next)
if (!r->locked && r->overlaps(ld))
r->locked = true;
}
 
// Prior loads from the location of @st are no longer valid.
// Stores to the location of @st may no longer be used to derive
// the value at it nor be coalesced into later stores.
void
MemoryOpt::purgeRecords(Instruction *const st, DataFile f)
{
if (st)
f = st->src(0).getFile();
 
for (Record *r = loads[f]; r; r = r->next)
if (!st || r->overlaps(st))
r->unlink(&loads[f]);
 
for (Record *r = stores[f]; r; r = r->next)
if (!st || r->overlaps(st))
r->unlink(&stores[f]);
}
 
bool
MemoryOpt::visit(BasicBlock *bb)
{
bool ret = runOpt(bb);
// Run again, one pass won't combine 4 32 bit ld/st to a single 128 bit ld/st
// where 96 bit memory operations are forbidden.
if (ret)
ret = runOpt(bb);
return ret;
}
 
bool
MemoryOpt::runOpt(BasicBlock *bb)
{
Instruction *ldst, *next;
Record *rec;
bool isAdjacent = true;
 
for (ldst = bb->getEntry(); ldst; ldst = next) {
bool keep = true;
bool isLoad = true;
next = ldst->next;
 
if (ldst->op == OP_LOAD || ldst->op == OP_VFETCH) {
if (ldst->isDead()) {
// might have been produced by earlier optimization
delete_Instruction(prog, ldst);
continue;
}
} else
if (ldst->op == OP_STORE || ldst->op == OP_EXPORT) {
isLoad = false;
} else {
// TODO: maybe have all fixed ops act as barrier ?
if (ldst->op == OP_CALL ||
ldst->op == OP_BAR ||
ldst->op == OP_MEMBAR) {
purgeRecords(NULL, FILE_MEMORY_LOCAL);
purgeRecords(NULL, FILE_MEMORY_GLOBAL);
purgeRecords(NULL, FILE_MEMORY_SHARED);
purgeRecords(NULL, FILE_SHADER_OUTPUT);
} else
if (ldst->op == OP_ATOM || ldst->op == OP_CCTL) {
if (ldst->src(0).getFile() == FILE_MEMORY_GLOBAL) {
purgeRecords(NULL, FILE_MEMORY_LOCAL);
purgeRecords(NULL, FILE_MEMORY_GLOBAL);
purgeRecords(NULL, FILE_MEMORY_SHARED);
} else {
purgeRecords(NULL, ldst->src(0).getFile());
}
} else
if (ldst->op == OP_EMIT || ldst->op == OP_RESTART) {
purgeRecords(NULL, FILE_SHADER_OUTPUT);
}
continue;
}
if (ldst->getPredicate()) // TODO: handle predicated ld/st
continue;
 
if (isLoad) {
DataFile file = ldst->src(0).getFile();
 
// if ld l[]/g[] look for previous store to eliminate the reload
if (file == FILE_MEMORY_GLOBAL || file == FILE_MEMORY_LOCAL) {
// TODO: shared memory ?
rec = findRecord(ldst, false, isAdjacent);
if (rec && !isAdjacent)
keep = !replaceLdFromSt(ldst, rec);
}
 
// or look for ld from the same location and replace this one
rec = keep ? findRecord(ldst, true, isAdjacent) : NULL;
if (rec) {
if (!isAdjacent)
keep = !replaceLdFromLd(ldst, rec);
else
// or combine a previous load with this one
keep = !combineLd(rec, ldst);
}
if (keep)
lockStores(ldst);
} else {
rec = findRecord(ldst, false, isAdjacent);
if (rec) {
if (!isAdjacent)
keep = !replaceStFromSt(ldst, rec);
else
keep = !combineSt(rec, ldst);
}
if (keep)
purgeRecords(ldst, DATA_FILE_COUNT);
}
if (keep)
addRecord(ldst);
}
reset();
 
return true;
}
 
// =============================================================================
 
// Turn control flow into predicated instructions (after register allocation !).
// TODO:
// Could move this to before register allocation on NVC0 and also handle nested
// constructs.
class FlatteningPass : public Pass
{
private:
virtual bool visit(BasicBlock *);
 
bool tryPredicateConditional(BasicBlock *);
void predicateInstructions(BasicBlock *, Value *pred, CondCode cc);
void tryPropagateBranch(BasicBlock *);
inline bool isConstantCondition(Value *pred);
inline bool mayPredicate(const Instruction *, const Value *pred) const;
inline void removeFlow(Instruction *);
};
 
bool
FlatteningPass::isConstantCondition(Value *pred)
{
Instruction *insn = pred->getUniqueInsn();
assert(insn);
if (insn->op != OP_SET || insn->srcExists(2))
return false;
 
for (int s = 0; s < 2 && insn->srcExists(s); ++s) {
Instruction *ld = insn->getSrc(s)->getUniqueInsn();
DataFile file;
if (ld) {
if (ld->op != OP_MOV && ld->op != OP_LOAD)
return false;
if (ld->src(0).isIndirect(0))
return false;
file = ld->src(0).getFile();
} else {
file = insn->src(s).getFile();
// catch $r63 on NVC0
if (file == FILE_GPR && insn->getSrc(s)->reg.data.id > prog->maxGPR)
file = FILE_IMMEDIATE;
}
if (file != FILE_IMMEDIATE && file != FILE_MEMORY_CONST)
return false;
}
return true;
}
 
void
FlatteningPass::removeFlow(Instruction *insn)
{
FlowInstruction *term = insn ? insn->asFlow() : NULL;
if (!term)
return;
Graph::Edge::Type ty = term->bb->cfg.outgoing().getType();
 
if (term->op == OP_BRA) {
// TODO: this might get more difficult when we get arbitrary BRAs
if (ty == Graph::Edge::CROSS || ty == Graph::Edge::BACK)
return;
} else
if (term->op != OP_JOIN)
return;
 
Value *pred = term->getPredicate();
 
delete_Instruction(prog, term);
 
if (pred && pred->refCount() == 0) {
Instruction *pSet = pred->getUniqueInsn();
pred->join->reg.data.id = -1; // deallocate
if (pSet->isDead())
delete_Instruction(prog, pSet);
}
}
 
void
FlatteningPass::predicateInstructions(BasicBlock *bb, Value *pred, CondCode cc)
{
for (Instruction *i = bb->getEntry(); i; i = i->next) {
if (i->isNop())
continue;
assert(!i->getPredicate());
i->setPredicate(cc, pred);
}
removeFlow(bb->getExit());
}
 
bool
FlatteningPass::mayPredicate(const Instruction *insn, const Value *pred) const
{
if (insn->isPseudo())
return true;
// TODO: calls where we don't know which registers are modified
 
if (!prog->getTarget()->mayPredicate(insn, pred))
return false;
for (int d = 0; insn->defExists(d); ++d)
if (insn->getDef(d)->equals(pred))
return false;
return true;
}
 
// If we jump to BRA/RET/EXIT, replace the jump with it.
// NOTE: We do not update the CFG anymore here !
//
// TODO: Handle cases where we skip over a branch (maybe do that elsewhere ?):
// BB:0
// @p0 bra BB:2 -> @!p0 bra BB:3 iff (!) BB:2 immediately adjoins BB:1
// BB1:
// bra BB:3
// BB2:
// ...
// BB3:
// ...
void
FlatteningPass::tryPropagateBranch(BasicBlock *bb)
{
for (Instruction *i = bb->getExit(); i && i->op == OP_BRA; i = i->prev) {
BasicBlock *bf = i->asFlow()->target.bb;
 
if (bf->getInsnCount() != 1)
continue;
 
FlowInstruction *bra = i->asFlow();
FlowInstruction *rep = bf->getExit()->asFlow();
 
if (!rep || rep->getPredicate())
continue;
if (rep->op != OP_BRA &&
rep->op != OP_JOIN &&
rep->op != OP_EXIT)
continue;
 
// TODO: If there are multiple branches to @rep, only the first would
// be replaced, so only remove them after this pass is done ?
// Also, need to check all incident blocks for fall-through exits and
// add the branch there.
bra->op = rep->op;
bra->target.bb = rep->target.bb;
if (bf->cfg.incidentCount() == 1)
bf->remove(rep);
}
}
 
bool
FlatteningPass::visit(BasicBlock *bb)
{
if (tryPredicateConditional(bb))
return true;
 
// try to attach join to previous instruction
Instruction *insn = bb->getExit();
if (insn && insn->op == OP_JOIN && !insn->getPredicate()) {
insn = insn->prev;
if (insn && !insn->getPredicate() &&
!insn->asFlow() &&
insn->op != OP_TEXBAR &&
!isTextureOp(insn->op) && // probably just nve4
!isSurfaceOp(insn->op) && // not confirmed
insn->op != OP_LINTERP && // probably just nve4
insn->op != OP_PINTERP && // probably just nve4
((insn->op != OP_LOAD && insn->op != OP_STORE) ||
typeSizeof(insn->dType) <= 4) &&
!insn->isNop()) {
insn->join = 1;
bb->remove(bb->getExit());
return true;
}
}
 
tryPropagateBranch(bb);
 
return true;
}
 
bool
FlatteningPass::tryPredicateConditional(BasicBlock *bb)
{
BasicBlock *bL = NULL, *bR = NULL;
unsigned int nL = 0, nR = 0, limit = 12;
Instruction *insn;
unsigned int mask;
 
mask = bb->initiatesSimpleConditional();
if (!mask)
return false;
 
assert(bb->getExit());
Value *pred = bb->getExit()->getPredicate();
assert(pred);
 
if (isConstantCondition(pred))
limit = 4;
 
Graph::EdgeIterator ei = bb->cfg.outgoing();
 
if (mask & 1) {
bL = BasicBlock::get(ei.getNode());
for (insn = bL->getEntry(); insn; insn = insn->next, ++nL)
if (!mayPredicate(insn, pred))
return false;
if (nL > limit)
return false; // too long, do a real branch
}
ei.next();
 
if (mask & 2) {
bR = BasicBlock::get(ei.getNode());
for (insn = bR->getEntry(); insn; insn = insn->next, ++nR)
if (!mayPredicate(insn, pred))
return false;
if (nR > limit)
return false; // too long, do a real branch
}
 
if (bL)
predicateInstructions(bL, pred, bb->getExit()->cc);
if (bR)
predicateInstructions(bR, pred, inverseCondCode(bb->getExit()->cc));
 
if (bb->joinAt) {
bb->remove(bb->joinAt);
bb->joinAt = NULL;
}
removeFlow(bb->getExit()); // delete the branch/join at the fork point
 
// remove potential join operations at the end of the conditional
if (prog->getTarget()->joinAnterior) {
bb = BasicBlock::get((bL ? bL : bR)->cfg.outgoing().getNode());
if (bb->getEntry() && bb->getEntry()->op == OP_JOIN)
removeFlow(bb->getEntry());
}
 
return true;
}
 
// =============================================================================
 
// Common subexpression elimination. Stupid O^2 implementation.
class LocalCSE : public Pass
{
private:
virtual bool visit(BasicBlock *);
 
inline bool tryReplace(Instruction **, Instruction *);
 
DLList ops[OP_LAST + 1];
};
 
class GlobalCSE : public Pass
{
private:
virtual bool visit(BasicBlock *);
};
 
bool
Instruction::isActionEqual(const Instruction *that) const
{
if (this->op != that->op ||
this->dType != that->dType ||
this->sType != that->sType)
return false;
if (this->cc != that->cc)
return false;
 
if (this->asTex()) {
if (memcmp(&this->asTex()->tex,
&that->asTex()->tex,
sizeof(this->asTex()->tex)))
return false;
} else
if (this->asCmp()) {
if (this->asCmp()->setCond != that->asCmp()->setCond)
return false;
} else
if (this->asFlow()) {
return false;
} else {
if (this->ipa != that->ipa ||
this->lanes != that->lanes ||
this->perPatch != that->perPatch)
return false;
if (this->postFactor != that->postFactor)
return false;
}
 
if (this->subOp != that->subOp ||
this->saturate != that->saturate ||
this->rnd != that->rnd ||
this->ftz != that->ftz ||
this->dnz != that->dnz ||
this->cache != that->cache ||
this->mask != that->mask)
return false;
 
return true;
}
 
bool
Instruction::isResultEqual(const Instruction *that) const
{
unsigned int d, s;
 
// NOTE: location of discard only affects tex with liveOnly and quadops
if (!this->defExists(0) && this->op != OP_DISCARD)
return false;
 
if (!isActionEqual(that))
return false;
 
if (this->predSrc != that->predSrc)
return false;
 
for (d = 0; this->defExists(d); ++d) {
if (!that->defExists(d) ||
!this->getDef(d)->equals(that->getDef(d), false))
return false;
}
if (that->defExists(d))
return false;
 
for (s = 0; this->srcExists(s); ++s) {
if (!that->srcExists(s))
return false;
if (this->src(s).mod != that->src(s).mod)
return false;
if (!this->getSrc(s)->equals(that->getSrc(s), true))
return false;
}
if (that->srcExists(s))
return false;
 
if (op == OP_LOAD || op == OP_VFETCH) {
switch (src(0).getFile()) {
case FILE_MEMORY_CONST:
case FILE_SHADER_INPUT:
return true;
default:
return false;
}
}
 
return true;
}
 
// pull through common expressions from different in-blocks
bool
GlobalCSE::visit(BasicBlock *bb)
{
Instruction *phi, *next, *ik;
int s;
 
// TODO: maybe do this with OP_UNION, too
 
for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = next) {
next = phi->next;
if (phi->getSrc(0)->refCount() > 1)
continue;
ik = phi->getSrc(0)->getInsn();
if (!ik)
continue; // probably a function input
for (s = 1; phi->srcExists(s); ++s) {
if (phi->getSrc(s)->refCount() > 1)
break;
if (!phi->getSrc(s)->getInsn() ||
!phi->getSrc(s)->getInsn()->isResultEqual(ik))
break;
}
if (!phi->srcExists(s)) {
Instruction *entry = bb->getEntry();
ik->bb->remove(ik);
if (!entry || entry->op != OP_JOIN)
bb->insertHead(ik);
else
bb->insertAfter(entry, ik);
ik->setDef(0, phi->getDef(0));
delete_Instruction(prog, phi);
}
}
 
return true;
}
 
bool
LocalCSE::tryReplace(Instruction **ptr, Instruction *i)
{
Instruction *old = *ptr;
 
// TODO: maybe relax this later (causes trouble with OP_UNION)
if (i->isPredicated())
return false;
 
if (!old->isResultEqual(i))
return false;
 
for (int d = 0; old->defExists(d); ++d)
old->def(d).replace(i->getDef(d), false);
delete_Instruction(prog, old);
*ptr = NULL;
return true;
}
 
bool
LocalCSE::visit(BasicBlock *bb)
{
unsigned int replaced;
 
do {
Instruction *ir, *next;
 
replaced = 0;
 
// will need to know the order of instructions
int serial = 0;
for (ir = bb->getFirst(); ir; ir = ir->next)
ir->serial = serial++;
 
for (ir = bb->getEntry(); ir; ir = next) {
int s;
Value *src = NULL;
 
next = ir->next;
 
if (ir->fixed) {
ops[ir->op].insert(ir);
continue;
}
 
for (s = 0; ir->srcExists(s); ++s)
if (ir->getSrc(s)->asLValue())
if (!src || ir->getSrc(s)->refCount() < src->refCount())
src = ir->getSrc(s);
 
if (src) {
for (Value::UseIterator it = src->uses.begin();
it != src->uses.end(); ++it) {
Instruction *ik = (*it)->getInsn();
if (ik && ik->bb == ir->bb && ik->serial < ir->serial)
if (tryReplace(&ir, ik))
break;
}
} else {
DLLIST_FOR_EACH(&ops[ir->op], iter)
{
Instruction *ik = reinterpret_cast<Instruction *>(iter.get());
if (tryReplace(&ir, ik))
break;
}
}
 
if (ir)
ops[ir->op].insert(ir);
else
++replaced;
}
for (unsigned int i = 0; i <= OP_LAST; ++i)
ops[i].clear();
 
} while (replaced);
 
return true;
}
 
// =============================================================================
 
// Remove computations of unused values.
class DeadCodeElim : public Pass
{
public:
bool buryAll(Program *);
 
private:
virtual bool visit(BasicBlock *);
 
void checkSplitLoad(Instruction *ld); // for partially dead loads
 
unsigned int deadCount;
};
 
bool
DeadCodeElim::buryAll(Program *prog)
{
do {
deadCount = 0;
if (!this->run(prog, false, false))
return false;
} while (deadCount);
 
return true;
}
 
bool
DeadCodeElim::visit(BasicBlock *bb)
{
Instruction *next;
 
for (Instruction *i = bb->getFirst(); i; i = next) {
next = i->next;
if (i->isDead()) {
++deadCount;
delete_Instruction(prog, i);
} else
if (i->defExists(1) && (i->op == OP_VFETCH || i->op == OP_LOAD)) {
checkSplitLoad(i);
} else
if (i->defExists(0) && !i->getDef(0)->refCount()) {
if (i->op == OP_ATOM ||
i->op == OP_SUREDP ||
i->op == OP_SUREDB)
i->setDef(0, NULL);
}
}
return true;
}
 
void
DeadCodeElim::checkSplitLoad(Instruction *ld1)
{
Instruction *ld2 = NULL; // can get at most 2 loads
Value *def1[4];
Value *def2[4];
int32_t addr1, addr2;
int32_t size1, size2;
int d, n1, n2;
uint32_t mask = 0xffffffff;
 
for (d = 0; ld1->defExists(d); ++d)
if (!ld1->getDef(d)->refCount() && ld1->getDef(d)->reg.data.id < 0)
mask &= ~(1 << d);
if (mask == 0xffffffff)
return;
 
addr1 = ld1->getSrc(0)->reg.data.offset;
n1 = n2 = 0;
size1 = size2 = 0;
for (d = 0; ld1->defExists(d); ++d) {
if (mask & (1 << d)) {
if (size1 && (addr1 & 0x7))
break;
def1[n1] = ld1->getDef(d);
size1 += def1[n1++]->reg.size;
} else
if (!n1) {
addr1 += ld1->getDef(d)->reg.size;
} else {
break;
}
}
for (addr2 = addr1 + size1; ld1->defExists(d); ++d) {
if (mask & (1 << d)) {
def2[n2] = ld1->getDef(d);
size2 += def2[n2++]->reg.size;
} else {
assert(!n2);
addr2 += ld1->getDef(d)->reg.size;
}
}
 
updateLdStOffset(ld1, addr1, func);
ld1->setType(typeOfSize(size1));
for (d = 0; d < 4; ++d)
ld1->setDef(d, (d < n1) ? def1[d] : NULL);
 
if (!n2)
return;
 
ld2 = cloneShallow(func, ld1);
updateLdStOffset(ld2, addr2, func);
ld2->setType(typeOfSize(size2));
for (d = 0; d < 4; ++d)
ld2->setDef(d, (d < n2) ? def2[d] : NULL);
 
ld1->bb->insertAfter(ld1, ld2);
}
 
// =============================================================================
 
#define RUN_PASS(l, n, f) \
if (level >= (l)) { \
if (dbgFlags & NV50_IR_DEBUG_VERBOSE) \
INFO("PEEPHOLE: %s\n", #n); \
n pass; \
if (!pass.f(this)) \
return false; \
}
 
bool
Program::optimizeSSA(int level)
{
RUN_PASS(1, DeadCodeElim, buryAll);
RUN_PASS(1, CopyPropagation, run);
RUN_PASS(2, GlobalCSE, run);
RUN_PASS(1, LocalCSE, run);
RUN_PASS(2, AlgebraicOpt, run);
RUN_PASS(2, ModifierFolding, run); // before load propagation -> less checks
RUN_PASS(1, ConstantFolding, foldAll);
RUN_PASS(1, LoadPropagation, run);
RUN_PASS(2, MemoryOpt, run);
RUN_PASS(2, LocalCSE, run);
RUN_PASS(0, DeadCodeElim, buryAll);
 
return true;
}
 
bool
Program::optimizePostRA(int level)
{
RUN_PASS(2, FlatteningPass, run);
return true;
}
 
}
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp
0,0 → 1,698
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_target.h"
 
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
 
namespace nv50_ir {
 
enum TextStyle
{
TXT_DEFAULT,
TXT_GPR,
TXT_REGISTER,
TXT_FLAGS,
TXT_MEM,
TXT_IMMD,
TXT_BRA,
TXT_INSN
};
 
static const char *_colour[8] =
{
"\x1b[00m",
"\x1b[34m",
"\x1b[35m",
"\x1b[35m",
"\x1b[36m",
"\x1b[33m",
"\x1b[37m",
"\x1b[32m"
};
 
static const char *_nocolour[8] =
{
"", "", "", "", "", "", "", ""
};
 
static const char **colour;
 
static void init_colours()
{
if (getenv("NV50_PROG_DEBUG_NO_COLORS") != NULL)
colour = _nocolour;
else
colour = _colour;
}
 
const char *operationStr[OP_LAST + 1] =
{
"nop",
"phi",
"union",
"split",
"merge",
"consec",
"mov",
"ld",
"st",
"add",
"sub",
"mul",
"div",
"mod",
"mad",
"fma",
"sad",
"abs",
"neg",
"not",
"and",
"or",
"xor",
"shl",
"shr",
"max",
"min",
"sat",
"ceil",
"floor",
"trunc",
"cvt",
"set and",
"set or",
"set xor",
"set",
"selp",
"slct",
"rcp",
"rsq",
"lg2",
"sin",
"cos",
"ex2",
"exp",
"log",
"presin",
"preex2",
"sqrt",
"pow",
"bra",
"call",
"ret",
"cont",
"break",
"preret",
"precont",
"prebreak",
"brkpt",
"joinat",
"join",
"discard",
"exit",
"membar",
"vfetch",
"pfetch",
"export",
"linterp",
"pinterp",
"emit",
"restart",
"tex",
"texbias",
"texlod",
"texfetch",
"texquery",
"texgrad",
"texgather",
"texcsaa",
"texprep",
"suldb",
"suldp",
"sustb",
"sustp",
"suredb",
"suredp",
"sulea",
"subfm",
"suclamp",
"sueau",
"madsp",
"texbar",
"dfdx",
"dfdy",
"rdsv",
"wrsv",
"quadop",
"quadon",
"quadpop",
"popcnt",
"insbf",
"extbf",
"permt",
"atom",
"bar",
"vadd",
"vavg",
"vmin",
"vmax",
"vsad",
"vset",
"vshr",
"vshl",
"vsel",
"cctl",
"(invalid)"
};
 
static const char *atomSubOpStr[] =
{
"add", "min", "max", "inc", "dec", "and", "or", "xor", "cas", "exch"
};
 
static const char *DataTypeStr[] =
{
"-",
"u8", "s8",
"u16", "s16",
"u32", "s32",
"u64", "s64",
"f16", "f32", "f64",
"b96", "b128"
};
 
static const char *RoundModeStr[] =
{
"", "rm", "rz", "rp", "rni", "rmi", "rzi", "rpi"
};
 
static const char *CondCodeStr[] =
{
"never",
"lt",
"eq",
"le",
"gt",
"ne",
"ge",
"",
"(invalid)",
"ltu",
"equ",
"leu",
"gtu",
"neu",
"geu",
"",
"no",
"nc",
"ns",
"na",
"a",
"s",
"c",
"o"
};
 
static const char *SemanticStr[SV_LAST + 1] =
{
"POSITION",
"VERTEX_ID",
"INSTANCE_ID",
"INVOCATION_ID",
"PRIMITIVE_ID",
"VERTEX_COUNT",
"LAYER",
"VIEWPORT_INDEX",
"Y_DIR",
"FACE",
"POINT_SIZE",
"POINT_COORD",
"CLIP_DISTANCE",
"SAMPLE_INDEX",
"TESS_FACTOR",
"TESS_COORD",
"TID",
"CTAID",
"NTID",
"GRIDID",
"NCTAID",
"LANEID",
"PHYSID",
"NPHYSID",
"CLOCK",
"LBASE",
"SBASE",
"?",
"(INVALID)"
};
 
static const char *interpStr[16] =
{
"pass",
"mul",
"flat",
"sc",
"cent pass",
"cent mul",
"cent flat",
"cent sc",
"off pass",
"off mul",
"off flat",
"off sc",
"samp pass",
"samp mul",
"samp flat",
"samp sc"
};
 
#define PRINT(args...) \
do { \
pos += snprintf(&buf[pos], size - pos, args); \
} while(0)
 
#define SPACE_PRINT(cond, args...) \
do { \
if (cond) \
buf[pos++] = ' '; \
pos += snprintf(&buf[pos], size - pos, args); \
} while(0)
 
#define SPACE() \
do { \
if (pos < size) \
buf[pos++] = ' '; \
} while(0)
 
int Modifier::print(char *buf, size_t size) const
{
size_t pos = 0;
 
if (bits)
PRINT("%s", colour[TXT_INSN]);
 
size_t base = pos;
 
if (bits & NV50_IR_MOD_NOT)
PRINT("not");
if (bits & NV50_IR_MOD_SAT)
SPACE_PRINT(pos > base && pos < size, "sat");
if (bits & NV50_IR_MOD_NEG)
SPACE_PRINT(pos > base && pos < size, "neg");
if (bits & NV50_IR_MOD_ABS)
SPACE_PRINT(pos > base && pos < size, "abs");
 
return pos;
}
 
int LValue::print(char *buf, size_t size, DataType ty) const
{
const char *postFix = "";
size_t pos = 0;
int idx = join->reg.data.id >= 0 ? join->reg.data.id : id;
char p = join->reg.data.id >= 0 ? '$' : '%';
char r;
int col = TXT_DEFAULT;
 
switch (reg.file) {
case FILE_GPR:
r = 'r'; col = TXT_GPR;
if (reg.size == 2) {
if (p == '$') {
postFix = (idx & 1) ? "h" : "l";
idx /= 2;
} else {
postFix = "s";
}
} else
if (reg.size == 8) {
postFix = "d";
} else
if (reg.size == 16) {
postFix = "q";
} else
if (reg.size == 12) {
postFix = "t";
}
break;
case FILE_PREDICATE:
r = 'p'; col = TXT_REGISTER;
if (reg.size == 2)
postFix = "d";
else
if (reg.size == 4)
postFix = "q";
break;
case FILE_FLAGS:
r = 'c'; col = TXT_FLAGS;
break;
case FILE_ADDRESS:
r = 'a'; col = TXT_REGISTER;
break;
default:
assert(!"invalid file for lvalue");
r = '?';
break;
}
 
PRINT("%s%c%c%i%s", colour[col], p, r, idx, postFix);
 
return pos;
}
 
int ImmediateValue::print(char *buf, size_t size, DataType ty) const
{
size_t pos = 0;
 
PRINT("%s", colour[TXT_IMMD]);
 
switch (ty) {
case TYPE_F32: PRINT("%f", reg.data.f32); break;
case TYPE_F64: PRINT("%f", reg.data.f64); break;
case TYPE_U8: PRINT("0x%02x", reg.data.u8); break;
case TYPE_S8: PRINT("%i", reg.data.s8); break;
case TYPE_U16: PRINT("0x%04x", reg.data.u16); break;
case TYPE_S16: PRINT("%i", reg.data.s16); break;
case TYPE_U32: PRINT("0x%08x", reg.data.u32); break;
case TYPE_S32: PRINT("%i", reg.data.s32); break;
case TYPE_U64:
case TYPE_S64:
default:
PRINT("0x%016"PRIx64, reg.data.u64);
break;
}
return pos;
}
 
int Symbol::print(char *buf, size_t size, DataType ty) const
{
return print(buf, size, NULL, NULL, ty);
}
 
int Symbol::print(char *buf, size_t size,
Value *rel, Value *dimRel, DataType ty) const
{
size_t pos = 0;
char c;
 
if (ty == TYPE_NONE)
ty = typeOfSize(reg.size);
 
if (reg.file == FILE_SYSTEM_VALUE) {
PRINT("%ssv[%s%s:%i%s", colour[TXT_MEM],
colour[TXT_REGISTER],
SemanticStr[reg.data.sv.sv], reg.data.sv.index, colour[TXT_MEM]);
if (rel) {
PRINT("%s+", colour[TXT_DEFAULT]);
pos += rel->print(&buf[pos], size - pos);
}
PRINT("%s]", colour[TXT_MEM]);
return pos;
}
 
switch (reg.file) {
case FILE_MEMORY_CONST: c = 'c'; break;
case FILE_SHADER_INPUT: c = 'a'; break;
case FILE_SHADER_OUTPUT: c = 'o'; break;
case FILE_MEMORY_GLOBAL: c = 'g'; break;
case FILE_MEMORY_SHARED: c = 's'; break;
case FILE_MEMORY_LOCAL: c = 'l'; break;
default:
assert(!"invalid file");
c = '?';
break;
}
 
if (c == 'c')
PRINT("%s%c%i[", colour[TXT_MEM], c, reg.fileIndex);
else
PRINT("%s%c[", colour[TXT_MEM], c);
 
if (dimRel) {
pos += dimRel->print(&buf[pos], size - pos, TYPE_S32);
PRINT("%s][", colour[TXT_MEM]);
}
 
if (rel) {
pos += rel->print(&buf[pos], size - pos);
PRINT("%s%c", colour[TXT_DEFAULT], (reg.data.offset < 0) ? '-' : '+');
} else {
assert(reg.data.offset >= 0);
}
PRINT("%s0x%x%s]", colour[TXT_IMMD], abs(reg.data.offset), colour[TXT_MEM]);
 
return pos;
}
 
void Instruction::print() const
{
#define BUFSZ 512
 
const size_t size = BUFSZ;
 
char buf[BUFSZ];
int s, d;
size_t pos = 0;
 
PRINT("%s", colour[TXT_INSN]);
 
if (join)
PRINT("join ");
 
if (predSrc >= 0) {
const size_t pre = pos;
if (getSrc(predSrc)->reg.file == FILE_PREDICATE) {
if (cc == CC_NOT_P)
PRINT("not");
} else {
PRINT("%s", CondCodeStr[cc]);
}
if (pos > pre)
SPACE();
pos += getSrc(predSrc)->print(&buf[pos], BUFSZ - pos);
PRINT(" %s", colour[TXT_INSN]);
}
 
if (saturate)
PRINT("sat ");
 
if (asFlow()) {
PRINT("%s", operationStr[op]);
if (asFlow()->indirect)
PRINT(" ind");
if (asFlow()->absolute)
PRINT(" abs");
if (op == OP_CALL && asFlow()->builtin) {
PRINT(" %sBUILTIN:%i", colour[TXT_BRA], asFlow()->target.builtin);
} else
if (op == OP_CALL && asFlow()->target.fn) {
PRINT(" %s%s:%i", colour[TXT_BRA],
asFlow()->target.fn->getName(),
asFlow()->target.fn->getLabel());
} else
if (asFlow()->target.bb)
PRINT(" %sBB:%i", colour[TXT_BRA], asFlow()->target.bb->getId());
} else {
PRINT("%s ", operationStr[op]);
if (op == OP_LINTERP || op == OP_PINTERP)
PRINT("%s ", interpStr[ipa]);
switch (op) {
case OP_SUREDP:
case OP_ATOM:
if (subOp < Elements(atomSubOpStr))
PRINT("%s ", atomSubOpStr[subOp]);
break;
default:
if (subOp)
PRINT("(SUBOP:%u) ", subOp);
break;
}
if (perPatch)
PRINT("patch ");
if (asTex())
PRINT("%s %s$r%u $s%u %s", asTex()->tex.target.getName(),
colour[TXT_MEM], asTex()->tex.r, asTex()->tex.s,
colour[TXT_INSN]);
if (postFactor)
PRINT("x2^%i ", postFactor);
PRINT("%s%s", dnz ? "dnz " : (ftz ? "ftz " : ""), DataTypeStr[dType]);
}
 
if (rnd != ROUND_N)
PRINT(" %s", RoundModeStr[rnd]);
 
if (defExists(1))
PRINT(" {");
for (d = 0; defExists(d); ++d) {
SPACE();
pos += getDef(d)->print(&buf[pos], size - pos);
}
if (d > 1)
PRINT(" %s}", colour[TXT_INSN]);
else
if (!d && !asFlow())
PRINT(" %s#", colour[TXT_INSN]);
 
if (asCmp())
PRINT(" %s%s", colour[TXT_INSN], CondCodeStr[asCmp()->setCond]);
 
if (sType != dType)
PRINT(" %s%s", colour[TXT_INSN], DataTypeStr[sType]);
 
for (s = 0; srcExists(s); ++s) {
if (s == predSrc || src(s).usedAsPtr)
continue;
const size_t pre = pos;
SPACE();
pos += src(s).mod.print(&buf[pos], BUFSZ - pos);
if (pos > pre + 1)
SPACE();
if (src(s).isIndirect(0) || src(s).isIndirect(1))
pos += getSrc(s)->asSym()->print(&buf[pos], BUFSZ - pos,
getIndirect(s, 0),
getIndirect(s, 1));
else
pos += getSrc(s)->print(&buf[pos], BUFSZ - pos, sType);
}
if (exit)
PRINT("%s exit", colour[TXT_INSN]);
 
PRINT("%s", colour[TXT_DEFAULT]);
 
buf[MIN2(pos, BUFSZ - 1)] = 0;
 
INFO("%s (%u)\n", buf, encSize);
}
 
class PrintPass : public Pass
{
public:
PrintPass() : serial(0) { }
 
virtual bool visit(Function *);
virtual bool visit(BasicBlock *);
virtual bool visit(Instruction *);
 
private:
int serial;
};
 
bool
PrintPass::visit(Function *fn)
{
char str[16];
 
INFO("\n%s:%i (", fn->getName(), fn->getLabel());
 
if (!fn->outs.empty())
INFO("out");
for (std::deque<ValueRef>::iterator it = fn->outs.begin();
it != fn->outs.end();
++it) {
it->get()->print(str, sizeof(str), typeOfSize(it->get()->reg.size));
INFO(" %s", str);
}
 
if (!fn->ins.empty())
INFO("%s%sin", colour[TXT_DEFAULT], fn->outs.empty() ? "" : ", ");
for (std::deque<ValueDef>::iterator it = fn->ins.begin();
it != fn->ins.end();
++it) {
it->get()->print(str, sizeof(str), typeOfSize(it->get()->reg.size));
INFO(" %s", str);
}
INFO("%s)\n", colour[TXT_DEFAULT]);
 
return true;
}
 
bool
PrintPass::visit(BasicBlock *bb)
{
#if 0
INFO("---\n");
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next())
INFO(" <- BB:%i (%s)\n",
BasicBlock::get(ei.getNode())->getId(),
ei.getEdge()->typeStr());
#endif
INFO("BB:%i (%u instructions) - ", bb->getId(), bb->getInsnCount());
 
if (bb->idom())
INFO("idom = BB:%i, ", bb->idom()->getId());
 
INFO("df = { ");
for (DLList::Iterator df = bb->getDF().iterator(); !df.end(); df.next())
INFO("BB:%i ", BasicBlock::get(df)->getId());
 
INFO("}\n");
 
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next())
INFO(" -> BB:%i (%s)\n",
BasicBlock::get(ei.getNode())->getId(),
ei.getEdge()->typeStr());
 
return true;
}
 
bool
PrintPass::visit(Instruction *insn)
{
INFO("%3i: ", serial++);
insn->print();
return true;
}
 
void
Function::print()
{
PrintPass pass;
pass.run(this, true, false);
}
 
void
Program::print()
{
PrintPass pass;
init_colours();
pass.run(this, true, false);
}
 
void
Function::printLiveIntervals() const
{
INFO("printing live intervals ...\n");
 
for (ArrayList::Iterator it = allLValues.iterator(); !it.end(); it.next()) {
const Value *lval = Value::get(it)->asLValue();
if (lval && !lval->livei.isEmpty()) {
INFO("livei(%%%i): ", lval->id);
lval->livei.print();
}
}
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_ra.cpp
0,0 → 1,2050
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_target.h"
 
#include <stack>
#include <limits>
 
namespace nv50_ir {
 
#define MAX_REGISTER_FILE_SIZE 256
 
class RegisterSet
{
public:
RegisterSet(const Target *);
 
void init(const Target *);
void reset(DataFile, bool resetMax = false);
 
void periodicMask(DataFile f, uint32_t lock, uint32_t unlock);
void intersect(DataFile f, const RegisterSet *);
 
bool assign(int32_t& reg, DataFile f, unsigned int size);
void release(DataFile f, int32_t reg, unsigned int size);
void occupy(DataFile f, int32_t reg, unsigned int size);
void occupy(const Value *);
void occupyMask(DataFile f, int32_t reg, uint8_t mask);
bool isOccupied(DataFile f, int32_t reg, unsigned int size) const;
bool testOccupy(const Value *);
bool testOccupy(DataFile f, int32_t reg, unsigned int size);
 
inline int getMaxAssigned(DataFile f) const { return fill[f]; }
 
inline unsigned int getFileSize(DataFile f, uint8_t regSize) const
{
if (restrictedGPR16Range && f == FILE_GPR && regSize == 2)
return (last[f] + 1) / 2;
return last[f] + 1;
}
 
inline unsigned int units(DataFile f, unsigned int size) const
{
return size >> unit[f];
}
// for regs of size >= 4, id is counted in 4-byte words (like nv50/c0 binary)
inline unsigned int idToBytes(const Value *v) const
{
return v->reg.data.id * MIN2(v->reg.size, 4);
}
inline unsigned int idToUnits(const Value *v) const
{
return units(v->reg.file, idToBytes(v));
}
inline int bytesToId(Value *v, unsigned int bytes) const
{
if (v->reg.size < 4)
return units(v->reg.file, bytes);
return bytes / 4;
}
inline int unitsToId(DataFile f, int u, uint8_t size) const
{
if (u < 0)
return -1;
return (size < 4) ? u : ((u << unit[f]) / 4);
}
 
void print() const;
 
private:
BitSet bits[LAST_REGISTER_FILE + 1];
 
int unit[LAST_REGISTER_FILE + 1]; // log2 of allocation granularity
 
int last[LAST_REGISTER_FILE + 1];
int fill[LAST_REGISTER_FILE + 1];
 
const bool restrictedGPR16Range;
};
 
void
RegisterSet::reset(DataFile f, bool resetMax)
{
bits[f].fill(0);
if (resetMax)
fill[f] = -1;
}
 
void
RegisterSet::init(const Target *targ)
{
for (unsigned int rf = 0; rf <= FILE_ADDRESS; ++rf) {
DataFile f = static_cast<DataFile>(rf);
last[rf] = targ->getFileSize(f) - 1;
unit[rf] = targ->getFileUnit(f);
fill[rf] = -1;
assert(last[rf] < MAX_REGISTER_FILE_SIZE);
bits[rf].allocate(last[rf] + 1, true);
}
}
 
RegisterSet::RegisterSet(const Target *targ)
: restrictedGPR16Range(targ->getChipset() < 0xc0)
{
init(targ);
for (unsigned int i = 0; i <= LAST_REGISTER_FILE; ++i)
reset(static_cast<DataFile>(i));
}
 
void
RegisterSet::periodicMask(DataFile f, uint32_t lock, uint32_t unlock)
{
bits[f].periodicMask32(lock, unlock);
}
 
void
RegisterSet::intersect(DataFile f, const RegisterSet *set)
{
bits[f] |= set->bits[f];
}
 
void
RegisterSet::print() const
{
INFO("GPR:");
bits[FILE_GPR].print();
INFO("\n");
}
 
bool
RegisterSet::assign(int32_t& reg, DataFile f, unsigned int size)
{
reg = bits[f].findFreeRange(size);
if (reg < 0)
return false;
fill[f] = MAX2(fill[f], (int32_t)(reg + size - 1));
return true;
}
 
bool
RegisterSet::isOccupied(DataFile f, int32_t reg, unsigned int size) const
{
return bits[f].testRange(reg, size);
}
 
void
RegisterSet::occupy(const Value *v)
{
occupy(v->reg.file, idToUnits(v), v->reg.size >> unit[v->reg.file]);
}
 
void
RegisterSet::occupyMask(DataFile f, int32_t reg, uint8_t mask)
{
bits[f].setMask(reg & ~31, static_cast<uint32_t>(mask) << (reg % 32));
}
 
void
RegisterSet::occupy(DataFile f, int32_t reg, unsigned int size)
{
bits[f].setRange(reg, size);
 
INFO_DBG(0, REG_ALLOC, "reg occupy: %u[%i] %u\n", f, reg, size);
 
fill[f] = MAX2(fill[f], (int32_t)(reg + size - 1));
}
 
bool
RegisterSet::testOccupy(const Value *v)
{
return testOccupy(v->reg.file,
idToUnits(v), v->reg.size >> unit[v->reg.file]);
}
 
bool
RegisterSet::testOccupy(DataFile f, int32_t reg, unsigned int size)
{
if (isOccupied(f, reg, size))
return false;
occupy(f, reg, size);
return true;
}
 
void
RegisterSet::release(DataFile f, int32_t reg, unsigned int size)
{
bits[f].clrRange(reg, size);
 
INFO_DBG(0, REG_ALLOC, "reg release: %u[%i] %u\n", f, reg, size);
}
 
class RegAlloc
{
public:
RegAlloc(Program *program) : prog(program), sequence(0) { }
 
bool exec();
bool execFunc();
 
private:
class PhiMovesPass : public Pass {
private:
virtual bool visit(BasicBlock *);
inline bool needNewElseBlock(BasicBlock *b, BasicBlock *p);
};
 
class ArgumentMovesPass : public Pass {
private:
virtual bool visit(BasicBlock *);
};
 
class BuildIntervalsPass : public Pass {
private:
virtual bool visit(BasicBlock *);
void collectLiveValues(BasicBlock *);
void addLiveRange(Value *, const BasicBlock *, int end);
};
 
class InsertConstraintsPass : public Pass {
public:
bool exec(Function *func);
private:
virtual bool visit(BasicBlock *);
 
bool insertConstraintMoves();
 
void condenseDefs(Instruction *);
void condenseSrcs(Instruction *, const int first, const int last);
 
void addHazard(Instruction *i, const ValueRef *src);
void textureMask(TexInstruction *);
void addConstraint(Instruction *, int s, int n);
bool detectConflict(Instruction *, int s);
 
// target specific functions, TODO: put in subclass or Target
void texConstraintNV50(TexInstruction *);
void texConstraintNVC0(TexInstruction *);
void texConstraintNVE0(TexInstruction *);
 
std::list<Instruction *> constrList;
 
const Target *targ;
};
 
bool buildLiveSets(BasicBlock *);
 
private:
Program *prog;
Function *func;
 
// instructions in control flow / chronological order
ArrayList insns;
 
int sequence; // for manual passes through CFG
};
 
typedef std::pair<Value *, Value *> ValuePair;
 
class SpillCodeInserter
{
public:
SpillCodeInserter(Function *fn) : func(fn), stackSize(0), stackBase(0) { }
 
bool run(const std::list<ValuePair>&);
 
Symbol *assignSlot(const Interval&, const unsigned int size);
inline int32_t getStackSize() const { return stackSize; }
 
private:
Function *func;
 
struct SpillSlot
{
Interval occup;
std::list<Value *> residents; // needed to recalculate occup
Symbol *sym;
int32_t offset;
inline uint8_t size() const { return sym->reg.size; }
};
std::list<SpillSlot> slots;
int32_t stackSize;
int32_t stackBase;
 
LValue *unspill(Instruction *usei, LValue *, Value *slot);
void spill(Instruction *defi, Value *slot, LValue *);
};
 
void
RegAlloc::BuildIntervalsPass::addLiveRange(Value *val,
const BasicBlock *bb,
int end)
{
Instruction *insn = val->getUniqueInsn();
 
if (!insn)
insn = bb->getFirst();
 
assert(bb->getFirst()->serial <= bb->getExit()->serial);
assert(bb->getExit()->serial + 1 >= end);
 
int begin = insn->serial;
if (begin < bb->getEntry()->serial || begin > bb->getExit()->serial)
begin = bb->getEntry()->serial;
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "%%%i <- live range [%i(%i), %i)\n",
val->id, begin, insn->serial, end);
 
if (begin != end) // empty ranges are only added as hazards for fixed regs
val->livei.extend(begin, end);
}
 
bool
RegAlloc::PhiMovesPass::needNewElseBlock(BasicBlock *b, BasicBlock *p)
{
if (b->cfg.incidentCount() <= 1)
return false;
 
int n = 0;
for (Graph::EdgeIterator ei = p->cfg.outgoing(); !ei.end(); ei.next())
if (ei.getType() == Graph::Edge::TREE ||
ei.getType() == Graph::Edge::FORWARD)
++n;
return (n == 2);
}
 
// For each operand of each PHI in b, generate a new value by inserting a MOV
// at the end of the block it is coming from and replace the operand with its
// result. This eliminates liveness conflicts and enables us to let values be
// copied to the right register if such a conflict exists nonetheless.
//
// These MOVs are also crucial in making sure the live intervals of phi srces
// are extended until the end of the loop, since they are not included in the
// live-in sets.
bool
RegAlloc::PhiMovesPass::visit(BasicBlock *bb)
{
Instruction *phi, *mov;
BasicBlock *pb, *pn;
 
std::stack<BasicBlock *> stack;
 
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
pb = BasicBlock::get(ei.getNode());
assert(pb);
if (needNewElseBlock(bb, pb))
stack.push(pb);
}
while (!stack.empty()) {
pb = stack.top();
pn = new BasicBlock(func);
stack.pop();
 
pb->cfg.detach(&bb->cfg);
pb->cfg.attach(&pn->cfg, Graph::Edge::TREE);
pn->cfg.attach(&bb->cfg, Graph::Edge::FORWARD);
 
assert(pb->getExit()->op != OP_CALL);
if (pb->getExit()->asFlow()->target.bb == bb)
pb->getExit()->asFlow()->target.bb = pn;
}
 
// insert MOVs (phi->src(j) should stem from j-th in-BB)
int j = 0;
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
pb = BasicBlock::get(ei.getNode());
if (!pb->isTerminated())
pb->insertTail(new_FlowInstruction(func, OP_BRA, bb));
 
for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = phi->next) {
mov = new_Instruction(func, OP_MOV, TYPE_U32);
 
mov->setSrc(0, phi->getSrc(j));
mov->setDef(0, new_LValue(func, phi->getDef(0)->asLValue()));
phi->setSrc(j, mov->getDef(0));
 
pb->insertBefore(pb->getExit(), mov);
}
++j;
}
 
return true;
}
 
bool
RegAlloc::ArgumentMovesPass::visit(BasicBlock *bb)
{
// Bind function call inputs/outputs to the same physical register
// the callee uses, inserting moves as appropriate for the case a
// conflict arises.
for (Instruction *i = bb->getEntry(); i; i = i->next) {
FlowInstruction *cal = i->asFlow();
// TODO: Handle indirect calls.
// Right now they should only be generated for builtins.
if (!cal || cal->op != OP_CALL || cal->builtin || cal->indirect)
continue;
RegisterSet clobberSet(prog->getTarget());
 
// Bind input values.
for (int s = cal->indirect ? 1 : 0; cal->srcExists(s); ++s) {
const int t = cal->indirect ? (s - 1) : s;
LValue *tmp = new_LValue(func, cal->getSrc(s)->asLValue());
tmp->reg.data.id = cal->target.fn->ins[t].rep()->reg.data.id;
 
Instruction *mov =
new_Instruction(func, OP_MOV, typeOfSize(tmp->reg.size));
mov->setDef(0, tmp);
mov->setSrc(0, cal->getSrc(s));
cal->setSrc(s, tmp);
 
bb->insertBefore(cal, mov);
}
 
// Bind output values.
for (int d = 0; cal->defExists(d); ++d) {
LValue *tmp = new_LValue(func, cal->getDef(d)->asLValue());
tmp->reg.data.id = cal->target.fn->outs[d].rep()->reg.data.id;
 
Instruction *mov =
new_Instruction(func, OP_MOV, typeOfSize(tmp->reg.size));
mov->setSrc(0, tmp);
mov->setDef(0, cal->getDef(d));
cal->setDef(d, tmp);
 
bb->insertAfter(cal, mov);
clobberSet.occupy(tmp);
}
 
// Bind clobbered values.
for (std::deque<Value *>::iterator it = cal->target.fn->clobbers.begin();
it != cal->target.fn->clobbers.end();
++it) {
if (clobberSet.testOccupy(*it)) {
Value *tmp = new_LValue(func, (*it)->asLValue());
tmp->reg.data.id = (*it)->reg.data.id;
cal->setDef(cal->defCount(), tmp);
}
}
}
 
// Update the clobber set of the function.
if (BasicBlock::get(func->cfgExit) == bb) {
func->buildDefSets();
for (unsigned int i = 0; i < bb->defSet.getSize(); ++i)
if (bb->defSet.test(i))
func->clobbers.push_back(func->getLValue(i));
}
 
return true;
}
 
// Build the set of live-in variables of bb.
bool
RegAlloc::buildLiveSets(BasicBlock *bb)
{
Function *f = bb->getFunction();
BasicBlock *bn;
Instruction *i;
unsigned int s, d;
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "buildLiveSets(BB:%i)\n", bb->getId());
 
bb->liveSet.allocate(func->allLValues.getSize(), false);
 
int n = 0;
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
bn = BasicBlock::get(ei.getNode());
if (bn == bb)
continue;
if (bn->cfg.visit(sequence))
if (!buildLiveSets(bn))
return false;
if (n++ || bb->liveSet.marker)
bb->liveSet |= bn->liveSet;
else
bb->liveSet = bn->liveSet;
}
if (!n && !bb->liveSet.marker)
bb->liveSet.fill(0);
bb->liveSet.marker = true;
 
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) {
INFO("BB:%i live set of out blocks:\n", bb->getId());
bb->liveSet.print();
}
 
// if (!bb->getEntry())
// return true;
 
if (bb == BasicBlock::get(f->cfgExit)) {
for (std::deque<ValueRef>::iterator it = f->outs.begin();
it != f->outs.end(); ++it) {
assert(it->get()->asLValue());
bb->liveSet.set(it->get()->id);
}
}
 
for (i = bb->getExit(); i && i != bb->getEntry()->prev; i = i->prev) {
for (d = 0; i->defExists(d); ++d)
bb->liveSet.clr(i->getDef(d)->id);
for (s = 0; i->srcExists(s); ++s)
if (i->getSrc(s)->asLValue())
bb->liveSet.set(i->getSrc(s)->id);
}
for (i = bb->getPhi(); i && i->op == OP_PHI; i = i->next)
bb->liveSet.clr(i->getDef(0)->id);
 
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) {
INFO("BB:%i live set after propagation:\n", bb->getId());
bb->liveSet.print();
}
 
return true;
}
 
void
RegAlloc::BuildIntervalsPass::collectLiveValues(BasicBlock *bb)
{
BasicBlock *bbA = NULL, *bbB = NULL;
 
if (bb->cfg.outgoingCount()) {
// trickery to save a loop of OR'ing liveSets
// aliasing works fine with BitSet::setOr
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
if (ei.getType() == Graph::Edge::DUMMY)
continue;
if (bbA) {
bb->liveSet.setOr(&bbA->liveSet, &bbB->liveSet);
bbA = bb;
} else {
bbA = bbB;
}
bbB = BasicBlock::get(ei.getNode());
}
bb->liveSet.setOr(&bbB->liveSet, bbA ? &bbA->liveSet : NULL);
} else
if (bb->cfg.incidentCount()) {
bb->liveSet.fill(0);
}
}
 
bool
RegAlloc::BuildIntervalsPass::visit(BasicBlock *bb)
{
collectLiveValues(bb);
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "BuildIntervals(BB:%i)\n", bb->getId());
 
// go through out blocks and delete phi sources that do not originate from
// the current block from the live set
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
BasicBlock *out = BasicBlock::get(ei.getNode());
 
for (Instruction *i = out->getPhi(); i && i->op == OP_PHI; i = i->next) {
bb->liveSet.clr(i->getDef(0)->id);
 
for (int s = 0; i->srcExists(s); ++s) {
assert(i->src(s).getInsn());
if (i->getSrc(s)->getUniqueInsn()->bb == bb) // XXX: reachableBy ?
bb->liveSet.set(i->getSrc(s)->id);
else
bb->liveSet.clr(i->getSrc(s)->id);
}
}
}
 
// remaining live-outs are live until end
if (bb->getExit()) {
for (unsigned int j = 0; j < bb->liveSet.getSize(); ++j)
if (bb->liveSet.test(j))
addLiveRange(func->getLValue(j), bb, bb->getExit()->serial + 1);
}
 
for (Instruction *i = bb->getExit(); i && i->op != OP_PHI; i = i->prev) {
for (int d = 0; i->defExists(d); ++d) {
bb->liveSet.clr(i->getDef(d)->id);
if (i->getDef(d)->reg.data.id >= 0) // add hazard for fixed regs
i->getDef(d)->livei.extend(i->serial, i->serial);
}
 
for (int s = 0; i->srcExists(s); ++s) {
if (!i->getSrc(s)->asLValue())
continue;
if (!bb->liveSet.test(i->getSrc(s)->id)) {
bb->liveSet.set(i->getSrc(s)->id);
addLiveRange(i->getSrc(s), bb, i->serial);
}
}
}
 
if (bb == BasicBlock::get(func->cfg.getRoot())) {
for (std::deque<ValueDef>::iterator it = func->ins.begin();
it != func->ins.end(); ++it) {
if (it->get()->reg.data.id >= 0) // add hazard for fixed regs
it->get()->livei.extend(0, 1);
}
}
 
return true;
}
 
 
#define JOIN_MASK_PHI (1 << 0)
#define JOIN_MASK_UNION (1 << 1)
#define JOIN_MASK_MOV (1 << 2)
#define JOIN_MASK_TEX (1 << 3)
 
class GCRA
{
public:
GCRA(Function *, SpillCodeInserter&);
~GCRA();
 
bool allocateRegisters(ArrayList& insns);
 
void printNodeInfo() const;
 
private:
class RIG_Node : public Graph::Node
{
public:
RIG_Node();
 
void init(const RegisterSet&, LValue *);
 
void addInterference(RIG_Node *);
void addRegPreference(RIG_Node *);
 
inline LValue *getValue() const
{
return reinterpret_cast<LValue *>(data);
}
inline void setValue(LValue *lval) { data = lval; }
 
inline uint8_t getCompMask() const
{
return ((1 << colors) - 1) << (reg & 7);
}
 
static inline RIG_Node *get(const Graph::EdgeIterator& ei)
{
return static_cast<RIG_Node *>(ei.getNode());
}
 
public:
uint32_t degree;
uint16_t degreeLimit; // if deg < degLimit, node is trivially colourable
uint16_t colors;
 
DataFile f;
int32_t reg;
 
float weight;
 
// list pointers for simplify() phase
RIG_Node *next;
RIG_Node *prev;
 
// union of the live intervals of all coalesced values (we want to retain
// the separate intervals for testing interference of compound values)
Interval livei;
 
std::list<RIG_Node *> prefRegs;
};
 
private:
inline RIG_Node *getNode(const LValue *v) const { return &nodes[v->id]; }
 
void buildRIG(ArrayList&);
bool coalesce(ArrayList&);
bool doCoalesce(ArrayList&, unsigned int mask);
void calculateSpillWeights();
void simplify();
bool selectRegisters();
void cleanup(const bool success);
 
void simplifyEdge(RIG_Node *, RIG_Node *);
void simplifyNode(RIG_Node *);
 
bool coalesceValues(Value *, Value *, bool force);
void resolveSplitsAndMerges();
void makeCompound(Instruction *, bool isSplit);
 
inline void checkInterference(const RIG_Node *, Graph::EdgeIterator&);
 
inline void insertOrderedTail(std::list<RIG_Node *>&, RIG_Node *);
void checkList(std::list<RIG_Node *>&);
 
private:
std::stack<uint32_t> stack;
 
// list headers for simplify() phase
RIG_Node lo[2];
RIG_Node hi;
 
Graph RIG;
RIG_Node *nodes;
unsigned int nodeCount;
 
Function *func;
Program *prog;
 
static uint8_t relDegree[17][17];
 
RegisterSet regs;
 
// need to fixup register id for participants of OP_MERGE/SPLIT
std::list<Instruction *> merges;
std::list<Instruction *> splits;
 
SpillCodeInserter& spill;
std::list<ValuePair> mustSpill;
};
 
uint8_t GCRA::relDegree[17][17];
 
GCRA::RIG_Node::RIG_Node() : Node(NULL), next(this), prev(this)
{
colors = 0;
}
 
void
GCRA::printNodeInfo() const
{
for (unsigned int i = 0; i < nodeCount; ++i) {
if (!nodes[i].colors)
continue;
INFO("RIG_Node[%%%i]($[%u]%i): %u colors, weight %f, deg %u/%u\n X",
i,
nodes[i].f,nodes[i].reg,nodes[i].colors,
nodes[i].weight,
nodes[i].degree, nodes[i].degreeLimit);
 
for (Graph::EdgeIterator ei = nodes[i].outgoing(); !ei.end(); ei.next())
INFO(" %%%i", RIG_Node::get(ei)->getValue()->id);
for (Graph::EdgeIterator ei = nodes[i].incident(); !ei.end(); ei.next())
INFO(" %%%i", RIG_Node::get(ei)->getValue()->id);
INFO("\n");
}
}
 
void
GCRA::RIG_Node::init(const RegisterSet& regs, LValue *lval)
{
setValue(lval);
if (lval->reg.data.id >= 0)
lval->noSpill = lval->fixedReg = 1;
 
colors = regs.units(lval->reg.file, lval->reg.size);
f = lval->reg.file;
reg = -1;
if (lval->reg.data.id >= 0)
reg = regs.idToUnits(lval);
 
weight = std::numeric_limits<float>::infinity();
degree = 0;
degreeLimit = regs.getFileSize(f, lval->reg.size);
 
livei.insert(lval->livei);
}
 
bool
GCRA::coalesceValues(Value *dst, Value *src, bool force)
{
LValue *rep = dst->join->asLValue();
LValue *val = src->join->asLValue();
 
if (!force && val->reg.data.id >= 0) {
rep = src->join->asLValue();
val = dst->join->asLValue();
}
RIG_Node *nRep = &nodes[rep->id];
RIG_Node *nVal = &nodes[val->id];
 
if (src->reg.file != dst->reg.file) {
if (!force)
return false;
WARN("forced coalescing of values in different files !\n");
}
if (!force && dst->reg.size != src->reg.size)
return false;
 
if ((rep->reg.data.id >= 0) && (rep->reg.data.id != val->reg.data.id)) {
if (force) {
if (val->reg.data.id >= 0)
WARN("forced coalescing of values in different fixed regs !\n");
} else {
if (val->reg.data.id >= 0)
return false;
// make sure that there is no overlap with the fixed register of rep
for (ArrayList::Iterator it = func->allLValues.iterator();
!it.end(); it.next()) {
Value *reg = reinterpret_cast<Value *>(it.get())->asLValue();
assert(reg);
if (reg->interfers(rep) && reg->livei.overlaps(nVal->livei))
return false;
}
}
}
 
if (!force && nRep->livei.overlaps(nVal->livei))
return false;
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "joining %%%i($%i) <- %%%i\n",
rep->id, rep->reg.data.id, val->id);
 
// set join pointer of all values joined with val
for (Value::DefIterator def = val->defs.begin(); def != val->defs.end();
++def)
(*def)->get()->join = rep;
assert(rep->join == rep && val->join == rep);
 
// add val's definitions to rep and extend the live interval of its RIG node
rep->defs.insert(rep->defs.end(), val->defs.begin(), val->defs.end());
nRep->livei.unify(nVal->livei);
return true;
}
 
bool
GCRA::coalesce(ArrayList& insns)
{
bool ret = doCoalesce(insns, JOIN_MASK_PHI);
if (!ret)
return false;
switch (func->getProgram()->getTarget()->getChipset() & ~0xf) {
case 0x50:
case 0x80:
case 0x90:
case 0xa0:
ret = doCoalesce(insns, JOIN_MASK_UNION | JOIN_MASK_TEX);
break;
case 0xc0:
case 0xd0:
case 0xe0:
ret = doCoalesce(insns, JOIN_MASK_UNION);
break;
default:
break;
}
if (!ret)
return false;
return doCoalesce(insns, JOIN_MASK_MOV);
}
 
static inline uint8_t makeCompMask(int compSize, int base, int size)
{
uint8_t m = ((1 << size) - 1) << base;
 
switch (compSize) {
case 1:
return 0xff;
case 2:
m |= (m << 2);
return (m << 4) | m;
case 3:
case 4:
return (m << 4) | m;
default:
assert(compSize <= 8);
return m;
}
}
 
// Used when coalescing moves. The non-compound value will become one, e.g.:
// mov b32 $r0 $r2 / merge b64 $r0d { $r0 $r1 }
// split b64 { $r0 $r1 } $r0d / mov b64 $r0d f64 $r2d
static inline void copyCompound(Value *dst, Value *src)
{
LValue *ldst = dst->asLValue();
LValue *lsrc = src->asLValue();
 
if (ldst->compound && !lsrc->compound) {
LValue *swap = lsrc;
lsrc = ldst;
ldst = swap;
}
 
ldst->compound = lsrc->compound;
ldst->compMask = lsrc->compMask;
}
 
void
GCRA::makeCompound(Instruction *insn, bool split)
{
LValue *rep = (split ? insn->getSrc(0) : insn->getDef(0))->asLValue();
 
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) {
INFO("makeCompound(split = %i): ", split);
insn->print();
}
 
const unsigned int size = getNode(rep)->colors;
unsigned int base = 0;
 
if (!rep->compound)
rep->compMask = 0xff;
rep->compound = 1;
 
for (int c = 0; split ? insn->defExists(c) : insn->srcExists(c); ++c) {
LValue *val = (split ? insn->getDef(c) : insn->getSrc(c))->asLValue();
 
val->compound = 1;
if (!val->compMask)
val->compMask = 0xff;
val->compMask &= makeCompMask(size, base, getNode(val)->colors);
assert(val->compMask);
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "compound: %%%i:%02x <- %%%i:%02x\n",
rep->id, rep->compMask, val->id, val->compMask);
 
base += getNode(val)->colors;
}
assert(base == size);
}
 
bool
GCRA::doCoalesce(ArrayList& insns, unsigned int mask)
{
int c, n;
 
for (n = 0; n < insns.getSize(); ++n) {
Instruction *i;
Instruction *insn = reinterpret_cast<Instruction *>(insns.get(n));
 
switch (insn->op) {
case OP_PHI:
if (!(mask & JOIN_MASK_PHI))
break;
for (c = 0; insn->srcExists(c); ++c)
if (!coalesceValues(insn->getDef(0), insn->getSrc(c), false)) {
// this is bad
ERROR("failed to coalesce phi operands\n");
return false;
}
break;
case OP_UNION:
case OP_MERGE:
if (!(mask & JOIN_MASK_UNION))
break;
for (c = 0; insn->srcExists(c); ++c)
coalesceValues(insn->getDef(0), insn->getSrc(c), true);
if (insn->op == OP_MERGE) {
merges.push_back(insn);
if (insn->srcExists(1))
makeCompound(insn, false);
}
break;
case OP_SPLIT:
if (!(mask & JOIN_MASK_UNION))
break;
splits.push_back(insn);
for (c = 0; insn->defExists(c); ++c)
coalesceValues(insn->getSrc(0), insn->getDef(c), true);
makeCompound(insn, true);
break;
case OP_MOV:
if (!(mask & JOIN_MASK_MOV))
break;
i = NULL;
if (!insn->getDef(0)->uses.empty())
i = insn->getDef(0)->uses.front()->getInsn();
// if this is a contraint-move there will only be a single use
if (i && i->op == OP_MERGE) // do we really still need this ?
break;
i = insn->getSrc(0)->getUniqueInsn();
if (i && !i->constrainedDefs()) {
if (coalesceValues(insn->getDef(0), insn->getSrc(0), false))
copyCompound(insn->getSrc(0), insn->getDef(0));
}
break;
case OP_TEX:
case OP_TXB:
case OP_TXL:
case OP_TXF:
case OP_TXQ:
case OP_TXD:
case OP_TXG:
case OP_TEXCSAA:
if (!(mask & JOIN_MASK_TEX))
break;
for (c = 0; insn->srcExists(c) && c != insn->predSrc; ++c)
coalesceValues(insn->getDef(c), insn->getSrc(c), true);
break;
default:
break;
}
}
return true;
}
 
void
GCRA::RIG_Node::addInterference(RIG_Node *node)
{
this->degree += relDegree[node->colors][colors];
node->degree += relDegree[colors][node->colors];
 
this->attach(node, Graph::Edge::CROSS);
}
 
void
GCRA::RIG_Node::addRegPreference(RIG_Node *node)
{
prefRegs.push_back(node);
}
 
GCRA::GCRA(Function *fn, SpillCodeInserter& spill) :
func(fn),
regs(fn->getProgram()->getTarget()),
spill(spill)
{
prog = func->getProgram();
 
// initialize relative degrees array - i takes away from j
for (int i = 1; i <= 16; ++i)
for (int j = 1; j <= 16; ++j)
relDegree[i][j] = j * ((i + j - 1) / j);
}
 
GCRA::~GCRA()
{
if (nodes)
delete[] nodes;
}
 
void
GCRA::checkList(std::list<RIG_Node *>& lst)
{
GCRA::RIG_Node *prev = NULL;
 
for (std::list<RIG_Node *>::iterator it = lst.begin();
it != lst.end();
++it) {
assert((*it)->getValue()->join == (*it)->getValue());
if (prev)
assert(prev->livei.begin() <= (*it)->livei.begin());
prev = *it;
}
}
 
void
GCRA::insertOrderedTail(std::list<RIG_Node *>& list, RIG_Node *node)
{
if (node->livei.isEmpty())
return;
// only the intervals of joined values don't necessarily arrive in order
std::list<RIG_Node *>::iterator prev, it;
for (it = list.end(); it != list.begin(); it = prev) {
prev = it;
--prev;
if ((*prev)->livei.begin() <= node->livei.begin())
break;
}
list.insert(it, node);
}
 
void
GCRA::buildRIG(ArrayList& insns)
{
std::list<RIG_Node *> values, active;
 
for (std::deque<ValueDef>::iterator it = func->ins.begin();
it != func->ins.end(); ++it)
insertOrderedTail(values, getNode(it->get()->asLValue()));
 
for (int i = 0; i < insns.getSize(); ++i) {
Instruction *insn = reinterpret_cast<Instruction *>(insns.get(i));
for (int d = 0; insn->defExists(d); ++d)
if (insn->getDef(d)->rep() == insn->getDef(d))
insertOrderedTail(values, getNode(insn->getDef(d)->asLValue()));
}
checkList(values);
 
while (!values.empty()) {
RIG_Node *cur = values.front();
 
for (std::list<RIG_Node *>::iterator it = active.begin();
it != active.end();) {
RIG_Node *node = *it;
 
if (node->livei.end() <= cur->livei.begin()) {
it = active.erase(it);
} else {
if (node->f == cur->f && node->livei.overlaps(cur->livei))
cur->addInterference(node);
++it;
}
}
values.pop_front();
active.push_back(cur);
}
}
 
void
GCRA::calculateSpillWeights()
{
for (unsigned int i = 0; i < nodeCount; ++i) {
RIG_Node *const n = &nodes[i];
if (!nodes[i].colors || nodes[i].livei.isEmpty())
continue;
if (nodes[i].reg >= 0) {
// update max reg
regs.occupy(n->f, n->reg, n->colors);
continue;
}
LValue *val = nodes[i].getValue();
 
if (!val->noSpill) {
int rc = 0;
for (Value::DefIterator it = val->defs.begin();
it != val->defs.end();
++it)
rc += (*it)->get()->refCount();
 
nodes[i].weight =
(float)rc * (float)rc / (float)nodes[i].livei.extent();
}
 
if (nodes[i].degree < nodes[i].degreeLimit) {
int l = 0;
if (val->reg.size > 4)
l = 1;
DLLIST_ADDHEAD(&lo[l], &nodes[i]);
} else {
DLLIST_ADDHEAD(&hi, &nodes[i]);
}
}
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC)
printNodeInfo();
}
 
void
GCRA::simplifyEdge(RIG_Node *a, RIG_Node *b)
{
bool move = b->degree >= b->degreeLimit;
 
INFO_DBG(prog->dbgFlags, REG_ALLOC,
"edge: (%%%i, deg %u/%u) >-< (%%%i, deg %u/%u)\n",
a->getValue()->id, a->degree, a->degreeLimit,
b->getValue()->id, b->degree, b->degreeLimit);
 
b->degree -= relDegree[a->colors][b->colors];
 
move = move && b->degree < b->degreeLimit;
if (move && !DLLIST_EMPTY(b)) {
int l = (b->getValue()->reg.size > 4) ? 1 : 0;
DLLIST_DEL(b);
DLLIST_ADDTAIL(&lo[l], b);
}
}
 
void
GCRA::simplifyNode(RIG_Node *node)
{
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next())
simplifyEdge(node, RIG_Node::get(ei));
 
for (Graph::EdgeIterator ei = node->incident(); !ei.end(); ei.next())
simplifyEdge(node, RIG_Node::get(ei));
 
DLLIST_DEL(node);
stack.push(node->getValue()->id);
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "SIMPLIFY: pushed %%%i%s\n",
node->getValue()->id,
(node->degree < node->degreeLimit) ? "" : "(spill)");
}
 
void
GCRA::simplify()
{
for (;;) {
if (!DLLIST_EMPTY(&lo[0])) {
do {
simplifyNode(lo[0].next);
} while (!DLLIST_EMPTY(&lo[0]));
} else
if (!DLLIST_EMPTY(&lo[1])) {
simplifyNode(lo[1].next);
} else
if (!DLLIST_EMPTY(&hi)) {
RIG_Node *best = hi.next;
float bestScore = best->weight / (float)best->degree;
// spill candidate
for (RIG_Node *it = best->next; it != &hi; it = it->next) {
float score = it->weight / (float)it->degree;
if (score < bestScore) {
best = it;
bestScore = score;
}
}
if (isinf(bestScore)) {
ERROR("no viable spill candidates left\n");
break;
}
simplifyNode(best);
} else {
break;
}
}
}
 
void
GCRA::checkInterference(const RIG_Node *node, Graph::EdgeIterator& ei)
{
const RIG_Node *intf = RIG_Node::get(ei);
 
if (intf->reg < 0)
return;
const LValue *vA = node->getValue();
const LValue *vB = intf->getValue();
 
const uint8_t intfMask = ((1 << intf->colors) - 1) << (intf->reg & 7);
 
if (vA->compound | vB->compound) {
// NOTE: this only works for >aligned< register tuples !
for (Value::DefCIterator D = vA->defs.begin(); D != vA->defs.end(); ++D) {
for (Value::DefCIterator d = vB->defs.begin(); d != vB->defs.end(); ++d) {
const LValue *vD = (*D)->get()->asLValue();
const LValue *vd = (*d)->get()->asLValue();
 
if (!vD->livei.overlaps(vd->livei)) {
INFO_DBG(prog->dbgFlags, REG_ALLOC, "(%%%i) X (%%%i): no overlap\n",
vD->id, vd->id);
continue;
}
 
uint8_t mask = vD->compound ? vD->compMask : ~0;
if (vd->compound) {
assert(vB->compound);
mask &= vd->compMask & vB->compMask;
} else {
mask &= intfMask;
}
 
INFO_DBG(prog->dbgFlags, REG_ALLOC,
"(%%%i)%02x X (%%%i)%02x & %02x: $r%i.%02x\n",
vD->id,
vD->compound ? vD->compMask : 0xff,
vd->id,
vd->compound ? vd->compMask : intfMask,
vB->compMask, intf->reg & ~7, mask);
if (mask)
regs.occupyMask(node->f, intf->reg & ~7, mask);
}
}
} else {
INFO_DBG(prog->dbgFlags, REG_ALLOC,
"(%%%i) X (%%%i): $r%i + %u\n",
vA->id, vB->id, intf->reg, intf->colors);
regs.occupy(node->f, intf->reg, intf->colors);
}
}
 
bool
GCRA::selectRegisters()
{
INFO_DBG(prog->dbgFlags, REG_ALLOC, "\nSELECT phase\n");
 
while (!stack.empty()) {
RIG_Node *node = &nodes[stack.top()];
stack.pop();
 
regs.reset(node->f);
 
INFO_DBG(prog->dbgFlags, REG_ALLOC, "\nNODE[%%%i, %u colors]\n",
node->getValue()->id, node->colors);
 
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next())
checkInterference(node, ei);
for (Graph::EdgeIterator ei = node->incident(); !ei.end(); ei.next())
checkInterference(node, ei);
 
if (!node->prefRegs.empty()) {
for (std::list<RIG_Node *>::const_iterator it = node->prefRegs.begin();
it != node->prefRegs.end();
++it) {
if ((*it)->reg >= 0 &&
regs.testOccupy(node->f, (*it)->reg, node->colors)) {
node->reg = (*it)->reg;
break;
}
}
}
if (node->reg >= 0)
continue;
LValue *lval = node->getValue();
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC)
regs.print();
bool ret = regs.assign(node->reg, node->f, node->colors);
if (ret) {
INFO_DBG(prog->dbgFlags, REG_ALLOC, "assigned reg %i\n", node->reg);
lval->compMask = node->getCompMask();
} else {
INFO_DBG(prog->dbgFlags, REG_ALLOC, "must spill: %%%i (size %u)\n",
lval->id, lval->reg.size);
Symbol *slot = NULL;
if (lval->reg.file == FILE_GPR)
slot = spill.assignSlot(node->livei, lval->reg.size);
mustSpill.push_back(ValuePair(lval, slot));
}
}
if (!mustSpill.empty())
return false;
for (unsigned int i = 0; i < nodeCount; ++i) {
LValue *lval = nodes[i].getValue();
if (nodes[i].reg >= 0 && nodes[i].colors > 0)
lval->reg.data.id =
regs.unitsToId(nodes[i].f, nodes[i].reg, lval->reg.size);
}
return true;
}
 
bool
GCRA::allocateRegisters(ArrayList& insns)
{
bool ret;
 
INFO_DBG(prog->dbgFlags, REG_ALLOC,
"allocateRegisters to %u instructions\n", insns.getSize());
 
nodeCount = func->allLValues.getSize();
nodes = new RIG_Node[nodeCount];
if (!nodes)
return false;
for (unsigned int i = 0; i < nodeCount; ++i) {
LValue *lval = reinterpret_cast<LValue *>(func->allLValues.get(i));
if (lval) {
nodes[i].init(regs, lval);
RIG.insert(&nodes[i]);
}
}
 
// coalesce first, we use only 1 RIG node for a group of joined values
ret = coalesce(insns);
if (!ret)
goto out;
 
if (func->getProgram()->dbgFlags & NV50_IR_DEBUG_REG_ALLOC)
func->printLiveIntervals();
 
buildRIG(insns);
calculateSpillWeights();
simplify();
 
ret = selectRegisters();
if (!ret) {
INFO_DBG(prog->dbgFlags, REG_ALLOC,
"selectRegisters failed, inserting spill code ...\n");
regs.reset(FILE_GPR, true);
spill.run(mustSpill);
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC)
func->print();
} else {
prog->maxGPR = std::max(prog->maxGPR, regs.getMaxAssigned(FILE_GPR));
}
 
out:
cleanup(ret);
return ret;
}
 
void
GCRA::cleanup(const bool success)
{
mustSpill.clear();
 
for (ArrayList::Iterator it = func->allLValues.iterator();
!it.end(); it.next()) {
LValue *lval = reinterpret_cast<LValue *>(it.get());
 
lval->livei.clear();
 
lval->compound = 0;
lval->compMask = 0;
 
if (lval->join == lval)
continue;
 
if (success) {
lval->reg.data.id = lval->join->reg.data.id;
} else {
for (Value::DefIterator d = lval->defs.begin(); d != lval->defs.end();
++d)
lval->join->defs.remove(*d);
lval->join = lval;
}
}
 
if (success)
resolveSplitsAndMerges();
splits.clear(); // avoid duplicate entries on next coalesce pass
merges.clear();
 
delete[] nodes;
nodes = NULL;
}
 
Symbol *
SpillCodeInserter::assignSlot(const Interval &livei, const unsigned int size)
{
SpillSlot slot;
int32_t offsetBase = stackSize;
int32_t offset;
std::list<SpillSlot>::iterator pos = slots.end(), it = slots.begin();
 
if (offsetBase % size)
offsetBase += size - (offsetBase % size);
 
slot.sym = NULL;
 
for (offset = offsetBase; offset < stackSize; offset += size) {
const int32_t entryEnd = offset + size;
while (it != slots.end() && it->offset < offset)
++it;
if (it == slots.end()) // no slots left
break;
std::list<SpillSlot>::iterator bgn = it;
 
while (it != slots.end() && it->offset < entryEnd) {
it->occup.print();
if (it->occup.overlaps(livei))
break;
++it;
}
if (it == slots.end() || it->offset >= entryEnd) {
// fits
for (; bgn != slots.end() && bgn->offset < entryEnd; ++bgn) {
bgn->occup.insert(livei);
if (bgn->size() == size)
slot.sym = bgn->sym;
}
break;
}
}
if (!slot.sym) {
stackSize = offset + size;
slot.offset = offset;
slot.sym = new_Symbol(func->getProgram(), FILE_MEMORY_LOCAL);
if (!func->stackPtr)
offset += func->tlsBase;
slot.sym->setAddress(NULL, offset);
slot.sym->reg.size = size;
slots.insert(pos, slot)->occup.insert(livei);
}
return slot.sym;
}
 
void
SpillCodeInserter::spill(Instruction *defi, Value *slot, LValue *lval)
{
const DataType ty = typeOfSize(slot->reg.size);
 
Instruction *st;
if (slot->reg.file == FILE_MEMORY_LOCAL) {
st = new_Instruction(func, OP_STORE, ty);
st->setSrc(0, slot);
st->setSrc(1, lval);
lval->noSpill = 1;
} else {
st = new_Instruction(func, OP_CVT, ty);
st->setDef(0, slot);
st->setSrc(0, lval);
}
defi->bb->insertAfter(defi, st);
}
 
LValue *
SpillCodeInserter::unspill(Instruction *usei, LValue *lval, Value *slot)
{
const DataType ty = typeOfSize(slot->reg.size);
 
lval = cloneShallow(func, lval);
 
Instruction *ld;
if (slot->reg.file == FILE_MEMORY_LOCAL) {
lval->noSpill = 1;
ld = new_Instruction(func, OP_LOAD, ty);
} else {
ld = new_Instruction(func, OP_CVT, ty);
}
ld->setDef(0, lval);
ld->setSrc(0, slot);
 
usei->bb->insertBefore(usei, ld);
return lval;
}
 
bool
SpillCodeInserter::run(const std::list<ValuePair>& lst)
{
for (std::list<ValuePair>::const_iterator it = lst.begin(); it != lst.end();
++it) {
LValue *lval = it->first->asLValue();
Symbol *mem = it->second ? it->second->asSym() : NULL;
 
for (Value::DefIterator d = lval->defs.begin(); d != lval->defs.end();
++d) {
Value *slot = mem ?
static_cast<Value *>(mem) : new_LValue(func, FILE_GPR);
Value *tmp = NULL;
Instruction *last = NULL;
 
LValue *dval = (*d)->get()->asLValue();
Instruction *defi = (*d)->getInsn();
 
// handle uses first or they'll contain the spill stores
while (!dval->uses.empty()) {
ValueRef *u = dval->uses.front();
Instruction *usei = u->getInsn();
assert(usei);
if (usei->op == OP_PHI) {
tmp = (slot->reg.file == FILE_MEMORY_LOCAL) ? NULL : slot;
last = NULL;
} else
if (!last || usei != last->next) { // TODO: sort uses
tmp = unspill(usei, dval, slot);
last = usei;
}
u->set(tmp);
}
 
assert(defi);
if (defi->op == OP_PHI) {
d = lval->defs.erase(d);
--d;
if (slot->reg.file == FILE_MEMORY_LOCAL)
delete_Instruction(func->getProgram(), defi);
else
defi->setDef(0, slot);
} else {
spill(defi, slot, dval);
}
}
 
}
 
// TODO: We're not trying to reuse old slots in a potential next iteration.
// We have to update the slots' livei intervals to be able to do that.
stackBase = stackSize;
slots.clear();
return true;
}
 
bool
RegAlloc::exec()
{
for (IteratorRef it = prog->calls.iteratorDFS(false);
!it->end(); it->next()) {
func = Function::get(reinterpret_cast<Graph::Node *>(it->get()));
 
func->tlsBase = prog->tlsSize;
if (!execFunc())
return false;
prog->tlsSize += func->tlsSize;
}
return true;
}
 
bool
RegAlloc::execFunc()
{
InsertConstraintsPass insertConstr;
PhiMovesPass insertPhiMoves;
ArgumentMovesPass insertArgMoves;
BuildIntervalsPass buildIntervals;
SpillCodeInserter insertSpills(func);
 
GCRA gcra(func, insertSpills);
 
unsigned int i, retries;
bool ret;
 
if (!func->ins.empty()) {
// Insert a nop at the entry so inputs only used by the first instruction
// don't count as having an empty live range.
Instruction *nop = new_Instruction(func, OP_NOP, TYPE_NONE);
BasicBlock::get(func->cfg.getRoot())->insertHead(nop);
}
 
ret = insertConstr.exec(func);
if (!ret)
goto out;
 
ret = insertPhiMoves.run(func);
if (!ret)
goto out;
 
ret = insertArgMoves.run(func);
if (!ret)
goto out;
 
// TODO: need to fix up spill slot usage ranges to support > 1 retry
for (retries = 0; retries < 3; ++retries) {
if (retries && (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC))
INFO("Retry: %i\n", retries);
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC)
func->print();
 
// spilling to registers may add live ranges, need to rebuild everything
ret = true;
for (sequence = func->cfg.nextSequence(), i = 0;
ret && i <= func->loopNestingBound;
sequence = func->cfg.nextSequence(), ++i)
ret = buildLiveSets(BasicBlock::get(func->cfg.getRoot()));
if (!ret)
break;
func->orderInstructions(this->insns);
 
ret = buildIntervals.run(func);
if (!ret)
break;
ret = gcra.allocateRegisters(insns);
if (ret)
break; // success
}
INFO_DBG(prog->dbgFlags, REG_ALLOC, "RegAlloc done: %i\n", ret);
 
func->tlsSize = insertSpills.getStackSize();
out:
return ret;
}
 
// TODO: check if modifying Instruction::join here breaks anything
void
GCRA::resolveSplitsAndMerges()
{
for (std::list<Instruction *>::iterator it = splits.begin();
it != splits.end();
++it) {
Instruction *split = *it;
unsigned int reg = regs.idToBytes(split->getSrc(0));
for (int d = 0; split->defExists(d); ++d) {
Value *v = split->getDef(d);
v->reg.data.id = regs.bytesToId(v, reg);
v->join = v;
reg += v->reg.size;
}
}
splits.clear();
 
for (std::list<Instruction *>::iterator it = merges.begin();
it != merges.end();
++it) {
Instruction *merge = *it;
unsigned int reg = regs.idToBytes(merge->getDef(0));
for (int s = 0; merge->srcExists(s); ++s) {
Value *v = merge->getSrc(s);
v->reg.data.id = regs.bytesToId(v, reg);
v->join = v;
reg += v->reg.size;
}
}
merges.clear();
}
 
bool Program::registerAllocation()
{
RegAlloc ra(this);
return ra.exec();
}
 
bool
RegAlloc::InsertConstraintsPass::exec(Function *ir)
{
constrList.clear();
 
bool ret = run(ir, true, true);
if (ret)
ret = insertConstraintMoves();
return ret;
}
 
// TODO: make part of texture insn
void
RegAlloc::InsertConstraintsPass::textureMask(TexInstruction *tex)
{
Value *def[4];
int c, k, d;
uint8_t mask = 0;
 
for (d = 0, k = 0, c = 0; c < 4; ++c) {
if (!(tex->tex.mask & (1 << c)))
continue;
if (tex->getDef(k)->refCount()) {
mask |= 1 << c;
def[d++] = tex->getDef(k);
}
++k;
}
tex->tex.mask = mask;
 
for (c = 0; c < d; ++c)
tex->setDef(c, def[c]);
for (; c < 4; ++c)
tex->setDef(c, NULL);
}
 
bool
RegAlloc::InsertConstraintsPass::detectConflict(Instruction *cst, int s)
{
Value *v = cst->getSrc(s);
 
// current register allocation can't handle it if a value participates in
// multiple constraints
for (Value::UseIterator it = v->uses.begin(); it != v->uses.end(); ++it) {
if (cst != (*it)->getInsn())
return true;
}
 
// can start at s + 1 because detectConflict is called on all sources
for (int c = s + 1; cst->srcExists(c); ++c)
if (v == cst->getSrc(c))
return true;
 
Instruction *defi = v->getInsn();
 
return (!defi || defi->constrainedDefs());
}
 
void
RegAlloc::InsertConstraintsPass::addConstraint(Instruction *i, int s, int n)
{
Instruction *cst;
int d;
 
// first, look for an existing identical constraint op
for (std::list<Instruction *>::iterator it = constrList.begin();
it != constrList.end();
++it) {
cst = (*it);
if (!i->bb->dominatedBy(cst->bb))
break;
for (d = 0; d < n; ++d)
if (cst->getSrc(d) != i->getSrc(d + s))
break;
if (d >= n) {
for (d = 0; d < n; ++d, ++s)
i->setSrc(s, cst->getDef(d));
return;
}
}
cst = new_Instruction(func, OP_CONSTRAINT, i->dType);
 
for (d = 0; d < n; ++s, ++d) {
cst->setDef(d, new_LValue(func, FILE_GPR));
cst->setSrc(d, i->getSrc(s));
i->setSrc(s, cst->getDef(d));
}
i->bb->insertBefore(i, cst);
 
constrList.push_back(cst);
}
 
// Add a dummy use of the pointer source of >= 8 byte loads after the load
// to prevent it from being assigned a register which overlapping the load's
// destination, which would produce random corruptions.
void
RegAlloc::InsertConstraintsPass::addHazard(Instruction *i, const ValueRef *src)
{
Instruction *hzd = new_Instruction(func, OP_NOP, TYPE_NONE);
hzd->setSrc(0, src->get());
i->bb->insertAfter(i, hzd);
 
}
 
// b32 { %r0 %r1 %r2 %r3 } -> b128 %r0q
void
RegAlloc::InsertConstraintsPass::condenseDefs(Instruction *insn)
{
uint8_t size = 0;
int n;
for (n = 0; insn->defExists(n) && insn->def(n).getFile() == FILE_GPR; ++n)
size += insn->getDef(n)->reg.size;
if (n < 2)
return;
LValue *lval = new_LValue(func, FILE_GPR);
lval->reg.size = size;
 
Instruction *split = new_Instruction(func, OP_SPLIT, typeOfSize(size));
split->setSrc(0, lval);
for (int d = 0; d < n; ++d) {
split->setDef(d, insn->getDef(d));
insn->setDef(d, NULL);
}
insn->setDef(0, lval);
 
for (int k = 1, d = n; insn->defExists(d); ++d, ++k) {
insn->setDef(k, insn->getDef(d));
insn->setDef(d, NULL);
}
// carry over predicate if any (mainly for OP_UNION uses)
split->setPredicate(insn->cc, insn->getPredicate());
 
insn->bb->insertAfter(insn, split);
constrList.push_back(split);
}
void
RegAlloc::InsertConstraintsPass::condenseSrcs(Instruction *insn,
const int a, const int b)
{
uint8_t size = 0;
if (a >= b)
return;
for (int s = a; s <= b; ++s)
size += insn->getSrc(s)->reg.size;
if (!size)
return;
LValue *lval = new_LValue(func, FILE_GPR);
lval->reg.size = size;
 
Value *save[3];
insn->takeExtraSources(0, save);
 
Instruction *merge = new_Instruction(func, OP_MERGE, typeOfSize(size));
merge->setDef(0, lval);
for (int s = a, i = 0; s <= b; ++s, ++i) {
merge->setSrc(i, insn->getSrc(s));
insn->setSrc(s, NULL);
}
insn->setSrc(a, lval);
 
for (int k = a + 1, s = b + 1; insn->srcExists(s); ++s, ++k) {
insn->setSrc(k, insn->getSrc(s));
insn->setSrc(s, NULL);
}
insn->bb->insertBefore(insn, merge);
 
insn->putExtraSources(0, save);
 
constrList.push_back(merge);
}
 
void
RegAlloc::InsertConstraintsPass::texConstraintNVE0(TexInstruction *tex)
{
if (isTextureOp(tex->op))
textureMask(tex);
condenseDefs(tex);
 
if (tex->op == OP_SUSTB || tex->op == OP_SUSTP) {
condenseSrcs(tex, 3, (3 + typeSizeof(tex->dType) / 4) - 1);
} else
if (isTextureOp(tex->op)) {
int n = tex->srcCount(0xff, true);
if (n > 4) {
condenseSrcs(tex, 0, 3);
if (n > 5) // NOTE: first call modified positions already
condenseSrcs(tex, 4 - (4 - 1), n - 1 - (4 - 1));
} else
if (n > 1) {
condenseSrcs(tex, 0, n - 1);
}
}
}
 
void
RegAlloc::InsertConstraintsPass::texConstraintNVC0(TexInstruction *tex)
{
int n, s;
 
textureMask(tex);
 
if (tex->op == OP_TXQ) {
s = tex->srcCount(0xff);
n = 0;
} else {
s = tex->tex.target.getArgCount();
if (!tex->tex.target.isArray() &&
(tex->tex.rIndirectSrc >= 0 || tex->tex.sIndirectSrc >= 0))
++s;
if (tex->op == OP_TXD && tex->tex.useOffsets)
++s;
n = tex->srcCount(0xff) - s;
assert(n <= 4);
}
 
if (s > 1)
condenseSrcs(tex, 0, s - 1);
if (n > 1) // NOTE: first call modified positions already
condenseSrcs(tex, 1, n);
 
condenseDefs(tex);
}
 
void
RegAlloc::InsertConstraintsPass::texConstraintNV50(TexInstruction *tex)
{
Value *pred = tex->getPredicate();
if (pred)
tex->setPredicate(tex->cc, NULL);
 
textureMask(tex);
 
assert(tex->defExists(0) && tex->srcExists(0));
// make src and def count match
int c;
for (c = 0; tex->srcExists(c) || tex->defExists(c); ++c) {
if (!tex->srcExists(c))
tex->setSrc(c, new_LValue(func, tex->getSrc(0)->asLValue()));
if (!tex->defExists(c))
tex->setDef(c, new_LValue(func, tex->getDef(0)->asLValue()));
}
if (pred)
tex->setPredicate(tex->cc, pred);
condenseDefs(tex);
condenseSrcs(tex, 0, c - 1);
}
 
// Insert constraint markers for instructions whose multiple sources must be
// located in consecutive registers.
bool
RegAlloc::InsertConstraintsPass::visit(BasicBlock *bb)
{
TexInstruction *tex;
Instruction *next;
int s, size;
 
targ = bb->getProgram()->getTarget();
 
for (Instruction *i = bb->getEntry(); i; i = next) {
next = i->next;
 
if ((tex = i->asTex())) {
switch (targ->getChipset() & ~0xf) {
case 0x50:
case 0x80:
case 0x90:
case 0xa0:
texConstraintNV50(tex);
break;
case 0xc0:
case 0xd0:
texConstraintNVC0(tex);
break;
case 0xe0:
case NVISA_GK110_CHIPSET:
texConstraintNVE0(tex);
break;
default:
break;
}
} else
if (i->op == OP_EXPORT || i->op == OP_STORE) {
for (size = typeSizeof(i->dType), s = 1; size > 0; ++s) {
assert(i->srcExists(s));
size -= i->getSrc(s)->reg.size;
}
condenseSrcs(i, 1, s - 1);
} else
if (i->op == OP_LOAD || i->op == OP_VFETCH) {
condenseDefs(i);
if (i->src(0).isIndirect(0) && typeSizeof(i->dType) >= 8)
addHazard(i, i->src(0).getIndirect(0));
} else
if (i->op == OP_UNION ||
i->op == OP_MERGE ||
i->op == OP_SPLIT) {
constrList.push_back(i);
}
}
return true;
}
 
// Insert extra moves so that, if multiple register constraints on a value are
// in conflict, these conflicts can be resolved.
bool
RegAlloc::InsertConstraintsPass::insertConstraintMoves()
{
for (std::list<Instruction *>::iterator it = constrList.begin();
it != constrList.end();
++it) {
Instruction *cst = *it;
Instruction *mov;
 
if (cst->op == OP_SPLIT && 0) {
// spilling splits is annoying, just make sure they're separate
for (int d = 0; cst->defExists(d); ++d) {
if (!cst->getDef(d)->refCount())
continue;
LValue *lval = new_LValue(func, cst->def(d).getFile());
const uint8_t size = cst->def(d).getSize();
lval->reg.size = size;
 
mov = new_Instruction(func, OP_MOV, typeOfSize(size));
mov->setSrc(0, lval);
mov->setDef(0, cst->getDef(d));
cst->setDef(d, mov->getSrc(0));
cst->bb->insertAfter(cst, mov);
 
cst->getSrc(0)->asLValue()->noSpill = 1;
mov->getSrc(0)->asLValue()->noSpill = 1;
}
} else
if (cst->op == OP_MERGE || cst->op == OP_UNION) {
for (int s = 0; cst->srcExists(s); ++s) {
const uint8_t size = cst->src(s).getSize();
 
if (!cst->getSrc(s)->defs.size()) {
mov = new_Instruction(func, OP_NOP, typeOfSize(size));
mov->setDef(0, cst->getSrc(s));
cst->bb->insertBefore(cst, mov);
continue;
}
assert(cst->getSrc(s)->defs.size() == 1); // still SSA
 
Instruction *defi = cst->getSrc(s)->defs.front()->getInsn();
// catch some cases where don't really need MOVs
if (cst->getSrc(s)->refCount() == 1 && !defi->constrainedDefs())
continue;
 
LValue *lval = new_LValue(func, cst->src(s).getFile());
lval->reg.size = size;
 
mov = new_Instruction(func, OP_MOV, typeOfSize(size));
mov->setDef(0, lval);
mov->setSrc(0, cst->getSrc(s));
cst->setSrc(s, mov->getDef(0));
cst->bb->insertBefore(cst, mov);
 
cst->getDef(0)->asLValue()->noSpill = 1; // doesn't help
 
if (cst->op == OP_UNION)
mov->setPredicate(defi->cc, defi->getPredicate());
}
}
}
 
return true;
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_ssa.cpp
0,0 → 1,552
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir.h"
#include "nv50_ir_target.h"
 
namespace nv50_ir {
 
// Converts nv50 IR generated from TGSI to SSA form.
 
// DominatorTree implements an algorithm for finding immediate dominators,
// as described by T. Lengauer & R. Tarjan.
class DominatorTree : public Graph
{
public:
DominatorTree(Graph *cfg);
~DominatorTree() { }
bool dominates(BasicBlock *, BasicBlock *);
 
void findDominanceFrontiers();
 
private:
void build();
void buildDFS(Node *);
 
void squash(int);
inline void link(int, int);
inline int eval(int);
 
void debugPrint();
 
Graph *cfg;
 
Node **vert;
int *data;
const int count;
 
#define SEMI(i) (data[(i) + 0 * count])
#define ANCESTOR(i) (data[(i) + 1 * count])
#define PARENT(i) (data[(i) + 2 * count])
#define LABEL(i) (data[(i) + 3 * count])
#define DOM(i) (data[(i) + 4 * count])
};
 
void DominatorTree::debugPrint()
{
for (int i = 0; i < count; ++i) {
INFO("SEMI(%i) = %i\n", i, SEMI(i));
INFO("ANCESTOR(%i) = %i\n", i, ANCESTOR(i));
INFO("PARENT(%i) = %i\n", i, PARENT(i));
INFO("LABEL(%i) = %i\n", i, LABEL(i));
INFO("DOM(%i) = %i\n", i, DOM(i));
}
}
 
DominatorTree::DominatorTree(Graph *cfgraph) : cfg(cfgraph),
count(cfg->getSize())
{
int i = 0;
 
vert = new Node * [count];
data = new int[5 * count];
 
for (IteratorRef it = cfg->iteratorDFS(true); !it->end(); it->next(), ++i) {
vert[i] = reinterpret_cast<Node *>(it->get());
vert[i]->tag = i;
LABEL(i) = i;
SEMI(i) = ANCESTOR(i) = -1;
}
 
build();
 
delete[] vert;
delete[] data;
}
 
void DominatorTree::buildDFS(Graph::Node *node)
{
SEMI(node->tag) = node->tag;
 
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) {
if (SEMI(ei.getNode()->tag) < 0) {
buildDFS(ei.getNode());
PARENT(ei.getNode()->tag) = node->tag;
}
}
}
 
void DominatorTree::squash(int v)
{
if (ANCESTOR(ANCESTOR(v)) >= 0) {
squash(ANCESTOR(v));
 
if (SEMI(LABEL(ANCESTOR(v))) < SEMI(LABEL(v)))
LABEL(v) = LABEL(ANCESTOR(v));
ANCESTOR(v) = ANCESTOR(ANCESTOR(v));
}
}
 
int DominatorTree::eval(int v)
{
if (ANCESTOR(v) < 0)
return v;
squash(v);
return LABEL(v);
}
 
void DominatorTree::link(int v, int w)
{
ANCESTOR(w) = v;
}
 
void DominatorTree::build()
{
DLList *bucket = new DLList[count];
Node *nv, *nw;
int p, u, v, w;
 
buildDFS(cfg->getRoot());
 
for (w = count - 1; w >= 1; --w) {
nw = vert[w];
assert(nw->tag == w);
for (Graph::EdgeIterator ei = nw->incident(); !ei.end(); ei.next()) {
nv = ei.getNode();
v = nv->tag;
u = eval(v);
if (SEMI(u) < SEMI(w))
SEMI(w) = SEMI(u);
}
p = PARENT(w);
bucket[SEMI(w)].insert(nw);
link(p, w);
 
for (DLList::Iterator it = bucket[p].iterator(); !it.end(); it.erase()) {
v = reinterpret_cast<Node *>(it.get())->tag;
u = eval(v);
DOM(v) = (SEMI(u) < SEMI(v)) ? u : p;
}
}
for (w = 1; w < count; ++w) {
if (DOM(w) != SEMI(w))
DOM(w) = DOM(DOM(w));
}
DOM(0) = 0;
 
insert(&BasicBlock::get(cfg->getRoot())->dom);
do {
p = 0;
for (v = 1; v < count; ++v) {
nw = &BasicBlock::get(vert[DOM(v)])->dom;;
nv = &BasicBlock::get(vert[v])->dom;
if (nw->getGraph() && !nv->getGraph()) {
++p;
nw->attach(nv, Graph::Edge::TREE);
}
}
} while (p);
 
delete[] bucket;
}
 
#undef SEMI
#undef ANCESTOR
#undef PARENT
#undef LABEL
#undef DOM
 
void DominatorTree::findDominanceFrontiers()
{
BasicBlock *bb;
 
for (IteratorRef dtIt = iteratorDFS(false); !dtIt->end(); dtIt->next()) {
EdgeIterator succIt, chldIt;
 
bb = BasicBlock::get(reinterpret_cast<Node *>(dtIt->get()));
bb->getDF().clear();
 
for (succIt = bb->cfg.outgoing(); !succIt.end(); succIt.next()) {
BasicBlock *dfLocal = BasicBlock::get(succIt.getNode());
if (dfLocal->idom() != bb)
bb->getDF().insert(dfLocal);
}
 
for (chldIt = bb->dom.outgoing(); !chldIt.end(); chldIt.next()) {
BasicBlock *cb = BasicBlock::get(chldIt.getNode());
 
DLList::Iterator dfIt = cb->getDF().iterator();
for (; !dfIt.end(); dfIt.next()) {
BasicBlock *dfUp = BasicBlock::get(dfIt);
if (dfUp->idom() != bb)
bb->getDF().insert(dfUp);
}
}
}
}
 
// liveIn(bb) = usedBeforeAssigned(bb) U (liveOut(bb) - assigned(bb))
void
Function::buildLiveSetsPreSSA(BasicBlock *bb, const int seq)
{
Function *f = bb->getFunction();
BitSet usedBeforeAssigned(allLValues.getSize(), true);
BitSet assigned(allLValues.getSize(), true);
 
bb->liveSet.allocate(allLValues.getSize(), false);
 
int n = 0;
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
BasicBlock *out = BasicBlock::get(ei.getNode());
if (out == bb)
continue;
if (out->cfg.visit(seq))
buildLiveSetsPreSSA(out, seq);
if (!n++)
bb->liveSet = out->liveSet;
else
bb->liveSet |= out->liveSet;
}
if (!n && !bb->liveSet.marker)
bb->liveSet.fill(0);
bb->liveSet.marker = true;
 
for (Instruction *i = bb->getEntry(); i; i = i->next) {
for (int s = 0; i->srcExists(s); ++s)
if (i->getSrc(s)->asLValue() && !assigned.test(i->getSrc(s)->id))
usedBeforeAssigned.set(i->getSrc(s)->id);
for (int d = 0; i->defExists(d); ++d)
assigned.set(i->getDef(d)->id);
}
 
if (bb == BasicBlock::get(f->cfgExit)) {
for (std::deque<ValueRef>::iterator it = f->outs.begin();
it != f->outs.end(); ++it) {
if (!assigned.test(it->get()->id))
usedBeforeAssigned.set(it->get()->id);
}
}
 
bb->liveSet.andNot(assigned);
bb->liveSet |= usedBeforeAssigned;
}
 
void
Function::buildDefSetsPreSSA(BasicBlock *bb, const int seq)
{
bb->defSet.allocate(allLValues.getSize(), !bb->liveSet.marker);
bb->liveSet.marker = true;
 
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
BasicBlock *in = BasicBlock::get(ei.getNode());
 
if (in->cfg.visit(seq))
buildDefSetsPreSSA(in, seq);
 
bb->defSet |= in->defSet;
}
 
for (Instruction *i = bb->getEntry(); i; i = i->next) {
for (int d = 0; i->defExists(d); ++d)
bb->defSet.set(i->getDef(d)->id);
}
}
 
class RenamePass
{
public:
RenamePass(Function *);
~RenamePass();
 
bool run();
void search(BasicBlock *);
 
inline LValue *getStackTop(Value *);
 
LValue *mkUndefined(Value *);
 
private:
Stack *stack;
Function *func;
Program *prog;
};
 
bool
Program::convertToSSA()
{
for (ArrayList::Iterator fi = allFuncs.iterator(); !fi.end(); fi.next()) {
Function *fn = reinterpret_cast<Function *>(fi.get());
if (!fn->convertToSSA())
return false;
}
return true;
}
 
// XXX: add edge from entry to exit ?
 
// Efficiently Computing Static Single Assignment Form and
// the Control Dependence Graph,
// R. Cytron, J. Ferrante, B. K. Rosen, M. N. Wegman, F. K. Zadeck
bool
Function::convertToSSA()
{
// 0. calculate live in variables (for pruned SSA)
buildLiveSets();
 
// 1. create the dominator tree
domTree = new DominatorTree(&cfg);
reinterpret_cast<DominatorTree *>(domTree)->findDominanceFrontiers();
 
// 2. insert PHI functions
DLList workList;
LValue *lval;
BasicBlock *bb;
int var;
int iterCount = 0;
int *hasAlready = new int[allBBlocks.getSize() * 2];
int *work = &hasAlready[allBBlocks.getSize()];
 
memset(hasAlready, 0, allBBlocks.getSize() * 2 * sizeof(int));
 
// for each variable
for (var = 0; var < allLValues.getSize(); ++var) {
if (!allLValues.get(var))
continue;
lval = reinterpret_cast<Value *>(allLValues.get(var))->asLValue();
if (!lval || lval->defs.empty())
continue;
++iterCount;
 
// TODO: don't add phi functions for values that aren't used outside
// the BB they're defined in
 
// gather blocks with assignments to lval in workList
for (Value::DefIterator d = lval->defs.begin();
d != lval->defs.end(); ++d) {
bb = ((*d)->getInsn() ? (*d)->getInsn()->bb : NULL);
if (!bb)
continue; // instruction likely been removed but not XXX deleted
 
if (work[bb->getId()] == iterCount)
continue;
work[bb->getId()] = iterCount;
workList.insert(bb);
}
 
// for each block in workList, insert a phi for lval in the block's
// dominance frontier (if we haven't already done so)
for (DLList::Iterator wI = workList.iterator(); !wI.end(); wI.erase()) {
bb = BasicBlock::get(wI);
 
DLList::Iterator dfIter = bb->getDF().iterator();
for (; !dfIter.end(); dfIter.next()) {
Instruction *phi;
BasicBlock *dfBB = BasicBlock::get(dfIter);
 
if (hasAlready[dfBB->getId()] >= iterCount)
continue;
hasAlready[dfBB->getId()] = iterCount;
 
// pruned SSA: don't need a phi if the value is not live-in
if (!dfBB->liveSet.test(lval->id))
continue;
 
phi = new_Instruction(this, OP_PHI, typeOfSize(lval->reg.size));
dfBB->insertTail(phi);
 
phi->setDef(0, lval);
for (int s = 0; s < dfBB->cfg.incidentCount(); ++s)
phi->setSrc(s, lval);
 
if (work[dfBB->getId()] < iterCount) {
work[dfBB->getId()] = iterCount;
wI.insert(dfBB);
}
}
}
}
delete[] hasAlready;
 
RenamePass rename(this);
return rename.run();
}
 
RenamePass::RenamePass(Function *fn) : func(fn), prog(fn->getProgram())
{
stack = new Stack[func->allLValues.getSize()];
}
 
RenamePass::~RenamePass()
{
if (stack)
delete[] stack;
}
 
LValue *
RenamePass::getStackTop(Value *val)
{
if (!stack[val->id].getSize())
return 0;
return reinterpret_cast<LValue *>(stack[val->id].peek().u.p);
}
 
LValue *
RenamePass::mkUndefined(Value *val)
{
LValue *lval = val->asLValue();
assert(lval);
LValue *ud = new_LValue(func, lval);
Instruction *nop = new_Instruction(func, OP_NOP, typeOfSize(lval->reg.size));
nop->setDef(0, ud);
BasicBlock::get(func->cfg.getRoot())->insertHead(nop);
return ud;
}
 
bool RenamePass::run()
{
if (!stack)
return false;
search(BasicBlock::get(func->domTree->getRoot()));
 
return true;
}
 
// Go through BBs in dominance order, create new values for each definition,
// and replace all sources with their current new values.
//
// NOTE: The values generated for function inputs/outputs have no connection
// to their corresponding outputs/inputs in other functions. Only allocation
// of physical registers will establish this connection.
//
void RenamePass::search(BasicBlock *bb)
{
LValue *lval, *ssa;
int d, s;
const Target *targ = prog->getTarget();
 
// Put current definitions for function inputs values on the stack.
// They can be used before any redefinitions are pushed.
if (bb == BasicBlock::get(func->cfg.getRoot())) {
for (std::deque<ValueDef>::iterator it = func->ins.begin();
it != func->ins.end(); ++it) {
lval = it->get()->asLValue();
assert(lval);
 
ssa = new_LValue(func, targ->nativeFile(lval->reg.file));
ssa->reg.size = lval->reg.size;
ssa->reg.data.id = lval->reg.data.id;
 
it->setSSA(ssa);
stack[lval->id].push(ssa);
}
}
 
for (Instruction *stmt = bb->getFirst(); stmt; stmt = stmt->next) {
// PHI sources get definitions from the passes through the incident BBs,
// so skip them here.
if (stmt->op != OP_PHI) {
for (s = 0; stmt->srcExists(s); ++s) {
lval = stmt->getSrc(s)->asLValue();
if (!lval)
continue;
// Values on the stack created in previously visited blocks, and
// function inputs, will be valid because they dominate this one.
lval = getStackTop(lval);
if (!lval)
lval = mkUndefined(stmt->getSrc(s));
stmt->setSrc(s, lval);
}
}
for (d = 0; stmt->defExists(d); ++d) {
lval = stmt->def(d).get()->asLValue();
assert(lval);
stmt->def(d).setSSA(
new_LValue(func, targ->nativeFile(lval->reg.file)));
stmt->def(d).get()->reg.size = lval->reg.size;
stmt->def(d).get()->reg.data.id = lval->reg.data.id;
stack[lval->id].push(stmt->def(d).get());
}
}
 
// Update sources of PHI ops corresponding to this BB in outgoing BBs.
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
Instruction *phi;
int p = 0;
BasicBlock *sb = BasicBlock::get(ei.getNode());
 
// which predecessor of sb is bb ?
for (Graph::EdgeIterator ei = sb->cfg.incident(); !ei.end(); ei.next()) {
if (ei.getNode() == &bb->cfg)
break;
++p;
}
assert(p < sb->cfg.incidentCount());
 
for (phi = sb->getPhi(); phi && phi->op == OP_PHI; phi = phi->next) {
lval = getStackTop(phi->getSrc(p));
if (!lval)
lval = mkUndefined(phi->getSrc(p));
phi->setSrc(p, lval);
}
}
 
// Visit the BBs we dominate.
for (Graph::EdgeIterator ei = bb->dom.outgoing(); !ei.end(); ei.next())
search(BasicBlock::get(ei.getNode()));
 
// Update function outputs to the last definitions of their pre-SSA values.
// I hope they're unique, i.e. that we get PHIs for all of them ...
if (bb == BasicBlock::get(func->cfgExit)) {
for (std::deque<ValueRef>::iterator it = func->outs.begin();
it != func->outs.end(); ++it) {
lval = it->get()->asLValue();
if (!lval)
continue;
lval = getStackTop(lval);
if (!lval)
lval = mkUndefined(it->get());
it->set(lval);
}
}
 
// Pop the values we created in this block from the stack because we will
// return to blocks that we do not dominate.
for (Instruction *stmt = bb->getFirst(); stmt; stmt = stmt->next) {
if (stmt->op == OP_NOP)
continue;
for (d = 0; stmt->defExists(d); ++d)
stack[stmt->def(d).preSSA()->id].pop();
}
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp
0,0 → 1,469
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50/codegen/nv50_ir.h"
#include "nv50/codegen/nv50_ir_target.h"
 
namespace nv50_ir {
 
const uint8_t Target::operationSrcNr[OP_LAST + 1] =
{
0, 0, // NOP, PHI
0, 0, 0, 0, // UNION, SPLIT, MERGE, CONSTRAINT
1, 1, 2, // MOV, LOAD, STORE
2, 2, 2, 2, 2, 3, 3, 3, // ADD, SUB, MUL, DIV, MOD, MAD, FMA, SAD
1, 1, 1, // ABS, NEG, NOT
2, 2, 2, 2, 2, // AND, OR, XOR, SHL, SHR
2, 2, 1, // MAX, MIN, SAT
1, 1, 1, 1, // CEIL, FLOOR, TRUNC, CVT
3, 3, 3, 2, 3, 3, // SET_AND,OR,XOR, SET, SELP, SLCT
1, 1, 1, 1, 1, 1, // RCP, RSQ, LG2, SIN, COS, EX2
1, 1, 1, 1, 1, 2, // EXP, LOG, PRESIN, PREEX2, SQRT, POW
0, 0, 0, 0, 0, // BRA, CALL, RET, CONT, BREAK,
0, 0, 0, // PRERET,CONT,BREAK
0, 0, 0, 0, 0, 0, // BRKPT, JOINAT, JOIN, DISCARD, EXIT, MEMBAR
1, 1, 2, 1, 2, // VFETCH, PFETCH, EXPORT, LINTERP, PINTERP
1, 1, // EMIT, RESTART
1, 1, 1, // TEX, TXB, TXL,
1, 1, 1, 1, 1, 2, // TXF, TXQ, TXD, TXG, TEXCSAA, TEXPREP
1, 1, 2, 2, 2, 2, 2, // SULDB, SULDP, SUSTB, SUSTP, SUREDB, SUREDP, SULEA
3, 3, 3, 3, // SUBFM, SUCLAMP, SUEAU, MADSP
0, // TEXBAR
1, 1, // DFDX, DFDY
1, 2, 2, 0, 0, // RDSV, WRSV, QUADOP, QUADON, QUADPOP
2, 3, 2, 3, // POPCNT, INSBF, EXTBF, PERMT
2, 2, // ATOM, BAR
2, 2, 2, 2, 3, 2, // VADD, VAVG, VMIN, VMAX, VSAD, VSET,
2, 2, 2, 1, // VSHR, VSHL, VSEL, CCTL
0
};
 
const OpClass Target::operationClass[OP_LAST + 1] =
{
// NOP; PHI; UNION, SPLIT, MERGE, CONSTRAINT
OPCLASS_OTHER,
OPCLASS_PSEUDO,
OPCLASS_PSEUDO, OPCLASS_PSEUDO, OPCLASS_PSEUDO, OPCLASS_PSEUDO,
// MOV; LOAD; STORE
OPCLASS_MOVE,
OPCLASS_LOAD,
OPCLASS_STORE,
// ADD, SUB, MUL; DIV, MOD; MAD, FMA, SAD
OPCLASS_ARITH, OPCLASS_ARITH, OPCLASS_ARITH,
OPCLASS_ARITH, OPCLASS_ARITH,
OPCLASS_ARITH, OPCLASS_ARITH, OPCLASS_ARITH,
// ABS, NEG; NOT, AND, OR, XOR; SHL, SHR
OPCLASS_CONVERT, OPCLASS_CONVERT,
OPCLASS_LOGIC, OPCLASS_LOGIC, OPCLASS_LOGIC, OPCLASS_LOGIC,
OPCLASS_SHIFT, OPCLASS_SHIFT,
// MAX, MIN
OPCLASS_COMPARE, OPCLASS_COMPARE,
// SAT, CEIL, FLOOR, TRUNC; CVT
OPCLASS_CONVERT, OPCLASS_CONVERT, OPCLASS_CONVERT, OPCLASS_CONVERT,
OPCLASS_CONVERT,
// SET(AND,OR,XOR); SELP, SLCT
OPCLASS_COMPARE, OPCLASS_COMPARE, OPCLASS_COMPARE, OPCLASS_COMPARE,
OPCLASS_COMPARE, OPCLASS_COMPARE,
// RCP, RSQ, LG2, SIN, COS; EX2, EXP, LOG, PRESIN, PREEX2; SQRT, POW
OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU,
OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU,
OPCLASS_SFU, OPCLASS_SFU,
// BRA, CALL, RET; CONT, BREAK, PRE(RET,CONT,BREAK); BRKPT, JOINAT, JOIN
OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW,
OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW,
OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW,
// DISCARD, EXIT
OPCLASS_FLOW, OPCLASS_FLOW,
// MEMBAR
OPCLASS_CONTROL,
// VFETCH, PFETCH, EXPORT
OPCLASS_LOAD, OPCLASS_OTHER, OPCLASS_STORE,
// LINTERP, PINTERP
OPCLASS_SFU, OPCLASS_SFU,
// EMIT, RESTART
OPCLASS_CONTROL, OPCLASS_CONTROL,
// TEX, TXB, TXL, TXF; TXQ, TXD, TXG, TEXCSAA; TEXPREP
OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE,
OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE,
OPCLASS_TEXTURE,
// SULDB, SULDP, SUSTB, SUSTP; SUREDB, SUREDP, SULEA
OPCLASS_SURFACE, OPCLASS_SURFACE, OPCLASS_ATOMIC, OPCLASS_SURFACE,
OPCLASS_SURFACE, OPCLASS_SURFACE, OPCLASS_SURFACE,
// SUBFM, SUCLAMP, SUEAU, MADSP
OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_ARITH,
// TEXBAR
OPCLASS_OTHER,
// DFDX, DFDY, RDSV, WRSV; QUADOP, QUADON, QUADPOP
OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER,
OPCLASS_OTHER, OPCLASS_CONTROL, OPCLASS_CONTROL,
// POPCNT, INSBF, EXTBF, PERMT
OPCLASS_BITFIELD, OPCLASS_BITFIELD, OPCLASS_BITFIELD, OPCLASS_BITFIELD,
// ATOM, BAR
OPCLASS_ATOMIC, OPCLASS_CONTROL,
// VADD, VAVG, VMIN, VMAX
OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR,
// VSAD, VSET, VSHR, VSHL
OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR,
// VSEL, CCTL
OPCLASS_VECTOR, OPCLASS_CONTROL,
OPCLASS_PSEUDO // LAST
};
 
 
extern Target *getTargetNVC0(unsigned int chipset);
extern Target *getTargetNV50(unsigned int chipset);
 
Target *Target::create(unsigned int chipset)
{
switch (chipset & 0xf0) {
case 0xc0:
case 0xd0:
case 0xe0:
case NVISA_GK110_CHIPSET:
return getTargetNVC0(chipset);
case 0x50:
case 0x80:
case 0x90:
case 0xa0:
return getTargetNV50(chipset);
default:
ERROR("unsupported target: NV%x\n", chipset);
return 0;
}
}
 
void Target::destroy(Target *targ)
{
delete targ;
}
 
CodeEmitter::CodeEmitter(const Target *target) : targ(target)
{
}
 
void
CodeEmitter::setCodeLocation(void *ptr, uint32_t size)
{
code = reinterpret_cast<uint32_t *>(ptr);
codeSize = 0;
codeSizeLimit = size;
}
 
void
CodeEmitter::printBinary() const
{
uint32_t *bin = code - codeSize / 4;
INFO("program binary (%u bytes)", codeSize);
for (unsigned int pos = 0; pos < codeSize / 4; ++pos) {
if ((pos % 8) == 0)
INFO("\n");
INFO("%08x ", bin[pos]);
}
INFO("\n");
}
 
static inline uint32_t sizeToBundlesNVE4(uint32_t size)
{
return (size + 55) / 56;
}
 
void
CodeEmitter::prepareEmission(Program *prog)
{
for (ArrayList::Iterator fi = prog->allFuncs.iterator();
!fi.end(); fi.next()) {
Function *func = reinterpret_cast<Function *>(fi.get());
func->binPos = prog->binSize;
prepareEmission(func);
 
// adjust sizes & positions for schedulding info:
if (prog->getTarget()->hasSWSched) {
uint32_t adjPos = func->binPos;
BasicBlock *bb = NULL;
for (int i = 0; i < func->bbCount; ++i) {
bb = func->bbArray[i];
int32_t adjSize = bb->binSize;
if (adjPos % 64) {
adjSize -= 64 - adjPos % 64;
if (adjSize < 0)
adjSize = 0;
}
adjSize = bb->binSize + sizeToBundlesNVE4(adjSize) * 8;
bb->binPos = adjPos;
bb->binSize = adjSize;
adjPos += adjSize;
}
if (bb)
func->binSize = adjPos - func->binPos;
}
 
prog->binSize += func->binSize;
}
}
 
void
CodeEmitter::prepareEmission(Function *func)
{
func->bbCount = 0;
func->bbArray = new BasicBlock * [func->cfg.getSize()];
 
BasicBlock::get(func->cfg.getRoot())->binPos = func->binPos;
 
for (IteratorRef it = func->cfg.iteratorCFG(); !it->end(); it->next())
prepareEmission(BasicBlock::get(*it));
}
 
void
CodeEmitter::prepareEmission(BasicBlock *bb)
{
Instruction *i, *next;
Function *func = bb->getFunction();
int j;
unsigned int nShort;
 
for (j = func->bbCount - 1; j >= 0 && !func->bbArray[j]->binSize; --j);
 
for (; j >= 0; --j) {
BasicBlock *in = func->bbArray[j];
Instruction *exit = in->getExit();
 
if (exit && exit->op == OP_BRA && exit->asFlow()->target.bb == bb) {
in->binSize -= 8;
func->binSize -= 8;
 
for (++j; j < func->bbCount; ++j)
func->bbArray[j]->binPos -= 8;
 
in->remove(exit);
}
bb->binPos = in->binPos + in->binSize;
if (in->binSize) // no more no-op branches to bb
break;
}
func->bbArray[func->bbCount++] = bb;
 
if (!bb->getExit())
return;
 
// determine encoding size, try to group short instructions
nShort = 0;
for (i = bb->getEntry(); i; i = next) {
next = i->next;
 
if (i->op == OP_MEMBAR && !targ->isOpSupported(OP_MEMBAR, TYPE_NONE)) {
bb->remove(i);
continue;
}
 
i->encSize = getMinEncodingSize(i);
if (next && i->encSize < 8)
++nShort;
else
if ((nShort & 1) && next && getMinEncodingSize(next) == 4) {
if (i->isCommutationLegal(i->next)) {
bb->permuteAdjacent(i, next);
next->encSize = 4;
next = i;
i = i->prev;
++nShort;
} else
if (i->isCommutationLegal(i->prev) && next->next) {
bb->permuteAdjacent(i->prev, i);
next->encSize = 4;
next = next->next;
bb->binSize += 4;
++nShort;
} else {
i->encSize = 8;
i->prev->encSize = 8;
bb->binSize += 4;
nShort = 0;
}
} else {
i->encSize = 8;
if (nShort & 1) {
i->prev->encSize = 8;
bb->binSize += 4;
}
nShort = 0;
}
bb->binSize += i->encSize;
}
 
if (bb->getExit()->encSize == 4) {
assert(nShort);
bb->getExit()->encSize = 8;
bb->binSize += 4;
 
if ((bb->getExit()->prev->encSize == 4) && !(nShort & 1)) {
bb->binSize += 8;
bb->getExit()->prev->encSize = 8;
}
}
assert(!bb->getEntry() || (bb->getExit() && bb->getExit()->encSize == 8));
 
func->binSize += bb->binSize;
}
 
void
Program::emitSymbolTable(struct nv50_ir_prog_info *info)
{
unsigned int n = 0, nMax = allFuncs.getSize();
 
info->bin.syms =
(struct nv50_ir_prog_symbol *)MALLOC(nMax * sizeof(*info->bin.syms));
 
for (ArrayList::Iterator fi = allFuncs.iterator();
!fi.end();
fi.next(), ++n) {
Function *f = (Function *)fi.get();
assert(n < nMax);
 
info->bin.syms[n].label = f->getLabel();
info->bin.syms[n].offset = f->binPos;
}
 
info->bin.numSyms = n;
}
 
bool
Program::emitBinary(struct nv50_ir_prog_info *info)
{
CodeEmitter *emit = target->getCodeEmitter(progType);
 
emit->prepareEmission(this);
 
if (dbgFlags & NV50_IR_DEBUG_BASIC)
this->print();
 
if (!binSize) {
code = NULL;
return false;
}
code = reinterpret_cast<uint32_t *>(MALLOC(binSize));
if (!code)
return false;
emit->setCodeLocation(code, binSize);
 
for (ArrayList::Iterator fi = allFuncs.iterator(); !fi.end(); fi.next()) {
Function *fn = reinterpret_cast<Function *>(fi.get());
 
assert(emit->getCodeSize() == fn->binPos);
 
for (int b = 0; b < fn->bbCount; ++b)
for (Instruction *i = fn->bbArray[b]->getEntry(); i; i = i->next)
emit->emitInstruction(i);
}
info->bin.relocData = emit->getRelocInfo();
 
emitSymbolTable(info);
 
// the nvc0 driver will print the binary iself together with the header
if ((dbgFlags & NV50_IR_DEBUG_BASIC) && getTarget()->getChipset() < 0xc0)
emit->printBinary();
 
delete emit;
return true;
}
 
#define RELOC_ALLOC_INCREMENT 8
 
bool
CodeEmitter::addReloc(RelocEntry::Type ty, int w, uint32_t data, uint32_t m,
int s)
{
unsigned int n = relocInfo ? relocInfo->count : 0;
 
if (!(n % RELOC_ALLOC_INCREMENT)) {
size_t size = sizeof(RelocInfo) + n * sizeof(RelocEntry);
relocInfo = reinterpret_cast<RelocInfo *>(
REALLOC(relocInfo, n ? size : 0,
size + RELOC_ALLOC_INCREMENT * sizeof(RelocEntry)));
if (!relocInfo)
return false;
if (n == 0)
memset(relocInfo, 0, sizeof(RelocInfo));
}
++relocInfo->count;
 
relocInfo->entry[n].data = data;
relocInfo->entry[n].mask = m;
relocInfo->entry[n].offset = codeSize + w * 4;
relocInfo->entry[n].bitPos = s;
relocInfo->entry[n].type = ty;
 
return true;
}
 
void
RelocEntry::apply(uint32_t *binary, const RelocInfo *info) const
{
uint32_t value = 0;
 
switch (type) {
case TYPE_CODE: value = info->codePos; break;
case TYPE_BUILTIN: value = info->libPos; break;
case TYPE_DATA: value = info->dataPos; break;
default:
assert(0);
break;
}
value += data;
value = (bitPos < 0) ? (value >> -bitPos) : (value << bitPos);
 
binary[offset / 4] &= ~mask;
binary[offset / 4] |= value & mask;
}
 
} // namespace nv50_ir
 
 
#include "nv50/codegen/nv50_ir_driver.h"
 
extern "C" {
 
void
nv50_ir_relocate_code(void *relocData, uint32_t *code,
uint32_t codePos,
uint32_t libPos,
uint32_t dataPos)
{
nv50_ir::RelocInfo *info = reinterpret_cast<nv50_ir::RelocInfo *>(relocData);
 
info->codePos = codePos;
info->libPos = libPos;
info->dataPos = dataPos;
 
for (unsigned int i = 0; i < info->count; ++i)
info->entry[i].apply(code, info);
}
 
void
nv50_ir_get_target_library(uint32_t chipset,
const uint32_t **code, uint32_t *size)
{
nv50_ir::Target *targ = nv50_ir::Target::create(chipset);
targ->getBuiltinCode(code, size);
nv50_ir::Target::destroy(targ);
}
 
}
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target.h
0,0 → 1,235
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_TARGET_H__
#define __NV50_IR_TARGET_H__
 
#include "nv50_ir.h"
 
namespace nv50_ir {
 
struct RelocInfo;
 
struct RelocEntry
{
enum Type
{
TYPE_CODE,
TYPE_BUILTIN,
TYPE_DATA
};
 
uint32_t data;
uint32_t mask;
uint32_t offset;
int8_t bitPos;
Type type;
 
inline void apply(uint32_t *binary, const RelocInfo *info) const;
};
 
struct RelocInfo
{
uint32_t codePos;
uint32_t libPos;
uint32_t dataPos;
 
uint32_t count;
 
RelocEntry entry[0];
};
 
class CodeEmitter
{
public:
CodeEmitter(const Target *);
virtual ~CodeEmitter() { }
 
// returns whether the instruction was encodable and written
virtual bool emitInstruction(Instruction *) = 0;
 
virtual uint32_t getMinEncodingSize(const Instruction *) const = 0;
 
void setCodeLocation(void *, uint32_t size);
inline void *getCodeLocation() const { return code; }
inline uint32_t getCodeSize() const { return codeSize; }
 
bool addReloc(RelocEntry::Type, int w, uint32_t data, uint32_t m,
int s);
 
inline void *getRelocInfo() const { return relocInfo; }
 
void prepareEmission(Program *);
virtual void prepareEmission(Function *);
virtual void prepareEmission(BasicBlock *);
 
void printBinary() const;
 
protected:
const Target *targ;
 
uint32_t *code;
uint32_t codeSize;
uint32_t codeSizeLimit;
 
RelocInfo *relocInfo;
};
 
 
enum OpClass
{
OPCLASS_MOVE = 0,
OPCLASS_LOAD = 1,
OPCLASS_STORE = 2,
OPCLASS_ARITH = 3,
OPCLASS_SHIFT = 4,
OPCLASS_SFU = 5,
OPCLASS_LOGIC = 6,
OPCLASS_COMPARE = 7,
OPCLASS_CONVERT = 8,
OPCLASS_ATOMIC = 9,
OPCLASS_TEXTURE = 10,
OPCLASS_SURFACE = 11,
OPCLASS_FLOW = 12,
OPCLASS_PSEUDO = 14,
OPCLASS_VECTOR = 15,
OPCLASS_BITFIELD = 16,
OPCLASS_CONTROL = 17,
OPCLASS_OTHER = 18
};
 
class Target
{
public:
Target(bool j, bool s) : joinAnterior(j), hasSWSched(s) { }
virtual ~Target() { }
 
static Target *create(uint32_t chipset);
static void destroy(Target *);
 
// 0x50 and 0x84 to 0xaf for nv50
// 0xc0 to 0xdf for nvc0
inline uint32_t getChipset() const { return chipset; }
 
virtual CodeEmitter *getCodeEmitter(Program::Type) = 0;
 
// Drivers should upload this so we can use it from all programs.
// The address chosen is supplied to the relocation routine.
virtual void getBuiltinCode(const uint32_t **code, uint32_t *size) const = 0;
 
virtual void parseDriverInfo(const struct nv50_ir_prog_info *info) { }
 
virtual bool runLegalizePass(Program *, CGStage stage) const = 0;
 
public:
struct OpInfo
{
OpInfo *variants;
operation op;
uint16_t srcTypes;
uint16_t dstTypes;
uint32_t immdBits;
uint8_t srcNr;
uint8_t srcMods[3];
uint8_t dstMods;
uint8_t srcFiles[3];
uint8_t dstFiles;
unsigned int minEncSize : 4;
unsigned int vector : 1;
unsigned int predicate : 1;
unsigned int commutative : 1;
unsigned int pseudo : 1;
unsigned int flow : 1;
unsigned int hasDest : 1;
unsigned int terminator : 1;
};
 
inline const OpInfo& getOpInfo(const Instruction *) const;
inline const OpInfo& getOpInfo(const operation) const;
 
inline DataFile nativeFile(DataFile f) const;
 
virtual bool insnCanLoad(const Instruction *insn, int s,
const Instruction *ld) const = 0;
virtual bool isOpSupported(operation, DataType) const = 0;
virtual bool isAccessSupported(DataFile, DataType) const = 0;
virtual bool isModSupported(const Instruction *,
int s, Modifier) const = 0;
virtual bool isSatSupported(const Instruction *) const = 0;
virtual bool isPostMultiplySupported(operation op, float f,
int& e) const { return false; }
virtual bool mayPredicate(const Instruction *,
const Value *) const = 0;
 
// whether @insn can be issued together with @next (order matters)
virtual bool canDualIssue(const Instruction *insn,
const Instruction *next) const { return false; }
virtual int getLatency(const Instruction *) const { return 1; }
virtual int getThroughput(const Instruction *) const { return 1; }
 
virtual unsigned int getFileSize(DataFile) const = 0;
virtual unsigned int getFileUnit(DataFile) const = 0;
 
virtual uint32_t getSVAddress(DataFile, const Symbol *) const = 0;
 
public:
const bool joinAnterior; // true if join is executed before the op
const bool hasSWSched; // true if code should provide scheduling data
 
static const uint8_t operationSrcNr[OP_LAST + 1];
static const OpClass operationClass[OP_LAST + 1];
 
static inline uint8_t getOpSrcNr(operation op)
{
return operationSrcNr[op];
}
static inline OpClass getOpClass(operation op)
{
return operationClass[op];
}
 
protected:
uint32_t chipset;
 
DataFile nativeFileMap[DATA_FILE_COUNT];
 
OpInfo opInfo[OP_LAST + 1];
};
 
const Target::OpInfo& Target::getOpInfo(const Instruction *insn) const
{
return opInfo[MIN2(insn->op, OP_LAST)];
}
 
const Target::OpInfo& Target::getOpInfo(const operation op) const
{
return opInfo[op];
}
 
inline DataFile Target::nativeFile(DataFile f) const
{
return nativeFileMap[f];
}
 
} // namespace nv50_ir
 
#endif // __NV50_IR_TARGET_H__
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.cpp
0,0 → 1,552
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir_target_nv50.h"
 
namespace nv50_ir {
 
Target *getTargetNV50(unsigned int chipset)
{
return new TargetNV50(chipset);
}
 
TargetNV50::TargetNV50(unsigned int card) : Target(true, false)
{
chipset = card;
 
wposMask = 0;
for (unsigned int i = 0; i <= SV_LAST; ++i)
sysvalLocation[i] = ~0;
 
initOpInfo();
}
 
#if 0
// BULTINS / LIBRARY FUNCTIONS:
 
// TODO
static const uint32_t nvc0_builtin_code[] =
{
};
 
static const uint16_t nvc0_builtin_offsets[NV50_BUILTIN_COUNT] =
{
};
#endif
 
void
TargetNV50::getBuiltinCode(const uint32_t **code, uint32_t *size) const
{
*code = NULL;
*size = 0;
}
 
uint32_t
TargetNV50::getBuiltinOffset(int builtin) const
{
return 0;
}
 
struct opProperties
{
operation op;
unsigned int mNeg : 4;
unsigned int mAbs : 4;
unsigned int mNot : 4;
unsigned int mSat : 4;
unsigned int fConst : 3;
unsigned int fShared : 3;
unsigned int fAttrib : 3;
unsigned int fImm : 3;
};
 
static const struct opProperties _initProps[] =
{
// neg abs not sat c[] s[], a[], imm
{ OP_ADD, 0x3, 0x0, 0x0, 0x8, 0x2, 0x1, 0x1, 0x2 },
{ OP_SUB, 0x3, 0x0, 0x0, 0x0, 0x2, 0x1, 0x1, 0x2 },
{ OP_MUL, 0x3, 0x0, 0x0, 0x0, 0x2, 0x1, 0x1, 0x2 },
{ OP_MAX, 0x3, 0x3, 0x0, 0x0, 0x2, 0x1, 0x1, 0x0 },
{ OP_MIN, 0x3, 0x3, 0x0, 0x0, 0x2, 0x1, 0x1, 0x0 },
{ OP_MAD, 0x7, 0x0, 0x0, 0x0, 0x6, 0x1, 0x1, 0x0 }, // special constraint
{ OP_ABS, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0 },
{ OP_NEG, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0 },
{ OP_CVT, 0x1, 0x1, 0x0, 0x8, 0x0, 0x1, 0x1, 0x0 },
{ OP_AND, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x2 },
{ OP_OR, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x2 },
{ OP_XOR, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x2 },
{ OP_SHL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 },
{ OP_SHR, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 },
{ OP_SET, 0x3, 0x3, 0x0, 0x0, 0x2, 0x1, 0x1, 0x0 },
{ OP_PREEX2, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
{ OP_PRESIN, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
{ OP_LG2, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
{ OP_RCP, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
{ OP_RSQ, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
{ OP_DFDX, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
{ OP_DFDY, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 },
};
 
void TargetNV50::initOpInfo()
{
unsigned int i, j;
 
static const uint32_t commutative[(OP_LAST + 31) / 32] =
{
// ADD,MAD,MUL,AND,OR,XOR,MAX,MIN
0x0670ca00, 0x0000003f, 0x00000000, 0x00000000
};
static const uint32_t shortForm[(OP_LAST + 31) / 32] =
{
// MOV,ADD,SUB,MUL,SAD,L/PINTERP,RCP,TEX,TXF
0x00010e40, 0x00000040, 0x00000498, 0x00000000
};
static const operation noDestList[] =
{
OP_STORE, OP_WRSV, OP_EXPORT, OP_BRA, OP_CALL, OP_RET, OP_EXIT,
OP_DISCARD, OP_CONT, OP_BREAK, OP_PRECONT, OP_PREBREAK, OP_PRERET,
OP_JOIN, OP_JOINAT, OP_BRKPT, OP_MEMBAR, OP_EMIT, OP_RESTART,
OP_QUADON, OP_QUADPOP, OP_TEXBAR, OP_SUSTB, OP_SUSTP, OP_SUREDP,
OP_SUREDB, OP_BAR
};
static const operation noPredList[] =
{
OP_CALL, OP_PREBREAK, OP_PRERET, OP_QUADON, OP_QUADPOP, OP_JOINAT
};
 
for (i = 0; i < DATA_FILE_COUNT; ++i)
nativeFileMap[i] = (DataFile)i;
nativeFileMap[FILE_PREDICATE] = FILE_FLAGS;
 
for (i = 0; i < OP_LAST; ++i) {
opInfo[i].variants = NULL;
opInfo[i].op = (operation)i;
opInfo[i].srcTypes = 1 << (int)TYPE_F32;
opInfo[i].dstTypes = 1 << (int)TYPE_F32;
opInfo[i].immdBits = 0xffffffff;
opInfo[i].srcNr = operationSrcNr[i];
 
for (j = 0; j < opInfo[i].srcNr; ++j) {
opInfo[i].srcMods[j] = 0;
opInfo[i].srcFiles[j] = 1 << (int)FILE_GPR;
}
opInfo[i].dstMods = 0;
opInfo[i].dstFiles = 1 << (int)FILE_GPR;
 
opInfo[i].hasDest = 1;
opInfo[i].vector = (i >= OP_TEX && i <= OP_TEXCSAA);
opInfo[i].commutative = (commutative[i / 32] >> (i % 32)) & 1;
opInfo[i].pseudo = (i < OP_MOV);
opInfo[i].predicate = !opInfo[i].pseudo;
opInfo[i].flow = (i >= OP_BRA && i <= OP_JOIN);
opInfo[i].minEncSize = (shortForm[i / 32] & (1 << (i % 32))) ? 4 : 8;
}
for (i = 0; i < sizeof(noDestList) / sizeof(noDestList[0]); ++i)
opInfo[noDestList[i]].hasDest = 0;
for (i = 0; i < sizeof(noPredList) / sizeof(noPredList[0]); ++i)
opInfo[noPredList[i]].predicate = 0;
 
for (i = 0; i < sizeof(_initProps) / sizeof(_initProps[0]); ++i) {
const struct opProperties *prop = &_initProps[i];
 
for (int s = 0; s < 3; ++s) {
if (prop->mNeg & (1 << s))
opInfo[prop->op].srcMods[s] |= NV50_IR_MOD_NEG;
if (prop->mAbs & (1 << s))
opInfo[prop->op].srcMods[s] |= NV50_IR_MOD_ABS;
if (prop->mNot & (1 << s))
opInfo[prop->op].srcMods[s] |= NV50_IR_MOD_NOT;
if (prop->fConst & (1 << s))
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_MEMORY_CONST;
if (prop->fShared & (1 << s))
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_MEMORY_SHARED;
if (prop->fAttrib & (1 << s))
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_SHADER_INPUT;
if (prop->fImm & (1 << s))
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_IMMEDIATE;
}
if (prop->mSat & 8)
opInfo[prop->op].dstMods = NV50_IR_MOD_SAT;
}
}
 
unsigned int
TargetNV50::getFileSize(DataFile file) const
{
switch (file) {
case FILE_NULL: return 0;
case FILE_GPR: return 256; // in 16-bit units **
case FILE_PREDICATE: return 0;
case FILE_FLAGS: return 4;
case FILE_ADDRESS: return 4;
case FILE_IMMEDIATE: return 0;
case FILE_MEMORY_CONST: return 65536;
case FILE_SHADER_INPUT: return 0x200;
case FILE_SHADER_OUTPUT: return 0x200;
case FILE_MEMORY_GLOBAL: return 0xffffffff;
case FILE_MEMORY_SHARED: return 16 << 10;
case FILE_MEMORY_LOCAL: return 48 << 10;
case FILE_SYSTEM_VALUE: return 16;
default:
assert(!"invalid file");
return 0;
}
// ** only first 128 units encodable for 16-bit regs
}
 
unsigned int
TargetNV50::getFileUnit(DataFile file) const
{
if (file == FILE_GPR || file == FILE_ADDRESS)
return 1;
if (file == FILE_SYSTEM_VALUE)
return 2;
return 0;
}
 
uint32_t
TargetNV50::getSVAddress(DataFile shaderFile, const Symbol *sym) const
{
switch (sym->reg.data.sv.sv) {
case SV_FACE:
return 0x3fc;
case SV_POSITION:
{
uint32_t addr = sysvalLocation[sym->reg.data.sv.sv];
for (int c = 0; c < sym->reg.data.sv.index; ++c)
if (wposMask & (1 << c))
addr += 4;
return addr;
}
case SV_NCTAID:
return 0x8 + 2 * sym->reg.data.sv.index;
case SV_CTAID:
return 0xc + 2 * sym->reg.data.sv.index;
case SV_NTID:
return 0x2 + 2 * sym->reg.data.sv.index;
case SV_TID:
return 0;
default:
return sysvalLocation[sym->reg.data.sv.sv];
}
}
 
// long: rrr, arr, rcr, acr, rrc, arc, gcr, grr
// short: rr, ar, rc, gr
// immd: ri, gi
bool
TargetNV50::insnCanLoad(const Instruction *i, int s,
const Instruction *ld) const
{
DataFile sf = ld->src(0).getFile();
 
if (sf == FILE_IMMEDIATE && (i->predSrc >= 0 || i->flagsDef >= 0))
return false;
if (s >= opInfo[i->op].srcNr)
return false;
if (!(opInfo[i->op].srcFiles[s] & (1 << (int)sf)))
return false;
if (s == 2 && i->src(1).getFile() != FILE_GPR)
return false;
 
// NOTE: don't rely on flagsDef
for (int d = 0; i->defExists(d); ++d)
if (i->def(d).getFile() == FILE_FLAGS)
return false;
 
unsigned mode = 0;
 
for (int z = 0; z < Target::operationSrcNr[i->op]; ++z) {
DataFile zf = (z == s) ? sf : i->src(z).getFile();
switch (zf) {
case FILE_GPR:
break;
case FILE_MEMORY_SHARED:
case FILE_SHADER_INPUT:
mode |= 1 << (z * 2);
break;
case FILE_MEMORY_CONST:
mode |= 2 << (z * 2);
break;
case FILE_IMMEDIATE:
mode |= 3 << (z * 2);
default:
break;
}
}
 
switch (mode) {
case 0x00:
case 0x01:
case 0x03:
case 0x08:
case 0x09:
case 0x0c:
case 0x20:
case 0x21:
break;
case 0x0d:
if (ld->bb->getProgram()->getType() != Program::TYPE_GEOMETRY)
return false;
default:
return false;
}
 
uint8_t ldSize;
 
if ((i->op == OP_MUL || i->op == OP_MAD) && !isFloatType(i->dType)) {
// 32-bit MUL will be split into 16-bit MULs
if (ld->src(0).isIndirect(0))
return false;
if (sf == FILE_IMMEDIATE)
return false;
ldSize = 2;
} else {
ldSize = typeSizeof(ld->dType);
}
 
if (sf == FILE_IMMEDIATE)
return true;
 
 
// Check if memory access is encodable:
 
if (ldSize < 4 && sf == FILE_SHADER_INPUT) // no < 4-byte aligned a[] access
return false;
if (ld->getSrc(0)->reg.data.offset > (int32_t)(127 * ldSize))
return false;
 
if (ld->src(0).isIndirect(0)) {
for (int z = 0; i->srcExists(z); ++z)
if (i->src(z).isIndirect(0))
return false;
 
// s[] access only possible in CP, $aX always applies
if (sf == FILE_MEMORY_SHARED)
return true;
if (!ld->bb) // can't check type ...
return false;
Program::Type pt = ld->bb->getProgram()->getType();
 
// $aX applies to c[] only in VP, FP, GP if p[] is not accessed
if (pt == Program::TYPE_COMPUTE)
return false;
if (pt == Program::TYPE_GEOMETRY) {
if (sf == FILE_MEMORY_CONST)
return i->src(s).getFile() != FILE_SHADER_INPUT;
return sf == FILE_SHADER_INPUT;
}
return sf == FILE_MEMORY_CONST;
}
return true;
}
 
bool
TargetNV50::isAccessSupported(DataFile file, DataType ty) const
{
if (ty == TYPE_B96 || ty == TYPE_NONE)
return false;
if (typeSizeof(ty) > 4)
return (file == FILE_MEMORY_LOCAL) || (file == FILE_MEMORY_GLOBAL);
return true;
}
 
bool
TargetNV50::isOpSupported(operation op, DataType ty) const
{
if (ty == TYPE_F64 && chipset < 0xa0)
return false;
 
switch (op) {
case OP_PRERET:
return chipset >= 0xa0;
case OP_TXG:
return chipset >= 0xa3;
case OP_POW:
case OP_SQRT:
case OP_DIV:
case OP_MOD:
case OP_SET_AND:
case OP_SET_OR:
case OP_SET_XOR:
case OP_SLCT:
case OP_SELP:
case OP_POPCNT:
case OP_INSBF:
case OP_EXTBF:
case OP_EXIT: // want exit modifier instead (on NOP if required)
case OP_MEMBAR:
return false;
case OP_SAD:
return ty == TYPE_S32;
default:
return true;
}
}
 
bool
TargetNV50::isModSupported(const Instruction *insn, int s, Modifier mod) const
{
if (!isFloatType(insn->dType)) {
switch (insn->op) {
case OP_ABS:
case OP_NEG:
case OP_CVT:
case OP_CEIL:
case OP_FLOOR:
case OP_TRUNC:
case OP_AND:
case OP_OR:
case OP_XOR:
break;
case OP_ADD:
if (insn->src(s ? 0 : 1).mod.neg())
return false;
break;
case OP_SUB:
if (s == 0)
return insn->src(1).mod.neg() ? false : true;
break;
case OP_SET:
if (insn->sType != TYPE_F32)
return false;
break;
default:
return false;
}
}
if (s > 3)
return false;
return (mod & Modifier(opInfo[insn->op].srcMods[s])) == mod;
}
 
bool
TargetNV50::mayPredicate(const Instruction *insn, const Value *pred) const
{
if (insn->getPredicate() || insn->flagsSrc >= 0)
return false;
for (int s = 0; insn->srcExists(s); ++s)
if (insn->src(s).getFile() == FILE_IMMEDIATE)
return false;
return opInfo[insn->op].predicate;
}
 
bool
TargetNV50::isSatSupported(const Instruction *insn) const
{
if (insn->op == OP_CVT)
return true;
if (insn->dType != TYPE_F32)
return false;
return opInfo[insn->op].dstMods & NV50_IR_MOD_SAT;
}
 
int TargetNV50::getLatency(const Instruction *i) const
{
// TODO: tune these values
if (i->op == OP_LOAD) {
switch (i->src(0).getFile()) {
case FILE_MEMORY_LOCAL:
case FILE_MEMORY_GLOBAL:
return 100; // really 400 to 800
default:
return 22;
}
}
return 22;
}
 
// These are "inverse" throughput values, i.e. the number of cycles required
// to issue a specific instruction for a full warp (32 threads).
//
// Assuming we have more than 1 warp in flight, a higher issue latency results
// in a lower result latency since the MP will have spent more time with other
// warps.
// This also helps to determine the number of cycles between instructions in
// a single warp.
//
int TargetNV50::getThroughput(const Instruction *i) const
{
// TODO: tune these values
if (i->dType == TYPE_F32) {
switch (i->op) {
case OP_RCP:
case OP_RSQ:
case OP_LG2:
case OP_SIN:
case OP_COS:
case OP_PRESIN:
case OP_PREEX2:
return 16;
default:
return 4;
}
} else
if (i->dType == TYPE_U32 || i->dType == TYPE_S32) {
return 4;
} else
if (i->dType == TYPE_F64) {
return 32;
} else {
return 1;
}
}
 
static void
recordLocation(uint16_t *locs, uint8_t *masks,
const struct nv50_ir_varying *var)
{
uint16_t addr = var->slot[0] * 4;
 
switch (var->sn) {
case TGSI_SEMANTIC_POSITION: locs[SV_POSITION] = addr; break;
case TGSI_SEMANTIC_INSTANCEID: locs[SV_INSTANCE_ID] = addr; break;
case TGSI_SEMANTIC_VERTEXID: locs[SV_VERTEX_ID] = addr; break;
case TGSI_SEMANTIC_PRIMID: locs[SV_PRIMITIVE_ID] = addr; break;
case NV50_SEMANTIC_LAYER: locs[SV_LAYER] = addr; break;
case NV50_SEMANTIC_VIEWPORTINDEX: locs[SV_VIEWPORT_INDEX] = addr; break;
default:
break;
}
if (var->sn == TGSI_SEMANTIC_POSITION && masks)
masks[0] = var->mask;
}
 
void
TargetNV50::parseDriverInfo(const struct nv50_ir_prog_info *info)
{
unsigned int i;
for (i = 0; i < info->numOutputs; ++i)
recordLocation(sysvalLocation, NULL, &info->out[i]);
for (i = 0; i < info->numInputs; ++i)
recordLocation(sysvalLocation, &wposMask, &info->in[i]);
for (i = 0; i < info->numSysVals; ++i)
recordLocation(sysvalLocation, NULL, &info->sv[i]);
 
if (sysvalLocation[SV_POSITION] >= 0x200) {
// not assigned by driver, but we need it internally
wposMask = 0x8;
sysvalLocation[SV_POSITION] = 0;
}
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.h
0,0 → 1,72
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50/codegen/nv50_ir_target.h"
 
namespace nv50_ir {
 
#define NVC0_BUILTIN_DIV_U32 0
#define NVC0_BUILTIN_DIV_S32 1
#define NVC0_BUILTIN_RCP_F64 2
#define NVC0_BUILTIN_RSQ_F64 3
 
#define NVC0_BUILTIN_COUNT 4
 
class TargetNV50 : public Target
{
public:
TargetNV50(unsigned int chipset);
 
virtual CodeEmitter *getCodeEmitter(Program::Type);
 
virtual bool runLegalizePass(Program *, CGStage stage) const;
 
virtual void getBuiltinCode(const uint32_t **code, uint32_t *size) const;
 
virtual void parseDriverInfo(const struct nv50_ir_prog_info *);
 
virtual bool insnCanLoad(const Instruction *insn, int s,
const Instruction *ld) const;
virtual bool isOpSupported(operation, DataType) const;
virtual bool isAccessSupported(DataFile, DataType) const;
virtual bool isModSupported(const Instruction *, int s, Modifier) const;
virtual bool isSatSupported(const Instruction *) const;
virtual bool mayPredicate(const Instruction *, const Value *) const;
 
virtual int getLatency(const Instruction *) const;
virtual int getThroughput(const Instruction *) const;
 
virtual unsigned int getFileSize(DataFile) const;
virtual unsigned int getFileUnit(DataFile) const;
 
virtual uint32_t getSVAddress(DataFile shaderFile, const Symbol *sv) const;
 
uint32_t getBuiltinOffset(int builtin) const;
 
private:
void initOpInfo();
 
uint16_t sysvalLocation[SV_LAST + 1];
uint8_t wposMask;
};
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_util.cpp
0,0 → 1,390
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#include "nv50_ir_util.h"
 
namespace nv50_ir {
 
void DLList::clear()
{
for (Item *next, *item = head.next; item != &head; item = next) {
next = item->next;
delete item;
}
head.next = head.prev = &head;
}
 
void
DLList::Iterator::erase()
{
Item *rem = pos;
 
if (rem == term)
return;
pos = pos->next;
 
DLLIST_DEL(rem);
delete rem;
}
 
void DLList::Iterator::moveToList(DLList& dest)
{
Item *item = pos;
 
assert(term != &dest.head);
assert(pos != term);
 
pos = pos->next;
 
DLLIST_DEL(item);
DLLIST_ADDHEAD(&dest.head, item);
}
 
bool
DLList::Iterator::insert(void *data)
{
Item *ins = new Item(data);
 
ins->next = pos->next;
ins->prev = pos;
pos->next->prev = ins;
pos->next = ins;
 
if (pos == term)
term = ins;
 
return true;
}
 
void
Stack::moveTo(Stack& that)
{
unsigned int newSize = this->size + that.size;
 
while (newSize > that.limit)
that.resize();
memcpy(&that.array[that.size], &array[0], this->size * sizeof(Item));
 
that.size = newSize;
this->size = 0;
}
 
Interval::Interval(const Interval& that) : head(NULL), tail(NULL)
{
this->insert(that);
}
 
Interval::~Interval()
{
clear();
}
 
void
Interval::clear()
{
for (Range *next, *r = head; r; r = next) {
next = r->next;
delete r;
}
head = tail = NULL;
}
 
bool
Interval::extend(int a, int b)
{
Range *r, **nextp = &head;
 
// NOTE: we need empty intervals for fixed registers
// if (a == b)
// return false;
assert(a <= b);
 
for (r = head; r; r = r->next) {
if (b < r->bgn)
break; // insert before
if (a > r->end) {
// insert after
nextp = &r->next;
continue;
}
 
// overlap
if (a < r->bgn) {
r->bgn = a;
if (b > r->end)
r->end = b;
r->coalesce(&tail);
return true;
}
if (b > r->end) {
r->end = b;
r->coalesce(&tail);
return true;
}
assert(a >= r->bgn);
assert(b <= r->end);
return true;
}
 
(*nextp) = new Range(a, b);
(*nextp)->next = r;
 
for (r = (*nextp); r->next; r = r->next);
tail = r;
return true;
}
 
bool Interval::contains(int pos) const
{
for (Range *r = head; r && r->bgn <= pos; r = r->next)
if (r->end > pos)
return true;
return false;
}
 
bool Interval::overlaps(const Interval &that) const
{
#if 1
Range *a = this->head;
Range *b = that.head;
 
while (a && b) {
if (b->bgn < a->end &&
b->end > a->bgn)
return true;
if (a->end <= b->bgn)
a = a->next;
else
b = b->next;
}
#else
for (Range *rA = this->head; rA; rA = rA->next)
for (Range *rB = iv.head; rB; rB = rB->next)
if (rB->bgn < rA->end &&
rB->end > rA->bgn)
return true;
#endif
return false;
}
 
void Interval::insert(const Interval &that)
{
for (Range *r = that.head; r; r = r->next)
this->extend(r->bgn, r->end);
}
 
void Interval::unify(Interval &that)
{
assert(this != &that);
for (Range *next, *r = that.head; r; r = next) {
next = r->next;
this->extend(r->bgn, r->end);
delete r;
}
that.head = NULL;
}
 
int Interval::length() const
{
int len = 0;
for (Range *r = head; r; r = r->next)
len += r->bgn - r->end;
return len;
}
 
void Interval::print() const
{
if (!head)
return;
INFO("[%i %i)", head->bgn, head->end);
for (const Range *r = head->next; r; r = r->next)
INFO(" [%i %i)", r->bgn, r->end);
INFO("\n");
}
 
void
BitSet::andNot(const BitSet &set)
{
assert(data && set.data);
assert(size >= set.size);
for (unsigned int i = 0; i < (set.size + 31) / 32; ++i)
data[i] &= ~set.data[i];
}
 
BitSet& BitSet::operator|=(const BitSet &set)
{
assert(data && set.data);
assert(size >= set.size);
for (unsigned int i = 0; i < (set.size + 31) / 32; ++i)
data[i] |= set.data[i];
return *this;
}
 
bool BitSet::resize(unsigned int nBits)
{
if (!data || !nBits)
return allocate(nBits, true);
const unsigned int p = (size + 31) / 32;
const unsigned int n = (nBits + 31) / 32;
if (n == p)
return true;
 
data = (uint32_t *)REALLOC(data, 4 * p, 4 * n);
if (!data) {
size = 0;
return false;
}
if (n > p)
memset(&data[4 * p + 4], 0, (n - p) * 4);
 
size = nBits;
return true;
}
 
bool BitSet::allocate(unsigned int nBits, bool zero)
{
if (data && size < nBits) {
FREE(data);
data = NULL;
}
size = nBits;
 
if (!data)
data = reinterpret_cast<uint32_t *>(CALLOC((size + 31) / 32, 4));
 
if (zero)
memset(data, 0, (size + 7) / 8);
else
if (nBits)
data[(size + 31) / 32 - 1] = 0; // clear unused bits (e.g. for popCount)
 
return data;
}
 
unsigned int BitSet::popCount() const
{
unsigned int count = 0;
 
for (unsigned int i = 0; i < (size + 31) / 32; ++i)
if (data[i])
count += util_bitcount(data[i]);
return count;
}
 
void BitSet::fill(uint32_t val)
{
unsigned int i;
for (i = 0; i < (size + 31) / 32; ++i)
data[i] = val;
if (val)
data[i] &= ~(0xffffffff << (size % 32)); // BE ?
}
 
void BitSet::setOr(BitSet *pA, BitSet *pB)
{
if (!pB) {
*this = *pA;
} else {
for (unsigned int i = 0; i < (size + 31) / 32; ++i)
data[i] = pA->data[i] | pB->data[i];
}
}
 
int BitSet::findFreeRange(unsigned int count) const
{
const uint32_t m = (1 << count) - 1;
int pos = size;
unsigned int i;
const unsigned int end = (size + 31) / 32;
 
if (count == 1) {
for (i = 0; i < end; ++i) {
pos = ffs(~data[i]) - 1;
if (pos >= 0)
break;
}
} else
if (count == 2) {
for (i = 0; i < end; ++i) {
if (data[i] != 0xffffffff) {
uint32_t b = data[i] | (data[i] >> 1) | 0xaaaaaaaa;
pos = ffs(~b) - 1;
if (pos >= 0)
break;
}
}
} else
if (count == 4 || count == 3) {
for (i = 0; i < end; ++i) {
if (data[i] != 0xffffffff) {
uint32_t b =
(data[i] >> 0) | (data[i] >> 1) |
(data[i] >> 2) | (data[i] >> 3) | 0xeeeeeeee;
pos = ffs(~b) - 1;
if (pos >= 0)
break;
}
}
} else {
if (count <= 8)
count = 8;
else
if (count <= 16)
count = 16;
else
count = 32;
 
for (i = 0; i < end; ++i) {
if (data[i] != 0xffffffff) {
for (pos = 0; pos < 32; pos += count)
if (!(data[i] & (m << pos)))
break;
if (pos < 32)
break;
}
}
}
pos += i * 32;
 
return ((pos + count) <= size) ? pos : -1;
}
 
void BitSet::print() const
{
unsigned int n = 0;
INFO("BitSet of size %u:\n", size);
for (unsigned int i = 0; i < (size + 31) / 32; ++i) {
uint32_t bits = data[i];
while (bits) {
int pos = ffs(bits) - 1;
bits &= ~(1 << pos);
INFO(" %i", i * 32 + pos);
++n;
if ((n % 16) == 0)
INFO("\n");
}
}
if (n % 16)
INFO("\n");
}
 
} // namespace nv50_ir
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_util.h
0,0 → 1,788
/*
* Copyright 2011 Christoph Bumiller
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
 
#ifndef __NV50_IR_UTIL_H__
#define __NV50_IR_UTIL_H__
 
#include <new>
#include <assert.h>
#include <stdio.h>
#include <memory>
#include <map>
 
#ifndef NDEBUG
# include <typeinfo>
#endif
 
#include "util/u_inlines.h"
#include "util/u_memory.h"
 
#define ERROR(args...) debug_printf("ERROR: " args)
#define WARN(args...) debug_printf("WARNING: " args)
#define INFO(args...) debug_printf(args)
 
#define INFO_DBG(m, f, args...) \
do { \
if (m & NV50_IR_DEBUG_##f) \
debug_printf(args); \
} while(0)
 
#define FATAL(args...) \
do { \
fprintf(stderr, args); \
abort(); \
} while(0)
 
 
#define NV50_IR_FUNC_ALLOC_OBJ_DEF(obj, f, args...) \
new ((f)->getProgram()->mem_##obj.allocate()) obj(f, args)
 
#define new_Instruction(f, args...) \
NV50_IR_FUNC_ALLOC_OBJ_DEF(Instruction, f, args)
#define new_CmpInstruction(f, args...) \
NV50_IR_FUNC_ALLOC_OBJ_DEF(CmpInstruction, f, args)
#define new_TexInstruction(f, args...) \
NV50_IR_FUNC_ALLOC_OBJ_DEF(TexInstruction, f, args)
#define new_FlowInstruction(f, args...) \
NV50_IR_FUNC_ALLOC_OBJ_DEF(FlowInstruction, f, args)
 
#define new_LValue(f, args...) \
NV50_IR_FUNC_ALLOC_OBJ_DEF(LValue, f, args)
 
 
#define NV50_IR_PROG_ALLOC_OBJ_DEF(obj, p, args...) \
new ((p)->mem_##obj.allocate()) obj(p, args)
 
#define new_Symbol(p, args...) \
NV50_IR_PROG_ALLOC_OBJ_DEF(Symbol, p, args)
#define new_ImmediateValue(p, args...) \
NV50_IR_PROG_ALLOC_OBJ_DEF(ImmediateValue, p, args)
 
 
#define delete_Instruction(p, insn) (p)->releaseInstruction(insn)
#define delete_Value(p, val) (p)->releaseValue(val)
 
 
namespace nv50_ir {
 
class Iterator
{
public:
virtual ~Iterator() { };
virtual void next() = 0;
virtual void *get() const = 0;
virtual bool end() const = 0; // if true, get will return 0
virtual void reset() { assert(0); } // only for graph iterators
};
 
typedef std::auto_ptr<Iterator> IteratorRef;
 
class ManipIterator : public Iterator
{
public:
virtual bool insert(void *) = 0; // insert after current position
virtual void erase() = 0;
};
 
// WARNING: do not use a->prev/next for __item or __list
 
#define DLLIST_DEL(__item) \
do { \
(__item)->prev->next = (__item)->next; \
(__item)->next->prev = (__item)->prev; \
(__item)->next = (__item); \
(__item)->prev = (__item); \
} while(0)
 
#define DLLIST_ADDTAIL(__list, __item) \
do { \
(__item)->next = (__list); \
(__item)->prev = (__list)->prev; \
(__list)->prev->next = (__item); \
(__list)->prev = (__item); \
} while(0)
 
#define DLLIST_ADDHEAD(__list, __item) \
do { \
(__item)->prev = (__list); \
(__item)->next = (__list)->next; \
(__list)->next->prev = (__item); \
(__list)->next = (__item); \
} while(0)
 
#define DLLIST_MERGE(__listA, __listB, ty) \
do { \
ty prevB = (__listB)->prev; \
(__listA)->prev->next = (__listB); \
(__listB)->prev->next = (__listA); \
(__listB)->prev = (__listA)->prev; \
(__listA)->prev = prevB; \
} while(0)
 
#define DLLIST_EMPTY(__list) ((__list)->next == (__list))
 
#define DLLIST_FOR_EACH(list, it) \
for (DLList::Iterator (it) = (list)->iterator(); !(it).end(); (it).next())
 
class DLList
{
public:
class Item
{
public:
Item(void *priv) : next(this), prev(this), data(priv) { }
 
public:
Item *next;
Item *prev;
void *data;
};
 
DLList() : head(0) { }
~DLList() { clear(); }
 
inline void insertHead(void *data)
{
Item *item = new Item(data);
 
assert(data);
 
item->prev = &head;
item->next = head.next;
head.next->prev = item;
head.next = item;
}
 
inline void insertTail(void *data)
{
Item *item = new Item(data);
 
assert(data);
 
DLLIST_ADDTAIL(&head, item);
}
 
inline void insert(void *data) { insertTail(data); }
 
void clear();
 
class Iterator : public ManipIterator
{
public:
Iterator(Item *head, bool r) : rev(r), pos(r ? head->prev : head->next),
term(head) { }
 
virtual void next() { if (!end()) pos = rev ? pos->prev : pos->next; }
virtual void *get() const { return pos->data; }
virtual bool end() const { return pos == term; }
 
// caution: if you're at end-2 and erase it, then do next, you're at end
virtual void erase();
virtual bool insert(void *data);
 
// move item to a another list, no consistency with its iterators though
void moveToList(DLList&);
 
private:
const bool rev;
Item *pos;
Item *term;
 
friend class DLList;
};
 
inline void erase(Iterator& pos)
{
pos.erase();
}
 
Iterator iterator()
{
return Iterator(&head, false);
}
 
Iterator revIterator()
{
return Iterator(&head, true);
}
 
private:
Item head;
};
 
class Stack
{
public:
class Item {
public:
union {
void *p;
int i;
unsigned int u;
float f;
double d;
} u;
 
Item() { memset(&u, 0, sizeof(u)); }
};
Stack() : size(0), limit(0), array(0) { }
~Stack() { if (array) FREE(array); }
 
inline void push(int i) { Item data; data.u.i = i; push(data); }
inline void push(unsigned int u) { Item data; data.u.u = u; push(data); }
inline void push(void *p) { Item data; data.u.p = p; push(data); }
inline void push(float f) { Item data; data.u.f = f; push(data); }
 
inline void push(Item data)
{
if (size == limit)
resize();
array[size++] = data;
}
 
inline Item pop()
{
if (!size) {
Item data;
assert(0);
return data;
}
return array[--size];
}
 
inline unsigned int getSize() { return size; }
 
inline Item& peek() { assert(size); return array[size - 1]; }
 
void clear(bool releaseStorage = false)
{
if (releaseStorage && array)
FREE(array);
size = limit = 0;
}
 
void moveTo(Stack&); // move all items to target (not like push(pop()))
 
private:
void resize()
{
unsigned int sizeOld, sizeNew;
 
sizeOld = limit * sizeof(Item);
limit = MAX2(4, limit + limit);
sizeNew = limit * sizeof(Item);
 
array = (Item *)REALLOC(array, sizeOld, sizeNew);
}
 
unsigned int size;
unsigned int limit;
Item *array;
};
 
class DynArray
{
public:
class Item
{
public:
union {
uint32_t u32;
void *p;
};
};
 
DynArray() : data(NULL), size(0) { }
 
~DynArray() { if (data) FREE(data); }
 
inline Item& operator[](unsigned int i)
{
if (i >= size)
resize(i);
return data[i];
}
 
inline const Item operator[](unsigned int i) const
{
return data[i];
}
 
void resize(unsigned int index)
{
const unsigned int oldSize = size * sizeof(Item);
 
if (!size)
size = 8;
while (size <= index)
size <<= 1;
 
data = (Item *)REALLOC(data, oldSize, size * sizeof(Item));
}
 
void clear()
{
FREE(data);
data = NULL;
size = 0;
}
 
private:
Item *data;
unsigned int size;
};
 
class ArrayList
{
public:
ArrayList() : size(0) { }
 
void insert(void *item, int& id)
{
id = ids.getSize() ? ids.pop().u.i : size++;
data[id].p = item;
}
 
void remove(int& id)
{
const unsigned int uid = id;
assert(uid < size && data[id].p);
ids.push(uid);
data[uid].p = NULL;
id = -1;
}
 
inline int getSize() const { return size; }
 
inline void *get(unsigned int id) { assert(id < size); return data[id].p; }
 
class Iterator : public nv50_ir::Iterator
{
public:
Iterator(const ArrayList *array) : pos(0), data(array->data)
{
size = array->getSize();
if (size)
nextValid();
}
 
void nextValid() { while ((pos < size) && !data[pos].p) ++pos; }
 
void next() { if (pos < size) { ++pos; nextValid(); } }
void *get() const { assert(pos < size); return data[pos].p; }
bool end() const { return pos >= size; }
 
private:
unsigned int pos;
unsigned int size;
const DynArray& data;
 
friend class ArrayList;
};
 
Iterator iterator() const { return Iterator(this); }
 
void clear()
{
data.clear();
ids.clear(true);
size = 0;
}
 
private:
DynArray data;
Stack ids;
unsigned int size;
};
 
class Interval
{
public:
Interval() : head(0), tail(0) { }
Interval(const Interval&);
~Interval();
 
bool extend(int, int);
void insert(const Interval&);
void unify(Interval&); // clears source interval
void clear();
 
inline int begin() const { return head ? head->bgn : -1; }
inline int end() const { checkTail(); return tail ? tail->end : -1; }
inline bool isEmpty() const { return !head; }
bool overlaps(const Interval&) const;
bool contains(int pos) const;
 
inline int extent() const { return end() - begin(); }
int length() const;
 
void print() const;
 
inline void checkTail() const;
 
private:
class Range
{
public:
Range(int a, int b) : next(0), bgn(a), end(b) { }
 
Range *next;
int bgn;
int end;
 
void coalesce(Range **ptail)
{
Range *rnn;
 
while (next && end >= next->bgn) {
assert(bgn <= next->bgn);
rnn = next->next;
end = MAX2(end, next->end);
delete next;
next = rnn;
}
if (!next)
*ptail = this;
}
};
 
Range *head;
Range *tail;
};
 
class BitSet
{
public:
BitSet() : marker(false), data(0), size(0) { }
BitSet(unsigned int nBits, bool zero) : marker(false), data(0), size(0)
{
allocate(nBits, zero);
}
~BitSet()
{
if (data)
FREE(data);
}
 
bool allocate(unsigned int nBits, bool zero);
bool resize(unsigned int nBits); // keep old data, zero additional bits
 
inline unsigned int getSize() const { return size; }
 
void fill(uint32_t val);
 
void setOr(BitSet *, BitSet *); // second BitSet may be NULL
 
inline void set(unsigned int i)
{
assert(i < size);
data[i / 32] |= 1 << (i % 32);
}
// NOTE: range may not cross 32 bit boundary (implies n <= 32)
inline void setRange(unsigned int i, unsigned int n)
{
assert((i + n) <= size && (((i % 32) + n) <= 32));
data[i / 32] |= ((1 << n) - 1) << (i % 32);
}
inline void setMask(unsigned int i, uint32_t m)
{
assert(i < size);
data[i / 32] |= m;
}
 
inline void clr(unsigned int i)
{
assert(i < size);
data[i / 32] &= ~(1 << (i % 32));
}
// NOTE: range may not cross 32 bit boundary (implies n <= 32)
inline void clrRange(unsigned int i, unsigned int n)
{
assert((i + n) <= size && (((i % 32) + n) <= 32));
data[i / 32] &= ~(((1 << n) - 1) << (i % 32));
}
 
inline bool test(unsigned int i) const
{
assert(i < size);
return data[i / 32] & (1 << (i % 32));
}
// NOTE: range may not cross 32 bit boundary (implies n <= 32)
inline bool testRange(unsigned int i, unsigned int n) const
{
assert((i + n) <= size && (((i % 32) + n) <= 32));
return data[i / 32] & (((1 << n) - 1) << (i % 32));
}
 
// Find a range of size (<= 32) clear bits aligned to roundup_pow2(size).
int findFreeRange(unsigned int size) const;
 
BitSet& operator|=(const BitSet&);
 
BitSet& operator=(const BitSet& set)
{
assert(data && set.data);
assert(size == set.size);
memcpy(data, set.data, (set.size + 7) / 8);
return *this;
}
 
void andNot(const BitSet&);
 
// bits = (bits | setMask) & ~clrMask
inline void periodicMask32(uint32_t setMask, uint32_t clrMask)
{
for (unsigned int i = 0; i < (size + 31) / 32; ++i)
data[i] = (data[i] | setMask) & ~clrMask;
}
 
unsigned int popCount() const;
 
void print() const;
 
public:
bool marker; // for user
 
private:
uint32_t *data;
unsigned int size;
};
 
void Interval::checkTail() const
{
#if NV50_DEBUG & NV50_DEBUG_PROG_RA
Range *r = head;
while (r->next)
r = r->next;
assert(tail == r);
#endif
}
 
class MemoryPool
{
private:
inline bool enlargeAllocationsArray(const unsigned int id, unsigned int nr)
{
const unsigned int size = sizeof(uint8_t *) * id;
const unsigned int incr = sizeof(uint8_t *) * nr;
 
uint8_t **alloc = (uint8_t **)REALLOC(allocArray, size, size + incr);
if (!alloc)
return false;
allocArray = alloc;
return true;
}
 
inline bool enlargeCapacity()
{
const unsigned int id = count >> objStepLog2;
 
uint8_t *const mem = (uint8_t *)MALLOC(objSize << objStepLog2);
if (!mem)
return false;
 
if (!(id % 32)) {
if (!enlargeAllocationsArray(id, 32)) {
FREE(mem);
return false;
}
}
allocArray[id] = mem;
return true;
}
 
public:
MemoryPool(unsigned int size, unsigned int incr) : objSize(size),
objStepLog2(incr)
{
allocArray = NULL;
released = NULL;
count = 0;
}
 
~MemoryPool()
{
unsigned int allocCount = (count + (1 << objStepLog2) - 1) >> objStepLog2;
for (unsigned int i = 0; i < allocCount && allocArray[i]; ++i)
FREE(allocArray[i]);
if (allocArray)
FREE(allocArray);
}
 
void *allocate()
{
void *ret;
const unsigned int mask = (1 << objStepLog2) - 1;
 
if (released) {
ret = released;
released = *(void **)released;
return ret;
}
 
if (!(count & mask))
if (!enlargeCapacity())
return NULL;
 
ret = allocArray[count >> objStepLog2] + (count & mask) * objSize;
++count;
return ret;
}
 
void release(void *ptr)
{
*(void **)ptr = released;
released = ptr;
}
 
private:
uint8_t **allocArray; // array (list) of MALLOC allocations
 
void *released; // list of released objects
 
unsigned int count; // highest allocated object
 
const unsigned int objSize;
const unsigned int objStepLog2;
};
 
/**
* Composite object cloning policy.
*
* Encapsulates how sub-objects are to be handled (if at all) when a
* composite object is being cloned.
*/
template<typename C>
class ClonePolicy
{
protected:
C *c;
 
public:
ClonePolicy(C *c) : c(c) {}
 
C *context() { return c; }
 
template<typename T> T *get(T *obj)
{
void *clone = lookup(obj);
if (!clone)
clone = obj->clone(*this);
return reinterpret_cast<T *>(clone);
}
 
template<typename T> void set(const T *obj, T *clone)
{
insert(obj, clone);
}
 
protected:
virtual void *lookup(void *obj) = 0;
virtual void insert(const void *obj, void *clone) = 0;
};
 
/**
* Shallow non-recursive cloning policy.
*
* Objects cloned with the "shallow" policy don't clone their
* children recursively, instead, the new copy shares its children
* with the original object.
*/
template<typename C>
class ShallowClonePolicy : public ClonePolicy<C>
{
public:
ShallowClonePolicy(C *c) : ClonePolicy<C>(c) {}
 
protected:
virtual void *lookup(void *obj)
{
return obj;
}
 
virtual void insert(const void *obj, void *clone)
{
}
};
 
template<typename C, typename T>
inline T *cloneShallow(C *c, T *obj)
{
ShallowClonePolicy<C> pol(c);
return obj->clone(pol);
}
 
/**
* Recursive cloning policy.
*
* Objects cloned with the "deep" policy clone their children
* recursively, keeping track of what has already been cloned to
* avoid making several new copies of the same object.
*/
template<typename C>
class DeepClonePolicy : public ClonePolicy<C>
{
public:
DeepClonePolicy(C *c) : ClonePolicy<C>(c) {}
 
private:
std::map<const void *, void *> map;
 
protected:
virtual void *lookup(void *obj)
{
return map[obj];
}
 
virtual void insert(const void *obj, void *clone)
{
map[obj] = clone;
}
};
 
template<typename S, typename T>
struct bimap
{
std::map<S, T> forth;
std::map<T, S> back;
 
public:
bimap() : l(back), r(forth) { }
bimap(const bimap<S, T> &m)
: forth(m.forth), back(m.back), l(back), r(forth) { }
 
void insert(const S &s, const T &t)
{
forth.insert(std::make_pair(s, t));
back.insert(std::make_pair(t, s));
}
 
typedef typename std::map<T, S>::const_iterator l_iterator;
const std::map<T, S> &l;
typedef typename std::map<S, T>::const_iterator r_iterator;
const std::map<S, T> &r;
};
 
} // namespace nv50_ir
 
#endif // __NV50_IR_UTIL_H__