/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir.cpp |
---|
0,0 → 1,1231 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_target.h" |
#include "nv50_ir_driver.h" |
extern "C" { |
#include "nv50/nv50_program.h" |
#include "nv50/nv50_debug.h" |
} |
namespace nv50_ir { |
Modifier::Modifier(operation op) |
{ |
switch (op) { |
case OP_NEG: bits = NV50_IR_MOD_NEG; break; |
case OP_ABS: bits = NV50_IR_MOD_ABS; break; |
case OP_SAT: bits = NV50_IR_MOD_SAT; break; |
case OP_NOT: bits = NV50_IR_MOD_NOT; break; |
default: |
bits = 0; |
break; |
} |
} |
Modifier Modifier::operator*(const Modifier m) const |
{ |
unsigned int a, b, c; |
b = m.bits; |
if (this->bits & NV50_IR_MOD_ABS) |
b &= ~NV50_IR_MOD_NEG; |
a = (this->bits ^ b) & (NV50_IR_MOD_NOT | NV50_IR_MOD_NEG); |
c = (this->bits | m.bits) & (NV50_IR_MOD_ABS | NV50_IR_MOD_SAT); |
return Modifier(a | c); |
} |
ValueRef::ValueRef(Value *v) : value(NULL), insn(NULL) |
{ |
indirect[0] = -1; |
indirect[1] = -1; |
usedAsPtr = false; |
set(v); |
} |
ValueRef::ValueRef(const ValueRef& ref) : value(NULL), insn(ref.insn) |
{ |
set(ref); |
usedAsPtr = ref.usedAsPtr; |
} |
ValueRef::~ValueRef() |
{ |
this->set(NULL); |
} |
bool ValueRef::getImmediate(ImmediateValue &imm) const |
{ |
const ValueRef *src = this; |
Modifier m; |
DataType type = src->insn->sType; |
while (src) { |
if (src->mod) { |
if (src->insn->sType != type) |
break; |
m *= src->mod; |
} |
if (src->getFile() == FILE_IMMEDIATE) { |
imm = *(src->value->asImm()); |
// The immediate's type isn't required to match its use, it's |
// more of a hint; applying a modifier makes use of that hint. |
imm.reg.type = type; |
m.applyTo(imm); |
return true; |
} |
Instruction *insn = src->value->getUniqueInsn(); |
if (insn && insn->op == OP_MOV) { |
src = &insn->src(0); |
if (src->mod) |
WARN("OP_MOV with modifier encountered !\n"); |
} else { |
src = NULL; |
} |
} |
return false; |
} |
ValueDef::ValueDef(Value *v) : value(NULL), insn(NULL) |
{ |
set(v); |
} |
ValueDef::ValueDef(const ValueDef& def) : value(NULL), insn(NULL) |
{ |
set(def.get()); |
} |
ValueDef::~ValueDef() |
{ |
this->set(NULL); |
} |
void |
ValueRef::set(const ValueRef &ref) |
{ |
this->set(ref.get()); |
mod = ref.mod; |
indirect[0] = ref.indirect[0]; |
indirect[1] = ref.indirect[1]; |
} |
void |
ValueRef::set(Value *refVal) |
{ |
if (value == refVal) |
return; |
if (value) |
value->uses.remove(this); |
if (refVal) |
refVal->uses.push_back(this); |
value = refVal; |
} |
void |
ValueDef::set(Value *defVal) |
{ |
if (value == defVal) |
return; |
if (value) |
value->defs.remove(this); |
if (defVal) |
defVal->defs.push_back(this); |
value = defVal; |
} |
// Check if we can replace this definition's value by the value in @rep, |
// including the source modifiers, i.e. make sure that all uses support |
// @rep.mod. |
bool |
ValueDef::mayReplace(const ValueRef &rep) |
{ |
if (!rep.mod) |
return true; |
if (!insn || !insn->bb) // Unbound instruction ? |
return false; |
const Target *target = insn->bb->getProgram()->getTarget(); |
for (Value::UseIterator it = value->uses.begin(); it != value->uses.end(); |
++it) { |
Instruction *insn = (*it)->getInsn(); |
int s = -1; |
for (int i = 0; insn->srcExists(i); ++i) { |
if (insn->src(i).get() == value) { |
// If there are multiple references to us we'd have to check if the |
// combination of mods is still supported, but just bail for now. |
if (&insn->src(i) != (*it)) |
return false; |
s = i; |
} |
} |
assert(s >= 0); // integrity of uses list |
if (!target->isModSupported(insn, s, rep.mod)) |
return false; |
} |
return true; |
} |
void |
ValueDef::replace(const ValueRef &repVal, bool doSet) |
{ |
assert(mayReplace(repVal)); |
if (value == repVal.get()) |
return; |
while (!value->uses.empty()) { |
ValueRef *ref = value->uses.front(); |
ref->set(repVal.get()); |
ref->mod *= repVal.mod; |
} |
if (doSet) |
set(repVal.get()); |
} |
Value::Value() |
{ |
join = this; |
memset(®, 0, sizeof(reg)); |
reg.size = 4; |
} |
LValue::LValue(Function *fn, DataFile file) |
{ |
reg.file = file; |
reg.size = (file != FILE_PREDICATE) ? 4 : 1; |
reg.data.id = -1; |
compMask = 0; |
compound = 0; |
ssa = 0; |
fixedReg = 0; |
noSpill = 0; |
fn->add(this, this->id); |
} |
LValue::LValue(Function *fn, LValue *lval) |
{ |
assert(lval); |
reg.file = lval->reg.file; |
reg.size = lval->reg.size; |
reg.data.id = -1; |
compMask = 0; |
compound = 0; |
ssa = 0; |
fixedReg = 0; |
noSpill = 0; |
fn->add(this, this->id); |
} |
LValue * |
LValue::clone(ClonePolicy<Function>& pol) const |
{ |
LValue *that = new_LValue(pol.context(), reg.file); |
pol.set<Value>(this, that); |
that->reg.size = this->reg.size; |
that->reg.type = this->reg.type; |
that->reg.data = this->reg.data; |
return that; |
} |
bool |
LValue::isUniform() const |
{ |
if (defs.size() > 1) |
return false; |
Instruction *insn = getInsn(); |
// let's not try too hard here for now ... |
return !insn->srcExists(1) && insn->getSrc(0)->isUniform(); |
} |
Symbol::Symbol(Program *prog, DataFile f, ubyte fidx) |
{ |
baseSym = NULL; |
reg.file = f; |
reg.fileIndex = fidx; |
reg.data.offset = 0; |
prog->add(this, this->id); |
} |
Symbol * |
Symbol::clone(ClonePolicy<Function>& pol) const |
{ |
Program *prog = pol.context()->getProgram(); |
Symbol *that = new_Symbol(prog, reg.file, reg.fileIndex); |
pol.set<Value>(this, that); |
that->reg.size = this->reg.size; |
that->reg.type = this->reg.type; |
that->reg.data = this->reg.data; |
that->baseSym = this->baseSym; |
return that; |
} |
bool |
Symbol::isUniform() const |
{ |
return |
reg.file != FILE_SYSTEM_VALUE && |
reg.file != FILE_MEMORY_LOCAL && |
reg.file != FILE_SHADER_INPUT; |
} |
ImmediateValue::ImmediateValue(Program *prog, uint32_t uval) |
{ |
memset(®, 0, sizeof(reg)); |
reg.file = FILE_IMMEDIATE; |
reg.size = 4; |
reg.type = TYPE_U32; |
reg.data.u32 = uval; |
prog->add(this, this->id); |
} |
ImmediateValue::ImmediateValue(Program *prog, float fval) |
{ |
memset(®, 0, sizeof(reg)); |
reg.file = FILE_IMMEDIATE; |
reg.size = 4; |
reg.type = TYPE_F32; |
reg.data.f32 = fval; |
prog->add(this, this->id); |
} |
ImmediateValue::ImmediateValue(Program *prog, double dval) |
{ |
memset(®, 0, sizeof(reg)); |
reg.file = FILE_IMMEDIATE; |
reg.size = 8; |
reg.type = TYPE_F64; |
reg.data.f64 = dval; |
prog->add(this, this->id); |
} |
ImmediateValue::ImmediateValue(const ImmediateValue *proto, DataType ty) |
{ |
reg = proto->reg; |
reg.type = ty; |
reg.size = typeSizeof(ty); |
} |
ImmediateValue * |
ImmediateValue::clone(ClonePolicy<Function>& pol) const |
{ |
Program *prog = pol.context()->getProgram(); |
ImmediateValue *that = new_ImmediateValue(prog, 0u); |
pol.set<Value>(this, that); |
that->reg.size = this->reg.size; |
that->reg.type = this->reg.type; |
that->reg.data = this->reg.data; |
return that; |
} |
bool |
ImmediateValue::isInteger(const int i) const |
{ |
switch (reg.type) { |
case TYPE_S8: |
return reg.data.s8 == i; |
case TYPE_U8: |
return reg.data.u8 == i; |
case TYPE_S16: |
return reg.data.s16 == i; |
case TYPE_U16: |
return reg.data.u16 == i; |
case TYPE_S32: |
case TYPE_U32: |
return reg.data.s32 == i; // as if ... |
case TYPE_F32: |
return reg.data.f32 == static_cast<float>(i); |
case TYPE_F64: |
return reg.data.f64 == static_cast<double>(i); |
default: |
return false; |
} |
} |
bool |
ImmediateValue::isNegative() const |
{ |
switch (reg.type) { |
case TYPE_S8: return reg.data.s8 < 0; |
case TYPE_S16: return reg.data.s16 < 0; |
case TYPE_S32: |
case TYPE_U32: return reg.data.s32 < 0; |
case TYPE_F32: return reg.data.u32 & (1 << 31); |
case TYPE_F64: return reg.data.u64 & (1ULL << 63); |
default: |
return false; |
} |
} |
bool |
ImmediateValue::isPow2() const |
{ |
switch (reg.type) { |
case TYPE_U8: |
case TYPE_U16: |
case TYPE_U32: return util_is_power_of_two(reg.data.u32); |
default: |
return false; |
} |
} |
void |
ImmediateValue::applyLog2() |
{ |
switch (reg.type) { |
case TYPE_S8: |
case TYPE_S16: |
case TYPE_S32: |
assert(!this->isNegative()); |
// fall through |
case TYPE_U8: |
case TYPE_U16: |
case TYPE_U32: |
reg.data.u32 = util_logbase2(reg.data.u32); |
break; |
case TYPE_F32: |
reg.data.f32 = log2f(reg.data.f32); |
break; |
case TYPE_F64: |
reg.data.f64 = log2(reg.data.f64); |
break; |
default: |
assert(0); |
break; |
} |
} |
bool |
ImmediateValue::compare(CondCode cc, float fval) const |
{ |
if (reg.type != TYPE_F32) |
ERROR("immediate value is not of type f32"); |
switch (static_cast<CondCode>(cc & 7)) { |
case CC_TR: return true; |
case CC_FL: return false; |
case CC_LT: return reg.data.f32 < fval; |
case CC_LE: return reg.data.f32 <= fval; |
case CC_GT: return reg.data.f32 > fval; |
case CC_GE: return reg.data.f32 >= fval; |
case CC_EQ: return reg.data.f32 == fval; |
case CC_NE: return reg.data.f32 != fval; |
default: |
assert(0); |
return false; |
} |
} |
ImmediateValue& |
ImmediateValue::operator=(const ImmediateValue &that) |
{ |
this->reg = that.reg; |
return (*this); |
} |
bool |
Value::interfers(const Value *that) const |
{ |
uint32_t idA, idB; |
if (that->reg.file != reg.file || that->reg.fileIndex != reg.fileIndex) |
return false; |
if (this->asImm()) |
return false; |
if (this->asSym()) { |
idA = this->join->reg.data.offset; |
idB = that->join->reg.data.offset; |
} else { |
idA = this->join->reg.data.id * MIN2(this->reg.size, 4); |
idB = that->join->reg.data.id * MIN2(that->reg.size, 4); |
} |
if (idA < idB) |
return (idA + this->reg.size > idB); |
else |
if (idA > idB) |
return (idB + that->reg.size > idA); |
else |
return (idA == idB); |
} |
bool |
Value::equals(const Value *that, bool strict) const |
{ |
if (strict) |
return this == that; |
if (that->reg.file != reg.file || that->reg.fileIndex != reg.fileIndex) |
return false; |
if (that->reg.size != this->reg.size) |
return false; |
if (that->reg.data.id != this->reg.data.id) |
return false; |
return true; |
} |
bool |
ImmediateValue::equals(const Value *that, bool strict) const |
{ |
const ImmediateValue *imm = that->asImm(); |
if (!imm) |
return false; |
return reg.data.u64 == imm->reg.data.u64; |
} |
bool |
Symbol::equals(const Value *that, bool strict) const |
{ |
if (reg.file != that->reg.file || reg.fileIndex != that->reg.fileIndex) |
return false; |
assert(that->asSym()); |
if (this->baseSym != that->asSym()->baseSym) |
return false; |
if (reg.file == FILE_SYSTEM_VALUE) |
return (this->reg.data.sv.sv == that->reg.data.sv.sv && |
this->reg.data.sv.index == that->reg.data.sv.index); |
return this->reg.data.offset == that->reg.data.offset; |
} |
void Instruction::init() |
{ |
next = prev = 0; |
cc = CC_ALWAYS; |
rnd = ROUND_N; |
cache = CACHE_CA; |
subOp = 0; |
saturate = 0; |
join = 0; |
exit = 0; |
terminator = 0; |
ftz = 0; |
dnz = 0; |
perPatch = 0; |
fixed = 0; |
encSize = 0; |
ipa = 0; |
mask = 0; |
lanes = 0xf; |
postFactor = 0; |
predSrc = -1; |
flagsDef = -1; |
flagsSrc = -1; |
} |
Instruction::Instruction() |
{ |
init(); |
op = OP_NOP; |
dType = sType = TYPE_F32; |
id = -1; |
bb = 0; |
} |
Instruction::Instruction(Function *fn, operation opr, DataType ty) |
{ |
init(); |
op = opr; |
dType = sType = ty; |
fn->add(this, id); |
} |
Instruction::~Instruction() |
{ |
if (bb) { |
Function *fn = bb->getFunction(); |
bb->remove(this); |
fn->allInsns.remove(id); |
} |
for (int s = 0; srcExists(s); ++s) |
setSrc(s, NULL); |
// must unlink defs too since the list pointers will get deallocated |
for (int d = 0; defExists(d); ++d) |
setDef(d, NULL); |
} |
void |
Instruction::setDef(int i, Value *val) |
{ |
int size = defs.size(); |
if (i >= size) { |
defs.resize(i + 1); |
while (size <= i) |
defs[size++].setInsn(this); |
} |
defs[i].set(val); |
} |
void |
Instruction::setSrc(int s, Value *val) |
{ |
int size = srcs.size(); |
if (s >= size) { |
srcs.resize(s + 1); |
while (size <= s) |
srcs[size++].setInsn(this); |
} |
srcs[s].set(val); |
} |
void |
Instruction::setSrc(int s, const ValueRef& ref) |
{ |
setSrc(s, ref.get()); |
srcs[s].mod = ref.mod; |
} |
void |
Instruction::swapSources(int a, int b) |
{ |
Value *value = srcs[a].get(); |
Modifier m = srcs[a].mod; |
setSrc(a, srcs[b]); |
srcs[b].set(value); |
srcs[b].mod = m; |
} |
static inline void moveSourcesAdjustIndex(int8_t &index, int s, int delta) |
{ |
if (index >= s) |
index += delta; |
else |
if ((delta < 0) && (index >= (s + delta))) |
index = -1; |
} |
// Moves sources [@s,last_source] by @delta. |
// If @delta < 0, sources [@s - abs(@delta), @s) are erased. |
void |
Instruction::moveSources(const int s, const int delta) |
{ |
if (delta == 0) |
return; |
assert(s + delta >= 0); |
int k; |
for (k = 0; srcExists(k); ++k) { |
for (int i = 0; i < 2; ++i) |
moveSourcesAdjustIndex(src(k).indirect[i], s, delta); |
} |
moveSourcesAdjustIndex(predSrc, s, delta); |
moveSourcesAdjustIndex(flagsSrc, s, delta); |
if (asTex()) { |
TexInstruction *tex = asTex(); |
moveSourcesAdjustIndex(tex->tex.rIndirectSrc, s, delta); |
moveSourcesAdjustIndex(tex->tex.sIndirectSrc, s, delta); |
} |
if (delta > 0) { |
--k; |
for (int p = k + delta; k >= s; --k, --p) |
setSrc(p, src(k)); |
} else { |
int p; |
for (p = s; p < k; ++p) |
setSrc(p + delta, src(p)); |
for (; (p + delta) < k; ++p) |
setSrc(p + delta, NULL); |
} |
} |
void |
Instruction::takeExtraSources(int s, Value *values[3]) |
{ |
values[0] = getIndirect(s, 0); |
if (values[0]) |
setIndirect(s, 0, NULL); |
values[1] = getIndirect(s, 1); |
if (values[1]) |
setIndirect(s, 1, NULL); |
values[2] = getPredicate(); |
if (values[2]) |
setPredicate(cc, NULL); |
} |
void |
Instruction::putExtraSources(int s, Value *values[3]) |
{ |
if (values[0]) |
setIndirect(s, 0, values[0]); |
if (values[1]) |
setIndirect(s, 1, values[1]); |
if (values[2]) |
setPredicate(cc, values[2]); |
} |
Instruction * |
Instruction::clone(ClonePolicy<Function>& pol, Instruction *i) const |
{ |
if (!i) |
i = new_Instruction(pol.context(), op, dType); |
#ifndef NDEBUG // non-conformant assert, so this is required |
assert(typeid(*i) == typeid(*this)); |
#endif |
pol.set<Instruction>(this, i); |
i->sType = sType; |
i->rnd = rnd; |
i->cache = cache; |
i->subOp = subOp; |
i->saturate = saturate; |
i->join = join; |
i->exit = exit; |
i->mask = mask; |
i->ftz = ftz; |
i->dnz = dnz; |
i->ipa = ipa; |
i->lanes = lanes; |
i->perPatch = perPatch; |
i->postFactor = postFactor; |
for (int d = 0; defExists(d); ++d) |
i->setDef(d, pol.get(getDef(d))); |
for (int s = 0; srcExists(s); ++s) { |
i->setSrc(s, pol.get(getSrc(s))); |
i->src(s).mod = src(s).mod; |
} |
i->cc = cc; |
i->predSrc = predSrc; |
i->flagsDef = flagsDef; |
i->flagsSrc = flagsSrc; |
return i; |
} |
unsigned int |
Instruction::defCount(unsigned int mask, bool singleFile) const |
{ |
unsigned int i, n; |
if (singleFile) { |
unsigned int d = ffs(mask); |
if (!d) |
return 0; |
for (i = d--; defExists(i); ++i) |
if (getDef(i)->reg.file != getDef(d)->reg.file) |
mask &= ~(1 << i); |
} |
for (n = 0, i = 0; this->defExists(i); ++i, mask >>= 1) |
n += mask & 1; |
return n; |
} |
unsigned int |
Instruction::srcCount(unsigned int mask, bool singleFile) const |
{ |
unsigned int i, n; |
if (singleFile) { |
unsigned int s = ffs(mask); |
if (!s) |
return 0; |
for (i = s--; srcExists(i); ++i) |
if (getSrc(i)->reg.file != getSrc(s)->reg.file) |
mask &= ~(1 << i); |
} |
for (n = 0, i = 0; this->srcExists(i); ++i, mask >>= 1) |
n += mask & 1; |
return n; |
} |
bool |
Instruction::setIndirect(int s, int dim, Value *value) |
{ |
assert(this->srcExists(s)); |
int p = srcs[s].indirect[dim]; |
if (p < 0) { |
if (!value) |
return true; |
p = srcs.size(); |
while (p > 0 && !srcExists(p - 1)) |
--p; |
} |
setSrc(p, value); |
srcs[p].usedAsPtr = (value != 0); |
srcs[s].indirect[dim] = value ? p : -1; |
return true; |
} |
bool |
Instruction::setPredicate(CondCode ccode, Value *value) |
{ |
cc = ccode; |
if (!value) { |
if (predSrc >= 0) { |
srcs[predSrc].set(NULL); |
predSrc = -1; |
} |
return true; |
} |
if (predSrc < 0) { |
predSrc = srcs.size(); |
while (predSrc > 0 && !srcExists(predSrc - 1)) |
--predSrc; |
} |
setSrc(predSrc, value); |
return true; |
} |
bool |
Instruction::writesPredicate() const |
{ |
for (int d = 0; defExists(d); ++d) |
if (getDef(d)->inFile(FILE_PREDICATE) || getDef(d)->inFile(FILE_FLAGS)) |
return true; |
return false; |
} |
static bool |
insnCheckCommutationDefSrc(const Instruction *a, const Instruction *b) |
{ |
for (int d = 0; a->defExists(d); ++d) |
for (int s = 0; b->srcExists(s); ++s) |
if (a->getDef(d)->interfers(b->getSrc(s))) |
return false; |
return true; |
} |
static bool |
insnCheckCommutationDefDef(const Instruction *a, const Instruction *b) |
{ |
for (int d = 0; a->defExists(d); ++d) |
for (int c = 0; b->defExists(c); ++c) |
if (a->getDef(d)->interfers(b->getDef(c))) |
return false; |
return true; |
} |
bool |
Instruction::isCommutationLegal(const Instruction *i) const |
{ |
bool ret = insnCheckCommutationDefDef(this, i); |
ret = ret && insnCheckCommutationDefSrc(this, i); |
ret = ret && insnCheckCommutationDefSrc(i, this); |
return ret; |
} |
TexInstruction::TexInstruction(Function *fn, operation op) |
: Instruction(fn, op, TYPE_F32) |
{ |
memset(&tex, 0, sizeof(tex)); |
tex.rIndirectSrc = -1; |
tex.sIndirectSrc = -1; |
} |
TexInstruction::~TexInstruction() |
{ |
for (int c = 0; c < 3; ++c) { |
dPdx[c].set(NULL); |
dPdy[c].set(NULL); |
} |
} |
TexInstruction * |
TexInstruction::clone(ClonePolicy<Function>& pol, Instruction *i) const |
{ |
TexInstruction *tex = (i ? static_cast<TexInstruction *>(i) : |
new_TexInstruction(pol.context(), op)); |
Instruction::clone(pol, tex); |
tex->tex = this->tex; |
if (op == OP_TXD) { |
for (unsigned int c = 0; c < tex->tex.target.getDim(); ++c) { |
tex->dPdx[c].set(dPdx[c]); |
tex->dPdy[c].set(dPdy[c]); |
} |
} |
return tex; |
} |
const struct TexInstruction::Target::Desc TexInstruction::Target::descTable[] = |
{ |
{ "1D", 1, 1, false, false, false }, |
{ "2D", 2, 2, false, false, false }, |
{ "2D_MS", 2, 3, false, false, false }, |
{ "3D", 3, 3, false, false, false }, |
{ "CUBE", 2, 3, false, true, false }, |
{ "1D_SHADOW", 1, 1, false, false, true }, |
{ "2D_SHADOW", 2, 2, false, false, true }, |
{ "CUBE_SHADOW", 2, 3, false, true, true }, |
{ "1D_ARRAY", 1, 2, true, false, false }, |
{ "2D_ARRAY", 2, 3, true, false, false }, |
{ "2D_MS_ARRAY", 2, 4, true, false, false }, |
{ "CUBE_ARRAY", 2, 4, true, true, false }, |
{ "1D_ARRAY_SHADOW", 1, 2, true, false, true }, |
{ "2D_ARRAY_SHADOW", 2, 3, true, false, true }, |
{ "RECT", 2, 2, false, false, false }, |
{ "RECT_SHADOW", 2, 2, false, false, true }, |
{ "CUBE_ARRAY_SHADOW", 2, 4, true, true, true }, |
{ "BUFFER", 1, 1, false, false, false }, |
}; |
void |
TexInstruction::setIndirectR(Value *v) |
{ |
int p = ((tex.rIndirectSrc < 0) && v) ? srcs.size() : tex.rIndirectSrc; |
if (p >= 0) { |
tex.rIndirectSrc = p; |
setSrc(p, v); |
srcs[p].usedAsPtr = !!v; |
} |
} |
void |
TexInstruction::setIndirectS(Value *v) |
{ |
int p = ((tex.sIndirectSrc < 0) && v) ? srcs.size() : tex.sIndirectSrc; |
if (p >= 0) { |
tex.sIndirectSrc = p; |
setSrc(p, v); |
srcs[p].usedAsPtr = !!v; |
} |
} |
CmpInstruction::CmpInstruction(Function *fn, operation op) |
: Instruction(fn, op, TYPE_F32) |
{ |
setCond = CC_ALWAYS; |
} |
CmpInstruction * |
CmpInstruction::clone(ClonePolicy<Function>& pol, Instruction *i) const |
{ |
CmpInstruction *cmp = (i ? static_cast<CmpInstruction *>(i) : |
new_CmpInstruction(pol.context(), op)); |
cmp->dType = dType; |
Instruction::clone(pol, cmp); |
cmp->setCond = setCond; |
return cmp; |
} |
FlowInstruction::FlowInstruction(Function *fn, operation op, void *targ) |
: Instruction(fn, op, TYPE_NONE) |
{ |
if (op == OP_CALL) |
target.fn = reinterpret_cast<Function *>(targ); |
else |
target.bb = reinterpret_cast<BasicBlock *>(targ); |
if (op == OP_BRA || |
op == OP_CONT || op == OP_BREAK || |
op == OP_RET || op == OP_EXIT) |
terminator = 1; |
else |
if (op == OP_JOIN) |
terminator = targ ? 1 : 0; |
allWarp = absolute = limit = builtin = indirect = 0; |
} |
FlowInstruction * |
FlowInstruction::clone(ClonePolicy<Function>& pol, Instruction *i) const |
{ |
FlowInstruction *flow = (i ? static_cast<FlowInstruction *>(i) : |
new_FlowInstruction(pol.context(), op, NULL)); |
Instruction::clone(pol, flow); |
flow->allWarp = allWarp; |
flow->absolute = absolute; |
flow->limit = limit; |
flow->builtin = builtin; |
if (builtin) |
flow->target.builtin = target.builtin; |
else |
if (op == OP_CALL) |
flow->target.fn = target.fn; |
else |
if (target.bb) |
flow->target.bb = pol.get<BasicBlock>(target.bb); |
return flow; |
} |
Program::Program(Type type, Target *arch) |
: progType(type), |
target(arch), |
mem_Instruction(sizeof(Instruction), 6), |
mem_CmpInstruction(sizeof(CmpInstruction), 4), |
mem_TexInstruction(sizeof(TexInstruction), 4), |
mem_FlowInstruction(sizeof(FlowInstruction), 4), |
mem_LValue(sizeof(LValue), 8), |
mem_Symbol(sizeof(Symbol), 7), |
mem_ImmediateValue(sizeof(ImmediateValue), 7) |
{ |
code = NULL; |
binSize = 0; |
maxGPR = -1; |
main = new Function(this, "MAIN", ~0); |
calls.insert(&main->call); |
dbgFlags = 0; |
optLevel = 0; |
targetPriv = NULL; |
} |
Program::~Program() |
{ |
for (ArrayList::Iterator it = allFuncs.iterator(); !it.end(); it.next()) |
delete reinterpret_cast<Function *>(it.get()); |
for (ArrayList::Iterator it = allRValues.iterator(); !it.end(); it.next()) |
releaseValue(reinterpret_cast<Value *>(it.get())); |
} |
void Program::releaseInstruction(Instruction *insn) |
{ |
// TODO: make this not suck so much |
insn->~Instruction(); |
if (insn->asCmp()) |
mem_CmpInstruction.release(insn); |
else |
if (insn->asTex()) |
mem_TexInstruction.release(insn); |
else |
if (insn->asFlow()) |
mem_FlowInstruction.release(insn); |
else |
mem_Instruction.release(insn); |
} |
void Program::releaseValue(Value *value) |
{ |
value->~Value(); |
if (value->asLValue()) |
mem_LValue.release(value); |
else |
if (value->asImm()) |
mem_ImmediateValue.release(value); |
else |
if (value->asSym()) |
mem_Symbol.release(value); |
} |
} // namespace nv50_ir |
extern "C" { |
static void |
nv50_ir_init_prog_info(struct nv50_ir_prog_info *info) |
{ |
#if defined(PIPE_SHADER_HULL) && defined(PIPE_SHADER_DOMAIN) |
if (info->type == PIPE_SHADER_HULL || info->type == PIPE_SHADER_DOMAIN) { |
info->prop.tp.domain = PIPE_PRIM_MAX; |
info->prop.tp.outputPrim = PIPE_PRIM_MAX; |
} |
#endif |
if (info->type == PIPE_SHADER_GEOMETRY) { |
info->prop.gp.instanceCount = 1; |
info->prop.gp.maxVertices = 1; |
} |
info->io.clipDistance = 0xff; |
info->io.pointSize = 0xff; |
info->io.instanceId = 0xff; |
info->io.vertexId = 0xff; |
info->io.edgeFlagIn = 0xff; |
info->io.edgeFlagOut = 0xff; |
info->io.fragDepth = 0xff; |
info->io.sampleMask = 0xff; |
info->io.backFaceColor[0] = info->io.backFaceColor[1] = 0xff; |
} |
int |
nv50_ir_generate_code(struct nv50_ir_prog_info *info) |
{ |
int ret = 0; |
nv50_ir::Program::Type type; |
nv50_ir_init_prog_info(info); |
#define PROG_TYPE_CASE(a, b) \ |
case PIPE_SHADER_##a: type = nv50_ir::Program::TYPE_##b; break |
switch (info->type) { |
PROG_TYPE_CASE(VERTEX, VERTEX); |
// PROG_TYPE_CASE(HULL, TESSELLATION_CONTROL); |
// PROG_TYPE_CASE(DOMAIN, TESSELLATION_EVAL); |
PROG_TYPE_CASE(GEOMETRY, GEOMETRY); |
PROG_TYPE_CASE(FRAGMENT, FRAGMENT); |
PROG_TYPE_CASE(COMPUTE, COMPUTE); |
default: |
type = nv50_ir::Program::TYPE_COMPUTE; |
break; |
} |
INFO_DBG(info->dbgFlags, VERBOSE, "translating program of type %u\n", type); |
nv50_ir::Target *targ = nv50_ir::Target::create(info->target); |
if (!targ) |
return -1; |
nv50_ir::Program *prog = new nv50_ir::Program(type, targ); |
if (!prog) |
return -1; |
prog->driver = info; |
prog->dbgFlags = info->dbgFlags; |
prog->optLevel = info->optLevel; |
switch (info->bin.sourceRep) { |
#if 0 |
case PIPE_IR_LLVM: |
case PIPE_IR_GLSL: |
return -1; |
case PIPE_IR_SM4: |
ret = prog->makeFromSM4(info) ? 0 : -2; |
break; |
case PIPE_IR_TGSI: |
#endif |
default: |
ret = prog->makeFromTGSI(info) ? 0 : -2; |
break; |
} |
if (ret < 0) |
goto out; |
if (prog->dbgFlags & NV50_IR_DEBUG_VERBOSE) |
prog->print(); |
targ->parseDriverInfo(info); |
prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_PRE_SSA); |
prog->convertToSSA(); |
if (prog->dbgFlags & NV50_IR_DEBUG_VERBOSE) |
prog->print(); |
prog->optimizeSSA(info->optLevel); |
prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_SSA); |
if (prog->dbgFlags & NV50_IR_DEBUG_BASIC) |
prog->print(); |
if (!prog->registerAllocation()) { |
ret = -4; |
goto out; |
} |
prog->getTarget()->runLegalizePass(prog, nv50_ir::CG_STAGE_POST_RA); |
prog->optimizePostRA(info->optLevel); |
if (!prog->emitBinary(info)) { |
ret = -5; |
goto out; |
} |
out: |
INFO_DBG(prog->dbgFlags, VERBOSE, "nv50_ir_generate_code: ret = %i\n", ret); |
info->bin.maxGPR = prog->maxGPR; |
info->bin.code = prog->code; |
info->bin.codeSize = prog->binSize; |
info->bin.tlsSpace = prog->tlsSize; |
delete prog; |
nv50_ir::Target::destroy(targ); |
return ret; |
} |
} // extern "C" |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir.h |
---|
0,0 → 1,1197 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_H__ |
#define __NV50_IR_H__ |
#include <stdio.h> |
#include <stdlib.h> |
#include <stdint.h> |
#include <deque> |
#include <list> |
#include <vector> |
#include "nv50_ir_util.h" |
#include "nv50_ir_graph.h" |
#include "nv50_ir_driver.h" |
namespace nv50_ir { |
enum operation |
{ |
OP_NOP = 0, |
OP_PHI, |
OP_UNION, // unify a new definition and several source values |
OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced) |
OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value |
OP_CONSTRAINT, // copy values into consecutive registers |
OP_MOV, // simple copy, no modifiers allowed |
OP_LOAD, |
OP_STORE, |
OP_ADD, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds |
OP_SUB, |
OP_MUL, |
OP_DIV, |
OP_MOD, |
OP_MAD, |
OP_FMA, |
OP_SAD, // abs(src0 - src1) + src2 |
OP_ABS, |
OP_NEG, |
OP_NOT, |
OP_AND, |
OP_OR, |
OP_XOR, |
OP_SHL, |
OP_SHR, |
OP_MAX, |
OP_MIN, |
OP_SAT, // CLAMP(f32, 0.0, 1.0) |
OP_CEIL, |
OP_FLOOR, |
OP_TRUNC, |
OP_CVT, |
OP_SET_AND, // dst = (src0 CMP src1) & src2 |
OP_SET_OR, |
OP_SET_XOR, |
OP_SET, |
OP_SELP, // dst = src2 ? src0 : src1 |
OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1 |
OP_RCP, |
OP_RSQ, |
OP_LG2, |
OP_SIN, |
OP_COS, |
OP_EX2, |
OP_EXP, // exponential (base M_E) |
OP_LOG, // natural logarithm |
OP_PRESIN, |
OP_PREEX2, |
OP_SQRT, |
OP_POW, |
OP_BRA, |
OP_CALL, |
OP_RET, |
OP_CONT, |
OP_BREAK, |
OP_PRERET, |
OP_PRECONT, |
OP_PREBREAK, |
OP_BRKPT, // breakpoint (not related to loops) |
OP_JOINAT, // push control flow convergence point |
OP_JOIN, // converge |
OP_DISCARD, |
OP_EXIT, |
OP_MEMBAR, // memory barrier (mfence, lfence, sfence) |
OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base |
OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1] |
OP_EXPORT, |
OP_LINTERP, |
OP_PINTERP, |
OP_EMIT, // emit vertex |
OP_RESTART, // restart primitive |
OP_TEX, |
OP_TXB, // texture bias |
OP_TXL, // texure lod |
OP_TXF, // texel fetch |
OP_TXQ, // texture size query |
OP_TXD, // texture derivatives |
OP_TXG, // texture gather |
OP_TEXCSAA, // texture op for coverage sampling |
OP_TEXPREP, // turn cube map array into 2d array coordinates |
OP_SULDB, // surface load (raw) |
OP_SULDP, // surface load (formatted) |
OP_SUSTB, // surface store (raw) |
OP_SUSTP, // surface store (formatted) |
OP_SUREDB, |
OP_SUREDP, // surface reduction (atomic op) |
OP_SULEA, // surface load effective address |
OP_SUBFM, // surface bitfield manipulation |
OP_SUCLAMP, // clamp surface coordinates |
OP_SUEAU, // surface effective address |
OP_MADSP, // special integer multiply-add |
OP_TEXBAR, // texture dependency barrier |
OP_DFDX, |
OP_DFDY, |
OP_RDSV, // read system value |
OP_WRSV, // write system value |
OP_QUADOP, |
OP_QUADON, |
OP_QUADPOP, |
OP_POPCNT, // bitcount(src0 & src1) |
OP_INSBF, // insert first src1[8:15] bits of src0 into src2 at src1[0:7] |
OP_EXTBF, // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK |
OP_PERMT, // dst = bytes from src2,src0 selected by src1 (nvc0's src order) |
OP_ATOM, |
OP_BAR, // execution barrier, sources = { id, thread count, predicate } |
OP_VADD, // byte/word vector operations |
OP_VAVG, |
OP_VMIN, |
OP_VMAX, |
OP_VSAD, |
OP_VSET, |
OP_VSHR, |
OP_VSHL, |
OP_VSEL, |
OP_CCTL, // cache control |
OP_LAST |
}; |
// various instruction-specific modifier definitions Instruction::subOp |
// MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs) |
#define NV50_IR_SUBOP_MUL_HIGH 1 |
#define NV50_IR_SUBOP_EMIT_RESTART 1 |
#define NV50_IR_SUBOP_LDC_IL 1 |
#define NV50_IR_SUBOP_LDC_IS 2 |
#define NV50_IR_SUBOP_LDC_ISL 3 |
#define NV50_IR_SUBOP_SHIFT_WRAP 1 |
#define NV50_IR_SUBOP_EMU_PRERET 1 |
#define NV50_IR_SUBOP_TEXBAR(n) n |
#define NV50_IR_SUBOP_MOV_FINAL 1 |
#define NV50_IR_SUBOP_EXTBF_REV 1 |
#define NV50_IR_SUBOP_PERMT_F4E 1 |
#define NV50_IR_SUBOP_PERMT_B4E 2 |
#define NV50_IR_SUBOP_PERMT_RC8 3 |
#define NV50_IR_SUBOP_PERMT_ECL 4 |
#define NV50_IR_SUBOP_PERMT_ECR 5 |
#define NV50_IR_SUBOP_PERMT_RC16 6 |
#define NV50_IR_SUBOP_BAR_SYNC 0 |
#define NV50_IR_SUBOP_BAR_ARRIVE 1 |
#define NV50_IR_SUBOP_BAR_RED_AND 2 |
#define NV50_IR_SUBOP_BAR_RED_OR 3 |
#define NV50_IR_SUBOP_BAR_RED_POPC 4 |
#define NV50_IR_SUBOP_MEMBAR_L 1 |
#define NV50_IR_SUBOP_MEMBAR_S 2 |
#define NV50_IR_SUBOP_MEMBAR_M 3 |
#define NV50_IR_SUBOP_MEMBAR_CTA (0 << 2) |
#define NV50_IR_SUBOP_MEMBAR_GL (1 << 2) |
#define NV50_IR_SUBOP_MEMBAR_SYS (2 << 2) |
#define NV50_IR_SUBOP_MEMBAR_DIR(m) ((m) & 0x3) |
#define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3) |
#define NV50_IR_SUBOP_MEMBAR(d,s) \ |
(NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s) |
#define NV50_IR_SUBOP_ATOM_ADD 0 |
#define NV50_IR_SUBOP_ATOM_MIN 1 |
#define NV50_IR_SUBOP_ATOM_MAX 2 |
#define NV50_IR_SUBOP_ATOM_INC 3 |
#define NV50_IR_SUBOP_ATOM_DEC 4 |
#define NV50_IR_SUBOP_ATOM_AND 5 |
#define NV50_IR_SUBOP_ATOM_OR 6 |
#define NV50_IR_SUBOP_ATOM_XOR 7 |
#define NV50_IR_SUBOP_ATOM_CAS 8 |
#define NV50_IR_SUBOP_ATOM_EXCH 9 |
#define NV50_IR_SUBOP_CCTL_IV 5 |
#define NV50_IR_SUBOP_CCTL_IVALL 6 |
#define NV50_IR_SUBOP_SUST_IGN 0 |
#define NV50_IR_SUBOP_SUST_TRAP 1 |
#define NV50_IR_SUBOP_SUST_SDCL 3 |
#define NV50_IR_SUBOP_SULD_ZERO 0 |
#define NV50_IR_SUBOP_SULD_TRAP 1 |
#define NV50_IR_SUBOP_SULD_SDCL 3 |
#define NV50_IR_SUBOP_SUBFM_3D 1 |
#define NV50_IR_SUBOP_SUCLAMP_2D 0x10 |
#define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0)) |
#define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0)) |
#define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0)) |
#define NV50_IR_SUBOP_MADSP_SD 0xffff |
// Yes, we could represent those with DataType. |
// Or put the type into operation and have a couple 1000 values in that enum. |
// This will have to do for now. |
// The bitfields are supposed to correspond to nve4 ISA. |
#define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a)) |
#define NV50_IR_SUBOP_V1(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x0000) |
#define NV50_IR_SUBOP_V2(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x4000) |
#define NV50_IR_SUBOP_V4(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x8000) |
#define NV50_IR_SUBOP_Vn(n) ((n) >> 14) |
enum DataType |
{ |
TYPE_NONE, |
TYPE_U8, |
TYPE_S8, |
TYPE_U16, |
TYPE_S16, |
TYPE_U32, |
TYPE_S32, |
TYPE_U64, // 64 bit operations are only lowered after register allocation |
TYPE_S64, |
TYPE_F16, |
TYPE_F32, |
TYPE_F64, |
TYPE_B96, |
TYPE_B128 |
}; |
enum CondCode |
{ |
CC_FL = 0, |
CC_NEVER = CC_FL, // when used with FILE_FLAGS |
CC_LT = 1, |
CC_EQ = 2, |
CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE |
CC_LE = 3, |
CC_GT = 4, |
CC_NE = 5, |
CC_P = CC_NE, |
CC_GE = 6, |
CC_TR = 7, |
CC_ALWAYS = CC_TR, |
CC_U = 8, |
CC_LTU = 9, |
CC_EQU = 10, |
CC_LEU = 11, |
CC_GTU = 12, |
CC_NEU = 13, |
CC_GEU = 14, |
CC_NO = 0x10, |
CC_NC = 0x11, |
CC_NS = 0x12, |
CC_NA = 0x13, |
CC_A = 0x14, |
CC_S = 0x15, |
CC_C = 0x16, |
CC_O = 0x17 |
}; |
enum RoundMode |
{ |
ROUND_N, // nearest |
ROUND_M, // towards -inf |
ROUND_Z, // towards 0 |
ROUND_P, // towards +inf |
ROUND_NI, // nearest integer |
ROUND_MI, // to integer towards -inf |
ROUND_ZI, // to integer towards 0 |
ROUND_PI, // to integer towards +inf |
}; |
enum CacheMode |
{ |
CACHE_CA, // cache at all levels |
CACHE_WB = CACHE_CA, // cache write back |
CACHE_CG, // cache at global level |
CACHE_CS, // cache streaming |
CACHE_CV, // cache as volatile |
CACHE_WT = CACHE_CV // cache write-through |
}; |
enum DataFile |
{ |
FILE_NULL = 0, |
FILE_GPR, |
FILE_PREDICATE, // boolean predicate |
FILE_FLAGS, // zero/sign/carry/overflow bits |
FILE_ADDRESS, |
LAST_REGISTER_FILE = FILE_ADDRESS, |
FILE_IMMEDIATE, |
FILE_MEMORY_CONST, |
FILE_SHADER_INPUT, |
FILE_SHADER_OUTPUT, |
FILE_MEMORY_GLOBAL, |
FILE_MEMORY_SHARED, |
FILE_MEMORY_LOCAL, |
FILE_SYSTEM_VALUE, |
DATA_FILE_COUNT |
}; |
enum TexTarget |
{ |
TEX_TARGET_1D, |
TEX_TARGET_2D, |
TEX_TARGET_2D_MS, |
TEX_TARGET_3D, |
TEX_TARGET_CUBE, |
TEX_TARGET_1D_SHADOW, |
TEX_TARGET_2D_SHADOW, |
TEX_TARGET_CUBE_SHADOW, |
TEX_TARGET_1D_ARRAY, |
TEX_TARGET_2D_ARRAY, |
TEX_TARGET_2D_MS_ARRAY, |
TEX_TARGET_CUBE_ARRAY, |
TEX_TARGET_1D_ARRAY_SHADOW, |
TEX_TARGET_2D_ARRAY_SHADOW, |
TEX_TARGET_RECT, |
TEX_TARGET_RECT_SHADOW, |
TEX_TARGET_CUBE_ARRAY_SHADOW, |
TEX_TARGET_BUFFER, |
TEX_TARGET_COUNT |
}; |
enum SVSemantic |
{ |
SV_POSITION, // WPOS |
SV_VERTEX_ID, |
SV_INSTANCE_ID, |
SV_INVOCATION_ID, |
SV_PRIMITIVE_ID, |
SV_VERTEX_COUNT, // gl_PatchVerticesIn |
SV_LAYER, |
SV_VIEWPORT_INDEX, |
SV_YDIR, |
SV_FACE, |
SV_POINT_SIZE, |
SV_POINT_COORD, |
SV_CLIP_DISTANCE, |
SV_SAMPLE_INDEX, |
SV_TESS_FACTOR, |
SV_TESS_COORD, |
SV_TID, |
SV_CTAID, |
SV_NTID, |
SV_GRIDID, |
SV_NCTAID, |
SV_LANEID, |
SV_PHYSID, |
SV_NPHYSID, |
SV_CLOCK, |
SV_LBASE, |
SV_SBASE, |
SV_UNDEFINED, |
SV_LAST |
}; |
class Program; |
class Function; |
class BasicBlock; |
class Target; |
class Instruction; |
class CmpInstruction; |
class TexInstruction; |
class FlowInstruction; |
class Value; |
class LValue; |
class Symbol; |
class ImmediateValue; |
struct Storage |
{ |
DataFile file; |
int8_t fileIndex; // signed, may be indirect for CONST[] |
uint8_t size; // this should match the Instruction type's size |
DataType type; // mainly for pretty printing |
union { |
uint64_t u64; // immediate values |
uint32_t u32; |
uint16_t u16; |
uint8_t u8; |
int64_t s64; |
int32_t s32; |
int16_t s16; |
int8_t s8; |
float f32; |
double f64; |
int32_t offset; // offset from 0 (base of address space) |
int32_t id; // register id (< 0 if virtual/unassigned, in units <= 4) |
struct { |
SVSemantic sv; |
int index; |
} sv; |
} data; |
}; |
// precedence: NOT after SAT after NEG after ABS |
#define NV50_IR_MOD_ABS (1 << 0) |
#define NV50_IR_MOD_NEG (1 << 1) |
#define NV50_IR_MOD_SAT (1 << 2) |
#define NV50_IR_MOD_NOT (1 << 3) |
#define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS) |
#define NV50_IR_INTERP_MODE_MASK 0x3 |
#define NV50_IR_INTERP_LINEAR (0 << 0) |
#define NV50_IR_INTERP_PERSPECTIVE (1 << 0) |
#define NV50_IR_INTERP_FLAT (2 << 0) |
#define NV50_IR_INTERP_SC (3 << 0) // what exactly is that ? |
#define NV50_IR_INTERP_SAMPLE_MASK 0xc |
#define NV50_IR_INTERP_DEFAULT (0 << 2) |
#define NV50_IR_INTERP_CENTROID (1 << 2) |
#define NV50_IR_INTERP_OFFSET (2 << 2) |
#define NV50_IR_INTERP_SAMPLEID (3 << 2) |
// do we really want this to be a class ? |
class Modifier |
{ |
public: |
Modifier() : bits(0) { } |
Modifier(unsigned int m) : bits(m) { } |
Modifier(operation op); |
// @return new Modifier applying a after b (asserts if unrepresentable) |
Modifier operator*(const Modifier) const; |
Modifier operator*=(const Modifier m) { *this = *this * m; return *this; } |
Modifier operator==(const Modifier m) const { return m.bits == bits; } |
Modifier operator!=(const Modifier m) const { return m.bits != bits; } |
inline Modifier operator&(const Modifier m) const { return bits & m.bits; } |
inline Modifier operator|(const Modifier m) const { return bits | m.bits; } |
inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; } |
operation getOp() const; |
inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; } |
inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; } |
inline operator bool() const { return bits ? true : false; } |
void applyTo(ImmediateValue &imm) const; |
int print(char *buf, size_t size) const; |
private: |
uint8_t bits; |
}; |
class ValueRef |
{ |
public: |
ValueRef(Value * = NULL); |
ValueRef(const ValueRef&); |
~ValueRef(); |
inline bool exists() const { return value != NULL; } |
void set(Value *); |
void set(const ValueRef&); |
inline Value *get() const { return value; } |
inline Value *rep() const; |
inline Instruction *getInsn() const { return insn; } |
inline void setInsn(Instruction *inst) { insn = inst; } |
inline bool isIndirect(int dim) const { return indirect[dim] >= 0; } |
inline const ValueRef *getIndirect(int dim) const; |
inline DataFile getFile() const; |
inline unsigned getSize() const; |
// SSA: return eventual (traverse MOVs) literal value, if it exists |
bool getImmediate(ImmediateValue&) const; |
public: |
Modifier mod; |
int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i]) |
uint8_t swizzle; |
bool usedAsPtr; // for printing |
private: |
Value *value; |
Instruction *insn; |
}; |
class ValueDef |
{ |
public: |
ValueDef(Value * = NULL); |
ValueDef(const ValueDef&); |
~ValueDef(); |
inline bool exists() const { return value != NULL; } |
inline Value *get() const { return value; } |
inline Value *rep() const; |
void set(Value *); |
bool mayReplace(const ValueRef &); |
void replace(const ValueRef &, bool doSet); // replace all uses of the old value |
inline Instruction *getInsn() const { return insn; } |
inline void setInsn(Instruction *inst) { insn = inst; } |
inline DataFile getFile() const; |
inline unsigned getSize() const; |
inline void setSSA(LValue *); |
inline const LValue *preSSA() const; |
private: |
Value *value; // should make this LValue * ... |
LValue *origin; // pre SSA value |
Instruction *insn; |
}; |
class Value |
{ |
public: |
Value(); |
virtual ~Value() { } |
virtual Value *clone(ClonePolicy<Function>&) const = 0; |
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0; |
virtual bool equals(const Value *, bool strict = false) const; |
virtual bool interfers(const Value *) const; |
virtual bool isUniform() const { return true; } |
inline Value *rep() const { return join; } |
inline Instruction *getUniqueInsn() const; |
inline Instruction *getInsn() const; // use when uniqueness is certain |
inline int refCount() { return uses.size(); } |
inline LValue *asLValue(); |
inline Symbol *asSym(); |
inline ImmediateValue *asImm(); |
inline const Symbol *asSym() const; |
inline const ImmediateValue *asImm() const; |
inline bool inFile(DataFile f) { return reg.file == f; } |
static inline Value *get(Iterator&); |
std::list<ValueRef *> uses; |
std::list<ValueDef *> defs; |
typedef std::list<ValueRef *>::iterator UseIterator; |
typedef std::list<ValueRef *>::const_iterator UseCIterator; |
typedef std::list<ValueDef *>::iterator DefIterator; |
typedef std::list<ValueDef *>::const_iterator DefCIterator; |
int id; |
Storage reg; |
// TODO: these should be in LValue: |
Interval livei; |
Value *join; |
}; |
class LValue : public Value |
{ |
public: |
LValue(Function *, DataFile file); |
LValue(Function *, LValue *); |
~LValue() { } |
virtual bool isUniform() const; |
virtual LValue *clone(ClonePolicy<Function>&) const; |
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const; |
public: |
unsigned compMask : 8; // compound/component mask |
unsigned compound : 1; // used by RA, value involved in split/merge |
unsigned ssa : 1; |
unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0) |
unsigned noSpill : 1; // do not spill (e.g. if spill temporary already) |
}; |
class Symbol : public Value |
{ |
public: |
Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0); |
~Symbol() { } |
virtual Symbol *clone(ClonePolicy<Function>&) const; |
virtual bool equals(const Value *that, bool strict) const; |
virtual bool isUniform() const; |
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const; |
// print with indirect values |
int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const; |
inline void setFile(DataFile file, ubyte fileIndex = 0) |
{ |
reg.file = file; |
reg.fileIndex = fileIndex; |
} |
inline void setOffset(int32_t offset); |
inline void setAddress(Symbol *base, int32_t offset); |
inline void setSV(SVSemantic sv, uint32_t idx = 0); |
inline const Symbol *getBase() const { return baseSym; } |
private: |
Symbol *baseSym; // array base for Symbols representing array elements |
}; |
class ImmediateValue : public Value |
{ |
public: |
ImmediateValue() { } |
ImmediateValue(Program *, uint32_t); |
ImmediateValue(Program *, float); |
ImmediateValue(Program *, double); |
// NOTE: not added to program with |
ImmediateValue(const ImmediateValue *, DataType ty); |
~ImmediateValue() { }; |
virtual ImmediateValue *clone(ClonePolicy<Function>&) const; |
virtual bool equals(const Value *that, bool strict) const; |
// these only work if 'type' is valid (we mostly use untyped literals): |
bool isInteger(const int ival) const; // ival is cast to this' type |
bool isNegative() const; |
bool isPow2() const; |
void applyLog2(); |
// for constant folding: |
ImmediateValue operator+(const ImmediateValue&) const; |
ImmediateValue operator-(const ImmediateValue&) const; |
ImmediateValue operator*(const ImmediateValue&) const; |
ImmediateValue operator/(const ImmediateValue&) const; |
ImmediateValue& operator=(const ImmediateValue&); // only sets value ! |
bool compare(CondCode cc, float fval) const; |
virtual int print(char *, size_t, DataType ty = TYPE_NONE) const; |
}; |
class Instruction |
{ |
public: |
Instruction(); |
Instruction(Function *, operation, DataType); |
virtual ~Instruction(); |
virtual Instruction *clone(ClonePolicy<Function>&, |
Instruction * = NULL) const; |
void setDef(int i, Value *); |
void setSrc(int s, Value *); |
void setSrc(int s, const ValueRef&); |
void swapSources(int a, int b); |
void moveSources(int s, int delta); |
bool setIndirect(int s, int dim, Value *); |
inline ValueRef& src(int s) { return srcs[s]; } |
inline ValueDef& def(int s) { return defs[s]; } |
inline const ValueRef& src(int s) const { return srcs[s]; } |
inline const ValueDef& def(int s) const { return defs[s]; } |
inline Value *getDef(int d) const { return defs[d].get(); } |
inline Value *getSrc(int s) const { return srcs[s].get(); } |
inline Value *getIndirect(int s, int dim) const; |
inline bool defExists(unsigned d) const |
{ |
return d < defs.size() && defs[d].exists(); |
} |
inline bool srcExists(unsigned s) const |
{ |
return s < srcs.size() && srcs[s].exists(); |
} |
inline bool constrainedDefs() const; |
bool setPredicate(CondCode ccode, Value *); |
inline Value *getPredicate() const; |
bool writesPredicate() const; |
inline bool isPredicated() const { return predSrc >= 0; } |
inline void setFlagsSrc(int s, Value *); |
inline void setFlagsDef(int d, Value *); |
inline bool usesFlags() const { return flagsSrc >= 0; } |
unsigned int defCount() const { return defs.size(); }; |
unsigned int defCount(unsigned int mask, bool singleFile = false) const; |
unsigned int srcCount() const { return srcs.size(); }; |
unsigned int srcCount(unsigned int mask, bool singleFile = false) const; |
// save & remove / set indirect[0,1] and predicate source |
void takeExtraSources(int s, Value *[3]); |
void putExtraSources(int s, Value *[3]); |
inline void setType(DataType type) { dType = sType = type; } |
inline void setType(DataType dtype, DataType stype) |
{ |
dType = dtype; |
sType = stype; |
} |
inline bool isPseudo() const { return op < OP_MOV; } |
bool isDead() const; |
bool isNop() const; |
bool isCommutationLegal(const Instruction *) const; // must be adjacent ! |
bool isActionEqual(const Instruction *) const; |
bool isResultEqual(const Instruction *) const; |
void print() const; |
inline CmpInstruction *asCmp(); |
inline TexInstruction *asTex(); |
inline FlowInstruction *asFlow(); |
inline const TexInstruction *asTex() const; |
inline const CmpInstruction *asCmp() const; |
inline const FlowInstruction *asFlow() const; |
public: |
Instruction *next; |
Instruction *prev; |
int id; |
int serial; // CFG order |
operation op; |
DataType dType; // destination or defining type |
DataType sType; // source or secondary type |
CondCode cc; |
RoundMode rnd; |
CacheMode cache; |
uint16_t subOp; // quadop, 1 for mul-high, etc. |
unsigned encSize : 4; // encoding size in bytes |
unsigned saturate : 1; // to [0.0f, 1.0f] |
unsigned join : 1; // converge control flow (use OP_JOIN until end) |
unsigned fixed : 1; // prevent dead code elimination |
unsigned terminator : 1; // end of basic block |
unsigned ftz : 1; // flush denormal to zero |
unsigned dnz : 1; // denormals, NaN are zero |
unsigned ipa : 4; // interpolation mode |
unsigned lanes : 4; |
unsigned perPatch : 1; |
unsigned exit : 1; // terminate program after insn |
unsigned mask : 4; // for vector ops |
int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor |
int8_t predSrc; |
int8_t flagsDef; |
int8_t flagsSrc; |
uint8_t sched; // scheduling data (NOTE: maybe move to separate storage) |
BasicBlock *bb; |
protected: |
std::deque<ValueDef> defs; // no gaps ! |
std::deque<ValueRef> srcs; // no gaps ! |
// instruction specific methods: |
// (don't want to subclass, would need more constructors and memory pools) |
public: |
inline void setInterpolate(unsigned int mode) { ipa = mode; } |
unsigned int getInterpMode() const { return ipa & 0x3; } |
unsigned int getSampleMode() const { return ipa & 0xc; } |
private: |
void init(); |
}; |
enum TexQuery |
{ |
TXQ_DIMS, |
TXQ_TYPE, |
TXQ_SAMPLE_POSITION, |
TXQ_FILTER, |
TXQ_LOD, |
TXQ_WRAP, |
TXQ_BORDER_COLOUR |
}; |
class TexInstruction : public Instruction |
{ |
public: |
class Target |
{ |
public: |
Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { } |
const char *getName() const { return descTable[target].name; } |
unsigned int getArgCount() const { return descTable[target].argc; } |
unsigned int getDim() const { return descTable[target].dim; } |
int isArray() const { return descTable[target].array ? 1 : 0; } |
int isCube() const { return descTable[target].cube ? 1 : 0; } |
int isShadow() const { return descTable[target].shadow ? 1 : 0; } |
int isMS() const { |
return target == TEX_TARGET_2D_MS || target == TEX_TARGET_2D_MS_ARRAY; } |
Target& operator=(TexTarget targ) |
{ |
assert(targ < TEX_TARGET_COUNT); |
target = targ; |
return *this; |
} |
inline bool operator==(TexTarget targ) const { return target == targ; } |
inline bool operator!=(TexTarget targ) const { return target != targ; } |
enum TexTarget getEnum() const { return target; } |
private: |
struct Desc |
{ |
char name[19]; |
uint8_t dim; |
uint8_t argc; |
bool array; |
bool cube; |
bool shadow; |
}; |
static const struct Desc descTable[TEX_TARGET_COUNT]; |
private: |
enum TexTarget target; |
}; |
public: |
TexInstruction(Function *, operation); |
virtual ~TexInstruction(); |
virtual TexInstruction *clone(ClonePolicy<Function>&, |
Instruction * = NULL) const; |
inline void setTexture(Target targ, uint8_t r, uint8_t s) |
{ |
tex.r = r; |
tex.s = s; |
tex.target = targ; |
} |
void setIndirectR(Value *); |
void setIndirectS(Value *); |
inline Value *getIndirectR() const; |
inline Value *getIndirectS() const; |
public: |
struct { |
Target target; |
uint16_t r; |
uint16_t s; |
int8_t rIndirectSrc; |
int8_t sIndirectSrc; |
uint8_t mask; |
uint8_t gatherComp; |
bool liveOnly; // only execute on live pixels of a quad (optimization) |
bool levelZero; |
bool derivAll; |
int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets |
int8_t offset[4][3]; |
enum TexQuery query; |
} tex; |
ValueRef dPdx[3]; |
ValueRef dPdy[3]; |
}; |
class CmpInstruction : public Instruction |
{ |
public: |
CmpInstruction(Function *, operation); |
virtual CmpInstruction *clone(ClonePolicy<Function>&, |
Instruction * = NULL) const; |
void setCondition(CondCode cond) { setCond = cond; } |
CondCode getCondition() const { return setCond; } |
public: |
CondCode setCond; |
}; |
class FlowInstruction : public Instruction |
{ |
public: |
FlowInstruction(Function *, operation, void *target); |
virtual FlowInstruction *clone(ClonePolicy<Function>&, |
Instruction * = NULL) const; |
public: |
unsigned allWarp : 1; |
unsigned absolute : 1; |
unsigned limit : 1; |
unsigned builtin : 1; // true for calls to emulation code |
unsigned indirect : 1; // target in src(0) |
union { |
BasicBlock *bb; |
int builtin; |
Function *fn; |
} target; |
}; |
class BasicBlock |
{ |
public: |
BasicBlock(Function *); |
~BasicBlock(); |
BasicBlock *clone(ClonePolicy<Function>&) const; |
inline int getId() const { return id; } |
inline unsigned int getInsnCount() const { return numInsns; } |
inline bool isTerminated() const { return exit && exit->terminator; } |
bool dominatedBy(BasicBlock *bb); |
inline bool reachableBy(const BasicBlock *by, const BasicBlock *term); |
// returns mask of conditional out blocks |
// e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF |
unsigned int initiatesSimpleConditional() const; |
public: |
Function *getFunction() const { return func; } |
Program *getProgram() const { return program; } |
Instruction *getEntry() const { return entry; } // first non-phi instruction |
Instruction *getPhi() const { return phi; } |
Instruction *getFirst() const { return phi ? phi : entry; } |
Instruction *getExit() const { return exit; } |
void insertHead(Instruction *); |
void insertTail(Instruction *); |
void insertBefore(Instruction *, Instruction *); |
void insertAfter(Instruction *, Instruction *); |
void remove(Instruction *); |
void permuteAdjacent(Instruction *, Instruction *); |
BasicBlock *idom() const; |
// NOTE: currently does not rebuild the dominator tree |
BasicBlock *splitBefore(Instruction *, bool attach = true); |
BasicBlock *splitAfter(Instruction *, bool attach = true); |
DLList& getDF() { return df; } |
DLList::Iterator iterDF() { return df.iterator(); } |
static inline BasicBlock *get(Iterator&); |
static inline BasicBlock *get(Graph::Node *); |
public: |
Graph::Node cfg; // first edge is branch *taken* (the ELSE branch) |
Graph::Node dom; |
BitSet liveSet; |
BitSet defSet; |
uint32_t binPos; |
uint32_t binSize; |
Instruction *joinAt; // for quick reference |
bool explicitCont; // loop headers: true if loop contains continue stmts |
private: |
int id; |
DLList df; |
Instruction *phi; |
Instruction *entry; |
Instruction *exit; |
unsigned int numInsns; |
private: |
Function *func; |
Program *program; |
void splitCommon(Instruction *, BasicBlock *, bool attach); |
}; |
class Function |
{ |
public: |
Function(Program *, const char *name, uint32_t label); |
~Function(); |
static inline Function *get(Graph::Node *node); |
inline Program *getProgram() const { return prog; } |
inline const char *getName() const { return name; } |
inline int getId() const { return id; } |
inline uint32_t getLabel() const { return label; } |
void print(); |
void printLiveIntervals() const; |
void printCFGraph(const char *filePath); |
bool setEntry(BasicBlock *); |
bool setExit(BasicBlock *); |
unsigned int orderInstructions(ArrayList&); |
inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); } |
inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); } |
inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); } |
inline LValue *getLValue(int id); |
void buildLiveSets(); |
void buildDefSets(); |
bool convertToSSA(); |
public: |
std::deque<ValueDef> ins; |
std::deque<ValueRef> outs; |
std::deque<Value *> clobbers; |
Graph cfg; |
Graph::Node *cfgExit; |
Graph *domTree; |
Graph::Node call; // node in the call graph |
BasicBlock **bbArray; // BBs in emission order |
int bbCount; |
unsigned int loopNestingBound; |
int regClobberMax; |
uint32_t binPos; |
uint32_t binSize; |
Value *stackPtr; |
uint32_t tlsBase; // base address for l[] space (if no stack pointer is used) |
uint32_t tlsSize; |
ArrayList allBBlocks; |
ArrayList allInsns; |
ArrayList allLValues; |
private: |
void buildLiveSetsPreSSA(BasicBlock *, const int sequence); |
void buildDefSetsPreSSA(BasicBlock *bb, const int seq); |
private: |
uint32_t label; |
int id; |
const char *const name; |
Program *prog; |
}; |
enum CGStage |
{ |
CG_STAGE_PRE_SSA, |
CG_STAGE_SSA, // expected directly before register allocation |
CG_STAGE_POST_RA |
}; |
class Program |
{ |
public: |
enum Type |
{ |
TYPE_VERTEX, |
TYPE_TESSELLATION_CONTROL, |
TYPE_TESSELLATION_EVAL, |
TYPE_GEOMETRY, |
TYPE_FRAGMENT, |
TYPE_COMPUTE |
}; |
Program(Type type, Target *targ); |
~Program(); |
void print(); |
Type getType() const { return progType; } |
inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); } |
inline void del(Function *fn, int& id) { allFuncs.remove(id); } |
inline void add(Value *rval, int& id) { allRValues.insert(rval, id); } |
bool makeFromTGSI(struct nv50_ir_prog_info *); |
bool makeFromSM4(struct nv50_ir_prog_info *); |
bool convertToSSA(); |
bool optimizeSSA(int level); |
bool optimizePostRA(int level); |
bool registerAllocation(); |
bool emitBinary(struct nv50_ir_prog_info *); |
const Target *getTarget() const { return target; } |
private: |
void emitSymbolTable(struct nv50_ir_prog_info *); |
Type progType; |
Target *target; |
public: |
Function *main; |
Graph calls; |
ArrayList allFuncs; |
ArrayList allRValues; |
uint32_t *code; |
uint32_t binSize; |
uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL |
int maxGPR; |
MemoryPool mem_Instruction; |
MemoryPool mem_CmpInstruction; |
MemoryPool mem_TexInstruction; |
MemoryPool mem_FlowInstruction; |
MemoryPool mem_LValue; |
MemoryPool mem_Symbol; |
MemoryPool mem_ImmediateValue; |
uint32_t dbgFlags; |
uint8_t optLevel; |
void *targetPriv; // e.g. to carry information between passes |
const struct nv50_ir_prog_info *driver; // for driver configuration |
void releaseInstruction(Instruction *); |
void releaseValue(Value *); |
}; |
// TODO: add const version |
class Pass |
{ |
public: |
bool run(Program *, bool ordered = false, bool skipPhi = false); |
bool run(Function *, bool ordered = false, bool skipPhi = false); |
private: |
// return false to continue with next entity on next higher level |
virtual bool visit(Function *) { return true; } |
virtual bool visit(BasicBlock *) { return true; } |
virtual bool visit(Instruction *) { return false; } |
bool doRun(Program *, bool ordered, bool skipPhi); |
bool doRun(Function *, bool ordered, bool skipPhi); |
protected: |
bool err; |
Function *func; |
Program *prog; |
}; |
// ============================================================================= |
#include "nv50_ir_inlines.h" |
} // namespace nv50_ir |
#endif // __NV50_IR_H__ |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_bb.cpp |
---|
0,0 → 1,550 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
namespace nv50_ir { |
Function::Function(Program *p, const char *fnName, uint32_t label) |
: call(this), |
label(label), |
name(fnName), |
prog(p) |
{ |
cfgExit = NULL; |
domTree = NULL; |
bbArray = NULL; |
bbCount = 0; |
loopNestingBound = 0; |
regClobberMax = 0; |
binPos = 0; |
binSize = 0; |
stackPtr = NULL; |
tlsBase = 0; |
tlsSize = 0; |
prog->add(this, id); |
} |
Function::~Function() |
{ |
prog->del(this, id); |
if (domTree) |
delete domTree; |
if (bbArray) |
delete[] bbArray; |
// clear value refs and defs |
ins.clear(); |
outs.clear(); |
for (ArrayList::Iterator it = allInsns.iterator(); !it.end(); it.next()) |
delete_Instruction(prog, reinterpret_cast<Instruction *>(it.get())); |
for (ArrayList::Iterator it = allLValues.iterator(); !it.end(); it.next()) |
delete_Value(prog, reinterpret_cast<LValue *>(it.get())); |
for (ArrayList::Iterator BBs = allBBlocks.iterator(); !BBs.end(); BBs.next()) |
delete reinterpret_cast<BasicBlock *>(BBs.get()); |
} |
BasicBlock::BasicBlock(Function *fn) : cfg(this), dom(this), func(fn) |
{ |
program = func->getProgram(); |
joinAt = phi = entry = exit = NULL; |
numInsns = 0; |
binPos = 0; |
binSize = 0; |
explicitCont = false; |
func->add(this, this->id); |
} |
BasicBlock::~BasicBlock() |
{ |
// nothing yet |
} |
BasicBlock * |
BasicBlock::clone(ClonePolicy<Function>& pol) const |
{ |
BasicBlock *bb = new BasicBlock(pol.context()); |
pol.set(this, bb); |
for (Instruction *i = getFirst(); i; i = i->next) |
bb->insertTail(i->clone(pol)); |
pol.context()->cfg.insert(&bb->cfg); |
for (Graph::EdgeIterator it = cfg.outgoing(); !it.end(); it.next()) { |
BasicBlock *obb = BasicBlock::get(it.getNode()); |
bb->cfg.attach(&pol.get(obb)->cfg, it.getType()); |
} |
return bb; |
} |
BasicBlock * |
BasicBlock::idom() const |
{ |
Graph::Node *dn = dom.parent(); |
return dn ? BasicBlock::get(dn) : NULL; |
} |
void |
BasicBlock::insertHead(Instruction *inst) |
{ |
assert(inst->next == 0 && inst->prev == 0); |
if (inst->op == OP_PHI) { |
if (phi) { |
insertBefore(phi, inst); |
} else { |
if (entry) { |
insertBefore(entry, inst); |
} else { |
assert(!exit); |
phi = exit = inst; |
inst->bb = this; |
++numInsns; |
} |
} |
} else { |
if (entry) { |
insertBefore(entry, inst); |
} else { |
if (phi) { |
insertAfter(exit, inst); // after last phi |
} else { |
assert(!exit); |
entry = exit = inst; |
inst->bb = this; |
++numInsns; |
} |
} |
} |
} |
void |
BasicBlock::insertTail(Instruction *inst) |
{ |
assert(inst->next == 0 && inst->prev == 0); |
if (inst->op == OP_PHI) { |
if (entry) { |
insertBefore(entry, inst); |
} else |
if (exit) { |
assert(phi); |
insertAfter(exit, inst); |
} else { |
assert(!phi); |
phi = exit = inst; |
inst->bb = this; |
++numInsns; |
} |
} else { |
if (exit) { |
insertAfter(exit, inst); |
} else { |
assert(!phi); |
entry = exit = inst; |
inst->bb = this; |
++numInsns; |
} |
} |
} |
void |
BasicBlock::insertBefore(Instruction *q, Instruction *p) |
{ |
assert(p && q); |
assert(p->next == 0 && p->prev == 0); |
if (q == entry) { |
if (p->op == OP_PHI) { |
if (!phi) |
phi = p; |
} else { |
entry = p; |
} |
} else |
if (q == phi) { |
assert(p->op == OP_PHI); |
phi = p; |
} |
p->next = q; |
p->prev = q->prev; |
if (p->prev) |
p->prev->next = p; |
q->prev = p; |
p->bb = this; |
++numInsns; |
} |
void |
BasicBlock::insertAfter(Instruction *p, Instruction *q) |
{ |
assert(p && q); |
assert(q->op != OP_PHI || p->op == OP_PHI); |
assert(q->next == 0 && q->prev == 0); |
if (p == exit) |
exit = q; |
if (p->op == OP_PHI && q->op != OP_PHI) |
entry = q; |
q->prev = p; |
q->next = p->next; |
if (q->next) |
q->next->prev = q; |
p->next = q; |
q->bb = this; |
++numInsns; |
} |
void |
BasicBlock::remove(Instruction *insn) |
{ |
assert(insn->bb == this); |
if (insn->prev) |
insn->prev->next = insn->next; |
if (insn->next) |
insn->next->prev = insn->prev; |
else |
exit = insn->prev; |
if (insn == entry) { |
if (insn->next) |
entry = insn->next; |
else |
if (insn->prev && insn->prev->op != OP_PHI) |
entry = insn->prev; |
else |
entry = NULL; |
} |
if (insn == phi) |
phi = (insn->next && insn->next->op == OP_PHI) ? insn->next : 0; |
--numInsns; |
insn->bb = NULL; |
insn->next = |
insn->prev = NULL; |
} |
void BasicBlock::permuteAdjacent(Instruction *a, Instruction *b) |
{ |
assert(a->bb == b->bb); |
if (a->next != b) { |
Instruction *i = a; |
a = b; |
b = i; |
} |
assert(a->next == b); |
assert(a->op != OP_PHI && b->op != OP_PHI); |
if (b == exit) |
exit = a; |
if (a == entry) |
entry = b; |
b->prev = a->prev; |
a->next = b->next; |
b->next = a; |
a->prev = b; |
if (b->prev) |
b->prev->next = b; |
if (a->prev) |
a->next->prev = a; |
} |
void |
BasicBlock::splitCommon(Instruction *insn, BasicBlock *bb, bool attach) |
{ |
bb->entry = insn; |
if (insn) { |
exit = insn->prev; |
insn->prev = NULL; |
} |
if (exit) |
exit->next = NULL; |
else |
entry = NULL; |
while (!cfg.outgoing(true).end()) { |
Graph::Edge *e = cfg.outgoing(true).getEdge(); |
bb->cfg.attach(e->getTarget(), e->getType()); |
this->cfg.detach(e->getTarget()); |
} |
for (; insn; insn = insn->next) { |
this->numInsns--; |
bb->numInsns++; |
insn->bb = bb; |
bb->exit = insn; |
} |
if (attach) |
this->cfg.attach(&bb->cfg, Graph::Edge::TREE); |
} |
BasicBlock * |
BasicBlock::splitBefore(Instruction *insn, bool attach) |
{ |
BasicBlock *bb = new BasicBlock(func); |
assert(!insn || insn->op != OP_PHI); |
splitCommon(insn, bb, attach); |
return bb; |
} |
BasicBlock * |
BasicBlock::splitAfter(Instruction *insn, bool attach) |
{ |
BasicBlock *bb = new BasicBlock(func); |
assert(!insn || insn->op != OP_PHI); |
bb->joinAt = joinAt; |
joinAt = NULL; |
splitCommon(insn ? insn->next : NULL, bb, attach); |
return bb; |
} |
bool |
BasicBlock::dominatedBy(BasicBlock *that) |
{ |
Graph::Node *bn = &that->dom; |
Graph::Node *dn = &this->dom; |
while (dn && dn != bn) |
dn = dn->parent(); |
return dn != NULL; |
} |
unsigned int |
BasicBlock::initiatesSimpleConditional() const |
{ |
Graph::Node *out[2]; |
int n; |
Graph::Edge::Type eR; |
if (cfg.outgoingCount() != 2) // -> if and -> else/endif |
return false; |
n = 0; |
for (Graph::EdgeIterator ei = cfg.outgoing(); !ei.end(); ei.next()) |
out[n++] = ei.getNode(); |
eR = out[1]->outgoing().getType(); |
// IF block is out edge to the right |
if (eR == Graph::Edge::CROSS || eR == Graph::Edge::BACK) |
return 0x2; |
if (out[1]->outgoingCount() != 1) // 0 is IF { RET; }, >1 is more divergence |
return 0x0; |
// do they reconverge immediately ? |
if (out[1]->outgoing().getNode() == out[0]) |
return 0x1; |
if (out[0]->outgoingCount() == 1) |
if (out[0]->outgoing().getNode() == out[1]->outgoing().getNode()) |
return 0x3; |
return 0x0; |
} |
bool |
Function::setEntry(BasicBlock *bb) |
{ |
if (cfg.getRoot()) |
return false; |
cfg.insert(&bb->cfg); |
return true; |
} |
bool |
Function::setExit(BasicBlock *bb) |
{ |
if (cfgExit) |
return false; |
cfgExit = &bb->cfg; |
return true; |
} |
unsigned int |
Function::orderInstructions(ArrayList &result) |
{ |
result.clear(); |
for (IteratorRef it = cfg.iteratorCFG(); !it->end(); it->next()) { |
BasicBlock *bb = |
BasicBlock::get(reinterpret_cast<Graph::Node *>(it->get())); |
for (Instruction *insn = bb->getFirst(); insn; insn = insn->next) |
result.insert(insn, insn->serial); |
} |
return result.getSize(); |
} |
void |
Function::buildLiveSets() |
{ |
for (unsigned i = 0; i <= loopNestingBound; ++i) |
buildLiveSetsPreSSA(BasicBlock::get(cfg.getRoot()), cfg.nextSequence()); |
for (ArrayList::Iterator bi = allBBlocks.iterator(); !bi.end(); bi.next()) |
BasicBlock::get(bi)->liveSet.marker = false; |
} |
void |
Function::buildDefSets() |
{ |
for (unsigned i = 0; i <= loopNestingBound; ++i) |
buildDefSetsPreSSA(BasicBlock::get(cfgExit), cfg.nextSequence()); |
for (ArrayList::Iterator bi = allBBlocks.iterator(); !bi.end(); bi.next()) |
BasicBlock::get(bi)->liveSet.marker = false; |
} |
bool |
Pass::run(Program *prog, bool ordered, bool skipPhi) |
{ |
this->prog = prog; |
err = false; |
return doRun(prog, ordered, skipPhi); |
} |
bool |
Pass::doRun(Program *prog, bool ordered, bool skipPhi) |
{ |
for (IteratorRef it = prog->calls.iteratorDFS(false); |
!it->end(); it->next()) { |
Graph::Node *n = reinterpret_cast<Graph::Node *>(it->get()); |
if (!doRun(Function::get(n), ordered, skipPhi)) |
return false; |
} |
return !err; |
} |
bool |
Pass::run(Function *func, bool ordered, bool skipPhi) |
{ |
prog = func->getProgram(); |
err = false; |
return doRun(func, ordered, skipPhi); |
} |
bool |
Pass::doRun(Function *func, bool ordered, bool skipPhi) |
{ |
IteratorRef bbIter; |
BasicBlock *bb; |
Instruction *insn, *next; |
this->func = func; |
if (!visit(func)) |
return false; |
bbIter = ordered ? func->cfg.iteratorCFG() : func->cfg.iteratorDFS(); |
for (; !bbIter->end(); bbIter->next()) { |
bb = BasicBlock::get(reinterpret_cast<Graph::Node *>(bbIter->get())); |
if (!visit(bb)) |
break; |
for (insn = skipPhi ? bb->getEntry() : bb->getFirst(); insn != NULL; |
insn = next) { |
next = insn->next; |
if (!visit(insn)) |
break; |
} |
} |
return !err; |
} |
void |
Function::printCFGraph(const char *filePath) |
{ |
FILE *out = fopen(filePath, "a"); |
if (!out) { |
ERROR("failed to open file: %s\n", filePath); |
return; |
} |
INFO("printing control flow graph to: %s\n", filePath); |
fprintf(out, "digraph G {\n"); |
for (IteratorRef it = cfg.iteratorDFS(); !it->end(); it->next()) { |
BasicBlock *bb = BasicBlock::get( |
reinterpret_cast<Graph::Node *>(it->get())); |
int idA = bb->getId(); |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
int idB = BasicBlock::get(ei.getNode())->getId(); |
switch (ei.getType()) { |
case Graph::Edge::TREE: |
fprintf(out, "\t%i -> %i;\n", idA, idB); |
break; |
case Graph::Edge::FORWARD: |
fprintf(out, "\t%i -> %i [color=green];\n", idA, idB); |
break; |
case Graph::Edge::CROSS: |
fprintf(out, "\t%i -> %i [color=red];\n", idA, idB); |
break; |
case Graph::Edge::BACK: |
fprintf(out, "\t%i -> %i;\n", idA, idB); |
break; |
case Graph::Edge::DUMMY: |
fprintf(out, "\t%i -> %i [style=dotted];\n", idA, idB); |
break; |
default: |
assert(0); |
break; |
} |
} |
} |
fprintf(out, "}\n"); |
fclose(out); |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.cpp |
---|
0,0 → 1,614 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_build_util.h" |
namespace nv50_ir { |
BuildUtil::BuildUtil() |
{ |
init(NULL); |
} |
BuildUtil::BuildUtil(Program *prog) |
{ |
init(prog); |
} |
void |
BuildUtil::init(Program *prog) |
{ |
this->prog = prog; |
func = NULL; |
bb = NULL; |
pos = NULL; |
memset(imms, 0, sizeof(imms)); |
immCount = 0; |
} |
void |
BuildUtil::addImmediate(ImmediateValue *imm) |
{ |
if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4) |
return; |
unsigned int pos = u32Hash(imm->reg.data.u32); |
while (imms[pos]) |
pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE; |
imms[pos] = imm; |
immCount++; |
} |
Instruction * |
BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src) |
{ |
Instruction *insn = new_Instruction(func, op, ty); |
insn->setDef(0, dst); |
insn->setSrc(0, src); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkOp2(operation op, DataType ty, Value *dst, |
Value *src0, Value *src1) |
{ |
Instruction *insn = new_Instruction(func, op, ty); |
insn->setDef(0, dst); |
insn->setSrc(0, src0); |
insn->setSrc(1, src1); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkOp3(operation op, DataType ty, Value *dst, |
Value *src0, Value *src1, Value *src2) |
{ |
Instruction *insn = new_Instruction(func, op, ty); |
insn->setDef(0, dst); |
insn->setSrc(0, src0); |
insn->setSrc(1, src1); |
insn->setSrc(2, src2); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr) |
{ |
Instruction *insn = new_Instruction(func, OP_LOAD, ty); |
insn->setDef(0, dst); |
insn->setSrc(0, mem); |
if (ptr) |
insn->setIndirect(0, 0, ptr); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr, |
Value *stVal) |
{ |
Instruction *insn = new_Instruction(func, op, ty); |
insn->setSrc(0, mem); |
insn->setSrc(1, stVal); |
if (ptr) |
insn->setIndirect(0, 0, ptr); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset, |
Value *attrRel, Value *primRel) |
{ |
Symbol *sym = mkSymbol(file, 0, ty, offset); |
Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym); |
insn->setIndirect(0, 0, attrRel); |
insn->setIndirect(0, 1, primRel); |
// already inserted |
return insn; |
} |
Instruction * |
BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel) |
{ |
operation op = OP_LINTERP; |
DataType ty = TYPE_F32; |
if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT) |
ty = TYPE_U32; |
else |
if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE) |
op = OP_PINTERP; |
Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset); |
Instruction *insn = mkOp1(op, ty, dst, sym); |
insn->setIndirect(0, 0, rel); |
return insn; |
} |
Instruction * |
BuildUtil::mkMov(Value *dst, Value *src, DataType ty) |
{ |
Instruction *insn = new_Instruction(func, OP_MOV, ty); |
insn->setDef(0, dst); |
insn->setSrc(0, src); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkMovToReg(int id, Value *src) |
{ |
Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size)); |
insn->setDef(0, new_LValue(func, FILE_GPR)); |
insn->getDef(0)->reg.data.id = id; |
insn->setSrc(0, src); |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkMovFromReg(Value *dst, int id) |
{ |
Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size)); |
insn->setDef(0, dst); |
insn->setSrc(0, new_LValue(func, FILE_GPR)); |
insn->getSrc(0)->reg.data.id = id; |
insert(insn); |
return insn; |
} |
Instruction * |
BuildUtil::mkCvt(operation op, |
DataType dstTy, Value *dst, DataType srcTy, Value *src) |
{ |
Instruction *insn = new_Instruction(func, op, dstTy); |
insn->setType(dstTy, srcTy); |
insn->setDef(0, dst); |
insn->setSrc(0, src); |
insert(insn); |
return insn; |
} |
CmpInstruction * |
BuildUtil::mkCmp(operation op, CondCode cc, DataType ty, Value *dst, |
Value *src0, Value *src1, Value *src2) |
{ |
CmpInstruction *insn = new_CmpInstruction(func, op); |
insn->setType((dst->reg.file == FILE_PREDICATE || |
dst->reg.file == FILE_FLAGS) ? TYPE_U8 : ty, ty); |
insn->setCondition(cc); |
insn->setDef(0, dst); |
insn->setSrc(0, src0); |
insn->setSrc(1, src1); |
if (src2) |
insn->setSrc(2, src2); |
if (dst->reg.file == FILE_FLAGS) |
insn->flagsDef = 0; |
insert(insn); |
return insn; |
} |
TexInstruction * |
BuildUtil::mkTex(operation op, TexTarget targ, |
uint16_t tic, uint16_t tsc, |
const std::vector<Value *> &def, |
const std::vector<Value *> &src) |
{ |
TexInstruction *tex = new_TexInstruction(func, op); |
for (size_t d = 0; d < def.size() && def[d]; ++d) |
tex->setDef(d, def[d]); |
for (size_t s = 0; s < src.size() && src[s]; ++s) |
tex->setSrc(s, src[s]); |
tex->setTexture(targ, tic, tsc); |
insert(tex); |
return tex; |
} |
Instruction * |
BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1) |
{ |
Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1); |
quadop->subOp = q; |
quadop->lanes = l; |
return quadop; |
} |
Instruction * |
BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc) |
{ |
LValue *def0 = getSSA(); |
LValue *def1 = getSSA(); |
mkMov(def0, trSrc)->setPredicate(CC_P, pred); |
mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred); |
return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1); |
} |
Instruction * |
BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val) |
{ |
Instruction *insn = NULL; |
const DataType fTy = typeOfSize(halfSize * 2); |
if (val->reg.file == FILE_IMMEDIATE) |
val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0); |
if (isMemoryFile(val->reg.file)) { |
h[0] = cloneShallow(getFunction(), val); |
h[1] = cloneShallow(getFunction(), val); |
h[0]->reg.size = halfSize; |
h[1]->reg.size = halfSize; |
h[1]->reg.data.offset += halfSize; |
} else { |
h[0] = getSSA(halfSize, val->reg.file); |
h[1] = getSSA(halfSize, val->reg.file); |
insn = mkOp1(OP_SPLIT, fTy, h[0], val); |
insn->setDef(1, h[1]); |
} |
return insn; |
} |
FlowInstruction * |
BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred) |
{ |
FlowInstruction *insn = new_FlowInstruction(func, op, targ); |
if (pred) |
insn->setPredicate(cc, pred); |
insert(insn); |
return insn; |
} |
void |
BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit) |
{ |
static const uint16_t baseSize2[16] = |
{ |
0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220, |
0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040, |
}; |
int base = 0; |
for (; rMask; rMask >>= 4, base += 4) { |
const uint32_t mask = rMask & 0xf; |
if (!mask) |
continue; |
int base1 = (baseSize2[mask] >> 0) & 0xf; |
int size1 = (baseSize2[mask] >> 4) & 0xf; |
int base2 = (baseSize2[mask] >> 8) & 0xf; |
int size2 = (baseSize2[mask] >> 12) & 0xf; |
Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL); |
if (1) { // size1 can't be 0 |
LValue *reg = new_LValue(func, f); |
reg->reg.size = size1 << unit; |
reg->reg.data.id = base + base1; |
insn->setDef(0, reg); |
} |
if (size2) { |
LValue *reg = new_LValue(func, f); |
reg->reg.size = size2 << unit; |
reg->reg.data.id = base + base2; |
insn->setDef(1, reg); |
} |
} |
} |
ImmediateValue * |
BuildUtil::mkImm(uint32_t u) |
{ |
unsigned int pos = u32Hash(u); |
while (imms[pos] && imms[pos]->reg.data.u32 != u) |
pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE; |
ImmediateValue *imm = imms[pos]; |
if (!imm) { |
imm = new_ImmediateValue(prog, u); |
addImmediate(imm); |
} |
return imm; |
} |
ImmediateValue * |
BuildUtil::mkImm(uint64_t u) |
{ |
ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0); |
imm->reg.size = 8; |
imm->reg.type = TYPE_U64; |
imm->reg.data.u64 = u; |
return imm; |
} |
ImmediateValue * |
BuildUtil::mkImm(float f) |
{ |
union { |
float f32; |
uint32_t u32; |
} u; |
u.f32 = f; |
return mkImm(u.u32); |
} |
Value * |
BuildUtil::loadImm(Value *dst, float f) |
{ |
return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f)); |
} |
Value * |
BuildUtil::loadImm(Value *dst, uint32_t u) |
{ |
return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u)); |
} |
Value * |
BuildUtil::loadImm(Value *dst, uint64_t u) |
{ |
return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u)); |
} |
Symbol * |
BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty, |
uint32_t baseAddr) |
{ |
Symbol *sym = new_Symbol(prog, file, fileIndex); |
sym->setOffset(baseAddr); |
sym->reg.type = ty; |
sym->reg.size = typeSizeof(ty); |
return sym; |
} |
Symbol * |
BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex) |
{ |
Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0); |
assert(svIndex < 4 || |
(svName == SV_CLIP_DISTANCE || svName == SV_TESS_FACTOR)); |
switch (svName) { |
case SV_POSITION: |
case SV_FACE: |
case SV_YDIR: |
case SV_POINT_SIZE: |
case SV_POINT_COORD: |
case SV_CLIP_DISTANCE: |
case SV_TESS_FACTOR: |
sym->reg.type = TYPE_F32; |
break; |
default: |
sym->reg.type = TYPE_U32; |
break; |
} |
sym->reg.size = typeSizeof(sym->reg.type); |
sym->reg.data.sv.sv = svName; |
sym->reg.data.sv.index = svIndex; |
return sym; |
} |
void |
BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx, |
uint32_t base, int len, int vecDim, int eltSize, |
DataFile file, int8_t fileIdx) |
{ |
this->array = array; |
this->arrayIdx = arrayIdx; |
this->baseAddr = base; |
this->arrayLen = len; |
this->vecDim = vecDim; |
this->eltSize = eltSize; |
this->file = file; |
this->regOnly = !isMemoryFile(file); |
if (!regOnly) { |
baseSym = new_Symbol(up->getProgram(), file, fileIdx); |
baseSym->setOffset(baseAddr); |
baseSym->reg.size = eltSize; |
} else { |
baseSym = NULL; |
} |
} |
Value * |
BuildUtil::DataArray::acquire(ValueMap &m, int i, int c) |
{ |
if (regOnly) { |
Value *v = lookup(m, i, c); |
if (!v) |
v = insert(m, i, c, new_LValue(up->getFunction(), file)); |
return v; |
} else { |
return up->getScratch(); |
} |
} |
Value * |
BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr) |
{ |
if (regOnly) { |
Value *v = lookup(m, i, c); |
if (!v) |
v = insert(m, i, c, new_LValue(up->getFunction(), file)); |
return v; |
} else { |
Value *sym = lookup(m, i, c); |
if (!sym) |
sym = insert(m, i, c, mkSymbol(i, c)); |
return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr); |
} |
} |
void |
BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value) |
{ |
if (regOnly) { |
assert(!ptr); |
if (!lookup(m, i, c)) |
insert(m, i, c, value); |
assert(lookup(m, i, c) == value); |
} else { |
Value *sym = lookup(m, i, c); |
if (!sym) |
sym = insert(m, i, c, mkSymbol(i, c)); |
const DataType stTy = typeOfSize(value->reg.size); |
up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value); |
} |
} |
Symbol * |
BuildUtil::DataArray::mkSymbol(int i, int c) |
{ |
const unsigned int idx = i * vecDim + c; |
Symbol *sym = new_Symbol(up->getProgram(), file, 0); |
assert(baseSym || (idx < arrayLen && c < vecDim)); |
sym->reg.size = eltSize; |
sym->reg.type = typeOfSize(eltSize); |
sym->setAddress(baseSym, baseAddr + idx * eltSize); |
return sym; |
} |
Instruction * |
BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i, |
Value *zero, |
Value *carry) |
{ |
DataType hTy; |
int srcNr; |
switch (i->dType) { |
case TYPE_U64: hTy = TYPE_U32; break; |
case TYPE_S64: hTy = TYPE_S32; break; |
default: |
return NULL; |
} |
switch (i->op) { |
case OP_MOV: srcNr = 1; break; |
case OP_ADD: |
case OP_SUB: |
if (!carry) |
return NULL; |
srcNr = 2; |
break; |
default: |
// TODO when needed |
return NULL; |
} |
i->setType(hTy); |
i->setDef(0, cloneShallow(fn, i->getDef(0))); |
i->getDef(0)->reg.size = 4; |
Instruction *lo = i; |
Instruction *hi = cloneForward(fn, i); |
lo->bb->insertAfter(lo, hi); |
hi->getDef(0)->reg.data.id++; |
for (int s = 0; s < srcNr; ++s) { |
if (lo->getSrc(s)->reg.size < 8) { |
hi->setSrc(s, zero); |
} else { |
if (lo->getSrc(s)->refCount() > 1) |
lo->setSrc(s, cloneShallow(fn, lo->getSrc(s))); |
lo->getSrc(s)->reg.size /= 2; |
hi->setSrc(s, cloneShallow(fn, lo->getSrc(s))); |
switch (hi->src(s).getFile()) { |
case FILE_IMMEDIATE: |
hi->getSrc(s)->reg.data.u64 >>= 32; |
break; |
case FILE_MEMORY_CONST: |
case FILE_MEMORY_SHARED: |
case FILE_SHADER_INPUT: |
hi->getSrc(s)->reg.data.offset += 4; |
break; |
default: |
assert(hi->src(s).getFile() == FILE_GPR); |
hi->getSrc(s)->reg.data.id++; |
break; |
} |
} |
} |
if (srcNr == 2) { |
lo->setDef(1, carry); |
hi->setFlagsSrc(hi->srcCount(), carry); |
} |
return hi; |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_build_util.h |
---|
0,0 → 1,324 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_BUILD_UTIL__ |
#define __NV50_IR_BUILD_UTIL__ |
namespace nv50_ir { |
class BuildUtil |
{ |
public: |
BuildUtil(); |
BuildUtil(Program *); |
inline void setProgram(Program *); |
inline Program *getProgram() const { return prog; } |
inline Function *getFunction() const { return func; } |
// keeps inserting at head/tail of block |
inline void setPosition(BasicBlock *, bool tail); |
// position advances only if @after is true |
inline void setPosition(Instruction *, bool after); |
inline BasicBlock *getBB() { return bb; } |
inline void insert(Instruction *); |
inline void remove(Instruction *i) { assert(i->bb == bb); bb->remove(i); } |
inline LValue *getScratch(int size = 4, DataFile = FILE_GPR); |
// scratch value for a single assignment: |
inline LValue *getSSA(int size = 4, DataFile = FILE_GPR); |
inline Instruction *mkOp(operation, DataType, Value *); |
Instruction *mkOp1(operation, DataType, Value *, Value *); |
Instruction *mkOp2(operation, DataType, Value *, Value *, Value *); |
Instruction *mkOp3(operation, DataType, Value *, Value *, Value *, Value *); |
LValue *mkOp1v(operation, DataType, Value *, Value *); |
LValue *mkOp2v(operation, DataType, Value *, Value *, Value *); |
LValue *mkOp3v(operation, DataType, Value *, Value *, Value *, Value *); |
Instruction *mkLoad(DataType, Value *dst, Symbol *, Value *ptr); |
Instruction *mkStore(operation, DataType, Symbol *, Value *ptr, Value *val); |
LValue *mkLoadv(DataType, Symbol *, Value *ptr); |
Instruction *mkMov(Value *, Value *, DataType = TYPE_U32); |
Instruction *mkMovToReg(int id, Value *); |
Instruction *mkMovFromReg(Value *, int id); |
Instruction *mkInterp(unsigned mode, Value *, int32_t offset, Value *rel); |
Instruction *mkFetch(Value *, DataType, DataFile, int32_t offset, |
Value *attrRel, Value *primRel); |
Instruction *mkCvt(operation, DataType, Value *, DataType, Value *); |
CmpInstruction *mkCmp(operation, CondCode, DataType, |
Value *, |
Value *, Value *, Value * = NULL); |
TexInstruction *mkTex(operation, TexTarget, |
uint16_t tic, uint16_t tsc, |
const std::vector<Value *> &def, |
const std::vector<Value *> &src); |
Instruction *mkQuadop(uint8_t qop, Value *, uint8_t l, Value *, Value *); |
FlowInstruction *mkFlow(operation, void *target, CondCode, Value *pred); |
Instruction *mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc); |
Instruction *mkSplit(Value *half[2], uint8_t halfSize, Value *); |
void mkClobber(DataFile file, uint32_t regMask, int regUnitLog2); |
ImmediateValue *mkImm(float); |
ImmediateValue *mkImm(uint32_t); |
ImmediateValue *mkImm(uint64_t); |
ImmediateValue *mkImm(int i) { return mkImm((uint32_t)i); } |
Value *loadImm(Value *dst, float); |
Value *loadImm(Value *dst, uint32_t); |
Value *loadImm(Value *dst, uint64_t); |
Value *loadImm(Value *dst, int i) { return loadImm(dst, (uint32_t)i); } |
// returns high part of the operation |
static Instruction *split64BitOpPostRA(Function *, Instruction *, |
Value *zero, Value *carry); |
struct Location |
{ |
Location(unsigned array, unsigned arrayIdx, unsigned i, unsigned c) |
: array(array), arrayIdx(arrayIdx), i(i), c(c) { } |
Location(const Location &l) |
: array(l.array), arrayIdx(l.arrayIdx), i(l.i), c(l.c) { } |
bool operator==(const Location &l) const |
{ |
return |
array == l.array && arrayIdx == l.arrayIdx && i == l.i && c == l.c; |
} |
bool operator<(const Location &l) const |
{ |
return array != l.array ? array < l.array : |
arrayIdx != l.arrayIdx ? arrayIdx < l.arrayIdx : |
i != l.i ? i < l.i : |
c != l.c ? c < l.c : |
false; |
} |
unsigned array, arrayIdx, i, c; |
}; |
typedef bimap<Location, Value *> ValueMap; |
class DataArray |
{ |
public: |
DataArray(BuildUtil *bld) : up(bld) { } |
void setup(unsigned array, unsigned arrayIdx, |
uint32_t base, int len, int vecDim, int eltSize, |
DataFile file, int8_t fileIdx); |
inline bool exists(ValueMap&, unsigned int i, unsigned int c); |
Value *load(ValueMap&, int i, int c, Value *ptr); |
void store(ValueMap&, int i, int c, Value *ptr, Value *value); |
Value *acquire(ValueMap&, int i, int c); |
private: |
inline Value *lookup(ValueMap&, unsigned i, unsigned c); |
inline Value *insert(ValueMap&, unsigned i, unsigned c, Value *v); |
Symbol *mkSymbol(int i, int c); |
private: |
BuildUtil *up; |
unsigned array, arrayIdx; |
uint32_t baseAddr; |
uint32_t arrayLen; |
Symbol *baseSym; |
uint8_t vecDim; |
uint8_t eltSize; // in bytes |
DataFile file; |
bool regOnly; |
}; |
Symbol *mkSymbol(DataFile file, int8_t fileIndex, |
DataType ty, uint32_t baseAddress); |
Symbol *mkSysVal(SVSemantic svName, uint32_t svIndex); |
private: |
void init(Program *); |
void addImmediate(ImmediateValue *); |
inline unsigned int u32Hash(uint32_t); |
protected: |
Program *prog; |
Function *func; |
Instruction *pos; |
BasicBlock *bb; |
bool tail; |
#define NV50_IR_BUILD_IMM_HT_SIZE 256 |
ImmediateValue *imms[NV50_IR_BUILD_IMM_HT_SIZE]; |
unsigned int immCount; |
}; |
unsigned int BuildUtil::u32Hash(uint32_t u) |
{ |
return (u % 273) % NV50_IR_BUILD_IMM_HT_SIZE; |
} |
void BuildUtil::setProgram(Program *program) |
{ |
prog = program; |
} |
void |
BuildUtil::setPosition(BasicBlock *block, bool atTail) |
{ |
bb = block; |
prog = bb->getProgram(); |
func = bb->getFunction(); |
pos = NULL; |
tail = atTail; |
} |
void |
BuildUtil::setPosition(Instruction *i, bool after) |
{ |
bb = i->bb; |
prog = bb->getProgram(); |
func = bb->getFunction(); |
pos = i; |
tail = after; |
assert(bb); |
} |
LValue * |
BuildUtil::getScratch(int size, DataFile f) |
{ |
LValue *lval = new_LValue(func, f); |
lval->reg.size = size; |
return lval; |
} |
LValue * |
BuildUtil::getSSA(int size, DataFile f) |
{ |
LValue *lval = new_LValue(func, f); |
lval->ssa = 1; |
lval->reg.size = size; |
return lval; |
} |
void BuildUtil::insert(Instruction *i) |
{ |
if (!pos) { |
tail ? bb->insertTail(i) : bb->insertHead(i); |
} else { |
if (tail) { |
bb->insertAfter(pos, i); |
pos = i; |
} else { |
bb->insertBefore(pos, i); |
} |
} |
} |
Instruction * |
BuildUtil::mkOp(operation op, DataType ty, Value *dst) |
{ |
Instruction *insn = new_Instruction(func, op, ty); |
insn->setDef(0, dst); |
insert(insn); |
if (op == OP_DISCARD || op == OP_EXIT || |
op == OP_JOIN || |
op == OP_QUADON || op == OP_QUADPOP || |
op == OP_EMIT || op == OP_RESTART) |
insn->fixed = 1; |
return insn; |
} |
inline LValue * |
BuildUtil::mkOp1v(operation op, DataType ty, Value *dst, Value *src) |
{ |
mkOp1(op, ty, dst, src); |
return dst->asLValue(); |
} |
inline LValue * |
BuildUtil::mkOp2v(operation op, DataType ty, Value *dst, |
Value *src0, Value *src1) |
{ |
mkOp2(op, ty, dst, src0, src1); |
return dst->asLValue(); |
} |
inline LValue * |
BuildUtil::mkOp3v(operation op, DataType ty, Value *dst, |
Value *src0, Value *src1, Value *src2) |
{ |
mkOp3(op, ty, dst, src0, src1, src2); |
return dst->asLValue(); |
} |
inline LValue * |
BuildUtil::mkLoadv(DataType ty, Symbol *mem, Value *ptr) |
{ |
LValue *dst = getScratch(); |
mkLoad(ty, dst, mem, ptr); |
return dst; |
} |
bool |
BuildUtil::DataArray::exists(ValueMap &m, unsigned int i, unsigned int c) |
{ |
assert(i < arrayLen && c < vecDim); |
return !regOnly || m.r.count(Location(array, arrayIdx, i, c)); |
} |
Value * |
BuildUtil::DataArray::lookup(ValueMap &m, unsigned i, unsigned c) |
{ |
ValueMap::r_iterator it = m.r.find(Location(array, arrayIdx, i, c)); |
return it != m.r.end() ? it->second : NULL; |
} |
Value * |
BuildUtil::DataArray::insert(ValueMap &m, unsigned i, unsigned c, Value *v) |
{ |
m.insert(Location(array, arrayIdx, i, c), v); |
return v; |
} |
} // namespace nv50_ir |
#endif // __NV50_IR_BUILD_UTIL_H__ |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_driver.h |
---|
0,0 → 1,220 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_DRIVER_H__ |
#define __NV50_IR_DRIVER_H__ |
#include "pipe/p_shader_tokens.h" |
#include "tgsi/tgsi_util.h" |
#include "tgsi/tgsi_parse.h" |
#include "tgsi/tgsi_scan.h" |
/* |
* This struct constitutes linkage information in TGSI terminology. |
* |
* It is created by the code generator and handed to the pipe driver |
* for input/output slot assignment. |
*/ |
struct nv50_ir_varying |
{ |
uint8_t slot[4]; /* native slots for xyzw (addresses in 32-bit words) */ |
unsigned mask : 4; /* vec4 mask */ |
unsigned linear : 1; /* linearly interpolated if true (and not flat) */ |
unsigned flat : 1; |
unsigned sc : 1; /* special colour interpolation mode (SHADE_MODEL) */ |
unsigned centroid : 1; |
unsigned patch : 1; /* patch constant value */ |
unsigned regular : 1; /* driver-specific meaning (e.g. input in sreg) */ |
unsigned input : 1; /* indicates direction of system values */ |
unsigned oread : 1; /* true if output is read from parallel TCP */ |
ubyte id; /* TGSI register index */ |
ubyte sn; /* TGSI semantic name */ |
ubyte si; /* TGSI semantic index */ |
}; |
#define NV50_PROGRAM_IR_TGSI 0 |
#define NV50_PROGRAM_IR_SM4 1 |
#define NV50_PROGRAM_IR_GLSL 2 |
#define NV50_PROGRAM_IR_LLVM 3 |
#ifdef DEBUG |
# define NV50_IR_DEBUG_BASIC (1 << 0) |
# define NV50_IR_DEBUG_VERBOSE (2 << 0) |
# define NV50_IR_DEBUG_REG_ALLOC (1 << 2) |
#else |
# define NV50_IR_DEBUG_BASIC 0 |
# define NV50_IR_DEBUG_VERBOSE 0 |
# define NV50_IR_DEBUG_REG_ALLOC 0 |
#endif |
#define NV50_SEMANTIC_CLIPDISTANCE (TGSI_SEMANTIC_COUNT + 0) |
#define NV50_SEMANTIC_VIEWPORTINDEX (TGSI_SEMANTIC_COUNT + 4) |
#define NV50_SEMANTIC_LAYER (TGSI_SEMANTIC_COUNT + 5) |
#define NV50_SEMANTIC_INVOCATIONID (TGSI_SEMANTIC_COUNT + 6) |
#define NV50_SEMANTIC_TESSFACTOR (TGSI_SEMANTIC_COUNT + 7) |
#define NV50_SEMANTIC_TESSCOORD (TGSI_SEMANTIC_COUNT + 8) |
#define NV50_SEMANTIC_SAMPLEMASK (TGSI_SEMANTIC_COUNT + 9) |
#define NV50_SEMANTIC_COUNT (TGSI_SEMANTIC_COUNT + 10) |
#define NV50_TESS_PART_FRACT_ODD 0 |
#define NV50_TESS_PART_FRACT_EVEN 1 |
#define NV50_TESS_PART_POW2 2 |
#define NV50_TESS_PART_INTEGER 3 |
#define NV50_PRIM_PATCHES PIPE_PRIM_MAX |
struct nv50_ir_prog_symbol |
{ |
uint32_t label; |
uint32_t offset; |
}; |
#define NVISA_GF100_CHIPSET_C0 0xc0 |
#define NVISA_GF100_CHIPSET_D0 0xd0 |
#define NVISA_GK104_CHIPSET 0xe0 |
#define NVISA_GK110_CHIPSET 0xf0 |
struct nv50_ir_prog_info |
{ |
uint16_t target; /* chipset (0x50, 0x84, 0xc0, ...) */ |
uint8_t type; /* PIPE_SHADER */ |
uint8_t optLevel; /* optimization level (0 to 3) */ |
uint8_t dbgFlags; |
struct { |
int16_t maxGPR; /* may be -1 if none used */ |
int16_t maxOutput; |
uint32_t tlsSpace; /* required local memory per thread */ |
uint32_t *code; |
uint32_t codeSize; |
uint8_t sourceRep; /* NV50_PROGRAM_IR */ |
const void *source; |
void *relocData; |
struct nv50_ir_prog_symbol *syms; |
uint16_t numSyms; |
} bin; |
struct nv50_ir_varying sv[PIPE_MAX_SHADER_INPUTS]; |
struct nv50_ir_varying in[PIPE_MAX_SHADER_INPUTS]; |
struct nv50_ir_varying out[PIPE_MAX_SHADER_OUTPUTS]; |
uint8_t numInputs; |
uint8_t numOutputs; |
uint8_t numPatchConstants; /* also included in numInputs/numOutputs */ |
uint8_t numSysVals; |
struct { |
uint32_t *buf; /* for IMMEDIATE_ARRAY */ |
uint16_t bufSize; /* size of immediate array */ |
uint16_t count; /* count of inline immediates */ |
uint32_t *data; /* inline immediate data */ |
uint8_t *type; /* for each vec4 (128 bit) */ |
} immd; |
union { |
struct { |
uint32_t inputMask[4]; /* mask of attributes read (1 bit per scalar) */ |
} vp; |
struct { |
uint8_t inputPatchSize; |
uint8_t outputPatchSize; |
uint8_t partitioning; /* PIPE_TESS_PART */ |
int8_t winding; /* +1 (clockwise) / -1 (counter-clockwise) */ |
uint8_t domain; /* PIPE_PRIM_{QUADS,TRIANGLES,LINES} */ |
uint8_t outputPrim; /* PIPE_PRIM_{TRIANGLES,LINES,POINTS} */ |
} tp; |
struct { |
uint8_t inputPrim; |
uint8_t outputPrim; |
unsigned instanceCount; |
unsigned maxVertices; |
} gp; |
struct { |
unsigned numColourResults; |
boolean writesDepth; |
boolean earlyFragTests; |
boolean separateFragData; |
boolean usesDiscard; |
} fp; |
struct { |
uint32_t inputOffset; /* base address for user args */ |
uint32_t sharedOffset; /* reserved space in s[] */ |
uint32_t gridInfoBase; /* base address for NTID,NCTAID */ |
} cp; |
} prop; |
uint8_t numBarriers; |
struct { |
uint8_t clipDistance; /* index of first clip distance output */ |
uint8_t clipDistanceMask; /* mask of clip distances defined */ |
uint8_t cullDistanceMask; /* clip distance mode (1 bit per output) */ |
int8_t genUserClip; /* request user clip planes for ClipVertex */ |
uint16_t ucpBase; /* base address for UCPs */ |
uint8_t ucpCBSlot; /* constant buffer index of UCP data */ |
uint8_t pointSize; /* output index for PointSize */ |
uint8_t instanceId; /* system value index of InstanceID */ |
uint8_t vertexId; /* system value index of VertexID */ |
uint8_t edgeFlagIn; |
uint8_t edgeFlagOut; |
uint8_t fragDepth; /* output index of FragDepth */ |
uint8_t sampleMask; /* output index of SampleMask */ |
uint8_t backFaceColor[2]; /* input/output indices of back face colour */ |
uint8_t globalAccess; /* 1 for read, 2 for wr, 3 for rw */ |
boolean nv50styleSurfaces; /* generate gX[] access for raw buffers */ |
uint8_t resInfoCBSlot; /* cX[] used for tex handles, surface info */ |
uint16_t texBindBase; /* base address for tex handles (nve4) */ |
uint16_t suInfoBase; /* base address for surface info (nve4) */ |
uint8_t msInfoCBSlot; /* cX[] used for multisample info */ |
uint16_t msInfoBase; /* base address for multisample info */ |
} io; |
/* driver callback to assign input/output locations */ |
int (*assignSlots)(struct nv50_ir_prog_info *); |
void *driverPriv; |
}; |
#ifdef __cplusplus |
extern "C" { |
#endif |
extern int nv50_ir_generate_code(struct nv50_ir_prog_info *); |
extern void nv50_ir_relocate_code(void *relocData, uint32_t *code, |
uint32_t codePos, |
uint32_t libPos, |
uint32_t dataPos); |
/* obtain code that will be shared among programs */ |
extern void nv50_ir_get_target_library(uint32_t chipset, |
const uint32_t **code, uint32_t *size); |
#ifdef __cplusplus |
} |
#endif |
#endif // __NV50_IR_DRIVER_H__ |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_emit_nv50.cpp |
---|
0,0 → 1,1962 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_target_nv50.h" |
namespace nv50_ir { |
#define NV50_OP_ENC_LONG 0 |
#define NV50_OP_ENC_SHORT 1 |
#define NV50_OP_ENC_IMM 2 |
#define NV50_OP_ENC_LONG_ALT 3 |
class CodeEmitterNV50 : public CodeEmitter |
{ |
public: |
CodeEmitterNV50(const TargetNV50 *); |
virtual bool emitInstruction(Instruction *); |
virtual uint32_t getMinEncodingSize(const Instruction *) const; |
inline void setProgramType(Program::Type pType) { progType = pType; } |
virtual void prepareEmission(Function *); |
private: |
Program::Type progType; |
const TargetNV50 *targNV50; |
private: |
inline void defId(const ValueDef&, const int pos); |
inline void srcId(const ValueRef&, const int pos); |
inline void srcId(const ValueRef *, const int pos); |
inline void srcAddr16(const ValueRef&, bool adj, const int pos); |
inline void srcAddr8(const ValueRef&, const int pos); |
void emitFlagsRd(const Instruction *); |
void emitFlagsWr(const Instruction *); |
void emitCondCode(CondCode cc, DataType ty, int pos); |
inline void setARegBits(unsigned int); |
void setAReg16(const Instruction *, int s); |
void setImmediate(const Instruction *, int s); |
void setDst(const Value *); |
void setDst(const Instruction *, int d); |
void setSrcFileBits(const Instruction *, int enc); |
void setSrc(const Instruction *, unsigned int s, int slot); |
void emitForm_MAD(const Instruction *); |
void emitForm_ADD(const Instruction *); |
void emitForm_MUL(const Instruction *); |
void emitForm_IMM(const Instruction *); |
void emitLoadStoreSizeLG(DataType ty, int pos); |
void emitLoadStoreSizeCS(DataType ty); |
void roundMode_MAD(const Instruction *); |
void roundMode_CVT(RoundMode); |
void emitMNeg12(const Instruction *); |
void emitLOAD(const Instruction *); |
void emitSTORE(const Instruction *); |
void emitMOV(const Instruction *); |
void emitNOP(); |
void emitINTERP(const Instruction *); |
void emitPFETCH(const Instruction *); |
void emitOUT(const Instruction *); |
void emitUADD(const Instruction *); |
void emitAADD(const Instruction *); |
void emitFADD(const Instruction *); |
void emitIMUL(const Instruction *); |
void emitFMUL(const Instruction *); |
void emitFMAD(const Instruction *); |
void emitIMAD(const Instruction *); |
void emitISAD(const Instruction *); |
void emitMINMAX(const Instruction *); |
void emitPreOp(const Instruction *); |
void emitSFnOp(const Instruction *, uint8_t subOp); |
void emitShift(const Instruction *); |
void emitARL(const Instruction *, unsigned int shl); |
void emitLogicOp(const Instruction *); |
void emitNOT(const Instruction *); |
void emitCVT(const Instruction *); |
void emitSET(const Instruction *); |
void emitTEX(const TexInstruction *); |
void emitTXQ(const TexInstruction *); |
void emitTEXPREP(const TexInstruction *); |
void emitQUADOP(const Instruction *, uint8_t lane, uint8_t quOp); |
void emitFlow(const Instruction *, uint8_t flowOp); |
void emitPRERETEmu(const FlowInstruction *); |
void emitBAR(const Instruction *); |
void emitATOM(const Instruction *); |
}; |
#define SDATA(a) ((a).rep()->reg.data) |
#define DDATA(a) ((a).rep()->reg.data) |
void CodeEmitterNV50::srcId(const ValueRef& src, const int pos) |
{ |
assert(src.get()); |
code[pos / 32] |= SDATA(src).id << (pos % 32); |
} |
void CodeEmitterNV50::srcId(const ValueRef *src, const int pos) |
{ |
assert(src->get()); |
code[pos / 32] |= SDATA(*src).id << (pos % 32); |
} |
void CodeEmitterNV50::srcAddr16(const ValueRef& src, bool adj, const int pos) |
{ |
assert(src.get()); |
int32_t offset = SDATA(src).offset; |
assert(!adj || src.get()->reg.size <= 4); |
if (adj) |
offset /= src.get()->reg.size; |
assert(offset <= 0x7fff && offset >= (int32_t)-0x8000 && (pos % 32) <= 16); |
if (offset < 0) |
offset &= adj ? (0xffff >> (src.get()->reg.size >> 1)) : 0xffff; |
code[pos / 32] |= offset << (pos % 32); |
} |
void CodeEmitterNV50::srcAddr8(const ValueRef& src, const int pos) |
{ |
assert(src.get()); |
uint32_t offset = SDATA(src).offset; |
assert((offset <= 0x1fc || offset == 0x3fc) && !(offset & 0x3)); |
code[pos / 32] |= (offset >> 2) << (pos % 32); |
} |
void CodeEmitterNV50::defId(const ValueDef& def, const int pos) |
{ |
assert(def.get() && def.getFile() != FILE_SHADER_OUTPUT); |
code[pos / 32] |= DDATA(def).id << (pos % 32); |
} |
void |
CodeEmitterNV50::roundMode_MAD(const Instruction *insn) |
{ |
switch (insn->rnd) { |
case ROUND_M: code[1] |= 1 << 22; break; |
case ROUND_P: code[1] |= 2 << 22; break; |
case ROUND_Z: code[1] |= 3 << 22; break; |
default: |
assert(insn->rnd == ROUND_N); |
break; |
} |
} |
void |
CodeEmitterNV50::emitMNeg12(const Instruction *i) |
{ |
code[1] |= i->src(0).mod.neg() << 26; |
code[1] |= i->src(1).mod.neg() << 27; |
} |
void CodeEmitterNV50::emitCondCode(CondCode cc, DataType ty, int pos) |
{ |
uint8_t enc; |
assert(pos >= 32 || pos <= 27); |
switch (cc) { |
case CC_LT: enc = 0x1; break; |
case CC_LTU: enc = 0x9; break; |
case CC_EQ: enc = 0x2; break; |
case CC_EQU: enc = 0xa; break; |
case CC_LE: enc = 0x3; break; |
case CC_LEU: enc = 0xb; break; |
case CC_GT: enc = 0x4; break; |
case CC_GTU: enc = 0xc; break; |
case CC_NE: enc = 0x5; break; |
case CC_NEU: enc = 0xd; break; |
case CC_GE: enc = 0x6; break; |
case CC_GEU: enc = 0xe; break; |
case CC_TR: enc = 0xf; break; |
case CC_FL: enc = 0x0; break; |
case CC_O: enc = 0x10; break; |
case CC_C: enc = 0x11; break; |
case CC_A: enc = 0x12; break; |
case CC_S: enc = 0x13; break; |
case CC_NS: enc = 0x1c; break; |
case CC_NA: enc = 0x1d; break; |
case CC_NC: enc = 0x1e; break; |
case CC_NO: enc = 0x1f; break; |
default: |
enc = 0; |
assert(!"invalid condition code"); |
break; |
} |
if (ty != TYPE_NONE && !isFloatType(ty)) |
enc &= ~0x8; // unordered only exists for float types |
code[pos / 32] |= enc << (pos % 32); |
} |
void |
CodeEmitterNV50::emitFlagsRd(const Instruction *i) |
{ |
int s = (i->flagsSrc >= 0) ? i->flagsSrc : i->predSrc; |
assert(!(code[1] & 0x00003f80)); |
if (s >= 0) { |
assert(i->getSrc(s)->reg.file == FILE_FLAGS); |
emitCondCode(i->cc, TYPE_NONE, 32 + 7); |
srcId(i->src(s), 32 + 12); |
} else { |
code[1] |= 0x0780; |
} |
} |
void |
CodeEmitterNV50::emitFlagsWr(const Instruction *i) |
{ |
assert(!(code[1] & 0x70)); |
int flagsDef = i->flagsDef; |
// find flags definition and check that it is the last def |
if (flagsDef < 0) { |
for (int d = 0; i->defExists(d); ++d) |
if (i->def(d).getFile() == FILE_FLAGS) |
flagsDef = d; |
if (flagsDef >= 0 && 0) // TODO: enforce use of flagsDef at some point |
WARN("Instruction::flagsDef was not set properly\n"); |
} |
if (flagsDef == 0 && i->defExists(1)) |
WARN("flags def should not be the primary definition\n"); |
if (flagsDef >= 0) |
code[1] |= (DDATA(i->def(flagsDef)).id << 4) | 0x40; |
} |
void |
CodeEmitterNV50::setARegBits(unsigned int u) |
{ |
code[0] |= (u & 3) << 26; |
code[1] |= (u & 4); |
} |
void |
CodeEmitterNV50::setAReg16(const Instruction *i, int s) |
{ |
if (i->srcExists(s)) { |
s = i->src(s).indirect[0]; |
if (s >= 0) |
setARegBits(SDATA(i->src(s)).id + 1); |
} |
} |
void |
CodeEmitterNV50::setImmediate(const Instruction *i, int s) |
{ |
const ImmediateValue *imm = i->src(s).get()->asImm(); |
assert(imm); |
uint32_t u = imm->reg.data.u32; |
if (i->src(s).mod & Modifier(NV50_IR_MOD_NOT)) |
u = ~u; |
code[1] |= 3; |
code[0] |= (u & 0x3f) << 16; |
code[1] |= (u >> 6) << 2; |
} |
void |
CodeEmitterNV50::setDst(const Value *dst) |
{ |
const Storage *reg = &dst->join->reg; |
assert(reg->file != FILE_ADDRESS); |
if (reg->data.id < 0 || reg->file == FILE_FLAGS) { |
code[0] |= (127 << 2) | 1; |
code[1] |= 8; |
} else { |
int id; |
if (reg->file == FILE_SHADER_OUTPUT) { |
code[1] |= 8; |
id = reg->data.offset / 4; |
} else { |
id = reg->data.id; |
} |
code[0] |= id << 2; |
} |
} |
void |
CodeEmitterNV50::setDst(const Instruction *i, int d) |
{ |
if (i->defExists(d)) { |
setDst(i->getDef(d)); |
} else |
if (!d) { |
code[0] |= 0x01fc; // bit bucket |
code[1] |= 0x0008; |
} |
} |
// 3 * 2 bits: |
// 0: r |
// 1: a/s |
// 2: c |
// 3: i |
void |
CodeEmitterNV50::setSrcFileBits(const Instruction *i, int enc) |
{ |
uint8_t mode = 0; |
for (unsigned int s = 0; s < Target::operationSrcNr[i->op]; ++s) { |
switch (i->src(s).getFile()) { |
case FILE_GPR: |
break; |
case FILE_MEMORY_SHARED: |
case FILE_SHADER_INPUT: |
mode |= 1 << (s * 2); |
break; |
case FILE_MEMORY_CONST: |
mode |= 2 << (s * 2); |
break; |
case FILE_IMMEDIATE: |
mode |= 3 << (s * 2); |
break; |
default: |
ERROR("invalid file on source %i: %u\n", s, i->src(s).getFile()); |
assert(0); |
break; |
} |
} |
switch (mode) { |
case 0x00: // rrr |
break; |
case 0x01: // arr/grr |
if (progType == Program::TYPE_GEOMETRY) { |
code[0] |= 0x01800000; |
if (enc == NV50_OP_ENC_LONG || enc == NV50_OP_ENC_LONG_ALT) |
code[1] |= 0x00200000; |
} else { |
if (enc == NV50_OP_ENC_SHORT) |
code[0] |= 0x01000000; |
else |
code[1] |= 0x00200000; |
} |
break; |
case 0x03: // irr |
assert(i->op == OP_MOV); |
return; |
case 0x0c: // rir |
break; |
case 0x0d: // gir |
code[0] |= 0x01000000; |
assert(progType == Program::TYPE_GEOMETRY || |
progType == Program::TYPE_COMPUTE); |
break; |
case 0x08: // rcr |
code[0] |= (enc == NV50_OP_ENC_LONG_ALT) ? 0x01000000 : 0x00800000; |
code[1] |= (i->getSrc(1)->reg.fileIndex << 22); |
break; |
case 0x09: // acr/gcr |
if (progType == Program::TYPE_GEOMETRY) { |
code[0] |= 0x01800000; |
} else { |
code[0] |= (enc == NV50_OP_ENC_LONG_ALT) ? 0x01000000 : 0x00800000; |
code[1] |= 0x00200000; |
} |
code[1] |= (i->getSrc(1)->reg.fileIndex << 22); |
break; |
case 0x20: // rrc |
code[0] |= 0x01000000; |
code[1] |= (i->getSrc(2)->reg.fileIndex << 22); |
break; |
case 0x21: // arc |
code[0] |= 0x01000000; |
code[1] |= 0x00200000 | (i->getSrc(2)->reg.fileIndex << 22); |
assert(progType != Program::TYPE_GEOMETRY); |
break; |
default: |
ERROR("not encodable: %x\n", mode); |
assert(0); |
break; |
} |
if (progType != Program::TYPE_COMPUTE) |
return; |
if ((mode & 3) == 1) { |
const int pos = i->src(1).getFile() == FILE_IMMEDIATE ? 13 : 14; |
switch (i->getSrc(0)->reg.type) { |
case TYPE_U8: |
break; |
case TYPE_U16: |
code[0] |= 1 << pos; |
break; |
case TYPE_S16: |
code[0] |= 2 << pos; |
break; |
default: |
code[0] |= 3 << pos; |
assert(i->getSrc(0)->reg.size == 4); |
break; |
} |
} |
} |
void |
CodeEmitterNV50::setSrc(const Instruction *i, unsigned int s, int slot) |
{ |
if (Target::operationSrcNr[i->op] <= s) |
return; |
const Storage *reg = &i->src(s).rep()->reg; |
unsigned int id = (reg->file == FILE_GPR) ? |
reg->data.id : |
reg->data.offset >> (reg->size >> 1); // no > 4 byte sources here |
switch (slot) { |
case 0: code[0] |= id << 9; break; |
case 1: code[0] |= id << 16; break; |
case 2: code[1] |= id << 14; break; |
default: |
assert(0); |
break; |
} |
} |
// the default form: |
// - long instruction |
// - 1 to 3 sources in slots 0, 1, 2 (rrr, arr, rcr, acr, rrc, arc, gcr, grr) |
// - address & flags |
void |
CodeEmitterNV50::emitForm_MAD(const Instruction *i) |
{ |
assert(i->encSize == 8); |
code[0] |= 1; |
emitFlagsRd(i); |
emitFlagsWr(i); |
setDst(i, 0); |
setSrcFileBits(i, NV50_OP_ENC_LONG); |
setSrc(i, 0, 0); |
setSrc(i, 1, 1); |
setSrc(i, 2, 2); |
setAReg16(i, 1); |
} |
// like default form, but 2nd source in slot 2, and no 3rd source |
void |
CodeEmitterNV50::emitForm_ADD(const Instruction *i) |
{ |
assert(i->encSize == 8); |
code[0] |= 1; |
emitFlagsRd(i); |
emitFlagsWr(i); |
setDst(i, 0); |
setSrcFileBits(i, NV50_OP_ENC_LONG_ALT); |
setSrc(i, 0, 0); |
setSrc(i, 1, 2); |
setAReg16(i, 1); |
} |
// default short form (rr, ar, rc, gr) |
void |
CodeEmitterNV50::emitForm_MUL(const Instruction *i) |
{ |
assert(i->encSize == 4 && !(code[0] & 1)); |
assert(i->defExists(0)); |
assert(!i->getPredicate()); |
setDst(i, 0); |
setSrcFileBits(i, NV50_OP_ENC_SHORT); |
setSrc(i, 0, 0); |
setSrc(i, 1, 1); |
} |
// usual immediate form |
// - 1 to 3 sources where last is immediate (rir, gir) |
// - no address or predicate possible |
void |
CodeEmitterNV50::emitForm_IMM(const Instruction *i) |
{ |
assert(i->encSize == 8); |
code[0] |= 1; |
assert(i->defExists(0) && i->srcExists(0)); |
setDst(i, 0); |
setSrcFileBits(i, NV50_OP_ENC_IMM); |
if (Target::operationSrcNr[i->op] > 1) { |
setSrc(i, 0, 0); |
setImmediate(i, 1); |
setSrc(i, 2, 1); |
} else { |
setImmediate(i, 0); |
} |
} |
void |
CodeEmitterNV50::emitLoadStoreSizeLG(DataType ty, int pos) |
{ |
uint8_t enc; |
switch (ty) { |
case TYPE_F32: // fall through |
case TYPE_S32: // fall through |
case TYPE_U32: enc = 0x6; break; |
case TYPE_B128: enc = 0x5; break; |
case TYPE_F64: // fall through |
case TYPE_S64: // fall through |
case TYPE_U64: enc = 0x4; break; |
case TYPE_S16: enc = 0x3; break; |
case TYPE_U16: enc = 0x2; break; |
case TYPE_S8: enc = 0x1; break; |
case TYPE_U8: enc = 0x0; break; |
default: |
enc = 0; |
assert(!"invalid load/store type"); |
break; |
} |
code[pos / 32] |= enc << (pos % 32); |
} |
void |
CodeEmitterNV50::emitLoadStoreSizeCS(DataType ty) |
{ |
switch (ty) { |
case TYPE_U8: break; |
case TYPE_U16: code[1] |= 0x4000; break; |
case TYPE_S16: code[1] |= 0x8000; break; |
case TYPE_F32: |
case TYPE_S32: |
case TYPE_U32: code[1] |= 0xc000; break; |
default: |
assert(0); |
break; |
} |
} |
void |
CodeEmitterNV50::emitLOAD(const Instruction *i) |
{ |
DataFile sf = i->src(0).getFile(); |
int32_t offset = i->getSrc(0)->reg.data.offset; |
switch (sf) { |
case FILE_SHADER_INPUT: |
// use 'mov' where we can |
code[0] = i->src(0).isIndirect(0) ? 0x00000001 : 0x10000001; |
code[1] = 0x00200000 | (i->lanes << 14); |
if (typeSizeof(i->dType) == 4) |
code[1] |= 0x04000000; |
break; |
case FILE_MEMORY_SHARED: |
if (targ->getChipset() >= 0x84) { |
assert(offset <= (int32_t)(0x3fff * typeSizeof(i->sType))); |
code[0] = 0x10000001; |
code[1] = 0x40000000; |
if (typeSizeof(i->dType) == 4) |
code[1] |= 0x04000000; |
emitLoadStoreSizeCS(i->sType); |
} else { |
assert(offset <= (int32_t)(0x1f * typeSizeof(i->sType))); |
code[0] = 0x10000001; |
code[1] = 0x00200000 | (i->lanes << 14); |
emitLoadStoreSizeCS(i->sType); |
} |
break; |
case FILE_MEMORY_CONST: |
code[0] = 0x10000001; |
code[1] = 0x20000000 | (i->getSrc(0)->reg.fileIndex << 22); |
if (typeSizeof(i->dType) == 4) |
code[1] |= 0x04000000; |
emitLoadStoreSizeCS(i->sType); |
break; |
case FILE_MEMORY_LOCAL: |
code[0] = 0xd0000001; |
code[1] = 0x40000000; |
break; |
case FILE_MEMORY_GLOBAL: |
code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16); |
code[1] = 0x80000000; |
break; |
default: |
assert(!"invalid load source file"); |
break; |
} |
if (sf == FILE_MEMORY_LOCAL || |
sf == FILE_MEMORY_GLOBAL) |
emitLoadStoreSizeLG(i->sType, 21 + 32); |
setDst(i, 0); |
emitFlagsRd(i); |
emitFlagsWr(i); |
if (i->src(0).getFile() == FILE_MEMORY_GLOBAL) { |
srcId(*i->src(0).getIndirect(0), 9); |
} else { |
setAReg16(i, 0); |
srcAddr16(i->src(0), i->src(0).getFile() != FILE_MEMORY_LOCAL, 9); |
} |
} |
void |
CodeEmitterNV50::emitSTORE(const Instruction *i) |
{ |
DataFile f = i->getSrc(0)->reg.file; |
int32_t offset = i->getSrc(0)->reg.data.offset; |
switch (f) { |
case FILE_SHADER_OUTPUT: |
code[0] = 0x00000001 | ((offset >> 2) << 9); |
code[1] = 0x80c00000; |
srcId(i->src(1), 32 + 14); |
break; |
case FILE_MEMORY_GLOBAL: |
code[0] = 0xd0000001 | (i->getSrc(0)->reg.fileIndex << 16); |
code[1] = 0xa0000000; |
emitLoadStoreSizeLG(i->dType, 21 + 32); |
srcId(i->src(1), 2); |
break; |
case FILE_MEMORY_LOCAL: |
code[0] = 0xd0000001; |
code[1] = 0x60000000; |
emitLoadStoreSizeLG(i->dType, 21 + 32); |
srcId(i->src(1), 2); |
break; |
case FILE_MEMORY_SHARED: |
code[0] = 0x00000001; |
code[1] = 0xe0000000; |
switch (typeSizeof(i->dType)) { |
case 1: |
code[0] |= offset << 9; |
code[1] |= 0x00400000; |
break; |
case 2: |
code[0] |= (offset >> 1) << 9; |
break; |
case 4: |
code[0] |= (offset >> 2) << 9; |
code[1] |= 0x04200000; |
break; |
default: |
assert(0); |
break; |
} |
srcId(i->src(1), 32 + 14); |
break; |
default: |
assert(!"invalid store destination file"); |
break; |
} |
if (f == FILE_MEMORY_GLOBAL) |
srcId(*i->src(0).getIndirect(0), 9); |
else |
setAReg16(i, 0); |
if (f == FILE_MEMORY_LOCAL) |
srcAddr16(i->src(0), false, 9); |
emitFlagsRd(i); |
} |
void |
CodeEmitterNV50::emitMOV(const Instruction *i) |
{ |
DataFile sf = i->getSrc(0)->reg.file; |
DataFile df = i->getDef(0)->reg.file; |
assert(sf == FILE_GPR || df == FILE_GPR); |
if (sf == FILE_FLAGS) { |
code[0] = 0x00000001; |
code[1] = 0x20000000; |
defId(i->def(0), 2); |
srcId(i->src(0), 12); |
emitFlagsRd(i); |
} else |
if (sf == FILE_ADDRESS) { |
code[0] = 0x00000001; |
code[1] = 0x40000000; |
defId(i->def(0), 2); |
setARegBits(SDATA(i->src(0)).id + 1); |
emitFlagsRd(i); |
} else |
if (df == FILE_FLAGS) { |
code[0] = 0x00000001; |
code[1] = 0xa0000000; |
defId(i->def(0), 4); |
srcId(i->src(0), 9); |
emitFlagsRd(i); |
} else |
if (sf == FILE_IMMEDIATE) { |
code[0] = 0x10008001; |
code[1] = 0x00000003; |
emitForm_IMM(i); |
} else { |
if (i->encSize == 4) { |
code[0] = 0x10008000; |
} else { |
code[0] = 0x10000001; |
code[1] = (typeSizeof(i->dType) == 2) ? 0 : 0x04000000; |
code[1] |= (i->lanes << 14); |
emitFlagsRd(i); |
} |
defId(i->def(0), 2); |
srcId(i->src(0), 9); |
} |
if (df == FILE_SHADER_OUTPUT) { |
assert(i->encSize == 8); |
code[1] |= 0x8; |
} |
} |
void |
CodeEmitterNV50::emitNOP() |
{ |
code[0] = 0xf0000001; |
code[1] = 0xe0000000; |
} |
void |
CodeEmitterNV50::emitQUADOP(const Instruction *i, uint8_t lane, uint8_t quOp) |
{ |
code[0] = 0xc0000000 | (lane << 16); |
code[1] = 0x80000000; |
code[0] |= (quOp & 0x03) << 20; |
code[1] |= (quOp & 0xfc) << 20; |
emitForm_ADD(i); |
if (!i->srcExists(1)) |
srcId(i->src(0), 32 + 14); |
} |
void |
CodeEmitterNV50::emitPFETCH(const Instruction *i) |
{ |
code[0] = 0x11800001; |
code[1] = 0x04200000 | (0xf << 14); |
defId(i->def(0), 2); |
srcAddr8(i->src(0), 9); |
setAReg16(i, 0); |
} |
void |
CodeEmitterNV50::emitINTERP(const Instruction *i) |
{ |
code[0] = 0x80000000; |
defId(i->def(0), 2); |
srcAddr8(i->src(0), 16); |
if (i->getInterpMode() == NV50_IR_INTERP_FLAT) { |
code[0] |= 1 << 8; |
} else { |
if (i->op == OP_PINTERP) { |
code[0] |= 1 << 25; |
srcId(i->src(1), 9); |
} |
if (i->getSampleMode() == NV50_IR_INTERP_CENTROID) |
code[0] |= 1 << 24; |
} |
if (i->encSize == 8) { |
code[1] = |
(code[0] & (3 << 24)) >> (24 - 16) | |
(code[0] & (1 << 8)) << (18 - 8); |
code[0] &= ~0x03000100; |
code[0] |= 1; |
emitFlagsRd(i); |
} |
} |
void |
CodeEmitterNV50::emitMINMAX(const Instruction *i) |
{ |
if (i->dType == TYPE_F64) { |
code[0] = 0xe0000000; |
code[1] = (i->op == OP_MIN) ? 0xa0000000 : 0xc0000000; |
} else { |
code[0] = 0x30000000; |
code[1] = 0x80000000; |
if (i->op == OP_MIN) |
code[1] |= 0x20000000; |
switch (i->dType) { |
case TYPE_F32: code[0] |= 0x80000000; break; |
case TYPE_S32: code[1] |= 0x8c000000; break; |
case TYPE_U32: code[1] |= 0x84000000; break; |
case TYPE_S16: code[1] |= 0x80000000; break; |
case TYPE_U16: break; |
default: |
assert(0); |
break; |
} |
code[1] |= i->src(0).mod.abs() << 20; |
code[1] |= i->src(1).mod.abs() << 19; |
} |
emitForm_MAD(i); |
} |
void |
CodeEmitterNV50::emitFMAD(const Instruction *i) |
{ |
const int neg_mul = i->src(0).mod.neg() ^ i->src(1).mod.neg(); |
const int neg_add = i->src(2).mod.neg(); |
code[0] = 0xe0000000; |
if (i->encSize == 4) { |
emitForm_MUL(i); |
assert(!neg_mul && !neg_add); |
} else { |
code[1] = neg_mul << 26; |
code[1] |= neg_add << 27; |
if (i->saturate) |
code[1] |= 1 << 29; |
emitForm_MAD(i); |
} |
} |
void |
CodeEmitterNV50::emitFADD(const Instruction *i) |
{ |
const int neg0 = i->src(0).mod.neg(); |
const int neg1 = i->src(1).mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0); |
code[0] = 0xb0000000; |
assert(!(i->src(0).mod | i->src(1).mod).abs()); |
if (i->src(1).getFile() == FILE_IMMEDIATE) { |
code[1] = 0; |
emitForm_IMM(i); |
code[0] |= neg0 << 15; |
code[0] |= neg1 << 22; |
if (i->saturate) |
code[0] |= 1 << 8; |
} else |
if (i->encSize == 8) { |
code[1] = 0; |
emitForm_ADD(i); |
code[1] |= neg0 << 26; |
code[1] |= neg1 << 27; |
if (i->saturate) |
code[1] |= 1 << 29; |
} else { |
emitForm_MUL(i); |
code[0] |= neg0 << 15; |
code[0] |= neg1 << 22; |
if (i->saturate) |
code[0] |= 1 << 8; |
} |
} |
void |
CodeEmitterNV50::emitUADD(const Instruction *i) |
{ |
const int neg0 = i->src(0).mod.neg(); |
const int neg1 = i->src(1).mod.neg() ^ ((i->op == OP_SUB) ? 1 : 0); |
code[0] = 0x20008000; |
if (i->src(1).getFile() == FILE_IMMEDIATE) { |
code[1] = 0; |
emitForm_IMM(i); |
} else |
if (i->encSize == 8) { |
code[0] = 0x20000000; |
code[1] = (typeSizeof(i->dType) == 2) ? 0 : 0x04000000; |
emitForm_ADD(i); |
} else { |
emitForm_MUL(i); |
} |
assert(!(neg0 && neg1)); |
code[0] |= neg0 << 28; |
code[0] |= neg1 << 22; |
if (i->flagsSrc >= 0) { |
// addc == sub | subr |
assert(!(code[0] & 0x10400000) && !i->getPredicate()); |
code[0] |= 0x10400000; |
srcId(i->src(i->flagsSrc), 32 + 12); |
} |
} |
void |
CodeEmitterNV50::emitAADD(const Instruction *i) |
{ |
const int s = (i->op == OP_MOV) ? 0 : 1; |
code[0] = 0xd0000001 | (i->getSrc(s)->reg.data.u16 << 9); |
code[1] = 0x20000000; |
code[0] |= (DDATA(i->def(0)).id + 1) << 2; |
emitFlagsRd(i); |
if (s && i->srcExists(0)) |
setARegBits(SDATA(i->src(0)).id + 1); |
} |
void |
CodeEmitterNV50::emitIMUL(const Instruction *i) |
{ |
code[0] = 0x40000000; |
if (i->encSize == 8) { |
code[1] = (i->sType == TYPE_S16) ? (0x8000 | 0x4000) : 0x0000; |
emitForm_MAD(i); |
} else { |
if (i->sType == TYPE_S16) |
code[0] |= 0x8100; |
emitForm_MUL(i); |
} |
} |
void |
CodeEmitterNV50::emitFMUL(const Instruction *i) |
{ |
const int neg = (i->src(0).mod ^ i->src(1).mod).neg(); |
code[0] = 0xc0000000; |
if (i->src(1).getFile() == FILE_IMMEDIATE) { |
code[1] = 0; |
emitForm_IMM(i); |
if (neg) |
code[0] |= 0x8000; |
} else |
if (i->encSize == 8) { |
code[1] = i->rnd == ROUND_Z ? 0x0000c000 : 0; |
if (neg) |
code[1] |= 0x08000000; |
emitForm_MAD(i); |
} else { |
emitForm_MUL(i); |
if (neg) |
code[0] |= 0x8000; |
} |
} |
void |
CodeEmitterNV50::emitIMAD(const Instruction *i) |
{ |
code[0] = 0x60000000; |
if (isSignedType(i->sType)) |
code[1] = i->saturate ? 0x40000000 : 0x20000000; |
else |
code[1] = 0x00000000; |
int neg1 = i->src(0).mod.neg() ^ i->src(1).mod.neg(); |
int neg2 = i->src(2).mod.neg(); |
assert(!(neg1 & neg2)); |
code[1] |= neg1 << 27; |
code[1] |= neg2 << 26; |
emitForm_MAD(i); |
if (i->flagsSrc >= 0) { |
// add with carry from $cX |
assert(!(code[1] & 0x0c000000) && !i->getPredicate()); |
code[1] |= 0xc << 24; |
srcId(i->src(i->flagsSrc), 32 + 12); |
} |
} |
void |
CodeEmitterNV50::emitISAD(const Instruction *i) |
{ |
if (i->encSize == 8) { |
code[0] = 0x50000000; |
switch (i->sType) { |
case TYPE_U32: code[1] = 0x04000000; break; |
case TYPE_S32: code[1] = 0x0c000000; break; |
case TYPE_U16: code[1] = 0x00000000; break; |
case TYPE_S16: code[1] = 0x08000000; break; |
default: |
assert(0); |
break; |
} |
emitForm_MAD(i); |
} else { |
switch (i->sType) { |
case TYPE_U32: code[0] = 0x50008000; break; |
case TYPE_S32: code[0] = 0x50008100; break; |
case TYPE_U16: code[0] = 0x50000000; break; |
case TYPE_S16: code[0] = 0x50000100; break; |
default: |
assert(0); |
break; |
} |
emitForm_MUL(i); |
} |
} |
void |
CodeEmitterNV50::emitSET(const Instruction *i) |
{ |
code[0] = 0x30000000; |
code[1] = 0x60000000; |
emitCondCode(i->asCmp()->setCond, i->sType, 32 + 14); |
switch (i->sType) { |
case TYPE_F32: code[0] |= 0x80000000; break; |
case TYPE_S32: code[1] |= 0x0c000000; break; |
case TYPE_U32: code[1] |= 0x04000000; break; |
case TYPE_S16: code[1] |= 0x08000000; break; |
case TYPE_U16: break; |
default: |
assert(0); |
break; |
} |
if (i->src(0).mod.neg()) code[1] |= 0x04000000; |
if (i->src(1).mod.neg()) code[1] |= 0x08000000; |
if (i->src(0).mod.abs()) code[1] |= 0x00100000; |
if (i->src(1).mod.abs()) code[1] |= 0x00080000; |
emitForm_MAD(i); |
} |
void |
CodeEmitterNV50::roundMode_CVT(RoundMode rnd) |
{ |
switch (rnd) { |
case ROUND_NI: code[1] |= 0x08000000; break; |
case ROUND_M: code[1] |= 0x00020000; break; |
case ROUND_MI: code[1] |= 0x08020000; break; |
case ROUND_P: code[1] |= 0x00040000; break; |
case ROUND_PI: code[1] |= 0x08040000; break; |
case ROUND_Z: code[1] |= 0x00060000; break; |
case ROUND_ZI: code[1] |= 0x08060000; break; |
default: |
assert(rnd == ROUND_N); |
break; |
} |
} |
void |
CodeEmitterNV50::emitCVT(const Instruction *i) |
{ |
const bool f2f = isFloatType(i->dType) && isFloatType(i->sType); |
RoundMode rnd; |
switch (i->op) { |
case OP_CEIL: rnd = f2f ? ROUND_PI : ROUND_P; break; |
case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break; |
case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break; |
default: |
rnd = i->rnd; |
break; |
} |
code[0] = 0xa0000000; |
switch (i->dType) { |
case TYPE_F64: |
switch (i->sType) { |
case TYPE_F64: code[1] = 0xc4404000; break; |
case TYPE_S64: code[1] = 0x44414000; break; |
case TYPE_U64: code[1] = 0x44404000; break; |
case TYPE_F32: code[1] = 0xc4400000; break; |
case TYPE_S32: code[1] = 0x44410000; break; |
case TYPE_U32: code[1] = 0x44400000; break; |
default: |
assert(0); |
break; |
} |
break; |
case TYPE_S64: |
switch (i->sType) { |
case TYPE_F64: code[1] = 0x8c404000; break; |
case TYPE_F32: code[1] = 0x8c400000; break; |
default: |
assert(0); |
break; |
} |
break; |
case TYPE_U64: |
switch (i->sType) { |
case TYPE_F64: code[1] = 0x84404000; break; |
case TYPE_F32: code[1] = 0x84400000; break; |
default: |
assert(0); |
break; |
} |
break; |
case TYPE_F32: |
switch (i->sType) { |
case TYPE_F64: code[1] = 0xc0404000; break; |
case TYPE_S64: code[1] = 0x40414000; break; |
case TYPE_U64: code[1] = 0x40404000; break; |
case TYPE_F32: code[1] = 0xc4004000; break; |
case TYPE_S32: code[1] = 0x44014000; break; |
case TYPE_U32: code[1] = 0x44004000; break; |
case TYPE_F16: code[1] = 0xc4000000; break; |
default: |
assert(0); |
break; |
} |
break; |
case TYPE_S32: |
switch (i->sType) { |
case TYPE_F64: code[1] = 0x88404000; break; |
case TYPE_F32: code[1] = 0x8c004000; break; |
case TYPE_S32: code[1] = 0x0c014000; break; |
case TYPE_U32: code[1] = 0x0c004000; break; |
case TYPE_F16: code[1] = 0x8c000000; break; |
case TYPE_S16: code[1] = 0x0c010000; break; |
case TYPE_U16: code[1] = 0x0c000000; break; |
case TYPE_S8: code[1] = 0x0c018000; break; |
case TYPE_U8: code[1] = 0x0c008000; break; |
default: |
assert(0); |
break; |
} |
break; |
case TYPE_U32: |
switch (i->sType) { |
case TYPE_F64: code[1] = 0x80404000; break; |
case TYPE_F32: code[1] = 0x84004000; break; |
case TYPE_S32: code[1] = 0x04014000; break; |
case TYPE_U32: code[1] = 0x04004000; break; |
case TYPE_F16: code[1] = 0x84000000; break; |
case TYPE_S16: code[1] = 0x04010000; break; |
case TYPE_U16: code[1] = 0x04000000; break; |
case TYPE_S8: code[1] = 0x04018000; break; |
case TYPE_U8: code[1] = 0x04008000; break; |
default: |
assert(0); |
break; |
} |
break; |
case TYPE_S16: |
case TYPE_U16: |
case TYPE_S8: |
case TYPE_U8: |
default: |
assert(0); |
break; |
} |
if (typeSizeof(i->sType) == 1 && i->getSrc(0)->reg.size == 4) |
code[1] |= 0x00004000; |
roundMode_CVT(rnd); |
switch (i->op) { |
case OP_ABS: code[1] |= 1 << 20; break; |
case OP_SAT: code[1] |= 1 << 19; break; |
case OP_NEG: code[1] |= 1 << 29; break; |
default: |
break; |
} |
code[1] ^= i->src(0).mod.neg() << 29; |
code[1] |= i->src(0).mod.abs() << 20; |
if (i->saturate) |
code[1] |= 1 << 19; |
assert(i->op != OP_ABS || !i->src(0).mod.neg()); |
emitForm_MAD(i); |
} |
void |
CodeEmitterNV50::emitPreOp(const Instruction *i) |
{ |
code[0] = 0xb0000000; |
code[1] = (i->op == OP_PREEX2) ? 0xc0004000 : 0xc0000000; |
code[1] |= i->src(0).mod.abs() << 20; |
code[1] |= i->src(0).mod.neg() << 26; |
emitForm_MAD(i); |
} |
void |
CodeEmitterNV50::emitSFnOp(const Instruction *i, uint8_t subOp) |
{ |
code[0] = 0x90000000; |
if (i->encSize == 4) { |
assert(i->op == OP_RCP); |
code[0] |= i->src(0).mod.abs() << 15; |
code[0] |= i->src(0).mod.neg() << 22; |
emitForm_MUL(i); |
} else { |
code[1] = subOp << 29; |
code[1] |= i->src(0).mod.abs() << 20; |
code[1] |= i->src(0).mod.neg() << 26; |
emitForm_MAD(i); |
} |
} |
void |
CodeEmitterNV50::emitNOT(const Instruction *i) |
{ |
code[0] = 0xd0000000; |
code[1] = 0x0002c000; |
switch (i->sType) { |
case TYPE_U32: |
case TYPE_S32: |
code[1] |= 0x04000000; |
break; |
default: |
break; |
} |
emitForm_MAD(i); |
setSrc(i, 0, 1); |
} |
void |
CodeEmitterNV50::emitLogicOp(const Instruction *i) |
{ |
code[0] = 0xd0000000; |
code[1] = 0; |
if (i->src(1).getFile() == FILE_IMMEDIATE) { |
switch (i->op) { |
case OP_OR: code[0] |= 0x0100; break; |
case OP_XOR: code[0] |= 0x8000; break; |
default: |
assert(i->op == OP_AND); |
break; |
} |
if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT)) |
code[0] |= 1 << 22; |
emitForm_IMM(i); |
} else { |
switch (i->op) { |
case OP_AND: code[1] = 0x04000000; break; |
case OP_OR: code[1] = 0x04004000; break; |
case OP_XOR: code[1] = 0x04008000; break; |
default: |
assert(0); |
break; |
} |
if (i->src(0).mod & Modifier(NV50_IR_MOD_NOT)) |
code[1] |= 1 << 16; |
if (i->src(1).mod & Modifier(NV50_IR_MOD_NOT)) |
code[1] |= 1 << 17; |
emitForm_MAD(i); |
} |
} |
void |
CodeEmitterNV50::emitARL(const Instruction *i, unsigned int shl) |
{ |
code[0] = 0x00000001 | (shl << 16); |
code[1] = 0xc0000000; |
code[0] |= (DDATA(i->def(0)).id + 1) << 2; |
setSrcFileBits(i, NV50_OP_ENC_IMM); |
setSrc(i, 0, 0); |
emitFlagsRd(i); |
} |
void |
CodeEmitterNV50::emitShift(const Instruction *i) |
{ |
if (i->def(0).getFile() == FILE_ADDRESS) { |
assert(i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE); |
emitARL(i, i->getSrc(1)->reg.data.u32 & 0x3f); |
} else { |
code[0] = 0x30000001; |
code[1] = (i->op == OP_SHR) ? 0xe4000000 : 0xc4000000; |
if (i->op == OP_SHR && isSignedType(i->sType)) |
code[1] |= 1 << 27; |
if (i->src(1).getFile() == FILE_IMMEDIATE) { |
code[1] |= 1 << 20; |
code[0] |= (i->getSrc(1)->reg.data.u32 & 0x7f) << 16; |
defId(i->def(0), 2); |
srcId(i->src(0), 9); |
emitFlagsRd(i); |
} else { |
emitForm_MAD(i); |
} |
} |
} |
void |
CodeEmitterNV50::emitOUT(const Instruction *i) |
{ |
code[0] = (i->op == OP_EMIT) ? 0xf0000200 : 0xf0000400; |
code[1] = 0xc0000001; |
emitFlagsRd(i); |
} |
void |
CodeEmitterNV50::emitTEX(const TexInstruction *i) |
{ |
code[0] = 0xf0000001; |
code[1] = 0x00000000; |
switch (i->op) { |
case OP_TXB: |
code[1] = 0x20000000; |
break; |
case OP_TXL: |
code[1] = 0x40000000; |
break; |
case OP_TXF: |
code[0] |= 0x01000000; |
break; |
case OP_TXG: |
code[0] = 0x01000000; |
code[1] = 0x80000000; |
break; |
default: |
assert(i->op == OP_TEX); |
break; |
} |
code[0] |= i->tex.r << 9; |
code[0] |= i->tex.s << 17; |
int argc = i->tex.target.getArgCount(); |
if (i->op == OP_TXB || i->op == OP_TXL || i->op == OP_TXF) |
argc += 1; |
if (i->tex.target.isShadow()) |
argc += 1; |
assert(argc <= 4); |
code[0] |= (argc - 1) << 22; |
if (i->tex.target.isCube()) { |
code[0] |= 0x08000000; |
} else |
if (i->tex.useOffsets) { |
code[1] |= (i->tex.offset[0][0] & 0xf) << 24; |
code[1] |= (i->tex.offset[0][1] & 0xf) << 20; |
code[1] |= (i->tex.offset[0][2] & 0xf) << 16; |
} |
code[0] |= (i->tex.mask & 0x3) << 25; |
code[1] |= (i->tex.mask & 0xc) << 12; |
if (i->tex.liveOnly) |
code[1] |= 4; |
defId(i->def(0), 2); |
emitFlagsRd(i); |
} |
void |
CodeEmitterNV50::emitTXQ(const TexInstruction *i) |
{ |
assert(i->tex.query == TXQ_DIMS); |
code[0] = 0xf0000001; |
code[1] = 0x60000000; |
code[0] |= i->tex.r << 9; |
code[0] |= i->tex.s << 17; |
code[0] |= (i->tex.mask & 0x3) << 25; |
code[1] |= (i->tex.mask & 0xc) << 12; |
defId(i->def(0), 2); |
emitFlagsRd(i); |
} |
void |
CodeEmitterNV50::emitTEXPREP(const TexInstruction *i) |
{ |
code[0] = 0xf8000001 | (3 << 22) | (i->tex.s << 17) | (i->tex.r << 9); |
code[1] = 0x60010000; |
code[0] |= (i->tex.mask & 0x3) << 25; |
code[1] |= (i->tex.mask & 0xc) << 12; |
defId(i->def(0), 2); |
emitFlagsRd(i); |
} |
void |
CodeEmitterNV50::emitPRERETEmu(const FlowInstruction *i) |
{ |
uint32_t pos = i->target.bb->binPos + 8; // +8 to skip an op */ |
code[0] = 0x10000003; // bra |
code[1] = 0x00000780; // always |
switch (i->subOp) { |
case NV50_IR_SUBOP_EMU_PRERET + 0: // bra to the call |
break; |
case NV50_IR_SUBOP_EMU_PRERET + 1: // bra to skip the call |
pos += 8; |
break; |
default: |
assert(i->subOp == (NV50_IR_SUBOP_EMU_PRERET + 2)); |
code[0] = 0x20000003; // call |
code[1] = 0x00000000; // no predicate |
break; |
} |
addReloc(RelocEntry::TYPE_CODE, 0, pos, 0x07fff800, 9); |
addReloc(RelocEntry::TYPE_CODE, 1, pos, 0x000fc000, -4); |
} |
void |
CodeEmitterNV50::emitFlow(const Instruction *i, uint8_t flowOp) |
{ |
const FlowInstruction *f = i->asFlow(); |
bool hasPred = false; |
bool hasTarg = false; |
code[0] = 0x00000003 | (flowOp << 28); |
code[1] = 0x00000000; |
switch (i->op) { |
case OP_BRA: |
hasPred = true; |
hasTarg = true; |
break; |
case OP_BREAK: |
case OP_BRKPT: |
case OP_DISCARD: |
case OP_RET: |
hasPred = true; |
break; |
case OP_CALL: |
case OP_PREBREAK: |
case OP_JOINAT: |
hasTarg = true; |
break; |
case OP_PRERET: |
hasTarg = true; |
if (i->subOp >= NV50_IR_SUBOP_EMU_PRERET) { |
emitPRERETEmu(f); |
return; |
} |
break; |
default: |
break; |
} |
if (hasPred) |
emitFlagsRd(i); |
if (hasTarg && f) { |
uint32_t pos; |
if (f->op == OP_CALL) { |
if (f->builtin) { |
pos = targNV50->getBuiltinOffset(f->target.builtin); |
} else { |
pos = f->target.fn->binPos; |
} |
} else { |
pos = f->target.bb->binPos; |
} |
code[0] |= ((pos >> 2) & 0xffff) << 11; |
code[1] |= ((pos >> 18) & 0x003f) << 14; |
RelocEntry::Type relocTy; |
relocTy = f->builtin ? RelocEntry::TYPE_BUILTIN : RelocEntry::TYPE_CODE; |
addReloc(relocTy, 0, pos, 0x07fff800, 9); |
addReloc(relocTy, 1, pos, 0x000fc000, -4); |
} |
} |
void |
CodeEmitterNV50::emitBAR(const Instruction *i) |
{ |
ImmediateValue *barId = i->getSrc(0)->asImm(); |
assert(barId); |
code[0] = 0x82000003 | (barId->reg.data.u32 << 21); |
code[1] = 0x00004000; |
if (i->subOp == NV50_IR_SUBOP_BAR_SYNC) |
code[0] |= 1 << 26; |
} |
void |
CodeEmitterNV50::emitATOM(const Instruction *i) |
{ |
uint8_t subOp; |
switch (i->subOp) { |
case NV50_IR_SUBOP_ATOM_ADD: subOp = 0x0; break; |
case NV50_IR_SUBOP_ATOM_MIN: subOp = 0x7; break; |
case NV50_IR_SUBOP_ATOM_MAX: subOp = 0x6; break; |
case NV50_IR_SUBOP_ATOM_INC: subOp = 0x4; break; |
case NV50_IR_SUBOP_ATOM_DEC: subOp = 0x5; break; |
case NV50_IR_SUBOP_ATOM_AND: subOp = 0xa; break; |
case NV50_IR_SUBOP_ATOM_OR: subOp = 0xb; break; |
case NV50_IR_SUBOP_ATOM_XOR: subOp = 0xc; break; |
case NV50_IR_SUBOP_ATOM_CAS: subOp = 0x2; break; |
case NV50_IR_SUBOP_ATOM_EXCH: subOp = 0x1; break; |
default: |
assert(!"invalid subop"); |
return; |
} |
code[0] = 0xd0000001; |
code[1] = 0xe0c00000 | (subOp << 2); |
if (isSignedType(i->dType)) |
code[1] |= 1 << 21; |
// args |
emitFlagsRd(i); |
setDst(i, 0); |
setSrc(i, 1, 1); |
if (i->subOp == NV50_IR_SUBOP_ATOM_CAS) |
setSrc(i, 2, 2); |
// g[] pointer |
code[0] |= i->getSrc(0)->reg.fileIndex << 23; |
srcId(i->getIndirect(0, 0), 9); |
} |
bool |
CodeEmitterNV50::emitInstruction(Instruction *insn) |
{ |
if (!insn->encSize) { |
ERROR("skipping unencodable instruction: "); insn->print(); |
return false; |
} else |
if (codeSize + insn->encSize > codeSizeLimit) { |
ERROR("code emitter output buffer too small\n"); |
return false; |
} |
if (insn->bb->getProgram()->dbgFlags & NV50_IR_DEBUG_BASIC) { |
INFO("EMIT: "); insn->print(); |
} |
switch (insn->op) { |
case OP_MOV: |
emitMOV(insn); |
break; |
case OP_EXIT: |
case OP_NOP: |
case OP_JOIN: |
emitNOP(); |
break; |
case OP_VFETCH: |
case OP_LOAD: |
emitLOAD(insn); |
break; |
case OP_EXPORT: |
case OP_STORE: |
emitSTORE(insn); |
break; |
case OP_PFETCH: |
emitPFETCH(insn); |
break; |
case OP_LINTERP: |
case OP_PINTERP: |
emitINTERP(insn); |
break; |
case OP_ADD: |
case OP_SUB: |
if (isFloatType(insn->dType)) |
emitFADD(insn); |
else if (insn->getDef(0)->reg.file == FILE_ADDRESS) |
emitAADD(insn); |
else |
emitUADD(insn); |
break; |
case OP_MUL: |
if (isFloatType(insn->dType)) |
emitFMUL(insn); |
else |
emitIMUL(insn); |
break; |
case OP_MAD: |
case OP_FMA: |
if (isFloatType(insn->dType)) |
emitFMAD(insn); |
else |
emitIMAD(insn); |
break; |
case OP_SAD: |
emitISAD(insn); |
break; |
case OP_NOT: |
emitNOT(insn); |
break; |
case OP_AND: |
case OP_OR: |
case OP_XOR: |
emitLogicOp(insn); |
break; |
case OP_SHL: |
case OP_SHR: |
emitShift(insn); |
break; |
case OP_SET: |
emitSET(insn); |
break; |
case OP_MIN: |
case OP_MAX: |
emitMINMAX(insn); |
break; |
case OP_CEIL: |
case OP_FLOOR: |
case OP_TRUNC: |
case OP_ABS: |
case OP_NEG: |
case OP_SAT: |
emitCVT(insn); |
break; |
case OP_CVT: |
if (insn->def(0).getFile() == FILE_ADDRESS) |
emitARL(insn, 0); |
else |
if (insn->def(0).getFile() == FILE_FLAGS || |
insn->src(0).getFile() == FILE_FLAGS || |
insn->src(0).getFile() == FILE_ADDRESS) |
emitMOV(insn); |
else |
emitCVT(insn); |
break; |
case OP_RCP: |
emitSFnOp(insn, 0); |
break; |
case OP_RSQ: |
emitSFnOp(insn, 2); |
break; |
case OP_LG2: |
emitSFnOp(insn, 3); |
break; |
case OP_SIN: |
emitSFnOp(insn, 4); |
break; |
case OP_COS: |
emitSFnOp(insn, 5); |
break; |
case OP_EX2: |
emitSFnOp(insn, 6); |
break; |
case OP_PRESIN: |
case OP_PREEX2: |
emitPreOp(insn); |
break; |
case OP_TEX: |
case OP_TXB: |
case OP_TXL: |
case OP_TXF: |
emitTEX(insn->asTex()); |
break; |
case OP_TXQ: |
emitTXQ(insn->asTex()); |
break; |
case OP_TEXPREP: |
emitTEXPREP(insn->asTex()); |
break; |
case OP_EMIT: |
case OP_RESTART: |
emitOUT(insn); |
break; |
case OP_DISCARD: |
emitFlow(insn, 0x0); |
break; |
case OP_BRA: |
emitFlow(insn, 0x1); |
break; |
case OP_CALL: |
emitFlow(insn, 0x2); |
break; |
case OP_RET: |
emitFlow(insn, 0x3); |
break; |
case OP_PREBREAK: |
emitFlow(insn, 0x4); |
break; |
case OP_BREAK: |
emitFlow(insn, 0x5); |
break; |
case OP_QUADON: |
emitFlow(insn, 0x6); |
break; |
case OP_QUADPOP: |
emitFlow(insn, 0x7); |
break; |
case OP_JOINAT: |
emitFlow(insn, 0xa); |
break; |
case OP_PRERET: |
emitFlow(insn, 0xd); |
break; |
case OP_QUADOP: |
emitQUADOP(insn, insn->lanes, insn->subOp); |
break; |
case OP_DFDX: |
emitQUADOP(insn, 4, insn->src(0).mod.neg() ? 0x66 : 0x99); |
break; |
case OP_DFDY: |
emitQUADOP(insn, 5, insn->src(0).mod.neg() ? 0x5a : 0xa5); |
break; |
case OP_ATOM: |
emitATOM(insn); |
break; |
case OP_BAR: |
emitBAR(insn); |
break; |
case OP_PHI: |
case OP_UNION: |
case OP_CONSTRAINT: |
ERROR("operation should have been eliminated\n"); |
return false; |
case OP_EXP: |
case OP_LOG: |
case OP_SQRT: |
case OP_POW: |
case OP_SELP: |
case OP_SLCT: |
case OP_TXD: |
case OP_PRECONT: |
case OP_CONT: |
case OP_POPCNT: |
case OP_INSBF: |
case OP_EXTBF: |
ERROR("operation should have been lowered\n"); |
return false; |
default: |
ERROR("unknown op: %u\n", insn->op); |
return false; |
} |
if (insn->join || insn->op == OP_JOIN) |
code[1] |= 0x2; |
else |
if (insn->exit || insn->op == OP_EXIT) |
code[1] |= 0x1; |
assert((insn->encSize == 8) == (code[0] & 1)); |
code += insn->encSize / 4; |
codeSize += insn->encSize; |
return true; |
} |
uint32_t |
CodeEmitterNV50::getMinEncodingSize(const Instruction *i) const |
{ |
const Target::OpInfo &info = targ->getOpInfo(i); |
if (info.minEncSize > 4) |
return 8; |
// check constraints on dst and src operands |
for (int d = 0; i->defExists(d); ++d) { |
if (i->def(d).rep()->reg.data.id > 63 || |
i->def(d).rep()->reg.file != FILE_GPR) |
return 8; |
} |
for (int s = 0; i->srcExists(s); ++s) { |
DataFile sf = i->src(s).getFile(); |
if (sf != FILE_GPR) |
if (sf != FILE_SHADER_INPUT || progType != Program::TYPE_FRAGMENT) |
return 8; |
if (i->src(s).rep()->reg.data.id > 63) |
return 8; |
} |
// check modifiers & rounding |
if (i->join || i->lanes != 0xf || i->exit) |
return 8; |
if (i->op == OP_MUL && i->rnd != ROUND_N) |
return 8; |
if (i->asTex()) |
return 8; // TODO: short tex encoding |
// check constraints on short MAD |
if (info.srcNr >= 2 && i->srcExists(2)) { |
if (i->saturate || i->src(2).mod) |
return 8; |
if ((i->src(0).mod ^ i->src(1).mod) || |
(i->src(0).mod | i->src(1).mod).abs()) |
return 8; |
if (!i->defExists(0) || |
i->def(0).rep()->reg.data.id != i->src(2).rep()->reg.data.id) |
return 8; |
} |
return info.minEncSize; |
} |
// Change the encoding size of an instruction after BBs have been scheduled. |
static void |
makeInstructionLong(Instruction *insn) |
{ |
if (insn->encSize == 8) |
return; |
Function *fn = insn->bb->getFunction(); |
int n = 0; |
int adj = 4; |
for (Instruction *i = insn->next; i && i->encSize == 4; ++n, i = i->next); |
if (n & 1) { |
adj = 8; |
insn->next->encSize = 8; |
} else |
if (insn->prev && insn->prev->encSize == 4) { |
adj = 8; |
insn->prev->encSize = 8; |
} |
insn->encSize = 8; |
for (int i = fn->bbCount - 1; i >= 0 && fn->bbArray[i] != insn->bb; --i) { |
fn->bbArray[i]->binPos += 4; |
} |
fn->binSize += adj; |
insn->bb->binSize += adj; |
} |
static bool |
trySetExitModifier(Instruction *insn) |
{ |
if (insn->op == OP_DISCARD || |
insn->op == OP_QUADON || |
insn->op == OP_QUADPOP) |
return false; |
for (int s = 0; insn->srcExists(s); ++s) |
if (insn->src(s).getFile() == FILE_IMMEDIATE) |
return false; |
if (insn->asFlow()) { |
if (insn->op == OP_CALL) // side effects ! |
return false; |
if (insn->getPredicate()) // cannot do conditional exit (or can we ?) |
return false; |
insn->op = OP_EXIT; |
} |
insn->exit = 1; |
makeInstructionLong(insn); |
return true; |
} |
static void |
replaceExitWithModifier(Function *func) |
{ |
BasicBlock *epilogue = BasicBlock::get(func->cfgExit); |
if (!epilogue->getExit() || |
epilogue->getExit()->op != OP_EXIT) // only main will use OP_EXIT |
return; |
if (epilogue->getEntry()->op != OP_EXIT) { |
Instruction *insn = epilogue->getExit()->prev; |
if (!insn || !trySetExitModifier(insn)) |
return; |
insn->exit = 1; |
} else { |
for (Graph::EdgeIterator ei = func->cfgExit->incident(); |
!ei.end(); ei.next()) { |
BasicBlock *bb = BasicBlock::get(ei.getNode()); |
Instruction *i = bb->getExit(); |
if (!i || !trySetExitModifier(i)) |
return; |
} |
} |
epilogue->binSize -= 8; |
func->binSize -= 8; |
delete_Instruction(func->getProgram(), epilogue->getExit()); |
} |
void |
CodeEmitterNV50::prepareEmission(Function *func) |
{ |
CodeEmitter::prepareEmission(func); |
replaceExitWithModifier(func); |
} |
CodeEmitterNV50::CodeEmitterNV50(const TargetNV50 *target) : |
CodeEmitter(target), targNV50(target) |
{ |
targ = target; // specialized |
code = NULL; |
codeSize = codeSizeLimit = 0; |
relocInfo = NULL; |
} |
CodeEmitter * |
TargetNV50::getCodeEmitter(Program::Type type) |
{ |
CodeEmitterNV50 *emit = new CodeEmitterNV50(this); |
emit->setProgramType(type); |
return emit; |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_from_tgsi.cpp |
---|
0,0 → 1,2835 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
extern "C" { |
#include "tgsi/tgsi_dump.h" |
#include "tgsi/tgsi_scan.h" |
#include "tgsi/tgsi_util.h" |
} |
#include <set> |
#include "nv50_ir.h" |
#include "nv50_ir_util.h" |
#include "nv50_ir_build_util.h" |
namespace tgsi { |
class Source; |
static nv50_ir::operation translateOpcode(uint opcode); |
static nv50_ir::DataFile translateFile(uint file); |
static nv50_ir::TexTarget translateTexture(uint texTarg); |
static nv50_ir::SVSemantic translateSysVal(uint sysval); |
class Instruction |
{ |
public: |
Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { } |
class SrcRegister |
{ |
public: |
SrcRegister(const struct tgsi_full_src_register *src) |
: reg(src->Register), |
fsr(src) |
{ } |
SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { } |
SrcRegister(const struct tgsi_ind_register& ind) |
: reg(tgsi_util_get_src_from_ind(&ind)), |
fsr(NULL) |
{ } |
struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off) |
{ |
struct tgsi_src_register reg; |
memset(®, 0, sizeof(reg)); |
reg.Index = off.Index; |
reg.File = off.File; |
reg.SwizzleX = off.SwizzleX; |
reg.SwizzleY = off.SwizzleY; |
reg.SwizzleZ = off.SwizzleZ; |
return reg; |
} |
SrcRegister(const struct tgsi_texture_offset& off) : |
reg(offsetToSrc(off)), |
fsr(NULL) |
{ } |
uint getFile() const { return reg.File; } |
bool is2D() const { return reg.Dimension; } |
bool isIndirect(int dim) const |
{ |
return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect; |
} |
int getIndex(int dim) const |
{ |
return (dim && fsr) ? fsr->Dimension.Index : reg.Index; |
} |
int getSwizzle(int chan) const |
{ |
return tgsi_util_get_src_register_swizzle(®, chan); |
} |
nv50_ir::Modifier getMod(int chan) const; |
SrcRegister getIndirect(int dim) const |
{ |
assert(fsr && isIndirect(dim)); |
if (dim) |
return SrcRegister(fsr->DimIndirect); |
return SrcRegister(fsr->Indirect); |
} |
uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const |
{ |
assert(reg.File == TGSI_FILE_IMMEDIATE); |
assert(!reg.Absolute); |
assert(!reg.Negate); |
return info->immd.data[reg.Index * 4 + getSwizzle(c)]; |
} |
private: |
const struct tgsi_src_register reg; |
const struct tgsi_full_src_register *fsr; |
}; |
class DstRegister |
{ |
public: |
DstRegister(const struct tgsi_full_dst_register *dst) |
: reg(dst->Register), |
fdr(dst) |
{ } |
DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { } |
uint getFile() const { return reg.File; } |
bool is2D() const { return reg.Dimension; } |
bool isIndirect(int dim) const |
{ |
return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect; |
} |
int getIndex(int dim) const |
{ |
return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index; |
} |
unsigned int getMask() const { return reg.WriteMask; } |
bool isMasked(int chan) const { return !(getMask() & (1 << chan)); } |
SrcRegister getIndirect(int dim) const |
{ |
assert(fdr && isIndirect(dim)); |
if (dim) |
return SrcRegister(fdr->DimIndirect); |
return SrcRegister(fdr->Indirect); |
} |
private: |
const struct tgsi_dst_register reg; |
const struct tgsi_full_dst_register *fdr; |
}; |
inline uint getOpcode() const { return insn->Instruction.Opcode; } |
unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; } |
unsigned int dstCount() const { return insn->Instruction.NumDstRegs; } |
// mask of used components of source s |
unsigned int srcMask(unsigned int s) const; |
SrcRegister getSrc(unsigned int s) const |
{ |
assert(s < srcCount()); |
return SrcRegister(&insn->Src[s]); |
} |
DstRegister getDst(unsigned int d) const |
{ |
assert(d < dstCount()); |
return DstRegister(&insn->Dst[d]); |
} |
SrcRegister getTexOffset(unsigned int i) const |
{ |
assert(i < TGSI_FULL_MAX_TEX_OFFSETS); |
return SrcRegister(insn->TexOffsets[i]); |
} |
unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; } |
bool checkDstSrcAliasing() const; |
inline nv50_ir::operation getOP() const { |
return translateOpcode(getOpcode()); } |
nv50_ir::DataType inferSrcType() const; |
nv50_ir::DataType inferDstType() const; |
nv50_ir::CondCode getSetCond() const; |
nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const; |
inline uint getLabel() { return insn->Label.Label; } |
unsigned getSaturate() const { return insn->Instruction.Saturate; } |
void print() const |
{ |
tgsi_dump_instruction(insn, 1); |
} |
private: |
const struct tgsi_full_instruction *insn; |
}; |
unsigned int Instruction::srcMask(unsigned int s) const |
{ |
unsigned int mask = insn->Dst[0].Register.WriteMask; |
switch (insn->Instruction.Opcode) { |
case TGSI_OPCODE_COS: |
case TGSI_OPCODE_SIN: |
return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0); |
case TGSI_OPCODE_DP2: |
return 0x3; |
case TGSI_OPCODE_DP3: |
return 0x7; |
case TGSI_OPCODE_DP4: |
case TGSI_OPCODE_DPH: |
case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */ |
return 0xf; |
case TGSI_OPCODE_DST: |
return mask & (s ? 0xa : 0x6); |
case TGSI_OPCODE_EX2: |
case TGSI_OPCODE_EXP: |
case TGSI_OPCODE_LG2: |
case TGSI_OPCODE_LOG: |
case TGSI_OPCODE_POW: |
case TGSI_OPCODE_RCP: |
case TGSI_OPCODE_RSQ: |
case TGSI_OPCODE_SCS: |
return 0x1; |
case TGSI_OPCODE_IF: |
case TGSI_OPCODE_UIF: |
return 0x1; |
case TGSI_OPCODE_LIT: |
return 0xb; |
case TGSI_OPCODE_TEX2: |
case TGSI_OPCODE_TXB2: |
case TGSI_OPCODE_TXL2: |
return (s == 0) ? 0xf : 0x3; |
case TGSI_OPCODE_TEX: |
case TGSI_OPCODE_TXB: |
case TGSI_OPCODE_TXD: |
case TGSI_OPCODE_TXL: |
case TGSI_OPCODE_TXP: |
{ |
const struct tgsi_instruction_texture *tex = &insn->Texture; |
assert(insn->Instruction.Texture); |
mask = 0x7; |
if (insn->Instruction.Opcode != TGSI_OPCODE_TEX && |
insn->Instruction.Opcode != TGSI_OPCODE_TXD) |
mask |= 0x8; /* bias, lod or proj */ |
switch (tex->Texture) { |
case TGSI_TEXTURE_1D: |
mask &= 0x9; |
break; |
case TGSI_TEXTURE_SHADOW1D: |
mask &= 0xd; |
break; |
case TGSI_TEXTURE_1D_ARRAY: |
case TGSI_TEXTURE_2D: |
case TGSI_TEXTURE_RECT: |
mask &= 0xb; |
break; |
case TGSI_TEXTURE_CUBE_ARRAY: |
case TGSI_TEXTURE_SHADOW2D_ARRAY: |
case TGSI_TEXTURE_SHADOWCUBE: |
case TGSI_TEXTURE_SHADOWCUBE_ARRAY: |
mask |= 0x8; |
break; |
default: |
break; |
} |
} |
return mask; |
case TGSI_OPCODE_XPD: |
{ |
unsigned int x = 0; |
if (mask & 1) x |= 0x6; |
if (mask & 2) x |= 0x5; |
if (mask & 4) x |= 0x3; |
return x; |
} |
default: |
break; |
} |
return mask; |
} |
nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const |
{ |
nv50_ir::Modifier m(0); |
if (reg.Absolute) |
m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS); |
if (reg.Negate) |
m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG); |
return m; |
} |
static nv50_ir::DataFile translateFile(uint file) |
{ |
switch (file) { |
case TGSI_FILE_CONSTANT: return nv50_ir::FILE_MEMORY_CONST; |
case TGSI_FILE_INPUT: return nv50_ir::FILE_SHADER_INPUT; |
case TGSI_FILE_OUTPUT: return nv50_ir::FILE_SHADER_OUTPUT; |
case TGSI_FILE_TEMPORARY: return nv50_ir::FILE_GPR; |
case TGSI_FILE_ADDRESS: return nv50_ir::FILE_ADDRESS; |
case TGSI_FILE_PREDICATE: return nv50_ir::FILE_PREDICATE; |
case TGSI_FILE_IMMEDIATE: return nv50_ir::FILE_IMMEDIATE; |
case TGSI_FILE_SYSTEM_VALUE: return nv50_ir::FILE_SYSTEM_VALUE; |
case TGSI_FILE_RESOURCE: return nv50_ir::FILE_MEMORY_GLOBAL; |
case TGSI_FILE_SAMPLER: |
case TGSI_FILE_NULL: |
default: |
return nv50_ir::FILE_NULL; |
} |
} |
static nv50_ir::SVSemantic translateSysVal(uint sysval) |
{ |
switch (sysval) { |
case TGSI_SEMANTIC_FACE: return nv50_ir::SV_FACE; |
case TGSI_SEMANTIC_PSIZE: return nv50_ir::SV_POINT_SIZE; |
case TGSI_SEMANTIC_PRIMID: return nv50_ir::SV_PRIMITIVE_ID; |
case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID; |
case TGSI_SEMANTIC_VERTEXID: return nv50_ir::SV_VERTEX_ID; |
case TGSI_SEMANTIC_GRID_SIZE: return nv50_ir::SV_NCTAID; |
case TGSI_SEMANTIC_BLOCK_ID: return nv50_ir::SV_CTAID; |
case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID; |
case TGSI_SEMANTIC_THREAD_ID: return nv50_ir::SV_TID; |
default: |
assert(0); |
return nv50_ir::SV_CLOCK; |
} |
} |
#define NV50_IR_TEX_TARG_CASE(a, b) \ |
case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b; |
static nv50_ir::TexTarget translateTexture(uint tex) |
{ |
switch (tex) { |
NV50_IR_TEX_TARG_CASE(1D, 1D); |
NV50_IR_TEX_TARG_CASE(2D, 2D); |
NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS); |
NV50_IR_TEX_TARG_CASE(3D, 3D); |
NV50_IR_TEX_TARG_CASE(CUBE, CUBE); |
NV50_IR_TEX_TARG_CASE(RECT, RECT); |
NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY); |
NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY); |
NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY); |
NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY); |
NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW); |
NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW); |
NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW); |
NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW); |
NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW); |
NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW); |
NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW); |
NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER); |
case TGSI_TEXTURE_UNKNOWN: |
default: |
assert(!"invalid texture target"); |
return nv50_ir::TEX_TARGET_2D; |
} |
} |
nv50_ir::DataType Instruction::inferSrcType() const |
{ |
switch (getOpcode()) { |
case TGSI_OPCODE_UIF: |
case TGSI_OPCODE_AND: |
case TGSI_OPCODE_OR: |
case TGSI_OPCODE_XOR: |
case TGSI_OPCODE_NOT: |
case TGSI_OPCODE_U2F: |
case TGSI_OPCODE_UADD: |
case TGSI_OPCODE_UDIV: |
case TGSI_OPCODE_UMOD: |
case TGSI_OPCODE_UMAD: |
case TGSI_OPCODE_UMUL: |
case TGSI_OPCODE_UMAX: |
case TGSI_OPCODE_UMIN: |
case TGSI_OPCODE_USEQ: |
case TGSI_OPCODE_USGE: |
case TGSI_OPCODE_USLT: |
case TGSI_OPCODE_USNE: |
case TGSI_OPCODE_USHR: |
case TGSI_OPCODE_UCMP: |
case TGSI_OPCODE_ATOMUADD: |
case TGSI_OPCODE_ATOMXCHG: |
case TGSI_OPCODE_ATOMCAS: |
case TGSI_OPCODE_ATOMAND: |
case TGSI_OPCODE_ATOMOR: |
case TGSI_OPCODE_ATOMXOR: |
case TGSI_OPCODE_ATOMUMIN: |
case TGSI_OPCODE_ATOMUMAX: |
return nv50_ir::TYPE_U32; |
case TGSI_OPCODE_I2F: |
case TGSI_OPCODE_IDIV: |
case TGSI_OPCODE_IMAX: |
case TGSI_OPCODE_IMIN: |
case TGSI_OPCODE_IABS: |
case TGSI_OPCODE_INEG: |
case TGSI_OPCODE_ISGE: |
case TGSI_OPCODE_ISHR: |
case TGSI_OPCODE_ISLT: |
case TGSI_OPCODE_ISSG: |
case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version |
case TGSI_OPCODE_MOD: |
case TGSI_OPCODE_UARL: |
case TGSI_OPCODE_ATOMIMIN: |
case TGSI_OPCODE_ATOMIMAX: |
return nv50_ir::TYPE_S32; |
default: |
return nv50_ir::TYPE_F32; |
} |
} |
nv50_ir::DataType Instruction::inferDstType() const |
{ |
switch (getOpcode()) { |
case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32; |
case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32; |
case TGSI_OPCODE_I2F: |
case TGSI_OPCODE_U2F: |
return nv50_ir::TYPE_F32; |
default: |
return inferSrcType(); |
} |
} |
nv50_ir::CondCode Instruction::getSetCond() const |
{ |
using namespace nv50_ir; |
switch (getOpcode()) { |
case TGSI_OPCODE_SLT: |
case TGSI_OPCODE_ISLT: |
case TGSI_OPCODE_USLT: |
return CC_LT; |
case TGSI_OPCODE_SLE: |
return CC_LE; |
case TGSI_OPCODE_SGE: |
case TGSI_OPCODE_ISGE: |
case TGSI_OPCODE_USGE: |
return CC_GE; |
case TGSI_OPCODE_SGT: |
return CC_GT; |
case TGSI_OPCODE_SEQ: |
case TGSI_OPCODE_USEQ: |
return CC_EQ; |
case TGSI_OPCODE_SNE: |
return CC_NEU; |
case TGSI_OPCODE_USNE: |
return CC_NE; |
case TGSI_OPCODE_SFL: |
return CC_NEVER; |
case TGSI_OPCODE_STR: |
default: |
return CC_ALWAYS; |
} |
} |
#define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b |
static nv50_ir::operation translateOpcode(uint opcode) |
{ |
switch (opcode) { |
NV50_IR_OPCODE_CASE(ARL, SHL); |
NV50_IR_OPCODE_CASE(MOV, MOV); |
NV50_IR_OPCODE_CASE(RCP, RCP); |
NV50_IR_OPCODE_CASE(RSQ, RSQ); |
NV50_IR_OPCODE_CASE(MUL, MUL); |
NV50_IR_OPCODE_CASE(ADD, ADD); |
NV50_IR_OPCODE_CASE(MIN, MIN); |
NV50_IR_OPCODE_CASE(MAX, MAX); |
NV50_IR_OPCODE_CASE(SLT, SET); |
NV50_IR_OPCODE_CASE(SGE, SET); |
NV50_IR_OPCODE_CASE(MAD, MAD); |
NV50_IR_OPCODE_CASE(SUB, SUB); |
NV50_IR_OPCODE_CASE(FLR, FLOOR); |
NV50_IR_OPCODE_CASE(ROUND, CVT); |
NV50_IR_OPCODE_CASE(EX2, EX2); |
NV50_IR_OPCODE_CASE(LG2, LG2); |
NV50_IR_OPCODE_CASE(POW, POW); |
NV50_IR_OPCODE_CASE(ABS, ABS); |
NV50_IR_OPCODE_CASE(COS, COS); |
NV50_IR_OPCODE_CASE(DDX, DFDX); |
NV50_IR_OPCODE_CASE(DDY, DFDY); |
NV50_IR_OPCODE_CASE(KILL, DISCARD); |
NV50_IR_OPCODE_CASE(SEQ, SET); |
NV50_IR_OPCODE_CASE(SFL, SET); |
NV50_IR_OPCODE_CASE(SGT, SET); |
NV50_IR_OPCODE_CASE(SIN, SIN); |
NV50_IR_OPCODE_CASE(SLE, SET); |
NV50_IR_OPCODE_CASE(SNE, SET); |
NV50_IR_OPCODE_CASE(STR, SET); |
NV50_IR_OPCODE_CASE(TEX, TEX); |
NV50_IR_OPCODE_CASE(TXD, TXD); |
NV50_IR_OPCODE_CASE(TXP, TEX); |
NV50_IR_OPCODE_CASE(BRA, BRA); |
NV50_IR_OPCODE_CASE(CAL, CALL); |
NV50_IR_OPCODE_CASE(RET, RET); |
NV50_IR_OPCODE_CASE(CMP, SLCT); |
NV50_IR_OPCODE_CASE(TXB, TXB); |
NV50_IR_OPCODE_CASE(DIV, DIV); |
NV50_IR_OPCODE_CASE(TXL, TXL); |
NV50_IR_OPCODE_CASE(CEIL, CEIL); |
NV50_IR_OPCODE_CASE(I2F, CVT); |
NV50_IR_OPCODE_CASE(NOT, NOT); |
NV50_IR_OPCODE_CASE(TRUNC, TRUNC); |
NV50_IR_OPCODE_CASE(SHL, SHL); |
NV50_IR_OPCODE_CASE(AND, AND); |
NV50_IR_OPCODE_CASE(OR, OR); |
NV50_IR_OPCODE_CASE(MOD, MOD); |
NV50_IR_OPCODE_CASE(XOR, XOR); |
NV50_IR_OPCODE_CASE(SAD, SAD); |
NV50_IR_OPCODE_CASE(TXF, TXF); |
NV50_IR_OPCODE_CASE(TXQ, TXQ); |
NV50_IR_OPCODE_CASE(EMIT, EMIT); |
NV50_IR_OPCODE_CASE(ENDPRIM, RESTART); |
NV50_IR_OPCODE_CASE(KILL_IF, DISCARD); |
NV50_IR_OPCODE_CASE(F2I, CVT); |
NV50_IR_OPCODE_CASE(IDIV, DIV); |
NV50_IR_OPCODE_CASE(IMAX, MAX); |
NV50_IR_OPCODE_CASE(IMIN, MIN); |
NV50_IR_OPCODE_CASE(IABS, ABS); |
NV50_IR_OPCODE_CASE(INEG, NEG); |
NV50_IR_OPCODE_CASE(ISGE, SET); |
NV50_IR_OPCODE_CASE(ISHR, SHR); |
NV50_IR_OPCODE_CASE(ISLT, SET); |
NV50_IR_OPCODE_CASE(F2U, CVT); |
NV50_IR_OPCODE_CASE(U2F, CVT); |
NV50_IR_OPCODE_CASE(UADD, ADD); |
NV50_IR_OPCODE_CASE(UDIV, DIV); |
NV50_IR_OPCODE_CASE(UMAD, MAD); |
NV50_IR_OPCODE_CASE(UMAX, MAX); |
NV50_IR_OPCODE_CASE(UMIN, MIN); |
NV50_IR_OPCODE_CASE(UMOD, MOD); |
NV50_IR_OPCODE_CASE(UMUL, MUL); |
NV50_IR_OPCODE_CASE(USEQ, SET); |
NV50_IR_OPCODE_CASE(USGE, SET); |
NV50_IR_OPCODE_CASE(USHR, SHR); |
NV50_IR_OPCODE_CASE(USLT, SET); |
NV50_IR_OPCODE_CASE(USNE, SET); |
NV50_IR_OPCODE_CASE(SAMPLE, TEX); |
NV50_IR_OPCODE_CASE(SAMPLE_B, TXB); |
NV50_IR_OPCODE_CASE(SAMPLE_C, TEX); |
NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX); |
NV50_IR_OPCODE_CASE(SAMPLE_D, TXD); |
NV50_IR_OPCODE_CASE(SAMPLE_L, TXL); |
NV50_IR_OPCODE_CASE(SAMPLE_I, TXF); |
NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF); |
NV50_IR_OPCODE_CASE(GATHER4, TXG); |
NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ); |
NV50_IR_OPCODE_CASE(ATOMUADD, ATOM); |
NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM); |
NV50_IR_OPCODE_CASE(ATOMCAS, ATOM); |
NV50_IR_OPCODE_CASE(ATOMAND, ATOM); |
NV50_IR_OPCODE_CASE(ATOMOR, ATOM); |
NV50_IR_OPCODE_CASE(ATOMXOR, ATOM); |
NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM); |
NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM); |
NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM); |
NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM); |
NV50_IR_OPCODE_CASE(TEX2, TEX); |
NV50_IR_OPCODE_CASE(TXB2, TXB); |
NV50_IR_OPCODE_CASE(TXL2, TXL); |
NV50_IR_OPCODE_CASE(END, EXIT); |
default: |
return nv50_ir::OP_NOP; |
} |
} |
static uint16_t opcodeToSubOp(uint opcode) |
{ |
switch (opcode) { |
case TGSI_OPCODE_LFENCE: return NV50_IR_SUBOP_MEMBAR(L, GL); |
case TGSI_OPCODE_SFENCE: return NV50_IR_SUBOP_MEMBAR(S, GL); |
case TGSI_OPCODE_MFENCE: return NV50_IR_SUBOP_MEMBAR(M, GL); |
case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD; |
case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH; |
case TGSI_OPCODE_ATOMCAS: return NV50_IR_SUBOP_ATOM_CAS; |
case TGSI_OPCODE_ATOMAND: return NV50_IR_SUBOP_ATOM_AND; |
case TGSI_OPCODE_ATOMOR: return NV50_IR_SUBOP_ATOM_OR; |
case TGSI_OPCODE_ATOMXOR: return NV50_IR_SUBOP_ATOM_XOR; |
case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN; |
case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN; |
case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX; |
case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX; |
default: |
return 0; |
} |
} |
bool Instruction::checkDstSrcAliasing() const |
{ |
if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory |
return false; |
for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) { |
if (insn->Src[s].Register.File == TGSI_FILE_NULL) |
break; |
if (insn->Src[s].Register.File == insn->Dst[0].Register.File && |
insn->Src[s].Register.Index == insn->Dst[0].Register.Index) |
return true; |
} |
return false; |
} |
class Source |
{ |
public: |
Source(struct nv50_ir_prog_info *); |
~Source(); |
public: |
bool scanSource(); |
unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; } |
public: |
struct tgsi_shader_info scan; |
struct tgsi_full_instruction *insns; |
const struct tgsi_token *tokens; |
struct nv50_ir_prog_info *info; |
nv50_ir::DynArray tempArrays; |
nv50_ir::DynArray immdArrays; |
typedef nv50_ir::BuildUtil::Location Location; |
// these registers are per-subroutine, cannot be used for parameter passing |
std::set<Location> locals; |
bool mainTempsInLMem; |
int clipVertexOutput; |
struct TextureView { |
uint8_t target; // TGSI_TEXTURE_* |
}; |
std::vector<TextureView> textureViews; |
struct Resource { |
uint8_t target; // TGSI_TEXTURE_* |
bool raw; |
uint8_t slot; // $surface index |
}; |
std::vector<Resource> resources; |
private: |
int inferSysValDirection(unsigned sn) const; |
bool scanDeclaration(const struct tgsi_full_declaration *); |
bool scanInstruction(const struct tgsi_full_instruction *); |
void scanProperty(const struct tgsi_full_property *); |
void scanImmediate(const struct tgsi_full_immediate *); |
inline bool isEdgeFlagPassthrough(const Instruction&) const; |
}; |
Source::Source(struct nv50_ir_prog_info *prog) : info(prog) |
{ |
tokens = (const struct tgsi_token *)info->bin.source; |
if (prog->dbgFlags & NV50_IR_DEBUG_BASIC) |
tgsi_dump(tokens, 0); |
mainTempsInLMem = FALSE; |
} |
Source::~Source() |
{ |
if (insns) |
FREE(insns); |
if (info->immd.data) |
FREE(info->immd.data); |
if (info->immd.type) |
FREE(info->immd.type); |
} |
bool Source::scanSource() |
{ |
unsigned insnCount = 0; |
struct tgsi_parse_context parse; |
tgsi_scan_shader(tokens, &scan); |
insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions * |
sizeof(insns[0])); |
if (!insns) |
return false; |
clipVertexOutput = -1; |
textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1); |
resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1); |
info->immd.bufSize = 0; |
info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1; |
info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1; |
info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1; |
if (info->type == PIPE_SHADER_FRAGMENT) { |
info->prop.fp.writesDepth = scan.writes_z; |
info->prop.fp.usesDiscard = scan.uses_kill; |
} else |
if (info->type == PIPE_SHADER_GEOMETRY) { |
info->prop.gp.instanceCount = 1; // default value |
} |
info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16); |
info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte)); |
tgsi_parse_init(&parse, tokens); |
while (!tgsi_parse_end_of_tokens(&parse)) { |
tgsi_parse_token(&parse); |
switch (parse.FullToken.Token.Type) { |
case TGSI_TOKEN_TYPE_IMMEDIATE: |
scanImmediate(&parse.FullToken.FullImmediate); |
break; |
case TGSI_TOKEN_TYPE_DECLARATION: |
scanDeclaration(&parse.FullToken.FullDeclaration); |
break; |
case TGSI_TOKEN_TYPE_INSTRUCTION: |
insns[insnCount++] = parse.FullToken.FullInstruction; |
scanInstruction(&parse.FullToken.FullInstruction); |
break; |
case TGSI_TOKEN_TYPE_PROPERTY: |
scanProperty(&parse.FullToken.FullProperty); |
break; |
default: |
INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type); |
break; |
} |
} |
tgsi_parse_free(&parse); |
if (mainTempsInLMem) |
info->bin.tlsSpace += (scan.file_max[TGSI_FILE_TEMPORARY] + 1) * 16; |
if (info->io.genUserClip > 0) { |
info->io.clipDistanceMask = (1 << info->io.genUserClip) - 1; |
const unsigned int nOut = (info->io.genUserClip + 3) / 4; |
for (unsigned int n = 0; n < nOut; ++n) { |
unsigned int i = info->numOutputs++; |
info->out[i].id = i; |
info->out[i].sn = TGSI_SEMANTIC_CLIPDIST; |
info->out[i].si = n; |
info->out[i].mask = info->io.clipDistanceMask >> (n * 4); |
} |
} |
return info->assignSlots(info) == 0; |
} |
void Source::scanProperty(const struct tgsi_full_property *prop) |
{ |
switch (prop->Property.PropertyName) { |
case TGSI_PROPERTY_GS_OUTPUT_PRIM: |
info->prop.gp.outputPrim = prop->u[0].Data; |
break; |
case TGSI_PROPERTY_GS_INPUT_PRIM: |
info->prop.gp.inputPrim = prop->u[0].Data; |
break; |
case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES: |
info->prop.gp.maxVertices = prop->u[0].Data; |
break; |
#if 0 |
case TGSI_PROPERTY_GS_INSTANCE_COUNT: |
info->prop.gp.instanceCount = prop->u[0].Data; |
break; |
#endif |
case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS: |
info->prop.fp.separateFragData = TRUE; |
break; |
case TGSI_PROPERTY_FS_COORD_ORIGIN: |
case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER: |
// we don't care |
break; |
case TGSI_PROPERTY_VS_PROHIBIT_UCPS: |
info->io.genUserClip = -1; |
break; |
default: |
INFO("unhandled TGSI property %d\n", prop->Property.PropertyName); |
break; |
} |
} |
void Source::scanImmediate(const struct tgsi_full_immediate *imm) |
{ |
const unsigned n = info->immd.count++; |
assert(n < scan.immediate_count); |
for (int c = 0; c < 4; ++c) |
info->immd.data[n * 4 + c] = imm->u[c].Uint; |
info->immd.type[n] = imm->Immediate.DataType; |
} |
int Source::inferSysValDirection(unsigned sn) const |
{ |
switch (sn) { |
case TGSI_SEMANTIC_INSTANCEID: |
case TGSI_SEMANTIC_VERTEXID: |
return 1; |
#if 0 |
case TGSI_SEMANTIC_LAYER: |
case TGSI_SEMANTIC_VIEWPORTINDEX: |
return 0; |
#endif |
case TGSI_SEMANTIC_PRIMID: |
return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0; |
default: |
return 0; |
} |
} |
bool Source::scanDeclaration(const struct tgsi_full_declaration *decl) |
{ |
unsigned i, c; |
unsigned sn = TGSI_SEMANTIC_GENERIC; |
unsigned si = 0; |
const unsigned first = decl->Range.First, last = decl->Range.Last; |
if (decl->Declaration.Semantic) { |
sn = decl->Semantic.Name; |
si = decl->Semantic.Index; |
} |
if (decl->Declaration.Local) { |
for (i = first; i <= last; ++i) { |
for (c = 0; c < 4; ++c) { |
locals.insert( |
Location(decl->Declaration.File, decl->Dim.Index2D, i, c)); |
} |
} |
} |
switch (decl->Declaration.File) { |
case TGSI_FILE_INPUT: |
if (info->type == PIPE_SHADER_VERTEX) { |
// all vertex attributes are equal |
for (i = first; i <= last; ++i) { |
info->in[i].sn = TGSI_SEMANTIC_GENERIC; |
info->in[i].si = i; |
} |
} else { |
for (i = first; i <= last; ++i, ++si) { |
info->in[i].id = i; |
info->in[i].sn = sn; |
info->in[i].si = si; |
if (info->type == PIPE_SHADER_FRAGMENT) { |
// translate interpolation mode |
switch (decl->Interp.Interpolate) { |
case TGSI_INTERPOLATE_CONSTANT: |
info->in[i].flat = 1; |
break; |
case TGSI_INTERPOLATE_COLOR: |
info->in[i].sc = 1; |
break; |
case TGSI_INTERPOLATE_LINEAR: |
info->in[i].linear = 1; |
break; |
default: |
break; |
} |
if (decl->Interp.Centroid) |
info->in[i].centroid = 1; |
} |
} |
} |
break; |
case TGSI_FILE_OUTPUT: |
for (i = first; i <= last; ++i, ++si) { |
switch (sn) { |
case TGSI_SEMANTIC_POSITION: |
if (info->type == PIPE_SHADER_FRAGMENT) |
info->io.fragDepth = i; |
else |
if (clipVertexOutput < 0) |
clipVertexOutput = i; |
break; |
case TGSI_SEMANTIC_COLOR: |
if (info->type == PIPE_SHADER_FRAGMENT) |
info->prop.fp.numColourResults++; |
break; |
case TGSI_SEMANTIC_EDGEFLAG: |
info->io.edgeFlagOut = i; |
break; |
case TGSI_SEMANTIC_CLIPVERTEX: |
clipVertexOutput = i; |
break; |
case TGSI_SEMANTIC_CLIPDIST: |
info->io.clipDistanceMask |= |
decl->Declaration.UsageMask << (si * 4); |
info->io.genUserClip = -1; |
break; |
default: |
break; |
} |
info->out[i].id = i; |
info->out[i].sn = sn; |
info->out[i].si = si; |
} |
break; |
case TGSI_FILE_SYSTEM_VALUE: |
switch (sn) { |
case TGSI_SEMANTIC_INSTANCEID: |
info->io.instanceId = first; |
break; |
case TGSI_SEMANTIC_VERTEXID: |
info->io.vertexId = first; |
break; |
default: |
break; |
} |
for (i = first; i <= last; ++i, ++si) { |
info->sv[i].sn = sn; |
info->sv[i].si = si; |
info->sv[i].input = inferSysValDirection(sn); |
} |
break; |
case TGSI_FILE_RESOURCE: |
for (i = first; i <= last; ++i) { |
resources[i].target = decl->Resource.Resource; |
resources[i].raw = decl->Resource.Raw; |
resources[i].slot = i; |
} |
break; |
case TGSI_FILE_SAMPLER_VIEW: |
for (i = first; i <= last; ++i) |
textureViews[i].target = decl->SamplerView.Resource; |
break; |
case TGSI_FILE_NULL: |
case TGSI_FILE_TEMPORARY: |
case TGSI_FILE_ADDRESS: |
case TGSI_FILE_CONSTANT: |
case TGSI_FILE_IMMEDIATE: |
case TGSI_FILE_PREDICATE: |
case TGSI_FILE_SAMPLER: |
break; |
default: |
ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File); |
return false; |
} |
return true; |
} |
inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const |
{ |
return insn.getOpcode() == TGSI_OPCODE_MOV && |
insn.getDst(0).getIndex(0) == info->io.edgeFlagOut && |
insn.getSrc(0).getFile() == TGSI_FILE_INPUT; |
} |
bool Source::scanInstruction(const struct tgsi_full_instruction *inst) |
{ |
Instruction insn(inst); |
if (insn.getOpcode() == TGSI_OPCODE_BARRIER) |
info->numBarriers = 1; |
if (insn.dstCount()) { |
if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) { |
Instruction::DstRegister dst = insn.getDst(0); |
if (dst.isIndirect(0)) |
for (unsigned i = 0; i < info->numOutputs; ++i) |
info->out[i].mask = 0xf; |
else |
info->out[dst.getIndex(0)].mask |= dst.getMask(); |
if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE || |
info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID || |
info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG) |
info->out[dst.getIndex(0)].mask &= 1; |
if (isEdgeFlagPassthrough(insn)) |
info->io.edgeFlagIn = insn.getSrc(0).getIndex(0); |
} else |
if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) { |
if (insn.getDst(0).isIndirect(0)) |
mainTempsInLMem = TRUE; |
} |
} |
for (unsigned s = 0; s < insn.srcCount(); ++s) { |
Instruction::SrcRegister src = insn.getSrc(s); |
if (src.getFile() == TGSI_FILE_TEMPORARY) { |
if (src.isIndirect(0)) |
mainTempsInLMem = TRUE; |
} else |
if (src.getFile() == TGSI_FILE_RESOURCE) { |
if (src.getIndex(0) == TGSI_RESOURCE_GLOBAL) |
info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ? |
0x1 : 0x2; |
} |
if (src.getFile() != TGSI_FILE_INPUT) |
continue; |
unsigned mask = insn.srcMask(s); |
if (src.isIndirect(0)) { |
for (unsigned i = 0; i < info->numInputs; ++i) |
info->in[i].mask = 0xf; |
} else { |
const int i = src.getIndex(0); |
for (unsigned c = 0; c < 4; ++c) { |
if (!(mask & (1 << c))) |
continue; |
int k = src.getSwizzle(c); |
if (k <= TGSI_SWIZZLE_W) |
info->in[i].mask |= 1 << k; |
} |
switch (info->in[i].sn) { |
case TGSI_SEMANTIC_PSIZE: |
case TGSI_SEMANTIC_PRIMID: |
case TGSI_SEMANTIC_FOG: |
info->in[i].mask &= 0x1; |
break; |
case TGSI_SEMANTIC_PCOORD: |
info->in[i].mask &= 0x3; |
break; |
default: |
break; |
} |
} |
} |
return true; |
} |
nv50_ir::TexInstruction::Target |
Instruction::getTexture(const tgsi::Source *code, int s) const |
{ |
// XXX: indirect access |
unsigned int r; |
switch (getSrc(s).getFile()) { |
case TGSI_FILE_RESOURCE: |
r = getSrc(s).getIndex(0); |
return translateTexture(code->resources.at(r).target); |
case TGSI_FILE_SAMPLER_VIEW: |
r = getSrc(s).getIndex(0); |
return translateTexture(code->textureViews.at(r).target); |
default: |
return translateTexture(insn->Texture.Texture); |
} |
} |
} // namespace tgsi |
namespace { |
using namespace nv50_ir; |
class Converter : public BuildUtil |
{ |
public: |
Converter(Program *, const tgsi::Source *); |
~Converter(); |
bool run(); |
private: |
struct Subroutine |
{ |
Subroutine(Function *f) : f(f) { } |
Function *f; |
ValueMap values; |
}; |
Value *getVertexBase(int s); |
DataArray *getArrayForFile(unsigned file, int idx); |
Value *fetchSrc(int s, int c); |
Value *acquireDst(int d, int c); |
void storeDst(int d, int c, Value *); |
Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr); |
void storeDst(const tgsi::Instruction::DstRegister dst, int c, |
Value *val, Value *ptr); |
Value *applySrcMod(Value *, int s, int c); |
Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr); |
Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c); |
Symbol *dstToSym(tgsi::Instruction::DstRegister, int c); |
bool handleInstruction(const struct tgsi_full_instruction *); |
void exportOutputs(); |
inline Subroutine *getSubroutine(unsigned ip); |
inline Subroutine *getSubroutine(Function *); |
inline bool isEndOfSubroutine(uint ip); |
void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask); |
// R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto) |
void setTexRS(TexInstruction *, unsigned int& s, int R, int S); |
void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy); |
void handleTXF(Value *dst0[4], int R, int L_M); |
void handleTXQ(Value *dst0[4], enum TexQuery); |
void handleLIT(Value *dst0[4]); |
void handleUserClipPlanes(); |
Symbol *getResourceBase(int r); |
void getResourceCoords(std::vector<Value *>&, int r, int s); |
void handleLOAD(Value *dst0[4]); |
void handleSTORE(); |
void handleATOM(Value *dst0[4], DataType, uint16_t subOp); |
Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr); |
void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork); |
Value *buildDot(int dim); |
class BindArgumentsPass : public Pass { |
public: |
BindArgumentsPass(Converter &conv) : conv(conv) { } |
private: |
Converter &conv; |
Subroutine *sub; |
inline const Location *getValueLocation(Subroutine *, Value *); |
template<typename T> inline void |
updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *), |
T (Function::*proto)); |
template<typename T> inline void |
updatePrototype(BitSet *set, void (Function::*updateSet)(), |
T (Function::*proto)); |
protected: |
bool visit(Function *); |
bool visit(BasicBlock *bb) { return false; } |
}; |
private: |
const struct tgsi::Source *code; |
const struct nv50_ir_prog_info *info; |
struct { |
std::map<unsigned, Subroutine> map; |
Subroutine *cur; |
} sub; |
uint ip; // instruction pointer |
tgsi::Instruction tgsi; |
DataType dstTy; |
DataType srcTy; |
DataArray tData; // TGSI_FILE_TEMPORARY |
DataArray aData; // TGSI_FILE_ADDRESS |
DataArray pData; // TGSI_FILE_PREDICATE |
DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers) |
Value *zero; |
Value *fragCoord[4]; |
Value *clipVtx[4]; |
Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP) |
uint8_t vtxBaseValid; |
Stack condBBs; // fork BB, then else clause BB |
Stack joinBBs; // fork BB, for inserting join ops on ENDIF |
Stack loopBBs; // loop headers |
Stack breakBBs; // end of / after loop |
}; |
Symbol * |
Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c) |
{ |
const int swz = src.getSwizzle(c); |
return makeSym(src.getFile(), |
src.is2D() ? src.getIndex(1) : 0, |
src.isIndirect(0) ? -1 : src.getIndex(0), swz, |
src.getIndex(0) * 16 + swz * 4); |
} |
Symbol * |
Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c) |
{ |
return makeSym(dst.getFile(), |
dst.is2D() ? dst.getIndex(1) : 0, |
dst.isIndirect(0) ? -1 : dst.getIndex(0), c, |
dst.getIndex(0) * 16 + c * 4); |
} |
Symbol * |
Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address) |
{ |
Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile)); |
sym->reg.fileIndex = fileIdx; |
if (idx >= 0) { |
if (sym->reg.file == FILE_SHADER_INPUT) |
sym->setOffset(info->in[idx].slot[c] * 4); |
else |
if (sym->reg.file == FILE_SHADER_OUTPUT) |
sym->setOffset(info->out[idx].slot[c] * 4); |
else |
if (sym->reg.file == FILE_SYSTEM_VALUE) |
sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c); |
else |
sym->setOffset(address); |
} else { |
sym->setOffset(address); |
} |
return sym; |
} |
static inline uint8_t |
translateInterpMode(const struct nv50_ir_varying *var, operation& op) |
{ |
uint8_t mode = NV50_IR_INTERP_PERSPECTIVE; |
if (var->flat) |
mode = NV50_IR_INTERP_FLAT; |
else |
if (var->linear) |
mode = NV50_IR_INTERP_LINEAR; |
else |
if (var->sc) |
mode = NV50_IR_INTERP_SC; |
op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC) |
? OP_PINTERP : OP_LINTERP; |
if (var->centroid) |
mode |= NV50_IR_INTERP_CENTROID; |
return mode; |
} |
Value * |
Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr) |
{ |
operation op; |
// XXX: no way to know interpolation mode if we don't know what's accessed |
const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 : |
src.getIndex(0)], op); |
Instruction *insn = new_Instruction(func, op, TYPE_F32); |
insn->setDef(0, getScratch()); |
insn->setSrc(0, srcToSym(src, c)); |
if (op == OP_PINTERP) |
insn->setSrc(1, fragCoord[3]); |
if (ptr) |
insn->setIndirect(0, 0, ptr); |
insn->setInterpolate(mode); |
bb->insertTail(insn); |
return insn->getDef(0); |
} |
Value * |
Converter::applySrcMod(Value *val, int s, int c) |
{ |
Modifier m = tgsi.getSrc(s).getMod(c); |
DataType ty = tgsi.inferSrcType(); |
if (m & Modifier(NV50_IR_MOD_ABS)) |
val = mkOp1v(OP_ABS, ty, getScratch(), val); |
if (m & Modifier(NV50_IR_MOD_NEG)) |
val = mkOp1v(OP_NEG, ty, getScratch(), val); |
return val; |
} |
Value * |
Converter::getVertexBase(int s) |
{ |
assert(s < 5); |
if (!(vtxBaseValid & (1 << s))) { |
const int index = tgsi.getSrc(s).getIndex(1); |
Value *rel = NULL; |
if (tgsi.getSrc(s).isIndirect(1)) |
rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL); |
vtxBaseValid |= 1 << s; |
vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(), mkImm(index), rel); |
} |
return vtxBase[s]; |
} |
Value * |
Converter::fetchSrc(int s, int c) |
{ |
Value *res; |
Value *ptr = NULL, *dimRel = NULL; |
tgsi::Instruction::SrcRegister src = tgsi.getSrc(s); |
if (src.isIndirect(0)) |
ptr = fetchSrc(src.getIndirect(0), 0, NULL); |
if (src.is2D()) { |
switch (src.getFile()) { |
case TGSI_FILE_INPUT: |
dimRel = getVertexBase(s); |
break; |
case TGSI_FILE_CONSTANT: |
// on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k] |
if (src.isIndirect(1)) |
dimRel = fetchSrc(src.getIndirect(1), 0, 0); |
break; |
default: |
break; |
} |
} |
res = fetchSrc(src, c, ptr); |
if (dimRel) |
res->getInsn()->setIndirect(0, 1, dimRel); |
return applySrcMod(res, s, c); |
} |
Converter::DataArray * |
Converter::getArrayForFile(unsigned file, int idx) |
{ |
switch (file) { |
case TGSI_FILE_TEMPORARY: |
return &tData; |
case TGSI_FILE_PREDICATE: |
return &pData; |
case TGSI_FILE_ADDRESS: |
return &aData; |
case TGSI_FILE_OUTPUT: |
assert(prog->getType() == Program::TYPE_FRAGMENT); |
return &oData; |
default: |
assert(!"invalid/unhandled TGSI source file"); |
return NULL; |
} |
} |
Value * |
Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr) |
{ |
const int idx2d = src.is2D() ? src.getIndex(1) : 0; |
const int idx = src.getIndex(0); |
const int swz = src.getSwizzle(c); |
switch (src.getFile()) { |
case TGSI_FILE_IMMEDIATE: |
assert(!ptr); |
return loadImm(NULL, info->immd.data[idx * 4 + swz]); |
case TGSI_FILE_CONSTANT: |
return mkLoadv(TYPE_U32, srcToSym(src, c), ptr); |
case TGSI_FILE_INPUT: |
if (prog->getType() == Program::TYPE_FRAGMENT) { |
// don't load masked inputs, won't be assigned a slot |
if (!ptr && !(info->in[idx].mask & (1 << swz))) |
return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f); |
if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_FACE) |
return mkOp1v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_FACE, 0)); |
return interpolate(src, c, ptr); |
} |
return mkLoadv(TYPE_U32, srcToSym(src, c), ptr); |
case TGSI_FILE_OUTPUT: |
assert(!"load from output file"); |
return NULL; |
case TGSI_FILE_SYSTEM_VALUE: |
assert(!ptr); |
return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c)); |
default: |
return getArrayForFile(src.getFile(), idx2d)->load( |
sub.cur->values, idx, swz, ptr); |
} |
} |
Value * |
Converter::acquireDst(int d, int c) |
{ |
const tgsi::Instruction::DstRegister dst = tgsi.getDst(d); |
const unsigned f = dst.getFile(); |
const int idx = dst.getIndex(0); |
const int idx2d = dst.is2D() ? dst.getIndex(1) : 0; |
if (dst.isMasked(c) || f == TGSI_FILE_RESOURCE) |
return NULL; |
if (dst.isIndirect(0) || |
f == TGSI_FILE_SYSTEM_VALUE || |
(f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT)) |
return getScratch(); |
return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c); |
} |
void |
Converter::storeDst(int d, int c, Value *val) |
{ |
const tgsi::Instruction::DstRegister dst = tgsi.getDst(d); |
switch (tgsi.getSaturate()) { |
case TGSI_SAT_NONE: |
break; |
case TGSI_SAT_ZERO_ONE: |
mkOp1(OP_SAT, dstTy, val, val); |
break; |
case TGSI_SAT_MINUS_PLUS_ONE: |
mkOp2(OP_MAX, dstTy, val, val, mkImm(-1.0f)); |
mkOp2(OP_MIN, dstTy, val, val, mkImm(+1.0f)); |
break; |
default: |
assert(!"invalid saturation mode"); |
break; |
} |
Value *ptr = dst.isIndirect(0) ? |
fetchSrc(dst.getIndirect(0), 0, NULL) : NULL; |
if (info->io.genUserClip > 0 && |
dst.getFile() == TGSI_FILE_OUTPUT && |
!dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) { |
mkMov(clipVtx[c], val); |
val = clipVtx[c]; |
} |
storeDst(dst, c, val, ptr); |
} |
void |
Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c, |
Value *val, Value *ptr) |
{ |
const unsigned f = dst.getFile(); |
const int idx = dst.getIndex(0); |
const int idx2d = dst.is2D() ? dst.getIndex(1) : 0; |
if (f == TGSI_FILE_SYSTEM_VALUE) { |
assert(!ptr); |
mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val); |
} else |
if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) { |
if (ptr || (info->out[idx].mask & (1 << c))) |
mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val); |
} else |
if (f == TGSI_FILE_TEMPORARY || |
f == TGSI_FILE_PREDICATE || |
f == TGSI_FILE_ADDRESS || |
f == TGSI_FILE_OUTPUT) { |
getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val); |
} else { |
assert(!"invalid dst file"); |
} |
} |
#define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \ |
for (chan = 0; chan < 4; ++chan) \ |
if (!inst.getDst(d).isMasked(chan)) |
Value * |
Converter::buildDot(int dim) |
{ |
assert(dim > 0); |
Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0); |
Value *dotp = getScratch(); |
mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1); |
for (int c = 1; c < dim; ++c) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp); |
} |
return dotp; |
} |
void |
Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork) |
{ |
FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL); |
join->fixed = 1; |
conv->insertHead(join); |
fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv); |
fork->insertBefore(fork->getExit(), fork->joinAt); |
} |
void |
Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S) |
{ |
unsigned rIdx = 0, sIdx = 0; |
if (R >= 0) |
rIdx = tgsi.getSrc(R).getIndex(0); |
if (S >= 0) |
sIdx = tgsi.getSrc(S).getIndex(0); |
tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx); |
if (tgsi.getSrc(R).isIndirect(0)) { |
tex->tex.rIndirectSrc = s; |
tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL)); |
} |
if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) { |
tex->tex.sIndirectSrc = s; |
tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL)); |
} |
} |
void |
Converter::handleTXQ(Value *dst0[4], enum TexQuery query) |
{ |
TexInstruction *tex = new_TexInstruction(func, OP_TXQ); |
tex->tex.query = query; |
unsigned int c, d; |
for (d = 0, c = 0; c < 4; ++c) { |
if (!dst0[c]) |
continue; |
tex->tex.mask |= 1 << c; |
tex->setDef(d++, dst0[c]); |
} |
tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level |
setTexRS(tex, c, 1, -1); |
bb->insertTail(tex); |
} |
void |
Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask) |
{ |
Value *proj = fetchSrc(0, 3); |
Instruction *insn = proj->getUniqueInsn(); |
int c; |
if (insn->op == OP_PINTERP) { |
bb->insertTail(insn = cloneForward(func, insn)); |
insn->op = OP_LINTERP; |
insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode()); |
insn->setSrc(1, NULL); |
proj = insn->getDef(0); |
} |
proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj); |
for (c = 0; c < 4; ++c) { |
if (!(mask & (1 << c))) |
continue; |
if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP) |
continue; |
mask &= ~(1 << c); |
bb->insertTail(insn = cloneForward(func, insn)); |
insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode()); |
insn->setSrc(1, proj); |
dst[c] = insn->getDef(0); |
} |
if (!mask) |
return; |
proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3)); |
for (c = 0; c < 4; ++c) |
if (mask & (1 << c)) |
dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj); |
} |
// order of nv50 ir sources: x y z layer lod/bias shadow |
// order of TGSI TEX sources: x y z layer shadow lod/bias |
// lowering will finally set the hw specific order (like array first on nvc0) |
void |
Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy) |
{ |
Value *val; |
Value *arg[4], *src[8]; |
Value *lod = NULL, *shd = NULL; |
unsigned int s, c, d; |
TexInstruction *texi = new_TexInstruction(func, tgsi.getOP()); |
TexInstruction::Target tgt = tgsi.getTexture(code, R); |
for (s = 0; s < tgt.getArgCount(); ++s) |
arg[s] = src[s] = fetchSrc(0, s); |
if (texi->op == OP_TXL || texi->op == OP_TXB) |
lod = fetchSrc(L >> 4, L & 3); |
if (C == 0x0f) |
C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src |
if (tgt.isShadow()) |
shd = fetchSrc(C >> 4, C & 3); |
if (texi->op == OP_TXD) { |
for (c = 0; c < tgt.getDim(); ++c) { |
texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c)); |
texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c)); |
} |
} |
// cube textures don't care about projection value, it's divided out |
if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) { |
unsigned int n = tgt.getDim(); |
if (shd) { |
arg[n] = shd; |
++n; |
assert(tgt.getDim() == tgt.getArgCount()); |
} |
loadProjTexCoords(src, arg, (1 << n) - 1); |
if (shd) |
shd = src[n - 1]; |
} |
if (tgt.isCube()) { |
for (c = 0; c < 3; ++c) |
src[c] = mkOp1v(OP_ABS, TYPE_F32, getSSA(), arg[c]); |
val = getScratch(); |
mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]); |
mkOp2(OP_MAX, TYPE_F32, val, src[2], val); |
mkOp1(OP_RCP, TYPE_F32, val, val); |
for (c = 0; c < 3; ++c) |
src[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), arg[c], val); |
} |
for (c = 0, d = 0; c < 4; ++c) { |
if (dst[c]) { |
texi->setDef(d++, dst[c]); |
texi->tex.mask |= 1 << c; |
} else { |
// NOTE: maybe hook up def too, for CSE |
} |
} |
for (s = 0; s < tgt.getArgCount(); ++s) |
texi->setSrc(s, src[s]); |
if (lod) |
texi->setSrc(s++, lod); |
if (shd) |
texi->setSrc(s++, shd); |
setTexRS(texi, s, R, S); |
if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ) |
texi->tex.levelZero = true; |
bb->insertTail(texi); |
} |
// 1st source: xyz = coordinates, w = lod/sample |
// 2nd source: offset |
void |
Converter::handleTXF(Value *dst[4], int R, int L_M) |
{ |
TexInstruction *texi = new_TexInstruction(func, tgsi.getOP()); |
int ms; |
unsigned int c, d, s; |
texi->tex.target = tgsi.getTexture(code, R); |
ms = texi->tex.target.isMS() ? 1 : 0; |
texi->tex.levelZero = ms; /* MS textures don't have mip-maps */ |
for (c = 0, d = 0; c < 4; ++c) { |
if (dst[c]) { |
texi->setDef(d++, dst[c]); |
texi->tex.mask |= 1 << c; |
} |
} |
for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c) |
texi->setSrc(c, fetchSrc(0, c)); |
texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms |
setTexRS(texi, c, R, -1); |
for (s = 0; s < tgsi.getNumTexOffsets(); ++s) { |
for (c = 0; c < 3; ++c) { |
texi->tex.offset[s][c] = tgsi.getTexOffset(s).getValueU32(c, info); |
if (texi->tex.offset[s][c]) |
texi->tex.useOffsets = s + 1; |
} |
} |
bb->insertTail(texi); |
} |
void |
Converter::handleLIT(Value *dst0[4]) |
{ |
Value *val0 = NULL; |
unsigned int mask = tgsi.getDst(0).getMask(); |
if (mask & (1 << 0)) |
loadImm(dst0[0], 1.0f); |
if (mask & (1 << 3)) |
loadImm(dst0[3], 1.0f); |
if (mask & (3 << 1)) { |
val0 = getScratch(); |
mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero); |
if (mask & (1 << 1)) |
mkMov(dst0[1], val0); |
} |
if (mask & (1 << 2)) { |
Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3); |
Value *val1 = getScratch(), *val3 = getScratch(); |
Value *pos128 = loadImm(NULL, +127.999999f); |
Value *neg128 = loadImm(NULL, -127.999999f); |
mkOp2(OP_MAX, TYPE_F32, val1, src1, zero); |
mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128); |
mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128); |
mkOp2(OP_POW, TYPE_F32, val3, val1, val3); |
mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], val3, zero, val0); |
} |
} |
static inline bool |
isResourceSpecial(const int r) |
{ |
return (r == TGSI_RESOURCE_GLOBAL || |
r == TGSI_RESOURCE_LOCAL || |
r == TGSI_RESOURCE_PRIVATE || |
r == TGSI_RESOURCE_INPUT); |
} |
static inline bool |
isResourceRaw(const struct tgsi::Source *code, const int r) |
{ |
return isResourceSpecial(r) || code->resources[r].raw; |
} |
static inline nv50_ir::TexTarget |
getResourceTarget(const struct tgsi::Source *code, int r) |
{ |
if (isResourceSpecial(r)) |
return nv50_ir::TEX_TARGET_BUFFER; |
return tgsi::translateTexture(code->resources.at(r).target); |
} |
Symbol * |
Converter::getResourceBase(const int r) |
{ |
Symbol *sym = NULL; |
switch (r) { |
case TGSI_RESOURCE_GLOBAL: |
sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL, 15); |
break; |
case TGSI_RESOURCE_LOCAL: |
assert(prog->getType() == Program::TYPE_COMPUTE); |
sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32, |
info->prop.cp.sharedOffset); |
break; |
case TGSI_RESOURCE_PRIVATE: |
sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32, |
info->bin.tlsSpace); |
break; |
case TGSI_RESOURCE_INPUT: |
assert(prog->getType() == Program::TYPE_COMPUTE); |
sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32, |
info->prop.cp.inputOffset); |
break; |
default: |
sym = new_Symbol(prog, |
nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot); |
break; |
} |
return sym; |
} |
void |
Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s) |
{ |
const int arg = |
TexInstruction::Target(getResourceTarget(code, r)).getArgCount(); |
for (int c = 0; c < arg; ++c) |
coords.push_back(fetchSrc(s, c)); |
// NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk |
if (r == TGSI_RESOURCE_LOCAL || |
r == TGSI_RESOURCE_PRIVATE || |
r == TGSI_RESOURCE_INPUT) |
coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS), |
coords[0]); |
} |
static inline int |
partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask) |
{ |
int n = 0; |
while (mask) { |
if (mask & 1) { |
size[n]++; |
} else { |
if (size[n]) |
comp[n = 1] = size[0] + 1; |
else |
comp[n]++; |
} |
mask >>= 1; |
} |
if (size[0] == 3) { |
n = 1; |
size[0] = (comp[0] == 1) ? 1 : 2; |
size[1] = 3 - size[0]; |
comp[1] = comp[0] + size[0]; |
} |
return n + 1; |
} |
// For raw loads, granularity is 4 byte. |
// Usage of the texture read mask on OP_SULDP is not allowed. |
void |
Converter::handleLOAD(Value *dst0[4]) |
{ |
const int r = tgsi.getSrc(0).getIndex(0); |
int c; |
std::vector<Value *> off, src, ldv, def; |
getResourceCoords(off, r, 1); |
if (isResourceRaw(code, r)) { |
uint8_t mask = 0; |
uint8_t comp[2] = { 0, 0 }; |
uint8_t size[2] = { 0, 0 }; |
Symbol *base = getResourceBase(r); |
// determine the base and size of the at most 2 load ops |
for (c = 0; c < 4; ++c) |
if (!tgsi.getDst(0).isMasked(c)) |
mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X); |
int n = partitionLoadStore(comp, size, mask); |
src = off; |
def.resize(4); // index by component, the ones we need will be non-NULL |
for (c = 0; c < 4; ++c) { |
if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c)) |
def[c] = dst0[c]; |
else |
if (mask & (1 << c)) |
def[c] = getScratch(); |
} |
const bool useLd = isResourceSpecial(r) || |
(info->io.nv50styleSurfaces && |
code->resources[r].target == TGSI_TEXTURE_BUFFER); |
for (int i = 0; i < n; ++i) { |
ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]); |
if (comp[i]) // adjust x component of source address if necessary |
src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file), |
off[0], mkImm(comp[i] * 4)); |
else |
src[0] = off[0]; |
if (useLd) { |
Instruction *ld = |
mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]); |
for (size_t c = 1; c < ldv.size(); ++c) |
ld->setDef(c, ldv[c]); |
} else { |
mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot, |
0, ldv, src)->dType = typeOfSize(size[i] * 4); |
} |
} |
} else { |
def.resize(4); |
for (c = 0; c < 4; ++c) { |
if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c)) |
def[c] = getScratch(); |
else |
def[c] = dst0[c]; |
} |
mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0, |
def, off); |
} |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
if (dst0[c] != def[c]) |
mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]); |
} |
// For formatted stores, the write mask on OP_SUSTP can be used. |
// Raw stores have to be split. |
void |
Converter::handleSTORE() |
{ |
const int r = tgsi.getDst(0).getIndex(0); |
int c; |
std::vector<Value *> off, src, dummy; |
getResourceCoords(off, r, 0); |
src = off; |
const int s = src.size(); |
if (isResourceRaw(code, r)) { |
uint8_t comp[2] = { 0, 0 }; |
uint8_t size[2] = { 0, 0 }; |
int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask()); |
Symbol *base = getResourceBase(r); |
const bool useSt = isResourceSpecial(r) || |
(info->io.nv50styleSurfaces && |
code->resources[r].target == TGSI_TEXTURE_BUFFER); |
for (int i = 0; i < n; ++i) { |
if (comp[i]) // adjust x component of source address if necessary |
src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file), |
off[0], mkImm(comp[i] * 4)); |
else |
src[0] = off[0]; |
const DataType stTy = typeOfSize(size[i] * 4); |
if (useSt) { |
Instruction *st = |
mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i])); |
for (c = 1; c < size[i]; ++c) |
st->setSrc(1 + c, fetchSrc(1, comp[i] + c)); |
st->setIndirect(0, 0, src[0]); |
} else { |
// attach values to be stored |
src.resize(s + size[i]); |
for (c = 0; c < size[i]; ++c) |
src[s + c] = fetchSrc(1, comp[i] + c); |
mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot, |
0, dummy, src)->setType(stTy); |
} |
} |
} else { |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
src.push_back(fetchSrc(1, c)); |
mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0, |
dummy, src)->tex.mask = tgsi.getDst(0).getMask(); |
} |
} |
// XXX: These only work on resources with the single-component u32/s32 formats. |
// Therefore the result is replicated. This might not be intended by TGSI, but |
// operating on more than 1 component would produce undefined results because |
// they do not exist. |
void |
Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp) |
{ |
const int r = tgsi.getSrc(0).getIndex(0); |
std::vector<Value *> srcv; |
std::vector<Value *> defv; |
LValue *dst = getScratch(); |
getResourceCoords(srcv, r, 1); |
if (isResourceSpecial(r)) { |
assert(r != TGSI_RESOURCE_INPUT); |
Instruction *insn; |
insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0)); |
insn->subOp = subOp; |
if (subOp == NV50_IR_SUBOP_ATOM_CAS) |
insn->setSrc(2, fetchSrc(3, 0)); |
insn->setIndirect(0, 0, srcv.at(0)); |
} else { |
operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP; |
TexTarget targ = getResourceTarget(code, r); |
int idx = code->resources[r].slot; |
defv.push_back(dst); |
srcv.push_back(fetchSrc(2, 0)); |
if (subOp == NV50_IR_SUBOP_ATOM_CAS) |
srcv.push_back(fetchSrc(3, 0)); |
TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv); |
tex->subOp = subOp; |
tex->tex.mask = 1; |
tex->setType(ty); |
} |
for (int c = 0; c < 4; ++c) |
if (dst0[c]) |
dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov |
} |
Converter::Subroutine * |
Converter::getSubroutine(unsigned ip) |
{ |
std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip); |
if (it == sub.map.end()) |
it = sub.map.insert(std::make_pair( |
ip, Subroutine(new Function(prog, "SUB", ip)))).first; |
return &it->second; |
} |
Converter::Subroutine * |
Converter::getSubroutine(Function *f) |
{ |
unsigned ip = f->getLabel(); |
std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip); |
if (it == sub.map.end()) |
it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first; |
return &it->second; |
} |
bool |
Converter::isEndOfSubroutine(uint ip) |
{ |
assert(ip < code->scan.num_instructions); |
tgsi::Instruction insn(&code->insns[ip]); |
return (insn.getOpcode() == TGSI_OPCODE_END || |
insn.getOpcode() == TGSI_OPCODE_ENDSUB || |
// does END occur at end of main or the very end ? |
insn.getOpcode() == TGSI_OPCODE_BGNSUB); |
} |
bool |
Converter::handleInstruction(const struct tgsi_full_instruction *insn) |
{ |
Instruction *geni; |
Value *dst0[4], *rDst0[4]; |
Value *src0, *src1, *src2; |
Value *val0, *val1; |
int c; |
tgsi = tgsi::Instruction(insn); |
bool useScratchDst = tgsi.checkDstSrcAliasing(); |
operation op = tgsi.getOP(); |
dstTy = tgsi.inferDstType(); |
srcTy = tgsi.inferSrcType(); |
unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0; |
if (tgsi.dstCount()) { |
for (c = 0; c < 4; ++c) { |
rDst0[c] = acquireDst(0, c); |
dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c]; |
} |
} |
switch (tgsi.getOpcode()) { |
case TGSI_OPCODE_ADD: |
case TGSI_OPCODE_UADD: |
case TGSI_OPCODE_AND: |
case TGSI_OPCODE_DIV: |
case TGSI_OPCODE_IDIV: |
case TGSI_OPCODE_UDIV: |
case TGSI_OPCODE_MAX: |
case TGSI_OPCODE_MIN: |
case TGSI_OPCODE_IMAX: |
case TGSI_OPCODE_IMIN: |
case TGSI_OPCODE_UMAX: |
case TGSI_OPCODE_UMIN: |
case TGSI_OPCODE_MOD: |
case TGSI_OPCODE_UMOD: |
case TGSI_OPCODE_MUL: |
case TGSI_OPCODE_UMUL: |
case TGSI_OPCODE_OR: |
case TGSI_OPCODE_POW: |
case TGSI_OPCODE_SHL: |
case TGSI_OPCODE_ISHR: |
case TGSI_OPCODE_USHR: |
case TGSI_OPCODE_SUB: |
case TGSI_OPCODE_XOR: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
mkOp2(op, dstTy, dst0[c], src0, src1); |
} |
break; |
case TGSI_OPCODE_MAD: |
case TGSI_OPCODE_UMAD: |
case TGSI_OPCODE_SAD: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
src2 = fetchSrc(2, c); |
mkOp3(op, dstTy, dst0[c], src0, src1, src2); |
} |
break; |
case TGSI_OPCODE_MOV: |
case TGSI_OPCODE_ABS: |
case TGSI_OPCODE_CEIL: |
case TGSI_OPCODE_FLR: |
case TGSI_OPCODE_TRUNC: |
case TGSI_OPCODE_RCP: |
case TGSI_OPCODE_IABS: |
case TGSI_OPCODE_INEG: |
case TGSI_OPCODE_NOT: |
case TGSI_OPCODE_DDX: |
case TGSI_OPCODE_DDY: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkOp1(op, dstTy, dst0[c], fetchSrc(0, c)); |
break; |
case TGSI_OPCODE_RSQ: |
src0 = fetchSrc(0, 0); |
val0 = getScratch(); |
mkOp1(OP_ABS, TYPE_F32, val0, src0); |
mkOp1(OP_RSQ, TYPE_F32, val0, val0); |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkMov(dst0[c], val0); |
break; |
case TGSI_OPCODE_ARL: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = ROUND_M; |
mkOp2(OP_SHL, TYPE_U32, dst0[c], dst0[c], mkImm(4)); |
} |
break; |
case TGSI_OPCODE_UARL: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkOp2(OP_SHL, TYPE_U32, dst0[c], fetchSrc(0, c), mkImm(4)); |
break; |
case TGSI_OPCODE_EX2: |
case TGSI_OPCODE_LG2: |
val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0); |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkOp1(OP_MOV, TYPE_F32, dst0[c], val0); |
break; |
case TGSI_OPCODE_COS: |
case TGSI_OPCODE_SIN: |
val0 = getScratch(); |
if (mask & 7) { |
mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0)); |
mkOp1(op, TYPE_F32, val0, val0); |
for (c = 0; c < 3; ++c) |
if (dst0[c]) |
mkMov(dst0[c], val0); |
} |
if (dst0[3]) { |
mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3)); |
mkOp1(op, TYPE_F32, dst0[3], val0); |
} |
break; |
case TGSI_OPCODE_SCS: |
if (mask & 3) { |
val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0)); |
if (dst0[0]) |
mkOp1(OP_COS, TYPE_F32, dst0[0], val0); |
if (dst0[1]) |
mkOp1(OP_SIN, TYPE_F32, dst0[1], val0); |
} |
if (dst0[2]) |
loadImm(dst0[2], 0.0f); |
if (dst0[3]) |
loadImm(dst0[3], 1.0f); |
break; |
case TGSI_OPCODE_EXP: |
src0 = fetchSrc(0, 0); |
val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0); |
if (dst0[1]) |
mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0); |
if (dst0[0]) |
mkOp1(OP_EX2, TYPE_F32, dst0[0], val0); |
if (dst0[2]) |
mkOp1(OP_EX2, TYPE_F32, dst0[2], src0); |
if (dst0[3]) |
loadImm(dst0[3], 1.0f); |
break; |
case TGSI_OPCODE_LOG: |
src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0)); |
val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0); |
if (dst0[0] || dst0[1]) |
val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0); |
if (dst0[1]) { |
mkOp1(OP_EX2, TYPE_F32, dst0[1], val1); |
mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]); |
mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0); |
} |
if (dst0[3]) |
loadImm(dst0[3], 1.0f); |
break; |
case TGSI_OPCODE_DP2: |
val0 = buildDot(2); |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkMov(dst0[c], val0); |
break; |
case TGSI_OPCODE_DP3: |
val0 = buildDot(3); |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkMov(dst0[c], val0); |
break; |
case TGSI_OPCODE_DP4: |
val0 = buildDot(4); |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkMov(dst0[c], val0); |
break; |
case TGSI_OPCODE_DPH: |
val0 = buildDot(3); |
src1 = fetchSrc(1, 3); |
mkOp2(OP_ADD, TYPE_F32, val0, val0, src1); |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkMov(dst0[c], val0); |
break; |
case TGSI_OPCODE_DST: |
if (dst0[0]) |
loadImm(dst0[0], 1.0f); |
if (dst0[1]) { |
src0 = fetchSrc(0, 1); |
src1 = fetchSrc(1, 1); |
mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1); |
} |
if (dst0[2]) |
mkMov(dst0[2], fetchSrc(0, 2)); |
if (dst0[3]) |
mkMov(dst0[3], fetchSrc(1, 3)); |
break; |
case TGSI_OPCODE_LRP: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
src2 = fetchSrc(2, c); |
mkOp3(OP_MAD, TYPE_F32, dst0[c], |
mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2); |
} |
break; |
case TGSI_OPCODE_LIT: |
handleLIT(dst0); |
break; |
case TGSI_OPCODE_XPD: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
if (c < 3) { |
val0 = getSSA(); |
src0 = fetchSrc(1, (c + 1) % 3); |
src1 = fetchSrc(0, (c + 2) % 3); |
mkOp2(OP_MUL, TYPE_F32, val0, src0, src1); |
mkOp1(OP_NEG, TYPE_F32, val0, val0); |
src0 = fetchSrc(0, (c + 1) % 3); |
src1 = fetchSrc(1, (c + 2) % 3); |
mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0); |
} else { |
loadImm(dst0[c], 1.0f); |
} |
} |
break; |
case TGSI_OPCODE_ISSG: |
case TGSI_OPCODE_SSG: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
val0 = getScratch(); |
val1 = getScratch(); |
mkCmp(OP_SET, CC_GT, srcTy, val0, src0, zero); |
mkCmp(OP_SET, CC_LT, srcTy, val1, src0, zero); |
if (srcTy == TYPE_F32) |
mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1); |
else |
mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0); |
} |
break; |
case TGSI_OPCODE_UCMP: |
case TGSI_OPCODE_CMP: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
src2 = fetchSrc(2, c); |
if (src1 == src2) |
mkMov(dst0[c], src1); |
else |
mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE, |
srcTy, dst0[c], src1, src2, src0); |
} |
break; |
case TGSI_OPCODE_FRC: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
val0 = getScratch(); |
mkOp1(OP_FLOOR, TYPE_F32, val0, src0); |
mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0); |
} |
break; |
case TGSI_OPCODE_ROUND: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c)) |
->rnd = ROUND_NI; |
break; |
case TGSI_OPCODE_CLAMP: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
src2 = fetchSrc(2, c); |
val0 = getScratch(); |
mkOp2(OP_MIN, TYPE_F32, val0, src0, src1); |
mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2); |
} |
break; |
case TGSI_OPCODE_SLT: |
case TGSI_OPCODE_SGE: |
case TGSI_OPCODE_SEQ: |
case TGSI_OPCODE_SFL: |
case TGSI_OPCODE_SGT: |
case TGSI_OPCODE_SLE: |
case TGSI_OPCODE_SNE: |
case TGSI_OPCODE_STR: |
case TGSI_OPCODE_ISGE: |
case TGSI_OPCODE_ISLT: |
case TGSI_OPCODE_USEQ: |
case TGSI_OPCODE_USGE: |
case TGSI_OPCODE_USLT: |
case TGSI_OPCODE_USNE: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) { |
src0 = fetchSrc(0, c); |
src1 = fetchSrc(1, c); |
mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], src0, src1); |
} |
break; |
case TGSI_OPCODE_KILL_IF: |
val0 = new_LValue(func, FILE_PREDICATE); |
for (c = 0; c < 4; ++c) { |
mkCmp(OP_SET, CC_LT, TYPE_F32, val0, fetchSrc(0, c), zero); |
mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0); |
} |
break; |
case TGSI_OPCODE_KILL: |
mkOp(OP_DISCARD, TYPE_NONE, NULL); |
break; |
case TGSI_OPCODE_TEX: |
case TGSI_OPCODE_TXB: |
case TGSI_OPCODE_TXL: |
case TGSI_OPCODE_TXP: |
// R S L C Dx Dy |
handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00); |
break; |
case TGSI_OPCODE_TXD: |
handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20); |
break; |
case TGSI_OPCODE_TEX2: |
handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00); |
break; |
case TGSI_OPCODE_TXB2: |
case TGSI_OPCODE_TXL2: |
handleTEX(dst0, 2, 2, 0x10, 0x11, 0x00, 0x00); |
break; |
case TGSI_OPCODE_SAMPLE: |
case TGSI_OPCODE_SAMPLE_B: |
case TGSI_OPCODE_SAMPLE_D: |
case TGSI_OPCODE_SAMPLE_L: |
case TGSI_OPCODE_SAMPLE_C: |
case TGSI_OPCODE_SAMPLE_C_LZ: |
handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40); |
break; |
case TGSI_OPCODE_TXF: |
handleTXF(dst0, 1, 0x03); |
break; |
case TGSI_OPCODE_SAMPLE_I: |
handleTXF(dst0, 1, 0x03); |
break; |
case TGSI_OPCODE_SAMPLE_I_MS: |
handleTXF(dst0, 1, 0x20); |
break; |
case TGSI_OPCODE_TXQ: |
case TGSI_OPCODE_SVIEWINFO: |
handleTXQ(dst0, TXQ_DIMS); |
break; |
case TGSI_OPCODE_F2I: |
case TGSI_OPCODE_F2U: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z; |
break; |
case TGSI_OPCODE_I2F: |
case TGSI_OPCODE_U2F: |
FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) |
mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c)); |
break; |
case TGSI_OPCODE_EMIT: |
case TGSI_OPCODE_ENDPRIM: |
// get vertex stream if specified (must be immediate) |
src0 = tgsi.srcCount() ? |
mkImm(tgsi.getSrc(0).getValueU32(0, info)) : zero; |
mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1; |
break; |
case TGSI_OPCODE_IF: |
case TGSI_OPCODE_UIF: |
{ |
BasicBlock *ifBB = new BasicBlock(func); |
bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE); |
condBBs.push(bb); |
joinBBs.push(bb); |
mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy); |
setPosition(ifBB, true); |
} |
break; |
case TGSI_OPCODE_ELSE: |
{ |
BasicBlock *elseBB = new BasicBlock(func); |
BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p); |
forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE); |
condBBs.push(bb); |
forkBB->getExit()->asFlow()->target.bb = elseBB; |
if (!bb->isTerminated()) |
mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL); |
setPosition(elseBB, true); |
} |
break; |
case TGSI_OPCODE_ENDIF: |
{ |
BasicBlock *convBB = new BasicBlock(func); |
BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p); |
BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p); |
if (!bb->isTerminated()) { |
// we only want join if none of the clauses ended with CONT/BREAK/RET |
if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6) |
insertConvergenceOps(convBB, forkBB); |
mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL); |
bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD); |
} |
if (prevBB->getExit()->op == OP_BRA) { |
prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD); |
prevBB->getExit()->asFlow()->target.bb = convBB; |
} |
setPosition(convBB, true); |
} |
break; |
case TGSI_OPCODE_BGNLOOP: |
{ |
BasicBlock *lbgnBB = new BasicBlock(func); |
BasicBlock *lbrkBB = new BasicBlock(func); |
loopBBs.push(lbgnBB); |
breakBBs.push(lbrkBB); |
if (loopBBs.getSize() > func->loopNestingBound) |
func->loopNestingBound++; |
mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL); |
bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE); |
setPosition(lbgnBB, true); |
mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL); |
} |
break; |
case TGSI_OPCODE_ENDLOOP: |
{ |
BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p); |
if (!bb->isTerminated()) { |
mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL); |
bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK); |
} |
setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true); |
} |
break; |
case TGSI_OPCODE_BRK: |
{ |
if (bb->isTerminated()) |
break; |
BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p); |
mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL); |
bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS); |
} |
break; |
case TGSI_OPCODE_CONT: |
{ |
if (bb->isTerminated()) |
break; |
BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p); |
mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL); |
contBB->explicitCont = true; |
bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK); |
} |
break; |
case TGSI_OPCODE_BGNSUB: |
{ |
Subroutine *s = getSubroutine(ip); |
BasicBlock *entry = new BasicBlock(s->f); |
BasicBlock *leave = new BasicBlock(s->f); |
// multiple entrypoints possible, keep the graph connected |
if (prog->getType() == Program::TYPE_COMPUTE) |
prog->main->call.attach(&s->f->call, Graph::Edge::TREE); |
sub.cur = s; |
s->f->setEntry(entry); |
s->f->setExit(leave); |
setPosition(entry, true); |
return true; |
} |
case TGSI_OPCODE_ENDSUB: |
{ |
sub.cur = getSubroutine(prog->main); |
setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true); |
return true; |
} |
case TGSI_OPCODE_CAL: |
{ |
Subroutine *s = getSubroutine(tgsi.getLabel()); |
mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL); |
func->call.attach(&s->f->call, Graph::Edge::TREE); |
return true; |
} |
case TGSI_OPCODE_RET: |
{ |
if (bb->isTerminated()) |
return true; |
BasicBlock *leave = BasicBlock::get(func->cfgExit); |
if (!isEndOfSubroutine(ip + 1)) { |
// insert a PRERET at the entry if this is an early return |
// (only needed for sharing code in the epilogue) |
BasicBlock *pos = getBB(); |
setPosition(BasicBlock::get(func->cfg.getRoot()), false); |
mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1; |
setPosition(pos, true); |
} |
mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1; |
bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS); |
} |
break; |
case TGSI_OPCODE_END: |
{ |
// attach and generate epilogue code |
BasicBlock *epilogue = BasicBlock::get(func->cfgExit); |
bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE); |
setPosition(epilogue, true); |
if (prog->getType() == Program::TYPE_FRAGMENT) |
exportOutputs(); |
if (info->io.genUserClip > 0) |
handleUserClipPlanes(); |
mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1; |
} |
break; |
case TGSI_OPCODE_SWITCH: |
case TGSI_OPCODE_CASE: |
ERROR("switch/case opcode encountered, should have been lowered\n"); |
abort(); |
break; |
case TGSI_OPCODE_LOAD: |
handleLOAD(dst0); |
break; |
case TGSI_OPCODE_STORE: |
handleSTORE(); |
break; |
case TGSI_OPCODE_BARRIER: |
geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0)); |
geni->fixed = 1; |
geni->subOp = NV50_IR_SUBOP_BAR_SYNC; |
break; |
case TGSI_OPCODE_MFENCE: |
case TGSI_OPCODE_LFENCE: |
case TGSI_OPCODE_SFENCE: |
geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL); |
geni->fixed = 1; |
geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode()); |
break; |
case TGSI_OPCODE_ATOMUADD: |
case TGSI_OPCODE_ATOMXCHG: |
case TGSI_OPCODE_ATOMCAS: |
case TGSI_OPCODE_ATOMAND: |
case TGSI_OPCODE_ATOMOR: |
case TGSI_OPCODE_ATOMXOR: |
case TGSI_OPCODE_ATOMUMIN: |
case TGSI_OPCODE_ATOMIMIN: |
case TGSI_OPCODE_ATOMUMAX: |
case TGSI_OPCODE_ATOMIMAX: |
handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode())); |
break; |
default: |
ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode()); |
assert(0); |
break; |
} |
if (tgsi.dstCount()) { |
for (c = 0; c < 4; ++c) { |
if (!dst0[c]) |
continue; |
if (dst0[c] != rDst0[c]) |
mkMov(rDst0[c], dst0[c]); |
storeDst(0, c, rDst0[c]); |
} |
} |
vtxBaseValid = 0; |
return true; |
} |
void |
Converter::handleUserClipPlanes() |
{ |
Value *res[8]; |
int n, i, c; |
for (c = 0; c < 4; ++c) { |
for (i = 0; i < info->io.genUserClip; ++i) { |
Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.ucpCBSlot, |
TYPE_F32, info->io.ucpBase + i * 16 + c * 4); |
Value *ucp = mkLoadv(TYPE_F32, sym, NULL); |
if (c == 0) |
res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp); |
else |
mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]); |
} |
} |
const int first = info->numOutputs - (info->io.genUserClip + 3) / 4; |
for (i = 0; i < info->io.genUserClip; ++i) { |
n = i / 4 + first; |
c = i % 4; |
Symbol *sym = |
mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4); |
mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]); |
} |
} |
void |
Converter::exportOutputs() |
{ |
for (unsigned int i = 0; i < info->numOutputs; ++i) { |
for (unsigned int c = 0; c < 4; ++c) { |
if (!oData.exists(sub.cur->values, i, c)) |
continue; |
Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, |
info->out[i].slot[c] * 4); |
Value *val = oData.load(sub.cur->values, i, c, NULL); |
if (val) |
mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val); |
} |
} |
} |
Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir), |
code(code), |
tgsi(NULL), |
tData(this), aData(this), pData(this), oData(this) |
{ |
info = code->info; |
const DataFile tFile = code->mainTempsInLMem ? FILE_MEMORY_LOCAL : FILE_GPR; |
const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY); |
const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE); |
const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS); |
const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT); |
tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, tFile, 0); |
pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0); |
aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_ADDRESS, 0); |
oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0); |
zero = mkImm((uint32_t)0); |
vtxBaseValid = 0; |
} |
Converter::~Converter() |
{ |
} |
inline const Converter::Location * |
Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v) |
{ |
ValueMap::l_iterator it = s->values.l.find(v); |
return it == s->values.l.end() ? NULL : &it->second; |
} |
template<typename T> inline void |
Converter::BindArgumentsPass::updateCallArgs( |
Instruction *i, void (Instruction::*setArg)(int, Value *), |
T (Function::*proto)) |
{ |
Function *g = i->asFlow()->target.fn; |
Subroutine *subg = conv.getSubroutine(g); |
for (unsigned a = 0; a < (g->*proto).size(); ++a) { |
Value *v = (g->*proto)[a].get(); |
const Converter::Location &l = *getValueLocation(subg, v); |
Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx); |
(i->*setArg)(a, array->acquire(sub->values, l.i, l.c)); |
} |
} |
template<typename T> inline void |
Converter::BindArgumentsPass::updatePrototype( |
BitSet *set, void (Function::*updateSet)(), T (Function::*proto)) |
{ |
(func->*updateSet)(); |
for (unsigned i = 0; i < set->getSize(); ++i) { |
Value *v = func->getLValue(i); |
const Converter::Location *l = getValueLocation(sub, v); |
// only include values with a matching TGSI register |
if (set->test(i) && l && !conv.code->locals.count(*l)) |
(func->*proto).push_back(v); |
} |
} |
bool |
Converter::BindArgumentsPass::visit(Function *f) |
{ |
sub = conv.getSubroutine(f); |
for (ArrayList::Iterator bi = f->allBBlocks.iterator(); |
!bi.end(); bi.next()) { |
for (Instruction *i = BasicBlock::get(bi)->getFirst(); |
i; i = i->next) { |
if (i->op == OP_CALL && !i->asFlow()->builtin) { |
updateCallArgs(i, &Instruction::setSrc, &Function::ins); |
updateCallArgs(i, &Instruction::setDef, &Function::outs); |
} |
} |
} |
if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE) |
return true; |
updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet, |
&Function::buildLiveSets, &Function::ins); |
updatePrototype(&BasicBlock::get(f->cfgExit)->defSet, |
&Function::buildDefSets, &Function::outs); |
return true; |
} |
bool |
Converter::run() |
{ |
BasicBlock *entry = new BasicBlock(prog->main); |
BasicBlock *leave = new BasicBlock(prog->main); |
prog->main->setEntry(entry); |
prog->main->setExit(leave); |
setPosition(entry, true); |
sub.cur = getSubroutine(prog->main); |
if (info->io.genUserClip > 0) { |
for (int c = 0; c < 4; ++c) |
clipVtx[c] = getScratch(); |
} |
if (prog->getType() == Program::TYPE_FRAGMENT) { |
Symbol *sv = mkSysVal(SV_POSITION, 3); |
fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv); |
mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]); |
} |
for (ip = 0; ip < code->scan.num_instructions; ++ip) { |
if (!handleInstruction(&code->insns[ip])) |
return false; |
} |
if (!BindArgumentsPass(*this).run(prog)) |
return false; |
return true; |
} |
} // unnamed namespace |
namespace nv50_ir { |
bool |
Program::makeFromTGSI(struct nv50_ir_prog_info *info) |
{ |
tgsi::Source src(info); |
if (!src.scanSource()) |
return false; |
tlsSize = info->bin.tlsSpace; |
Converter builder(this, &src); |
return builder.run(); |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_graph.cpp |
---|
0,0 → 1,436 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir_graph.h" |
#include <limits> |
#include <list> |
#include <stack> |
#include "nv50_ir.h" |
namespace nv50_ir { |
Graph::Graph() |
{ |
root = NULL; |
size = 0; |
sequence = 0; |
} |
Graph::~Graph() |
{ |
for (IteratorRef it = safeIteratorDFS(); !it->end(); it->next()) |
reinterpret_cast<Node *>(it->get())->cut(); |
} |
void Graph::insert(Node *node) |
{ |
if (!root) |
root = node; |
node->graph = this; |
size++; |
} |
void Graph::Edge::unlink() |
{ |
if (origin) { |
prev[0]->next[0] = next[0]; |
next[0]->prev[0] = prev[0]; |
if (origin->out == this) |
origin->out = (next[0] == this) ? NULL : next[0]; |
--origin->outCount; |
} |
if (target) { |
prev[1]->next[1] = next[1]; |
next[1]->prev[1] = prev[1]; |
if (target->in == this) |
target->in = (next[1] == this) ? NULL : next[1]; |
--target->inCount; |
} |
} |
const char *Graph::Edge::typeStr() const |
{ |
switch (type) { |
case TREE: return "tree"; |
case FORWARD: return "forward"; |
case BACK: return "back"; |
case CROSS: return "cross"; |
case DUMMY: return "dummy"; |
case UNKNOWN: |
default: |
return "unk"; |
} |
} |
Graph::Node::Node(void *priv) : data(priv), |
in(0), out(0), graph(0), |
visited(0), |
inCount(0), outCount(0) |
{ |
// nothing to do |
} |
void Graph::Node::attach(Node *node, Edge::Type kind) |
{ |
Edge *edge = new Edge(this, node, kind); |
// insert head |
if (this->out) { |
edge->next[0] = this->out; |
edge->prev[0] = this->out->prev[0]; |
edge->prev[0]->next[0] = edge; |
this->out->prev[0] = edge; |
} |
this->out = edge; |
if (node->in) { |
edge->next[1] = node->in; |
edge->prev[1] = node->in->prev[1]; |
edge->prev[1]->next[1] = edge; |
node->in->prev[1] = edge; |
} |
node->in = edge; |
++this->outCount; |
++node->inCount; |
assert(graph || node->graph); |
if (!node->graph) |
graph->insert(node); |
if (!graph) |
node->graph->insert(this); |
if (kind == Edge::UNKNOWN) |
graph->classifyEdges(); |
} |
bool Graph::Node::detach(Graph::Node *node) |
{ |
EdgeIterator ei = this->outgoing(); |
for (; !ei.end(); ei.next()) |
if (ei.getNode() == node) |
break; |
if (ei.end()) { |
ERROR("no such node attached\n"); |
return false; |
} |
delete ei.getEdge(); |
return true; |
} |
// Cut a node from the graph, deleting all attached edges. |
void Graph::Node::cut() |
{ |
while (out) |
delete out; |
while (in) |
delete in; |
if (graph) { |
if (graph->root == this) |
graph->root = NULL; |
graph = NULL; |
} |
} |
Graph::Edge::Edge(Node *org, Node *tgt, Type kind) |
{ |
target = tgt; |
origin = org; |
type = kind; |
next[0] = next[1] = this; |
prev[0] = prev[1] = this; |
} |
bool |
Graph::Node::reachableBy(const Node *node, const Node *term) const |
{ |
std::stack<const Node *> stack; |
const Node *pos = NULL; |
const int seq = graph->nextSequence(); |
stack.push(node); |
while (!stack.empty()) { |
pos = stack.top(); |
stack.pop(); |
if (pos == this) |
return true; |
if (pos == term) |
continue; |
for (EdgeIterator ei = pos->outgoing(); !ei.end(); ei.next()) { |
if (ei.getType() == Edge::BACK || ei.getType() == Edge::DUMMY) |
continue; |
if (ei.getNode()->visit(seq)) |
stack.push(ei.getNode()); |
} |
} |
return pos == this; |
} |
class DFSIterator : public Iterator |
{ |
public: |
DFSIterator(Graph *graph, const bool preorder) |
{ |
unsigned int seq = graph->nextSequence(); |
nodes = new Graph::Node * [graph->getSize() + 1]; |
count = 0; |
pos = 0; |
nodes[graph->getSize()] = 0; |
if (graph->getRoot()) { |
graph->getRoot()->visit(seq); |
search(graph->getRoot(), preorder, seq); |
} |
} |
~DFSIterator() |
{ |
if (nodes) |
delete[] nodes; |
} |
void search(Graph::Node *node, const bool preorder, const int sequence) |
{ |
if (preorder) |
nodes[count++] = node; |
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) |
if (ei.getNode()->visit(sequence)) |
search(ei.getNode(), preorder, sequence); |
if (!preorder) |
nodes[count++] = node; |
} |
virtual bool end() const { return pos >= count; } |
virtual void next() { if (pos < count) ++pos; } |
virtual void *get() const { return nodes[pos]; } |
virtual void reset() { pos = 0; } |
protected: |
Graph::Node **nodes; |
int count; |
int pos; |
}; |
IteratorRef Graph::iteratorDFS(bool preorder) |
{ |
return IteratorRef(new DFSIterator(this, preorder)); |
} |
IteratorRef Graph::safeIteratorDFS(bool preorder) |
{ |
return this->iteratorDFS(preorder); |
} |
class CFGIterator : public Iterator |
{ |
public: |
CFGIterator(Graph *graph) |
{ |
nodes = new Graph::Node * [graph->getSize() + 1]; |
count = 0; |
pos = 0; |
nodes[graph->getSize()] = 0; |
// TODO: argh, use graph->sequence instead of tag and just raise it by > 1 |
for (IteratorRef it = graph->iteratorDFS(); !it->end(); it->next()) |
reinterpret_cast<Graph::Node *>(it->get())->tag = 0; |
if (graph->getRoot()) |
search(graph->getRoot(), graph->nextSequence()); |
} |
~CFGIterator() |
{ |
if (nodes) |
delete[] nodes; |
} |
virtual void *get() const { return nodes[pos]; } |
virtual bool end() const { return pos >= count; } |
virtual void next() { if (pos < count) ++pos; } |
virtual void reset() { pos = 0; } |
private: |
void search(Graph::Node *node, const int sequence) |
{ |
Stack bb, cross; |
bb.push(node); |
while (bb.getSize()) { |
node = reinterpret_cast<Graph::Node *>(bb.pop().u.p); |
assert(node); |
if (!node->visit(sequence)) |
continue; |
node->tag = 0; |
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) { |
switch (ei.getType()) { |
case Graph::Edge::TREE: |
case Graph::Edge::FORWARD: |
case Graph::Edge::DUMMY: |
if (++(ei.getNode()->tag) == ei.getNode()->incidentCountFwd()) |
bb.push(ei.getNode()); |
break; |
case Graph::Edge::BACK: |
continue; |
case Graph::Edge::CROSS: |
if (++(ei.getNode()->tag) == 1) |
cross.push(ei.getNode()); |
break; |
default: |
assert(!"unknown edge kind in CFG"); |
break; |
} |
} |
nodes[count++] = node; |
if (bb.getSize() == 0) |
cross.moveTo(bb); |
} |
} |
private: |
Graph::Node **nodes; |
int count; |
int pos; |
}; |
IteratorRef Graph::iteratorCFG() |
{ |
return IteratorRef(new CFGIterator(this)); |
} |
IteratorRef Graph::safeIteratorCFG() |
{ |
return this->iteratorCFG(); |
} |
void Graph::classifyEdges() |
{ |
int seq; |
for (IteratorRef it = iteratorDFS(true); !it->end(); it->next()) { |
Node *node = reinterpret_cast<Node *>(it->get()); |
node->visit(0); |
node->tag = 0; |
} |
classifyDFS(root, (seq = 0)); |
sequence = seq; |
} |
void Graph::classifyDFS(Node *curr, int& seq) |
{ |
Graph::Edge *edge; |
Graph::Node *node; |
curr->visit(++seq); |
curr->tag = 1; |
for (edge = curr->out; edge; edge = edge->next[0]) { |
node = edge->target; |
if (edge->type == Edge::DUMMY) |
continue; |
if (node->getSequence() == 0) { |
edge->type = Edge::TREE; |
classifyDFS(node, seq); |
} else |
if (node->getSequence() > curr->getSequence()) { |
edge->type = Edge::FORWARD; |
} else { |
edge->type = node->tag ? Edge::BACK : Edge::CROSS; |
} |
} |
for (edge = curr->in; edge; edge = edge->next[1]) { |
node = edge->origin; |
if (edge->type == Edge::DUMMY) |
continue; |
if (node->getSequence() == 0) { |
edge->type = Edge::TREE; |
classifyDFS(node, seq); |
} else |
if (node->getSequence() > curr->getSequence()) { |
edge->type = Edge::FORWARD; |
} else { |
edge->type = node->tag ? Edge::BACK : Edge::CROSS; |
} |
} |
curr->tag = 0; |
} |
// @dist is indexed by Node::tag, returns -1 if no path found |
int |
Graph::findLightestPathWeight(Node *a, Node *b, const std::vector<int> &weight) |
{ |
std::vector<int> path(weight.size(), std::numeric_limits<int>::max()); |
std::list<Node *> nodeList; |
const int seq = nextSequence(); |
path[a->tag] = 0; |
for (Node *c = a; c && c != b;) { |
const int p = path[c->tag] + weight[c->tag]; |
for (EdgeIterator ei = c->outgoing(); !ei.end(); ei.next()) { |
Node *t = ei.getNode(); |
if (t->getSequence() < seq) { |
if (path[t->tag] == std::numeric_limits<int>::max()) |
nodeList.push_front(t); |
if (p < path[t->tag]) |
path[t->tag] = p; |
} |
} |
c->visit(seq); |
Node *next = NULL; |
for (std::list<Node *>::iterator n = nodeList.begin(); |
n != nodeList.end(); ++n) { |
if (!next || path[(*n)->tag] < path[next->tag]) |
next = *n; |
if ((*n) == c) { |
// erase visited |
n = nodeList.erase(n); |
--n; |
} |
} |
c = next; |
} |
if (path[b->tag] == std::numeric_limits<int>::max()) |
return -1; |
return path[b->tag]; |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_graph.h |
---|
0,0 → 1,228 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_GRAPH_H__ |
#define __NV50_IR_GRAPH_H__ |
#include "nv50_ir_util.h" |
#include <vector> |
namespace nv50_ir { |
#define ITER_NODE(x) reinterpret_cast<Graph::Node *>((x).get()) |
#define ITER_EDGE(x) reinterpret_cast<Graph::Edge *>((x).get()) |
// A connected graph. |
class Graph |
{ |
public: |
class Node; |
class Edge |
{ |
public: |
enum Type |
{ |
UNKNOWN, |
TREE, |
FORWARD, |
BACK, |
CROSS, // e.g. loop break |
DUMMY |
}; |
Edge(Node *dst, Node *src, Type kind); |
~Edge() { unlink(); } |
inline Node *getOrigin() const { return origin; } |
inline Node *getTarget() const { return target; } |
inline Type getType() const { return type; } |
const char *typeStr() const; |
private: |
Node *origin; |
Node *target; |
Type type; |
Edge *next[2]; // next edge outgoing/incident from/to origin/target |
Edge *prev[2]; |
void unlink(); |
friend class Graph; |
}; |
class EdgeIterator : public Iterator |
{ |
public: |
EdgeIterator() : e(0), t(0), d(0), rev(false) { } |
EdgeIterator(Graph::Edge *first, int dir, bool reverse) |
: d(dir), rev(reverse) |
{ |
t = e = ((rev && first) ? first->prev[d] : first); |
} |
virtual void next() |
{ |
Graph::Edge *n = (rev ? e->prev[d] : e->next[d]); |
e = (n == t ? NULL : n); |
} |
virtual bool end() const { return !e; } |
virtual void *get() const { return e; } |
inline Node *getNode() const { assert(e); return d ? |
e->origin : e->target; } |
inline Edge *getEdge() const { return e; } |
inline Edge::Type getType() { return e ? e->getType() : Edge::UNKNOWN; } |
private: |
Graph::Edge *e; |
Graph::Edge *t; |
int d; |
bool rev; |
}; |
class Node |
{ |
public: |
Node(void *); |
~Node() { cut(); } |
void attach(Node *, Edge::Type); |
bool detach(Node *); |
void cut(); |
inline EdgeIterator outgoing(bool reverse = false) const; |
inline EdgeIterator incident(bool reverse = false) const; |
inline Node *parent() const; // returns NULL if count(incident edges) != 1 |
bool reachableBy(const Node *node, const Node *term) const; |
inline bool visit(int); |
inline int getSequence() const; |
inline int incidentCountFwd() const; // count of incident non-back edges |
inline int incidentCount() const { return inCount; } |
inline int outgoingCount() const { return outCount; } |
Graph *getGraph() const { return graph; } |
void *data; |
private: |
Edge *in; |
Edge *out; |
Graph *graph; |
int visited; |
int16_t inCount; |
int16_t outCount; |
public: |
int tag; // for temporary use |
friend class Graph; |
}; |
public: |
Graph(); |
~Graph(); // does *not* free the nodes (make it an option ?) |
inline Node *getRoot() const { return root; } |
inline unsigned int getSize() const { return size; } |
inline int nextSequence(); |
void insert(Node *node); // attach to or set as root |
IteratorRef iteratorDFS(bool preorder = true); |
IteratorRef iteratorCFG(); |
// safe iterators are unaffected by changes to the *edges* of the graph |
IteratorRef safeIteratorDFS(bool preorder = true); |
IteratorRef safeIteratorCFG(); |
void classifyEdges(); |
// @weights: indexed by Node::tag |
int findLightestPathWeight(Node *, Node *, const std::vector<int>& weights); |
private: |
void classifyDFS(Node *, int&); |
private: |
Node *root; |
unsigned int size; |
int sequence; |
}; |
int Graph::nextSequence() |
{ |
return ++sequence; |
} |
Graph::Node *Graph::Node::parent() const |
{ |
if (inCount != 1) |
return NULL; |
assert(in); |
return in->origin; |
} |
bool Graph::Node::visit(int v) |
{ |
if (visited == v) |
return false; |
visited = v; |
return true; |
} |
int Graph::Node::getSequence() const |
{ |
return visited; |
} |
Graph::EdgeIterator Graph::Node::outgoing(bool reverse) const |
{ |
return EdgeIterator(out, 0, reverse); |
} |
Graph::EdgeIterator Graph::Node::incident(bool reverse) const |
{ |
return EdgeIterator(in, 1, reverse); |
} |
int Graph::Node::incidentCountFwd() const |
{ |
int n = 0; |
for (EdgeIterator ei = incident(); !ei.end(); ei.next()) |
if (ei.getType() != Edge::BACK) |
++n; |
return n; |
} |
} // namespace nv50_ir |
#endif // __NV50_IR_GRAPH_H__ |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_inlines.h |
---|
0,0 → 1,420 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_INLINES_H__ |
#define __NV50_IR_INLINES_H__ |
static inline CondCode reverseCondCode(CondCode cc) |
{ |
static const uint8_t ccRev[8] = { 0, 4, 2, 6, 1, 5, 3, 7 }; |
return static_cast<CondCode>(ccRev[cc & 7] | (cc & ~7)); |
} |
static inline CondCode inverseCondCode(CondCode cc) |
{ |
return static_cast<CondCode>(cc ^ 7); |
} |
static inline bool isMemoryFile(DataFile f) |
{ |
return (f >= FILE_MEMORY_CONST && f <= FILE_MEMORY_LOCAL); |
} |
// contrary to asTex(), this will never include SULD/SUST |
static inline bool isTextureOp(operation op) |
{ |
return (op >= OP_TEX && op <= OP_TEXPREP); |
} |
static inline bool isSurfaceOp(operation op) |
{ |
return (op >= OP_SULDB && op <= OP_SULEA); |
} |
static inline unsigned int typeSizeof(DataType ty) |
{ |
switch (ty) { |
case TYPE_U8: |
case TYPE_S8: |
return 1; |
case TYPE_F16: |
case TYPE_U16: |
case TYPE_S16: |
return 2; |
case TYPE_F32: |
case TYPE_U32: |
case TYPE_S32: |
return 4; |
case TYPE_F64: |
case TYPE_U64: |
case TYPE_S64: |
return 8; |
case TYPE_B96: |
return 12; |
case TYPE_B128: |
return 16; |
default: |
return 0; |
} |
} |
static inline unsigned int typeSizeofLog2(DataType ty) |
{ |
switch (ty) { |
case TYPE_F16: |
case TYPE_U16: |
case TYPE_S16: |
return 1; |
case TYPE_F32: |
case TYPE_U32: |
case TYPE_S32: |
return 2; |
case TYPE_F64: |
case TYPE_U64: |
case TYPE_S64: |
return 3; |
case TYPE_B96: |
case TYPE_B128: |
return 4; |
case TYPE_U8: |
case TYPE_S8: |
default: |
return 0; |
} |
} |
static inline DataType typeOfSize(unsigned int size, |
bool flt = false, bool sgn = false) |
{ |
switch (size) { |
case 1: return sgn ? TYPE_S8 : TYPE_U8; |
case 2: return flt ? TYPE_F16 : (sgn ? TYPE_S16 : TYPE_U16); |
case 8: return flt ? TYPE_F64 : (sgn ? TYPE_S64 : TYPE_U64); |
case 12: return TYPE_B96; |
case 16: return TYPE_B128; |
case 4: |
return flt ? TYPE_F32 : (sgn ? TYPE_S32 : TYPE_U32); |
default: |
return TYPE_NONE; |
} |
} |
static inline bool isFloatType(DataType ty) |
{ |
return (ty >= TYPE_F16 && ty <= TYPE_F64); |
} |
static inline bool isSignedIntType(DataType ty) |
{ |
return (ty == TYPE_S8 || ty == TYPE_S16 || ty == TYPE_S32); |
} |
static inline bool isSignedType(DataType ty) |
{ |
switch (ty) { |
case TYPE_NONE: |
case TYPE_U8: |
case TYPE_U16: |
case TYPE_U32: |
case TYPE_B96: |
case TYPE_B128: |
return false; |
default: |
return true; |
} |
} |
static inline DataType intTypeToSigned(DataType ty) |
{ |
switch (ty) { |
case TYPE_U32: return TYPE_S32; |
case TYPE_U16: return TYPE_S16; |
case TYPE_U8: return TYPE_S8; |
default: |
return ty; |
} |
} |
const ValueRef *ValueRef::getIndirect(int dim) const |
{ |
return isIndirect(dim) ? &insn->src(indirect[dim]) : NULL; |
} |
DataFile ValueRef::getFile() const |
{ |
return value ? value->reg.file : FILE_NULL; |
} |
unsigned int ValueRef::getSize() const |
{ |
return value ? value->reg.size : 0; |
} |
Value *ValueRef::rep() const |
{ |
assert(value); |
return value->join; |
} |
Value *ValueDef::rep() const |
{ |
assert(value); |
return value->join; |
} |
DataFile ValueDef::getFile() const |
{ |
return value ? value->reg.file : FILE_NULL; |
} |
unsigned int ValueDef::getSize() const |
{ |
return value ? value->reg.size : 0; |
} |
void ValueDef::setSSA(LValue *lval) |
{ |
origin = value->asLValue(); |
set(lval); |
} |
const LValue *ValueDef::preSSA() const |
{ |
return origin; |
} |
Instruction *Value::getInsn() const |
{ |
return defs.empty() ? NULL : defs.front()->getInsn(); |
} |
Instruction *Value::getUniqueInsn() const |
{ |
if (defs.empty()) |
return NULL; |
// after regalloc, the definitions of coalesced values are linked |
if (join != this) { |
for (DefCIterator it = defs.begin(); it != defs.end(); ++it) |
if ((*it)->get() == this) |
return (*it)->getInsn(); |
// should be unreachable and trigger assertion at the end |
} |
#ifdef DEBUG |
if (reg.data.id < 0) { |
int n = 0; |
for (DefCIterator it = defs.begin(); n < 2 && it != defs.end(); ++it) |
if ((*it)->get() == this) // don't count joined values |
++n; |
if (n > 1) |
WARN("value %%%i not uniquely defined\n", id); // return NULL ? |
} |
#endif |
assert(defs.front()->get() == this); |
return defs.front()->getInsn(); |
} |
inline bool Instruction::constrainedDefs() const |
{ |
return defExists(1) || op == OP_UNION; |
} |
Value *Instruction::getIndirect(int s, int dim) const |
{ |
return srcs[s].isIndirect(dim) ? getSrc(srcs[s].indirect[dim]) : NULL; |
} |
Value *Instruction::getPredicate() const |
{ |
return (predSrc >= 0) ? getSrc(predSrc) : NULL; |
} |
void Instruction::setFlagsDef(int d, Value *val) |
{ |
if (val) { |
if (flagsDef < 0) |
flagsDef = d; |
setDef(flagsDef, val); |
} else { |
if (flagsDef >= 0) { |
setDef(flagsDef, NULL); |
flagsDef = -1; |
} |
} |
} |
void Instruction::setFlagsSrc(int s, Value *val) |
{ |
flagsSrc = s; |
setSrc(flagsSrc, val); |
} |
Value *TexInstruction::getIndirectR() const |
{ |
return tex.rIndirectSrc >= 0 ? getSrc(tex.rIndirectSrc) : NULL; |
} |
Value *TexInstruction::getIndirectS() const |
{ |
return tex.rIndirectSrc >= 0 ? getSrc(tex.rIndirectSrc) : NULL; |
} |
CmpInstruction *Instruction::asCmp() |
{ |
if (op >= OP_SET_AND && op <= OP_SLCT && op != OP_SELP) |
return static_cast<CmpInstruction *>(this); |
return NULL; |
} |
const CmpInstruction *Instruction::asCmp() const |
{ |
if (op >= OP_SET_AND && op <= OP_SLCT && op != OP_SELP) |
return static_cast<const CmpInstruction *>(this); |
return NULL; |
} |
FlowInstruction *Instruction::asFlow() |
{ |
if (op >= OP_BRA && op <= OP_JOIN) |
return static_cast<FlowInstruction *>(this); |
return NULL; |
} |
const FlowInstruction *Instruction::asFlow() const |
{ |
if (op >= OP_BRA && op <= OP_JOINAT) |
return static_cast<const FlowInstruction *>(this); |
return NULL; |
} |
TexInstruction *Instruction::asTex() |
{ |
if (op >= OP_TEX && op <= OP_SULEA) |
return static_cast<TexInstruction *>(this); |
return NULL; |
} |
const TexInstruction *Instruction::asTex() const |
{ |
if (op >= OP_TEX && op <= OP_SULEA) |
return static_cast<const TexInstruction *>(this); |
return NULL; |
} |
static inline Instruction *cloneForward(Function *ctx, Instruction *obj) |
{ |
DeepClonePolicy<Function> pol(ctx); |
for (int i = 0; obj->srcExists(i); ++i) |
pol.set(obj->getSrc(i), obj->getSrc(i)); |
return obj->clone(pol); |
} |
// XXX: use a virtual function so we're really really safe ? |
LValue *Value::asLValue() |
{ |
if (reg.file >= FILE_GPR && reg.file <= FILE_ADDRESS) |
return static_cast<LValue *>(this); |
return NULL; |
} |
Symbol *Value::asSym() |
{ |
if (reg.file >= FILE_MEMORY_CONST) |
return static_cast<Symbol *>(this); |
return NULL; |
} |
const Symbol *Value::asSym() const |
{ |
if (reg.file >= FILE_MEMORY_CONST) |
return static_cast<const Symbol *>(this); |
return NULL; |
} |
void Symbol::setOffset(int32_t offset) |
{ |
reg.data.offset = offset; |
} |
void Symbol::setAddress(Symbol *base, int32_t offset) |
{ |
baseSym = base; |
reg.data.offset = offset; |
} |
void Symbol::setSV(SVSemantic sv, uint32_t index) |
{ |
reg.data.sv.sv = sv; |
reg.data.sv.index = index; |
} |
ImmediateValue *Value::asImm() |
{ |
if (reg.file == FILE_IMMEDIATE) |
return static_cast<ImmediateValue *>(this); |
return NULL; |
} |
const ImmediateValue *Value::asImm() const |
{ |
if (reg.file == FILE_IMMEDIATE) |
return static_cast<const ImmediateValue *>(this); |
return NULL; |
} |
Value *Value::get(Iterator &it) |
{ |
return reinterpret_cast<Value *>(it.get()); |
} |
bool BasicBlock::reachableBy(const BasicBlock *by, const BasicBlock *term) |
{ |
return cfg.reachableBy(&by->cfg, &term->cfg); |
} |
BasicBlock *BasicBlock::get(Iterator &iter) |
{ |
return reinterpret_cast<BasicBlock *>(iter.get()); |
} |
BasicBlock *BasicBlock::get(Graph::Node *node) |
{ |
assert(node); |
return reinterpret_cast<BasicBlock *>(node->data); |
} |
Function *Function::get(Graph::Node *node) |
{ |
assert(node); |
return reinterpret_cast<Function *>(node->data); |
} |
LValue *Function::getLValue(int id) |
{ |
assert((unsigned int)id < (unsigned int)allLValues.getSize()); |
return reinterpret_cast<LValue *>(allLValues.get(id)); |
} |
#endif // __NV50_IR_INLINES_H__ |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_lowering_nv50.cpp |
---|
0,0 → 1,1101 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50/codegen/nv50_ir.h" |
#include "nv50/codegen/nv50_ir_build_util.h" |
#include "nv50_ir_target_nv50.h" |
namespace nv50_ir { |
// nv50 doesn't support 32 bit integer multiplication |
// |
// ah al * bh bl = LO32: (al * bh + ah * bl) << 16 + (al * bl) |
// ------------------- |
// al*bh 00 HI32: (al * bh + ah * bl) >> 16 + (ah * bh) + |
// ah*bh 00 00 ( carry1) << 16 + ( carry2) |
// al*bl |
// ah*bl 00 |
// |
// fffe0001 + fffe0001 |
static bool |
expandIntegerMUL(BuildUtil *bld, Instruction *mul) |
{ |
const bool highResult = mul->subOp == NV50_IR_SUBOP_MUL_HIGH; |
DataType fTy = mul->sType; // full type |
DataType hTy; |
switch (fTy) { |
case TYPE_S32: hTy = TYPE_S16; break; |
case TYPE_U32: hTy = TYPE_U16; break; |
case TYPE_U64: hTy = TYPE_U32; break; |
case TYPE_S64: hTy = TYPE_S32; break; |
default: |
return false; |
} |
unsigned int fullSize = typeSizeof(fTy); |
unsigned int halfSize = typeSizeof(hTy); |
Instruction *i[9]; |
bld->setPosition(mul, true); |
Value *a[2], *b[2]; |
Value *c[2]; |
Value *t[4]; |
for (int j = 0; j < 4; ++j) |
t[j] = bld->getSSA(fullSize); |
// split sources into halves |
i[0] = bld->mkSplit(a, halfSize, mul->getSrc(0)); |
i[1] = bld->mkSplit(b, halfSize, mul->getSrc(1)); |
i[2] = bld->mkOp2(OP_MUL, fTy, t[0], a[0], b[1]); |
i[3] = bld->mkOp3(OP_MAD, fTy, t[1], a[1], b[0], t[0]); |
i[7] = bld->mkOp2(OP_SHL, fTy, t[2], t[1], bld->mkImm(halfSize * 8)); |
i[4] = bld->mkOp3(OP_MAD, fTy, t[3], a[0], b[0], t[2]); |
if (highResult) { |
Value *r[3]; |
Value *imm = bld->loadImm(NULL, 1 << (halfSize * 8)); |
c[0] = bld->getSSA(1, FILE_FLAGS); |
c[1] = bld->getSSA(1, FILE_FLAGS); |
for (int j = 0; j < 3; ++j) |
r[j] = bld->getSSA(fullSize); |
i[8] = bld->mkOp2(OP_SHR, fTy, r[0], t[1], bld->mkImm(halfSize * 8)); |
i[6] = bld->mkOp2(OP_ADD, fTy, r[1], r[0], imm); |
bld->mkOp2(OP_UNION, TYPE_U32, r[2], r[1], r[0]); |
i[5] = bld->mkOp3(OP_MAD, fTy, mul->getDef(0), a[1], b[1], r[2]); |
// set carry defs / sources |
i[3]->setFlagsDef(1, c[0]); |
i[4]->setFlagsDef(0, c[1]); // actual result not required, just the carry |
i[6]->setPredicate(CC_C, c[0]); |
i[5]->setFlagsSrc(3, c[1]); |
} else { |
bld->mkMov(mul->getDef(0), t[3]); |
} |
delete_Instruction(bld->getProgram(), mul); |
for (int j = 2; j <= (highResult ? 5 : 4); ++j) |
if (i[j]) |
i[j]->sType = hTy; |
return true; |
} |
#define QOP_ADD 0 |
#define QOP_SUBR 1 |
#define QOP_SUB 2 |
#define QOP_MOV2 3 |
// UL UR LL LR |
#define QUADOP(q, r, s, t) \ |
((QOP_##q << 6) | (QOP_##r << 4) | \ |
(QOP_##s << 2) | (QOP_##t << 0)) |
class NV50LegalizePostRA : public Pass |
{ |
private: |
virtual bool visit(Function *); |
virtual bool visit(BasicBlock *); |
void handlePRERET(FlowInstruction *); |
void replaceZero(Instruction *); |
LValue *r63; |
}; |
bool |
NV50LegalizePostRA::visit(Function *fn) |
{ |
Program *prog = fn->getProgram(); |
r63 = new_LValue(fn, FILE_GPR); |
r63->reg.data.id = 63; |
// this is actually per-program, but we can do it all on visiting main() |
std::list<Instruction *> *outWrites = |
reinterpret_cast<std::list<Instruction *> *>(prog->targetPriv); |
if (outWrites) { |
for (std::list<Instruction *>::iterator it = outWrites->begin(); |
it != outWrites->end(); ++it) |
(*it)->getSrc(1)->defs.front()->getInsn()->setDef(0, (*it)->getSrc(0)); |
// instructions will be deleted on exit |
outWrites->clear(); |
} |
return true; |
} |
void |
NV50LegalizePostRA::replaceZero(Instruction *i) |
{ |
for (int s = 0; i->srcExists(s); ++s) { |
ImmediateValue *imm = i->getSrc(s)->asImm(); |
if (imm && imm->reg.data.u64 == 0) |
i->setSrc(s, r63); |
} |
} |
// Emulate PRERET: jump to the target and call to the origin from there |
// |
// WARNING: atm only works if BBs are affected by at most a single PRERET |
// |
// BB:0 |
// preret BB:3 |
// (...) |
// BB:3 |
// (...) |
// ---> |
// BB:0 |
// bra BB:3 + n0 (directly to the call; move to beginning of BB and fixate) |
// (...) |
// BB:3 |
// bra BB:3 + n1 (skip the call) |
// call BB:0 + n2 (skip bra at beginning of BB:0) |
// (...) |
void |
NV50LegalizePostRA::handlePRERET(FlowInstruction *pre) |
{ |
BasicBlock *bbE = pre->bb; |
BasicBlock *bbT = pre->target.bb; |
pre->subOp = NV50_IR_SUBOP_EMU_PRERET + 0; |
bbE->remove(pre); |
bbE->insertHead(pre); |
Instruction *skip = new_FlowInstruction(func, OP_PRERET, bbT); |
Instruction *call = new_FlowInstruction(func, OP_PRERET, bbE); |
bbT->insertHead(call); |
bbT->insertHead(skip); |
// NOTE: maybe split blocks to prevent the instructions from moving ? |
skip->subOp = NV50_IR_SUBOP_EMU_PRERET + 1; |
call->subOp = NV50_IR_SUBOP_EMU_PRERET + 2; |
} |
bool |
NV50LegalizePostRA::visit(BasicBlock *bb) |
{ |
Instruction *i, *next; |
// remove pseudo operations and non-fixed no-ops, split 64 bit operations |
for (i = bb->getFirst(); i; i = next) { |
next = i->next; |
if (i->isNop()) { |
bb->remove(i); |
} else |
if (i->op == OP_PRERET && prog->getTarget()->getChipset() < 0xa0) { |
handlePRERET(i->asFlow()); |
} else { |
// TODO: We will want to do this before register allocation, |
// since have to use a $c register for the carry flag. |
if (typeSizeof(i->dType) == 8) { |
Instruction *hi = BuildUtil::split64BitOpPostRA(func, i, r63, NULL); |
if (hi) |
next = hi; |
} |
if (i->op != OP_MOV && i->op != OP_PFETCH && |
i->op != OP_BAR && |
(!i->defExists(0) || i->def(0).getFile() != FILE_ADDRESS)) |
replaceZero(i); |
} |
} |
if (!bb->getEntry()) |
return true; |
return true; |
} |
class NV50LegalizeSSA : public Pass |
{ |
public: |
NV50LegalizeSSA(Program *); |
virtual bool visit(BasicBlock *bb); |
private: |
void propagateWriteToOutput(Instruction *); |
void handleDIV(Instruction *); |
void handleMOD(Instruction *); |
void handleMUL(Instruction *); |
void handleAddrDef(Instruction *); |
inline bool isARL(const Instruction *) const; |
BuildUtil bld; |
std::list<Instruction *> *outWrites; |
}; |
NV50LegalizeSSA::NV50LegalizeSSA(Program *prog) |
{ |
bld.setProgram(prog); |
if (prog->optLevel >= 2 && |
(prog->getType() == Program::TYPE_GEOMETRY || |
prog->getType() == Program::TYPE_VERTEX)) |
outWrites = |
reinterpret_cast<std::list<Instruction *> *>(prog->targetPriv); |
else |
outWrites = NULL; |
} |
void |
NV50LegalizeSSA::propagateWriteToOutput(Instruction *st) |
{ |
if (st->src(0).isIndirect(0) || st->getSrc(1)->refCount() != 1) |
return; |
// check def instruction can store |
Instruction *di = st->getSrc(1)->defs.front()->getInsn(); |
// TODO: move exports (if beneficial) in common opt pass |
if (di->isPseudo() || isTextureOp(di->op) || di->defCount(0xff, true) > 1) |
return; |
for (int s = 0; di->srcExists(s); ++s) |
if (di->src(s).getFile() == FILE_IMMEDIATE) |
return; |
// We cannot set defs to non-lvalues before register allocation, so |
// save & remove (to save registers) the exports and replace later. |
outWrites->push_back(st); |
st->bb->remove(st); |
} |
bool |
NV50LegalizeSSA::isARL(const Instruction *i) const |
{ |
ImmediateValue imm; |
if (i->op != OP_SHL || i->src(0).getFile() != FILE_GPR) |
return false; |
if (!i->src(1).getImmediate(imm)) |
return false; |
return imm.isInteger(0); |
} |
void |
NV50LegalizeSSA::handleAddrDef(Instruction *i) |
{ |
Instruction *arl; |
i->getDef(0)->reg.size = 2; // $aX are only 16 bit |
// only ADDR <- SHL(GPR, IMM) and ADDR <- ADD(ADDR, IMM) are valid |
if (i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE) { |
if (i->op == OP_SHL && i->src(0).getFile() == FILE_GPR) |
return; |
if (i->op == OP_ADD && i->src(0).getFile() == FILE_ADDRESS) |
return; |
} |
// turn $a sources into $r sources (can't operate on $a) |
for (int s = 0; i->srcExists(s); ++s) { |
Value *a = i->getSrc(s); |
Value *r; |
if (a->reg.file == FILE_ADDRESS) { |
if (a->getInsn() && isARL(a->getInsn())) { |
i->setSrc(s, a->getInsn()->getSrc(0)); |
} else { |
bld.setPosition(i, false); |
r = bld.getSSA(); |
bld.mkMov(r, a); |
i->setSrc(s, r); |
} |
} |
} |
if (i->op == OP_SHL && i->src(1).getFile() == FILE_IMMEDIATE) |
return; |
// turn result back into $a |
bld.setPosition(i, true); |
arl = bld.mkOp2(OP_SHL, TYPE_U32, i->getDef(0), bld.getSSA(), bld.mkImm(0)); |
i->setDef(0, arl->getSrc(0)); |
} |
void |
NV50LegalizeSSA::handleMUL(Instruction *mul) |
{ |
if (isFloatType(mul->sType) || typeSizeof(mul->sType) <= 2) |
return; |
Value *def = mul->getDef(0); |
Value *pred = mul->getPredicate(); |
CondCode cc = mul->cc; |
if (pred) |
mul->setPredicate(CC_ALWAYS, NULL); |
if (mul->op == OP_MAD) { |
Instruction *add = mul; |
bld.setPosition(add, false); |
Value *res = cloneShallow(func, mul->getDef(0)); |
mul = bld.mkOp2(OP_MUL, add->sType, res, add->getSrc(0), add->getSrc(1)); |
add->op = OP_ADD; |
add->setSrc(0, mul->getDef(0)); |
add->setSrc(1, add->getSrc(2)); |
for (int s = 2; add->srcExists(s); ++s) |
add->setSrc(s, NULL); |
mul->subOp = add->subOp; |
add->subOp = 0; |
} |
expandIntegerMUL(&bld, mul); |
if (pred) |
def->getInsn()->setPredicate(cc, pred); |
} |
// Use f32 division: first compute an approximate result, use it to reduce |
// the dividend, which should then be representable as f32, divide the reduced |
// dividend, and add the quotients. |
void |
NV50LegalizeSSA::handleDIV(Instruction *div) |
{ |
const DataType ty = div->sType; |
if (ty != TYPE_U32 && ty != TYPE_S32) |
return; |
Value *q, *q0, *qf, *aR, *aRf, *qRf, *qR, *t, *s, *m, *cond; |
bld.setPosition(div, false); |
Value *a, *af = bld.getSSA(); |
Value *b, *bf = bld.getSSA(); |
bld.mkCvt(OP_CVT, TYPE_F32, af, ty, div->getSrc(0)); |
bld.mkCvt(OP_CVT, TYPE_F32, bf, ty, div->getSrc(1)); |
if (isSignedType(ty)) { |
af->getInsn()->src(0).mod = Modifier(NV50_IR_MOD_ABS); |
bf->getInsn()->src(0).mod = Modifier(NV50_IR_MOD_ABS); |
a = bld.getSSA(); |
b = bld.getSSA(); |
bld.mkOp1(OP_ABS, ty, a, div->getSrc(0)); |
bld.mkOp1(OP_ABS, ty, b, div->getSrc(1)); |
} else { |
a = div->getSrc(0); |
b = div->getSrc(1); |
} |
bf = bld.mkOp1v(OP_RCP, TYPE_F32, bld.getSSA(), bf); |
bf = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), bf, bld.mkImm(-2)); |
bld.mkOp2(OP_MUL, TYPE_F32, (qf = bld.getSSA()), af, bf)->rnd = ROUND_Z; |
bld.mkCvt(OP_CVT, ty, (q0 = bld.getSSA()), TYPE_F32, qf)->rnd = ROUND_Z; |
// get error of 1st result |
expandIntegerMUL(&bld, |
bld.mkOp2(OP_MUL, TYPE_U32, (t = bld.getSSA()), q0, b)); |
bld.mkOp2(OP_SUB, TYPE_U32, (aRf = bld.getSSA()), a, t); |
bld.mkCvt(OP_CVT, TYPE_F32, (aR = bld.getSSA()), TYPE_U32, aRf); |
bld.mkOp2(OP_MUL, TYPE_F32, (qRf = bld.getSSA()), aR, bf)->rnd = ROUND_Z; |
bld.mkCvt(OP_CVT, TYPE_U32, (qR = bld.getSSA()), TYPE_F32, qRf) |
->rnd = ROUND_Z; |
bld.mkOp2(OP_ADD, ty, (q = bld.getSSA()), q0, qR); // add quotients |
// correction: if modulus >= divisor, add 1 |
expandIntegerMUL(&bld, |
bld.mkOp2(OP_MUL, TYPE_U32, (t = bld.getSSA()), q, b)); |
bld.mkOp2(OP_SUB, TYPE_U32, (m = bld.getSSA()), a, t); |
bld.mkCmp(OP_SET, CC_GE, TYPE_U32, (s = bld.getSSA()), m, b); |
if (!isSignedType(ty)) { |
div->op = OP_SUB; |
div->setSrc(0, q); |
div->setSrc(1, s); |
} else { |
t = q; |
bld.mkOp2(OP_SUB, TYPE_U32, (q = bld.getSSA()), t, s); |
s = bld.getSSA(); |
t = bld.getSSA(); |
// fix the sign |
bld.mkOp2(OP_XOR, TYPE_U32, NULL, div->getSrc(0), div->getSrc(1)) |
->setFlagsDef(0, (cond = bld.getSSA(1, FILE_FLAGS))); |
bld.mkOp1(OP_NEG, ty, s, q)->setPredicate(CC_S, cond); |
bld.mkOp1(OP_MOV, ty, t, q)->setPredicate(CC_NS, cond); |
div->op = OP_UNION; |
div->setSrc(0, s); |
div->setSrc(1, t); |
} |
} |
void |
NV50LegalizeSSA::handleMOD(Instruction *mod) |
{ |
if (mod->dType != TYPE_U32 && mod->dType != TYPE_S32) |
return; |
bld.setPosition(mod, false); |
Value *q = bld.getSSA(); |
Value *m = bld.getSSA(); |
bld.mkOp2(OP_DIV, mod->dType, q, mod->getSrc(0), mod->getSrc(1)); |
handleDIV(q->getInsn()); |
bld.setPosition(mod, false); |
expandIntegerMUL(&bld, bld.mkOp2(OP_MUL, TYPE_U32, m, q, mod->getSrc(1))); |
mod->op = OP_SUB; |
mod->setSrc(1, m); |
} |
bool |
NV50LegalizeSSA::visit(BasicBlock *bb) |
{ |
Instruction *insn, *next; |
// skipping PHIs (don't pass them to handleAddrDef) ! |
for (insn = bb->getEntry(); insn; insn = next) { |
next = insn->next; |
switch (insn->op) { |
case OP_EXPORT: |
if (outWrites) |
propagateWriteToOutput(insn); |
break; |
case OP_DIV: |
handleDIV(insn); |
break; |
case OP_MOD: |
handleMOD(insn); |
break; |
case OP_MAD: |
case OP_MUL: |
handleMUL(insn); |
break; |
default: |
break; |
} |
if (insn->defExists(0) && insn->getDef(0)->reg.file == FILE_ADDRESS) |
handleAddrDef(insn); |
} |
return true; |
} |
class NV50LoweringPreSSA : public Pass |
{ |
public: |
NV50LoweringPreSSA(Program *); |
private: |
virtual bool visit(Instruction *); |
virtual bool visit(Function *); |
bool handleRDSV(Instruction *); |
bool handleWRSV(Instruction *); |
bool handleEXPORT(Instruction *); |
bool handleDIV(Instruction *); |
bool handleSQRT(Instruction *); |
bool handlePOW(Instruction *); |
bool handleSET(Instruction *); |
bool handleSLCT(CmpInstruction *); |
bool handleSELP(Instruction *); |
bool handleTEX(TexInstruction *); |
bool handleTXB(TexInstruction *); // I really |
bool handleTXL(TexInstruction *); // hate |
bool handleTXD(TexInstruction *); // these 3 |
bool handleCALL(Instruction *); |
bool handlePRECONT(Instruction *); |
bool handleCONT(Instruction *); |
void checkPredicate(Instruction *); |
private: |
const Target *const targ; |
BuildUtil bld; |
Value *tid; |
}; |
NV50LoweringPreSSA::NV50LoweringPreSSA(Program *prog) : |
targ(prog->getTarget()), tid(NULL) |
{ |
bld.setProgram(prog); |
} |
bool |
NV50LoweringPreSSA::visit(Function *f) |
{ |
BasicBlock *root = BasicBlock::get(func->cfg.getRoot()); |
if (prog->getType() == Program::TYPE_COMPUTE) { |
// Add implicit "thread id" argument in $r0 to the function |
Value *arg = new_LValue(func, FILE_GPR); |
arg->reg.data.id = 0; |
f->ins.push_back(arg); |
bld.setPosition(root, false); |
tid = bld.mkMov(bld.getScratch(), arg, TYPE_U32)->getDef(0); |
} |
return true; |
} |
bool |
NV50LoweringPreSSA::handleTEX(TexInstruction *i) |
{ |
const int arg = i->tex.target.getArgCount(); |
const int dref = arg; |
const int lod = i->tex.target.isShadow() ? (arg + 1) : arg; |
// dref comes before bias/lod |
if (i->tex.target.isShadow()) |
if (i->op == OP_TXB || i->op == OP_TXL) |
i->swapSources(dref, lod); |
// array index must be converted to u32 |
if (i->tex.target.isArray()) { |
Value *layer = i->getSrc(arg - 1); |
LValue *src = new_LValue(func, FILE_GPR); |
bld.mkCvt(OP_CVT, TYPE_U32, src, TYPE_F32, layer); |
bld.mkOp2(OP_MIN, TYPE_U32, src, src, bld.loadImm(NULL, 511)); |
i->setSrc(arg - 1, src); |
if (i->tex.target.isCube()) { |
std::vector<Value *> acube, a2d; |
int c; |
acube.resize(4); |
for (c = 0; c < 4; ++c) |
acube[c] = i->getSrc(c); |
a2d.resize(4); |
for (c = 0; c < 3; ++c) |
a2d[c] = new_LValue(func, FILE_GPR); |
a2d[3] = NULL; |
bld.mkTex(OP_TEXPREP, TEX_TARGET_CUBE_ARRAY, i->tex.r, i->tex.s, |
a2d, acube)->asTex()->tex.mask = 0x7; |
for (c = 0; c < 3; ++c) |
i->setSrc(c, a2d[c]); |
i->setSrc(c, NULL); |
for (; i->srcExists(c + 1); ++c) |
i->setSrc(c, i->getSrc(c + 1)); |
i->tex.target = i->tex.target.isShadow() ? |
TEX_TARGET_2D_ARRAY_SHADOW : TEX_TARGET_2D_ARRAY; |
} |
} |
// texel offsets are 3 immediate fields in the instruction, |
// nv50 cannot do textureGatherOffsets |
assert(i->tex.useOffsets <= 1); |
return true; |
} |
// Bias must be equal for all threads of a quad or lod calculation will fail. |
// |
// The lanes of a quad are grouped by the bit in the condition register they |
// have set, which is selected by differing bias values. |
// Move the input values for TEX into a new register set for each group and |
// execute TEX only for a specific group. |
// We always need to use 4 new registers for the inputs/outputs because the |
// implicitly calculated derivatives must be correct. |
// |
// TODO: move to SSA phase so we can easily determine whether bias is constant |
bool |
NV50LoweringPreSSA::handleTXB(TexInstruction *i) |
{ |
const CondCode cc[4] = { CC_EQU, CC_S, CC_C, CC_O }; |
int l, d; |
handleTEX(i); |
Value *bias = i->getSrc(i->tex.target.getArgCount()); |
if (bias->isUniform()) |
return true; |
Instruction *cond = bld.mkOp1(OP_UNION, TYPE_U32, bld.getScratch(), |
bld.loadImm(NULL, 1)); |
bld.setPosition(cond, false); |
for (l = 1; l < 4; ++l) { |
const uint8_t qop = QUADOP(SUBR, SUBR, SUBR, SUBR); |
Value *bit = bld.getSSA(); |
Value *pred = bld.getScratch(1, FILE_FLAGS); |
Value *imm = bld.loadImm(NULL, (1 << l)); |
bld.mkQuadop(qop, pred, l, bias, bias)->flagsDef = 0; |
bld.mkMov(bit, imm)->setPredicate(CC_EQ, pred); |
cond->setSrc(l, bit); |
} |
Value *flags = bld.getScratch(1, FILE_FLAGS); |
bld.setPosition(cond, true); |
bld.mkCvt(OP_CVT, TYPE_U8, flags, TYPE_U32, cond->getDef(0)); |
Instruction *tex[4]; |
for (l = 0; l < 4; ++l) { |
(tex[l] = cloneForward(func, i))->setPredicate(cc[l], flags); |
bld.insert(tex[l]); |
} |
Value *res[4][4]; |
for (d = 0; i->defExists(d); ++d) |
res[0][d] = tex[0]->getDef(d); |
for (l = 1; l < 4; ++l) { |
for (d = 0; tex[l]->defExists(d); ++d) { |
res[l][d] = cloneShallow(func, res[0][d]); |
bld.mkMov(res[l][d], tex[l]->getDef(d))->setPredicate(cc[l], flags); |
} |
} |
for (d = 0; i->defExists(d); ++d) { |
Instruction *dst = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(d)); |
for (l = 0; l < 4; ++l) |
dst->setSrc(l, res[l][d]); |
} |
delete_Instruction(prog, i); |
return true; |
} |
// LOD must be equal for all threads of a quad. |
// Unlike with TXB, here we can just diverge since there's no LOD calculation |
// that would require all 4 threads' sources to be set up properly. |
bool |
NV50LoweringPreSSA::handleTXL(TexInstruction *i) |
{ |
handleTEX(i); |
Value *lod = i->getSrc(i->tex.target.getArgCount()); |
if (lod->isUniform()) |
return true; |
BasicBlock *currBB = i->bb; |
BasicBlock *texiBB = i->bb->splitBefore(i, false); |
BasicBlock *joinBB = i->bb->splitAfter(i); |
bld.setPosition(currBB, true); |
currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL); |
for (int l = 0; l <= 3; ++l) { |
const uint8_t qop = QUADOP(SUBR, SUBR, SUBR, SUBR); |
Value *pred = bld.getScratch(1, FILE_FLAGS); |
bld.setPosition(currBB, true); |
bld.mkQuadop(qop, pred, l, lod, lod)->flagsDef = 0; |
bld.mkFlow(OP_BRA, texiBB, CC_EQ, pred)->fixed = 1; |
currBB->cfg.attach(&texiBB->cfg, Graph::Edge::FORWARD); |
if (l <= 2) { |
BasicBlock *laneBB = new BasicBlock(func); |
currBB->cfg.attach(&laneBB->cfg, Graph::Edge::TREE); |
currBB = laneBB; |
} |
} |
bld.setPosition(joinBB, false); |
bld.mkOp(OP_JOIN, TYPE_NONE, NULL); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleTXD(TexInstruction *i) |
{ |
static const uint8_t qOps[4][2] = |
{ |
{ QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }, // l0 |
{ QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD, ADD) }, // l1 |
{ QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2 |
{ QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3 |
}; |
Value *def[4][4]; |
Value *crd[3]; |
Instruction *tex; |
Value *zero = bld.loadImm(bld.getSSA(), 0); |
int l, c; |
const int dim = i->tex.target.getDim(); |
handleTEX(i); |
i->op = OP_TEX; // no need to clone dPdx/dPdy later |
for (c = 0; c < dim; ++c) |
crd[c] = bld.getScratch(); |
bld.mkOp(OP_QUADON, TYPE_NONE, NULL); |
for (l = 0; l < 4; ++l) { |
// mov coordinates from lane l to all lanes |
for (c = 0; c < dim; ++c) |
bld.mkQuadop(0x00, crd[c], l, i->getSrc(c), zero); |
// add dPdx from lane l to lanes dx |
for (c = 0; c < dim; ++c) |
bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]); |
// add dPdy from lane l to lanes dy |
for (c = 0; c < dim; ++c) |
bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]); |
// texture |
bld.insert(tex = cloneForward(func, i)); |
for (c = 0; c < dim; ++c) |
tex->setSrc(c, crd[c]); |
// save results |
for (c = 0; i->defExists(c); ++c) { |
Instruction *mov; |
def[c][l] = bld.getSSA(); |
mov = bld.mkMov(def[c][l], tex->getDef(c)); |
mov->fixed = 1; |
mov->lanes = 1 << l; |
} |
} |
bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL); |
for (c = 0; i->defExists(c); ++c) { |
Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c)); |
for (l = 0; l < 4; ++l) |
u->setSrc(l, def[c][l]); |
} |
i->bb->remove(i); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleSET(Instruction *i) |
{ |
if (i->dType == TYPE_F32) { |
bld.setPosition(i, true); |
i->dType = TYPE_U32; |
bld.mkOp1(OP_ABS, TYPE_S32, i->getDef(0), i->getDef(0)); |
bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(0), TYPE_S32, i->getDef(0)); |
} |
return true; |
} |
bool |
NV50LoweringPreSSA::handleSLCT(CmpInstruction *i) |
{ |
Value *src0 = bld.getSSA(); |
Value *src1 = bld.getSSA(); |
Value *pred = bld.getScratch(1, FILE_FLAGS); |
Value *v0 = i->getSrc(0); |
Value *v1 = i->getSrc(1); |
// XXX: these probably shouldn't be immediates in the first place ... |
if (v0->asImm()) |
v0 = bld.mkMov(bld.getSSA(), v0)->getDef(0); |
if (v1->asImm()) |
v1 = bld.mkMov(bld.getSSA(), v1)->getDef(0); |
bld.setPosition(i, true); |
bld.mkMov(src0, v0)->setPredicate(CC_NE, pred); |
bld.mkMov(src1, v1)->setPredicate(CC_EQ, pred); |
bld.mkOp2(OP_UNION, i->dType, i->getDef(0), src0, src1); |
bld.setPosition(i, false); |
i->op = OP_SET; |
i->setFlagsDef(0, pred); |
i->dType = TYPE_U8; |
i->setSrc(0, i->getSrc(2)); |
i->setSrc(2, NULL); |
i->setSrc(1, bld.loadImm(NULL, 0)); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleSELP(Instruction *i) |
{ |
Value *src0 = bld.getSSA(); |
Value *src1 = bld.getSSA(); |
Value *v0 = i->getSrc(0); |
Value *v1 = i->getSrc(1); |
if (v0->asImm()) |
v0 = bld.mkMov(bld.getSSA(), v0)->getDef(0); |
if (v1->asImm()) |
v1 = bld.mkMov(bld.getSSA(), v1)->getDef(0); |
bld.mkMov(src0, v0)->setPredicate(CC_NE, i->getSrc(2)); |
bld.mkMov(src1, v1)->setPredicate(CC_EQ, i->getSrc(2)); |
bld.mkOp2(OP_UNION, i->dType, i->getDef(0), src0, src1); |
delete_Instruction(prog, i); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleWRSV(Instruction *i) |
{ |
Symbol *sym = i->getSrc(0)->asSym(); |
// these are all shader outputs, $sreg are not writeable |
uint32_t addr = targ->getSVAddress(FILE_SHADER_OUTPUT, sym); |
if (addr >= 0x400) |
return false; |
sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr); |
bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0), i->getSrc(1)); |
bld.getBB()->remove(i); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleCALL(Instruction *i) |
{ |
if (prog->getType() == Program::TYPE_COMPUTE) { |
// Add implicit "thread id" argument in $r0 to the function |
i->setSrc(i->srcCount(), tid); |
} |
return true; |
} |
bool |
NV50LoweringPreSSA::handlePRECONT(Instruction *i) |
{ |
delete_Instruction(prog, i); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleCONT(Instruction *i) |
{ |
i->op = OP_BRA; |
return true; |
} |
bool |
NV50LoweringPreSSA::handleRDSV(Instruction *i) |
{ |
Symbol *sym = i->getSrc(0)->asSym(); |
uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym); |
Value *def = i->getDef(0); |
SVSemantic sv = sym->reg.data.sv.sv; |
int idx = sym->reg.data.sv.index; |
if (addr >= 0x400) // mov $sreg |
return true; |
switch (sv) { |
case SV_POSITION: |
assert(prog->getType() == Program::TYPE_FRAGMENT); |
bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL); |
break; |
case SV_FACE: |
bld.mkInterp(NV50_IR_INTERP_FLAT, def, addr, NULL); |
if (i->dType == TYPE_F32) { |
bld.mkOp2(OP_AND, TYPE_U32, def, def, bld.mkImm(0x80000000)); |
bld.mkOp2(OP_XOR, TYPE_U32, def, def, bld.mkImm(0xbf800000)); |
} |
break; |
case SV_NCTAID: |
case SV_CTAID: |
case SV_NTID: |
if ((sv == SV_NCTAID && idx >= 2) || |
(sv == SV_NTID && idx >= 3)) { |
bld.mkMov(def, bld.mkImm(1)); |
} else if (sv == SV_CTAID && idx >= 2) { |
bld.mkMov(def, bld.mkImm(0)); |
} else { |
Value *x = bld.getSSA(2); |
bld.mkOp1(OP_LOAD, TYPE_U16, x, |
bld.mkSymbol(FILE_MEMORY_SHARED, 0, TYPE_U16, addr)); |
bld.mkCvt(OP_CVT, TYPE_U32, def, TYPE_U16, x); |
} |
break; |
case SV_TID: |
if (idx == 0) { |
bld.mkOp2(OP_AND, TYPE_U32, def, tid, bld.mkImm(0x0000ffff)); |
} else if (idx == 1) { |
bld.mkOp2(OP_AND, TYPE_U32, def, tid, bld.mkImm(0x03ff0000)); |
bld.mkOp2(OP_SHR, TYPE_U32, def, def, bld.mkImm(16)); |
} else if (idx == 2) { |
bld.mkOp2(OP_SHR, TYPE_U32, def, tid, bld.mkImm(26)); |
} else { |
bld.mkMov(def, bld.mkImm(0)); |
} |
break; |
default: |
bld.mkFetch(i->getDef(0), i->dType, |
FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), NULL); |
break; |
} |
bld.getBB()->remove(i); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleDIV(Instruction *i) |
{ |
if (!isFloatType(i->dType)) |
return true; |
bld.setPosition(i, false); |
Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(), i->getSrc(1)); |
i->op = OP_MUL; |
i->setSrc(1, rcp->getDef(0)); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleSQRT(Instruction *i) |
{ |
Instruction *rsq = bld.mkOp1(OP_RSQ, TYPE_F32, |
bld.getSSA(), i->getSrc(0)); |
i->op = OP_MUL; |
i->setSrc(1, rsq->getDef(0)); |
return true; |
} |
bool |
NV50LoweringPreSSA::handlePOW(Instruction *i) |
{ |
LValue *val = bld.getScratch(); |
bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0)); |
bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1; |
bld.mkOp1(OP_PREEX2, TYPE_F32, val, val); |
i->op = OP_EX2; |
i->setSrc(0, val); |
i->setSrc(1, NULL); |
return true; |
} |
bool |
NV50LoweringPreSSA::handleEXPORT(Instruction *i) |
{ |
if (prog->getType() == Program::TYPE_FRAGMENT) { |
if (i->getIndirect(0, 0)) { |
// TODO: redirect to l[] here, load to GPRs at exit |
return false; |
} else { |
int id = i->getSrc(0)->reg.data.offset / 4; // in 32 bit reg units |
i->op = OP_MOV; |
i->subOp = NV50_IR_SUBOP_MOV_FINAL; |
i->src(0).set(i->src(1)); |
i->setSrc(1, NULL); |
i->setDef(0, new_LValue(func, FILE_GPR)); |
i->getDef(0)->reg.data.id = id; |
prog->maxGPR = MAX2(prog->maxGPR, id); |
} |
} |
return true; |
} |
// Set flags according to predicate and make the instruction read $cX. |
void |
NV50LoweringPreSSA::checkPredicate(Instruction *insn) |
{ |
Value *pred = insn->getPredicate(); |
Value *cdst; |
if (!pred || pred->reg.file == FILE_FLAGS) |
return; |
cdst = bld.getSSA(1, FILE_FLAGS); |
bld.mkCmp(OP_SET, CC_NEU, insn->dType, cdst, bld.loadImm(NULL, 0), pred); |
insn->setPredicate(insn->cc, cdst); |
} |
// |
// - add quadop dance for texturing |
// - put FP outputs in GPRs |
// - convert instruction sequences |
// |
bool |
NV50LoweringPreSSA::visit(Instruction *i) |
{ |
bld.setPosition(i, false); |
if (i->cc != CC_ALWAYS) |
checkPredicate(i); |
switch (i->op) { |
case OP_TEX: |
case OP_TXF: |
case OP_TXG: |
return handleTEX(i->asTex()); |
case OP_TXB: |
return handleTXB(i->asTex()); |
case OP_TXL: |
return handleTXL(i->asTex()); |
case OP_TXD: |
return handleTXD(i->asTex()); |
case OP_EX2: |
bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0)); |
i->setSrc(0, i->getDef(0)); |
break; |
case OP_SET: |
return handleSET(i); |
case OP_SLCT: |
return handleSLCT(i->asCmp()); |
case OP_SELP: |
return handleSELP(i); |
case OP_POW: |
return handlePOW(i); |
case OP_DIV: |
return handleDIV(i); |
case OP_SQRT: |
return handleSQRT(i); |
case OP_EXPORT: |
return handleEXPORT(i); |
case OP_RDSV: |
return handleRDSV(i); |
case OP_WRSV: |
return handleWRSV(i); |
case OP_CALL: |
return handleCALL(i); |
case OP_PRECONT: |
return handlePRECONT(i); |
case OP_CONT: |
return handleCONT(i); |
default: |
break; |
} |
return true; |
} |
bool |
TargetNV50::runLegalizePass(Program *prog, CGStage stage) const |
{ |
bool ret = false; |
if (stage == CG_STAGE_PRE_SSA) { |
NV50LoweringPreSSA pass(prog); |
ret = pass.run(prog, false, true); |
} else |
if (stage == CG_STAGE_SSA) { |
if (!prog->targetPriv) |
prog->targetPriv = new std::list<Instruction *>(); |
NV50LegalizeSSA pass(prog); |
ret = pass.run(prog, false, true); |
} else |
if (stage == CG_STAGE_POST_RA) { |
NV50LegalizePostRA pass; |
ret = pass.run(prog, false, true); |
if (prog->targetPriv) |
delete reinterpret_cast<std::list<Instruction *> *>(prog->targetPriv); |
} |
return ret; |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_peephole.cpp |
---|
0,0 → 1,2464 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_target.h" |
#include "nv50_ir_build_util.h" |
extern "C" { |
#include "util/u_math.h" |
} |
namespace nv50_ir { |
bool |
Instruction::isNop() const |
{ |
if (op == OP_PHI || op == OP_SPLIT || op == OP_MERGE || op == OP_CONSTRAINT) |
return true; |
if (terminator || join) // XXX: should terminator imply flow ? |
return false; |
if (op == OP_ATOM) |
return false; |
if (!fixed && op == OP_NOP) |
return true; |
if (defExists(0) && def(0).rep()->reg.data.id < 0) { |
for (int d = 1; defExists(d); ++d) |
if (def(d).rep()->reg.data.id >= 0) |
WARN("part of vector result is unused !\n"); |
return true; |
} |
if (op == OP_MOV || op == OP_UNION) { |
if (!getDef(0)->equals(getSrc(0))) |
return false; |
if (op == OP_UNION) |
if (!def(0).rep()->equals(getSrc(1))) |
return false; |
return true; |
} |
return false; |
} |
bool Instruction::isDead() const |
{ |
if (op == OP_STORE || |
op == OP_EXPORT || |
op == OP_ATOM || |
op == OP_SUSTB || op == OP_SUSTP || op == OP_SUREDP || op == OP_SUREDB || |
op == OP_WRSV) |
return false; |
for (int d = 0; defExists(d); ++d) |
if (getDef(d)->refCount() || getDef(d)->reg.data.id >= 0) |
return false; |
if (terminator || asFlow()) |
return false; |
if (fixed) |
return false; |
return true; |
}; |
// ============================================================================= |
class CopyPropagation : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
}; |
// Propagate all MOVs forward to make subsequent optimization easier, except if |
// the sources stem from a phi, in which case we don't want to mess up potential |
// swaps $rX <-> $rY, i.e. do not create live range overlaps of phi src and def. |
bool |
CopyPropagation::visit(BasicBlock *bb) |
{ |
Instruction *mov, *si, *next; |
for (mov = bb->getEntry(); mov; mov = next) { |
next = mov->next; |
if (mov->op != OP_MOV || mov->fixed || !mov->getSrc(0)->asLValue()) |
continue; |
if (mov->getPredicate()) |
continue; |
if (mov->def(0).getFile() != mov->src(0).getFile()) |
continue; |
si = mov->getSrc(0)->getInsn(); |
if (mov->getDef(0)->reg.data.id < 0 && si && si->op != OP_PHI) { |
// propagate |
mov->def(0).replace(mov->getSrc(0), false); |
delete_Instruction(prog, mov); |
} |
} |
return true; |
} |
// ============================================================================= |
class LoadPropagation : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
void checkSwapSrc01(Instruction *); |
bool isCSpaceLoad(Instruction *); |
bool isImmd32Load(Instruction *); |
bool isAttribOrSharedLoad(Instruction *); |
}; |
bool |
LoadPropagation::isCSpaceLoad(Instruction *ld) |
{ |
return ld && ld->op == OP_LOAD && ld->src(0).getFile() == FILE_MEMORY_CONST; |
} |
bool |
LoadPropagation::isImmd32Load(Instruction *ld) |
{ |
if (!ld || (ld->op != OP_MOV) || (typeSizeof(ld->dType) != 4)) |
return false; |
return ld->src(0).getFile() == FILE_IMMEDIATE; |
} |
bool |
LoadPropagation::isAttribOrSharedLoad(Instruction *ld) |
{ |
return ld && |
(ld->op == OP_VFETCH || |
(ld->op == OP_LOAD && |
(ld->src(0).getFile() == FILE_SHADER_INPUT || |
ld->src(0).getFile() == FILE_MEMORY_SHARED))); |
} |
void |
LoadPropagation::checkSwapSrc01(Instruction *insn) |
{ |
if (!prog->getTarget()->getOpInfo(insn).commutative) |
if (insn->op != OP_SET && insn->op != OP_SLCT) |
return; |
if (insn->src(1).getFile() != FILE_GPR) |
return; |
Instruction *i0 = insn->getSrc(0)->getInsn(); |
Instruction *i1 = insn->getSrc(1)->getInsn(); |
if (isCSpaceLoad(i0)) { |
if (!isCSpaceLoad(i1)) |
insn->swapSources(0, 1); |
else |
return; |
} else |
if (isImmd32Load(i0)) { |
if (!isCSpaceLoad(i1) && !isImmd32Load(i1)) |
insn->swapSources(0, 1); |
else |
return; |
} else |
if (isAttribOrSharedLoad(i1)) { |
if (!isAttribOrSharedLoad(i0)) |
insn->swapSources(0, 1); |
else |
return; |
} else { |
return; |
} |
if (insn->op == OP_SET) |
insn->asCmp()->setCond = reverseCondCode(insn->asCmp()->setCond); |
else |
if (insn->op == OP_SLCT) |
insn->asCmp()->setCond = inverseCondCode(insn->asCmp()->setCond); |
} |
bool |
LoadPropagation::visit(BasicBlock *bb) |
{ |
const Target *targ = prog->getTarget(); |
Instruction *next; |
for (Instruction *i = bb->getEntry(); i; i = next) { |
next = i->next; |
if (i->op == OP_CALL) // calls have args as sources, they must be in regs |
continue; |
if (i->srcExists(1)) |
checkSwapSrc01(i); |
for (int s = 0; i->srcExists(s); ++s) { |
Instruction *ld = i->getSrc(s)->getInsn(); |
if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV)) |
continue; |
if (!targ->insnCanLoad(i, s, ld)) |
continue; |
// propagate ! |
i->setSrc(s, ld->getSrc(0)); |
if (ld->src(0).isIndirect(0)) |
i->setIndirect(s, 0, ld->getIndirect(0, 0)); |
if (ld->getDef(0)->refCount() == 0) |
delete_Instruction(prog, ld); |
} |
} |
return true; |
} |
// ============================================================================= |
// Evaluate constant expressions. |
class ConstantFolding : public Pass |
{ |
public: |
bool foldAll(Program *); |
private: |
virtual bool visit(BasicBlock *); |
void expr(Instruction *, ImmediateValue&, ImmediateValue&); |
void opnd(Instruction *, ImmediateValue&, int s); |
void unary(Instruction *, const ImmediateValue&); |
void tryCollapseChainedMULs(Instruction *, const int s, ImmediateValue&); |
// TGSI 'true' is converted to -1 by F2I(NEG(SET)), track back to SET |
CmpInstruction *findOriginForTestWithZero(Value *); |
unsigned int foldCount; |
BuildUtil bld; |
}; |
// TODO: remember generated immediates and only revisit these |
bool |
ConstantFolding::foldAll(Program *prog) |
{ |
unsigned int iterCount = 0; |
do { |
foldCount = 0; |
if (!run(prog)) |
return false; |
} while (foldCount && ++iterCount < 2); |
return true; |
} |
bool |
ConstantFolding::visit(BasicBlock *bb) |
{ |
Instruction *i, *next; |
for (i = bb->getEntry(); i; i = next) { |
next = i->next; |
if (i->op == OP_MOV || i->op == OP_CALL) |
continue; |
ImmediateValue src0, src1; |
if (i->srcExists(1) && |
i->src(0).getImmediate(src0) && i->src(1).getImmediate(src1)) |
expr(i, src0, src1); |
else |
if (i->srcExists(0) && i->src(0).getImmediate(src0)) |
opnd(i, src0, 0); |
else |
if (i->srcExists(1) && i->src(1).getImmediate(src1)) |
opnd(i, src1, 1); |
} |
return true; |
} |
CmpInstruction * |
ConstantFolding::findOriginForTestWithZero(Value *value) |
{ |
if (!value) |
return NULL; |
Instruction *insn = value->getInsn(); |
while (insn && insn->op != OP_SET) { |
Instruction *next = NULL; |
switch (insn->op) { |
case OP_NEG: |
case OP_ABS: |
case OP_CVT: |
next = insn->getSrc(0)->getInsn(); |
if (insn->sType != next->dType) |
return NULL; |
break; |
case OP_MOV: |
next = insn->getSrc(0)->getInsn(); |
break; |
default: |
return NULL; |
} |
insn = next; |
} |
return insn ? insn->asCmp() : NULL; |
} |
void |
Modifier::applyTo(ImmediateValue& imm) const |
{ |
if (!bits) // avoid failure if imm.reg.type is unhandled (e.g. b128) |
return; |
switch (imm.reg.type) { |
case TYPE_F32: |
if (bits & NV50_IR_MOD_ABS) |
imm.reg.data.f32 = fabsf(imm.reg.data.f32); |
if (bits & NV50_IR_MOD_NEG) |
imm.reg.data.f32 = -imm.reg.data.f32; |
if (bits & NV50_IR_MOD_SAT) { |
if (imm.reg.data.f32 < 0.0f) |
imm.reg.data.f32 = 0.0f; |
else |
if (imm.reg.data.f32 > 1.0f) |
imm.reg.data.f32 = 1.0f; |
} |
assert(!(bits & NV50_IR_MOD_NOT)); |
break; |
case TYPE_S8: // NOTE: will be extended |
case TYPE_S16: |
case TYPE_S32: |
case TYPE_U8: // NOTE: treated as signed |
case TYPE_U16: |
case TYPE_U32: |
if (bits & NV50_IR_MOD_ABS) |
imm.reg.data.s32 = (imm.reg.data.s32 >= 0) ? |
imm.reg.data.s32 : -imm.reg.data.s32; |
if (bits & NV50_IR_MOD_NEG) |
imm.reg.data.s32 = -imm.reg.data.s32; |
if (bits & NV50_IR_MOD_NOT) |
imm.reg.data.s32 = ~imm.reg.data.s32; |
break; |
case TYPE_F64: |
if (bits & NV50_IR_MOD_ABS) |
imm.reg.data.f64 = fabs(imm.reg.data.f64); |
if (bits & NV50_IR_MOD_NEG) |
imm.reg.data.f64 = -imm.reg.data.f64; |
if (bits & NV50_IR_MOD_SAT) { |
if (imm.reg.data.f64 < 0.0) |
imm.reg.data.f64 = 0.0; |
else |
if (imm.reg.data.f64 > 1.0) |
imm.reg.data.f64 = 1.0; |
} |
assert(!(bits & NV50_IR_MOD_NOT)); |
break; |
default: |
assert(!"invalid/unhandled type"); |
imm.reg.data.u64 = 0; |
break; |
} |
} |
operation |
Modifier::getOp() const |
{ |
switch (bits) { |
case NV50_IR_MOD_ABS: return OP_ABS; |
case NV50_IR_MOD_NEG: return OP_NEG; |
case NV50_IR_MOD_SAT: return OP_SAT; |
case NV50_IR_MOD_NOT: return OP_NOT; |
case 0: |
return OP_MOV; |
default: |
return OP_CVT; |
} |
} |
void |
ConstantFolding::expr(Instruction *i, |
ImmediateValue &imm0, ImmediateValue &imm1) |
{ |
struct Storage *const a = &imm0.reg, *const b = &imm1.reg; |
struct Storage res; |
memset(&res.data, 0, sizeof(res.data)); |
switch (i->op) { |
case OP_MAD: |
case OP_FMA: |
case OP_MUL: |
if (i->dnz && i->dType == TYPE_F32) { |
if (!isfinite(a->data.f32)) |
a->data.f32 = 0.0f; |
if (!isfinite(b->data.f32)) |
b->data.f32 = 0.0f; |
} |
switch (i->dType) { |
case TYPE_F32: res.data.f32 = a->data.f32 * b->data.f32; break; |
case TYPE_F64: res.data.f64 = a->data.f64 * b->data.f64; break; |
case TYPE_S32: |
case TYPE_U32: res.data.u32 = a->data.u32 * b->data.u32; break; |
default: |
return; |
} |
break; |
case OP_DIV: |
if (b->data.u32 == 0) |
break; |
switch (i->dType) { |
case TYPE_F32: res.data.f32 = a->data.f32 / b->data.f32; break; |
case TYPE_F64: res.data.f64 = a->data.f64 / b->data.f64; break; |
case TYPE_S32: res.data.s32 = a->data.s32 / b->data.s32; break; |
case TYPE_U32: res.data.u32 = a->data.u32 / b->data.u32; break; |
default: |
return; |
} |
break; |
case OP_ADD: |
switch (i->dType) { |
case TYPE_F32: res.data.f32 = a->data.f32 + b->data.f32; break; |
case TYPE_F64: res.data.f64 = a->data.f64 + b->data.f64; break; |
case TYPE_S32: |
case TYPE_U32: res.data.u32 = a->data.u32 + b->data.u32; break; |
default: |
return; |
} |
break; |
case OP_POW: |
switch (i->dType) { |
case TYPE_F32: res.data.f32 = pow(a->data.f32, b->data.f32); break; |
case TYPE_F64: res.data.f64 = pow(a->data.f64, b->data.f64); break; |
default: |
return; |
} |
break; |
case OP_MAX: |
switch (i->dType) { |
case TYPE_F32: res.data.f32 = MAX2(a->data.f32, b->data.f32); break; |
case TYPE_F64: res.data.f64 = MAX2(a->data.f64, b->data.f64); break; |
case TYPE_S32: res.data.s32 = MAX2(a->data.s32, b->data.s32); break; |
case TYPE_U32: res.data.u32 = MAX2(a->data.u32, b->data.u32); break; |
default: |
return; |
} |
break; |
case OP_MIN: |
switch (i->dType) { |
case TYPE_F32: res.data.f32 = MIN2(a->data.f32, b->data.f32); break; |
case TYPE_F64: res.data.f64 = MIN2(a->data.f64, b->data.f64); break; |
case TYPE_S32: res.data.s32 = MIN2(a->data.s32, b->data.s32); break; |
case TYPE_U32: res.data.u32 = MIN2(a->data.u32, b->data.u32); break; |
default: |
return; |
} |
break; |
case OP_AND: |
res.data.u64 = a->data.u64 & b->data.u64; |
break; |
case OP_OR: |
res.data.u64 = a->data.u64 | b->data.u64; |
break; |
case OP_XOR: |
res.data.u64 = a->data.u64 ^ b->data.u64; |
break; |
case OP_SHL: |
res.data.u32 = a->data.u32 << b->data.u32; |
break; |
case OP_SHR: |
switch (i->dType) { |
case TYPE_S32: res.data.s32 = a->data.s32 >> b->data.u32; break; |
case TYPE_U32: res.data.u32 = a->data.u32 >> b->data.u32; break; |
default: |
return; |
} |
break; |
case OP_SLCT: |
if (a->data.u32 != b->data.u32) |
return; |
res.data.u32 = a->data.u32; |
break; |
default: |
return; |
} |
++foldCount; |
i->src(0).mod = Modifier(0); |
i->src(1).mod = Modifier(0); |
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32)); |
i->setSrc(1, NULL); |
i->getSrc(0)->reg.data = res.data; |
if (i->op == OP_MAD || i->op == OP_FMA) { |
i->op = OP_ADD; |
i->setSrc(1, i->getSrc(0)); |
i->src(1).mod = i->src(2).mod; |
i->setSrc(0, i->getSrc(2)); |
i->setSrc(2, NULL); |
ImmediateValue src0; |
if (i->src(0).getImmediate(src0)) |
expr(i, src0, *i->getSrc(1)->asImm()); |
} else { |
i->op = OP_MOV; |
} |
} |
void |
ConstantFolding::unary(Instruction *i, const ImmediateValue &imm) |
{ |
Storage res; |
if (i->dType != TYPE_F32) |
return; |
switch (i->op) { |
case OP_NEG: res.data.f32 = -imm.reg.data.f32; break; |
case OP_ABS: res.data.f32 = fabsf(imm.reg.data.f32); break; |
case OP_RCP: res.data.f32 = 1.0f / imm.reg.data.f32; break; |
case OP_RSQ: res.data.f32 = 1.0f / sqrtf(imm.reg.data.f32); break; |
case OP_LG2: res.data.f32 = log2f(imm.reg.data.f32); break; |
case OP_EX2: res.data.f32 = exp2f(imm.reg.data.f32); break; |
case OP_SIN: res.data.f32 = sinf(imm.reg.data.f32); break; |
case OP_COS: res.data.f32 = cosf(imm.reg.data.f32); break; |
case OP_SQRT: res.data.f32 = sqrtf(imm.reg.data.f32); break; |
case OP_PRESIN: |
case OP_PREEX2: |
// these should be handled in subsequent OP_SIN/COS/EX2 |
res.data.f32 = imm.reg.data.f32; |
break; |
default: |
return; |
} |
i->op = OP_MOV; |
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.f32)); |
i->src(0).mod = Modifier(0); |
} |
void |
ConstantFolding::tryCollapseChainedMULs(Instruction *mul2, |
const int s, ImmediateValue& imm2) |
{ |
const int t = s ? 0 : 1; |
Instruction *insn; |
Instruction *mul1 = NULL; // mul1 before mul2 |
int e = 0; |
float f = imm2.reg.data.f32; |
ImmediateValue imm1; |
assert(mul2->op == OP_MUL && mul2->dType == TYPE_F32); |
if (mul2->getSrc(t)->refCount() == 1) { |
insn = mul2->getSrc(t)->getInsn(); |
if (!mul2->src(t).mod && insn->op == OP_MUL && insn->dType == TYPE_F32) |
mul1 = insn; |
if (mul1 && !mul1->saturate) { |
int s1; |
if (mul1->src(s1 = 0).getImmediate(imm1) || |
mul1->src(s1 = 1).getImmediate(imm1)) { |
bld.setPosition(mul1, false); |
// a = mul r, imm1 |
// d = mul a, imm2 -> d = mul r, (imm1 * imm2) |
mul1->setSrc(s1, bld.loadImm(NULL, f * imm1.reg.data.f32)); |
mul1->src(s1).mod = Modifier(0); |
mul2->def(0).replace(mul1->getDef(0), false); |
} else |
if (prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) { |
// c = mul a, b |
// d = mul c, imm -> d = mul_x_imm a, b |
mul1->postFactor = e; |
mul2->def(0).replace(mul1->getDef(0), false); |
if (f < 0) |
mul1->src(0).mod *= Modifier(NV50_IR_MOD_NEG); |
} |
mul1->saturate = mul2->saturate; |
return; |
} |
} |
if (mul2->getDef(0)->refCount() == 1 && !mul2->saturate) { |
// b = mul a, imm |
// d = mul b, c -> d = mul_x_imm a, c |
int s2, t2; |
insn = mul2->getDef(0)->uses.front()->getInsn(); |
if (!insn) |
return; |
mul1 = mul2; |
mul2 = NULL; |
s2 = insn->getSrc(0) == mul1->getDef(0) ? 0 : 1; |
t2 = s2 ? 0 : 1; |
if (insn->op == OP_MUL && insn->dType == TYPE_F32) |
if (!insn->src(s2).mod && !insn->src(t2).getImmediate(imm1)) |
mul2 = insn; |
if (mul2 && prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) { |
mul2->postFactor = e; |
mul2->setSrc(s2, mul1->src(t)); |
if (f < 0) |
mul2->src(s2).mod *= Modifier(NV50_IR_MOD_NEG); |
} |
} |
} |
void |
ConstantFolding::opnd(Instruction *i, ImmediateValue &imm0, int s) |
{ |
const int t = !s; |
const operation op = i->op; |
switch (i->op) { |
case OP_MUL: |
if (i->dType == TYPE_F32) |
tryCollapseChainedMULs(i, s, imm0); |
if (imm0.isInteger(0)) { |
i->op = OP_MOV; |
i->setSrc(0, new_ImmediateValue(prog, 0u)); |
i->src(0).mod = Modifier(0); |
i->setSrc(1, NULL); |
} else |
if (imm0.isInteger(1) || imm0.isInteger(-1)) { |
if (imm0.isNegative()) |
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG); |
i->op = i->src(t).mod.getOp(); |
if (s == 0) { |
i->setSrc(0, i->getSrc(1)); |
i->src(0).mod = i->src(1).mod; |
i->src(1).mod = 0; |
} |
if (i->op != OP_CVT) |
i->src(0).mod = 0; |
i->setSrc(1, NULL); |
} else |
if (imm0.isInteger(2) || imm0.isInteger(-2)) { |
if (imm0.isNegative()) |
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG); |
i->op = OP_ADD; |
i->setSrc(s, i->getSrc(t)); |
i->src(s).mod = i->src(t).mod; |
} else |
if (!isFloatType(i->sType) && !imm0.isNegative() && imm0.isPow2()) { |
i->op = OP_SHL; |
imm0.applyLog2(); |
i->setSrc(0, i->getSrc(t)); |
i->src(0).mod = i->src(t).mod; |
i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32)); |
i->src(1).mod = 0; |
} |
break; |
case OP_ADD: |
if (i->usesFlags()) |
break; |
if (imm0.isInteger(0)) { |
if (s == 0) { |
i->setSrc(0, i->getSrc(1)); |
i->src(0).mod = i->src(1).mod; |
} |
i->setSrc(1, NULL); |
i->op = i->src(0).mod.getOp(); |
if (i->op != OP_CVT) |
i->src(0).mod = Modifier(0); |
} |
break; |
case OP_DIV: |
if (s != 1 || (i->dType != TYPE_S32 && i->dType != TYPE_U32)) |
break; |
bld.setPosition(i, false); |
if (imm0.reg.data.u32 == 0) { |
break; |
} else |
if (imm0.reg.data.u32 == 1) { |
i->op = OP_MOV; |
i->setSrc(1, NULL); |
} else |
if (i->dType == TYPE_U32 && imm0.isPow2()) { |
i->op = OP_SHR; |
i->setSrc(1, bld.mkImm(util_logbase2(imm0.reg.data.u32))); |
} else |
if (i->dType == TYPE_U32) { |
Instruction *mul; |
Value *tA, *tB; |
const uint32_t d = imm0.reg.data.u32; |
uint32_t m; |
int r, s; |
uint32_t l = util_logbase2(d); |
if (((uint32_t)1 << l) < d) |
++l; |
m = (((uint64_t)1 << 32) * (((uint64_t)1 << l) - d)) / d + 1; |
r = l ? 1 : 0; |
s = l ? (l - 1) : 0; |
tA = bld.getSSA(); |
tB = bld.getSSA(); |
mul = bld.mkOp2(OP_MUL, TYPE_U32, tA, i->getSrc(0), |
bld.loadImm(NULL, m)); |
mul->subOp = NV50_IR_SUBOP_MUL_HIGH; |
bld.mkOp2(OP_SUB, TYPE_U32, tB, i->getSrc(0), tA); |
tA = bld.getSSA(); |
if (r) |
bld.mkOp2(OP_SHR, TYPE_U32, tA, tB, bld.mkImm(r)); |
else |
tA = tB; |
tB = s ? bld.getSSA() : i->getDef(0); |
bld.mkOp2(OP_ADD, TYPE_U32, tB, mul->getDef(0), tA); |
if (s) |
bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), tB, bld.mkImm(s)); |
delete_Instruction(prog, i); |
} else |
if (imm0.reg.data.s32 == -1) { |
i->op = OP_NEG; |
i->setSrc(1, NULL); |
} else { |
LValue *tA, *tB; |
LValue *tD; |
const int32_t d = imm0.reg.data.s32; |
int32_t m; |
int32_t l = util_logbase2(static_cast<unsigned>(abs(d))); |
if ((1 << l) < abs(d)) |
++l; |
if (!l) |
l = 1; |
m = ((uint64_t)1 << (32 + l - 1)) / abs(d) + 1 - ((uint64_t)1 << 32); |
tA = bld.getSSA(); |
tB = bld.getSSA(); |
bld.mkOp3(OP_MAD, TYPE_S32, tA, i->getSrc(0), bld.loadImm(NULL, m), |
i->getSrc(0))->subOp = NV50_IR_SUBOP_MUL_HIGH; |
if (l > 1) |
bld.mkOp2(OP_SHR, TYPE_S32, tB, tA, bld.mkImm(l - 1)); |
else |
tB = tA; |
tA = bld.getSSA(); |
bld.mkCmp(OP_SET, CC_LT, TYPE_S32, tA, i->getSrc(0), bld.mkImm(0)); |
tD = (d < 0) ? bld.getSSA() : i->getDef(0)->asLValue(); |
bld.mkOp2(OP_SUB, TYPE_U32, tD, tB, tA); |
if (d < 0) |
bld.mkOp1(OP_NEG, TYPE_S32, i->getDef(0), tB); |
delete_Instruction(prog, i); |
} |
break; |
case OP_MOD: |
if (i->sType == TYPE_U32 && imm0.isPow2()) { |
bld.setPosition(i, false); |
i->op = OP_AND; |
i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 - 1)); |
} |
break; |
case OP_SET: // TODO: SET_AND,OR,XOR |
{ |
CmpInstruction *si = findOriginForTestWithZero(i->getSrc(t)); |
CondCode cc, ccZ; |
if (i->src(t).mod != Modifier(0)) |
return; |
if (imm0.reg.data.u32 != 0 || !si || si->op != OP_SET) |
return; |
cc = si->setCond; |
ccZ = (CondCode)((unsigned int)i->asCmp()->setCond & ~CC_U); |
if (s == 0) |
ccZ = reverseCondCode(ccZ); |
switch (ccZ) { |
case CC_LT: cc = CC_FL; break; |
case CC_GE: cc = CC_TR; break; |
case CC_EQ: cc = inverseCondCode(cc); break; |
case CC_LE: cc = inverseCondCode(cc); break; |
case CC_GT: break; |
case CC_NE: break; |
default: |
return; |
} |
i->asCmp()->setCond = cc; |
i->setSrc(0, si->src(0)); |
i->setSrc(1, si->src(1)); |
i->sType = si->sType; |
} |
break; |
case OP_SHL: |
{ |
if (s != 1 || i->src(0).mod != Modifier(0)) |
break; |
// try to concatenate shifts |
Instruction *si = i->getSrc(0)->getInsn(); |
if (!si || si->op != OP_SHL) |
break; |
ImmediateValue imm1; |
if (si->src(1).getImmediate(imm1)) { |
bld.setPosition(i, false); |
i->setSrc(0, si->getSrc(0)); |
i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 + imm1.reg.data.u32)); |
} |
} |
break; |
case OP_ABS: |
case OP_NEG: |
case OP_LG2: |
case OP_RCP: |
case OP_SQRT: |
case OP_RSQ: |
case OP_PRESIN: |
case OP_SIN: |
case OP_COS: |
case OP_PREEX2: |
case OP_EX2: |
unary(i, imm0); |
break; |
default: |
return; |
} |
if (i->op != op) |
foldCount++; |
} |
// ============================================================================= |
// Merge modifier operations (ABS, NEG, NOT) into ValueRefs where allowed. |
class ModifierFolding : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
}; |
bool |
ModifierFolding::visit(BasicBlock *bb) |
{ |
const Target *target = prog->getTarget(); |
Instruction *i, *next, *mi; |
Modifier mod; |
for (i = bb->getEntry(); i; i = next) { |
next = i->next; |
if (0 && i->op == OP_SUB) { |
// turn "sub" into "add neg" (do we really want this ?) |
i->op = OP_ADD; |
i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG); |
} |
for (int s = 0; s < 3 && i->srcExists(s); ++s) { |
mi = i->getSrc(s)->getInsn(); |
if (!mi || |
mi->predSrc >= 0 || mi->getDef(0)->refCount() > 8) |
continue; |
if (i->sType == TYPE_U32 && mi->dType == TYPE_S32) { |
if ((i->op != OP_ADD && |
i->op != OP_MUL) || |
(mi->op != OP_ABS && |
mi->op != OP_NEG)) |
continue; |
} else |
if (i->sType != mi->dType) { |
continue; |
} |
if ((mod = Modifier(mi->op)) == Modifier(0)) |
continue; |
mod *= mi->src(0).mod; |
if ((i->op == OP_ABS) || i->src(s).mod.abs()) { |
// abs neg [abs] = abs |
mod = mod & Modifier(~(NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)); |
} else |
if ((i->op == OP_NEG) && mod.neg()) { |
assert(s == 0); |
// neg as both opcode and modifier on same insn is prohibited |
// neg neg abs = abs, neg neg = identity |
mod = mod & Modifier(~NV50_IR_MOD_NEG); |
i->op = mod.getOp(); |
mod = mod & Modifier(~NV50_IR_MOD_ABS); |
if (mod == Modifier(0)) |
i->op = OP_MOV; |
} |
if (target->isModSupported(i, s, mod)) { |
i->setSrc(s, mi->getSrc(0)); |
i->src(s).mod *= mod; |
} |
} |
if (i->op == OP_SAT) { |
mi = i->getSrc(0)->getInsn(); |
if (mi && |
mi->getDef(0)->refCount() <= 1 && target->isSatSupported(mi)) { |
mi->saturate = 1; |
mi->setDef(0, i->getDef(0)); |
delete_Instruction(prog, i); |
} |
} |
} |
return true; |
} |
// ============================================================================= |
// MUL + ADD -> MAD/FMA |
// MIN/MAX(a, a) -> a, etc. |
// SLCT(a, b, const) -> cc(const) ? a : b |
// RCP(RCP(a)) -> a |
// MUL(MUL(a, b), const) -> MUL_Xconst(a, b) |
class AlgebraicOpt : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
void handleABS(Instruction *); |
bool handleADD(Instruction *); |
bool tryADDToMADOrSAD(Instruction *, operation toOp); |
void handleMINMAX(Instruction *); |
void handleRCP(Instruction *); |
void handleSLCT(Instruction *); |
void handleLOGOP(Instruction *); |
void handleCVT(Instruction *); |
void handleSUCLAMP(Instruction *); |
BuildUtil bld; |
}; |
void |
AlgebraicOpt::handleABS(Instruction *abs) |
{ |
Instruction *sub = abs->getSrc(0)->getInsn(); |
DataType ty; |
if (!sub || |
!prog->getTarget()->isOpSupported(OP_SAD, abs->dType)) |
return; |
// expect not to have mods yet, if we do, bail |
if (sub->src(0).mod || sub->src(1).mod) |
return; |
// hidden conversion ? |
ty = intTypeToSigned(sub->dType); |
if (abs->dType != abs->sType || ty != abs->sType) |
return; |
if ((sub->op != OP_ADD && sub->op != OP_SUB) || |
sub->src(0).getFile() != FILE_GPR || sub->src(0).mod || |
sub->src(1).getFile() != FILE_GPR || sub->src(1).mod) |
return; |
Value *src0 = sub->getSrc(0); |
Value *src1 = sub->getSrc(1); |
if (sub->op == OP_ADD) { |
Instruction *neg = sub->getSrc(1)->getInsn(); |
if (neg && neg->op != OP_NEG) { |
neg = sub->getSrc(0)->getInsn(); |
src0 = sub->getSrc(1); |
} |
if (!neg || neg->op != OP_NEG || |
neg->dType != neg->sType || neg->sType != ty) |
return; |
src1 = neg->getSrc(0); |
} |
// found ABS(SUB)) |
abs->moveSources(1, 2); // move sources >=1 up by 2 |
abs->op = OP_SAD; |
abs->setType(sub->dType); |
abs->setSrc(0, src0); |
abs->setSrc(1, src1); |
bld.setPosition(abs, false); |
abs->setSrc(2, bld.loadImm(bld.getSSA(typeSizeof(ty)), 0)); |
} |
bool |
AlgebraicOpt::handleADD(Instruction *add) |
{ |
Value *src0 = add->getSrc(0); |
Value *src1 = add->getSrc(1); |
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR) |
return false; |
bool changed = false; |
if (!changed && prog->getTarget()->isOpSupported(OP_MAD, add->dType)) |
changed = tryADDToMADOrSAD(add, OP_MAD); |
if (!changed && prog->getTarget()->isOpSupported(OP_SAD, add->dType)) |
changed = tryADDToMADOrSAD(add, OP_SAD); |
return changed; |
} |
// ADD(SAD(a,b,0), c) -> SAD(a,b,c) |
// ADD(MUL(a,b), c) -> MAD(a,b,c) |
bool |
AlgebraicOpt::tryADDToMADOrSAD(Instruction *add, operation toOp) |
{ |
Value *src0 = add->getSrc(0); |
Value *src1 = add->getSrc(1); |
Value *src; |
int s; |
const operation srcOp = toOp == OP_SAD ? OP_SAD : OP_MUL; |
const Modifier modBad = Modifier(~((toOp == OP_MAD) ? NV50_IR_MOD_NEG : 0)); |
Modifier mod[4]; |
if (src0->refCount() == 1 && |
src0->getUniqueInsn() && src0->getUniqueInsn()->op == srcOp) |
s = 0; |
else |
if (src1->refCount() == 1 && |
src1->getUniqueInsn() && src1->getUniqueInsn()->op == srcOp) |
s = 1; |
else |
return false; |
if ((src0->getUniqueInsn() && src0->getUniqueInsn()->bb != add->bb) || |
(src1->getUniqueInsn() && src1->getUniqueInsn()->bb != add->bb)) |
return false; |
src = add->getSrc(s); |
if (src->getInsn()->postFactor) |
return false; |
if (toOp == OP_SAD) { |
ImmediateValue imm; |
if (!src->getInsn()->src(2).getImmediate(imm)) |
return false; |
if (!imm.isInteger(0)) |
return false; |
} |
mod[0] = add->src(0).mod; |
mod[1] = add->src(1).mod; |
mod[2] = src->getUniqueInsn()->src(0).mod; |
mod[3] = src->getUniqueInsn()->src(1).mod; |
if (((mod[0] | mod[1]) | (mod[2] | mod[3])) & modBad) |
return false; |
add->op = toOp; |
add->subOp = src->getInsn()->subOp; // potentially mul-high |
add->setSrc(2, add->src(s ? 0 : 1)); |
add->setSrc(0, src->getInsn()->getSrc(0)); |
add->src(0).mod = mod[2] ^ mod[s]; |
add->setSrc(1, src->getInsn()->getSrc(1)); |
add->src(1).mod = mod[3]; |
return true; |
} |
void |
AlgebraicOpt::handleMINMAX(Instruction *minmax) |
{ |
Value *src0 = minmax->getSrc(0); |
Value *src1 = minmax->getSrc(1); |
if (src0 != src1 || src0->reg.file != FILE_GPR) |
return; |
if (minmax->src(0).mod == minmax->src(1).mod) { |
if (minmax->def(0).mayReplace(minmax->src(0))) { |
minmax->def(0).replace(minmax->src(0), false); |
minmax->bb->remove(minmax); |
} else { |
minmax->op = OP_CVT; |
minmax->setSrc(1, NULL); |
} |
} else { |
// TODO: |
// min(x, -x) = -abs(x) |
// min(x, -abs(x)) = -abs(x) |
// min(x, abs(x)) = x |
// max(x, -abs(x)) = x |
// max(x, abs(x)) = abs(x) |
// max(x, -x) = abs(x) |
} |
} |
void |
AlgebraicOpt::handleRCP(Instruction *rcp) |
{ |
Instruction *si = rcp->getSrc(0)->getUniqueInsn(); |
if (si && si->op == OP_RCP) { |
Modifier mod = rcp->src(0).mod * si->src(0).mod; |
rcp->op = mod.getOp(); |
rcp->setSrc(0, si->getSrc(0)); |
} |
} |
void |
AlgebraicOpt::handleSLCT(Instruction *slct) |
{ |
if (slct->getSrc(2)->reg.file == FILE_IMMEDIATE) { |
if (slct->getSrc(2)->asImm()->compare(slct->asCmp()->setCond, 0.0f)) |
slct->setSrc(0, slct->getSrc(1)); |
} else |
if (slct->getSrc(0) != slct->getSrc(1)) { |
return; |
} |
slct->op = OP_MOV; |
slct->setSrc(1, NULL); |
slct->setSrc(2, NULL); |
} |
void |
AlgebraicOpt::handleLOGOP(Instruction *logop) |
{ |
Value *src0 = logop->getSrc(0); |
Value *src1 = logop->getSrc(1); |
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR) |
return; |
if (src0 == src1) { |
if ((logop->op == OP_AND || logop->op == OP_OR) && |
logop->def(0).mayReplace(logop->src(0))) { |
logop->def(0).replace(logop->src(0), false); |
delete_Instruction(prog, logop); |
} |
} else { |
// try AND(SET, SET) -> SET_AND(SET) |
Instruction *set0 = src0->getInsn(); |
Instruction *set1 = src1->getInsn(); |
if (!set0 || set0->fixed || !set1 || set1->fixed) |
return; |
if (set1->op != OP_SET) { |
Instruction *xchg = set0; |
set0 = set1; |
set1 = xchg; |
if (set1->op != OP_SET) |
return; |
} |
operation redOp = (logop->op == OP_AND ? OP_SET_AND : |
logop->op == OP_XOR ? OP_SET_XOR : OP_SET_OR); |
if (!prog->getTarget()->isOpSupported(redOp, set1->sType)) |
return; |
if (set0->op != OP_SET && |
set0->op != OP_SET_AND && |
set0->op != OP_SET_OR && |
set0->op != OP_SET_XOR) |
return; |
if (set0->getDef(0)->refCount() > 1 && |
set1->getDef(0)->refCount() > 1) |
return; |
if (set0->getPredicate() || set1->getPredicate()) |
return; |
// check that they don't source each other |
for (int s = 0; s < 2; ++s) |
if (set0->getSrc(s) == set1->getDef(0) || |
set1->getSrc(s) == set0->getDef(0)) |
return; |
set0 = cloneForward(func, set0); |
set1 = cloneShallow(func, set1); |
logop->bb->insertAfter(logop, set1); |
logop->bb->insertAfter(logop, set0); |
set0->dType = TYPE_U8; |
set0->getDef(0)->reg.file = FILE_PREDICATE; |
set0->getDef(0)->reg.size = 1; |
set1->setSrc(2, set0->getDef(0)); |
set1->op = redOp; |
set1->setDef(0, logop->getDef(0)); |
delete_Instruction(prog, logop); |
} |
} |
// F2I(NEG(SET with result 1.0f/0.0f)) -> SET with result -1/0 |
// nv50: |
// F2I(NEG(I2F(ABS(SET)))) |
void |
AlgebraicOpt::handleCVT(Instruction *cvt) |
{ |
if (cvt->sType != TYPE_F32 || |
cvt->dType != TYPE_S32 || cvt->src(0).mod != Modifier(0)) |
return; |
Instruction *insn = cvt->getSrc(0)->getInsn(); |
if (!insn || insn->op != OP_NEG || insn->dType != TYPE_F32) |
return; |
if (insn->src(0).mod != Modifier(0)) |
return; |
insn = insn->getSrc(0)->getInsn(); |
// check for nv50 SET(-1,0) -> SET(1.0f/0.0f) chain and nvc0's f32 SET |
if (insn && insn->op == OP_CVT && |
insn->dType == TYPE_F32 && |
insn->sType == TYPE_S32) { |
insn = insn->getSrc(0)->getInsn(); |
if (!insn || insn->op != OP_ABS || insn->sType != TYPE_S32 || |
insn->src(0).mod) |
return; |
insn = insn->getSrc(0)->getInsn(); |
if (!insn || insn->op != OP_SET || insn->dType != TYPE_U32) |
return; |
} else |
if (!insn || insn->op != OP_SET || insn->dType != TYPE_F32) { |
return; |
} |
Instruction *bset = cloneShallow(func, insn); |
bset->dType = TYPE_U32; |
bset->setDef(0, cvt->getDef(0)); |
cvt->bb->insertAfter(cvt, bset); |
delete_Instruction(prog, cvt); |
} |
// SUCLAMP dst, (ADD b imm), k, 0 -> SUCLAMP dst, b, k, imm (if imm fits s6) |
void |
AlgebraicOpt::handleSUCLAMP(Instruction *insn) |
{ |
ImmediateValue imm; |
int32_t val = insn->getSrc(2)->asImm()->reg.data.s32; |
int s; |
Instruction *add; |
assert(insn->srcExists(0) && insn->src(0).getFile() == FILE_GPR); |
// look for ADD (TODO: only count references by non-SUCLAMP) |
if (insn->getSrc(0)->refCount() > 1) |
return; |
add = insn->getSrc(0)->getInsn(); |
if (!add || add->op != OP_ADD || |
(add->dType != TYPE_U32 && |
add->dType != TYPE_S32)) |
return; |
// look for immediate |
for (s = 0; s < 2; ++s) |
if (add->src(s).getImmediate(imm)) |
break; |
if (s >= 2) |
return; |
s = s ? 0 : 1; |
// determine if immediate fits |
val += imm.reg.data.s32; |
if (val > 31 || val < -32) |
return; |
// determine if other addend fits |
if (add->src(s).getFile() != FILE_GPR || add->src(s).mod != Modifier(0)) |
return; |
bld.setPosition(insn, false); // make sure bld is init'ed |
// replace sources |
insn->setSrc(2, bld.mkImm(val)); |
insn->setSrc(0, add->getSrc(s)); |
} |
bool |
AlgebraicOpt::visit(BasicBlock *bb) |
{ |
Instruction *next; |
for (Instruction *i = bb->getEntry(); i; i = next) { |
next = i->next; |
switch (i->op) { |
case OP_ABS: |
handleABS(i); |
break; |
case OP_ADD: |
handleADD(i); |
break; |
case OP_RCP: |
handleRCP(i); |
break; |
case OP_MIN: |
case OP_MAX: |
handleMINMAX(i); |
break; |
case OP_SLCT: |
handleSLCT(i); |
break; |
case OP_AND: |
case OP_OR: |
case OP_XOR: |
handleLOGOP(i); |
break; |
case OP_CVT: |
handleCVT(i); |
break; |
case OP_SUCLAMP: |
handleSUCLAMP(i); |
break; |
default: |
break; |
} |
} |
return true; |
} |
// ============================================================================= |
static inline void |
updateLdStOffset(Instruction *ldst, int32_t offset, Function *fn) |
{ |
if (offset != ldst->getSrc(0)->reg.data.offset) { |
if (ldst->getSrc(0)->refCount() > 1) |
ldst->setSrc(0, cloneShallow(fn, ldst->getSrc(0))); |
ldst->getSrc(0)->reg.data.offset = offset; |
} |
} |
// Combine loads and stores, forward stores to loads where possible. |
class MemoryOpt : public Pass |
{ |
private: |
class Record |
{ |
public: |
Record *next; |
Instruction *insn; |
const Value *rel[2]; |
const Value *base; |
int32_t offset; |
int8_t fileIndex; |
uint8_t size; |
bool locked; |
Record *prev; |
bool overlaps(const Instruction *ldst) const; |
inline void link(Record **); |
inline void unlink(Record **); |
inline void set(const Instruction *ldst); |
}; |
public: |
MemoryOpt(); |
Record *loads[DATA_FILE_COUNT]; |
Record *stores[DATA_FILE_COUNT]; |
MemoryPool recordPool; |
private: |
virtual bool visit(BasicBlock *); |
bool runOpt(BasicBlock *); |
Record **getList(const Instruction *); |
Record *findRecord(const Instruction *, bool load, bool& isAdjacent) const; |
// merge @insn into load/store instruction from @rec |
bool combineLd(Record *rec, Instruction *ld); |
bool combineSt(Record *rec, Instruction *st); |
bool replaceLdFromLd(Instruction *ld, Record *ldRec); |
bool replaceLdFromSt(Instruction *ld, Record *stRec); |
bool replaceStFromSt(Instruction *restrict st, Record *stRec); |
void addRecord(Instruction *ldst); |
void purgeRecords(Instruction *const st, DataFile); |
void lockStores(Instruction *const ld); |
void reset(); |
private: |
Record *prevRecord; |
}; |
MemoryOpt::MemoryOpt() : recordPool(sizeof(MemoryOpt::Record), 6) |
{ |
for (int i = 0; i < DATA_FILE_COUNT; ++i) { |
loads[i] = NULL; |
stores[i] = NULL; |
} |
prevRecord = NULL; |
} |
void |
MemoryOpt::reset() |
{ |
for (unsigned int i = 0; i < DATA_FILE_COUNT; ++i) { |
Record *it, *next; |
for (it = loads[i]; it; it = next) { |
next = it->next; |
recordPool.release(it); |
} |
loads[i] = NULL; |
for (it = stores[i]; it; it = next) { |
next = it->next; |
recordPool.release(it); |
} |
stores[i] = NULL; |
} |
} |
bool |
MemoryOpt::combineLd(Record *rec, Instruction *ld) |
{ |
int32_t offRc = rec->offset; |
int32_t offLd = ld->getSrc(0)->reg.data.offset; |
int sizeRc = rec->size; |
int sizeLd = typeSizeof(ld->dType); |
int size = sizeRc + sizeLd; |
int d, j; |
if (!prog->getTarget()-> |
isAccessSupported(ld->getSrc(0)->reg.file, typeOfSize(size))) |
return false; |
// no unaligned loads |
if (((size == 0x8) && (MIN2(offLd, offRc) & 0x7)) || |
((size == 0xc) && (MIN2(offLd, offRc) & 0xf))) |
return false; |
assert(sizeRc + sizeLd <= 16 && offRc != offLd); |
for (j = 0; sizeRc; sizeRc -= rec->insn->getDef(j)->reg.size, ++j); |
if (offLd < offRc) { |
int sz; |
for (sz = 0, d = 0; sz < sizeLd; sz += ld->getDef(d)->reg.size, ++d); |
// d: nr of definitions in ld |
// j: nr of definitions in rec->insn, move: |
for (d = d + j - 1; j > 0; --j, --d) |
rec->insn->setDef(d, rec->insn->getDef(j - 1)); |
if (rec->insn->getSrc(0)->refCount() > 1) |
rec->insn->setSrc(0, cloneShallow(func, rec->insn->getSrc(0))); |
rec->offset = rec->insn->getSrc(0)->reg.data.offset = offLd; |
d = 0; |
} else { |
d = j; |
} |
// move definitions of @ld to @rec->insn |
for (j = 0; sizeLd; ++j, ++d) { |
sizeLd -= ld->getDef(j)->reg.size; |
rec->insn->setDef(d, ld->getDef(j)); |
} |
rec->size = size; |
rec->insn->getSrc(0)->reg.size = size; |
rec->insn->setType(typeOfSize(size)); |
delete_Instruction(prog, ld); |
return true; |
} |
bool |
MemoryOpt::combineSt(Record *rec, Instruction *st) |
{ |
int32_t offRc = rec->offset; |
int32_t offSt = st->getSrc(0)->reg.data.offset; |
int sizeRc = rec->size; |
int sizeSt = typeSizeof(st->dType); |
int s = sizeSt / 4; |
int size = sizeRc + sizeSt; |
int j, k; |
Value *src[4]; // no modifiers in ValueRef allowed for st |
Value *extra[3]; |
if (!prog->getTarget()-> |
isAccessSupported(st->getSrc(0)->reg.file, typeOfSize(size))) |
return false; |
if (size == 8 && MIN2(offRc, offSt) & 0x7) |
return false; |
st->takeExtraSources(0, extra); // save predicate and indirect address |
if (offRc < offSt) { |
// save values from @st |
for (s = 0; sizeSt; ++s) { |
sizeSt -= st->getSrc(s + 1)->reg.size; |
src[s] = st->getSrc(s + 1); |
} |
// set record's values as low sources of @st |
for (j = 1; sizeRc; ++j) { |
sizeRc -= rec->insn->getSrc(j)->reg.size; |
st->setSrc(j, rec->insn->getSrc(j)); |
} |
// set saved values as high sources of @st |
for (k = j, j = 0; j < s; ++j) |
st->setSrc(k++, src[j]); |
updateLdStOffset(st, offRc, func); |
} else { |
for (j = 1; sizeSt; ++j) |
sizeSt -= st->getSrc(j)->reg.size; |
for (s = 1; sizeRc; ++j, ++s) { |
sizeRc -= rec->insn->getSrc(s)->reg.size; |
st->setSrc(j, rec->insn->getSrc(s)); |
} |
rec->offset = offSt; |
} |
st->putExtraSources(0, extra); // restore pointer and predicate |
delete_Instruction(prog, rec->insn); |
rec->insn = st; |
rec->size = size; |
rec->insn->getSrc(0)->reg.size = size; |
rec->insn->setType(typeOfSize(size)); |
return true; |
} |
void |
MemoryOpt::Record::set(const Instruction *ldst) |
{ |
const Symbol *mem = ldst->getSrc(0)->asSym(); |
fileIndex = mem->reg.fileIndex; |
rel[0] = ldst->getIndirect(0, 0); |
rel[1] = ldst->getIndirect(0, 1); |
offset = mem->reg.data.offset; |
base = mem->getBase(); |
size = typeSizeof(ldst->sType); |
} |
void |
MemoryOpt::Record::link(Record **list) |
{ |
next = *list; |
if (next) |
next->prev = this; |
prev = NULL; |
*list = this; |
} |
void |
MemoryOpt::Record::unlink(Record **list) |
{ |
if (next) |
next->prev = prev; |
if (prev) |
prev->next = next; |
else |
*list = next; |
} |
MemoryOpt::Record ** |
MemoryOpt::getList(const Instruction *insn) |
{ |
if (insn->op == OP_LOAD || insn->op == OP_VFETCH) |
return &loads[insn->src(0).getFile()]; |
return &stores[insn->src(0).getFile()]; |
} |
void |
MemoryOpt::addRecord(Instruction *i) |
{ |
Record **list = getList(i); |
Record *it = reinterpret_cast<Record *>(recordPool.allocate()); |
it->link(list); |
it->set(i); |
it->insn = i; |
it->locked = false; |
} |
MemoryOpt::Record * |
MemoryOpt::findRecord(const Instruction *insn, bool load, bool& isAdj) const |
{ |
const Symbol *sym = insn->getSrc(0)->asSym(); |
const int size = typeSizeof(insn->sType); |
Record *rec = NULL; |
Record *it = load ? loads[sym->reg.file] : stores[sym->reg.file]; |
for (; it; it = it->next) { |
if (it->locked && insn->op != OP_LOAD) |
continue; |
if ((it->offset >> 4) != (sym->reg.data.offset >> 4) || |
it->rel[0] != insn->getIndirect(0, 0) || |
it->fileIndex != sym->reg.fileIndex || |
it->rel[1] != insn->getIndirect(0, 1)) |
continue; |
if (it->offset < sym->reg.data.offset) { |
if (it->offset + it->size >= sym->reg.data.offset) { |
isAdj = (it->offset + it->size == sym->reg.data.offset); |
if (!isAdj) |
return it; |
if (!(it->offset & 0x7)) |
rec = it; |
} |
} else { |
isAdj = it->offset != sym->reg.data.offset; |
if (size <= it->size && !isAdj) |
return it; |
else |
if (!(sym->reg.data.offset & 0x7)) |
if (it->offset - size <= sym->reg.data.offset) |
rec = it; |
} |
} |
return rec; |
} |
bool |
MemoryOpt::replaceLdFromSt(Instruction *ld, Record *rec) |
{ |
Instruction *st = rec->insn; |
int32_t offSt = rec->offset; |
int32_t offLd = ld->getSrc(0)->reg.data.offset; |
int d, s; |
for (s = 1; offSt != offLd && st->srcExists(s); ++s) |
offSt += st->getSrc(s)->reg.size; |
if (offSt != offLd) |
return false; |
for (d = 0; ld->defExists(d) && st->srcExists(s); ++d, ++s) { |
if (ld->getDef(d)->reg.size != st->getSrc(s)->reg.size) |
return false; |
if (st->getSrc(s)->reg.file != FILE_GPR) |
return false; |
ld->def(d).replace(st->src(s), false); |
} |
ld->bb->remove(ld); |
return true; |
} |
bool |
MemoryOpt::replaceLdFromLd(Instruction *ldE, Record *rec) |
{ |
Instruction *ldR = rec->insn; |
int32_t offR = rec->offset; |
int32_t offE = ldE->getSrc(0)->reg.data.offset; |
int dR, dE; |
assert(offR <= offE); |
for (dR = 0; offR < offE && ldR->defExists(dR); ++dR) |
offR += ldR->getDef(dR)->reg.size; |
if (offR != offE) |
return false; |
for (dE = 0; ldE->defExists(dE) && ldR->defExists(dR); ++dE, ++dR) { |
if (ldE->getDef(dE)->reg.size != ldR->getDef(dR)->reg.size) |
return false; |
ldE->def(dE).replace(ldR->getDef(dR), false); |
} |
delete_Instruction(prog, ldE); |
return true; |
} |
bool |
MemoryOpt::replaceStFromSt(Instruction *restrict st, Record *rec) |
{ |
const Instruction *const ri = rec->insn; |
Value *extra[3]; |
int32_t offS = st->getSrc(0)->reg.data.offset; |
int32_t offR = rec->offset; |
int32_t endS = offS + typeSizeof(st->dType); |
int32_t endR = offR + typeSizeof(ri->dType); |
rec->size = MAX2(endS, endR) - MIN2(offS, offR); |
st->takeExtraSources(0, extra); |
if (offR < offS) { |
Value *vals[10]; |
int s, n; |
int k = 0; |
// get non-replaced sources of ri |
for (s = 1; offR < offS; offR += ri->getSrc(s)->reg.size, ++s) |
vals[k++] = ri->getSrc(s); |
n = s; |
// get replaced sources of st |
for (s = 1; st->srcExists(s); offS += st->getSrc(s)->reg.size, ++s) |
vals[k++] = st->getSrc(s); |
// skip replaced sources of ri |
for (s = n; offR < endS; offR += ri->getSrc(s)->reg.size, ++s); |
// get non-replaced sources after values covered by st |
for (; offR < endR; offR += ri->getSrc(s)->reg.size, ++s) |
vals[k++] = ri->getSrc(s); |
assert((unsigned int)k <= Elements(vals)); |
for (s = 0; s < k; ++s) |
st->setSrc(s + 1, vals[s]); |
st->setSrc(0, ri->getSrc(0)); |
} else |
if (endR > endS) { |
int j, s; |
for (j = 1; offR < endS; offR += ri->getSrc(j++)->reg.size); |
for (s = 1; offS < endS; offS += st->getSrc(s++)->reg.size); |
for (; offR < endR; offR += ri->getSrc(j++)->reg.size) |
st->setSrc(s++, ri->getSrc(j)); |
} |
st->putExtraSources(0, extra); |
delete_Instruction(prog, rec->insn); |
rec->insn = st; |
rec->offset = st->getSrc(0)->reg.data.offset; |
st->setType(typeOfSize(rec->size)); |
return true; |
} |
bool |
MemoryOpt::Record::overlaps(const Instruction *ldst) const |
{ |
Record that; |
that.set(ldst); |
if (this->fileIndex != that.fileIndex) |
return false; |
if (this->rel[0] || that.rel[0]) |
return this->base == that.base; |
return |
(this->offset < that.offset + that.size) && |
(this->offset + this->size > that.offset); |
} |
// We must not eliminate stores that affect the result of @ld if |
// we find later stores to the same location, and we may no longer |
// merge them with later stores. |
// The stored value can, however, still be used to determine the value |
// returned by future loads. |
void |
MemoryOpt::lockStores(Instruction *const ld) |
{ |
for (Record *r = stores[ld->src(0).getFile()]; r; r = r->next) |
if (!r->locked && r->overlaps(ld)) |
r->locked = true; |
} |
// Prior loads from the location of @st are no longer valid. |
// Stores to the location of @st may no longer be used to derive |
// the value at it nor be coalesced into later stores. |
void |
MemoryOpt::purgeRecords(Instruction *const st, DataFile f) |
{ |
if (st) |
f = st->src(0).getFile(); |
for (Record *r = loads[f]; r; r = r->next) |
if (!st || r->overlaps(st)) |
r->unlink(&loads[f]); |
for (Record *r = stores[f]; r; r = r->next) |
if (!st || r->overlaps(st)) |
r->unlink(&stores[f]); |
} |
bool |
MemoryOpt::visit(BasicBlock *bb) |
{ |
bool ret = runOpt(bb); |
// Run again, one pass won't combine 4 32 bit ld/st to a single 128 bit ld/st |
// where 96 bit memory operations are forbidden. |
if (ret) |
ret = runOpt(bb); |
return ret; |
} |
bool |
MemoryOpt::runOpt(BasicBlock *bb) |
{ |
Instruction *ldst, *next; |
Record *rec; |
bool isAdjacent = true; |
for (ldst = bb->getEntry(); ldst; ldst = next) { |
bool keep = true; |
bool isLoad = true; |
next = ldst->next; |
if (ldst->op == OP_LOAD || ldst->op == OP_VFETCH) { |
if (ldst->isDead()) { |
// might have been produced by earlier optimization |
delete_Instruction(prog, ldst); |
continue; |
} |
} else |
if (ldst->op == OP_STORE || ldst->op == OP_EXPORT) { |
isLoad = false; |
} else { |
// TODO: maybe have all fixed ops act as barrier ? |
if (ldst->op == OP_CALL || |
ldst->op == OP_BAR || |
ldst->op == OP_MEMBAR) { |
purgeRecords(NULL, FILE_MEMORY_LOCAL); |
purgeRecords(NULL, FILE_MEMORY_GLOBAL); |
purgeRecords(NULL, FILE_MEMORY_SHARED); |
purgeRecords(NULL, FILE_SHADER_OUTPUT); |
} else |
if (ldst->op == OP_ATOM || ldst->op == OP_CCTL) { |
if (ldst->src(0).getFile() == FILE_MEMORY_GLOBAL) { |
purgeRecords(NULL, FILE_MEMORY_LOCAL); |
purgeRecords(NULL, FILE_MEMORY_GLOBAL); |
purgeRecords(NULL, FILE_MEMORY_SHARED); |
} else { |
purgeRecords(NULL, ldst->src(0).getFile()); |
} |
} else |
if (ldst->op == OP_EMIT || ldst->op == OP_RESTART) { |
purgeRecords(NULL, FILE_SHADER_OUTPUT); |
} |
continue; |
} |
if (ldst->getPredicate()) // TODO: handle predicated ld/st |
continue; |
if (isLoad) { |
DataFile file = ldst->src(0).getFile(); |
// if ld l[]/g[] look for previous store to eliminate the reload |
if (file == FILE_MEMORY_GLOBAL || file == FILE_MEMORY_LOCAL) { |
// TODO: shared memory ? |
rec = findRecord(ldst, false, isAdjacent); |
if (rec && !isAdjacent) |
keep = !replaceLdFromSt(ldst, rec); |
} |
// or look for ld from the same location and replace this one |
rec = keep ? findRecord(ldst, true, isAdjacent) : NULL; |
if (rec) { |
if (!isAdjacent) |
keep = !replaceLdFromLd(ldst, rec); |
else |
// or combine a previous load with this one |
keep = !combineLd(rec, ldst); |
} |
if (keep) |
lockStores(ldst); |
} else { |
rec = findRecord(ldst, false, isAdjacent); |
if (rec) { |
if (!isAdjacent) |
keep = !replaceStFromSt(ldst, rec); |
else |
keep = !combineSt(rec, ldst); |
} |
if (keep) |
purgeRecords(ldst, DATA_FILE_COUNT); |
} |
if (keep) |
addRecord(ldst); |
} |
reset(); |
return true; |
} |
// ============================================================================= |
// Turn control flow into predicated instructions (after register allocation !). |
// TODO: |
// Could move this to before register allocation on NVC0 and also handle nested |
// constructs. |
class FlatteningPass : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
bool tryPredicateConditional(BasicBlock *); |
void predicateInstructions(BasicBlock *, Value *pred, CondCode cc); |
void tryPropagateBranch(BasicBlock *); |
inline bool isConstantCondition(Value *pred); |
inline bool mayPredicate(const Instruction *, const Value *pred) const; |
inline void removeFlow(Instruction *); |
}; |
bool |
FlatteningPass::isConstantCondition(Value *pred) |
{ |
Instruction *insn = pred->getUniqueInsn(); |
assert(insn); |
if (insn->op != OP_SET || insn->srcExists(2)) |
return false; |
for (int s = 0; s < 2 && insn->srcExists(s); ++s) { |
Instruction *ld = insn->getSrc(s)->getUniqueInsn(); |
DataFile file; |
if (ld) { |
if (ld->op != OP_MOV && ld->op != OP_LOAD) |
return false; |
if (ld->src(0).isIndirect(0)) |
return false; |
file = ld->src(0).getFile(); |
} else { |
file = insn->src(s).getFile(); |
// catch $r63 on NVC0 |
if (file == FILE_GPR && insn->getSrc(s)->reg.data.id > prog->maxGPR) |
file = FILE_IMMEDIATE; |
} |
if (file != FILE_IMMEDIATE && file != FILE_MEMORY_CONST) |
return false; |
} |
return true; |
} |
void |
FlatteningPass::removeFlow(Instruction *insn) |
{ |
FlowInstruction *term = insn ? insn->asFlow() : NULL; |
if (!term) |
return; |
Graph::Edge::Type ty = term->bb->cfg.outgoing().getType(); |
if (term->op == OP_BRA) { |
// TODO: this might get more difficult when we get arbitrary BRAs |
if (ty == Graph::Edge::CROSS || ty == Graph::Edge::BACK) |
return; |
} else |
if (term->op != OP_JOIN) |
return; |
Value *pred = term->getPredicate(); |
delete_Instruction(prog, term); |
if (pred && pred->refCount() == 0) { |
Instruction *pSet = pred->getUniqueInsn(); |
pred->join->reg.data.id = -1; // deallocate |
if (pSet->isDead()) |
delete_Instruction(prog, pSet); |
} |
} |
void |
FlatteningPass::predicateInstructions(BasicBlock *bb, Value *pred, CondCode cc) |
{ |
for (Instruction *i = bb->getEntry(); i; i = i->next) { |
if (i->isNop()) |
continue; |
assert(!i->getPredicate()); |
i->setPredicate(cc, pred); |
} |
removeFlow(bb->getExit()); |
} |
bool |
FlatteningPass::mayPredicate(const Instruction *insn, const Value *pred) const |
{ |
if (insn->isPseudo()) |
return true; |
// TODO: calls where we don't know which registers are modified |
if (!prog->getTarget()->mayPredicate(insn, pred)) |
return false; |
for (int d = 0; insn->defExists(d); ++d) |
if (insn->getDef(d)->equals(pred)) |
return false; |
return true; |
} |
// If we jump to BRA/RET/EXIT, replace the jump with it. |
// NOTE: We do not update the CFG anymore here ! |
// |
// TODO: Handle cases where we skip over a branch (maybe do that elsewhere ?): |
// BB:0 |
// @p0 bra BB:2 -> @!p0 bra BB:3 iff (!) BB:2 immediately adjoins BB:1 |
// BB1: |
// bra BB:3 |
// BB2: |
// ... |
// BB3: |
// ... |
void |
FlatteningPass::tryPropagateBranch(BasicBlock *bb) |
{ |
for (Instruction *i = bb->getExit(); i && i->op == OP_BRA; i = i->prev) { |
BasicBlock *bf = i->asFlow()->target.bb; |
if (bf->getInsnCount() != 1) |
continue; |
FlowInstruction *bra = i->asFlow(); |
FlowInstruction *rep = bf->getExit()->asFlow(); |
if (!rep || rep->getPredicate()) |
continue; |
if (rep->op != OP_BRA && |
rep->op != OP_JOIN && |
rep->op != OP_EXIT) |
continue; |
// TODO: If there are multiple branches to @rep, only the first would |
// be replaced, so only remove them after this pass is done ? |
// Also, need to check all incident blocks for fall-through exits and |
// add the branch there. |
bra->op = rep->op; |
bra->target.bb = rep->target.bb; |
if (bf->cfg.incidentCount() == 1) |
bf->remove(rep); |
} |
} |
bool |
FlatteningPass::visit(BasicBlock *bb) |
{ |
if (tryPredicateConditional(bb)) |
return true; |
// try to attach join to previous instruction |
Instruction *insn = bb->getExit(); |
if (insn && insn->op == OP_JOIN && !insn->getPredicate()) { |
insn = insn->prev; |
if (insn && !insn->getPredicate() && |
!insn->asFlow() && |
insn->op != OP_TEXBAR && |
!isTextureOp(insn->op) && // probably just nve4 |
!isSurfaceOp(insn->op) && // not confirmed |
insn->op != OP_LINTERP && // probably just nve4 |
insn->op != OP_PINTERP && // probably just nve4 |
((insn->op != OP_LOAD && insn->op != OP_STORE) || |
typeSizeof(insn->dType) <= 4) && |
!insn->isNop()) { |
insn->join = 1; |
bb->remove(bb->getExit()); |
return true; |
} |
} |
tryPropagateBranch(bb); |
return true; |
} |
bool |
FlatteningPass::tryPredicateConditional(BasicBlock *bb) |
{ |
BasicBlock *bL = NULL, *bR = NULL; |
unsigned int nL = 0, nR = 0, limit = 12; |
Instruction *insn; |
unsigned int mask; |
mask = bb->initiatesSimpleConditional(); |
if (!mask) |
return false; |
assert(bb->getExit()); |
Value *pred = bb->getExit()->getPredicate(); |
assert(pred); |
if (isConstantCondition(pred)) |
limit = 4; |
Graph::EdgeIterator ei = bb->cfg.outgoing(); |
if (mask & 1) { |
bL = BasicBlock::get(ei.getNode()); |
for (insn = bL->getEntry(); insn; insn = insn->next, ++nL) |
if (!mayPredicate(insn, pred)) |
return false; |
if (nL > limit) |
return false; // too long, do a real branch |
} |
ei.next(); |
if (mask & 2) { |
bR = BasicBlock::get(ei.getNode()); |
for (insn = bR->getEntry(); insn; insn = insn->next, ++nR) |
if (!mayPredicate(insn, pred)) |
return false; |
if (nR > limit) |
return false; // too long, do a real branch |
} |
if (bL) |
predicateInstructions(bL, pred, bb->getExit()->cc); |
if (bR) |
predicateInstructions(bR, pred, inverseCondCode(bb->getExit()->cc)); |
if (bb->joinAt) { |
bb->remove(bb->joinAt); |
bb->joinAt = NULL; |
} |
removeFlow(bb->getExit()); // delete the branch/join at the fork point |
// remove potential join operations at the end of the conditional |
if (prog->getTarget()->joinAnterior) { |
bb = BasicBlock::get((bL ? bL : bR)->cfg.outgoing().getNode()); |
if (bb->getEntry() && bb->getEntry()->op == OP_JOIN) |
removeFlow(bb->getEntry()); |
} |
return true; |
} |
// ============================================================================= |
// Common subexpression elimination. Stupid O^2 implementation. |
class LocalCSE : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
inline bool tryReplace(Instruction **, Instruction *); |
DLList ops[OP_LAST + 1]; |
}; |
class GlobalCSE : public Pass |
{ |
private: |
virtual bool visit(BasicBlock *); |
}; |
bool |
Instruction::isActionEqual(const Instruction *that) const |
{ |
if (this->op != that->op || |
this->dType != that->dType || |
this->sType != that->sType) |
return false; |
if (this->cc != that->cc) |
return false; |
if (this->asTex()) { |
if (memcmp(&this->asTex()->tex, |
&that->asTex()->tex, |
sizeof(this->asTex()->tex))) |
return false; |
} else |
if (this->asCmp()) { |
if (this->asCmp()->setCond != that->asCmp()->setCond) |
return false; |
} else |
if (this->asFlow()) { |
return false; |
} else { |
if (this->ipa != that->ipa || |
this->lanes != that->lanes || |
this->perPatch != that->perPatch) |
return false; |
if (this->postFactor != that->postFactor) |
return false; |
} |
if (this->subOp != that->subOp || |
this->saturate != that->saturate || |
this->rnd != that->rnd || |
this->ftz != that->ftz || |
this->dnz != that->dnz || |
this->cache != that->cache || |
this->mask != that->mask) |
return false; |
return true; |
} |
bool |
Instruction::isResultEqual(const Instruction *that) const |
{ |
unsigned int d, s; |
// NOTE: location of discard only affects tex with liveOnly and quadops |
if (!this->defExists(0) && this->op != OP_DISCARD) |
return false; |
if (!isActionEqual(that)) |
return false; |
if (this->predSrc != that->predSrc) |
return false; |
for (d = 0; this->defExists(d); ++d) { |
if (!that->defExists(d) || |
!this->getDef(d)->equals(that->getDef(d), false)) |
return false; |
} |
if (that->defExists(d)) |
return false; |
for (s = 0; this->srcExists(s); ++s) { |
if (!that->srcExists(s)) |
return false; |
if (this->src(s).mod != that->src(s).mod) |
return false; |
if (!this->getSrc(s)->equals(that->getSrc(s), true)) |
return false; |
} |
if (that->srcExists(s)) |
return false; |
if (op == OP_LOAD || op == OP_VFETCH) { |
switch (src(0).getFile()) { |
case FILE_MEMORY_CONST: |
case FILE_SHADER_INPUT: |
return true; |
default: |
return false; |
} |
} |
return true; |
} |
// pull through common expressions from different in-blocks |
bool |
GlobalCSE::visit(BasicBlock *bb) |
{ |
Instruction *phi, *next, *ik; |
int s; |
// TODO: maybe do this with OP_UNION, too |
for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = next) { |
next = phi->next; |
if (phi->getSrc(0)->refCount() > 1) |
continue; |
ik = phi->getSrc(0)->getInsn(); |
if (!ik) |
continue; // probably a function input |
for (s = 1; phi->srcExists(s); ++s) { |
if (phi->getSrc(s)->refCount() > 1) |
break; |
if (!phi->getSrc(s)->getInsn() || |
!phi->getSrc(s)->getInsn()->isResultEqual(ik)) |
break; |
} |
if (!phi->srcExists(s)) { |
Instruction *entry = bb->getEntry(); |
ik->bb->remove(ik); |
if (!entry || entry->op != OP_JOIN) |
bb->insertHead(ik); |
else |
bb->insertAfter(entry, ik); |
ik->setDef(0, phi->getDef(0)); |
delete_Instruction(prog, phi); |
} |
} |
return true; |
} |
bool |
LocalCSE::tryReplace(Instruction **ptr, Instruction *i) |
{ |
Instruction *old = *ptr; |
// TODO: maybe relax this later (causes trouble with OP_UNION) |
if (i->isPredicated()) |
return false; |
if (!old->isResultEqual(i)) |
return false; |
for (int d = 0; old->defExists(d); ++d) |
old->def(d).replace(i->getDef(d), false); |
delete_Instruction(prog, old); |
*ptr = NULL; |
return true; |
} |
bool |
LocalCSE::visit(BasicBlock *bb) |
{ |
unsigned int replaced; |
do { |
Instruction *ir, *next; |
replaced = 0; |
// will need to know the order of instructions |
int serial = 0; |
for (ir = bb->getFirst(); ir; ir = ir->next) |
ir->serial = serial++; |
for (ir = bb->getEntry(); ir; ir = next) { |
int s; |
Value *src = NULL; |
next = ir->next; |
if (ir->fixed) { |
ops[ir->op].insert(ir); |
continue; |
} |
for (s = 0; ir->srcExists(s); ++s) |
if (ir->getSrc(s)->asLValue()) |
if (!src || ir->getSrc(s)->refCount() < src->refCount()) |
src = ir->getSrc(s); |
if (src) { |
for (Value::UseIterator it = src->uses.begin(); |
it != src->uses.end(); ++it) { |
Instruction *ik = (*it)->getInsn(); |
if (ik && ik->bb == ir->bb && ik->serial < ir->serial) |
if (tryReplace(&ir, ik)) |
break; |
} |
} else { |
DLLIST_FOR_EACH(&ops[ir->op], iter) |
{ |
Instruction *ik = reinterpret_cast<Instruction *>(iter.get()); |
if (tryReplace(&ir, ik)) |
break; |
} |
} |
if (ir) |
ops[ir->op].insert(ir); |
else |
++replaced; |
} |
for (unsigned int i = 0; i <= OP_LAST; ++i) |
ops[i].clear(); |
} while (replaced); |
return true; |
} |
// ============================================================================= |
// Remove computations of unused values. |
class DeadCodeElim : public Pass |
{ |
public: |
bool buryAll(Program *); |
private: |
virtual bool visit(BasicBlock *); |
void checkSplitLoad(Instruction *ld); // for partially dead loads |
unsigned int deadCount; |
}; |
bool |
DeadCodeElim::buryAll(Program *prog) |
{ |
do { |
deadCount = 0; |
if (!this->run(prog, false, false)) |
return false; |
} while (deadCount); |
return true; |
} |
bool |
DeadCodeElim::visit(BasicBlock *bb) |
{ |
Instruction *next; |
for (Instruction *i = bb->getFirst(); i; i = next) { |
next = i->next; |
if (i->isDead()) { |
++deadCount; |
delete_Instruction(prog, i); |
} else |
if (i->defExists(1) && (i->op == OP_VFETCH || i->op == OP_LOAD)) { |
checkSplitLoad(i); |
} else |
if (i->defExists(0) && !i->getDef(0)->refCount()) { |
if (i->op == OP_ATOM || |
i->op == OP_SUREDP || |
i->op == OP_SUREDB) |
i->setDef(0, NULL); |
} |
} |
return true; |
} |
void |
DeadCodeElim::checkSplitLoad(Instruction *ld1) |
{ |
Instruction *ld2 = NULL; // can get at most 2 loads |
Value *def1[4]; |
Value *def2[4]; |
int32_t addr1, addr2; |
int32_t size1, size2; |
int d, n1, n2; |
uint32_t mask = 0xffffffff; |
for (d = 0; ld1->defExists(d); ++d) |
if (!ld1->getDef(d)->refCount() && ld1->getDef(d)->reg.data.id < 0) |
mask &= ~(1 << d); |
if (mask == 0xffffffff) |
return; |
addr1 = ld1->getSrc(0)->reg.data.offset; |
n1 = n2 = 0; |
size1 = size2 = 0; |
for (d = 0; ld1->defExists(d); ++d) { |
if (mask & (1 << d)) { |
if (size1 && (addr1 & 0x7)) |
break; |
def1[n1] = ld1->getDef(d); |
size1 += def1[n1++]->reg.size; |
} else |
if (!n1) { |
addr1 += ld1->getDef(d)->reg.size; |
} else { |
break; |
} |
} |
for (addr2 = addr1 + size1; ld1->defExists(d); ++d) { |
if (mask & (1 << d)) { |
def2[n2] = ld1->getDef(d); |
size2 += def2[n2++]->reg.size; |
} else { |
assert(!n2); |
addr2 += ld1->getDef(d)->reg.size; |
} |
} |
updateLdStOffset(ld1, addr1, func); |
ld1->setType(typeOfSize(size1)); |
for (d = 0; d < 4; ++d) |
ld1->setDef(d, (d < n1) ? def1[d] : NULL); |
if (!n2) |
return; |
ld2 = cloneShallow(func, ld1); |
updateLdStOffset(ld2, addr2, func); |
ld2->setType(typeOfSize(size2)); |
for (d = 0; d < 4; ++d) |
ld2->setDef(d, (d < n2) ? def2[d] : NULL); |
ld1->bb->insertAfter(ld1, ld2); |
} |
// ============================================================================= |
#define RUN_PASS(l, n, f) \ |
if (level >= (l)) { \ |
if (dbgFlags & NV50_IR_DEBUG_VERBOSE) \ |
INFO("PEEPHOLE: %s\n", #n); \ |
n pass; \ |
if (!pass.f(this)) \ |
return false; \ |
} |
bool |
Program::optimizeSSA(int level) |
{ |
RUN_PASS(1, DeadCodeElim, buryAll); |
RUN_PASS(1, CopyPropagation, run); |
RUN_PASS(2, GlobalCSE, run); |
RUN_PASS(1, LocalCSE, run); |
RUN_PASS(2, AlgebraicOpt, run); |
RUN_PASS(2, ModifierFolding, run); // before load propagation -> less checks |
RUN_PASS(1, ConstantFolding, foldAll); |
RUN_PASS(1, LoadPropagation, run); |
RUN_PASS(2, MemoryOpt, run); |
RUN_PASS(2, LocalCSE, run); |
RUN_PASS(0, DeadCodeElim, buryAll); |
return true; |
} |
bool |
Program::optimizePostRA(int level) |
{ |
RUN_PASS(2, FlatteningPass, run); |
return true; |
} |
} |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_print.cpp |
---|
0,0 → 1,698 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_target.h" |
#define __STDC_FORMAT_MACROS |
#include <inttypes.h> |
namespace nv50_ir { |
enum TextStyle |
{ |
TXT_DEFAULT, |
TXT_GPR, |
TXT_REGISTER, |
TXT_FLAGS, |
TXT_MEM, |
TXT_IMMD, |
TXT_BRA, |
TXT_INSN |
}; |
static const char *_colour[8] = |
{ |
"\x1b[00m", |
"\x1b[34m", |
"\x1b[35m", |
"\x1b[35m", |
"\x1b[36m", |
"\x1b[33m", |
"\x1b[37m", |
"\x1b[32m" |
}; |
static const char *_nocolour[8] = |
{ |
"", "", "", "", "", "", "", "" |
}; |
static const char **colour; |
static void init_colours() |
{ |
if (getenv("NV50_PROG_DEBUG_NO_COLORS") != NULL) |
colour = _nocolour; |
else |
colour = _colour; |
} |
const char *operationStr[OP_LAST + 1] = |
{ |
"nop", |
"phi", |
"union", |
"split", |
"merge", |
"consec", |
"mov", |
"ld", |
"st", |
"add", |
"sub", |
"mul", |
"div", |
"mod", |
"mad", |
"fma", |
"sad", |
"abs", |
"neg", |
"not", |
"and", |
"or", |
"xor", |
"shl", |
"shr", |
"max", |
"min", |
"sat", |
"ceil", |
"floor", |
"trunc", |
"cvt", |
"set and", |
"set or", |
"set xor", |
"set", |
"selp", |
"slct", |
"rcp", |
"rsq", |
"lg2", |
"sin", |
"cos", |
"ex2", |
"exp", |
"log", |
"presin", |
"preex2", |
"sqrt", |
"pow", |
"bra", |
"call", |
"ret", |
"cont", |
"break", |
"preret", |
"precont", |
"prebreak", |
"brkpt", |
"joinat", |
"join", |
"discard", |
"exit", |
"membar", |
"vfetch", |
"pfetch", |
"export", |
"linterp", |
"pinterp", |
"emit", |
"restart", |
"tex", |
"texbias", |
"texlod", |
"texfetch", |
"texquery", |
"texgrad", |
"texgather", |
"texcsaa", |
"texprep", |
"suldb", |
"suldp", |
"sustb", |
"sustp", |
"suredb", |
"suredp", |
"sulea", |
"subfm", |
"suclamp", |
"sueau", |
"madsp", |
"texbar", |
"dfdx", |
"dfdy", |
"rdsv", |
"wrsv", |
"quadop", |
"quadon", |
"quadpop", |
"popcnt", |
"insbf", |
"extbf", |
"permt", |
"atom", |
"bar", |
"vadd", |
"vavg", |
"vmin", |
"vmax", |
"vsad", |
"vset", |
"vshr", |
"vshl", |
"vsel", |
"cctl", |
"(invalid)" |
}; |
static const char *atomSubOpStr[] = |
{ |
"add", "min", "max", "inc", "dec", "and", "or", "xor", "cas", "exch" |
}; |
static const char *DataTypeStr[] = |
{ |
"-", |
"u8", "s8", |
"u16", "s16", |
"u32", "s32", |
"u64", "s64", |
"f16", "f32", "f64", |
"b96", "b128" |
}; |
static const char *RoundModeStr[] = |
{ |
"", "rm", "rz", "rp", "rni", "rmi", "rzi", "rpi" |
}; |
static const char *CondCodeStr[] = |
{ |
"never", |
"lt", |
"eq", |
"le", |
"gt", |
"ne", |
"ge", |
"", |
"(invalid)", |
"ltu", |
"equ", |
"leu", |
"gtu", |
"neu", |
"geu", |
"", |
"no", |
"nc", |
"ns", |
"na", |
"a", |
"s", |
"c", |
"o" |
}; |
static const char *SemanticStr[SV_LAST + 1] = |
{ |
"POSITION", |
"VERTEX_ID", |
"INSTANCE_ID", |
"INVOCATION_ID", |
"PRIMITIVE_ID", |
"VERTEX_COUNT", |
"LAYER", |
"VIEWPORT_INDEX", |
"Y_DIR", |
"FACE", |
"POINT_SIZE", |
"POINT_COORD", |
"CLIP_DISTANCE", |
"SAMPLE_INDEX", |
"TESS_FACTOR", |
"TESS_COORD", |
"TID", |
"CTAID", |
"NTID", |
"GRIDID", |
"NCTAID", |
"LANEID", |
"PHYSID", |
"NPHYSID", |
"CLOCK", |
"LBASE", |
"SBASE", |
"?", |
"(INVALID)" |
}; |
static const char *interpStr[16] = |
{ |
"pass", |
"mul", |
"flat", |
"sc", |
"cent pass", |
"cent mul", |
"cent flat", |
"cent sc", |
"off pass", |
"off mul", |
"off flat", |
"off sc", |
"samp pass", |
"samp mul", |
"samp flat", |
"samp sc" |
}; |
#define PRINT(args...) \ |
do { \ |
pos += snprintf(&buf[pos], size - pos, args); \ |
} while(0) |
#define SPACE_PRINT(cond, args...) \ |
do { \ |
if (cond) \ |
buf[pos++] = ' '; \ |
pos += snprintf(&buf[pos], size - pos, args); \ |
} while(0) |
#define SPACE() \ |
do { \ |
if (pos < size) \ |
buf[pos++] = ' '; \ |
} while(0) |
int Modifier::print(char *buf, size_t size) const |
{ |
size_t pos = 0; |
if (bits) |
PRINT("%s", colour[TXT_INSN]); |
size_t base = pos; |
if (bits & NV50_IR_MOD_NOT) |
PRINT("not"); |
if (bits & NV50_IR_MOD_SAT) |
SPACE_PRINT(pos > base && pos < size, "sat"); |
if (bits & NV50_IR_MOD_NEG) |
SPACE_PRINT(pos > base && pos < size, "neg"); |
if (bits & NV50_IR_MOD_ABS) |
SPACE_PRINT(pos > base && pos < size, "abs"); |
return pos; |
} |
int LValue::print(char *buf, size_t size, DataType ty) const |
{ |
const char *postFix = ""; |
size_t pos = 0; |
int idx = join->reg.data.id >= 0 ? join->reg.data.id : id; |
char p = join->reg.data.id >= 0 ? '$' : '%'; |
char r; |
int col = TXT_DEFAULT; |
switch (reg.file) { |
case FILE_GPR: |
r = 'r'; col = TXT_GPR; |
if (reg.size == 2) { |
if (p == '$') { |
postFix = (idx & 1) ? "h" : "l"; |
idx /= 2; |
} else { |
postFix = "s"; |
} |
} else |
if (reg.size == 8) { |
postFix = "d"; |
} else |
if (reg.size == 16) { |
postFix = "q"; |
} else |
if (reg.size == 12) { |
postFix = "t"; |
} |
break; |
case FILE_PREDICATE: |
r = 'p'; col = TXT_REGISTER; |
if (reg.size == 2) |
postFix = "d"; |
else |
if (reg.size == 4) |
postFix = "q"; |
break; |
case FILE_FLAGS: |
r = 'c'; col = TXT_FLAGS; |
break; |
case FILE_ADDRESS: |
r = 'a'; col = TXT_REGISTER; |
break; |
default: |
assert(!"invalid file for lvalue"); |
r = '?'; |
break; |
} |
PRINT("%s%c%c%i%s", colour[col], p, r, idx, postFix); |
return pos; |
} |
int ImmediateValue::print(char *buf, size_t size, DataType ty) const |
{ |
size_t pos = 0; |
PRINT("%s", colour[TXT_IMMD]); |
switch (ty) { |
case TYPE_F32: PRINT("%f", reg.data.f32); break; |
case TYPE_F64: PRINT("%f", reg.data.f64); break; |
case TYPE_U8: PRINT("0x%02x", reg.data.u8); break; |
case TYPE_S8: PRINT("%i", reg.data.s8); break; |
case TYPE_U16: PRINT("0x%04x", reg.data.u16); break; |
case TYPE_S16: PRINT("%i", reg.data.s16); break; |
case TYPE_U32: PRINT("0x%08x", reg.data.u32); break; |
case TYPE_S32: PRINT("%i", reg.data.s32); break; |
case TYPE_U64: |
case TYPE_S64: |
default: |
PRINT("0x%016"PRIx64, reg.data.u64); |
break; |
} |
return pos; |
} |
int Symbol::print(char *buf, size_t size, DataType ty) const |
{ |
return print(buf, size, NULL, NULL, ty); |
} |
int Symbol::print(char *buf, size_t size, |
Value *rel, Value *dimRel, DataType ty) const |
{ |
size_t pos = 0; |
char c; |
if (ty == TYPE_NONE) |
ty = typeOfSize(reg.size); |
if (reg.file == FILE_SYSTEM_VALUE) { |
PRINT("%ssv[%s%s:%i%s", colour[TXT_MEM], |
colour[TXT_REGISTER], |
SemanticStr[reg.data.sv.sv], reg.data.sv.index, colour[TXT_MEM]); |
if (rel) { |
PRINT("%s+", colour[TXT_DEFAULT]); |
pos += rel->print(&buf[pos], size - pos); |
} |
PRINT("%s]", colour[TXT_MEM]); |
return pos; |
} |
switch (reg.file) { |
case FILE_MEMORY_CONST: c = 'c'; break; |
case FILE_SHADER_INPUT: c = 'a'; break; |
case FILE_SHADER_OUTPUT: c = 'o'; break; |
case FILE_MEMORY_GLOBAL: c = 'g'; break; |
case FILE_MEMORY_SHARED: c = 's'; break; |
case FILE_MEMORY_LOCAL: c = 'l'; break; |
default: |
assert(!"invalid file"); |
c = '?'; |
break; |
} |
if (c == 'c') |
PRINT("%s%c%i[", colour[TXT_MEM], c, reg.fileIndex); |
else |
PRINT("%s%c[", colour[TXT_MEM], c); |
if (dimRel) { |
pos += dimRel->print(&buf[pos], size - pos, TYPE_S32); |
PRINT("%s][", colour[TXT_MEM]); |
} |
if (rel) { |
pos += rel->print(&buf[pos], size - pos); |
PRINT("%s%c", colour[TXT_DEFAULT], (reg.data.offset < 0) ? '-' : '+'); |
} else { |
assert(reg.data.offset >= 0); |
} |
PRINT("%s0x%x%s]", colour[TXT_IMMD], abs(reg.data.offset), colour[TXT_MEM]); |
return pos; |
} |
void Instruction::print() const |
{ |
#define BUFSZ 512 |
const size_t size = BUFSZ; |
char buf[BUFSZ]; |
int s, d; |
size_t pos = 0; |
PRINT("%s", colour[TXT_INSN]); |
if (join) |
PRINT("join "); |
if (predSrc >= 0) { |
const size_t pre = pos; |
if (getSrc(predSrc)->reg.file == FILE_PREDICATE) { |
if (cc == CC_NOT_P) |
PRINT("not"); |
} else { |
PRINT("%s", CondCodeStr[cc]); |
} |
if (pos > pre) |
SPACE(); |
pos += getSrc(predSrc)->print(&buf[pos], BUFSZ - pos); |
PRINT(" %s", colour[TXT_INSN]); |
} |
if (saturate) |
PRINT("sat "); |
if (asFlow()) { |
PRINT("%s", operationStr[op]); |
if (asFlow()->indirect) |
PRINT(" ind"); |
if (asFlow()->absolute) |
PRINT(" abs"); |
if (op == OP_CALL && asFlow()->builtin) { |
PRINT(" %sBUILTIN:%i", colour[TXT_BRA], asFlow()->target.builtin); |
} else |
if (op == OP_CALL && asFlow()->target.fn) { |
PRINT(" %s%s:%i", colour[TXT_BRA], |
asFlow()->target.fn->getName(), |
asFlow()->target.fn->getLabel()); |
} else |
if (asFlow()->target.bb) |
PRINT(" %sBB:%i", colour[TXT_BRA], asFlow()->target.bb->getId()); |
} else { |
PRINT("%s ", operationStr[op]); |
if (op == OP_LINTERP || op == OP_PINTERP) |
PRINT("%s ", interpStr[ipa]); |
switch (op) { |
case OP_SUREDP: |
case OP_ATOM: |
if (subOp < Elements(atomSubOpStr)) |
PRINT("%s ", atomSubOpStr[subOp]); |
break; |
default: |
if (subOp) |
PRINT("(SUBOP:%u) ", subOp); |
break; |
} |
if (perPatch) |
PRINT("patch "); |
if (asTex()) |
PRINT("%s %s$r%u $s%u %s", asTex()->tex.target.getName(), |
colour[TXT_MEM], asTex()->tex.r, asTex()->tex.s, |
colour[TXT_INSN]); |
if (postFactor) |
PRINT("x2^%i ", postFactor); |
PRINT("%s%s", dnz ? "dnz " : (ftz ? "ftz " : ""), DataTypeStr[dType]); |
} |
if (rnd != ROUND_N) |
PRINT(" %s", RoundModeStr[rnd]); |
if (defExists(1)) |
PRINT(" {"); |
for (d = 0; defExists(d); ++d) { |
SPACE(); |
pos += getDef(d)->print(&buf[pos], size - pos); |
} |
if (d > 1) |
PRINT(" %s}", colour[TXT_INSN]); |
else |
if (!d && !asFlow()) |
PRINT(" %s#", colour[TXT_INSN]); |
if (asCmp()) |
PRINT(" %s%s", colour[TXT_INSN], CondCodeStr[asCmp()->setCond]); |
if (sType != dType) |
PRINT(" %s%s", colour[TXT_INSN], DataTypeStr[sType]); |
for (s = 0; srcExists(s); ++s) { |
if (s == predSrc || src(s).usedAsPtr) |
continue; |
const size_t pre = pos; |
SPACE(); |
pos += src(s).mod.print(&buf[pos], BUFSZ - pos); |
if (pos > pre + 1) |
SPACE(); |
if (src(s).isIndirect(0) || src(s).isIndirect(1)) |
pos += getSrc(s)->asSym()->print(&buf[pos], BUFSZ - pos, |
getIndirect(s, 0), |
getIndirect(s, 1)); |
else |
pos += getSrc(s)->print(&buf[pos], BUFSZ - pos, sType); |
} |
if (exit) |
PRINT("%s exit", colour[TXT_INSN]); |
PRINT("%s", colour[TXT_DEFAULT]); |
buf[MIN2(pos, BUFSZ - 1)] = 0; |
INFO("%s (%u)\n", buf, encSize); |
} |
class PrintPass : public Pass |
{ |
public: |
PrintPass() : serial(0) { } |
virtual bool visit(Function *); |
virtual bool visit(BasicBlock *); |
virtual bool visit(Instruction *); |
private: |
int serial; |
}; |
bool |
PrintPass::visit(Function *fn) |
{ |
char str[16]; |
INFO("\n%s:%i (", fn->getName(), fn->getLabel()); |
if (!fn->outs.empty()) |
INFO("out"); |
for (std::deque<ValueRef>::iterator it = fn->outs.begin(); |
it != fn->outs.end(); |
++it) { |
it->get()->print(str, sizeof(str), typeOfSize(it->get()->reg.size)); |
INFO(" %s", str); |
} |
if (!fn->ins.empty()) |
INFO("%s%sin", colour[TXT_DEFAULT], fn->outs.empty() ? "" : ", "); |
for (std::deque<ValueDef>::iterator it = fn->ins.begin(); |
it != fn->ins.end(); |
++it) { |
it->get()->print(str, sizeof(str), typeOfSize(it->get()->reg.size)); |
INFO(" %s", str); |
} |
INFO("%s)\n", colour[TXT_DEFAULT]); |
return true; |
} |
bool |
PrintPass::visit(BasicBlock *bb) |
{ |
#if 0 |
INFO("---\n"); |
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) |
INFO(" <- BB:%i (%s)\n", |
BasicBlock::get(ei.getNode())->getId(), |
ei.getEdge()->typeStr()); |
#endif |
INFO("BB:%i (%u instructions) - ", bb->getId(), bb->getInsnCount()); |
if (bb->idom()) |
INFO("idom = BB:%i, ", bb->idom()->getId()); |
INFO("df = { "); |
for (DLList::Iterator df = bb->getDF().iterator(); !df.end(); df.next()) |
INFO("BB:%i ", BasicBlock::get(df)->getId()); |
INFO("}\n"); |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) |
INFO(" -> BB:%i (%s)\n", |
BasicBlock::get(ei.getNode())->getId(), |
ei.getEdge()->typeStr()); |
return true; |
} |
bool |
PrintPass::visit(Instruction *insn) |
{ |
INFO("%3i: ", serial++); |
insn->print(); |
return true; |
} |
void |
Function::print() |
{ |
PrintPass pass; |
pass.run(this, true, false); |
} |
void |
Program::print() |
{ |
PrintPass pass; |
init_colours(); |
pass.run(this, true, false); |
} |
void |
Function::printLiveIntervals() const |
{ |
INFO("printing live intervals ...\n"); |
for (ArrayList::Iterator it = allLValues.iterator(); !it.end(); it.next()) { |
const Value *lval = Value::get(it)->asLValue(); |
if (lval && !lval->livei.isEmpty()) { |
INFO("livei(%%%i): ", lval->id); |
lval->livei.print(); |
} |
} |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_ra.cpp |
---|
0,0 → 1,2050 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_target.h" |
#include <stack> |
#include <limits> |
namespace nv50_ir { |
#define MAX_REGISTER_FILE_SIZE 256 |
class RegisterSet |
{ |
public: |
RegisterSet(const Target *); |
void init(const Target *); |
void reset(DataFile, bool resetMax = false); |
void periodicMask(DataFile f, uint32_t lock, uint32_t unlock); |
void intersect(DataFile f, const RegisterSet *); |
bool assign(int32_t& reg, DataFile f, unsigned int size); |
void release(DataFile f, int32_t reg, unsigned int size); |
void occupy(DataFile f, int32_t reg, unsigned int size); |
void occupy(const Value *); |
void occupyMask(DataFile f, int32_t reg, uint8_t mask); |
bool isOccupied(DataFile f, int32_t reg, unsigned int size) const; |
bool testOccupy(const Value *); |
bool testOccupy(DataFile f, int32_t reg, unsigned int size); |
inline int getMaxAssigned(DataFile f) const { return fill[f]; } |
inline unsigned int getFileSize(DataFile f, uint8_t regSize) const |
{ |
if (restrictedGPR16Range && f == FILE_GPR && regSize == 2) |
return (last[f] + 1) / 2; |
return last[f] + 1; |
} |
inline unsigned int units(DataFile f, unsigned int size) const |
{ |
return size >> unit[f]; |
} |
// for regs of size >= 4, id is counted in 4-byte words (like nv50/c0 binary) |
inline unsigned int idToBytes(const Value *v) const |
{ |
return v->reg.data.id * MIN2(v->reg.size, 4); |
} |
inline unsigned int idToUnits(const Value *v) const |
{ |
return units(v->reg.file, idToBytes(v)); |
} |
inline int bytesToId(Value *v, unsigned int bytes) const |
{ |
if (v->reg.size < 4) |
return units(v->reg.file, bytes); |
return bytes / 4; |
} |
inline int unitsToId(DataFile f, int u, uint8_t size) const |
{ |
if (u < 0) |
return -1; |
return (size < 4) ? u : ((u << unit[f]) / 4); |
} |
void print() const; |
private: |
BitSet bits[LAST_REGISTER_FILE + 1]; |
int unit[LAST_REGISTER_FILE + 1]; // log2 of allocation granularity |
int last[LAST_REGISTER_FILE + 1]; |
int fill[LAST_REGISTER_FILE + 1]; |
const bool restrictedGPR16Range; |
}; |
void |
RegisterSet::reset(DataFile f, bool resetMax) |
{ |
bits[f].fill(0); |
if (resetMax) |
fill[f] = -1; |
} |
void |
RegisterSet::init(const Target *targ) |
{ |
for (unsigned int rf = 0; rf <= FILE_ADDRESS; ++rf) { |
DataFile f = static_cast<DataFile>(rf); |
last[rf] = targ->getFileSize(f) - 1; |
unit[rf] = targ->getFileUnit(f); |
fill[rf] = -1; |
assert(last[rf] < MAX_REGISTER_FILE_SIZE); |
bits[rf].allocate(last[rf] + 1, true); |
} |
} |
RegisterSet::RegisterSet(const Target *targ) |
: restrictedGPR16Range(targ->getChipset() < 0xc0) |
{ |
init(targ); |
for (unsigned int i = 0; i <= LAST_REGISTER_FILE; ++i) |
reset(static_cast<DataFile>(i)); |
} |
void |
RegisterSet::periodicMask(DataFile f, uint32_t lock, uint32_t unlock) |
{ |
bits[f].periodicMask32(lock, unlock); |
} |
void |
RegisterSet::intersect(DataFile f, const RegisterSet *set) |
{ |
bits[f] |= set->bits[f]; |
} |
void |
RegisterSet::print() const |
{ |
INFO("GPR:"); |
bits[FILE_GPR].print(); |
INFO("\n"); |
} |
bool |
RegisterSet::assign(int32_t& reg, DataFile f, unsigned int size) |
{ |
reg = bits[f].findFreeRange(size); |
if (reg < 0) |
return false; |
fill[f] = MAX2(fill[f], (int32_t)(reg + size - 1)); |
return true; |
} |
bool |
RegisterSet::isOccupied(DataFile f, int32_t reg, unsigned int size) const |
{ |
return bits[f].testRange(reg, size); |
} |
void |
RegisterSet::occupy(const Value *v) |
{ |
occupy(v->reg.file, idToUnits(v), v->reg.size >> unit[v->reg.file]); |
} |
void |
RegisterSet::occupyMask(DataFile f, int32_t reg, uint8_t mask) |
{ |
bits[f].setMask(reg & ~31, static_cast<uint32_t>(mask) << (reg % 32)); |
} |
void |
RegisterSet::occupy(DataFile f, int32_t reg, unsigned int size) |
{ |
bits[f].setRange(reg, size); |
INFO_DBG(0, REG_ALLOC, "reg occupy: %u[%i] %u\n", f, reg, size); |
fill[f] = MAX2(fill[f], (int32_t)(reg + size - 1)); |
} |
bool |
RegisterSet::testOccupy(const Value *v) |
{ |
return testOccupy(v->reg.file, |
idToUnits(v), v->reg.size >> unit[v->reg.file]); |
} |
bool |
RegisterSet::testOccupy(DataFile f, int32_t reg, unsigned int size) |
{ |
if (isOccupied(f, reg, size)) |
return false; |
occupy(f, reg, size); |
return true; |
} |
void |
RegisterSet::release(DataFile f, int32_t reg, unsigned int size) |
{ |
bits[f].clrRange(reg, size); |
INFO_DBG(0, REG_ALLOC, "reg release: %u[%i] %u\n", f, reg, size); |
} |
class RegAlloc |
{ |
public: |
RegAlloc(Program *program) : prog(program), sequence(0) { } |
bool exec(); |
bool execFunc(); |
private: |
class PhiMovesPass : public Pass { |
private: |
virtual bool visit(BasicBlock *); |
inline bool needNewElseBlock(BasicBlock *b, BasicBlock *p); |
}; |
class ArgumentMovesPass : public Pass { |
private: |
virtual bool visit(BasicBlock *); |
}; |
class BuildIntervalsPass : public Pass { |
private: |
virtual bool visit(BasicBlock *); |
void collectLiveValues(BasicBlock *); |
void addLiveRange(Value *, const BasicBlock *, int end); |
}; |
class InsertConstraintsPass : public Pass { |
public: |
bool exec(Function *func); |
private: |
virtual bool visit(BasicBlock *); |
bool insertConstraintMoves(); |
void condenseDefs(Instruction *); |
void condenseSrcs(Instruction *, const int first, const int last); |
void addHazard(Instruction *i, const ValueRef *src); |
void textureMask(TexInstruction *); |
void addConstraint(Instruction *, int s, int n); |
bool detectConflict(Instruction *, int s); |
// target specific functions, TODO: put in subclass or Target |
void texConstraintNV50(TexInstruction *); |
void texConstraintNVC0(TexInstruction *); |
void texConstraintNVE0(TexInstruction *); |
std::list<Instruction *> constrList; |
const Target *targ; |
}; |
bool buildLiveSets(BasicBlock *); |
private: |
Program *prog; |
Function *func; |
// instructions in control flow / chronological order |
ArrayList insns; |
int sequence; // for manual passes through CFG |
}; |
typedef std::pair<Value *, Value *> ValuePair; |
class SpillCodeInserter |
{ |
public: |
SpillCodeInserter(Function *fn) : func(fn), stackSize(0), stackBase(0) { } |
bool run(const std::list<ValuePair>&); |
Symbol *assignSlot(const Interval&, const unsigned int size); |
inline int32_t getStackSize() const { return stackSize; } |
private: |
Function *func; |
struct SpillSlot |
{ |
Interval occup; |
std::list<Value *> residents; // needed to recalculate occup |
Symbol *sym; |
int32_t offset; |
inline uint8_t size() const { return sym->reg.size; } |
}; |
std::list<SpillSlot> slots; |
int32_t stackSize; |
int32_t stackBase; |
LValue *unspill(Instruction *usei, LValue *, Value *slot); |
void spill(Instruction *defi, Value *slot, LValue *); |
}; |
void |
RegAlloc::BuildIntervalsPass::addLiveRange(Value *val, |
const BasicBlock *bb, |
int end) |
{ |
Instruction *insn = val->getUniqueInsn(); |
if (!insn) |
insn = bb->getFirst(); |
assert(bb->getFirst()->serial <= bb->getExit()->serial); |
assert(bb->getExit()->serial + 1 >= end); |
int begin = insn->serial; |
if (begin < bb->getEntry()->serial || begin > bb->getExit()->serial) |
begin = bb->getEntry()->serial; |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "%%%i <- live range [%i(%i), %i)\n", |
val->id, begin, insn->serial, end); |
if (begin != end) // empty ranges are only added as hazards for fixed regs |
val->livei.extend(begin, end); |
} |
bool |
RegAlloc::PhiMovesPass::needNewElseBlock(BasicBlock *b, BasicBlock *p) |
{ |
if (b->cfg.incidentCount() <= 1) |
return false; |
int n = 0; |
for (Graph::EdgeIterator ei = p->cfg.outgoing(); !ei.end(); ei.next()) |
if (ei.getType() == Graph::Edge::TREE || |
ei.getType() == Graph::Edge::FORWARD) |
++n; |
return (n == 2); |
} |
// For each operand of each PHI in b, generate a new value by inserting a MOV |
// at the end of the block it is coming from and replace the operand with its |
// result. This eliminates liveness conflicts and enables us to let values be |
// copied to the right register if such a conflict exists nonetheless. |
// |
// These MOVs are also crucial in making sure the live intervals of phi srces |
// are extended until the end of the loop, since they are not included in the |
// live-in sets. |
bool |
RegAlloc::PhiMovesPass::visit(BasicBlock *bb) |
{ |
Instruction *phi, *mov; |
BasicBlock *pb, *pn; |
std::stack<BasicBlock *> stack; |
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) { |
pb = BasicBlock::get(ei.getNode()); |
assert(pb); |
if (needNewElseBlock(bb, pb)) |
stack.push(pb); |
} |
while (!stack.empty()) { |
pb = stack.top(); |
pn = new BasicBlock(func); |
stack.pop(); |
pb->cfg.detach(&bb->cfg); |
pb->cfg.attach(&pn->cfg, Graph::Edge::TREE); |
pn->cfg.attach(&bb->cfg, Graph::Edge::FORWARD); |
assert(pb->getExit()->op != OP_CALL); |
if (pb->getExit()->asFlow()->target.bb == bb) |
pb->getExit()->asFlow()->target.bb = pn; |
} |
// insert MOVs (phi->src(j) should stem from j-th in-BB) |
int j = 0; |
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) { |
pb = BasicBlock::get(ei.getNode()); |
if (!pb->isTerminated()) |
pb->insertTail(new_FlowInstruction(func, OP_BRA, bb)); |
for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = phi->next) { |
mov = new_Instruction(func, OP_MOV, TYPE_U32); |
mov->setSrc(0, phi->getSrc(j)); |
mov->setDef(0, new_LValue(func, phi->getDef(0)->asLValue())); |
phi->setSrc(j, mov->getDef(0)); |
pb->insertBefore(pb->getExit(), mov); |
} |
++j; |
} |
return true; |
} |
bool |
RegAlloc::ArgumentMovesPass::visit(BasicBlock *bb) |
{ |
// Bind function call inputs/outputs to the same physical register |
// the callee uses, inserting moves as appropriate for the case a |
// conflict arises. |
for (Instruction *i = bb->getEntry(); i; i = i->next) { |
FlowInstruction *cal = i->asFlow(); |
// TODO: Handle indirect calls. |
// Right now they should only be generated for builtins. |
if (!cal || cal->op != OP_CALL || cal->builtin || cal->indirect) |
continue; |
RegisterSet clobberSet(prog->getTarget()); |
// Bind input values. |
for (int s = cal->indirect ? 1 : 0; cal->srcExists(s); ++s) { |
const int t = cal->indirect ? (s - 1) : s; |
LValue *tmp = new_LValue(func, cal->getSrc(s)->asLValue()); |
tmp->reg.data.id = cal->target.fn->ins[t].rep()->reg.data.id; |
Instruction *mov = |
new_Instruction(func, OP_MOV, typeOfSize(tmp->reg.size)); |
mov->setDef(0, tmp); |
mov->setSrc(0, cal->getSrc(s)); |
cal->setSrc(s, tmp); |
bb->insertBefore(cal, mov); |
} |
// Bind output values. |
for (int d = 0; cal->defExists(d); ++d) { |
LValue *tmp = new_LValue(func, cal->getDef(d)->asLValue()); |
tmp->reg.data.id = cal->target.fn->outs[d].rep()->reg.data.id; |
Instruction *mov = |
new_Instruction(func, OP_MOV, typeOfSize(tmp->reg.size)); |
mov->setSrc(0, tmp); |
mov->setDef(0, cal->getDef(d)); |
cal->setDef(d, tmp); |
bb->insertAfter(cal, mov); |
clobberSet.occupy(tmp); |
} |
// Bind clobbered values. |
for (std::deque<Value *>::iterator it = cal->target.fn->clobbers.begin(); |
it != cal->target.fn->clobbers.end(); |
++it) { |
if (clobberSet.testOccupy(*it)) { |
Value *tmp = new_LValue(func, (*it)->asLValue()); |
tmp->reg.data.id = (*it)->reg.data.id; |
cal->setDef(cal->defCount(), tmp); |
} |
} |
} |
// Update the clobber set of the function. |
if (BasicBlock::get(func->cfgExit) == bb) { |
func->buildDefSets(); |
for (unsigned int i = 0; i < bb->defSet.getSize(); ++i) |
if (bb->defSet.test(i)) |
func->clobbers.push_back(func->getLValue(i)); |
} |
return true; |
} |
// Build the set of live-in variables of bb. |
bool |
RegAlloc::buildLiveSets(BasicBlock *bb) |
{ |
Function *f = bb->getFunction(); |
BasicBlock *bn; |
Instruction *i; |
unsigned int s, d; |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "buildLiveSets(BB:%i)\n", bb->getId()); |
bb->liveSet.allocate(func->allLValues.getSize(), false); |
int n = 0; |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
bn = BasicBlock::get(ei.getNode()); |
if (bn == bb) |
continue; |
if (bn->cfg.visit(sequence)) |
if (!buildLiveSets(bn)) |
return false; |
if (n++ || bb->liveSet.marker) |
bb->liveSet |= bn->liveSet; |
else |
bb->liveSet = bn->liveSet; |
} |
if (!n && !bb->liveSet.marker) |
bb->liveSet.fill(0); |
bb->liveSet.marker = true; |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) { |
INFO("BB:%i live set of out blocks:\n", bb->getId()); |
bb->liveSet.print(); |
} |
// if (!bb->getEntry()) |
// return true; |
if (bb == BasicBlock::get(f->cfgExit)) { |
for (std::deque<ValueRef>::iterator it = f->outs.begin(); |
it != f->outs.end(); ++it) { |
assert(it->get()->asLValue()); |
bb->liveSet.set(it->get()->id); |
} |
} |
for (i = bb->getExit(); i && i != bb->getEntry()->prev; i = i->prev) { |
for (d = 0; i->defExists(d); ++d) |
bb->liveSet.clr(i->getDef(d)->id); |
for (s = 0; i->srcExists(s); ++s) |
if (i->getSrc(s)->asLValue()) |
bb->liveSet.set(i->getSrc(s)->id); |
} |
for (i = bb->getPhi(); i && i->op == OP_PHI; i = i->next) |
bb->liveSet.clr(i->getDef(0)->id); |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) { |
INFO("BB:%i live set after propagation:\n", bb->getId()); |
bb->liveSet.print(); |
} |
return true; |
} |
void |
RegAlloc::BuildIntervalsPass::collectLiveValues(BasicBlock *bb) |
{ |
BasicBlock *bbA = NULL, *bbB = NULL; |
if (bb->cfg.outgoingCount()) { |
// trickery to save a loop of OR'ing liveSets |
// aliasing works fine with BitSet::setOr |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
if (ei.getType() == Graph::Edge::DUMMY) |
continue; |
if (bbA) { |
bb->liveSet.setOr(&bbA->liveSet, &bbB->liveSet); |
bbA = bb; |
} else { |
bbA = bbB; |
} |
bbB = BasicBlock::get(ei.getNode()); |
} |
bb->liveSet.setOr(&bbB->liveSet, bbA ? &bbA->liveSet : NULL); |
} else |
if (bb->cfg.incidentCount()) { |
bb->liveSet.fill(0); |
} |
} |
bool |
RegAlloc::BuildIntervalsPass::visit(BasicBlock *bb) |
{ |
collectLiveValues(bb); |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "BuildIntervals(BB:%i)\n", bb->getId()); |
// go through out blocks and delete phi sources that do not originate from |
// the current block from the live set |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
BasicBlock *out = BasicBlock::get(ei.getNode()); |
for (Instruction *i = out->getPhi(); i && i->op == OP_PHI; i = i->next) { |
bb->liveSet.clr(i->getDef(0)->id); |
for (int s = 0; i->srcExists(s); ++s) { |
assert(i->src(s).getInsn()); |
if (i->getSrc(s)->getUniqueInsn()->bb == bb) // XXX: reachableBy ? |
bb->liveSet.set(i->getSrc(s)->id); |
else |
bb->liveSet.clr(i->getSrc(s)->id); |
} |
} |
} |
// remaining live-outs are live until end |
if (bb->getExit()) { |
for (unsigned int j = 0; j < bb->liveSet.getSize(); ++j) |
if (bb->liveSet.test(j)) |
addLiveRange(func->getLValue(j), bb, bb->getExit()->serial + 1); |
} |
for (Instruction *i = bb->getExit(); i && i->op != OP_PHI; i = i->prev) { |
for (int d = 0; i->defExists(d); ++d) { |
bb->liveSet.clr(i->getDef(d)->id); |
if (i->getDef(d)->reg.data.id >= 0) // add hazard for fixed regs |
i->getDef(d)->livei.extend(i->serial, i->serial); |
} |
for (int s = 0; i->srcExists(s); ++s) { |
if (!i->getSrc(s)->asLValue()) |
continue; |
if (!bb->liveSet.test(i->getSrc(s)->id)) { |
bb->liveSet.set(i->getSrc(s)->id); |
addLiveRange(i->getSrc(s), bb, i->serial); |
} |
} |
} |
if (bb == BasicBlock::get(func->cfg.getRoot())) { |
for (std::deque<ValueDef>::iterator it = func->ins.begin(); |
it != func->ins.end(); ++it) { |
if (it->get()->reg.data.id >= 0) // add hazard for fixed regs |
it->get()->livei.extend(0, 1); |
} |
} |
return true; |
} |
#define JOIN_MASK_PHI (1 << 0) |
#define JOIN_MASK_UNION (1 << 1) |
#define JOIN_MASK_MOV (1 << 2) |
#define JOIN_MASK_TEX (1 << 3) |
class GCRA |
{ |
public: |
GCRA(Function *, SpillCodeInserter&); |
~GCRA(); |
bool allocateRegisters(ArrayList& insns); |
void printNodeInfo() const; |
private: |
class RIG_Node : public Graph::Node |
{ |
public: |
RIG_Node(); |
void init(const RegisterSet&, LValue *); |
void addInterference(RIG_Node *); |
void addRegPreference(RIG_Node *); |
inline LValue *getValue() const |
{ |
return reinterpret_cast<LValue *>(data); |
} |
inline void setValue(LValue *lval) { data = lval; } |
inline uint8_t getCompMask() const |
{ |
return ((1 << colors) - 1) << (reg & 7); |
} |
static inline RIG_Node *get(const Graph::EdgeIterator& ei) |
{ |
return static_cast<RIG_Node *>(ei.getNode()); |
} |
public: |
uint32_t degree; |
uint16_t degreeLimit; // if deg < degLimit, node is trivially colourable |
uint16_t colors; |
DataFile f; |
int32_t reg; |
float weight; |
// list pointers for simplify() phase |
RIG_Node *next; |
RIG_Node *prev; |
// union of the live intervals of all coalesced values (we want to retain |
// the separate intervals for testing interference of compound values) |
Interval livei; |
std::list<RIG_Node *> prefRegs; |
}; |
private: |
inline RIG_Node *getNode(const LValue *v) const { return &nodes[v->id]; } |
void buildRIG(ArrayList&); |
bool coalesce(ArrayList&); |
bool doCoalesce(ArrayList&, unsigned int mask); |
void calculateSpillWeights(); |
void simplify(); |
bool selectRegisters(); |
void cleanup(const bool success); |
void simplifyEdge(RIG_Node *, RIG_Node *); |
void simplifyNode(RIG_Node *); |
bool coalesceValues(Value *, Value *, bool force); |
void resolveSplitsAndMerges(); |
void makeCompound(Instruction *, bool isSplit); |
inline void checkInterference(const RIG_Node *, Graph::EdgeIterator&); |
inline void insertOrderedTail(std::list<RIG_Node *>&, RIG_Node *); |
void checkList(std::list<RIG_Node *>&); |
private: |
std::stack<uint32_t> stack; |
// list headers for simplify() phase |
RIG_Node lo[2]; |
RIG_Node hi; |
Graph RIG; |
RIG_Node *nodes; |
unsigned int nodeCount; |
Function *func; |
Program *prog; |
static uint8_t relDegree[17][17]; |
RegisterSet regs; |
// need to fixup register id for participants of OP_MERGE/SPLIT |
std::list<Instruction *> merges; |
std::list<Instruction *> splits; |
SpillCodeInserter& spill; |
std::list<ValuePair> mustSpill; |
}; |
uint8_t GCRA::relDegree[17][17]; |
GCRA::RIG_Node::RIG_Node() : Node(NULL), next(this), prev(this) |
{ |
colors = 0; |
} |
void |
GCRA::printNodeInfo() const |
{ |
for (unsigned int i = 0; i < nodeCount; ++i) { |
if (!nodes[i].colors) |
continue; |
INFO("RIG_Node[%%%i]($[%u]%i): %u colors, weight %f, deg %u/%u\n X", |
i, |
nodes[i].f,nodes[i].reg,nodes[i].colors, |
nodes[i].weight, |
nodes[i].degree, nodes[i].degreeLimit); |
for (Graph::EdgeIterator ei = nodes[i].outgoing(); !ei.end(); ei.next()) |
INFO(" %%%i", RIG_Node::get(ei)->getValue()->id); |
for (Graph::EdgeIterator ei = nodes[i].incident(); !ei.end(); ei.next()) |
INFO(" %%%i", RIG_Node::get(ei)->getValue()->id); |
INFO("\n"); |
} |
} |
void |
GCRA::RIG_Node::init(const RegisterSet& regs, LValue *lval) |
{ |
setValue(lval); |
if (lval->reg.data.id >= 0) |
lval->noSpill = lval->fixedReg = 1; |
colors = regs.units(lval->reg.file, lval->reg.size); |
f = lval->reg.file; |
reg = -1; |
if (lval->reg.data.id >= 0) |
reg = regs.idToUnits(lval); |
weight = std::numeric_limits<float>::infinity(); |
degree = 0; |
degreeLimit = regs.getFileSize(f, lval->reg.size); |
livei.insert(lval->livei); |
} |
bool |
GCRA::coalesceValues(Value *dst, Value *src, bool force) |
{ |
LValue *rep = dst->join->asLValue(); |
LValue *val = src->join->asLValue(); |
if (!force && val->reg.data.id >= 0) { |
rep = src->join->asLValue(); |
val = dst->join->asLValue(); |
} |
RIG_Node *nRep = &nodes[rep->id]; |
RIG_Node *nVal = &nodes[val->id]; |
if (src->reg.file != dst->reg.file) { |
if (!force) |
return false; |
WARN("forced coalescing of values in different files !\n"); |
} |
if (!force && dst->reg.size != src->reg.size) |
return false; |
if ((rep->reg.data.id >= 0) && (rep->reg.data.id != val->reg.data.id)) { |
if (force) { |
if (val->reg.data.id >= 0) |
WARN("forced coalescing of values in different fixed regs !\n"); |
} else { |
if (val->reg.data.id >= 0) |
return false; |
// make sure that there is no overlap with the fixed register of rep |
for (ArrayList::Iterator it = func->allLValues.iterator(); |
!it.end(); it.next()) { |
Value *reg = reinterpret_cast<Value *>(it.get())->asLValue(); |
assert(reg); |
if (reg->interfers(rep) && reg->livei.overlaps(nVal->livei)) |
return false; |
} |
} |
} |
if (!force && nRep->livei.overlaps(nVal->livei)) |
return false; |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "joining %%%i($%i) <- %%%i\n", |
rep->id, rep->reg.data.id, val->id); |
// set join pointer of all values joined with val |
for (Value::DefIterator def = val->defs.begin(); def != val->defs.end(); |
++def) |
(*def)->get()->join = rep; |
assert(rep->join == rep && val->join == rep); |
// add val's definitions to rep and extend the live interval of its RIG node |
rep->defs.insert(rep->defs.end(), val->defs.begin(), val->defs.end()); |
nRep->livei.unify(nVal->livei); |
return true; |
} |
bool |
GCRA::coalesce(ArrayList& insns) |
{ |
bool ret = doCoalesce(insns, JOIN_MASK_PHI); |
if (!ret) |
return false; |
switch (func->getProgram()->getTarget()->getChipset() & ~0xf) { |
case 0x50: |
case 0x80: |
case 0x90: |
case 0xa0: |
ret = doCoalesce(insns, JOIN_MASK_UNION | JOIN_MASK_TEX); |
break; |
case 0xc0: |
case 0xd0: |
case 0xe0: |
ret = doCoalesce(insns, JOIN_MASK_UNION); |
break; |
default: |
break; |
} |
if (!ret) |
return false; |
return doCoalesce(insns, JOIN_MASK_MOV); |
} |
static inline uint8_t makeCompMask(int compSize, int base, int size) |
{ |
uint8_t m = ((1 << size) - 1) << base; |
switch (compSize) { |
case 1: |
return 0xff; |
case 2: |
m |= (m << 2); |
return (m << 4) | m; |
case 3: |
case 4: |
return (m << 4) | m; |
default: |
assert(compSize <= 8); |
return m; |
} |
} |
// Used when coalescing moves. The non-compound value will become one, e.g.: |
// mov b32 $r0 $r2 / merge b64 $r0d { $r0 $r1 } |
// split b64 { $r0 $r1 } $r0d / mov b64 $r0d f64 $r2d |
static inline void copyCompound(Value *dst, Value *src) |
{ |
LValue *ldst = dst->asLValue(); |
LValue *lsrc = src->asLValue(); |
if (ldst->compound && !lsrc->compound) { |
LValue *swap = lsrc; |
lsrc = ldst; |
ldst = swap; |
} |
ldst->compound = lsrc->compound; |
ldst->compMask = lsrc->compMask; |
} |
void |
GCRA::makeCompound(Instruction *insn, bool split) |
{ |
LValue *rep = (split ? insn->getSrc(0) : insn->getDef(0))->asLValue(); |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) { |
INFO("makeCompound(split = %i): ", split); |
insn->print(); |
} |
const unsigned int size = getNode(rep)->colors; |
unsigned int base = 0; |
if (!rep->compound) |
rep->compMask = 0xff; |
rep->compound = 1; |
for (int c = 0; split ? insn->defExists(c) : insn->srcExists(c); ++c) { |
LValue *val = (split ? insn->getDef(c) : insn->getSrc(c))->asLValue(); |
val->compound = 1; |
if (!val->compMask) |
val->compMask = 0xff; |
val->compMask &= makeCompMask(size, base, getNode(val)->colors); |
assert(val->compMask); |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "compound: %%%i:%02x <- %%%i:%02x\n", |
rep->id, rep->compMask, val->id, val->compMask); |
base += getNode(val)->colors; |
} |
assert(base == size); |
} |
bool |
GCRA::doCoalesce(ArrayList& insns, unsigned int mask) |
{ |
int c, n; |
for (n = 0; n < insns.getSize(); ++n) { |
Instruction *i; |
Instruction *insn = reinterpret_cast<Instruction *>(insns.get(n)); |
switch (insn->op) { |
case OP_PHI: |
if (!(mask & JOIN_MASK_PHI)) |
break; |
for (c = 0; insn->srcExists(c); ++c) |
if (!coalesceValues(insn->getDef(0), insn->getSrc(c), false)) { |
// this is bad |
ERROR("failed to coalesce phi operands\n"); |
return false; |
} |
break; |
case OP_UNION: |
case OP_MERGE: |
if (!(mask & JOIN_MASK_UNION)) |
break; |
for (c = 0; insn->srcExists(c); ++c) |
coalesceValues(insn->getDef(0), insn->getSrc(c), true); |
if (insn->op == OP_MERGE) { |
merges.push_back(insn); |
if (insn->srcExists(1)) |
makeCompound(insn, false); |
} |
break; |
case OP_SPLIT: |
if (!(mask & JOIN_MASK_UNION)) |
break; |
splits.push_back(insn); |
for (c = 0; insn->defExists(c); ++c) |
coalesceValues(insn->getSrc(0), insn->getDef(c), true); |
makeCompound(insn, true); |
break; |
case OP_MOV: |
if (!(mask & JOIN_MASK_MOV)) |
break; |
i = NULL; |
if (!insn->getDef(0)->uses.empty()) |
i = insn->getDef(0)->uses.front()->getInsn(); |
// if this is a contraint-move there will only be a single use |
if (i && i->op == OP_MERGE) // do we really still need this ? |
break; |
i = insn->getSrc(0)->getUniqueInsn(); |
if (i && !i->constrainedDefs()) { |
if (coalesceValues(insn->getDef(0), insn->getSrc(0), false)) |
copyCompound(insn->getSrc(0), insn->getDef(0)); |
} |
break; |
case OP_TEX: |
case OP_TXB: |
case OP_TXL: |
case OP_TXF: |
case OP_TXQ: |
case OP_TXD: |
case OP_TXG: |
case OP_TEXCSAA: |
if (!(mask & JOIN_MASK_TEX)) |
break; |
for (c = 0; insn->srcExists(c) && c != insn->predSrc; ++c) |
coalesceValues(insn->getDef(c), insn->getSrc(c), true); |
break; |
default: |
break; |
} |
} |
return true; |
} |
void |
GCRA::RIG_Node::addInterference(RIG_Node *node) |
{ |
this->degree += relDegree[node->colors][colors]; |
node->degree += relDegree[colors][node->colors]; |
this->attach(node, Graph::Edge::CROSS); |
} |
void |
GCRA::RIG_Node::addRegPreference(RIG_Node *node) |
{ |
prefRegs.push_back(node); |
} |
GCRA::GCRA(Function *fn, SpillCodeInserter& spill) : |
func(fn), |
regs(fn->getProgram()->getTarget()), |
spill(spill) |
{ |
prog = func->getProgram(); |
// initialize relative degrees array - i takes away from j |
for (int i = 1; i <= 16; ++i) |
for (int j = 1; j <= 16; ++j) |
relDegree[i][j] = j * ((i + j - 1) / j); |
} |
GCRA::~GCRA() |
{ |
if (nodes) |
delete[] nodes; |
} |
void |
GCRA::checkList(std::list<RIG_Node *>& lst) |
{ |
GCRA::RIG_Node *prev = NULL; |
for (std::list<RIG_Node *>::iterator it = lst.begin(); |
it != lst.end(); |
++it) { |
assert((*it)->getValue()->join == (*it)->getValue()); |
if (prev) |
assert(prev->livei.begin() <= (*it)->livei.begin()); |
prev = *it; |
} |
} |
void |
GCRA::insertOrderedTail(std::list<RIG_Node *>& list, RIG_Node *node) |
{ |
if (node->livei.isEmpty()) |
return; |
// only the intervals of joined values don't necessarily arrive in order |
std::list<RIG_Node *>::iterator prev, it; |
for (it = list.end(); it != list.begin(); it = prev) { |
prev = it; |
--prev; |
if ((*prev)->livei.begin() <= node->livei.begin()) |
break; |
} |
list.insert(it, node); |
} |
void |
GCRA::buildRIG(ArrayList& insns) |
{ |
std::list<RIG_Node *> values, active; |
for (std::deque<ValueDef>::iterator it = func->ins.begin(); |
it != func->ins.end(); ++it) |
insertOrderedTail(values, getNode(it->get()->asLValue())); |
for (int i = 0; i < insns.getSize(); ++i) { |
Instruction *insn = reinterpret_cast<Instruction *>(insns.get(i)); |
for (int d = 0; insn->defExists(d); ++d) |
if (insn->getDef(d)->rep() == insn->getDef(d)) |
insertOrderedTail(values, getNode(insn->getDef(d)->asLValue())); |
} |
checkList(values); |
while (!values.empty()) { |
RIG_Node *cur = values.front(); |
for (std::list<RIG_Node *>::iterator it = active.begin(); |
it != active.end();) { |
RIG_Node *node = *it; |
if (node->livei.end() <= cur->livei.begin()) { |
it = active.erase(it); |
} else { |
if (node->f == cur->f && node->livei.overlaps(cur->livei)) |
cur->addInterference(node); |
++it; |
} |
} |
values.pop_front(); |
active.push_back(cur); |
} |
} |
void |
GCRA::calculateSpillWeights() |
{ |
for (unsigned int i = 0; i < nodeCount; ++i) { |
RIG_Node *const n = &nodes[i]; |
if (!nodes[i].colors || nodes[i].livei.isEmpty()) |
continue; |
if (nodes[i].reg >= 0) { |
// update max reg |
regs.occupy(n->f, n->reg, n->colors); |
continue; |
} |
LValue *val = nodes[i].getValue(); |
if (!val->noSpill) { |
int rc = 0; |
for (Value::DefIterator it = val->defs.begin(); |
it != val->defs.end(); |
++it) |
rc += (*it)->get()->refCount(); |
nodes[i].weight = |
(float)rc * (float)rc / (float)nodes[i].livei.extent(); |
} |
if (nodes[i].degree < nodes[i].degreeLimit) { |
int l = 0; |
if (val->reg.size > 4) |
l = 1; |
DLLIST_ADDHEAD(&lo[l], &nodes[i]); |
} else { |
DLLIST_ADDHEAD(&hi, &nodes[i]); |
} |
} |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) |
printNodeInfo(); |
} |
void |
GCRA::simplifyEdge(RIG_Node *a, RIG_Node *b) |
{ |
bool move = b->degree >= b->degreeLimit; |
INFO_DBG(prog->dbgFlags, REG_ALLOC, |
"edge: (%%%i, deg %u/%u) >-< (%%%i, deg %u/%u)\n", |
a->getValue()->id, a->degree, a->degreeLimit, |
b->getValue()->id, b->degree, b->degreeLimit); |
b->degree -= relDegree[a->colors][b->colors]; |
move = move && b->degree < b->degreeLimit; |
if (move && !DLLIST_EMPTY(b)) { |
int l = (b->getValue()->reg.size > 4) ? 1 : 0; |
DLLIST_DEL(b); |
DLLIST_ADDTAIL(&lo[l], b); |
} |
} |
void |
GCRA::simplifyNode(RIG_Node *node) |
{ |
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) |
simplifyEdge(node, RIG_Node::get(ei)); |
for (Graph::EdgeIterator ei = node->incident(); !ei.end(); ei.next()) |
simplifyEdge(node, RIG_Node::get(ei)); |
DLLIST_DEL(node); |
stack.push(node->getValue()->id); |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "SIMPLIFY: pushed %%%i%s\n", |
node->getValue()->id, |
(node->degree < node->degreeLimit) ? "" : "(spill)"); |
} |
void |
GCRA::simplify() |
{ |
for (;;) { |
if (!DLLIST_EMPTY(&lo[0])) { |
do { |
simplifyNode(lo[0].next); |
} while (!DLLIST_EMPTY(&lo[0])); |
} else |
if (!DLLIST_EMPTY(&lo[1])) { |
simplifyNode(lo[1].next); |
} else |
if (!DLLIST_EMPTY(&hi)) { |
RIG_Node *best = hi.next; |
float bestScore = best->weight / (float)best->degree; |
// spill candidate |
for (RIG_Node *it = best->next; it != &hi; it = it->next) { |
float score = it->weight / (float)it->degree; |
if (score < bestScore) { |
best = it; |
bestScore = score; |
} |
} |
if (isinf(bestScore)) { |
ERROR("no viable spill candidates left\n"); |
break; |
} |
simplifyNode(best); |
} else { |
break; |
} |
} |
} |
void |
GCRA::checkInterference(const RIG_Node *node, Graph::EdgeIterator& ei) |
{ |
const RIG_Node *intf = RIG_Node::get(ei); |
if (intf->reg < 0) |
return; |
const LValue *vA = node->getValue(); |
const LValue *vB = intf->getValue(); |
const uint8_t intfMask = ((1 << intf->colors) - 1) << (intf->reg & 7); |
if (vA->compound | vB->compound) { |
// NOTE: this only works for >aligned< register tuples ! |
for (Value::DefCIterator D = vA->defs.begin(); D != vA->defs.end(); ++D) { |
for (Value::DefCIterator d = vB->defs.begin(); d != vB->defs.end(); ++d) { |
const LValue *vD = (*D)->get()->asLValue(); |
const LValue *vd = (*d)->get()->asLValue(); |
if (!vD->livei.overlaps(vd->livei)) { |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "(%%%i) X (%%%i): no overlap\n", |
vD->id, vd->id); |
continue; |
} |
uint8_t mask = vD->compound ? vD->compMask : ~0; |
if (vd->compound) { |
assert(vB->compound); |
mask &= vd->compMask & vB->compMask; |
} else { |
mask &= intfMask; |
} |
INFO_DBG(prog->dbgFlags, REG_ALLOC, |
"(%%%i)%02x X (%%%i)%02x & %02x: $r%i.%02x\n", |
vD->id, |
vD->compound ? vD->compMask : 0xff, |
vd->id, |
vd->compound ? vd->compMask : intfMask, |
vB->compMask, intf->reg & ~7, mask); |
if (mask) |
regs.occupyMask(node->f, intf->reg & ~7, mask); |
} |
} |
} else { |
INFO_DBG(prog->dbgFlags, REG_ALLOC, |
"(%%%i) X (%%%i): $r%i + %u\n", |
vA->id, vB->id, intf->reg, intf->colors); |
regs.occupy(node->f, intf->reg, intf->colors); |
} |
} |
bool |
GCRA::selectRegisters() |
{ |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "\nSELECT phase\n"); |
while (!stack.empty()) { |
RIG_Node *node = &nodes[stack.top()]; |
stack.pop(); |
regs.reset(node->f); |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "\nNODE[%%%i, %u colors]\n", |
node->getValue()->id, node->colors); |
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) |
checkInterference(node, ei); |
for (Graph::EdgeIterator ei = node->incident(); !ei.end(); ei.next()) |
checkInterference(node, ei); |
if (!node->prefRegs.empty()) { |
for (std::list<RIG_Node *>::const_iterator it = node->prefRegs.begin(); |
it != node->prefRegs.end(); |
++it) { |
if ((*it)->reg >= 0 && |
regs.testOccupy(node->f, (*it)->reg, node->colors)) { |
node->reg = (*it)->reg; |
break; |
} |
} |
} |
if (node->reg >= 0) |
continue; |
LValue *lval = node->getValue(); |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) |
regs.print(); |
bool ret = regs.assign(node->reg, node->f, node->colors); |
if (ret) { |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "assigned reg %i\n", node->reg); |
lval->compMask = node->getCompMask(); |
} else { |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "must spill: %%%i (size %u)\n", |
lval->id, lval->reg.size); |
Symbol *slot = NULL; |
if (lval->reg.file == FILE_GPR) |
slot = spill.assignSlot(node->livei, lval->reg.size); |
mustSpill.push_back(ValuePair(lval, slot)); |
} |
} |
if (!mustSpill.empty()) |
return false; |
for (unsigned int i = 0; i < nodeCount; ++i) { |
LValue *lval = nodes[i].getValue(); |
if (nodes[i].reg >= 0 && nodes[i].colors > 0) |
lval->reg.data.id = |
regs.unitsToId(nodes[i].f, nodes[i].reg, lval->reg.size); |
} |
return true; |
} |
bool |
GCRA::allocateRegisters(ArrayList& insns) |
{ |
bool ret; |
INFO_DBG(prog->dbgFlags, REG_ALLOC, |
"allocateRegisters to %u instructions\n", insns.getSize()); |
nodeCount = func->allLValues.getSize(); |
nodes = new RIG_Node[nodeCount]; |
if (!nodes) |
return false; |
for (unsigned int i = 0; i < nodeCount; ++i) { |
LValue *lval = reinterpret_cast<LValue *>(func->allLValues.get(i)); |
if (lval) { |
nodes[i].init(regs, lval); |
RIG.insert(&nodes[i]); |
} |
} |
// coalesce first, we use only 1 RIG node for a group of joined values |
ret = coalesce(insns); |
if (!ret) |
goto out; |
if (func->getProgram()->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) |
func->printLiveIntervals(); |
buildRIG(insns); |
calculateSpillWeights(); |
simplify(); |
ret = selectRegisters(); |
if (!ret) { |
INFO_DBG(prog->dbgFlags, REG_ALLOC, |
"selectRegisters failed, inserting spill code ...\n"); |
regs.reset(FILE_GPR, true); |
spill.run(mustSpill); |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) |
func->print(); |
} else { |
prog->maxGPR = std::max(prog->maxGPR, regs.getMaxAssigned(FILE_GPR)); |
} |
out: |
cleanup(ret); |
return ret; |
} |
void |
GCRA::cleanup(const bool success) |
{ |
mustSpill.clear(); |
for (ArrayList::Iterator it = func->allLValues.iterator(); |
!it.end(); it.next()) { |
LValue *lval = reinterpret_cast<LValue *>(it.get()); |
lval->livei.clear(); |
lval->compound = 0; |
lval->compMask = 0; |
if (lval->join == lval) |
continue; |
if (success) { |
lval->reg.data.id = lval->join->reg.data.id; |
} else { |
for (Value::DefIterator d = lval->defs.begin(); d != lval->defs.end(); |
++d) |
lval->join->defs.remove(*d); |
lval->join = lval; |
} |
} |
if (success) |
resolveSplitsAndMerges(); |
splits.clear(); // avoid duplicate entries on next coalesce pass |
merges.clear(); |
delete[] nodes; |
nodes = NULL; |
} |
Symbol * |
SpillCodeInserter::assignSlot(const Interval &livei, const unsigned int size) |
{ |
SpillSlot slot; |
int32_t offsetBase = stackSize; |
int32_t offset; |
std::list<SpillSlot>::iterator pos = slots.end(), it = slots.begin(); |
if (offsetBase % size) |
offsetBase += size - (offsetBase % size); |
slot.sym = NULL; |
for (offset = offsetBase; offset < stackSize; offset += size) { |
const int32_t entryEnd = offset + size; |
while (it != slots.end() && it->offset < offset) |
++it; |
if (it == slots.end()) // no slots left |
break; |
std::list<SpillSlot>::iterator bgn = it; |
while (it != slots.end() && it->offset < entryEnd) { |
it->occup.print(); |
if (it->occup.overlaps(livei)) |
break; |
++it; |
} |
if (it == slots.end() || it->offset >= entryEnd) { |
// fits |
for (; bgn != slots.end() && bgn->offset < entryEnd; ++bgn) { |
bgn->occup.insert(livei); |
if (bgn->size() == size) |
slot.sym = bgn->sym; |
} |
break; |
} |
} |
if (!slot.sym) { |
stackSize = offset + size; |
slot.offset = offset; |
slot.sym = new_Symbol(func->getProgram(), FILE_MEMORY_LOCAL); |
if (!func->stackPtr) |
offset += func->tlsBase; |
slot.sym->setAddress(NULL, offset); |
slot.sym->reg.size = size; |
slots.insert(pos, slot)->occup.insert(livei); |
} |
return slot.sym; |
} |
void |
SpillCodeInserter::spill(Instruction *defi, Value *slot, LValue *lval) |
{ |
const DataType ty = typeOfSize(slot->reg.size); |
Instruction *st; |
if (slot->reg.file == FILE_MEMORY_LOCAL) { |
st = new_Instruction(func, OP_STORE, ty); |
st->setSrc(0, slot); |
st->setSrc(1, lval); |
lval->noSpill = 1; |
} else { |
st = new_Instruction(func, OP_CVT, ty); |
st->setDef(0, slot); |
st->setSrc(0, lval); |
} |
defi->bb->insertAfter(defi, st); |
} |
LValue * |
SpillCodeInserter::unspill(Instruction *usei, LValue *lval, Value *slot) |
{ |
const DataType ty = typeOfSize(slot->reg.size); |
lval = cloneShallow(func, lval); |
Instruction *ld; |
if (slot->reg.file == FILE_MEMORY_LOCAL) { |
lval->noSpill = 1; |
ld = new_Instruction(func, OP_LOAD, ty); |
} else { |
ld = new_Instruction(func, OP_CVT, ty); |
} |
ld->setDef(0, lval); |
ld->setSrc(0, slot); |
usei->bb->insertBefore(usei, ld); |
return lval; |
} |
bool |
SpillCodeInserter::run(const std::list<ValuePair>& lst) |
{ |
for (std::list<ValuePair>::const_iterator it = lst.begin(); it != lst.end(); |
++it) { |
LValue *lval = it->first->asLValue(); |
Symbol *mem = it->second ? it->second->asSym() : NULL; |
for (Value::DefIterator d = lval->defs.begin(); d != lval->defs.end(); |
++d) { |
Value *slot = mem ? |
static_cast<Value *>(mem) : new_LValue(func, FILE_GPR); |
Value *tmp = NULL; |
Instruction *last = NULL; |
LValue *dval = (*d)->get()->asLValue(); |
Instruction *defi = (*d)->getInsn(); |
// handle uses first or they'll contain the spill stores |
while (!dval->uses.empty()) { |
ValueRef *u = dval->uses.front(); |
Instruction *usei = u->getInsn(); |
assert(usei); |
if (usei->op == OP_PHI) { |
tmp = (slot->reg.file == FILE_MEMORY_LOCAL) ? NULL : slot; |
last = NULL; |
} else |
if (!last || usei != last->next) { // TODO: sort uses |
tmp = unspill(usei, dval, slot); |
last = usei; |
} |
u->set(tmp); |
} |
assert(defi); |
if (defi->op == OP_PHI) { |
d = lval->defs.erase(d); |
--d; |
if (slot->reg.file == FILE_MEMORY_LOCAL) |
delete_Instruction(func->getProgram(), defi); |
else |
defi->setDef(0, slot); |
} else { |
spill(defi, slot, dval); |
} |
} |
} |
// TODO: We're not trying to reuse old slots in a potential next iteration. |
// We have to update the slots' livei intervals to be able to do that. |
stackBase = stackSize; |
slots.clear(); |
return true; |
} |
bool |
RegAlloc::exec() |
{ |
for (IteratorRef it = prog->calls.iteratorDFS(false); |
!it->end(); it->next()) { |
func = Function::get(reinterpret_cast<Graph::Node *>(it->get())); |
func->tlsBase = prog->tlsSize; |
if (!execFunc()) |
return false; |
prog->tlsSize += func->tlsSize; |
} |
return true; |
} |
bool |
RegAlloc::execFunc() |
{ |
InsertConstraintsPass insertConstr; |
PhiMovesPass insertPhiMoves; |
ArgumentMovesPass insertArgMoves; |
BuildIntervalsPass buildIntervals; |
SpillCodeInserter insertSpills(func); |
GCRA gcra(func, insertSpills); |
unsigned int i, retries; |
bool ret; |
if (!func->ins.empty()) { |
// Insert a nop at the entry so inputs only used by the first instruction |
// don't count as having an empty live range. |
Instruction *nop = new_Instruction(func, OP_NOP, TYPE_NONE); |
BasicBlock::get(func->cfg.getRoot())->insertHead(nop); |
} |
ret = insertConstr.exec(func); |
if (!ret) |
goto out; |
ret = insertPhiMoves.run(func); |
if (!ret) |
goto out; |
ret = insertArgMoves.run(func); |
if (!ret) |
goto out; |
// TODO: need to fix up spill slot usage ranges to support > 1 retry |
for (retries = 0; retries < 3; ++retries) { |
if (retries && (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC)) |
INFO("Retry: %i\n", retries); |
if (prog->dbgFlags & NV50_IR_DEBUG_REG_ALLOC) |
func->print(); |
// spilling to registers may add live ranges, need to rebuild everything |
ret = true; |
for (sequence = func->cfg.nextSequence(), i = 0; |
ret && i <= func->loopNestingBound; |
sequence = func->cfg.nextSequence(), ++i) |
ret = buildLiveSets(BasicBlock::get(func->cfg.getRoot())); |
if (!ret) |
break; |
func->orderInstructions(this->insns); |
ret = buildIntervals.run(func); |
if (!ret) |
break; |
ret = gcra.allocateRegisters(insns); |
if (ret) |
break; // success |
} |
INFO_DBG(prog->dbgFlags, REG_ALLOC, "RegAlloc done: %i\n", ret); |
func->tlsSize = insertSpills.getStackSize(); |
out: |
return ret; |
} |
// TODO: check if modifying Instruction::join here breaks anything |
void |
GCRA::resolveSplitsAndMerges() |
{ |
for (std::list<Instruction *>::iterator it = splits.begin(); |
it != splits.end(); |
++it) { |
Instruction *split = *it; |
unsigned int reg = regs.idToBytes(split->getSrc(0)); |
for (int d = 0; split->defExists(d); ++d) { |
Value *v = split->getDef(d); |
v->reg.data.id = regs.bytesToId(v, reg); |
v->join = v; |
reg += v->reg.size; |
} |
} |
splits.clear(); |
for (std::list<Instruction *>::iterator it = merges.begin(); |
it != merges.end(); |
++it) { |
Instruction *merge = *it; |
unsigned int reg = regs.idToBytes(merge->getDef(0)); |
for (int s = 0; merge->srcExists(s); ++s) { |
Value *v = merge->getSrc(s); |
v->reg.data.id = regs.bytesToId(v, reg); |
v->join = v; |
reg += v->reg.size; |
} |
} |
merges.clear(); |
} |
bool Program::registerAllocation() |
{ |
RegAlloc ra(this); |
return ra.exec(); |
} |
bool |
RegAlloc::InsertConstraintsPass::exec(Function *ir) |
{ |
constrList.clear(); |
bool ret = run(ir, true, true); |
if (ret) |
ret = insertConstraintMoves(); |
return ret; |
} |
// TODO: make part of texture insn |
void |
RegAlloc::InsertConstraintsPass::textureMask(TexInstruction *tex) |
{ |
Value *def[4]; |
int c, k, d; |
uint8_t mask = 0; |
for (d = 0, k = 0, c = 0; c < 4; ++c) { |
if (!(tex->tex.mask & (1 << c))) |
continue; |
if (tex->getDef(k)->refCount()) { |
mask |= 1 << c; |
def[d++] = tex->getDef(k); |
} |
++k; |
} |
tex->tex.mask = mask; |
for (c = 0; c < d; ++c) |
tex->setDef(c, def[c]); |
for (; c < 4; ++c) |
tex->setDef(c, NULL); |
} |
bool |
RegAlloc::InsertConstraintsPass::detectConflict(Instruction *cst, int s) |
{ |
Value *v = cst->getSrc(s); |
// current register allocation can't handle it if a value participates in |
// multiple constraints |
for (Value::UseIterator it = v->uses.begin(); it != v->uses.end(); ++it) { |
if (cst != (*it)->getInsn()) |
return true; |
} |
// can start at s + 1 because detectConflict is called on all sources |
for (int c = s + 1; cst->srcExists(c); ++c) |
if (v == cst->getSrc(c)) |
return true; |
Instruction *defi = v->getInsn(); |
return (!defi || defi->constrainedDefs()); |
} |
void |
RegAlloc::InsertConstraintsPass::addConstraint(Instruction *i, int s, int n) |
{ |
Instruction *cst; |
int d; |
// first, look for an existing identical constraint op |
for (std::list<Instruction *>::iterator it = constrList.begin(); |
it != constrList.end(); |
++it) { |
cst = (*it); |
if (!i->bb->dominatedBy(cst->bb)) |
break; |
for (d = 0; d < n; ++d) |
if (cst->getSrc(d) != i->getSrc(d + s)) |
break; |
if (d >= n) { |
for (d = 0; d < n; ++d, ++s) |
i->setSrc(s, cst->getDef(d)); |
return; |
} |
} |
cst = new_Instruction(func, OP_CONSTRAINT, i->dType); |
for (d = 0; d < n; ++s, ++d) { |
cst->setDef(d, new_LValue(func, FILE_GPR)); |
cst->setSrc(d, i->getSrc(s)); |
i->setSrc(s, cst->getDef(d)); |
} |
i->bb->insertBefore(i, cst); |
constrList.push_back(cst); |
} |
// Add a dummy use of the pointer source of >= 8 byte loads after the load |
// to prevent it from being assigned a register which overlapping the load's |
// destination, which would produce random corruptions. |
void |
RegAlloc::InsertConstraintsPass::addHazard(Instruction *i, const ValueRef *src) |
{ |
Instruction *hzd = new_Instruction(func, OP_NOP, TYPE_NONE); |
hzd->setSrc(0, src->get()); |
i->bb->insertAfter(i, hzd); |
} |
// b32 { %r0 %r1 %r2 %r3 } -> b128 %r0q |
void |
RegAlloc::InsertConstraintsPass::condenseDefs(Instruction *insn) |
{ |
uint8_t size = 0; |
int n; |
for (n = 0; insn->defExists(n) && insn->def(n).getFile() == FILE_GPR; ++n) |
size += insn->getDef(n)->reg.size; |
if (n < 2) |
return; |
LValue *lval = new_LValue(func, FILE_GPR); |
lval->reg.size = size; |
Instruction *split = new_Instruction(func, OP_SPLIT, typeOfSize(size)); |
split->setSrc(0, lval); |
for (int d = 0; d < n; ++d) { |
split->setDef(d, insn->getDef(d)); |
insn->setDef(d, NULL); |
} |
insn->setDef(0, lval); |
for (int k = 1, d = n; insn->defExists(d); ++d, ++k) { |
insn->setDef(k, insn->getDef(d)); |
insn->setDef(d, NULL); |
} |
// carry over predicate if any (mainly for OP_UNION uses) |
split->setPredicate(insn->cc, insn->getPredicate()); |
insn->bb->insertAfter(insn, split); |
constrList.push_back(split); |
} |
void |
RegAlloc::InsertConstraintsPass::condenseSrcs(Instruction *insn, |
const int a, const int b) |
{ |
uint8_t size = 0; |
if (a >= b) |
return; |
for (int s = a; s <= b; ++s) |
size += insn->getSrc(s)->reg.size; |
if (!size) |
return; |
LValue *lval = new_LValue(func, FILE_GPR); |
lval->reg.size = size; |
Value *save[3]; |
insn->takeExtraSources(0, save); |
Instruction *merge = new_Instruction(func, OP_MERGE, typeOfSize(size)); |
merge->setDef(0, lval); |
for (int s = a, i = 0; s <= b; ++s, ++i) { |
merge->setSrc(i, insn->getSrc(s)); |
insn->setSrc(s, NULL); |
} |
insn->setSrc(a, lval); |
for (int k = a + 1, s = b + 1; insn->srcExists(s); ++s, ++k) { |
insn->setSrc(k, insn->getSrc(s)); |
insn->setSrc(s, NULL); |
} |
insn->bb->insertBefore(insn, merge); |
insn->putExtraSources(0, save); |
constrList.push_back(merge); |
} |
void |
RegAlloc::InsertConstraintsPass::texConstraintNVE0(TexInstruction *tex) |
{ |
if (isTextureOp(tex->op)) |
textureMask(tex); |
condenseDefs(tex); |
if (tex->op == OP_SUSTB || tex->op == OP_SUSTP) { |
condenseSrcs(tex, 3, (3 + typeSizeof(tex->dType) / 4) - 1); |
} else |
if (isTextureOp(tex->op)) { |
int n = tex->srcCount(0xff, true); |
if (n > 4) { |
condenseSrcs(tex, 0, 3); |
if (n > 5) // NOTE: first call modified positions already |
condenseSrcs(tex, 4 - (4 - 1), n - 1 - (4 - 1)); |
} else |
if (n > 1) { |
condenseSrcs(tex, 0, n - 1); |
} |
} |
} |
void |
RegAlloc::InsertConstraintsPass::texConstraintNVC0(TexInstruction *tex) |
{ |
int n, s; |
textureMask(tex); |
if (tex->op == OP_TXQ) { |
s = tex->srcCount(0xff); |
n = 0; |
} else { |
s = tex->tex.target.getArgCount(); |
if (!tex->tex.target.isArray() && |
(tex->tex.rIndirectSrc >= 0 || tex->tex.sIndirectSrc >= 0)) |
++s; |
if (tex->op == OP_TXD && tex->tex.useOffsets) |
++s; |
n = tex->srcCount(0xff) - s; |
assert(n <= 4); |
} |
if (s > 1) |
condenseSrcs(tex, 0, s - 1); |
if (n > 1) // NOTE: first call modified positions already |
condenseSrcs(tex, 1, n); |
condenseDefs(tex); |
} |
void |
RegAlloc::InsertConstraintsPass::texConstraintNV50(TexInstruction *tex) |
{ |
Value *pred = tex->getPredicate(); |
if (pred) |
tex->setPredicate(tex->cc, NULL); |
textureMask(tex); |
assert(tex->defExists(0) && tex->srcExists(0)); |
// make src and def count match |
int c; |
for (c = 0; tex->srcExists(c) || tex->defExists(c); ++c) { |
if (!tex->srcExists(c)) |
tex->setSrc(c, new_LValue(func, tex->getSrc(0)->asLValue())); |
if (!tex->defExists(c)) |
tex->setDef(c, new_LValue(func, tex->getDef(0)->asLValue())); |
} |
if (pred) |
tex->setPredicate(tex->cc, pred); |
condenseDefs(tex); |
condenseSrcs(tex, 0, c - 1); |
} |
// Insert constraint markers for instructions whose multiple sources must be |
// located in consecutive registers. |
bool |
RegAlloc::InsertConstraintsPass::visit(BasicBlock *bb) |
{ |
TexInstruction *tex; |
Instruction *next; |
int s, size; |
targ = bb->getProgram()->getTarget(); |
for (Instruction *i = bb->getEntry(); i; i = next) { |
next = i->next; |
if ((tex = i->asTex())) { |
switch (targ->getChipset() & ~0xf) { |
case 0x50: |
case 0x80: |
case 0x90: |
case 0xa0: |
texConstraintNV50(tex); |
break; |
case 0xc0: |
case 0xd0: |
texConstraintNVC0(tex); |
break; |
case 0xe0: |
case NVISA_GK110_CHIPSET: |
texConstraintNVE0(tex); |
break; |
default: |
break; |
} |
} else |
if (i->op == OP_EXPORT || i->op == OP_STORE) { |
for (size = typeSizeof(i->dType), s = 1; size > 0; ++s) { |
assert(i->srcExists(s)); |
size -= i->getSrc(s)->reg.size; |
} |
condenseSrcs(i, 1, s - 1); |
} else |
if (i->op == OP_LOAD || i->op == OP_VFETCH) { |
condenseDefs(i); |
if (i->src(0).isIndirect(0) && typeSizeof(i->dType) >= 8) |
addHazard(i, i->src(0).getIndirect(0)); |
} else |
if (i->op == OP_UNION || |
i->op == OP_MERGE || |
i->op == OP_SPLIT) { |
constrList.push_back(i); |
} |
} |
return true; |
} |
// Insert extra moves so that, if multiple register constraints on a value are |
// in conflict, these conflicts can be resolved. |
bool |
RegAlloc::InsertConstraintsPass::insertConstraintMoves() |
{ |
for (std::list<Instruction *>::iterator it = constrList.begin(); |
it != constrList.end(); |
++it) { |
Instruction *cst = *it; |
Instruction *mov; |
if (cst->op == OP_SPLIT && 0) { |
// spilling splits is annoying, just make sure they're separate |
for (int d = 0; cst->defExists(d); ++d) { |
if (!cst->getDef(d)->refCount()) |
continue; |
LValue *lval = new_LValue(func, cst->def(d).getFile()); |
const uint8_t size = cst->def(d).getSize(); |
lval->reg.size = size; |
mov = new_Instruction(func, OP_MOV, typeOfSize(size)); |
mov->setSrc(0, lval); |
mov->setDef(0, cst->getDef(d)); |
cst->setDef(d, mov->getSrc(0)); |
cst->bb->insertAfter(cst, mov); |
cst->getSrc(0)->asLValue()->noSpill = 1; |
mov->getSrc(0)->asLValue()->noSpill = 1; |
} |
} else |
if (cst->op == OP_MERGE || cst->op == OP_UNION) { |
for (int s = 0; cst->srcExists(s); ++s) { |
const uint8_t size = cst->src(s).getSize(); |
if (!cst->getSrc(s)->defs.size()) { |
mov = new_Instruction(func, OP_NOP, typeOfSize(size)); |
mov->setDef(0, cst->getSrc(s)); |
cst->bb->insertBefore(cst, mov); |
continue; |
} |
assert(cst->getSrc(s)->defs.size() == 1); // still SSA |
Instruction *defi = cst->getSrc(s)->defs.front()->getInsn(); |
// catch some cases where don't really need MOVs |
if (cst->getSrc(s)->refCount() == 1 && !defi->constrainedDefs()) |
continue; |
LValue *lval = new_LValue(func, cst->src(s).getFile()); |
lval->reg.size = size; |
mov = new_Instruction(func, OP_MOV, typeOfSize(size)); |
mov->setDef(0, lval); |
mov->setSrc(0, cst->getSrc(s)); |
cst->setSrc(s, mov->getDef(0)); |
cst->bb->insertBefore(cst, mov); |
cst->getDef(0)->asLValue()->noSpill = 1; // doesn't help |
if (cst->op == OP_UNION) |
mov->setPredicate(defi->cc, defi->getPredicate()); |
} |
} |
} |
return true; |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_ssa.cpp |
---|
0,0 → 1,552 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir.h" |
#include "nv50_ir_target.h" |
namespace nv50_ir { |
// Converts nv50 IR generated from TGSI to SSA form. |
// DominatorTree implements an algorithm for finding immediate dominators, |
// as described by T. Lengauer & R. Tarjan. |
class DominatorTree : public Graph |
{ |
public: |
DominatorTree(Graph *cfg); |
~DominatorTree() { } |
bool dominates(BasicBlock *, BasicBlock *); |
void findDominanceFrontiers(); |
private: |
void build(); |
void buildDFS(Node *); |
void squash(int); |
inline void link(int, int); |
inline int eval(int); |
void debugPrint(); |
Graph *cfg; |
Node **vert; |
int *data; |
const int count; |
#define SEMI(i) (data[(i) + 0 * count]) |
#define ANCESTOR(i) (data[(i) + 1 * count]) |
#define PARENT(i) (data[(i) + 2 * count]) |
#define LABEL(i) (data[(i) + 3 * count]) |
#define DOM(i) (data[(i) + 4 * count]) |
}; |
void DominatorTree::debugPrint() |
{ |
for (int i = 0; i < count; ++i) { |
INFO("SEMI(%i) = %i\n", i, SEMI(i)); |
INFO("ANCESTOR(%i) = %i\n", i, ANCESTOR(i)); |
INFO("PARENT(%i) = %i\n", i, PARENT(i)); |
INFO("LABEL(%i) = %i\n", i, LABEL(i)); |
INFO("DOM(%i) = %i\n", i, DOM(i)); |
} |
} |
DominatorTree::DominatorTree(Graph *cfgraph) : cfg(cfgraph), |
count(cfg->getSize()) |
{ |
int i = 0; |
vert = new Node * [count]; |
data = new int[5 * count]; |
for (IteratorRef it = cfg->iteratorDFS(true); !it->end(); it->next(), ++i) { |
vert[i] = reinterpret_cast<Node *>(it->get()); |
vert[i]->tag = i; |
LABEL(i) = i; |
SEMI(i) = ANCESTOR(i) = -1; |
} |
build(); |
delete[] vert; |
delete[] data; |
} |
void DominatorTree::buildDFS(Graph::Node *node) |
{ |
SEMI(node->tag) = node->tag; |
for (Graph::EdgeIterator ei = node->outgoing(); !ei.end(); ei.next()) { |
if (SEMI(ei.getNode()->tag) < 0) { |
buildDFS(ei.getNode()); |
PARENT(ei.getNode()->tag) = node->tag; |
} |
} |
} |
void DominatorTree::squash(int v) |
{ |
if (ANCESTOR(ANCESTOR(v)) >= 0) { |
squash(ANCESTOR(v)); |
if (SEMI(LABEL(ANCESTOR(v))) < SEMI(LABEL(v))) |
LABEL(v) = LABEL(ANCESTOR(v)); |
ANCESTOR(v) = ANCESTOR(ANCESTOR(v)); |
} |
} |
int DominatorTree::eval(int v) |
{ |
if (ANCESTOR(v) < 0) |
return v; |
squash(v); |
return LABEL(v); |
} |
void DominatorTree::link(int v, int w) |
{ |
ANCESTOR(w) = v; |
} |
void DominatorTree::build() |
{ |
DLList *bucket = new DLList[count]; |
Node *nv, *nw; |
int p, u, v, w; |
buildDFS(cfg->getRoot()); |
for (w = count - 1; w >= 1; --w) { |
nw = vert[w]; |
assert(nw->tag == w); |
for (Graph::EdgeIterator ei = nw->incident(); !ei.end(); ei.next()) { |
nv = ei.getNode(); |
v = nv->tag; |
u = eval(v); |
if (SEMI(u) < SEMI(w)) |
SEMI(w) = SEMI(u); |
} |
p = PARENT(w); |
bucket[SEMI(w)].insert(nw); |
link(p, w); |
for (DLList::Iterator it = bucket[p].iterator(); !it.end(); it.erase()) { |
v = reinterpret_cast<Node *>(it.get())->tag; |
u = eval(v); |
DOM(v) = (SEMI(u) < SEMI(v)) ? u : p; |
} |
} |
for (w = 1; w < count; ++w) { |
if (DOM(w) != SEMI(w)) |
DOM(w) = DOM(DOM(w)); |
} |
DOM(0) = 0; |
insert(&BasicBlock::get(cfg->getRoot())->dom); |
do { |
p = 0; |
for (v = 1; v < count; ++v) { |
nw = &BasicBlock::get(vert[DOM(v)])->dom;; |
nv = &BasicBlock::get(vert[v])->dom; |
if (nw->getGraph() && !nv->getGraph()) { |
++p; |
nw->attach(nv, Graph::Edge::TREE); |
} |
} |
} while (p); |
delete[] bucket; |
} |
#undef SEMI |
#undef ANCESTOR |
#undef PARENT |
#undef LABEL |
#undef DOM |
void DominatorTree::findDominanceFrontiers() |
{ |
BasicBlock *bb; |
for (IteratorRef dtIt = iteratorDFS(false); !dtIt->end(); dtIt->next()) { |
EdgeIterator succIt, chldIt; |
bb = BasicBlock::get(reinterpret_cast<Node *>(dtIt->get())); |
bb->getDF().clear(); |
for (succIt = bb->cfg.outgoing(); !succIt.end(); succIt.next()) { |
BasicBlock *dfLocal = BasicBlock::get(succIt.getNode()); |
if (dfLocal->idom() != bb) |
bb->getDF().insert(dfLocal); |
} |
for (chldIt = bb->dom.outgoing(); !chldIt.end(); chldIt.next()) { |
BasicBlock *cb = BasicBlock::get(chldIt.getNode()); |
DLList::Iterator dfIt = cb->getDF().iterator(); |
for (; !dfIt.end(); dfIt.next()) { |
BasicBlock *dfUp = BasicBlock::get(dfIt); |
if (dfUp->idom() != bb) |
bb->getDF().insert(dfUp); |
} |
} |
} |
} |
// liveIn(bb) = usedBeforeAssigned(bb) U (liveOut(bb) - assigned(bb)) |
void |
Function::buildLiveSetsPreSSA(BasicBlock *bb, const int seq) |
{ |
Function *f = bb->getFunction(); |
BitSet usedBeforeAssigned(allLValues.getSize(), true); |
BitSet assigned(allLValues.getSize(), true); |
bb->liveSet.allocate(allLValues.getSize(), false); |
int n = 0; |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
BasicBlock *out = BasicBlock::get(ei.getNode()); |
if (out == bb) |
continue; |
if (out->cfg.visit(seq)) |
buildLiveSetsPreSSA(out, seq); |
if (!n++) |
bb->liveSet = out->liveSet; |
else |
bb->liveSet |= out->liveSet; |
} |
if (!n && !bb->liveSet.marker) |
bb->liveSet.fill(0); |
bb->liveSet.marker = true; |
for (Instruction *i = bb->getEntry(); i; i = i->next) { |
for (int s = 0; i->srcExists(s); ++s) |
if (i->getSrc(s)->asLValue() && !assigned.test(i->getSrc(s)->id)) |
usedBeforeAssigned.set(i->getSrc(s)->id); |
for (int d = 0; i->defExists(d); ++d) |
assigned.set(i->getDef(d)->id); |
} |
if (bb == BasicBlock::get(f->cfgExit)) { |
for (std::deque<ValueRef>::iterator it = f->outs.begin(); |
it != f->outs.end(); ++it) { |
if (!assigned.test(it->get()->id)) |
usedBeforeAssigned.set(it->get()->id); |
} |
} |
bb->liveSet.andNot(assigned); |
bb->liveSet |= usedBeforeAssigned; |
} |
void |
Function::buildDefSetsPreSSA(BasicBlock *bb, const int seq) |
{ |
bb->defSet.allocate(allLValues.getSize(), !bb->liveSet.marker); |
bb->liveSet.marker = true; |
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) { |
BasicBlock *in = BasicBlock::get(ei.getNode()); |
if (in->cfg.visit(seq)) |
buildDefSetsPreSSA(in, seq); |
bb->defSet |= in->defSet; |
} |
for (Instruction *i = bb->getEntry(); i; i = i->next) { |
for (int d = 0; i->defExists(d); ++d) |
bb->defSet.set(i->getDef(d)->id); |
} |
} |
class RenamePass |
{ |
public: |
RenamePass(Function *); |
~RenamePass(); |
bool run(); |
void search(BasicBlock *); |
inline LValue *getStackTop(Value *); |
LValue *mkUndefined(Value *); |
private: |
Stack *stack; |
Function *func; |
Program *prog; |
}; |
bool |
Program::convertToSSA() |
{ |
for (ArrayList::Iterator fi = allFuncs.iterator(); !fi.end(); fi.next()) { |
Function *fn = reinterpret_cast<Function *>(fi.get()); |
if (!fn->convertToSSA()) |
return false; |
} |
return true; |
} |
// XXX: add edge from entry to exit ? |
// Efficiently Computing Static Single Assignment Form and |
// the Control Dependence Graph, |
// R. Cytron, J. Ferrante, B. K. Rosen, M. N. Wegman, F. K. Zadeck |
bool |
Function::convertToSSA() |
{ |
// 0. calculate live in variables (for pruned SSA) |
buildLiveSets(); |
// 1. create the dominator tree |
domTree = new DominatorTree(&cfg); |
reinterpret_cast<DominatorTree *>(domTree)->findDominanceFrontiers(); |
// 2. insert PHI functions |
DLList workList; |
LValue *lval; |
BasicBlock *bb; |
int var; |
int iterCount = 0; |
int *hasAlready = new int[allBBlocks.getSize() * 2]; |
int *work = &hasAlready[allBBlocks.getSize()]; |
memset(hasAlready, 0, allBBlocks.getSize() * 2 * sizeof(int)); |
// for each variable |
for (var = 0; var < allLValues.getSize(); ++var) { |
if (!allLValues.get(var)) |
continue; |
lval = reinterpret_cast<Value *>(allLValues.get(var))->asLValue(); |
if (!lval || lval->defs.empty()) |
continue; |
++iterCount; |
// TODO: don't add phi functions for values that aren't used outside |
// the BB they're defined in |
// gather blocks with assignments to lval in workList |
for (Value::DefIterator d = lval->defs.begin(); |
d != lval->defs.end(); ++d) { |
bb = ((*d)->getInsn() ? (*d)->getInsn()->bb : NULL); |
if (!bb) |
continue; // instruction likely been removed but not XXX deleted |
if (work[bb->getId()] == iterCount) |
continue; |
work[bb->getId()] = iterCount; |
workList.insert(bb); |
} |
// for each block in workList, insert a phi for lval in the block's |
// dominance frontier (if we haven't already done so) |
for (DLList::Iterator wI = workList.iterator(); !wI.end(); wI.erase()) { |
bb = BasicBlock::get(wI); |
DLList::Iterator dfIter = bb->getDF().iterator(); |
for (; !dfIter.end(); dfIter.next()) { |
Instruction *phi; |
BasicBlock *dfBB = BasicBlock::get(dfIter); |
if (hasAlready[dfBB->getId()] >= iterCount) |
continue; |
hasAlready[dfBB->getId()] = iterCount; |
// pruned SSA: don't need a phi if the value is not live-in |
if (!dfBB->liveSet.test(lval->id)) |
continue; |
phi = new_Instruction(this, OP_PHI, typeOfSize(lval->reg.size)); |
dfBB->insertTail(phi); |
phi->setDef(0, lval); |
for (int s = 0; s < dfBB->cfg.incidentCount(); ++s) |
phi->setSrc(s, lval); |
if (work[dfBB->getId()] < iterCount) { |
work[dfBB->getId()] = iterCount; |
wI.insert(dfBB); |
} |
} |
} |
} |
delete[] hasAlready; |
RenamePass rename(this); |
return rename.run(); |
} |
RenamePass::RenamePass(Function *fn) : func(fn), prog(fn->getProgram()) |
{ |
stack = new Stack[func->allLValues.getSize()]; |
} |
RenamePass::~RenamePass() |
{ |
if (stack) |
delete[] stack; |
} |
LValue * |
RenamePass::getStackTop(Value *val) |
{ |
if (!stack[val->id].getSize()) |
return 0; |
return reinterpret_cast<LValue *>(stack[val->id].peek().u.p); |
} |
LValue * |
RenamePass::mkUndefined(Value *val) |
{ |
LValue *lval = val->asLValue(); |
assert(lval); |
LValue *ud = new_LValue(func, lval); |
Instruction *nop = new_Instruction(func, OP_NOP, typeOfSize(lval->reg.size)); |
nop->setDef(0, ud); |
BasicBlock::get(func->cfg.getRoot())->insertHead(nop); |
return ud; |
} |
bool RenamePass::run() |
{ |
if (!stack) |
return false; |
search(BasicBlock::get(func->domTree->getRoot())); |
return true; |
} |
// Go through BBs in dominance order, create new values for each definition, |
// and replace all sources with their current new values. |
// |
// NOTE: The values generated for function inputs/outputs have no connection |
// to their corresponding outputs/inputs in other functions. Only allocation |
// of physical registers will establish this connection. |
// |
void RenamePass::search(BasicBlock *bb) |
{ |
LValue *lval, *ssa; |
int d, s; |
const Target *targ = prog->getTarget(); |
// Put current definitions for function inputs values on the stack. |
// They can be used before any redefinitions are pushed. |
if (bb == BasicBlock::get(func->cfg.getRoot())) { |
for (std::deque<ValueDef>::iterator it = func->ins.begin(); |
it != func->ins.end(); ++it) { |
lval = it->get()->asLValue(); |
assert(lval); |
ssa = new_LValue(func, targ->nativeFile(lval->reg.file)); |
ssa->reg.size = lval->reg.size; |
ssa->reg.data.id = lval->reg.data.id; |
it->setSSA(ssa); |
stack[lval->id].push(ssa); |
} |
} |
for (Instruction *stmt = bb->getFirst(); stmt; stmt = stmt->next) { |
// PHI sources get definitions from the passes through the incident BBs, |
// so skip them here. |
if (stmt->op != OP_PHI) { |
for (s = 0; stmt->srcExists(s); ++s) { |
lval = stmt->getSrc(s)->asLValue(); |
if (!lval) |
continue; |
// Values on the stack created in previously visited blocks, and |
// function inputs, will be valid because they dominate this one. |
lval = getStackTop(lval); |
if (!lval) |
lval = mkUndefined(stmt->getSrc(s)); |
stmt->setSrc(s, lval); |
} |
} |
for (d = 0; stmt->defExists(d); ++d) { |
lval = stmt->def(d).get()->asLValue(); |
assert(lval); |
stmt->def(d).setSSA( |
new_LValue(func, targ->nativeFile(lval->reg.file))); |
stmt->def(d).get()->reg.size = lval->reg.size; |
stmt->def(d).get()->reg.data.id = lval->reg.data.id; |
stack[lval->id].push(stmt->def(d).get()); |
} |
} |
// Update sources of PHI ops corresponding to this BB in outgoing BBs. |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
Instruction *phi; |
int p = 0; |
BasicBlock *sb = BasicBlock::get(ei.getNode()); |
// which predecessor of sb is bb ? |
for (Graph::EdgeIterator ei = sb->cfg.incident(); !ei.end(); ei.next()) { |
if (ei.getNode() == &bb->cfg) |
break; |
++p; |
} |
assert(p < sb->cfg.incidentCount()); |
for (phi = sb->getPhi(); phi && phi->op == OP_PHI; phi = phi->next) { |
lval = getStackTop(phi->getSrc(p)); |
if (!lval) |
lval = mkUndefined(phi->getSrc(p)); |
phi->setSrc(p, lval); |
} |
} |
// Visit the BBs we dominate. |
for (Graph::EdgeIterator ei = bb->dom.outgoing(); !ei.end(); ei.next()) |
search(BasicBlock::get(ei.getNode())); |
// Update function outputs to the last definitions of their pre-SSA values. |
// I hope they're unique, i.e. that we get PHIs for all of them ... |
if (bb == BasicBlock::get(func->cfgExit)) { |
for (std::deque<ValueRef>::iterator it = func->outs.begin(); |
it != func->outs.end(); ++it) { |
lval = it->get()->asLValue(); |
if (!lval) |
continue; |
lval = getStackTop(lval); |
if (!lval) |
lval = mkUndefined(it->get()); |
it->set(lval); |
} |
} |
// Pop the values we created in this block from the stack because we will |
// return to blocks that we do not dominate. |
for (Instruction *stmt = bb->getFirst(); stmt; stmt = stmt->next) { |
if (stmt->op == OP_NOP) |
continue; |
for (d = 0; stmt->defExists(d); ++d) |
stack[stmt->def(d).preSSA()->id].pop(); |
} |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target.cpp |
---|
0,0 → 1,469 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50/codegen/nv50_ir.h" |
#include "nv50/codegen/nv50_ir_target.h" |
namespace nv50_ir { |
const uint8_t Target::operationSrcNr[OP_LAST + 1] = |
{ |
0, 0, // NOP, PHI |
0, 0, 0, 0, // UNION, SPLIT, MERGE, CONSTRAINT |
1, 1, 2, // MOV, LOAD, STORE |
2, 2, 2, 2, 2, 3, 3, 3, // ADD, SUB, MUL, DIV, MOD, MAD, FMA, SAD |
1, 1, 1, // ABS, NEG, NOT |
2, 2, 2, 2, 2, // AND, OR, XOR, SHL, SHR |
2, 2, 1, // MAX, MIN, SAT |
1, 1, 1, 1, // CEIL, FLOOR, TRUNC, CVT |
3, 3, 3, 2, 3, 3, // SET_AND,OR,XOR, SET, SELP, SLCT |
1, 1, 1, 1, 1, 1, // RCP, RSQ, LG2, SIN, COS, EX2 |
1, 1, 1, 1, 1, 2, // EXP, LOG, PRESIN, PREEX2, SQRT, POW |
0, 0, 0, 0, 0, // BRA, CALL, RET, CONT, BREAK, |
0, 0, 0, // PRERET,CONT,BREAK |
0, 0, 0, 0, 0, 0, // BRKPT, JOINAT, JOIN, DISCARD, EXIT, MEMBAR |
1, 1, 2, 1, 2, // VFETCH, PFETCH, EXPORT, LINTERP, PINTERP |
1, 1, // EMIT, RESTART |
1, 1, 1, // TEX, TXB, TXL, |
1, 1, 1, 1, 1, 2, // TXF, TXQ, TXD, TXG, TEXCSAA, TEXPREP |
1, 1, 2, 2, 2, 2, 2, // SULDB, SULDP, SUSTB, SUSTP, SUREDB, SUREDP, SULEA |
3, 3, 3, 3, // SUBFM, SUCLAMP, SUEAU, MADSP |
0, // TEXBAR |
1, 1, // DFDX, DFDY |
1, 2, 2, 0, 0, // RDSV, WRSV, QUADOP, QUADON, QUADPOP |
2, 3, 2, 3, // POPCNT, INSBF, EXTBF, PERMT |
2, 2, // ATOM, BAR |
2, 2, 2, 2, 3, 2, // VADD, VAVG, VMIN, VMAX, VSAD, VSET, |
2, 2, 2, 1, // VSHR, VSHL, VSEL, CCTL |
0 |
}; |
const OpClass Target::operationClass[OP_LAST + 1] = |
{ |
// NOP; PHI; UNION, SPLIT, MERGE, CONSTRAINT |
OPCLASS_OTHER, |
OPCLASS_PSEUDO, |
OPCLASS_PSEUDO, OPCLASS_PSEUDO, OPCLASS_PSEUDO, OPCLASS_PSEUDO, |
// MOV; LOAD; STORE |
OPCLASS_MOVE, |
OPCLASS_LOAD, |
OPCLASS_STORE, |
// ADD, SUB, MUL; DIV, MOD; MAD, FMA, SAD |
OPCLASS_ARITH, OPCLASS_ARITH, OPCLASS_ARITH, |
OPCLASS_ARITH, OPCLASS_ARITH, |
OPCLASS_ARITH, OPCLASS_ARITH, OPCLASS_ARITH, |
// ABS, NEG; NOT, AND, OR, XOR; SHL, SHR |
OPCLASS_CONVERT, OPCLASS_CONVERT, |
OPCLASS_LOGIC, OPCLASS_LOGIC, OPCLASS_LOGIC, OPCLASS_LOGIC, |
OPCLASS_SHIFT, OPCLASS_SHIFT, |
// MAX, MIN |
OPCLASS_COMPARE, OPCLASS_COMPARE, |
// SAT, CEIL, FLOOR, TRUNC; CVT |
OPCLASS_CONVERT, OPCLASS_CONVERT, OPCLASS_CONVERT, OPCLASS_CONVERT, |
OPCLASS_CONVERT, |
// SET(AND,OR,XOR); SELP, SLCT |
OPCLASS_COMPARE, OPCLASS_COMPARE, OPCLASS_COMPARE, OPCLASS_COMPARE, |
OPCLASS_COMPARE, OPCLASS_COMPARE, |
// RCP, RSQ, LG2, SIN, COS; EX2, EXP, LOG, PRESIN, PREEX2; SQRT, POW |
OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, |
OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, OPCLASS_SFU, |
OPCLASS_SFU, OPCLASS_SFU, |
// BRA, CALL, RET; CONT, BREAK, PRE(RET,CONT,BREAK); BRKPT, JOINAT, JOIN |
OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, |
OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, |
OPCLASS_FLOW, OPCLASS_FLOW, OPCLASS_FLOW, |
// DISCARD, EXIT |
OPCLASS_FLOW, OPCLASS_FLOW, |
// MEMBAR |
OPCLASS_CONTROL, |
// VFETCH, PFETCH, EXPORT |
OPCLASS_LOAD, OPCLASS_OTHER, OPCLASS_STORE, |
// LINTERP, PINTERP |
OPCLASS_SFU, OPCLASS_SFU, |
// EMIT, RESTART |
OPCLASS_CONTROL, OPCLASS_CONTROL, |
// TEX, TXB, TXL, TXF; TXQ, TXD, TXG, TEXCSAA; TEXPREP |
OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, |
OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, OPCLASS_TEXTURE, |
OPCLASS_TEXTURE, |
// SULDB, SULDP, SUSTB, SUSTP; SUREDB, SUREDP, SULEA |
OPCLASS_SURFACE, OPCLASS_SURFACE, OPCLASS_ATOMIC, OPCLASS_SURFACE, |
OPCLASS_SURFACE, OPCLASS_SURFACE, OPCLASS_SURFACE, |
// SUBFM, SUCLAMP, SUEAU, MADSP |
OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_ARITH, |
// TEXBAR |
OPCLASS_OTHER, |
// DFDX, DFDY, RDSV, WRSV; QUADOP, QUADON, QUADPOP |
OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, OPCLASS_OTHER, |
OPCLASS_OTHER, OPCLASS_CONTROL, OPCLASS_CONTROL, |
// POPCNT, INSBF, EXTBF, PERMT |
OPCLASS_BITFIELD, OPCLASS_BITFIELD, OPCLASS_BITFIELD, OPCLASS_BITFIELD, |
// ATOM, BAR |
OPCLASS_ATOMIC, OPCLASS_CONTROL, |
// VADD, VAVG, VMIN, VMAX |
OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR, |
// VSAD, VSET, VSHR, VSHL |
OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR, OPCLASS_VECTOR, |
// VSEL, CCTL |
OPCLASS_VECTOR, OPCLASS_CONTROL, |
OPCLASS_PSEUDO // LAST |
}; |
extern Target *getTargetNVC0(unsigned int chipset); |
extern Target *getTargetNV50(unsigned int chipset); |
Target *Target::create(unsigned int chipset) |
{ |
switch (chipset & 0xf0) { |
case 0xc0: |
case 0xd0: |
case 0xe0: |
case NVISA_GK110_CHIPSET: |
return getTargetNVC0(chipset); |
case 0x50: |
case 0x80: |
case 0x90: |
case 0xa0: |
return getTargetNV50(chipset); |
default: |
ERROR("unsupported target: NV%x\n", chipset); |
return 0; |
} |
} |
void Target::destroy(Target *targ) |
{ |
delete targ; |
} |
CodeEmitter::CodeEmitter(const Target *target) : targ(target) |
{ |
} |
void |
CodeEmitter::setCodeLocation(void *ptr, uint32_t size) |
{ |
code = reinterpret_cast<uint32_t *>(ptr); |
codeSize = 0; |
codeSizeLimit = size; |
} |
void |
CodeEmitter::printBinary() const |
{ |
uint32_t *bin = code - codeSize / 4; |
INFO("program binary (%u bytes)", codeSize); |
for (unsigned int pos = 0; pos < codeSize / 4; ++pos) { |
if ((pos % 8) == 0) |
INFO("\n"); |
INFO("%08x ", bin[pos]); |
} |
INFO("\n"); |
} |
static inline uint32_t sizeToBundlesNVE4(uint32_t size) |
{ |
return (size + 55) / 56; |
} |
void |
CodeEmitter::prepareEmission(Program *prog) |
{ |
for (ArrayList::Iterator fi = prog->allFuncs.iterator(); |
!fi.end(); fi.next()) { |
Function *func = reinterpret_cast<Function *>(fi.get()); |
func->binPos = prog->binSize; |
prepareEmission(func); |
// adjust sizes & positions for schedulding info: |
if (prog->getTarget()->hasSWSched) { |
uint32_t adjPos = func->binPos; |
BasicBlock *bb = NULL; |
for (int i = 0; i < func->bbCount; ++i) { |
bb = func->bbArray[i]; |
int32_t adjSize = bb->binSize; |
if (adjPos % 64) { |
adjSize -= 64 - adjPos % 64; |
if (adjSize < 0) |
adjSize = 0; |
} |
adjSize = bb->binSize + sizeToBundlesNVE4(adjSize) * 8; |
bb->binPos = adjPos; |
bb->binSize = adjSize; |
adjPos += adjSize; |
} |
if (bb) |
func->binSize = adjPos - func->binPos; |
} |
prog->binSize += func->binSize; |
} |
} |
void |
CodeEmitter::prepareEmission(Function *func) |
{ |
func->bbCount = 0; |
func->bbArray = new BasicBlock * [func->cfg.getSize()]; |
BasicBlock::get(func->cfg.getRoot())->binPos = func->binPos; |
for (IteratorRef it = func->cfg.iteratorCFG(); !it->end(); it->next()) |
prepareEmission(BasicBlock::get(*it)); |
} |
void |
CodeEmitter::prepareEmission(BasicBlock *bb) |
{ |
Instruction *i, *next; |
Function *func = bb->getFunction(); |
int j; |
unsigned int nShort; |
for (j = func->bbCount - 1; j >= 0 && !func->bbArray[j]->binSize; --j); |
for (; j >= 0; --j) { |
BasicBlock *in = func->bbArray[j]; |
Instruction *exit = in->getExit(); |
if (exit && exit->op == OP_BRA && exit->asFlow()->target.bb == bb) { |
in->binSize -= 8; |
func->binSize -= 8; |
for (++j; j < func->bbCount; ++j) |
func->bbArray[j]->binPos -= 8; |
in->remove(exit); |
} |
bb->binPos = in->binPos + in->binSize; |
if (in->binSize) // no more no-op branches to bb |
break; |
} |
func->bbArray[func->bbCount++] = bb; |
if (!bb->getExit()) |
return; |
// determine encoding size, try to group short instructions |
nShort = 0; |
for (i = bb->getEntry(); i; i = next) { |
next = i->next; |
if (i->op == OP_MEMBAR && !targ->isOpSupported(OP_MEMBAR, TYPE_NONE)) { |
bb->remove(i); |
continue; |
} |
i->encSize = getMinEncodingSize(i); |
if (next && i->encSize < 8) |
++nShort; |
else |
if ((nShort & 1) && next && getMinEncodingSize(next) == 4) { |
if (i->isCommutationLegal(i->next)) { |
bb->permuteAdjacent(i, next); |
next->encSize = 4; |
next = i; |
i = i->prev; |
++nShort; |
} else |
if (i->isCommutationLegal(i->prev) && next->next) { |
bb->permuteAdjacent(i->prev, i); |
next->encSize = 4; |
next = next->next; |
bb->binSize += 4; |
++nShort; |
} else { |
i->encSize = 8; |
i->prev->encSize = 8; |
bb->binSize += 4; |
nShort = 0; |
} |
} else { |
i->encSize = 8; |
if (nShort & 1) { |
i->prev->encSize = 8; |
bb->binSize += 4; |
} |
nShort = 0; |
} |
bb->binSize += i->encSize; |
} |
if (bb->getExit()->encSize == 4) { |
assert(nShort); |
bb->getExit()->encSize = 8; |
bb->binSize += 4; |
if ((bb->getExit()->prev->encSize == 4) && !(nShort & 1)) { |
bb->binSize += 8; |
bb->getExit()->prev->encSize = 8; |
} |
} |
assert(!bb->getEntry() || (bb->getExit() && bb->getExit()->encSize == 8)); |
func->binSize += bb->binSize; |
} |
void |
Program::emitSymbolTable(struct nv50_ir_prog_info *info) |
{ |
unsigned int n = 0, nMax = allFuncs.getSize(); |
info->bin.syms = |
(struct nv50_ir_prog_symbol *)MALLOC(nMax * sizeof(*info->bin.syms)); |
for (ArrayList::Iterator fi = allFuncs.iterator(); |
!fi.end(); |
fi.next(), ++n) { |
Function *f = (Function *)fi.get(); |
assert(n < nMax); |
info->bin.syms[n].label = f->getLabel(); |
info->bin.syms[n].offset = f->binPos; |
} |
info->bin.numSyms = n; |
} |
bool |
Program::emitBinary(struct nv50_ir_prog_info *info) |
{ |
CodeEmitter *emit = target->getCodeEmitter(progType); |
emit->prepareEmission(this); |
if (dbgFlags & NV50_IR_DEBUG_BASIC) |
this->print(); |
if (!binSize) { |
code = NULL; |
return false; |
} |
code = reinterpret_cast<uint32_t *>(MALLOC(binSize)); |
if (!code) |
return false; |
emit->setCodeLocation(code, binSize); |
for (ArrayList::Iterator fi = allFuncs.iterator(); !fi.end(); fi.next()) { |
Function *fn = reinterpret_cast<Function *>(fi.get()); |
assert(emit->getCodeSize() == fn->binPos); |
for (int b = 0; b < fn->bbCount; ++b) |
for (Instruction *i = fn->bbArray[b]->getEntry(); i; i = i->next) |
emit->emitInstruction(i); |
} |
info->bin.relocData = emit->getRelocInfo(); |
emitSymbolTable(info); |
// the nvc0 driver will print the binary iself together with the header |
if ((dbgFlags & NV50_IR_DEBUG_BASIC) && getTarget()->getChipset() < 0xc0) |
emit->printBinary(); |
delete emit; |
return true; |
} |
#define RELOC_ALLOC_INCREMENT 8 |
bool |
CodeEmitter::addReloc(RelocEntry::Type ty, int w, uint32_t data, uint32_t m, |
int s) |
{ |
unsigned int n = relocInfo ? relocInfo->count : 0; |
if (!(n % RELOC_ALLOC_INCREMENT)) { |
size_t size = sizeof(RelocInfo) + n * sizeof(RelocEntry); |
relocInfo = reinterpret_cast<RelocInfo *>( |
REALLOC(relocInfo, n ? size : 0, |
size + RELOC_ALLOC_INCREMENT * sizeof(RelocEntry))); |
if (!relocInfo) |
return false; |
if (n == 0) |
memset(relocInfo, 0, sizeof(RelocInfo)); |
} |
++relocInfo->count; |
relocInfo->entry[n].data = data; |
relocInfo->entry[n].mask = m; |
relocInfo->entry[n].offset = codeSize + w * 4; |
relocInfo->entry[n].bitPos = s; |
relocInfo->entry[n].type = ty; |
return true; |
} |
void |
RelocEntry::apply(uint32_t *binary, const RelocInfo *info) const |
{ |
uint32_t value = 0; |
switch (type) { |
case TYPE_CODE: value = info->codePos; break; |
case TYPE_BUILTIN: value = info->libPos; break; |
case TYPE_DATA: value = info->dataPos; break; |
default: |
assert(0); |
break; |
} |
value += data; |
value = (bitPos < 0) ? (value >> -bitPos) : (value << bitPos); |
binary[offset / 4] &= ~mask; |
binary[offset / 4] |= value & mask; |
} |
} // namespace nv50_ir |
#include "nv50/codegen/nv50_ir_driver.h" |
extern "C" { |
void |
nv50_ir_relocate_code(void *relocData, uint32_t *code, |
uint32_t codePos, |
uint32_t libPos, |
uint32_t dataPos) |
{ |
nv50_ir::RelocInfo *info = reinterpret_cast<nv50_ir::RelocInfo *>(relocData); |
info->codePos = codePos; |
info->libPos = libPos; |
info->dataPos = dataPos; |
for (unsigned int i = 0; i < info->count; ++i) |
info->entry[i].apply(code, info); |
} |
void |
nv50_ir_get_target_library(uint32_t chipset, |
const uint32_t **code, uint32_t *size) |
{ |
nv50_ir::Target *targ = nv50_ir::Target::create(chipset); |
targ->getBuiltinCode(code, size); |
nv50_ir::Target::destroy(targ); |
} |
} |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target.h |
---|
0,0 → 1,235 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_TARGET_H__ |
#define __NV50_IR_TARGET_H__ |
#include "nv50_ir.h" |
namespace nv50_ir { |
struct RelocInfo; |
struct RelocEntry |
{ |
enum Type |
{ |
TYPE_CODE, |
TYPE_BUILTIN, |
TYPE_DATA |
}; |
uint32_t data; |
uint32_t mask; |
uint32_t offset; |
int8_t bitPos; |
Type type; |
inline void apply(uint32_t *binary, const RelocInfo *info) const; |
}; |
struct RelocInfo |
{ |
uint32_t codePos; |
uint32_t libPos; |
uint32_t dataPos; |
uint32_t count; |
RelocEntry entry[0]; |
}; |
class CodeEmitter |
{ |
public: |
CodeEmitter(const Target *); |
virtual ~CodeEmitter() { } |
// returns whether the instruction was encodable and written |
virtual bool emitInstruction(Instruction *) = 0; |
virtual uint32_t getMinEncodingSize(const Instruction *) const = 0; |
void setCodeLocation(void *, uint32_t size); |
inline void *getCodeLocation() const { return code; } |
inline uint32_t getCodeSize() const { return codeSize; } |
bool addReloc(RelocEntry::Type, int w, uint32_t data, uint32_t m, |
int s); |
inline void *getRelocInfo() const { return relocInfo; } |
void prepareEmission(Program *); |
virtual void prepareEmission(Function *); |
virtual void prepareEmission(BasicBlock *); |
void printBinary() const; |
protected: |
const Target *targ; |
uint32_t *code; |
uint32_t codeSize; |
uint32_t codeSizeLimit; |
RelocInfo *relocInfo; |
}; |
enum OpClass |
{ |
OPCLASS_MOVE = 0, |
OPCLASS_LOAD = 1, |
OPCLASS_STORE = 2, |
OPCLASS_ARITH = 3, |
OPCLASS_SHIFT = 4, |
OPCLASS_SFU = 5, |
OPCLASS_LOGIC = 6, |
OPCLASS_COMPARE = 7, |
OPCLASS_CONVERT = 8, |
OPCLASS_ATOMIC = 9, |
OPCLASS_TEXTURE = 10, |
OPCLASS_SURFACE = 11, |
OPCLASS_FLOW = 12, |
OPCLASS_PSEUDO = 14, |
OPCLASS_VECTOR = 15, |
OPCLASS_BITFIELD = 16, |
OPCLASS_CONTROL = 17, |
OPCLASS_OTHER = 18 |
}; |
class Target |
{ |
public: |
Target(bool j, bool s) : joinAnterior(j), hasSWSched(s) { } |
virtual ~Target() { } |
static Target *create(uint32_t chipset); |
static void destroy(Target *); |
// 0x50 and 0x84 to 0xaf for nv50 |
// 0xc0 to 0xdf for nvc0 |
inline uint32_t getChipset() const { return chipset; } |
virtual CodeEmitter *getCodeEmitter(Program::Type) = 0; |
// Drivers should upload this so we can use it from all programs. |
// The address chosen is supplied to the relocation routine. |
virtual void getBuiltinCode(const uint32_t **code, uint32_t *size) const = 0; |
virtual void parseDriverInfo(const struct nv50_ir_prog_info *info) { } |
virtual bool runLegalizePass(Program *, CGStage stage) const = 0; |
public: |
struct OpInfo |
{ |
OpInfo *variants; |
operation op; |
uint16_t srcTypes; |
uint16_t dstTypes; |
uint32_t immdBits; |
uint8_t srcNr; |
uint8_t srcMods[3]; |
uint8_t dstMods; |
uint8_t srcFiles[3]; |
uint8_t dstFiles; |
unsigned int minEncSize : 4; |
unsigned int vector : 1; |
unsigned int predicate : 1; |
unsigned int commutative : 1; |
unsigned int pseudo : 1; |
unsigned int flow : 1; |
unsigned int hasDest : 1; |
unsigned int terminator : 1; |
}; |
inline const OpInfo& getOpInfo(const Instruction *) const; |
inline const OpInfo& getOpInfo(const operation) const; |
inline DataFile nativeFile(DataFile f) const; |
virtual bool insnCanLoad(const Instruction *insn, int s, |
const Instruction *ld) const = 0; |
virtual bool isOpSupported(operation, DataType) const = 0; |
virtual bool isAccessSupported(DataFile, DataType) const = 0; |
virtual bool isModSupported(const Instruction *, |
int s, Modifier) const = 0; |
virtual bool isSatSupported(const Instruction *) const = 0; |
virtual bool isPostMultiplySupported(operation op, float f, |
int& e) const { return false; } |
virtual bool mayPredicate(const Instruction *, |
const Value *) const = 0; |
// whether @insn can be issued together with @next (order matters) |
virtual bool canDualIssue(const Instruction *insn, |
const Instruction *next) const { return false; } |
virtual int getLatency(const Instruction *) const { return 1; } |
virtual int getThroughput(const Instruction *) const { return 1; } |
virtual unsigned int getFileSize(DataFile) const = 0; |
virtual unsigned int getFileUnit(DataFile) const = 0; |
virtual uint32_t getSVAddress(DataFile, const Symbol *) const = 0; |
public: |
const bool joinAnterior; // true if join is executed before the op |
const bool hasSWSched; // true if code should provide scheduling data |
static const uint8_t operationSrcNr[OP_LAST + 1]; |
static const OpClass operationClass[OP_LAST + 1]; |
static inline uint8_t getOpSrcNr(operation op) |
{ |
return operationSrcNr[op]; |
} |
static inline OpClass getOpClass(operation op) |
{ |
return operationClass[op]; |
} |
protected: |
uint32_t chipset; |
DataFile nativeFileMap[DATA_FILE_COUNT]; |
OpInfo opInfo[OP_LAST + 1]; |
}; |
const Target::OpInfo& Target::getOpInfo(const Instruction *insn) const |
{ |
return opInfo[MIN2(insn->op, OP_LAST)]; |
} |
const Target::OpInfo& Target::getOpInfo(const operation op) const |
{ |
return opInfo[op]; |
} |
inline DataFile Target::nativeFile(DataFile f) const |
{ |
return nativeFileMap[f]; |
} |
} // namespace nv50_ir |
#endif // __NV50_IR_TARGET_H__ |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.cpp |
---|
0,0 → 1,552 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir_target_nv50.h" |
namespace nv50_ir { |
Target *getTargetNV50(unsigned int chipset) |
{ |
return new TargetNV50(chipset); |
} |
TargetNV50::TargetNV50(unsigned int card) : Target(true, false) |
{ |
chipset = card; |
wposMask = 0; |
for (unsigned int i = 0; i <= SV_LAST; ++i) |
sysvalLocation[i] = ~0; |
initOpInfo(); |
} |
#if 0 |
// BULTINS / LIBRARY FUNCTIONS: |
// TODO |
static const uint32_t nvc0_builtin_code[] = |
{ |
}; |
static const uint16_t nvc0_builtin_offsets[NV50_BUILTIN_COUNT] = |
{ |
}; |
#endif |
void |
TargetNV50::getBuiltinCode(const uint32_t **code, uint32_t *size) const |
{ |
*code = NULL; |
*size = 0; |
} |
uint32_t |
TargetNV50::getBuiltinOffset(int builtin) const |
{ |
return 0; |
} |
struct opProperties |
{ |
operation op; |
unsigned int mNeg : 4; |
unsigned int mAbs : 4; |
unsigned int mNot : 4; |
unsigned int mSat : 4; |
unsigned int fConst : 3; |
unsigned int fShared : 3; |
unsigned int fAttrib : 3; |
unsigned int fImm : 3; |
}; |
static const struct opProperties _initProps[] = |
{ |
// neg abs not sat c[] s[], a[], imm |
{ OP_ADD, 0x3, 0x0, 0x0, 0x8, 0x2, 0x1, 0x1, 0x2 }, |
{ OP_SUB, 0x3, 0x0, 0x0, 0x0, 0x2, 0x1, 0x1, 0x2 }, |
{ OP_MUL, 0x3, 0x0, 0x0, 0x0, 0x2, 0x1, 0x1, 0x2 }, |
{ OP_MAX, 0x3, 0x3, 0x0, 0x0, 0x2, 0x1, 0x1, 0x0 }, |
{ OP_MIN, 0x3, 0x3, 0x0, 0x0, 0x2, 0x1, 0x1, 0x0 }, |
{ OP_MAD, 0x7, 0x0, 0x0, 0x0, 0x6, 0x1, 0x1, 0x0 }, // special constraint |
{ OP_ABS, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0 }, |
{ OP_NEG, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0 }, |
{ OP_CVT, 0x1, 0x1, 0x0, 0x8, 0x0, 0x1, 0x1, 0x0 }, |
{ OP_AND, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x2 }, |
{ OP_OR, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x2 }, |
{ OP_XOR, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x2 }, |
{ OP_SHL, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 }, |
{ OP_SHR, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2 }, |
{ OP_SET, 0x3, 0x3, 0x0, 0x0, 0x2, 0x1, 0x1, 0x0 }, |
{ OP_PREEX2, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
{ OP_PRESIN, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
{ OP_LG2, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
{ OP_RCP, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
{ OP_RSQ, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
{ OP_DFDX, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
{ OP_DFDY, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }, |
}; |
void TargetNV50::initOpInfo() |
{ |
unsigned int i, j; |
static const uint32_t commutative[(OP_LAST + 31) / 32] = |
{ |
// ADD,MAD,MUL,AND,OR,XOR,MAX,MIN |
0x0670ca00, 0x0000003f, 0x00000000, 0x00000000 |
}; |
static const uint32_t shortForm[(OP_LAST + 31) / 32] = |
{ |
// MOV,ADD,SUB,MUL,SAD,L/PINTERP,RCP,TEX,TXF |
0x00010e40, 0x00000040, 0x00000498, 0x00000000 |
}; |
static const operation noDestList[] = |
{ |
OP_STORE, OP_WRSV, OP_EXPORT, OP_BRA, OP_CALL, OP_RET, OP_EXIT, |
OP_DISCARD, OP_CONT, OP_BREAK, OP_PRECONT, OP_PREBREAK, OP_PRERET, |
OP_JOIN, OP_JOINAT, OP_BRKPT, OP_MEMBAR, OP_EMIT, OP_RESTART, |
OP_QUADON, OP_QUADPOP, OP_TEXBAR, OP_SUSTB, OP_SUSTP, OP_SUREDP, |
OP_SUREDB, OP_BAR |
}; |
static const operation noPredList[] = |
{ |
OP_CALL, OP_PREBREAK, OP_PRERET, OP_QUADON, OP_QUADPOP, OP_JOINAT |
}; |
for (i = 0; i < DATA_FILE_COUNT; ++i) |
nativeFileMap[i] = (DataFile)i; |
nativeFileMap[FILE_PREDICATE] = FILE_FLAGS; |
for (i = 0; i < OP_LAST; ++i) { |
opInfo[i].variants = NULL; |
opInfo[i].op = (operation)i; |
opInfo[i].srcTypes = 1 << (int)TYPE_F32; |
opInfo[i].dstTypes = 1 << (int)TYPE_F32; |
opInfo[i].immdBits = 0xffffffff; |
opInfo[i].srcNr = operationSrcNr[i]; |
for (j = 0; j < opInfo[i].srcNr; ++j) { |
opInfo[i].srcMods[j] = 0; |
opInfo[i].srcFiles[j] = 1 << (int)FILE_GPR; |
} |
opInfo[i].dstMods = 0; |
opInfo[i].dstFiles = 1 << (int)FILE_GPR; |
opInfo[i].hasDest = 1; |
opInfo[i].vector = (i >= OP_TEX && i <= OP_TEXCSAA); |
opInfo[i].commutative = (commutative[i / 32] >> (i % 32)) & 1; |
opInfo[i].pseudo = (i < OP_MOV); |
opInfo[i].predicate = !opInfo[i].pseudo; |
opInfo[i].flow = (i >= OP_BRA && i <= OP_JOIN); |
opInfo[i].minEncSize = (shortForm[i / 32] & (1 << (i % 32))) ? 4 : 8; |
} |
for (i = 0; i < sizeof(noDestList) / sizeof(noDestList[0]); ++i) |
opInfo[noDestList[i]].hasDest = 0; |
for (i = 0; i < sizeof(noPredList) / sizeof(noPredList[0]); ++i) |
opInfo[noPredList[i]].predicate = 0; |
for (i = 0; i < sizeof(_initProps) / sizeof(_initProps[0]); ++i) { |
const struct opProperties *prop = &_initProps[i]; |
for (int s = 0; s < 3; ++s) { |
if (prop->mNeg & (1 << s)) |
opInfo[prop->op].srcMods[s] |= NV50_IR_MOD_NEG; |
if (prop->mAbs & (1 << s)) |
opInfo[prop->op].srcMods[s] |= NV50_IR_MOD_ABS; |
if (prop->mNot & (1 << s)) |
opInfo[prop->op].srcMods[s] |= NV50_IR_MOD_NOT; |
if (prop->fConst & (1 << s)) |
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_MEMORY_CONST; |
if (prop->fShared & (1 << s)) |
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_MEMORY_SHARED; |
if (prop->fAttrib & (1 << s)) |
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_SHADER_INPUT; |
if (prop->fImm & (1 << s)) |
opInfo[prop->op].srcFiles[s] |= 1 << (int)FILE_IMMEDIATE; |
} |
if (prop->mSat & 8) |
opInfo[prop->op].dstMods = NV50_IR_MOD_SAT; |
} |
} |
unsigned int |
TargetNV50::getFileSize(DataFile file) const |
{ |
switch (file) { |
case FILE_NULL: return 0; |
case FILE_GPR: return 256; // in 16-bit units ** |
case FILE_PREDICATE: return 0; |
case FILE_FLAGS: return 4; |
case FILE_ADDRESS: return 4; |
case FILE_IMMEDIATE: return 0; |
case FILE_MEMORY_CONST: return 65536; |
case FILE_SHADER_INPUT: return 0x200; |
case FILE_SHADER_OUTPUT: return 0x200; |
case FILE_MEMORY_GLOBAL: return 0xffffffff; |
case FILE_MEMORY_SHARED: return 16 << 10; |
case FILE_MEMORY_LOCAL: return 48 << 10; |
case FILE_SYSTEM_VALUE: return 16; |
default: |
assert(!"invalid file"); |
return 0; |
} |
// ** only first 128 units encodable for 16-bit regs |
} |
unsigned int |
TargetNV50::getFileUnit(DataFile file) const |
{ |
if (file == FILE_GPR || file == FILE_ADDRESS) |
return 1; |
if (file == FILE_SYSTEM_VALUE) |
return 2; |
return 0; |
} |
uint32_t |
TargetNV50::getSVAddress(DataFile shaderFile, const Symbol *sym) const |
{ |
switch (sym->reg.data.sv.sv) { |
case SV_FACE: |
return 0x3fc; |
case SV_POSITION: |
{ |
uint32_t addr = sysvalLocation[sym->reg.data.sv.sv]; |
for (int c = 0; c < sym->reg.data.sv.index; ++c) |
if (wposMask & (1 << c)) |
addr += 4; |
return addr; |
} |
case SV_NCTAID: |
return 0x8 + 2 * sym->reg.data.sv.index; |
case SV_CTAID: |
return 0xc + 2 * sym->reg.data.sv.index; |
case SV_NTID: |
return 0x2 + 2 * sym->reg.data.sv.index; |
case SV_TID: |
return 0; |
default: |
return sysvalLocation[sym->reg.data.sv.sv]; |
} |
} |
// long: rrr, arr, rcr, acr, rrc, arc, gcr, grr |
// short: rr, ar, rc, gr |
// immd: ri, gi |
bool |
TargetNV50::insnCanLoad(const Instruction *i, int s, |
const Instruction *ld) const |
{ |
DataFile sf = ld->src(0).getFile(); |
if (sf == FILE_IMMEDIATE && (i->predSrc >= 0 || i->flagsDef >= 0)) |
return false; |
if (s >= opInfo[i->op].srcNr) |
return false; |
if (!(opInfo[i->op].srcFiles[s] & (1 << (int)sf))) |
return false; |
if (s == 2 && i->src(1).getFile() != FILE_GPR) |
return false; |
// NOTE: don't rely on flagsDef |
for (int d = 0; i->defExists(d); ++d) |
if (i->def(d).getFile() == FILE_FLAGS) |
return false; |
unsigned mode = 0; |
for (int z = 0; z < Target::operationSrcNr[i->op]; ++z) { |
DataFile zf = (z == s) ? sf : i->src(z).getFile(); |
switch (zf) { |
case FILE_GPR: |
break; |
case FILE_MEMORY_SHARED: |
case FILE_SHADER_INPUT: |
mode |= 1 << (z * 2); |
break; |
case FILE_MEMORY_CONST: |
mode |= 2 << (z * 2); |
break; |
case FILE_IMMEDIATE: |
mode |= 3 << (z * 2); |
default: |
break; |
} |
} |
switch (mode) { |
case 0x00: |
case 0x01: |
case 0x03: |
case 0x08: |
case 0x09: |
case 0x0c: |
case 0x20: |
case 0x21: |
break; |
case 0x0d: |
if (ld->bb->getProgram()->getType() != Program::TYPE_GEOMETRY) |
return false; |
default: |
return false; |
} |
uint8_t ldSize; |
if ((i->op == OP_MUL || i->op == OP_MAD) && !isFloatType(i->dType)) { |
// 32-bit MUL will be split into 16-bit MULs |
if (ld->src(0).isIndirect(0)) |
return false; |
if (sf == FILE_IMMEDIATE) |
return false; |
ldSize = 2; |
} else { |
ldSize = typeSizeof(ld->dType); |
} |
if (sf == FILE_IMMEDIATE) |
return true; |
// Check if memory access is encodable: |
if (ldSize < 4 && sf == FILE_SHADER_INPUT) // no < 4-byte aligned a[] access |
return false; |
if (ld->getSrc(0)->reg.data.offset > (int32_t)(127 * ldSize)) |
return false; |
if (ld->src(0).isIndirect(0)) { |
for (int z = 0; i->srcExists(z); ++z) |
if (i->src(z).isIndirect(0)) |
return false; |
// s[] access only possible in CP, $aX always applies |
if (sf == FILE_MEMORY_SHARED) |
return true; |
if (!ld->bb) // can't check type ... |
return false; |
Program::Type pt = ld->bb->getProgram()->getType(); |
// $aX applies to c[] only in VP, FP, GP if p[] is not accessed |
if (pt == Program::TYPE_COMPUTE) |
return false; |
if (pt == Program::TYPE_GEOMETRY) { |
if (sf == FILE_MEMORY_CONST) |
return i->src(s).getFile() != FILE_SHADER_INPUT; |
return sf == FILE_SHADER_INPUT; |
} |
return sf == FILE_MEMORY_CONST; |
} |
return true; |
} |
bool |
TargetNV50::isAccessSupported(DataFile file, DataType ty) const |
{ |
if (ty == TYPE_B96 || ty == TYPE_NONE) |
return false; |
if (typeSizeof(ty) > 4) |
return (file == FILE_MEMORY_LOCAL) || (file == FILE_MEMORY_GLOBAL); |
return true; |
} |
bool |
TargetNV50::isOpSupported(operation op, DataType ty) const |
{ |
if (ty == TYPE_F64 && chipset < 0xa0) |
return false; |
switch (op) { |
case OP_PRERET: |
return chipset >= 0xa0; |
case OP_TXG: |
return chipset >= 0xa3; |
case OP_POW: |
case OP_SQRT: |
case OP_DIV: |
case OP_MOD: |
case OP_SET_AND: |
case OP_SET_OR: |
case OP_SET_XOR: |
case OP_SLCT: |
case OP_SELP: |
case OP_POPCNT: |
case OP_INSBF: |
case OP_EXTBF: |
case OP_EXIT: // want exit modifier instead (on NOP if required) |
case OP_MEMBAR: |
return false; |
case OP_SAD: |
return ty == TYPE_S32; |
default: |
return true; |
} |
} |
bool |
TargetNV50::isModSupported(const Instruction *insn, int s, Modifier mod) const |
{ |
if (!isFloatType(insn->dType)) { |
switch (insn->op) { |
case OP_ABS: |
case OP_NEG: |
case OP_CVT: |
case OP_CEIL: |
case OP_FLOOR: |
case OP_TRUNC: |
case OP_AND: |
case OP_OR: |
case OP_XOR: |
break; |
case OP_ADD: |
if (insn->src(s ? 0 : 1).mod.neg()) |
return false; |
break; |
case OP_SUB: |
if (s == 0) |
return insn->src(1).mod.neg() ? false : true; |
break; |
case OP_SET: |
if (insn->sType != TYPE_F32) |
return false; |
break; |
default: |
return false; |
} |
} |
if (s > 3) |
return false; |
return (mod & Modifier(opInfo[insn->op].srcMods[s])) == mod; |
} |
bool |
TargetNV50::mayPredicate(const Instruction *insn, const Value *pred) const |
{ |
if (insn->getPredicate() || insn->flagsSrc >= 0) |
return false; |
for (int s = 0; insn->srcExists(s); ++s) |
if (insn->src(s).getFile() == FILE_IMMEDIATE) |
return false; |
return opInfo[insn->op].predicate; |
} |
bool |
TargetNV50::isSatSupported(const Instruction *insn) const |
{ |
if (insn->op == OP_CVT) |
return true; |
if (insn->dType != TYPE_F32) |
return false; |
return opInfo[insn->op].dstMods & NV50_IR_MOD_SAT; |
} |
int TargetNV50::getLatency(const Instruction *i) const |
{ |
// TODO: tune these values |
if (i->op == OP_LOAD) { |
switch (i->src(0).getFile()) { |
case FILE_MEMORY_LOCAL: |
case FILE_MEMORY_GLOBAL: |
return 100; // really 400 to 800 |
default: |
return 22; |
} |
} |
return 22; |
} |
// These are "inverse" throughput values, i.e. the number of cycles required |
// to issue a specific instruction for a full warp (32 threads). |
// |
// Assuming we have more than 1 warp in flight, a higher issue latency results |
// in a lower result latency since the MP will have spent more time with other |
// warps. |
// This also helps to determine the number of cycles between instructions in |
// a single warp. |
// |
int TargetNV50::getThroughput(const Instruction *i) const |
{ |
// TODO: tune these values |
if (i->dType == TYPE_F32) { |
switch (i->op) { |
case OP_RCP: |
case OP_RSQ: |
case OP_LG2: |
case OP_SIN: |
case OP_COS: |
case OP_PRESIN: |
case OP_PREEX2: |
return 16; |
default: |
return 4; |
} |
} else |
if (i->dType == TYPE_U32 || i->dType == TYPE_S32) { |
return 4; |
} else |
if (i->dType == TYPE_F64) { |
return 32; |
} else { |
return 1; |
} |
} |
static void |
recordLocation(uint16_t *locs, uint8_t *masks, |
const struct nv50_ir_varying *var) |
{ |
uint16_t addr = var->slot[0] * 4; |
switch (var->sn) { |
case TGSI_SEMANTIC_POSITION: locs[SV_POSITION] = addr; break; |
case TGSI_SEMANTIC_INSTANCEID: locs[SV_INSTANCE_ID] = addr; break; |
case TGSI_SEMANTIC_VERTEXID: locs[SV_VERTEX_ID] = addr; break; |
case TGSI_SEMANTIC_PRIMID: locs[SV_PRIMITIVE_ID] = addr; break; |
case NV50_SEMANTIC_LAYER: locs[SV_LAYER] = addr; break; |
case NV50_SEMANTIC_VIEWPORTINDEX: locs[SV_VIEWPORT_INDEX] = addr; break; |
default: |
break; |
} |
if (var->sn == TGSI_SEMANTIC_POSITION && masks) |
masks[0] = var->mask; |
} |
void |
TargetNV50::parseDriverInfo(const struct nv50_ir_prog_info *info) |
{ |
unsigned int i; |
for (i = 0; i < info->numOutputs; ++i) |
recordLocation(sysvalLocation, NULL, &info->out[i]); |
for (i = 0; i < info->numInputs; ++i) |
recordLocation(sysvalLocation, &wposMask, &info->in[i]); |
for (i = 0; i < info->numSysVals; ++i) |
recordLocation(sysvalLocation, NULL, &info->sv[i]); |
if (sysvalLocation[SV_POSITION] >= 0x200) { |
// not assigned by driver, but we need it internally |
wposMask = 0x8; |
sysvalLocation[SV_POSITION] = 0; |
} |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_target_nv50.h |
---|
0,0 → 1,72 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50/codegen/nv50_ir_target.h" |
namespace nv50_ir { |
#define NVC0_BUILTIN_DIV_U32 0 |
#define NVC0_BUILTIN_DIV_S32 1 |
#define NVC0_BUILTIN_RCP_F64 2 |
#define NVC0_BUILTIN_RSQ_F64 3 |
#define NVC0_BUILTIN_COUNT 4 |
class TargetNV50 : public Target |
{ |
public: |
TargetNV50(unsigned int chipset); |
virtual CodeEmitter *getCodeEmitter(Program::Type); |
virtual bool runLegalizePass(Program *, CGStage stage) const; |
virtual void getBuiltinCode(const uint32_t **code, uint32_t *size) const; |
virtual void parseDriverInfo(const struct nv50_ir_prog_info *); |
virtual bool insnCanLoad(const Instruction *insn, int s, |
const Instruction *ld) const; |
virtual bool isOpSupported(operation, DataType) const; |
virtual bool isAccessSupported(DataFile, DataType) const; |
virtual bool isModSupported(const Instruction *, int s, Modifier) const; |
virtual bool isSatSupported(const Instruction *) const; |
virtual bool mayPredicate(const Instruction *, const Value *) const; |
virtual int getLatency(const Instruction *) const; |
virtual int getThroughput(const Instruction *) const; |
virtual unsigned int getFileSize(DataFile) const; |
virtual unsigned int getFileUnit(DataFile) const; |
virtual uint32_t getSVAddress(DataFile shaderFile, const Symbol *sv) const; |
uint32_t getBuiltinOffset(int builtin) const; |
private: |
void initOpInfo(); |
uint16_t sysvalLocation[SV_LAST + 1]; |
uint8_t wposMask; |
}; |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_util.cpp |
---|
0,0 → 1,390 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#include "nv50_ir_util.h" |
namespace nv50_ir { |
void DLList::clear() |
{ |
for (Item *next, *item = head.next; item != &head; item = next) { |
next = item->next; |
delete item; |
} |
head.next = head.prev = &head; |
} |
void |
DLList::Iterator::erase() |
{ |
Item *rem = pos; |
if (rem == term) |
return; |
pos = pos->next; |
DLLIST_DEL(rem); |
delete rem; |
} |
void DLList::Iterator::moveToList(DLList& dest) |
{ |
Item *item = pos; |
assert(term != &dest.head); |
assert(pos != term); |
pos = pos->next; |
DLLIST_DEL(item); |
DLLIST_ADDHEAD(&dest.head, item); |
} |
bool |
DLList::Iterator::insert(void *data) |
{ |
Item *ins = new Item(data); |
ins->next = pos->next; |
ins->prev = pos; |
pos->next->prev = ins; |
pos->next = ins; |
if (pos == term) |
term = ins; |
return true; |
} |
void |
Stack::moveTo(Stack& that) |
{ |
unsigned int newSize = this->size + that.size; |
while (newSize > that.limit) |
that.resize(); |
memcpy(&that.array[that.size], &array[0], this->size * sizeof(Item)); |
that.size = newSize; |
this->size = 0; |
} |
Interval::Interval(const Interval& that) : head(NULL), tail(NULL) |
{ |
this->insert(that); |
} |
Interval::~Interval() |
{ |
clear(); |
} |
void |
Interval::clear() |
{ |
for (Range *next, *r = head; r; r = next) { |
next = r->next; |
delete r; |
} |
head = tail = NULL; |
} |
bool |
Interval::extend(int a, int b) |
{ |
Range *r, **nextp = &head; |
// NOTE: we need empty intervals for fixed registers |
// if (a == b) |
// return false; |
assert(a <= b); |
for (r = head; r; r = r->next) { |
if (b < r->bgn) |
break; // insert before |
if (a > r->end) { |
// insert after |
nextp = &r->next; |
continue; |
} |
// overlap |
if (a < r->bgn) { |
r->bgn = a; |
if (b > r->end) |
r->end = b; |
r->coalesce(&tail); |
return true; |
} |
if (b > r->end) { |
r->end = b; |
r->coalesce(&tail); |
return true; |
} |
assert(a >= r->bgn); |
assert(b <= r->end); |
return true; |
} |
(*nextp) = new Range(a, b); |
(*nextp)->next = r; |
for (r = (*nextp); r->next; r = r->next); |
tail = r; |
return true; |
} |
bool Interval::contains(int pos) const |
{ |
for (Range *r = head; r && r->bgn <= pos; r = r->next) |
if (r->end > pos) |
return true; |
return false; |
} |
bool Interval::overlaps(const Interval &that) const |
{ |
#if 1 |
Range *a = this->head; |
Range *b = that.head; |
while (a && b) { |
if (b->bgn < a->end && |
b->end > a->bgn) |
return true; |
if (a->end <= b->bgn) |
a = a->next; |
else |
b = b->next; |
} |
#else |
for (Range *rA = this->head; rA; rA = rA->next) |
for (Range *rB = iv.head; rB; rB = rB->next) |
if (rB->bgn < rA->end && |
rB->end > rA->bgn) |
return true; |
#endif |
return false; |
} |
void Interval::insert(const Interval &that) |
{ |
for (Range *r = that.head; r; r = r->next) |
this->extend(r->bgn, r->end); |
} |
void Interval::unify(Interval &that) |
{ |
assert(this != &that); |
for (Range *next, *r = that.head; r; r = next) { |
next = r->next; |
this->extend(r->bgn, r->end); |
delete r; |
} |
that.head = NULL; |
} |
int Interval::length() const |
{ |
int len = 0; |
for (Range *r = head; r; r = r->next) |
len += r->bgn - r->end; |
return len; |
} |
void Interval::print() const |
{ |
if (!head) |
return; |
INFO("[%i %i)", head->bgn, head->end); |
for (const Range *r = head->next; r; r = r->next) |
INFO(" [%i %i)", r->bgn, r->end); |
INFO("\n"); |
} |
void |
BitSet::andNot(const BitSet &set) |
{ |
assert(data && set.data); |
assert(size >= set.size); |
for (unsigned int i = 0; i < (set.size + 31) / 32; ++i) |
data[i] &= ~set.data[i]; |
} |
BitSet& BitSet::operator|=(const BitSet &set) |
{ |
assert(data && set.data); |
assert(size >= set.size); |
for (unsigned int i = 0; i < (set.size + 31) / 32; ++i) |
data[i] |= set.data[i]; |
return *this; |
} |
bool BitSet::resize(unsigned int nBits) |
{ |
if (!data || !nBits) |
return allocate(nBits, true); |
const unsigned int p = (size + 31) / 32; |
const unsigned int n = (nBits + 31) / 32; |
if (n == p) |
return true; |
data = (uint32_t *)REALLOC(data, 4 * p, 4 * n); |
if (!data) { |
size = 0; |
return false; |
} |
if (n > p) |
memset(&data[4 * p + 4], 0, (n - p) * 4); |
size = nBits; |
return true; |
} |
bool BitSet::allocate(unsigned int nBits, bool zero) |
{ |
if (data && size < nBits) { |
FREE(data); |
data = NULL; |
} |
size = nBits; |
if (!data) |
data = reinterpret_cast<uint32_t *>(CALLOC((size + 31) / 32, 4)); |
if (zero) |
memset(data, 0, (size + 7) / 8); |
else |
if (nBits) |
data[(size + 31) / 32 - 1] = 0; // clear unused bits (e.g. for popCount) |
return data; |
} |
unsigned int BitSet::popCount() const |
{ |
unsigned int count = 0; |
for (unsigned int i = 0; i < (size + 31) / 32; ++i) |
if (data[i]) |
count += util_bitcount(data[i]); |
return count; |
} |
void BitSet::fill(uint32_t val) |
{ |
unsigned int i; |
for (i = 0; i < (size + 31) / 32; ++i) |
data[i] = val; |
if (val) |
data[i] &= ~(0xffffffff << (size % 32)); // BE ? |
} |
void BitSet::setOr(BitSet *pA, BitSet *pB) |
{ |
if (!pB) { |
*this = *pA; |
} else { |
for (unsigned int i = 0; i < (size + 31) / 32; ++i) |
data[i] = pA->data[i] | pB->data[i]; |
} |
} |
int BitSet::findFreeRange(unsigned int count) const |
{ |
const uint32_t m = (1 << count) - 1; |
int pos = size; |
unsigned int i; |
const unsigned int end = (size + 31) / 32; |
if (count == 1) { |
for (i = 0; i < end; ++i) { |
pos = ffs(~data[i]) - 1; |
if (pos >= 0) |
break; |
} |
} else |
if (count == 2) { |
for (i = 0; i < end; ++i) { |
if (data[i] != 0xffffffff) { |
uint32_t b = data[i] | (data[i] >> 1) | 0xaaaaaaaa; |
pos = ffs(~b) - 1; |
if (pos >= 0) |
break; |
} |
} |
} else |
if (count == 4 || count == 3) { |
for (i = 0; i < end; ++i) { |
if (data[i] != 0xffffffff) { |
uint32_t b = |
(data[i] >> 0) | (data[i] >> 1) | |
(data[i] >> 2) | (data[i] >> 3) | 0xeeeeeeee; |
pos = ffs(~b) - 1; |
if (pos >= 0) |
break; |
} |
} |
} else { |
if (count <= 8) |
count = 8; |
else |
if (count <= 16) |
count = 16; |
else |
count = 32; |
for (i = 0; i < end; ++i) { |
if (data[i] != 0xffffffff) { |
for (pos = 0; pos < 32; pos += count) |
if (!(data[i] & (m << pos))) |
break; |
if (pos < 32) |
break; |
} |
} |
} |
pos += i * 32; |
return ((pos + count) <= size) ? pos : -1; |
} |
void BitSet::print() const |
{ |
unsigned int n = 0; |
INFO("BitSet of size %u:\n", size); |
for (unsigned int i = 0; i < (size + 31) / 32; ++i) { |
uint32_t bits = data[i]; |
while (bits) { |
int pos = ffs(bits) - 1; |
bits &= ~(1 << pos); |
INFO(" %i", i * 32 + pos); |
++n; |
if ((n % 16) == 0) |
INFO("\n"); |
} |
} |
if (n % 16) |
INFO("\n"); |
} |
} // namespace nv50_ir |
/contrib/sdk/sources/Mesa/src/gallium/drivers/nv50/codegen/nv50_ir_util.h |
---|
0,0 → 1,788 |
/* |
* Copyright 2011 Christoph Bumiller |
* |
* Permission is hereby granted, free of charge, to any person obtaining a |
* copy of this software and associated documentation files (the "Software"), |
* to deal in the Software without restriction, including without limitation |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
* and/or sell copies of the Software, and to permit persons to whom the |
* Software is furnished to do so, subject to the following conditions: |
* |
* The above copyright notice and this permission notice shall be included in |
* all copies or substantial portions of the Software. |
* |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
* OTHER DEALINGS IN THE SOFTWARE. |
*/ |
#ifndef __NV50_IR_UTIL_H__ |
#define __NV50_IR_UTIL_H__ |
#include <new> |
#include <assert.h> |
#include <stdio.h> |
#include <memory> |
#include <map> |
#ifndef NDEBUG |
# include <typeinfo> |
#endif |
#include "util/u_inlines.h" |
#include "util/u_memory.h" |
#define ERROR(args...) debug_printf("ERROR: " args) |
#define WARN(args...) debug_printf("WARNING: " args) |
#define INFO(args...) debug_printf(args) |
#define INFO_DBG(m, f, args...) \ |
do { \ |
if (m & NV50_IR_DEBUG_##f) \ |
debug_printf(args); \ |
} while(0) |
#define FATAL(args...) \ |
do { \ |
fprintf(stderr, args); \ |
abort(); \ |
} while(0) |
#define NV50_IR_FUNC_ALLOC_OBJ_DEF(obj, f, args...) \ |
new ((f)->getProgram()->mem_##obj.allocate()) obj(f, args) |
#define new_Instruction(f, args...) \ |
NV50_IR_FUNC_ALLOC_OBJ_DEF(Instruction, f, args) |
#define new_CmpInstruction(f, args...) \ |
NV50_IR_FUNC_ALLOC_OBJ_DEF(CmpInstruction, f, args) |
#define new_TexInstruction(f, args...) \ |
NV50_IR_FUNC_ALLOC_OBJ_DEF(TexInstruction, f, args) |
#define new_FlowInstruction(f, args...) \ |
NV50_IR_FUNC_ALLOC_OBJ_DEF(FlowInstruction, f, args) |
#define new_LValue(f, args...) \ |
NV50_IR_FUNC_ALLOC_OBJ_DEF(LValue, f, args) |
#define NV50_IR_PROG_ALLOC_OBJ_DEF(obj, p, args...) \ |
new ((p)->mem_##obj.allocate()) obj(p, args) |
#define new_Symbol(p, args...) \ |
NV50_IR_PROG_ALLOC_OBJ_DEF(Symbol, p, args) |
#define new_ImmediateValue(p, args...) \ |
NV50_IR_PROG_ALLOC_OBJ_DEF(ImmediateValue, p, args) |
#define delete_Instruction(p, insn) (p)->releaseInstruction(insn) |
#define delete_Value(p, val) (p)->releaseValue(val) |
namespace nv50_ir { |
class Iterator |
{ |
public: |
virtual ~Iterator() { }; |
virtual void next() = 0; |
virtual void *get() const = 0; |
virtual bool end() const = 0; // if true, get will return 0 |
virtual void reset() { assert(0); } // only for graph iterators |
}; |
typedef std::auto_ptr<Iterator> IteratorRef; |
class ManipIterator : public Iterator |
{ |
public: |
virtual bool insert(void *) = 0; // insert after current position |
virtual void erase() = 0; |
}; |
// WARNING: do not use a->prev/next for __item or __list |
#define DLLIST_DEL(__item) \ |
do { \ |
(__item)->prev->next = (__item)->next; \ |
(__item)->next->prev = (__item)->prev; \ |
(__item)->next = (__item); \ |
(__item)->prev = (__item); \ |
} while(0) |
#define DLLIST_ADDTAIL(__list, __item) \ |
do { \ |
(__item)->next = (__list); \ |
(__item)->prev = (__list)->prev; \ |
(__list)->prev->next = (__item); \ |
(__list)->prev = (__item); \ |
} while(0) |
#define DLLIST_ADDHEAD(__list, __item) \ |
do { \ |
(__item)->prev = (__list); \ |
(__item)->next = (__list)->next; \ |
(__list)->next->prev = (__item); \ |
(__list)->next = (__item); \ |
} while(0) |
#define DLLIST_MERGE(__listA, __listB, ty) \ |
do { \ |
ty prevB = (__listB)->prev; \ |
(__listA)->prev->next = (__listB); \ |
(__listB)->prev->next = (__listA); \ |
(__listB)->prev = (__listA)->prev; \ |
(__listA)->prev = prevB; \ |
} while(0) |
#define DLLIST_EMPTY(__list) ((__list)->next == (__list)) |
#define DLLIST_FOR_EACH(list, it) \ |
for (DLList::Iterator (it) = (list)->iterator(); !(it).end(); (it).next()) |
class DLList |
{ |
public: |
class Item |
{ |
public: |
Item(void *priv) : next(this), prev(this), data(priv) { } |
public: |
Item *next; |
Item *prev; |
void *data; |
}; |
DLList() : head(0) { } |
~DLList() { clear(); } |
inline void insertHead(void *data) |
{ |
Item *item = new Item(data); |
assert(data); |
item->prev = &head; |
item->next = head.next; |
head.next->prev = item; |
head.next = item; |
} |
inline void insertTail(void *data) |
{ |
Item *item = new Item(data); |
assert(data); |
DLLIST_ADDTAIL(&head, item); |
} |
inline void insert(void *data) { insertTail(data); } |
void clear(); |
class Iterator : public ManipIterator |
{ |
public: |
Iterator(Item *head, bool r) : rev(r), pos(r ? head->prev : head->next), |
term(head) { } |
virtual void next() { if (!end()) pos = rev ? pos->prev : pos->next; } |
virtual void *get() const { return pos->data; } |
virtual bool end() const { return pos == term; } |
// caution: if you're at end-2 and erase it, then do next, you're at end |
virtual void erase(); |
virtual bool insert(void *data); |
// move item to a another list, no consistency with its iterators though |
void moveToList(DLList&); |
private: |
const bool rev; |
Item *pos; |
Item *term; |
friend class DLList; |
}; |
inline void erase(Iterator& pos) |
{ |
pos.erase(); |
} |
Iterator iterator() |
{ |
return Iterator(&head, false); |
} |
Iterator revIterator() |
{ |
return Iterator(&head, true); |
} |
private: |
Item head; |
}; |
class Stack |
{ |
public: |
class Item { |
public: |
union { |
void *p; |
int i; |
unsigned int u; |
float f; |
double d; |
} u; |
Item() { memset(&u, 0, sizeof(u)); } |
}; |
Stack() : size(0), limit(0), array(0) { } |
~Stack() { if (array) FREE(array); } |
inline void push(int i) { Item data; data.u.i = i; push(data); } |
inline void push(unsigned int u) { Item data; data.u.u = u; push(data); } |
inline void push(void *p) { Item data; data.u.p = p; push(data); } |
inline void push(float f) { Item data; data.u.f = f; push(data); } |
inline void push(Item data) |
{ |
if (size == limit) |
resize(); |
array[size++] = data; |
} |
inline Item pop() |
{ |
if (!size) { |
Item data; |
assert(0); |
return data; |
} |
return array[--size]; |
} |
inline unsigned int getSize() { return size; } |
inline Item& peek() { assert(size); return array[size - 1]; } |
void clear(bool releaseStorage = false) |
{ |
if (releaseStorage && array) |
FREE(array); |
size = limit = 0; |
} |
void moveTo(Stack&); // move all items to target (not like push(pop())) |
private: |
void resize() |
{ |
unsigned int sizeOld, sizeNew; |
sizeOld = limit * sizeof(Item); |
limit = MAX2(4, limit + limit); |
sizeNew = limit * sizeof(Item); |
array = (Item *)REALLOC(array, sizeOld, sizeNew); |
} |
unsigned int size; |
unsigned int limit; |
Item *array; |
}; |
class DynArray |
{ |
public: |
class Item |
{ |
public: |
union { |
uint32_t u32; |
void *p; |
}; |
}; |
DynArray() : data(NULL), size(0) { } |
~DynArray() { if (data) FREE(data); } |
inline Item& operator[](unsigned int i) |
{ |
if (i >= size) |
resize(i); |
return data[i]; |
} |
inline const Item operator[](unsigned int i) const |
{ |
return data[i]; |
} |
void resize(unsigned int index) |
{ |
const unsigned int oldSize = size * sizeof(Item); |
if (!size) |
size = 8; |
while (size <= index) |
size <<= 1; |
data = (Item *)REALLOC(data, oldSize, size * sizeof(Item)); |
} |
void clear() |
{ |
FREE(data); |
data = NULL; |
size = 0; |
} |
private: |
Item *data; |
unsigned int size; |
}; |
class ArrayList |
{ |
public: |
ArrayList() : size(0) { } |
void insert(void *item, int& id) |
{ |
id = ids.getSize() ? ids.pop().u.i : size++; |
data[id].p = item; |
} |
void remove(int& id) |
{ |
const unsigned int uid = id; |
assert(uid < size && data[id].p); |
ids.push(uid); |
data[uid].p = NULL; |
id = -1; |
} |
inline int getSize() const { return size; } |
inline void *get(unsigned int id) { assert(id < size); return data[id].p; } |
class Iterator : public nv50_ir::Iterator |
{ |
public: |
Iterator(const ArrayList *array) : pos(0), data(array->data) |
{ |
size = array->getSize(); |
if (size) |
nextValid(); |
} |
void nextValid() { while ((pos < size) && !data[pos].p) ++pos; } |
void next() { if (pos < size) { ++pos; nextValid(); } } |
void *get() const { assert(pos < size); return data[pos].p; } |
bool end() const { return pos >= size; } |
private: |
unsigned int pos; |
unsigned int size; |
const DynArray& data; |
friend class ArrayList; |
}; |
Iterator iterator() const { return Iterator(this); } |
void clear() |
{ |
data.clear(); |
ids.clear(true); |
size = 0; |
} |
private: |
DynArray data; |
Stack ids; |
unsigned int size; |
}; |
class Interval |
{ |
public: |
Interval() : head(0), tail(0) { } |
Interval(const Interval&); |
~Interval(); |
bool extend(int, int); |
void insert(const Interval&); |
void unify(Interval&); // clears source interval |
void clear(); |
inline int begin() const { return head ? head->bgn : -1; } |
inline int end() const { checkTail(); return tail ? tail->end : -1; } |
inline bool isEmpty() const { return !head; } |
bool overlaps(const Interval&) const; |
bool contains(int pos) const; |
inline int extent() const { return end() - begin(); } |
int length() const; |
void print() const; |
inline void checkTail() const; |
private: |
class Range |
{ |
public: |
Range(int a, int b) : next(0), bgn(a), end(b) { } |
Range *next; |
int bgn; |
int end; |
void coalesce(Range **ptail) |
{ |
Range *rnn; |
while (next && end >= next->bgn) { |
assert(bgn <= next->bgn); |
rnn = next->next; |
end = MAX2(end, next->end); |
delete next; |
next = rnn; |
} |
if (!next) |
*ptail = this; |
} |
}; |
Range *head; |
Range *tail; |
}; |
class BitSet |
{ |
public: |
BitSet() : marker(false), data(0), size(0) { } |
BitSet(unsigned int nBits, bool zero) : marker(false), data(0), size(0) |
{ |
allocate(nBits, zero); |
} |
~BitSet() |
{ |
if (data) |
FREE(data); |
} |
bool allocate(unsigned int nBits, bool zero); |
bool resize(unsigned int nBits); // keep old data, zero additional bits |
inline unsigned int getSize() const { return size; } |
void fill(uint32_t val); |
void setOr(BitSet *, BitSet *); // second BitSet may be NULL |
inline void set(unsigned int i) |
{ |
assert(i < size); |
data[i / 32] |= 1 << (i % 32); |
} |
// NOTE: range may not cross 32 bit boundary (implies n <= 32) |
inline void setRange(unsigned int i, unsigned int n) |
{ |
assert((i + n) <= size && (((i % 32) + n) <= 32)); |
data[i / 32] |= ((1 << n) - 1) << (i % 32); |
} |
inline void setMask(unsigned int i, uint32_t m) |
{ |
assert(i < size); |
data[i / 32] |= m; |
} |
inline void clr(unsigned int i) |
{ |
assert(i < size); |
data[i / 32] &= ~(1 << (i % 32)); |
} |
// NOTE: range may not cross 32 bit boundary (implies n <= 32) |
inline void clrRange(unsigned int i, unsigned int n) |
{ |
assert((i + n) <= size && (((i % 32) + n) <= 32)); |
data[i / 32] &= ~(((1 << n) - 1) << (i % 32)); |
} |
inline bool test(unsigned int i) const |
{ |
assert(i < size); |
return data[i / 32] & (1 << (i % 32)); |
} |
// NOTE: range may not cross 32 bit boundary (implies n <= 32) |
inline bool testRange(unsigned int i, unsigned int n) const |
{ |
assert((i + n) <= size && (((i % 32) + n) <= 32)); |
return data[i / 32] & (((1 << n) - 1) << (i % 32)); |
} |
// Find a range of size (<= 32) clear bits aligned to roundup_pow2(size). |
int findFreeRange(unsigned int size) const; |
BitSet& operator|=(const BitSet&); |
BitSet& operator=(const BitSet& set) |
{ |
assert(data && set.data); |
assert(size == set.size); |
memcpy(data, set.data, (set.size + 7) / 8); |
return *this; |
} |
void andNot(const BitSet&); |
// bits = (bits | setMask) & ~clrMask |
inline void periodicMask32(uint32_t setMask, uint32_t clrMask) |
{ |
for (unsigned int i = 0; i < (size + 31) / 32; ++i) |
data[i] = (data[i] | setMask) & ~clrMask; |
} |
unsigned int popCount() const; |
void print() const; |
public: |
bool marker; // for user |
private: |
uint32_t *data; |
unsigned int size; |
}; |
void Interval::checkTail() const |
{ |
#if NV50_DEBUG & NV50_DEBUG_PROG_RA |
Range *r = head; |
while (r->next) |
r = r->next; |
assert(tail == r); |
#endif |
} |
class MemoryPool |
{ |
private: |
inline bool enlargeAllocationsArray(const unsigned int id, unsigned int nr) |
{ |
const unsigned int size = sizeof(uint8_t *) * id; |
const unsigned int incr = sizeof(uint8_t *) * nr; |
uint8_t **alloc = (uint8_t **)REALLOC(allocArray, size, size + incr); |
if (!alloc) |
return false; |
allocArray = alloc; |
return true; |
} |
inline bool enlargeCapacity() |
{ |
const unsigned int id = count >> objStepLog2; |
uint8_t *const mem = (uint8_t *)MALLOC(objSize << objStepLog2); |
if (!mem) |
return false; |
if (!(id % 32)) { |
if (!enlargeAllocationsArray(id, 32)) { |
FREE(mem); |
return false; |
} |
} |
allocArray[id] = mem; |
return true; |
} |
public: |
MemoryPool(unsigned int size, unsigned int incr) : objSize(size), |
objStepLog2(incr) |
{ |
allocArray = NULL; |
released = NULL; |
count = 0; |
} |
~MemoryPool() |
{ |
unsigned int allocCount = (count + (1 << objStepLog2) - 1) >> objStepLog2; |
for (unsigned int i = 0; i < allocCount && allocArray[i]; ++i) |
FREE(allocArray[i]); |
if (allocArray) |
FREE(allocArray); |
} |
void *allocate() |
{ |
void *ret; |
const unsigned int mask = (1 << objStepLog2) - 1; |
if (released) { |
ret = released; |
released = *(void **)released; |
return ret; |
} |
if (!(count & mask)) |
if (!enlargeCapacity()) |
return NULL; |
ret = allocArray[count >> objStepLog2] + (count & mask) * objSize; |
++count; |
return ret; |
} |
void release(void *ptr) |
{ |
*(void **)ptr = released; |
released = ptr; |
} |
private: |
uint8_t **allocArray; // array (list) of MALLOC allocations |
void *released; // list of released objects |
unsigned int count; // highest allocated object |
const unsigned int objSize; |
const unsigned int objStepLog2; |
}; |
/** |
* Composite object cloning policy. |
* |
* Encapsulates how sub-objects are to be handled (if at all) when a |
* composite object is being cloned. |
*/ |
template<typename C> |
class ClonePolicy |
{ |
protected: |
C *c; |
public: |
ClonePolicy(C *c) : c(c) {} |
C *context() { return c; } |
template<typename T> T *get(T *obj) |
{ |
void *clone = lookup(obj); |
if (!clone) |
clone = obj->clone(*this); |
return reinterpret_cast<T *>(clone); |
} |
template<typename T> void set(const T *obj, T *clone) |
{ |
insert(obj, clone); |
} |
protected: |
virtual void *lookup(void *obj) = 0; |
virtual void insert(const void *obj, void *clone) = 0; |
}; |
/** |
* Shallow non-recursive cloning policy. |
* |
* Objects cloned with the "shallow" policy don't clone their |
* children recursively, instead, the new copy shares its children |
* with the original object. |
*/ |
template<typename C> |
class ShallowClonePolicy : public ClonePolicy<C> |
{ |
public: |
ShallowClonePolicy(C *c) : ClonePolicy<C>(c) {} |
protected: |
virtual void *lookup(void *obj) |
{ |
return obj; |
} |
virtual void insert(const void *obj, void *clone) |
{ |
} |
}; |
template<typename C, typename T> |
inline T *cloneShallow(C *c, T *obj) |
{ |
ShallowClonePolicy<C> pol(c); |
return obj->clone(pol); |
} |
/** |
* Recursive cloning policy. |
* |
* Objects cloned with the "deep" policy clone their children |
* recursively, keeping track of what has already been cloned to |
* avoid making several new copies of the same object. |
*/ |
template<typename C> |
class DeepClonePolicy : public ClonePolicy<C> |
{ |
public: |
DeepClonePolicy(C *c) : ClonePolicy<C>(c) {} |
private: |
std::map<const void *, void *> map; |
protected: |
virtual void *lookup(void *obj) |
{ |
return map[obj]; |
} |
virtual void insert(const void *obj, void *clone) |
{ |
map[obj] = clone; |
} |
}; |
template<typename S, typename T> |
struct bimap |
{ |
std::map<S, T> forth; |
std::map<T, S> back; |
public: |
bimap() : l(back), r(forth) { } |
bimap(const bimap<S, T> &m) |
: forth(m.forth), back(m.back), l(back), r(forth) { } |
void insert(const S &s, const T &t) |
{ |
forth.insert(std::make_pair(s, t)); |
back.insert(std::make_pair(t, s)); |
} |
typedef typename std::map<T, S>::const_iterator l_iterator; |
const std::map<T, S> &l; |
typedef typename std::map<S, T>::const_iterator r_iterator; |
const std::map<S, T> &r; |
}; |
} // namespace nv50_ir |
#endif // __NV50_IR_UTIL_H__ |