Subversion Repositories Kolibri OS

/*

 * Copyright 2011 Christoph Bumiller

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the "Software"),

 * to deal in the Software without restriction, including without limitation

 * the rights to use, copy, modify, merge, publish, distribute, sublicense,

 * and/or sell copies of the Software, and to permit persons to whom the

 * Software is furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice shall be included in

 * all copies or substantial portions of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR

 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,

 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

 * OTHER DEALINGS IN THE SOFTWARE.

 */

#include "codegen/nv50_ir.h"

#include "codegen/nv50_ir_build_util.h"

#include "codegen/nv50_ir_target_nvc0.h"

#include "codegen/nv50_ir_lowering_nvc0.h"

#include 

namespace nv50_ir {

#define QOP_ADD  0

#define QOP_SUBR 1

#define QOP_SUB  2

#define QOP_MOV2 3

//             UL UR LL LR

#define QUADOP(q, r, s, t)                      \

   ((QOP_##q << 6) | (QOP_##r << 4) |           \

    (QOP_##s << 2) | (QOP_##t << 0))

void

NVC0LegalizeSSA::handleDIV(Instruction *i)

{

   FlowInstruction *call;

   int builtin;

   Value *def[2];

   bld.setPosition(i, false);

   def[0] = bld.mkMovToReg(0, i->getSrc(0))->getDef(0);

   def[1] = bld.mkMovToReg(1, i->getSrc(1))->getDef(0);

   switch (i->dType) {

   case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break;

   case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break;

   default:

      return;

   }

   call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);

   bld.mkMov(i->getDef(0), def[(i->op == OP_DIV) ? 0 : 1]);

   bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2);

   bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0);

   call->fixed = 1;

   call->absolute = call->builtin = 1;

   call->target.builtin = builtin;

   delete_Instruction(prog, i);

}

void

NVC0LegalizeSSA::handleRCPRSQ(Instruction *i)

{

   assert(i->dType == TYPE_F64);

   // There are instructions that will compute the high 32 bits of the 64-bit

   // float. We will just stick 0 in the bottom 32 bits.

   bld.setPosition(i, false);

   // 1. Take the source and it up.

   Value *src[2], *dst[2], *def = i->getDef(0);

   bld.mkSplit(src, 4, i->getSrc(0));

   // 2. We don't care about the low 32 bits of the destination. Stick a 0 in.

   dst[0] = bld.loadImm(NULL, 0);

   dst[1] = bld.getSSA();

   // 3. The new version of the instruction takes the high 32 bits of the

   // source and outputs the high 32 bits of the destination.

   i->setSrc(0, src[1]);

   i->setDef(0, dst[1]);

   i->setType(TYPE_F32);

   i->subOp = NV50_IR_SUBOP_RCPRSQ_64H;

   // 4. Recombine the two dst pieces back into the original destination.

   bld.setPosition(i, true);

   bld.mkOp2(OP_MERGE, TYPE_U64, def, dst[0], dst[1]);

}

void

NVC0LegalizeSSA::handleFTZ(Instruction *i)

{

   // Only want to flush float inputs

   assert(i->sType == TYPE_F32);

   // If we're already flushing denorms (and NaN's) to zero, no need for this.

   if (i->dnz)

      return;

   // Only certain classes of operations can flush

   OpClass cls = prog->getTarget()->getOpClass(i->op);

   if (cls != OPCLASS_ARITH && cls != OPCLASS_COMPARE &&

       cls != OPCLASS_CONVERT)

      return;

   i->ftz = true;

}

bool

NVC0LegalizeSSA::visit(Function *fn)

{

   bld.setProgram(fn->getProgram());

   return true;

}

bool

NVC0LegalizeSSA::visit(BasicBlock *bb)

{

   Instruction *next;

   for (Instruction *i = bb->getEntry(); i; i = next) {

      next = i->next;

      if (i->sType == TYPE_F32) {

         if (prog->getType() != Program::TYPE_COMPUTE)

            handleFTZ(i);

         continue;

      }

      switch (i->op) {

      case OP_DIV:

      case OP_MOD:

         handleDIV(i);

         break;

      case OP_RCP:

      case OP_RSQ:

         if (i->dType == TYPE_F64)

            handleRCPRSQ(i);

         break;

      default:

         break;

      }

   }

   return true;

}

NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog)

   : rZero(NULL),

     carry(NULL),

     needTexBar(prog->getTarget()->getChipset() >= 0xe0)

{

}

bool

NVC0LegalizePostRA::insnDominatedBy(const Instruction *later,

                                    const Instruction *early) const

{

   if (early->bb == later->bb)

      return early->serial < later->serial;

   return later->bb->dominatedBy(early->bb);

}

void

NVC0LegalizePostRA::addTexUse(std::list &uses,

                              Instruction *usei, const Instruction *insn)

{

   bool add = true;

   for (std::list::iterator it = uses.begin();

        it != uses.end();) {

      if (insnDominatedBy(usei, it->insn)) {

         add = false;

         break;

      }

      if (insnDominatedBy(it->insn, usei))

         it = uses.erase(it);

      else

         ++it;

   }

   if (add)

      uses.push_back(TexUse(usei, insn));

}

void

NVC0LegalizePostRA::findOverwritingDefs(const Instruction *texi,

                                        Instruction *insn,

                                        const BasicBlock *term,

                                        std::list &uses)

{

   while (insn->op == OP_MOV && insn->getDef(0)->equals(insn->getSrc(0)))

      insn = insn->getSrc(0)->getUniqueInsn();

   if (!insn->bb->reachableBy(texi->bb, term))

      return;

   switch (insn->op) {

   /* Values not connected to the tex's definition through any of these should

    * not be conflicting.

    */

   case OP_SPLIT:

   case OP_MERGE:

   case OP_PHI:

   case OP_UNION:

      /* recurse again */

      for (int s = 0; insn->srcExists(s); ++s)

         findOverwritingDefs(texi, insn->getSrc(s)->getUniqueInsn(), term,

                             uses);

      break;

   default:

      // if (!isTextureOp(insn->op)) // TODO: are TEXes always ordered ?

      addTexUse(uses, insn, texi);

      break;

   }

}

void

NVC0LegalizePostRA::findFirstUses(

      const Instruction *texi,

      const Instruction *insn,

      std::list &uses,

      std::tr1::unordered_set& visited)

{

   for (int d = 0; insn->defExists(d); ++d) {

      Value *v = insn->getDef(d);

      for (Value::UseIterator u = v->uses.begin(); u != v->uses.end(); ++u) {

         Instruction *usei = (*u)->getInsn();

         // NOTE: In case of a loop that overwrites a value but never uses

         // it, it can happen that we have a cycle of uses that consists only

         // of phis and no-op moves and will thus cause an infinite loop here

         // since these are not considered actual uses.

         // The most obvious (and perhaps the only) way to prevent this is to

         // remember which instructions we've already visited.

         if (visited.find(usei) != visited.end())

            continue;

         visited.insert(usei);

         if (usei->op == OP_PHI || usei->op == OP_UNION) {

            // need a barrier before WAW cases

            for (int s = 0; usei->srcExists(s); ++s) {

               Instruction *defi = usei->getSrc(s)->getUniqueInsn();

               if (defi && &usei->src(s) != *u)

                  findOverwritingDefs(texi, defi, usei->bb, uses);

            }

         }

         if (usei->op == OP_SPLIT ||

             usei->op == OP_MERGE ||

             usei->op == OP_PHI ||

             usei->op == OP_UNION) {

            // these uses don't manifest in the machine code

            findFirstUses(texi, usei, uses, visited);

         } else

         if (usei->op == OP_MOV && usei->getDef(0)->equals(usei->getSrc(0)) &&

             usei->subOp != NV50_IR_SUBOP_MOV_FINAL) {

            findFirstUses(texi, usei, uses, visited);

         } else {

            addTexUse(uses, usei, insn);

         }

      }

   }

}

// Texture barriers:

// This pass is a bit long and ugly and can probably be optimized.

//

// 1. obtain a list of TEXes and their outputs' first use(s)

// 2. calculate the barrier level of each first use (minimal number of TEXes,

//    over all paths, between the TEX and the use in question)

// 3. for each barrier, if all paths from the source TEX to that barrier

//    contain a barrier of lesser level, it can be culled

bool

NVC0LegalizePostRA::insertTextureBarriers(Function *fn)

{

   std::list *uses;

   std::vector texes;

   std::vector bbFirstTex;

   std::vector bbFirstUse;

   std::vector texCounts;

   std::vector useVec;

   ArrayList insns;

   fn->orderInstructions(insns);

   texCounts.resize(fn->allBBlocks.getSize(), 0);

   bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize());

   bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize());

   // tag BB CFG nodes by their id for later

   for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) {

      BasicBlock *bb = reinterpret_cast(i.get());

      if (bb)

         bb->cfg.tag = bb->getId();

   }

   // gather the first uses for each TEX

   for (int i = 0; i < insns.getSize(); ++i) {

      Instruction *tex = reinterpret_cast(insns.get(i));

      if (isTextureOp(tex->op)) {

         texes.push_back(tex);

         if (!texCounts.at(tex->bb->getId()))

            bbFirstTex[tex->bb->getId()] = texes.size() - 1;

         texCounts[tex->bb->getId()]++;

      }

   }

   insns.clear();

   if (texes.empty())

      return false;

   uses = new std::list[texes.size()];

   if (!uses)

      return false;

   for (size_t i = 0; i < texes.size(); ++i) {

      std::tr1::unordered_set visited;

      findFirstUses(texes[i], texes[i], uses[i], visited);

   }

   // determine the barrier level at each use

   for (size_t i = 0; i < texes.size(); ++i) {

      for (std::list::iterator u = uses[i].begin(); u != uses[i].end();

           ++u) {

         BasicBlock *tb = texes[i]->bb;

         BasicBlock *ub = u->insn->bb;

         if (tb == ub) {

            u->level = 0;

            for (size_t j = i + 1; j < texes.size() &&

                    texes[j]->bb == tb && texes[j]->serial < u->insn->serial;

                 ++j)

               u->level++;

         } else {

            u->level = fn->cfg.findLightestPathWeight(&tb->cfg,

                                                      &ub->cfg, texCounts);

            if (u->level < 0) {

               WARN("Failed to find path TEX -> TEXBAR\n");

               u->level = 0;

               continue;

            }

            // this counted all TEXes in the origin block, correct that

            u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */;

            // and did not count the TEXes in the destination block, add those

            for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() &&

                    texes[j]->bb == ub && texes[j]->serial < u->insn->serial;

                 ++j)

               u->level++;

         }

         assert(u->level >= 0);

         useVec.push_back(*u);

      }

   }

   delete[] uses;

   // insert the barriers

   for (size_t i = 0; i < useVec.size(); ++i) {

      Instruction *prev = useVec[i].insn->prev;

      if (useVec[i].level < 0)

         continue;

      if (prev && prev->op == OP_TEXBAR) {

         if (prev->subOp > useVec[i].level)

            prev->subOp = useVec[i].level;

         prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0));

      } else {

         Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE);

         bar->fixed = 1;

         bar->subOp = useVec[i].level;

         // make use explicit to ease latency calculation

         bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0));

         useVec[i].insn->bb->insertBefore(useVec[i].insn, bar);

      }

   }

   if (fn->getProgram()->optLevel < 3)

      return true;

   std::vector limitT, limitB, limitS; // entry, exit, single

   limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0));

   limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0));

   limitS.resize(fn->allBBlocks.getSize());

   // cull unneeded barriers (should do that earlier, but for simplicity)

   IteratorRef bi = fn->cfg.iteratorCFG();

   // first calculate min/max outstanding TEXes for each BB

   for (bi->reset(); !bi->end(); bi->next()) {

      Graph::Node *n = reinterpret_cast(bi->get());

      BasicBlock *bb = BasicBlock::get(n);

      int min = 0;

      int max = std::numeric_limits::max();

      for (Instruction *i = bb->getFirst(); i; i = i->next) {

         if (isTextureOp(i->op)) {

            min++;

            if (max < std::numeric_limits::max())

               max++;

         } else

         if (i->op == OP_TEXBAR) {

            min = MIN2(min, i->subOp);

            max = MIN2(max, i->subOp);

         }

      }

      // limits when looking at an isolated block

      limitS[bb->getId()].min = min;

      limitS[bb->getId()].max = max;

   }

   // propagate the min/max values

   for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) {

      for (bi->reset(); !bi->end(); bi->next()) {

         Graph::Node *n = reinterpret_cast(bi->get());

         BasicBlock *bb = BasicBlock::get(n);

         const int bbId = bb->getId();

         for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) {

            BasicBlock *in = BasicBlock::get(ei.getNode());

            const int inId = in->getId();

            limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min);

            limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max);

         }

         // I just hope this is correct ...

         if (limitS[bbId].max == std::numeric_limits::max()) {

            // no barrier

            limitB[bbId].min = limitT[bbId].min + limitS[bbId].min;

            limitB[bbId].max = limitT[bbId].max + limitS[bbId].min;

         } else {

            // block contained a barrier

            limitB[bbId].min = MIN2(limitS[bbId].max,

                                    limitT[bbId].min + limitS[bbId].min);

            limitB[bbId].max = MIN2(limitS[bbId].max,

                                    limitT[bbId].max + limitS[bbId].min);

         }

      }

   }

   // finally delete unnecessary barriers

   for (bi->reset(); !bi->end(); bi->next()) {

      Graph::Node *n = reinterpret_cast(bi->get());

      BasicBlock *bb = BasicBlock::get(n);

      Instruction *prev = NULL;

      Instruction *next;

      int max = limitT[bb->getId()].max;

      for (Instruction *i = bb->getFirst(); i; i = next) {

         next = i->next;

         if (i->op == OP_TEXBAR) {

            if (i->subOp >= max) {

               delete_Instruction(prog, i);

               i = NULL;

            } else {

               max = i->subOp;

               if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) {

                  delete_Instruction(prog, prev);

                  prev = NULL;

               }

            }

         } else

         if (isTextureOp(i->op)) {

            max++;

         }

         if (i && !i->isNop())

            prev = i;

      }

   }

   return true;

}

bool

NVC0LegalizePostRA::visit(Function *fn)

{

   if (needTexBar)

      insertTextureBarriers(fn);

   rZero = new_LValue(fn, FILE_GPR);

   carry = new_LValue(fn, FILE_FLAGS);

   rZero->reg.data.id = prog->getTarget()->getFileSize(FILE_GPR);

   carry->reg.data.id = 0;

   return true;

}

void

NVC0LegalizePostRA::replaceZero(Instruction *i)

{

   for (int s = 0; i->srcExists(s); ++s) {

      if (s == 2 && i->op == OP_SUCLAMP)

         continue;

      ImmediateValue *imm = i->getSrc(s)->asImm();

      if (imm && imm->reg.data.u64 == 0)

         i->setSrc(s, rZero);

   }

}

// replace CONT with BRA for single unconditional continue

bool

NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb)

{

   if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT)

      return false;

   Graph::EdgeIterator ei = bb->cfg.incident();

   if (ei.getType() != Graph::Edge::BACK)

      ei.next();

   if (ei.getType() != Graph::Edge::BACK)

      return false;

   BasicBlock *contBB = BasicBlock::get(ei.getNode());

   if (!contBB->getExit() || contBB->getExit()->op != OP_CONT ||

       contBB->getExit()->getPredicate())

      return false;

   contBB->getExit()->op = OP_BRA;

   bb->remove(bb->getEntry()); // delete PRECONT

   ei.next();

   assert(ei.end() || ei.getType() != Graph::Edge::BACK);

   return true;

}

// replace branches to join blocks with join ops

void

NVC0LegalizePostRA::propagateJoin(BasicBlock *bb)

{

   if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit)

      return;

   for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {

      BasicBlock *in = BasicBlock::get(ei.getNode());

      Instruction *exit = in->getExit();

      if (!exit) {

         in->insertTail(new FlowInstruction(func, OP_JOIN, bb));

         // there should always be a terminator instruction

         WARN("inserted missing terminator in BB:%i\n", in->getId());

      } else

      if (exit->op == OP_BRA) {

         exit->op = OP_JOIN;

         exit->asFlow()->limit = 1; // must-not-propagate marker

      }

   }

   bb->remove(bb->getEntry());

}

bool

NVC0LegalizePostRA::visit(BasicBlock *bb)

{

   Instruction *i, *next;

   // remove pseudo operations and non-fixed no-ops, split 64 bit operations

   for (i = bb->getFirst(); i; i = next) {

      next = i->next;

      if (i->op == OP_EMIT || i->op == OP_RESTART) {

         if (!i->getDef(0)->refCount())

            i->setDef(0, NULL);

         if (i->src(0).getFile() == FILE_IMMEDIATE)

            i->setSrc(0, rZero); // initial value must be 0

         replaceZero(i);

      } else

      if (i->isNop()) {

         bb->remove(i);

      } else {

         // TODO: Move this to before register allocation for operations that

         // need the $c register !

         if (typeSizeof(i->dType) == 8) {

            Instruction *hi;

            hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry);

            if (hi)

               next = hi;

         }

         if (i->op != OP_MOV && i->op != OP_PFETCH)

            replaceZero(i);

      }

   }

   if (!bb->getEntry())

      return true;

   if (!tryReplaceContWithBra(bb))

      propagateJoin(bb);

   return true;

}

NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget())

{

   bld.setProgram(prog);

   gMemBase = NULL;

}

bool

NVC0LoweringPass::visit(Function *fn)

{

   if (prog->getType() == Program::TYPE_GEOMETRY) {

      assert(!strncmp(fn->getName(), "MAIN", 4));

      // TODO: when we generate actual functions pass this value along somehow

      bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false);

      gpEmitAddress = bld.loadImm(NULL, 0)->asLValue();

      if (fn->cfgExit) {

         bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false);

         bld.mkMovToReg(0, gpEmitAddress);

      }

   }

   return true;

}

bool

NVC0LoweringPass::visit(BasicBlock *bb)

{

   return true;

}

inline Value *

NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot)

{

   uint8_t b = prog->driver->io.resInfoCBSlot;

   uint32_t off = prog->driver->io.texBindBase + slot * 4;

   return bld.

      mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);

}

// move array source to first slot, convert to u16, add indirections

bool

NVC0LoweringPass::handleTEX(TexInstruction *i)

{

   const int dim = i->tex.target.getDim() + i->tex.target.isCube();

   const int arg = i->tex.target.getArgCount();

   const int lyr = arg - (i->tex.target.isMS() ? 2 : 1);

   const int chipset = prog->getTarget()->getChipset();

   // Arguments to the TEX instruction are a little insane. Even though the

   // encoding is identical between SM20 and SM30, the arguments mean

   // different things between Fermi and Kepler+. A lot of arguments are

   // optional based on flags passed to the instruction. This summarizes the

   // order of things.

   //

   // Fermi:

   //  array/indirect

   //  coords

   //  sample

   //  lod bias

   //  depth compare

   //  offsets:

   //    - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg)

   //    - other: 4 bits each, single reg

   //

   // Kepler+:

   //  indirect handle

   //  array (+ offsets for txd in upper 16 bits)

   //  coords

   //  sample

   //  lod bias

   //  depth compare

   //  offsets (same as fermi, except txd which takes it with array)

   //

   // Maxwell (tex):

   //  array

   //  coords

   //  indirect handle

   //  sample

   //  lod bias

   //  depth compare

   //  offsets

   //

   // Maxwell (txd):

   //  indirect handle

   //  coords

   //  array + offsets

   //  derivatives

   if (chipset >= NVISA_GK104_CHIPSET) {

      if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {

         // XXX this ignores tsc, and assumes a 1:1 mapping

         assert(i->tex.rIndirectSrc >= 0);

         Value *hnd = loadTexHandle(

               bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),

                          i->getIndirectR(), bld.mkImm(2)),

               i->tex.r);

         i->tex.r = 0xff;

         i->tex.s = 0x1f;

         i->setIndirectR(hnd);

         i->setIndirectS(NULL);

      } else if (i->tex.r == i->tex.s) {

         i->tex.r += prog->driver->io.texBindBase / 4;

         i->tex.s  = 0; // only a single cX[] value possible here

      } else {

         Value *hnd = bld.getScratch();

         Value *rHnd = loadTexHandle(NULL, i->tex.r);

         Value *sHnd = loadTexHandle(NULL, i->tex.s);

         bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd);

         i->tex.r = 0; // not used for indirect tex

         i->tex.s = 0;

         i->setIndirectR(hnd);

      }

      if (i->tex.target.isArray()) {

         LValue *layer = new_LValue(func, FILE_GPR);

         Value *src = i->getSrc(lyr);

         const int sat = (i->op == OP_TXF) ? 1 : 0;

         DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;

         bld.mkCvt(OP_CVT, TYPE_U16, layer, sTy, src)->saturate = sat;

         if (i->op != OP_TXD || chipset < NVISA_GM107_CHIPSET) {

            for (int s = dim; s >= 1; --s)

               i->setSrc(s, i->getSrc(s - 1));

            i->setSrc(0, layer);

         } else {

            i->setSrc(dim, layer);

         }

      }

      // Move the indirect reference to the first place

      if (i->tex.rIndirectSrc >= 0 && (

                i->op == OP_TXD || chipset < NVISA_GM107_CHIPSET)) {

         Value *hnd = i->getIndirectR();

         i->setIndirectR(NULL);

         i->moveSources(0, 1);

         i->setSrc(0, hnd);

         i->tex.rIndirectSrc = 0;

         i->tex.sIndirectSrc = -1;

      }

   } else

   // (nvc0) generate and move the tsc/tic/array source to the front

   if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {

      LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa

      Value *ticRel = i->getIndirectR();

      Value *tscRel = i->getIndirectS();

      if (ticRel) {

         i->setSrc(i->tex.rIndirectSrc, NULL);

         if (i->tex.r)

            ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),

                                ticRel, bld.mkImm(i->tex.r));

      }

      if (tscRel) {

         i->setSrc(i->tex.sIndirectSrc, NULL);

         if (i->tex.s)

            tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),

                                tscRel, bld.mkImm(i->tex.s));

      }

      Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL;

      for (int s = dim; s >= 1; --s)

         i->setSrc(s, i->getSrc(s - 1));

      i->setSrc(0, arrayIndex);

      if (arrayIndex) {

         int sat = (i->op == OP_TXF) ? 1 : 0;

         DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;

         bld.mkCvt(OP_CVT, TYPE_U16, src, sTy, arrayIndex)->saturate = sat;

      } else {

         bld.loadImm(src, 0);

      }

      if (ticRel)

         bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src);

      if (tscRel)

         bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src);

      i->setSrc(0, src);

   }

   // For nvc0, the sample id has to be in the second operand, as the offset

   // does. Right now we don't know how to pass both in, and this case can't

   // happen with OpenGL. On nve0, the sample id is part of the texture

   // coordinate argument.

   assert(chipset >= NVISA_GK104_CHIPSET ||

          !i->tex.useOffsets || !i->tex.target.isMS());

   // offset is between lod and dc

   if (i->tex.useOffsets) {

      int n, c;

      int s = i->srcCount(0xff, true);

      if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) {

         if (i->tex.target.isShadow())

            s--;

         if (i->srcExists(s)) // move potential predicate out of the way

            i->moveSources(s, 1);

         if (i->tex.useOffsets == 4 && i->srcExists(s + 1))

            i->moveSources(s + 1, 1);

      }

      if (i->op == OP_TXG) {

         // Either there is 1 offset, which goes into the 2 low bytes of the

         // first source, or there are 4 offsets, which go into 2 sources (8

         // values, 1 byte each).

         Value *offs[2] = {NULL, NULL};

         for (n = 0; n < i->tex.useOffsets; n++) {

            for (c = 0; c < 2; ++c) {

               if ((n % 2) == 0 && c == 0)

                  offs[n / 2] = i->offset[n][c].get();

               else

                  bld.mkOp3(OP_INSBF, TYPE_U32,

                            offs[n / 2],

                            i->offset[n][c].get(),

                            bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)),

                            offs[n / 2]);

            }

         }

         i->setSrc(s, offs[0]);

         if (offs[1])

            i->setSrc(s + 1, offs[1]);

      } else {

         unsigned imm = 0;

         assert(i->tex.useOffsets == 1);

         for (c = 0; c < 3; ++c) {

            ImmediateValue val;

            if (!i->offset[0][c].getImmediate(val))

               assert(!"non-immediate offset passed to non-TXG");

            imm |= (val.reg.data.u32 & 0xf) << (c * 4);

         }

         if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) {

            // The offset goes into the upper 16 bits of the array index. So

            // create it if it's not already there, and INSBF it if it already

            // is.

            s = (i->tex.rIndirectSrc >= 0) ? 1 : 0;

            if (chipset >= NVISA_GM107_CHIPSET)

               s += dim;

            if (i->tex.target.isArray()) {

               bld.mkOp3(OP_INSBF, TYPE_U32, i->getSrc(s),

                         bld.loadImm(NULL, imm), bld.mkImm(0xc10),

                         i->getSrc(s));

            } else {

               i->moveSources(s, 1);

               i->setSrc(s, bld.loadImm(NULL, imm << 16));

            }

         } else {

            i->setSrc(s, bld.loadImm(NULL, imm));

         }

      }

   }

   if (chipset >= NVISA_GK104_CHIPSET) {

      //

      // If TEX requires more than 4 sources, the 2nd register tuple must be

      // aligned to 4, even if it consists of just a single 4-byte register.

      //

      // XXX HACK: We insert 0 sources to avoid the 5 or 6 regs case.

      //

      int s = i->srcCount(0xff, true);

      if (s > 4 && s < 7) {

         if (i->srcExists(s)) // move potential predicate out of the way

            i->moveSources(s, 7 - s);

         while (s < 7)

            i->setSrc(s++, bld.loadImm(NULL, 0));

      }

   }

   return true;

}

bool

NVC0LoweringPass::handleManualTXD(TexInstruction *i)

{

   static const uint8_t qOps[4][2] =

   {

      { QUADOP(MOV2, ADD,  MOV2, ADD),  QUADOP(MOV2, MOV2, ADD,  ADD) }, // l0

      { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD,  ADD) }, // l1

      { QUADOP(MOV2, ADD,  MOV2, ADD),  QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2

      { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3

   };

   Value *def[4][4];

   Value *crd[3];

   Instruction *tex;

   Value *zero = bld.loadImm(bld.getSSA(), 0);

   int l, c;

   const int dim = i->tex.target.getDim();

   const int array = i->tex.target.isArray();

   i->op = OP_TEX; // no need to clone dPdx/dPdy later

   for (c = 0; c < dim; ++c)

      crd[c] = bld.getScratch();

   bld.mkOp(OP_QUADON, TYPE_NONE, NULL);

   for (l = 0; l < 4; ++l) {

      // mov coordinates from lane l to all lanes

      for (c = 0; c < dim; ++c)

         bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero);

      // add dPdx from lane l to lanes dx

      for (c = 0; c < dim; ++c)

         bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]);

      // add dPdy from lane l to lanes dy

      for (c = 0; c < dim; ++c)

         bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]);

      // texture

      bld.insert(tex = cloneForward(func, i));

      for (c = 0; c < dim; ++c)

         tex->setSrc(c + array, crd[c]);

      // save results

      for (c = 0; i->defExists(c); ++c) {

         Instruction *mov;

         def[c][l] = bld.getSSA();

         mov = bld.mkMov(def[c][l], tex->getDef(c));

         mov->fixed = 1;

         mov->lanes = 1 << l;

      }

   }

   bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);

   for (c = 0; i->defExists(c); ++c) {

      Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));

      for (l = 0; l < 4; ++l)

         u->setSrc(l, def[c][l]);

   }

   i->bb->remove(i);

   return true;

}

bool

NVC0LoweringPass::handleTXD(TexInstruction *txd)

{

   int dim = txd->tex.target.getDim();

   unsigned arg = txd->tex.target.getArgCount();

   unsigned expected_args = arg;

   const int chipset = prog->getTarget()->getChipset();

   if (chipset >= NVISA_GK104_CHIPSET) {

      if (!txd->tex.target.isArray() && txd->tex.useOffsets)

         expected_args++;

      if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)

         expected_args++;

   } else {

      if (txd->tex.useOffsets)

         expected_args++;

      if (!txd->tex.target.isArray() && (

                txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0))

         expected_args++;

   }

   if (expected_args > 4 ||

       dim > 2 ||

       txd->tex.target.isShadow() ||

       txd->tex.target.isCube())

      txd->op = OP_TEX;

   handleTEX(txd);

   while (txd->srcExists(arg))

      ++arg;

   txd->tex.derivAll = true;

   if (txd->op == OP_TEX)

      return handleManualTXD(txd);

   assert(arg == expected_args);

   for (int c = 0; c < dim; ++c) {

      txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]);

      txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]);

      txd->dPdx[c].set(NULL);

      txd->dPdy[c].set(NULL);

   }

   return true;

}

bool

NVC0LoweringPass::handleTXQ(TexInstruction *txq)

{

   // TODO: indirect resource/sampler index

   return true;

}

bool

NVC0LoweringPass::handleTXLQ(TexInstruction *i)

{

   /* The outputs are inverted compared to what the TGSI instruction

    * expects. Take that into account in the mask.

    */

   assert((i->tex.mask & ~3) == 0);

   if (i->tex.mask == 1)

      i->tex.mask = 2;

   else if (i->tex.mask == 2)

      i->tex.mask = 1;

   handleTEX(i);

   bld.setPosition(i, true);

   /* The returned values are not quite what we want:

    * (a) convert from s16/u16 to f32

    * (b) multiply by 1/256

    */

   for (int def = 0; def < 2; ++def) {

      if (!i->defExists(def))

         continue;

      enum DataType type = TYPE_S16;

      if (i->tex.mask == 2 || def > 0)

         type = TYPE_U16;

      bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def));

      bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def),

                i->getDef(def), bld.loadImm(NULL, 1.0f / 256));

   }

   if (i->tex.mask == 3) {

      LValue *t = new_LValue(func, FILE_GPR);

      bld.mkMov(t, i->getDef(0));

      bld.mkMov(i->getDef(0), i->getDef(1));

      bld.mkMov(i->getDef(1), t);

   }

   return true;

}

bool

NVC0LoweringPass::handleATOM(Instruction *atom)

{

   SVSemantic sv;

   switch (atom->src(0).getFile()) {

   case FILE_MEMORY_LOCAL:

      sv = SV_LBASE;

      break;

   case FILE_MEMORY_SHARED:

      sv = SV_SBASE;

      break;

   default:

      assert(atom->src(0).getFile() == FILE_MEMORY_GLOBAL);

      return true;

   }

   Value *base =

      bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0));

   Value *ptr = atom->getIndirect(0, 0);

   atom->setSrc(0, cloneShallow(func, atom->getSrc(0)));

   atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;

   if (ptr)

      base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr);

   atom->setIndirect(0, 0, base);

   return true;

}

bool

NVC0LoweringPass::handleCasExch(Instruction *cas, bool needCctl)

{

   if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS &&

       cas->subOp != NV50_IR_SUBOP_ATOM_EXCH)

      return false;

   bld.setPosition(cas, true);

   if (needCctl) {

      Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, cas->getSrc(0));

      cctl->setIndirect(0, 0, cas->getIndirect(0, 0));

      cctl->fixed = 1;

      cctl->subOp = NV50_IR_SUBOP_CCTL_IV;

      if (cas->isPredicated())

         cctl->setPredicate(cas->cc, cas->getPredicate());

   }

   if (cas->defExists(0) && cas->subOp == NV50_IR_SUBOP_ATOM_CAS) {

      // CAS is crazy. It's 2nd source is a double reg, and the 3rd source

      // should be set to the high part of the double reg or bad things will

      // happen elsewhere in the universe.

      // Also, it sometimes returns the new value instead of the old one

      // under mysterious circumstances.

      Value *dreg = bld.getSSA(8);

      bld.setPosition(cas, false);

      bld.mkOp2(OP_MERGE, TYPE_U64, dreg, cas->getSrc(1), cas->getSrc(2));

      cas->setSrc(1, dreg);

   }

   return true;

}

inline Value *

NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off)

{

   uint8_t b = prog->driver->io.resInfoCBSlot;

   off += prog->driver->io.suInfoBase;

   return bld.

      mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);

}

inline Value *

NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off)

{

   uint8_t b = prog->driver->io.msInfoCBSlot;

   off += prog->driver->io.msInfoBase;

   return bld.

      mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);

}

/* On nvc0, surface info is obtained via the surface binding points passed

 * to the SULD/SUST instructions.

 * On nve4, surface info is stored in c[] and is used by various special

 * instructions, e.g. for clamping coordiantes or generating an address.

 * They couldn't just have added an equivalent to TIC now, couldn't they ?

 */

#define NVE4_SU_INFO_ADDR   0x00

#define NVE4_SU_INFO_FMT    0x04

#define NVE4_SU_INFO_DIM_X  0x08

#define NVE4_SU_INFO_PITCH  0x0c

#define NVE4_SU_INFO_DIM_Y  0x10

#define NVE4_SU_INFO_ARRAY  0x14

#define NVE4_SU_INFO_DIM_Z  0x18

#define NVE4_SU_INFO_UNK1C  0x1c

#define NVE4_SU_INFO_WIDTH  0x20

#define NVE4_SU_INFO_HEIGHT 0x24

#define NVE4_SU_INFO_DEPTH  0x28

#define NVE4_SU_INFO_TARGET 0x2c

#define NVE4_SU_INFO_CALL   0x30

#define NVE4_SU_INFO_RAW_X  0x34

#define NVE4_SU_INFO_MS_X   0x38

#define NVE4_SU_INFO_MS_Y   0x3c

#define NVE4_SU_INFO__STRIDE 0x40

#define NVE4_SU_INFO_DIM(i)  (0x08 + (i) * 8)

#define NVE4_SU_INFO_SIZE(i) (0x20 + (i) * 4)

#define NVE4_SU_INFO_MS(i)   (0x38 + (i) * 4)

static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c)

{

   switch (su->tex.target.getEnum()) {

   case TEX_TARGET_BUFFER:      return NV50_IR_SUBOP_SUCLAMP_PL(0, 1);

   case TEX_TARGET_RECT:        return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_1D:          return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_1D_ARRAY:    return (c == 1) ?

                                   NV50_IR_SUBOP_SUCLAMP_PL(0, 2) :

                                   NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_2D:          return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);

   case TEX_TARGET_2D_MS:       return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);

   case TEX_TARGET_2D_ARRAY:    return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_3D:          return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_CUBE:        return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   case TEX_TARGET_CUBE_ARRAY:  return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);

   default:

      assert(0);

      return 0;

   }

}

void