Subversion Repositories Kolibri OS

/*

 * Copyright 2012 Christoph Bumiller

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the "Software"),

 * to deal in the Software without restriction, including without limitation

 * the rights to use, copy, modify, merge, publish, distribute, sublicense,

 * and/or sell copies of the Software, and to permit persons to whom the

 * Software is furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice shall be included in

 * all copies or substantial portions of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR

 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,

 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

 * OTHER DEALINGS IN THE SOFTWARE.

 */

#include "nv50_ir_target_nvc0.h"

// CodeEmitter for GK110 encoding of the Fermi/Kepler ISA.

namespace nv50_ir {

class CodeEmitterGK110 : public CodeEmitter

{

public:

   CodeEmitterGK110(const TargetNVC0 *);

   virtual bool emitInstruction(Instruction *);

   virtual uint32_t getMinEncodingSize(const Instruction *) const;

   virtual void prepareEmission(Function *);

   inline void setProgramType(Program::Type pType) { progType = pType; }

private:

   const TargetNVC0 *targNVC0;

   Program::Type progType;

   const bool writeIssueDelays;

private:

   void emitForm_21(const Instruction *, uint32_t opc2, uint32_t opc1);

   void emitForm_C(const Instruction *, uint32_t opc, uint8_t ctg);

   void emitForm_L(const Instruction *, uint32_t opc, uint8_t ctg, Modifier);

   void emitPredicate(const Instruction *);

   void setCAddress14(const ValueRef&);

   void setShortImmediate(const Instruction *, const int s);

   void setImmediate32(const Instruction *, const int s, Modifier);

   void modNegAbsF32_3b(const Instruction *, const int s);

   void emitCondCode(CondCode cc, int pos, uint8_t mask);

   void emitInterpMode(const Instruction *);

   void emitLoadStoreType(DataType ty, const int pos);

   void emitCachingMode(CacheMode c, const int pos);

   inline uint8_t getSRegEncoding(const ValueRef&);

   void emitRoundMode(RoundMode, const int pos, const int rintPos);

   void emitRoundModeF(RoundMode, const int pos);

   void emitRoundModeI(RoundMode, const int pos);

   void emitNegAbs12(const Instruction *);

   void emitNOP(const Instruction *);

   void emitLOAD(const Instruction *);

   void emitSTORE(const Instruction *);

   void emitMOV(const Instruction *);

   void emitINTERP(const Instruction *);

   void emitPFETCH(const Instruction *);

   void emitVFETCH(const Instruction *);

   void emitEXPORT(const Instruction *);

   void emitOUT(const Instruction *);

   void emitUADD(const Instruction *);

   void emitFADD(const Instruction *);

   void emitIMUL(const Instruction *);

   void emitFMUL(const Instruction *);

   void emitIMAD(const Instruction *);

   void emitISAD(const Instruction *);

   void emitFMAD(const Instruction *);

   void emitNOT(const Instruction *);

   void emitLogicOp(const Instruction *, uint8_t subOp);

   void emitPOPC(const Instruction *);

   void emitINSBF(const Instruction *);

   void emitShift(const Instruction *);

   void emitSFnOp(const Instruction *, uint8_t subOp);

   void emitCVT(const Instruction *);

   void emitMINMAX(const Instruction *);

   void emitPreOp(const Instruction *);

   void emitSET(const CmpInstruction *);

   void emitSLCT(const CmpInstruction *);

   void emitSELP(const Instruction *);

   void emitTEXBAR(const Instruction *);

   void emitTEX(const TexInstruction *);

   void emitTEXCSAA(const TexInstruction *);

   void emitTXQ(const TexInstruction *);

   void emitQUADOP(const Instruction *, uint8_t qOp, uint8_t laneMask);

   void emitFlow(const Instruction *);

   inline void defId(const ValueDef&, const int pos);

   inline void srcId(const ValueRef&, const int pos);

   inline void srcId(const ValueRef *, const int pos);

   inline void srcId(const Instruction *, int s, const int pos);

   inline void srcAddr32(const ValueRef&, const int pos); // address / 4

   inline bool isLIMM(const ValueRef&, DataType ty, bool mod = false);

};

#define GK110_GPR_ZERO 255

#define NEG_(b, s) \

   if (i->src(s).mod.neg()) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)

#define ABS_(b, s) \

   if (i->src(s).mod.abs()) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)

#define NOT_(b, s) if (i->src(s).mod & Modifier(NV50_IR_MOD_NOT))       \

   code[(0x##b) / 32] |= 1 << ((0x##b) % 32)

#define FTZ_(b) if (i->ftz) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)

#define SAT_(b) if (i->saturate) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)

#define RND_(b, t) emitRoundMode##t(i->rnd, 0x##b)

#define SDATA(a) ((a).rep()->reg.data)

#define DDATA(a) ((a).rep()->reg.data)

void CodeEmitterGK110::srcId(const ValueRef& src, const int pos)

{

   code[pos / 32] |= (src.get() ? SDATA(src).id : GK110_GPR_ZERO) << (pos % 32);

}

void CodeEmitterGK110::srcId(const ValueRef *src, const int pos)

{

   code[pos / 32] |= (src ? SDATA(*src).id : GK110_GPR_ZERO) << (pos % 32);

}

void CodeEmitterGK110::srcId(const Instruction *insn, int s, int pos)

{

   int r = insn->srcExists(s) ? SDATA(insn->src(s)).id : GK110_GPR_ZERO;

   code[pos / 32] |= r << (pos % 32);

}

void CodeEmitterGK110::srcAddr32(const ValueRef& src, const int pos)

{

   code[pos / 32] |= (SDATA(src).offset >> 2) << (pos % 32);

}

void CodeEmitterGK110::defId(const ValueDef& def, const int pos)

{

   code[pos / 32] |= (def.get() ? DDATA(def).id : GK110_GPR_ZERO) << (pos % 32);

}

bool CodeEmitterGK110::isLIMM(const ValueRef& ref, DataType ty, bool mod)

{

   const ImmediateValue *imm = ref.get()->asImm();

   return imm && (imm->reg.data.u32 & ((ty == TYPE_F32) ? 0xfff : 0xfff00000));

}

void

CodeEmitterGK110::emitRoundMode(RoundMode rnd, const int pos, const int rintPos)

{

   bool rint = false;

   uint8_t n;

   switch (rnd) {

   case ROUND_MI: rint = true; /* fall through */ case ROUND_M: n = 1; break;

   case ROUND_PI: rint = true; /* fall through */ case ROUND_P: n = 2; break;

   case ROUND_ZI: rint = true; /* fall through */ case ROUND_Z: n = 3; break;

   default:

      rint = rnd == ROUND_NI;

      n = 0;

      assert(rnd == ROUND_N || rnd == ROUND_NI);

      break;

   }

   code[pos / 32] |= n << (pos % 32);

   if (rint && rintPos >= 0)

      code[rintPos / 32] |= 1 << (rintPos % 32);

}

void

CodeEmitterGK110::emitRoundModeF(RoundMode rnd, const int pos)

{

   uint8_t n;

   switch (rnd) {

   case ROUND_M: n = 1; break;

   case ROUND_P: n = 2; break;

   case ROUND_Z: n = 3; break;

   default:

      n = 0;

      assert(rnd == ROUND_N);

      break;

   }

   code[pos / 32] |= n << (pos % 32);

}

void

CodeEmitterGK110::emitRoundModeI(RoundMode rnd, const int pos)

{

   uint8_t n;

   switch (rnd) {

   case ROUND_MI: n = 1; break;

   case ROUND_PI: n = 2; break;

   case ROUND_ZI: n = 3; break;

   default:

      n = 0;

      assert(rnd == ROUND_NI);

      break;

   }

   code[pos / 32] |= n << (pos % 32);

}

void CodeEmitterGK110::emitCondCode(CondCode cc, int pos, uint8_t mask)

{

   uint8_t n;

   switch (cc) {

   case CC_FL:  n = 0x00; break;

   case CC_LT:  n = 0x01; break;

   case CC_EQ:  n = 0x02; break;

   case CC_LE:  n = 0x03; break;

   case CC_GT:  n = 0x04; break;

   case CC_NE:  n = 0x05; break;

   case CC_GE:  n = 0x06; break;

   case CC_LTU: n = 0x09; break;

   case CC_EQU: n = 0x0a; break;

   case CC_LEU: n = 0x0b; break;

   case CC_GTU: n = 0x0c; break;

   case CC_NEU: n = 0x0d; break;

   case CC_GEU: n = 0x0e; break;

   case CC_TR:  n = 0x0f; break;

   case CC_NO:  n = 0x10; break;

   case CC_NC:  n = 0x11; break;

   case CC_NS:  n = 0x12; break;

   case CC_NA:  n = 0x13; break;

   case CC_A:   n = 0x14; break;

   case CC_S:   n = 0x15; break;

   case CC_C:   n = 0x16; break;

   case CC_O:   n = 0x17; break;

   default:

      n = 0;

      assert(!"invalid condition code");

      break;

   }

   code[pos / 32] |= (n & mask) << (pos % 32);

}

void

CodeEmitterGK110::emitPredicate(const Instruction *i)

{

   if (i->predSrc >= 0) {

      srcId(i->src(i->predSrc), 18);

      if (i->cc == CC_NOT_P)

         code[0] |= 8 << 18; // negate

      assert(i->getPredicate()->reg.file == FILE_PREDICATE);

   } else {

      code[0] |= 7 << 18;

   }

}

void

CodeEmitterGK110::setCAddress14(const ValueRef& src)

{

   const int32_t addr = src.get()->asSym()->reg.data.offset / 4;

   code[0] |= (addr & 0x01ff) << 23;

   code[1] |= (addr & 0x3e00) >> 9;

}

void

CodeEmitterGK110::setShortImmediate(const Instruction *i, const int s)

{

   const uint32_t u32 = i->getSrc(s)->asImm()->reg.data.u32;

   const uint64_t u64 = i->getSrc(s)->asImm()->reg.data.u64;

   if (i->sType == TYPE_F32) {

      assert(!(u32 & 0x00000fff));

      code[0] |= ((u32 & 0x001ff000) >> 12) << 23;

      code[1] |= ((u32 & 0x7fe00000) >> 21);

      code[1] |= ((u32 & 0x80000000) >> 4);

   } else

   if (i->sType == TYPE_F64) {

      assert(!(u64 & 0x00000fffffffffffULL));

      code[0] |= ((u64 & 0x001ff00000000000ULL) >> 44) << 23;

      code[1] |= ((u64 & 0x7fe0000000000000ULL) >> 53);

      code[1] |= ((u64 & 0x8000000000000000ULL) >> 36);

   } else {

      assert((u32 & 0xfff00000) == 0 || (u32 & 0xfff00000) == 0xfff00000);

      code[0] |= (u32 & 0x001ff) << 23;

      code[1] |= (u32 & 0x7fe00) >> 9;

      code[1] |= (u32 & 0x80000) << 8;

   }

}

void

CodeEmitterGK110::setImmediate32(const Instruction *i, const int s,

                                 Modifier mod)

{

   uint32_t u32 = i->getSrc(s)->asImm()->reg.data.u32;

   if (mod) {

      ImmediateValue imm(i->getSrc(s)->asImm(), i->sType);

      mod.applyTo(imm);

      u32 = imm.reg.data.u32;

   }

   code[0] |= u32 << 23;

   code[1] |= u32 >> 9;

}

void

CodeEmitterGK110::emitForm_L(const Instruction *i, uint32_t opc, uint8_t ctg,

                             Modifier mod)

{

   code[0] = ctg;

   code[1] = opc << 20;

   emitPredicate(i);

   defId(i->def(0), 2);

   for (int s = 0; s < 3 && i->srcExists(s); ++s) {

      switch (i->src(s).getFile()) {

      case FILE_GPR:

         srcId(i->src(s), s ? 42 : 10);

         break;

      case FILE_IMMEDIATE:

         setImmediate32(i, s, mod);

         break;

      default:

         break;

      }

   }

}

void

CodeEmitterGK110::emitForm_C(const Instruction *i, uint32_t opc, uint8_t ctg)

{

   code[0] = ctg;

   code[1] = opc << 20;

   emitPredicate(i);

   defId(i->def(0), 2);

   switch (i->src(0).getFile()) {

   case FILE_MEMORY_CONST:

      code[1] |= 0x4 << 28;

      setCAddress14(i->src(0));

      break;

   case FILE_GPR:

      code[1] |= 0xc << 28;

      srcId(i->src(0), 23);

      break;

   default:

      assert(0);

      break;

   }

}

// 0x2 for GPR, c[] and 0x1 for short immediate

void

CodeEmitterGK110::emitForm_21(const Instruction *i, uint32_t opc2,

                              uint32_t opc1)

{

   const bool imm = i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE;

   int s1 = 23;

   if (i->srcExists(2) && i->src(2).getFile() == FILE_MEMORY_CONST)

      s1 = 42;

   if (imm) {

      code[0] = 0x1;

      code[1] = opc1 << 20;

   } else {

      code[0] = 0x2;

      code[1] = (0xc << 28) | (opc2 << 20);

   }

   emitPredicate(i);

   defId(i->def(0), 2);

   for (int s = 0; s < 3 && i->srcExists(s); ++s) {

      switch (i->src(s).getFile()) {

      case FILE_MEMORY_CONST:

         code[1] &= (s == 2) ? ~(0x4 << 28) : ~(0x8 << 28);

         setCAddress14(i->src(s));

         code[1] |= i->getSrc(s)->reg.fileIndex << 5;

         break;

      case FILE_IMMEDIATE:

         setShortImmediate(i, s);

         break;

      case FILE_GPR:

         srcId(i->src(s), s ? ((s == 2) ? 42 : s1) : 10);

         break;

      default:

         // ignore here, can be predicate or flags, but must not be address

         break;

      }

   }

   // 0x0 = invalid

   // 0xc = rrr

   // 0x8 = rrc

   // 0x4 = rcr

   assert(imm || (code[1] & (0xc << 28)));

}

inline void

CodeEmitterGK110::modNegAbsF32_3b(const Instruction *i, const int s)

{

   if (i->src(s).mod.abs()) code[1] &= ~(1 << 27);

   if (i->src(s).mod.neg()) code[1] ^=  (1 << 27);

}

void

CodeEmitterGK110::emitNOP(const Instruction *i)

{

   code[0] = 0x00003c02;

   code[1] = 0x85800000;

   if (i)

      emitPredicate(i);

   else

      code[0] = 0x001c3c02;

}

void

CodeEmitterGK110::emitFMAD(const Instruction *i)

{

   assert(!isLIMM(i->src(1), TYPE_F32));

   emitForm_21(i, 0x0c0, 0x940);

   NEG_(34, 2);

   SAT_(35);

   RND_(36, F);

   FTZ_(38);

   bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();

   if (code[0] & 0x1) {

      if (neg1)

         code[1] ^= 1 << 27;

   } else

   if (neg1) {

      code[1] |= 1 << 19;

   }

}

void

CodeEmitterGK110::emitFMUL(const Instruction *i)

{

   bool neg = (i->src(0).mod ^ i->src(1).mod).neg();

   assert(i->postFactor >= -3 && i->postFactor <= 3);

   if (isLIMM(i->src(1), TYPE_F32)) {

      emitForm_L(i, 0x200, 0x2, Modifier(0));

      FTZ_(38);

      SAT_(3a);

      if (neg)

         code[1] ^= 1 << 22;

      assert(i->postFactor == 0);

   } else {

      emitForm_21(i, 0x234, 0xc34);

      RND_(2a, F);

      FTZ_(2f);

      SAT_(35);

      if (code[0] & 0x1) {

         if (neg)

            code[1] ^= 1 << 27;

      } else

      if (neg) {

         code[1] |= 1 << 19;

      }

   }

}

void

CodeEmitterGK110::emitIMUL(const Instruction *i)

{

   assert(!i->src(0).mod.neg() && !i->src(1).mod.neg());

   assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());

   if (isLIMM(i->src(1), TYPE_S32)) {

      emitForm_L(i, 0x280, 2, Modifier(0));

      assert(i->subOp != NV50_IR_SUBOP_MUL_HIGH);

      if (i->sType == TYPE_S32)

         code[1] |= 3 << 25;

   } else {

      emitForm_21(i, 0x21c, 0xc1c);

      if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)

         code[1] |= 1 << 10;

      if (i->sType == TYPE_S32)

         code[1] |= 3 << 11;

   }

}

void

CodeEmitterGK110::emitFADD(const Instruction *i)

{

   if (isLIMM(i->src(1), TYPE_F32)) {

      assert(i->rnd == ROUND_N);

      assert(!i->saturate);

      emitForm_L(i, 0x400, 0, i->src(1).mod);

      FTZ_(3a);

      NEG_(3b, 0);

      ABS_(39, 0);

   } else {

      emitForm_21(i, 0x22c, 0xc2c);

      FTZ_(2f);

      RND_(2a, F);

      ABS_(31, 0);

      NEG_(33, 0);

      if (code[0] & 0x1) {

         modNegAbsF32_3b(i, 1);

      } else {

         ABS_(34, 1);

         NEG_(30, 1);

      }

   }

}

void

CodeEmitterGK110::emitUADD(const Instruction *i)

{

   uint8_t addOp = (i->src(0).mod.neg() << 1) | i->src(1).mod.neg();

   if (i->op == OP_SUB)

      addOp ^= 1;

   assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());

   if (isLIMM(i->src(1), TYPE_S32)) {

      emitForm_L(i, 0x400, 1, Modifier((addOp & 1) ? NV50_IR_MOD_NEG : 0));

      if (addOp & 2)

         code[1] |= 1 << 27;

      assert(!i->defExists(1));

      assert(i->flagsSrc < 0);

      SAT_(39);

   } else {

      emitForm_21(i, 0x208, 0xc08);

      assert(addOp != 3); // would be add-plus-one

      code[1] |= addOp << 19;

      if (i->defExists(1))

         code[1] |= 1 << 18; // write carry

      if (i->flagsSrc >= 0)

         code[1] |= 1 << 14; // add carry

      SAT_(35);

   }

}

// TODO: shl-add

void

CodeEmitterGK110::emitIMAD(const Instruction *i)

{

   uint8_t addOp =

      (i->src(2).mod.neg() << 1) | (i->src(0).mod.neg() ^ i->src(1).mod.neg());

   emitForm_21(i, 0x100, 0xa00);

   assert(addOp != 3);

   code[1] |= addOp << 26;

   if (i->sType == TYPE_S32)

      code[1] |= (1 << 19) | (1 << 24);

   if (code[0] & 0x1) {

      assert(!i->subOp);

      SAT_(39);

   } else {

      if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)

         code[1] |= 1 << 25;

      SAT_(35);

   }

}

void

CodeEmitterGK110::emitISAD(const Instruction *i)

{

   assert(i->dType == TYPE_S32 || i->dType == TYPE_U32);

   emitForm_21(i, 0x1fc, 0xb74);

   if (i->dType == TYPE_S32)

      code[1] |= 1 << 19;

}

void

CodeEmitterGK110::emitNOT(const Instruction *i)

{

   code[0] = 0x0003fc02; // logop(mov2) dst, 0, not src

   code[1] = 0x22003800;

   emitPredicate(i);

   defId(i->def(0), 2);

   switch (i->src(0).getFile()) {

   case FILE_GPR:

      code[1] |= 0xc << 28;

      srcId(i->src(0), 23);

      break;

   case FILE_MEMORY_CONST:

      code[1] |= 0x4 << 28;

      setCAddress14(i->src(1));

      break;

   default:

      assert(0);

      break;

   }

}

void

CodeEmitterGK110::emitLogicOp(const Instruction *i, uint8_t subOp)

{

   assert(!(i->src(0).mod & Modifier(NV50_IR_MOD_NOT))); // XXX: find me

   if (isLIMM(i->src(1), TYPE_S32)) {

      emitForm_L(i, 0x200, 0, i->src(1).mod);

      code[1] |= subOp << 24;

   } else {

      emitForm_21(i, 0x220, 0xc20);

      code[1] |= subOp << 12;

      NOT_(2b, 1);

   }

   assert(!(code[0] & 0x1) || !(i->src(1).mod & Modifier(NV50_IR_MOD_NOT)));

}

void

CodeEmitterGK110::emitPOPC(const Instruction *i)

{

   assert(!isLIMM(i->src(1), TYPE_S32, true));

   emitForm_21(i, 0x204, 0xc04);

   NOT_(2a, 0);

   if (!(code[0] & 0x1))

      NOT_(2b, 1);

}

void

CodeEmitterGK110::emitINSBF(const Instruction *i)

{

   emitForm_21(i, 0x1f8, 0xb78);

}

void

CodeEmitterGK110::emitShift(const Instruction *i)

{

   const bool sar = i->op == OP_SHR && isSignedType(i->sType);

   if (sar) {

      emitForm_21(i, 0x214, 0x014);

      code[1] |= 1 << 19;

   } else

   if (i->op == OP_SHR) {

      // this is actually RSHF

      emitForm_21(i, 0x27c, 0x87c);

      code[1] |= GK110_GPR_ZERO << 10;

   } else {

      // this is actually LSHF

      emitForm_21(i, 0x1fc, 0xb7c);

      code[1] |= GK110_GPR_ZERO << 10;

   }

   if (i->subOp == NV50_IR_SUBOP_SHIFT_WRAP) {

      if (!sar)

         code[1] |= 1 << 21;

      // XXX: find wrap modifier for SHR S32

   }

}

void

CodeEmitterGK110::emitPreOp(const Instruction *i)

{

   emitForm_21(i, 0x248, -1);

   if (i->op == OP_PREEX2)

      code[1] |= 1 << 10;

   NEG_(30, 0);

   ABS_(34, 0);

}

void

CodeEmitterGK110::emitSFnOp(const Instruction *i, uint8_t subOp)

{

   code[0] = 0x00000002 | (subOp << 23);

   code[1] = 0x84000000;

   emitPredicate(i);

   defId(i->def(0), 2);

   srcId(i->src(0), 10);

   NEG_(33, 0);

   ABS_(31, 0);

   // XXX: find saturate

}

void

CodeEmitterGK110::emitMINMAX(const Instruction *i)

{

   uint32_t op2, op1;

   switch (i->dType) {

   case TYPE_U32:

   case TYPE_S32:

      op2 = 0x210;

      op1 = 0xc10;

      break;

   case TYPE_F32:

      op2 = 0x230;

      op1 = 0xc30;

      break;

   case TYPE_F64:

      op2 = 0x228;

      op1 = 0xc28;

      break;

   default:

      assert(0);

      op2 = 0;

      op1 = 0;

      break;

   }

   emitForm_21(i, op2, op1);

   if (i->dType == TYPE_S32)

      code[1] |= 1 << 19;

   code[1] |= (i->op == OP_MIN) ? 0x1c00 : 0x3c00; // [!]pt

   FTZ_(2f);

   ABS_(31, 0);

   NEG_(33, 0);

   if (code[0] & 0x1) {

      modNegAbsF32_3b(i, 1);

   } else {

      ABS_(34, 1);

      NEG_(30, 1);

   }

}

void

CodeEmitterGK110::emitCVT(const Instruction *i)

{

   const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);

   const bool f2i = !isFloatType(i->dType) && isFloatType(i->sType);

   const bool i2f = isFloatType(i->dType) && !isFloatType(i->sType);

   bool sat = i->saturate;

   bool abs = i->src(0).mod.abs();

   bool neg = i->src(0).mod.neg();

   RoundMode rnd = i->rnd;

   switch (i->op) {

   case OP_CEIL:  rnd = f2f ? ROUND_PI : ROUND_P; break;

   case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;

   case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;

   case OP_SAT: sat = true; break;

   case OP_NEG: neg = !neg; break;

   case OP_ABS: abs = true; neg = false; break;

   default:

      break;

   }

   uint32_t op;

   if      (f2f) op = 0x254;

   else if (f2i) op = 0x258;

   else if (i2f) op = 0x25c;

   else          op = 0x260;

   emitForm_C(i, op, 0x2);

   FTZ_(2f);

   if (neg) code[1] |= 1 << 16;

   if (abs) code[1] |= 1 << 20;

   if (sat) code[1] |= 1 << 21;

   emitRoundMode(rnd, 32 + 10, f2f ? (32 + 13) : -1);

   code[0] |= typeSizeofLog2(i->dType) << 10;

   code[0] |= typeSizeofLog2(i->sType) << 12;

   if (isSignedIntType(i->dType))

      code[0] |= 0x4000;

   if (isSignedIntType(i->sType))

      code[0] |= 0x8000;

}

void

CodeEmitterGK110::emitSET(const CmpInstruction *i)

{

   uint16_t op1, op2;

   if (i->def(0).getFile() == FILE_PREDICATE) {

      switch (i->sType) {

      case TYPE_F32: op2 = 0x1d8; op1 = 0xb58; break;

      case TYPE_F64: op2 = 0x1c0; op1 = 0xb40; break;

      default:

         op2 = 0x1b0;

         op1 = 0xb30;

         break;

      }

      emitForm_21(i, op2, op1);

      NEG_(2e, 0);

      ABS_(9, 0);

      if (!(code[0] & 0x1)) {

         NEG_(8, 1);

         ABS_(2f, 1);

      } else {

         modNegAbsF32_3b(i, 1);

      }

      FTZ_(32);

      // normal DST field is negated predicate result

      code[0] = (code[0] & ~0xfc) | ((code[0] << 3) & 0xe0);

      if (i->defExists(1))

         defId(i->def(1), 2);

   else

      code[0] |= 0x1c;

   } else {

      switch (i->sType) {

      case TYPE_F32: op2 = 0x000; op1 = 0x820; break;

      case TYPE_F64: op2 = 0x080; op1 = 0x900; break;

      default:

         op2 = 0x1a8;

         op1 = 0xb28;

         break;

      }

      emitForm_21(i, op2, op1);

      NEG_(2e, 0);

      ABS_(39, 0);

      if (!(code[0] & 0x1)) {

         NEG_(38, 1);

         ABS_(2f, 1);

      } else {

         modNegAbsF32_3b(i, 1);

      }

      FTZ_(3a);

   }

   if (i->sType == TYPE_S32)

      code[1] |= 1 << 19;

   if (i->op != OP_SET) {

      switch (i->op) {

      case OP_SET_AND: code[1] |= 0x0 << 16; break;

      case OP_SET_OR:  code[1] |= 0x1 << 16; break;

      case OP_SET_XOR: code[1] |= 0x2 << 16; break;

      default:

         assert(0);

         break;

      }

      srcId(i->src(2), 0x2a);

   } else {

      code[1] |= 0x7 << 10;

   }

   emitCondCode(i->setCond,

                isFloatType(i->sType) ? 0x33 : 0x34,

                isFloatType(i->sType) ? 0xf : 0x7);

}

void

CodeEmitterGK110::emitSLCT(const CmpInstruction *i)

{

   CondCode cc = i->setCond;

   if (i->src(2).mod.neg())

      cc = reverseCondCode(cc);

   if (i->dType == TYPE_F32) {

      emitForm_21(i, 0x1d0, 0xb50);

      FTZ_(32);

      emitCondCode(cc, 0x33, 0xf);

   } else {

      emitForm_21(i, 0x1a4, 0xb20);

      emitCondCode(cc, 0x34, 0x7);

   }

}

void CodeEmitterGK110::emitSELP(const Instruction *i)

{

   emitForm_21(i, 0x250, 0x050);

   if ((i->cc == CC_NOT_P) ^ (bool)(i->src(2).mod & Modifier(NV50_IR_MOD_NOT)))

      code[1] |= 1 << 13;

}

void CodeEmitterGK110::emitTEXBAR(const Instruction *i)

{

   code[0] = 0x00000002 | (i->subOp << 23);

   code[1] = 0x77000000;

   emitPredicate(i);

}

void CodeEmitterGK110::emitTEXCSAA(const TexInstruction *i)

{

   emitNOP(i); // TODO

}

static inline bool

isNextIndependentTex(const TexInstruction *i)

{

   if (!i->next || !isTextureOp(i->next->op))

      return false;

   if (i->getDef(0)->interfers(i->next->getSrc(0)))

      return false;

   return !i->next->srcExists(1) || !i->getDef(0)->interfers(i->next->getSrc(1));

}

void

CodeEmitterGK110::emitTEX(const TexInstruction *i)

{

   const bool ind = i->tex.rIndirectSrc >= 0;

   if (ind) {

      code[0] = 0x00000002;

      switch (i->op) {

      case OP_TXD:

         code[1] = 0x7e000000;

         break;

      default:

         code[1] = 0x7d800000;

         break;

      }

   } else {

      switch (i->op) {

      case OP_TXD:

         code[0] = 0x00000002;

         code[1] = 0x76000000;

         break;

      default:

         code[0] = 0x00000001;

         code[1] = 0x60000000;

         break;

      }

      code[1] |= i->tex.r << 15;

   }

   code[1] |= isNextIndependentTex(i) ? 0x1 : 0x2; // t : p mode

   // if (i->tex.liveOnly)

   //    ?

   switch (i->op) {

   case OP_TEX: break;

   case OP_TXB: code[1] |= 0x2000; break;

   case OP_TXL: code[1] |= 0x3000; break;

   case OP_TXF: break; // XXX

   case OP_TXG: break; // XXX

   case OP_TXD: break;

   default:

      assert(!"invalid texture op");

      break;

   }

   /*

   if (i->op == OP_TXF) {

      if (!i->tex.levelZero)

         code[1] |= 0x02000000;

   } else */

   if (i->tex.levelZero) {

      code[1] |= 0x1000;

   }

   // if (i->op != OP_TXD && i->tex.derivAll)

   //   code[1] |= 1 << 13;

   emitPredicate(i);

   code[1] |= i->tex.mask << 2;

   const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)

   defId(i->def(0), 2);

   srcId(i->src(0), 10);

   srcId(i, src1, 23);

   // if (i->op == OP_TXG) code[0] |= i->tex.gatherComp << 5;

   // texture target:

   code[1] |= (i->tex.target.isCube() ? 3 : (i->tex.target.getDim() - 1)) << 7;

   if (i->tex.target.isArray())

      code[1] |= 0x40;

   // if (i->tex.target.isShadow())

   //   ?

   // if (i->tex.target == TEX_TARGET_2D_MS ||

   //     i->tex.target == TEX_TARGET_2D_MS_ARRAY)

   //   ?

   if (i->srcExists(src1) && i->src(src1).getFile() == FILE_IMMEDIATE) {

      // ?

   }

   // if (i->tex.useOffsets)

   //   ?

}

void

CodeEmitterGK110::emitTXQ(const TexInstruction *i)

{

   emitNOP(i); // TODO

}

void

CodeEmitterGK110::emitQUADOP(const Instruction *i, uint8_t qOp, uint8_t laneMask)

{

   emitNOP(i); // TODO

}

void

CodeEmitterGK110::emitFlow(const Instruction *i)

{

   const FlowInstruction *f = i->asFlow();

   unsigned mask; // bit 0: predicate, bit 1: target

   code[0] = 0x00000000;

   switch (i->op) {

   case OP_BRA:

      code[1] = f->absolute ? 0x00000 : 0x12000000; // XXX

      // if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)

      //   code[0] |= 0x4000;

      mask = 3;

      break;

   case OP_CALL:

      code[1] = f->absolute ? 0x00000 : 0x13000000; // XXX

      // if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)

      //   code[0] |= 0x4000;

      mask = 2;

      break;

   case OP_EXIT:    code[1] = 0x18000000; mask = 1; break;

   case OP_RET:     code[1] = 0x19000000; mask = 1; break;

   case OP_DISCARD: code[1] = 0x19800000; mask = 1; break; // XXX: guess

   case OP_BREAK:   code[1] = 0x1a800000; mask = 1; break; // XXX: guess

   case OP_CONT:    code[1] = 0x1b000000; mask = 1; break; // XXX: guess

   case OP_JOINAT:   code[1] = 0x14800000; mask = 2; break;

   case OP_PREBREAK: code[1] = 0x15000000; mask = 2; break; // XXX: guess

   case OP_PRECONT:  code[1] = 0x15800000; mask = 2; break; // XXX: guess

   case OP_PRERET:   code[1] = 0x16000000; mask = 2; break; // XXX: guess

   case OP_QUADON:  code[1] = 0x1c000000; mask = 0; break; // XXX: guess

   case OP_QUADPOP: code[1] = 0x1c800000; mask = 0; break; // XXX: guess

   case OP_BRKPT:   code[1] = 0x1d000000; mask = 0; break; // XXX: guess

   default:

      assert(!"invalid flow operation");

      return;

   }

   if (mask & 1) {

      emitPredicate(i);

      if (i->flagsSrc < 0)

         code[0] |= 0x3c;

   }

   if (!f)

      return;

   // TODO

   /*

   if (f->allWarp)

      code[0] |= 1 << 15;

   if (f->limit)

      code[0] |= 1 << 16;

   */

   if (f->op == OP_CALL) {

      if (f->builtin) {

         assert(f->absolute);

         uint32_t pcAbs = targNVC0->getBuiltinOffset(f->target.builtin);

         addReloc(RelocEntry::TYPE_BUILTIN, 0, pcAbs, 0xff800000, 23);

         addReloc(RelocEntry::TYPE_BUILTIN, 1, pcAbs, 0x007fffff, -9);

      } else {

         assert(!f->absolute);

         int32_t pcRel = f->target.fn->binPos - (codeSize + 8);

         code[0] |= (pcRel & 0x1ff) << 23;

         code[1] |= (pcRel >> 9) & 0x7fff;

      }

   } else

   if (mask & 2) {