Subversion Repositories Kolibri OS

Rev

Rev 5060 | Rev 5139 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008-2010 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *    Zou Nan hai <nanhai.zou@intel.com>
  26.  *    Xiang Hai hao<haihao.xiang@intel.com>
  27.  *
  28.  */
  29.  
  30. #include <drm/drmP.h>
  31. #include "i915_drv.h"
  32. #include <drm/i915_drm.h>
  33. #include "i915_trace.h"
  34. #include "intel_drv.h"
  35.  
  36. /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
  37.  * but keeps the logic simple. Indeed, the whole purpose of this macro is just
  38.  * to give some inclination as to some of the magic values used in the various
  39.  * workarounds!
  40.  */
  41. #define CACHELINE_BYTES 64
  42.  
  43. static inline int __ring_space(int head, int tail, int size)
  44. {
  45.         int space = head - (tail + I915_RING_FREE_SPACE);
  46.         if (space < 0)
  47.                 space += size;
  48.         return space;
  49. }
  50.  
  51. static inline int ring_space(struct intel_ringbuffer *ringbuf)
  52. {
  53.         return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size);
  54. }
  55.  
  56. static bool intel_ring_stopped(struct intel_engine_cs *ring)
  57. {
  58.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  59.         return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
  60. }
  61.  
  62. void __intel_ring_advance(struct intel_engine_cs *ring)
  63. {
  64.         struct intel_ringbuffer *ringbuf = ring->buffer;
  65.         ringbuf->tail &= ringbuf->size - 1;
  66.         if (intel_ring_stopped(ring))
  67.                 return;
  68.         ring->write_tail(ring, ringbuf->tail);
  69. }
  70.  
  71. static int
  72. gen2_render_ring_flush(struct intel_engine_cs *ring,
  73.                        u32      invalidate_domains,
  74.                        u32      flush_domains)
  75. {
  76.         u32 cmd;
  77.         int ret;
  78.  
  79.         cmd = MI_FLUSH;
  80.         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  81.                 cmd |= MI_NO_WRITE_FLUSH;
  82.  
  83.         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  84.                 cmd |= MI_READ_FLUSH;
  85.  
  86.         ret = intel_ring_begin(ring, 2);
  87.         if (ret)
  88.                 return ret;
  89.  
  90.         intel_ring_emit(ring, cmd);
  91.         intel_ring_emit(ring, MI_NOOP);
  92.         intel_ring_advance(ring);
  93.  
  94.         return 0;
  95. }
  96.  
  97. static int
  98. gen4_render_ring_flush(struct intel_engine_cs *ring,
  99.                   u32   invalidate_domains,
  100.                   u32   flush_domains)
  101. {
  102.         struct drm_device *dev = ring->dev;
  103.         u32 cmd;
  104.         int ret;
  105.  
  106.         /*
  107.          * read/write caches:
  108.          *
  109.          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  110.          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  111.          * also flushed at 2d versus 3d pipeline switches.
  112.          *
  113.          * read-only caches:
  114.          *
  115.          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  116.          * MI_READ_FLUSH is set, and is always flushed on 965.
  117.          *
  118.          * I915_GEM_DOMAIN_COMMAND may not exist?
  119.          *
  120.          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  121.          * invalidated when MI_EXE_FLUSH is set.
  122.          *
  123.          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  124.          * invalidated with every MI_FLUSH.
  125.          *
  126.          * TLBs:
  127.          *
  128.          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  129.          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  130.          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  131.          * are flushed at any MI_FLUSH.
  132.          */
  133.  
  134.         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  135.         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
  136.                 cmd &= ~MI_NO_WRITE_FLUSH;
  137.         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  138.                 cmd |= MI_EXE_FLUSH;
  139.  
  140.         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
  141.             (IS_G4X(dev) || IS_GEN5(dev)))
  142.                 cmd |= MI_INVALIDATE_ISP;
  143.  
  144.         ret = intel_ring_begin(ring, 2);
  145.         if (ret)
  146.                 return ret;
  147.  
  148.         intel_ring_emit(ring, cmd);
  149.         intel_ring_emit(ring, MI_NOOP);
  150.         intel_ring_advance(ring);
  151.  
  152.         return 0;
  153. }
  154.  
  155. /**
  156.  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
  157.  * implementing two workarounds on gen6.  From section 1.4.7.1
  158.  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
  159.  *
  160.  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
  161.  * produced by non-pipelined state commands), software needs to first
  162.  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
  163.  * 0.
  164.  *
  165.  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
  166.  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
  167.  *
  168.  * And the workaround for these two requires this workaround first:
  169.  *
  170.  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
  171.  * BEFORE the pipe-control with a post-sync op and no write-cache
  172.  * flushes.
  173.  *
  174.  * And this last workaround is tricky because of the requirements on
  175.  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
  176.  * volume 2 part 1:
  177.  *
  178.  *     "1 of the following must also be set:
  179.  *      - Render Target Cache Flush Enable ([12] of DW1)
  180.  *      - Depth Cache Flush Enable ([0] of DW1)
  181.  *      - Stall at Pixel Scoreboard ([1] of DW1)
  182.  *      - Depth Stall ([13] of DW1)
  183.  *      - Post-Sync Operation ([13] of DW1)
  184.  *      - Notify Enable ([8] of DW1)"
  185.  *
  186.  * The cache flushes require the workaround flush that triggered this
  187.  * one, so we can't use it.  Depth stall would trigger the same.
  188.  * Post-sync nonzero is what triggered this second workaround, so we
  189.  * can't use that one either.  Notify enable is IRQs, which aren't
  190.  * really our business.  That leaves only stall at scoreboard.
  191.  */
  192. static int
  193. intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring)
  194. {
  195.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  196.         int ret;
  197.  
  198.  
  199.         ret = intel_ring_begin(ring, 6);
  200.         if (ret)
  201.                 return ret;
  202.  
  203.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  204.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  205.                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
  206.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  207.         intel_ring_emit(ring, 0); /* low dword */
  208.         intel_ring_emit(ring, 0); /* high dword */
  209.         intel_ring_emit(ring, MI_NOOP);
  210.         intel_ring_advance(ring);
  211.  
  212.         ret = intel_ring_begin(ring, 6);
  213.         if (ret)
  214.                 return ret;
  215.  
  216.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  217.         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
  218.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  219.         intel_ring_emit(ring, 0);
  220.         intel_ring_emit(ring, 0);
  221.         intel_ring_emit(ring, MI_NOOP);
  222.         intel_ring_advance(ring);
  223.  
  224.         return 0;
  225. }
  226.  
  227. static int
  228. gen6_render_ring_flush(struct intel_engine_cs *ring,
  229.                          u32 invalidate_domains, u32 flush_domains)
  230. {
  231.         u32 flags = 0;
  232.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  233.         int ret;
  234.  
  235.         /* Force SNB workarounds for PIPE_CONTROL flushes */
  236.         ret = intel_emit_post_sync_nonzero_flush(ring);
  237.         if (ret)
  238.                 return ret;
  239.  
  240.         /* Just flush everything.  Experiments have shown that reducing the
  241.          * number of bits based on the write domains has little performance
  242.          * impact.
  243.          */
  244.         if (flush_domains) {
  245.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  246.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  247.                 /*
  248.                  * Ensure that any following seqno writes only happen
  249.                  * when the render cache is indeed flushed.
  250.                  */
  251.                 flags |= PIPE_CONTROL_CS_STALL;
  252.         }
  253.         if (invalidate_domains) {
  254.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  255.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  256.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  257.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  258.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  259.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  260.                 /*
  261.                  * TLB invalidate requires a post-sync write.
  262.                  */
  263.                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
  264.         }
  265.  
  266.         ret = intel_ring_begin(ring, 4);
  267.         if (ret)
  268.                 return ret;
  269.  
  270.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  271.         intel_ring_emit(ring, flags);
  272.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
  273.         intel_ring_emit(ring, 0);
  274.         intel_ring_advance(ring);
  275.  
  276.         return 0;
  277. }
  278.  
  279. static int
  280. gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
  281. {
  282.         int ret;
  283.  
  284.         ret = intel_ring_begin(ring, 4);
  285.         if (ret)
  286.                 return ret;
  287.  
  288.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  289.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  290.                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
  291.         intel_ring_emit(ring, 0);
  292.         intel_ring_emit(ring, 0);
  293.         intel_ring_advance(ring);
  294.  
  295.         return 0;
  296. }
  297.  
  298. static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value)
  299. {
  300.         int ret;
  301.  
  302.         if (!ring->fbc_dirty)
  303.                 return 0;
  304.  
  305.         ret = intel_ring_begin(ring, 6);
  306.         if (ret)
  307.                 return ret;
  308.         /* WaFbcNukeOn3DBlt:ivb/hsw */
  309.         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  310.         intel_ring_emit(ring, MSG_FBC_REND_STATE);
  311.         intel_ring_emit(ring, value);
  312.         intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
  313.         intel_ring_emit(ring, MSG_FBC_REND_STATE);
  314.         intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
  315.         intel_ring_advance(ring);
  316.  
  317.         ring->fbc_dirty = false;
  318.         return 0;
  319. }
  320.  
  321. static int
  322. gen7_render_ring_flush(struct intel_engine_cs *ring,
  323.                        u32 invalidate_domains, u32 flush_domains)
  324. {
  325.         u32 flags = 0;
  326.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  327.         int ret;
  328.  
  329.         /*
  330.          * Ensure that any following seqno writes only happen when the render
  331.          * cache is indeed flushed.
  332.          *
  333.          * Workaround: 4th PIPE_CONTROL command (except the ones with only
  334.          * read-cache invalidate bits set) must have the CS_STALL bit set. We
  335.          * don't try to be clever and just set it unconditionally.
  336.          */
  337.         flags |= PIPE_CONTROL_CS_STALL;
  338.  
  339.         /* Just flush everything.  Experiments have shown that reducing the
  340.          * number of bits based on the write domains has little performance
  341.          * impact.
  342.          */
  343.         if (flush_domains) {
  344.         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  345.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  346.         }
  347.         if (invalidate_domains) {
  348.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  349.         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  350.         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  351.         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  352.         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  353.         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  354.                 /*
  355.                  * TLB invalidate requires a post-sync write.
  356.                  */
  357.                 flags |= PIPE_CONTROL_QW_WRITE;
  358.                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  359.  
  360.                 /* Workaround: we must issue a pipe_control with CS-stall bit
  361.                  * set before a pipe_control command that has the state cache
  362.                  * invalidate bit set. */
  363.                 gen7_render_ring_cs_stall_wa(ring);
  364.         }
  365.  
  366.         ret = intel_ring_begin(ring, 4);
  367.         if (ret)
  368.                 return ret;
  369.  
  370.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  371.         intel_ring_emit(ring, flags);
  372.         intel_ring_emit(ring, scratch_addr);
  373.         intel_ring_emit(ring, 0);
  374.         intel_ring_advance(ring);
  375.  
  376.         if (!invalidate_domains && flush_domains)
  377.                 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
  378.  
  379.         return 0;
  380. }
  381.  
  382. static int
  383. gen8_emit_pipe_control(struct intel_engine_cs *ring,
  384.                        u32 flags, u32 scratch_addr)
  385. {
  386.         int ret;
  387.  
  388.         ret = intel_ring_begin(ring, 6);
  389.         if (ret)
  390.                 return ret;
  391.  
  392.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
  393.         intel_ring_emit(ring, flags);
  394.         intel_ring_emit(ring, scratch_addr);
  395.         intel_ring_emit(ring, 0);
  396.         intel_ring_emit(ring, 0);
  397.         intel_ring_emit(ring, 0);
  398.         intel_ring_advance(ring);
  399.  
  400.         return 0;
  401. }
  402.  
  403. static int
  404. gen8_render_ring_flush(struct intel_engine_cs *ring,
  405.                        u32 invalidate_domains, u32 flush_domains)
  406. {
  407.         u32 flags = 0;
  408.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  409.         int ret;
  410.  
  411.         flags |= PIPE_CONTROL_CS_STALL;
  412.  
  413.         if (flush_domains) {
  414.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  415.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  416.         }
  417.         if (invalidate_domains) {
  418.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  419.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  420.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  421.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  422.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  423.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  424.                 flags |= PIPE_CONTROL_QW_WRITE;
  425.                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  426.  
  427.                 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
  428.                 ret = gen8_emit_pipe_control(ring,
  429.                                              PIPE_CONTROL_CS_STALL |
  430.                                              PIPE_CONTROL_STALL_AT_SCOREBOARD,
  431.                                              0);
  432.         if (ret)
  433.                 return ret;
  434.         }
  435.  
  436.         return gen8_emit_pipe_control(ring, flags, scratch_addr);
  437. }
  438.  
  439. static void ring_write_tail(struct intel_engine_cs *ring,
  440.                             u32 value)
  441. {
  442.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  443.         I915_WRITE_TAIL(ring, value);
  444. }
  445.  
  446. u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
  447. {
  448.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  449.         u64 acthd;
  450.  
  451.         if (INTEL_INFO(ring->dev)->gen >= 8)
  452.                 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
  453.                                          RING_ACTHD_UDW(ring->mmio_base));
  454.         else if (INTEL_INFO(ring->dev)->gen >= 4)
  455.                 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
  456.         else
  457.                 acthd = I915_READ(ACTHD);
  458.  
  459.         return acthd;
  460. }
  461.  
  462. static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
  463. {
  464.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  465.         u32 addr;
  466.  
  467.         addr = dev_priv->status_page_dmah->busaddr;
  468.         if (INTEL_INFO(ring->dev)->gen >= 4)
  469.                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
  470.         I915_WRITE(HWS_PGA, addr);
  471. }
  472.  
  473. static bool stop_ring(struct intel_engine_cs *ring)
  474. {
  475.         struct drm_i915_private *dev_priv = to_i915(ring->dev);
  476.  
  477.         if (!IS_GEN2(ring->dev)) {
  478.                 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
  479.                 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
  480.                         DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
  481.                         return false;
  482.                 }
  483.         }
  484.  
  485.         I915_WRITE_CTL(ring, 0);
  486.         I915_WRITE_HEAD(ring, 0);
  487.         ring->write_tail(ring, 0);
  488.  
  489.         if (!IS_GEN2(ring->dev)) {
  490.                 (void)I915_READ_CTL(ring);
  491.                 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
  492.         }
  493.  
  494.         return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
  495. }
  496.  
  497. static int init_ring_common(struct intel_engine_cs *ring)
  498. {
  499.         struct drm_device *dev = ring->dev;
  500.         struct drm_i915_private *dev_priv = dev->dev_private;
  501.         struct intel_ringbuffer *ringbuf = ring->buffer;
  502.         struct drm_i915_gem_object *obj = ringbuf->obj;
  503.         int ret = 0;
  504.  
  505.         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
  506.  
  507.         if (!stop_ring(ring)) {
  508.                 /* G45 ring initialization often fails to reset head to zero */
  509.                 DRM_DEBUG_KMS("%s head not reset to zero "
  510.                               "ctl %08x head %08x tail %08x start %08x\n",
  511.                               ring->name,
  512.                               I915_READ_CTL(ring),
  513.                               I915_READ_HEAD(ring),
  514.                               I915_READ_TAIL(ring),
  515.                               I915_READ_START(ring));
  516.  
  517.                 if (!stop_ring(ring)) {
  518.                         DRM_ERROR("failed to set %s head to zero "
  519.                                   "ctl %08x head %08x tail %08x start %08x\n",
  520.                                   ring->name,
  521.                                   I915_READ_CTL(ring),
  522.                                   I915_READ_HEAD(ring),
  523.                                   I915_READ_TAIL(ring),
  524.                                   I915_READ_START(ring));
  525.                         ret = -EIO;
  526.                         goto out;
  527.                 }
  528.         }
  529.  
  530.         if (I915_NEED_GFX_HWS(dev))
  531.                 intel_ring_setup_status_page(ring);
  532.         else
  533.                 ring_setup_phys_status_page(ring);
  534.  
  535.         /* Enforce ordering by reading HEAD register back */
  536.         I915_READ_HEAD(ring);
  537.  
  538.         /* Initialize the ring. This must happen _after_ we've cleared the ring
  539.          * registers with the above sequence (the readback of the HEAD registers
  540.          * also enforces ordering), otherwise the hw might lose the new ring
  541.          * register values. */
  542.         I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
  543.         I915_WRITE_CTL(ring,
  544.                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
  545.                         | RING_VALID);
  546.  
  547.         /* If the head is still not zero, the ring is dead */
  548.         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
  549.                      I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
  550.                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
  551.                 DRM_ERROR("%s initialization failed "
  552.                           "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
  553.                                 ring->name,
  554.                           I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
  555.                           I915_READ_HEAD(ring), I915_READ_TAIL(ring),
  556.                           I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
  557.                 ret = -EIO;
  558.                 goto out;
  559.         }
  560.  
  561.  
  562.                 ringbuf->head = I915_READ_HEAD(ring);
  563.                 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
  564.                 ringbuf->space = ring_space(ringbuf);
  565.                 ringbuf->last_retired_head = -1;
  566.  
  567.         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
  568.  
  569. out:
  570.         gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
  571.  
  572.         return ret;
  573. }
  574.  
  575. static int
  576. init_pipe_control(struct intel_engine_cs *ring)
  577. {
  578.         int ret;
  579.  
  580.         if (ring->scratch.obj)
  581.                 return 0;
  582.  
  583.         ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
  584.         if (ring->scratch.obj == NULL) {
  585.                 DRM_ERROR("Failed to allocate seqno page\n");
  586.                 ret = -ENOMEM;
  587.                 goto err;
  588.         }
  589.  
  590.         ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
  591.         if (ret)
  592.                 goto err_unref;
  593.  
  594.         ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
  595.         if (ret)
  596.                 goto err_unref;
  597.  
  598.         ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
  599.     ring->scratch.cpu_page = (void*)MapIoMem((addr_t)sg_page(ring->scratch.obj->pages->sgl),4096, PG_SW|0x100);
  600.         if (ring->scratch.cpu_page == NULL) {
  601.                 ret = -ENOMEM;
  602.                 goto err_unpin;
  603.         }
  604.  
  605.         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
  606.                          ring->name, ring->scratch.gtt_offset);
  607.         return 0;
  608.  
  609. err_unpin:
  610.         i915_gem_object_ggtt_unpin(ring->scratch.obj);
  611. err_unref:
  612.         drm_gem_object_unreference(&ring->scratch.obj->base);
  613. err:
  614.         return ret;
  615. }
  616.  
  617. static int init_render_ring(struct intel_engine_cs *ring)
  618. {
  619.         struct drm_device *dev = ring->dev;
  620.         struct drm_i915_private *dev_priv = dev->dev_private;
  621.         int ret = init_ring_common(ring);
  622.         if (ret)
  623.                 return ret;
  624.  
  625.         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
  626.         if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
  627.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
  628.  
  629.         /* We need to disable the AsyncFlip performance optimisations in order
  630.          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
  631.          * programmed to '1' on all products.
  632.          *
  633.          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
  634.          */
  635.         if (INTEL_INFO(dev)->gen >= 6)
  636.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
  637.  
  638.         /* Required for the hardware to program scanline values for waiting */
  639.         /* WaEnableFlushTlbInvalidationMode:snb */
  640.         if (INTEL_INFO(dev)->gen == 6)
  641.                 I915_WRITE(GFX_MODE,
  642.                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
  643.  
  644.         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
  645.                 if (IS_GEN7(dev))
  646.                         I915_WRITE(GFX_MODE_GEN7,
  647.                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
  648.                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
  649.  
  650.         if (INTEL_INFO(dev)->gen >= 5) {
  651.                 ret = init_pipe_control(ring);
  652.                 if (ret)
  653.                         return ret;
  654.         }
  655.  
  656.         if (IS_GEN6(dev)) {
  657.                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
  658.                  * "If this bit is set, STCunit will have LRA as replacement
  659.                  *  policy. [...] This bit must be reset.  LRA replacement
  660.                  *  policy is not supported."
  661.                  */
  662.                 I915_WRITE(CACHE_MODE_0,
  663.                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
  664.         }
  665.  
  666.         if (INTEL_INFO(dev)->gen >= 6)
  667.                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
  668.  
  669.         if (HAS_L3_DPF(dev))
  670.                 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
  671.  
  672.         return ret;
  673. }
  674.  
  675. static void render_ring_cleanup(struct intel_engine_cs *ring)
  676. {
  677.         struct drm_device *dev = ring->dev;
  678.         struct drm_i915_private *dev_priv = dev->dev_private;
  679.  
  680.         if (dev_priv->semaphore_obj) {
  681.                 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
  682.                 drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
  683.                 dev_priv->semaphore_obj = NULL;
  684.         }
  685.  
  686.         if (ring->scratch.obj == NULL)
  687.                 return;
  688.  
  689.         if (INTEL_INFO(dev)->gen >= 5) {
  690. //              kunmap(sg_page(ring->scratch.obj->pages->sgl));
  691.                 i915_gem_object_ggtt_unpin(ring->scratch.obj);
  692.         }
  693.  
  694.         drm_gem_object_unreference(&ring->scratch.obj->base);
  695.         ring->scratch.obj = NULL;
  696. }
  697.  
  698. static int gen8_rcs_signal(struct intel_engine_cs *signaller,
  699.                            unsigned int num_dwords)
  700. {
  701. #define MBOX_UPDATE_DWORDS 8
  702.         struct drm_device *dev = signaller->dev;
  703.         struct drm_i915_private *dev_priv = dev->dev_private;
  704.         struct intel_engine_cs *waiter;
  705.         int i, ret, num_rings;
  706.  
  707.         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
  708.         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
  709. #undef MBOX_UPDATE_DWORDS
  710.  
  711.         ret = intel_ring_begin(signaller, num_dwords);
  712.         if (ret)
  713.                 return ret;
  714.  
  715.         for_each_ring(waiter, dev_priv, i) {
  716.                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
  717.                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
  718.                         continue;
  719.  
  720.                 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
  721.                 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
  722.                                            PIPE_CONTROL_QW_WRITE |
  723.                                            PIPE_CONTROL_FLUSH_ENABLE);
  724.                 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
  725.                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
  726.                 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
  727.                 intel_ring_emit(signaller, 0);
  728.                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
  729.                                            MI_SEMAPHORE_TARGET(waiter->id));
  730.                 intel_ring_emit(signaller, 0);
  731.         }
  732.  
  733.         return 0;
  734. }
  735.  
  736. static int gen8_xcs_signal(struct intel_engine_cs *signaller,
  737.                            unsigned int num_dwords)
  738. {
  739. #define MBOX_UPDATE_DWORDS 6
  740.         struct drm_device *dev = signaller->dev;
  741.         struct drm_i915_private *dev_priv = dev->dev_private;
  742.         struct intel_engine_cs *waiter;
  743.         int i, ret, num_rings;
  744.  
  745.         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
  746.         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
  747. #undef MBOX_UPDATE_DWORDS
  748.  
  749.         ret = intel_ring_begin(signaller, num_dwords);
  750.         if (ret)
  751.                 return ret;
  752.  
  753.         for_each_ring(waiter, dev_priv, i) {
  754.                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
  755.                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
  756.                         continue;
  757.  
  758.                 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
  759.                                            MI_FLUSH_DW_OP_STOREDW);
  760.                 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
  761.                                            MI_FLUSH_DW_USE_GTT);
  762.                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
  763.                 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
  764.                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
  765.                                            MI_SEMAPHORE_TARGET(waiter->id));
  766.                 intel_ring_emit(signaller, 0);
  767.         }
  768.  
  769.         return 0;
  770. }
  771.  
  772. static int gen6_signal(struct intel_engine_cs *signaller,
  773.                        unsigned int num_dwords)
  774. {
  775.         struct drm_device *dev = signaller->dev;
  776.         struct drm_i915_private *dev_priv = dev->dev_private;
  777.         struct intel_engine_cs *useless;
  778.         int i, ret, num_rings;
  779.  
  780. #define MBOX_UPDATE_DWORDS 3
  781.         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
  782.         num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
  783. #undef MBOX_UPDATE_DWORDS
  784.  
  785.         ret = intel_ring_begin(signaller, num_dwords);
  786.         if (ret)
  787.                 return ret;
  788.  
  789.         for_each_ring(useless, dev_priv, i) {
  790.                 u32 mbox_reg = signaller->semaphore.mbox.signal[i];
  791.                 if (mbox_reg != GEN6_NOSYNC) {
  792.                         intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
  793.                         intel_ring_emit(signaller, mbox_reg);
  794.                         intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
  795.                 }
  796.         }
  797.  
  798.         /* If num_dwords was rounded, make sure the tail pointer is correct */
  799.         if (num_rings % 2 == 0)
  800.                 intel_ring_emit(signaller, MI_NOOP);
  801.  
  802.         return 0;
  803. }
  804.  
  805. /**
  806.  * gen6_add_request - Update the semaphore mailbox registers
  807.  *
  808.  * @ring - ring that is adding a request
  809.  * @seqno - return seqno stuck into the ring
  810.  *
  811.  * Update the mailbox registers in the *other* rings with the current seqno.
  812.  * This acts like a signal in the canonical semaphore.
  813.  */
  814. static int
  815. gen6_add_request(struct intel_engine_cs *ring)
  816. {
  817.         int ret;
  818.  
  819.         if (ring->semaphore.signal)
  820.         ret = ring->semaphore.signal(ring, 4);
  821.         else
  822.                 ret = intel_ring_begin(ring, 4);
  823.  
  824.         if (ret)
  825.                 return ret;
  826.  
  827.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  828.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  829.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  830.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  831.         __intel_ring_advance(ring);
  832.  
  833.         return 0;
  834. }
  835.  
  836. static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
  837.                                               u32 seqno)
  838. {
  839.         struct drm_i915_private *dev_priv = dev->dev_private;
  840.         return dev_priv->last_seqno < seqno;
  841. }
  842.  
  843. /**
  844.  * intel_ring_sync - sync the waiter to the signaller on seqno
  845.  *
  846.  * @waiter - ring that is waiting
  847.  * @signaller - ring which has, or will signal
  848.  * @seqno - seqno which the waiter will block on
  849.  */
  850.  
  851. static int
  852. gen8_ring_sync(struct intel_engine_cs *waiter,
  853.                struct intel_engine_cs *signaller,
  854.                u32 seqno)
  855. {
  856.         struct drm_i915_private *dev_priv = waiter->dev->dev_private;
  857.         int ret;
  858.  
  859.         ret = intel_ring_begin(waiter, 4);
  860.         if (ret)
  861.                 return ret;
  862.  
  863.         intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
  864.                                 MI_SEMAPHORE_GLOBAL_GTT |
  865.                                 MI_SEMAPHORE_POLL |
  866.                                 MI_SEMAPHORE_SAD_GTE_SDD);
  867.         intel_ring_emit(waiter, seqno);
  868.         intel_ring_emit(waiter,
  869.                         lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
  870.         intel_ring_emit(waiter,
  871.                         upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
  872.         intel_ring_advance(waiter);
  873.         return 0;
  874. }
  875.  
  876. static int
  877. gen6_ring_sync(struct intel_engine_cs *waiter,
  878.                struct intel_engine_cs *signaller,
  879.                 u32 seqno)
  880. {
  881.         u32 dw1 = MI_SEMAPHORE_MBOX |
  882.                   MI_SEMAPHORE_COMPARE |
  883.                   MI_SEMAPHORE_REGISTER;
  884.         u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
  885.         int ret;
  886.  
  887.         /* Throughout all of the GEM code, seqno passed implies our current
  888.          * seqno is >= the last seqno executed. However for hardware the
  889.          * comparison is strictly greater than.
  890.          */
  891.         seqno -= 1;
  892.  
  893.         WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
  894.  
  895.         ret = intel_ring_begin(waiter, 4);
  896.         if (ret)
  897.                 return ret;
  898.  
  899.         /* If seqno wrap happened, omit the wait with no-ops */
  900.         if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
  901.                 intel_ring_emit(waiter, dw1 | wait_mbox);
  902.         intel_ring_emit(waiter, seqno);
  903.         intel_ring_emit(waiter, 0);
  904.         intel_ring_emit(waiter, MI_NOOP);
  905.         } else {
  906.                 intel_ring_emit(waiter, MI_NOOP);
  907.                 intel_ring_emit(waiter, MI_NOOP);
  908.                 intel_ring_emit(waiter, MI_NOOP);
  909.                 intel_ring_emit(waiter, MI_NOOP);
  910.         }
  911.         intel_ring_advance(waiter);
  912.  
  913.         return 0;
  914. }
  915.  
  916. #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
  917. do {                                                                    \
  918.         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
  919.                  PIPE_CONTROL_DEPTH_STALL);                             \
  920.         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
  921.         intel_ring_emit(ring__, 0);                                                     \
  922.         intel_ring_emit(ring__, 0);                                                     \
  923. } while (0)
  924.  
  925. static int
  926. pc_render_add_request(struct intel_engine_cs *ring)
  927. {
  928.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  929.         int ret;
  930.  
  931.         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
  932.          * incoherent with writes to memory, i.e. completely fubar,
  933.          * so we need to use PIPE_NOTIFY instead.
  934.          *
  935.          * However, we also need to workaround the qword write
  936.          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
  937.          * memory before requesting an interrupt.
  938.          */
  939.         ret = intel_ring_begin(ring, 32);
  940.         if (ret)
  941.                 return ret;
  942.  
  943.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  944.                         PIPE_CONTROL_WRITE_FLUSH |
  945.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
  946.         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  947.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  948.         intel_ring_emit(ring, 0);
  949.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  950.         scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
  951.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  952.         scratch_addr += 2 * CACHELINE_BYTES;
  953.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  954.         scratch_addr += 2 * CACHELINE_BYTES;
  955.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  956.         scratch_addr += 2 * CACHELINE_BYTES;
  957.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  958.         scratch_addr += 2 * CACHELINE_BYTES;
  959.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  960.  
  961.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  962.                         PIPE_CONTROL_WRITE_FLUSH |
  963.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
  964.                         PIPE_CONTROL_NOTIFY);
  965.         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  966.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  967.         intel_ring_emit(ring, 0);
  968.         __intel_ring_advance(ring);
  969.  
  970.         return 0;
  971. }
  972.  
  973. static u32
  974. gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
  975. {
  976.         /* Workaround to force correct ordering between irq and seqno writes on
  977.          * ivb (and maybe also on snb) by reading from a CS register (like
  978.          * ACTHD) before reading the status page. */
  979.         if (!lazy_coherency) {
  980.                 struct drm_i915_private *dev_priv = ring->dev->dev_private;
  981.                 POSTING_READ(RING_ACTHD(ring->mmio_base));
  982.         }
  983.  
  984.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  985. }
  986.  
  987. static u32
  988. ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
  989. {
  990.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  991. }
  992.  
  993. static void
  994. ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
  995. {
  996.         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
  997. }
  998.  
  999. static u32
  1000. pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
  1001. {
  1002.         return ring->scratch.cpu_page[0];
  1003. }
  1004.  
  1005. static void
  1006. pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
  1007. {
  1008.         ring->scratch.cpu_page[0] = seqno;
  1009. }
  1010.  
  1011. static bool
  1012. gen5_ring_get_irq(struct intel_engine_cs *ring)
  1013. {
  1014.         struct drm_device *dev = ring->dev;
  1015.         struct drm_i915_private *dev_priv = dev->dev_private;
  1016.         unsigned long flags;
  1017.  
  1018.         if (!dev->irq_enabled)
  1019.                 return false;
  1020.  
  1021.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1022.         if (ring->irq_refcount++ == 0)
  1023.                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  1024.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1025.  
  1026.         return true;
  1027. }
  1028.  
  1029. static void
  1030. gen5_ring_put_irq(struct intel_engine_cs *ring)
  1031. {
  1032.         struct drm_device *dev = ring->dev;
  1033.         struct drm_i915_private *dev_priv = dev->dev_private;
  1034.         unsigned long flags;
  1035.  
  1036.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1037.         if (--ring->irq_refcount == 0)
  1038.                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  1039.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1040. }
  1041.  
  1042. static bool
  1043. i9xx_ring_get_irq(struct intel_engine_cs *ring)
  1044. {
  1045.         struct drm_device *dev = ring->dev;
  1046.         struct drm_i915_private *dev_priv = dev->dev_private;
  1047.         unsigned long flags;
  1048.  
  1049.         if (!dev->irq_enabled)
  1050.                 return false;
  1051.  
  1052.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1053.         if (ring->irq_refcount++ == 0) {
  1054.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  1055.         I915_WRITE(IMR, dev_priv->irq_mask);
  1056.         POSTING_READ(IMR);
  1057.         }
  1058.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1059.  
  1060.         return true;
  1061. }
  1062.  
  1063. static void
  1064. i9xx_ring_put_irq(struct intel_engine_cs *ring)
  1065. {
  1066.         struct drm_device *dev = ring->dev;
  1067.         struct drm_i915_private *dev_priv = dev->dev_private;
  1068.         unsigned long flags;
  1069.  
  1070.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1071.         if (--ring->irq_refcount == 0) {
  1072.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  1073.         I915_WRITE(IMR, dev_priv->irq_mask);
  1074.         POSTING_READ(IMR);
  1075.         }
  1076.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1077. }
  1078.  
  1079. static bool
  1080. i8xx_ring_get_irq(struct intel_engine_cs *ring)
  1081. {
  1082.         struct drm_device *dev = ring->dev;
  1083.         struct drm_i915_private *dev_priv = dev->dev_private;
  1084.         unsigned long flags;
  1085.  
  1086.         if (!dev->irq_enabled)
  1087.                 return false;
  1088.  
  1089.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1090.         if (ring->irq_refcount++ == 0) {
  1091.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  1092.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  1093.                 POSTING_READ16(IMR);
  1094.         }
  1095.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1096.  
  1097.         return true;
  1098. }
  1099.  
  1100. static void
  1101. i8xx_ring_put_irq(struct intel_engine_cs *ring)
  1102. {
  1103.         struct drm_device *dev = ring->dev;
  1104.         struct drm_i915_private *dev_priv = dev->dev_private;
  1105.         unsigned long flags;
  1106.  
  1107.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1108.         if (--ring->irq_refcount == 0) {
  1109.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  1110.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  1111.                 POSTING_READ16(IMR);
  1112.         }
  1113.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1114. }
  1115.  
  1116. void intel_ring_setup_status_page(struct intel_engine_cs *ring)
  1117. {
  1118.         struct drm_device *dev = ring->dev;
  1119.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1120.         u32 mmio = 0;
  1121.  
  1122.         /* The ring status page addresses are no longer next to the rest of
  1123.          * the ring registers as of gen7.
  1124.          */
  1125.         if (IS_GEN7(dev)) {
  1126.                 switch (ring->id) {
  1127.                 case RCS:
  1128.                         mmio = RENDER_HWS_PGA_GEN7;
  1129.                         break;
  1130.                 case BCS:
  1131.                         mmio = BLT_HWS_PGA_GEN7;
  1132.                         break;
  1133.                 /*
  1134.                  * VCS2 actually doesn't exist on Gen7. Only shut up
  1135.                  * gcc switch check warning
  1136.                  */
  1137.                 case VCS2:
  1138.                 case VCS:
  1139.                         mmio = BSD_HWS_PGA_GEN7;
  1140.                         break;
  1141.                 case VECS:
  1142.                         mmio = VEBOX_HWS_PGA_GEN7;
  1143.                         break;
  1144.                 }
  1145.         } else if (IS_GEN6(ring->dev)) {
  1146.                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
  1147.         } else {
  1148.                 /* XXX: gen8 returns to sanity */
  1149.                 mmio = RING_HWS_PGA(ring->mmio_base);
  1150.         }
  1151.  
  1152.         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
  1153.         POSTING_READ(mmio);
  1154.  
  1155.         /*
  1156.          * Flush the TLB for this page
  1157.          *
  1158.          * FIXME: These two bits have disappeared on gen8, so a question
  1159.          * arises: do we still need this and if so how should we go about
  1160.          * invalidating the TLB?
  1161.          */
  1162.         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
  1163.                 u32 reg = RING_INSTPM(ring->mmio_base);
  1164.  
  1165.                 /* ring should be idle before issuing a sync flush*/
  1166.                 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
  1167.  
  1168.                 I915_WRITE(reg,
  1169.                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
  1170.                                               INSTPM_SYNC_FLUSH));
  1171.                 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
  1172.                              1000))
  1173.                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
  1174.                                   ring->name);
  1175.         }
  1176. }
  1177.  
  1178. static int
  1179. bsd_ring_flush(struct intel_engine_cs *ring,
  1180.                u32     invalidate_domains,
  1181.                u32     flush_domains)
  1182. {
  1183.         int ret;
  1184.  
  1185.         ret = intel_ring_begin(ring, 2);
  1186.         if (ret)
  1187.                 return ret;
  1188.  
  1189.         intel_ring_emit(ring, MI_FLUSH);
  1190.         intel_ring_emit(ring, MI_NOOP);
  1191.         intel_ring_advance(ring);
  1192.         return 0;
  1193. }
  1194.  
  1195. static int
  1196. i9xx_add_request(struct intel_engine_cs *ring)
  1197. {
  1198.         int ret;
  1199.  
  1200.         ret = intel_ring_begin(ring, 4);
  1201.         if (ret)
  1202.                 return ret;
  1203.  
  1204.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  1205.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  1206.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  1207.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  1208.         __intel_ring_advance(ring);
  1209.  
  1210.         return 0;
  1211. }
  1212.  
  1213. static bool
  1214. gen6_ring_get_irq(struct intel_engine_cs *ring)
  1215. {
  1216.         struct drm_device *dev = ring->dev;
  1217.         struct drm_i915_private *dev_priv = dev->dev_private;
  1218.         unsigned long flags;
  1219.  
  1220.         if (!dev->irq_enabled)
  1221.                return false;
  1222.  
  1223.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1224.         if (ring->irq_refcount++ == 0) {
  1225.                 if (HAS_L3_DPF(dev) && ring->id == RCS)
  1226.                         I915_WRITE_IMR(ring,
  1227.                                        ~(ring->irq_enable_mask |
  1228.                                          GT_PARITY_ERROR(dev)));
  1229.                 else
  1230.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1231.                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  1232.         }
  1233.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1234.  
  1235.     return true;
  1236. }
  1237.  
  1238. static void
  1239. gen6_ring_put_irq(struct intel_engine_cs *ring)
  1240. {
  1241.         struct drm_device *dev = ring->dev;
  1242.         struct drm_i915_private *dev_priv = dev->dev_private;
  1243.         unsigned long flags;
  1244.  
  1245.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1246.         if (--ring->irq_refcount == 0) {
  1247.                 if (HAS_L3_DPF(dev) && ring->id == RCS)
  1248.                         I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
  1249.                 else
  1250.                         I915_WRITE_IMR(ring, ~0);
  1251.                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  1252.         }
  1253.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1254. }
  1255.  
  1256. static bool
  1257. hsw_vebox_get_irq(struct intel_engine_cs *ring)
  1258. {
  1259.         struct drm_device *dev = ring->dev;
  1260.         struct drm_i915_private *dev_priv = dev->dev_private;
  1261.         unsigned long flags;
  1262.  
  1263.         if (!dev->irq_enabled)
  1264.                 return false;
  1265.  
  1266.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1267.         if (ring->irq_refcount++ == 0) {
  1268.                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1269.                 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
  1270.         }
  1271.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1272.  
  1273.         return true;
  1274. }
  1275.  
  1276. static void
  1277. hsw_vebox_put_irq(struct intel_engine_cs *ring)
  1278. {
  1279.         struct drm_device *dev = ring->dev;
  1280.         struct drm_i915_private *dev_priv = dev->dev_private;
  1281.         unsigned long flags;
  1282.  
  1283.         if (!dev->irq_enabled)
  1284.                 return;
  1285.  
  1286.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1287.         if (--ring->irq_refcount == 0) {
  1288.                 I915_WRITE_IMR(ring, ~0);
  1289.                 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
  1290.         }
  1291.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1292. }
  1293.  
  1294. static bool
  1295. gen8_ring_get_irq(struct intel_engine_cs *ring)
  1296. {
  1297.         struct drm_device *dev = ring->dev;
  1298.         struct drm_i915_private *dev_priv = dev->dev_private;
  1299.         unsigned long flags;
  1300.  
  1301.         if (!dev->irq_enabled)
  1302.                 return false;
  1303.  
  1304.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1305.         if (ring->irq_refcount++ == 0) {
  1306.                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
  1307.                         I915_WRITE_IMR(ring,
  1308.                                        ~(ring->irq_enable_mask |
  1309.                                          GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
  1310.                 } else {
  1311.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1312.                 }
  1313.                 POSTING_READ(RING_IMR(ring->mmio_base));
  1314.         }
  1315.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1316.  
  1317.         return true;
  1318. }
  1319.  
  1320. static void
  1321. gen8_ring_put_irq(struct intel_engine_cs *ring)
  1322. {
  1323.         struct drm_device *dev = ring->dev;
  1324.         struct drm_i915_private *dev_priv = dev->dev_private;
  1325.         unsigned long flags;
  1326.  
  1327.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1328.         if (--ring->irq_refcount == 0) {
  1329.                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
  1330.                         I915_WRITE_IMR(ring,
  1331.                                        ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
  1332.                 } else {
  1333.                         I915_WRITE_IMR(ring, ~0);
  1334.                 }
  1335.                 POSTING_READ(RING_IMR(ring->mmio_base));
  1336.         }
  1337.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1338. }
  1339.  
  1340. static int
  1341. i965_dispatch_execbuffer(struct intel_engine_cs *ring,
  1342.                          u64 offset, u32 length,
  1343.                          unsigned flags)
  1344. {
  1345.         int ret;
  1346.  
  1347.         ret = intel_ring_begin(ring, 2);
  1348.         if (ret)
  1349.                 return ret;
  1350.  
  1351.         intel_ring_emit(ring,
  1352.                         MI_BATCH_BUFFER_START |
  1353.                         MI_BATCH_GTT |
  1354.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  1355.         intel_ring_emit(ring, offset);
  1356.         intel_ring_advance(ring);
  1357.  
  1358.         return 0;
  1359. }
  1360.  
  1361. /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
  1362. #define I830_BATCH_LIMIT (256*1024)
  1363. #define I830_TLB_ENTRIES (2)
  1364. #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
  1365. static int
  1366. i830_dispatch_execbuffer(struct intel_engine_cs *ring,
  1367.                                 u64 offset, u32 len,
  1368.                                 unsigned flags)
  1369. {
  1370.         u32 cs_offset = ring->scratch.gtt_offset;
  1371.         int ret;
  1372.  
  1373.         ret = intel_ring_begin(ring, 6);
  1374.                 if (ret)
  1375.                         return ret;
  1376.  
  1377.         /* Evict the invalid PTE TLBs */
  1378.         intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
  1379.         intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
  1380.         intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
  1381.         intel_ring_emit(ring, cs_offset);
  1382.         intel_ring_emit(ring, 0xdeadbeef);
  1383.                 intel_ring_emit(ring, MI_NOOP);
  1384.                 intel_ring_advance(ring);
  1385.  
  1386.         if ((flags & I915_DISPATCH_PINNED) == 0) {
  1387.                 if (len > I830_BATCH_LIMIT)
  1388.                         return -ENOSPC;
  1389.  
  1390.                 ret = intel_ring_begin(ring, 6 + 2);
  1391.                 if (ret)
  1392.                         return ret;
  1393.  
  1394.                 /* Blit the batch (which has now all relocs applied) to the
  1395.                  * stable batch scratch bo area (so that the CS never
  1396.                  * stumbles over its tlb invalidation bug) ...
  1397.                  */
  1398.                 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
  1399.                 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
  1400.                 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 1024);
  1401.                 intel_ring_emit(ring, cs_offset);
  1402.                 intel_ring_emit(ring, 4096);
  1403.                 intel_ring_emit(ring, offset);
  1404.  
  1405.                 intel_ring_emit(ring, MI_FLUSH);
  1406.                 intel_ring_emit(ring, MI_NOOP);
  1407.                 intel_ring_advance(ring);
  1408.  
  1409.                 /* ... and execute it. */
  1410.                 offset = cs_offset;
  1411.         }
  1412.  
  1413.         ret = intel_ring_begin(ring, 4);
  1414.         if (ret)
  1415.                 return ret;
  1416.  
  1417.                 intel_ring_emit(ring, MI_BATCH_BUFFER);
  1418.         intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1419.         intel_ring_emit(ring, offset + len - 8);
  1420.         intel_ring_emit(ring, MI_NOOP);
  1421.         intel_ring_advance(ring);
  1422.  
  1423.         return 0;
  1424. }
  1425.  
  1426. static int
  1427. i915_dispatch_execbuffer(struct intel_engine_cs *ring,
  1428.                          u64 offset, u32 len,
  1429.                          unsigned flags)
  1430. {
  1431.         int ret;
  1432.  
  1433.                 ret = intel_ring_begin(ring, 2);
  1434.                 if (ret)
  1435.                         return ret;
  1436.  
  1437.         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
  1438.         intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1439.         intel_ring_advance(ring);
  1440.  
  1441.         return 0;
  1442. }
  1443.  
  1444. static void cleanup_status_page(struct intel_engine_cs *ring)
  1445. {
  1446.         struct drm_i915_gem_object *obj;
  1447.  
  1448.         obj = ring->status_page.obj;
  1449.         if (obj == NULL)
  1450.                 return;
  1451.  
  1452. //      kunmap(sg_page(obj->pages->sgl));
  1453.         i915_gem_object_ggtt_unpin(obj);
  1454.         drm_gem_object_unreference(&obj->base);
  1455.         ring->status_page.obj = NULL;
  1456. }
  1457.  
  1458. static int init_status_page(struct intel_engine_cs *ring)
  1459. {
  1460.         struct drm_i915_gem_object *obj;
  1461.  
  1462.         if ((obj = ring->status_page.obj) == NULL) {
  1463.                 unsigned flags;
  1464.         int ret;
  1465.  
  1466.                 obj = i915_gem_alloc_object(ring->dev, 4096);
  1467.         if (obj == NULL) {
  1468.                 DRM_ERROR("Failed to allocate status page\n");
  1469.                         return -ENOMEM;
  1470.         }
  1471.  
  1472.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  1473.         if (ret)
  1474.                 goto err_unref;
  1475.  
  1476.                 flags = 0;
  1477.                 if (!HAS_LLC(ring->dev))
  1478.                         /* On g33, we cannot place HWS above 256MiB, so
  1479.                          * restrict its pinning to the low mappable arena.
  1480.                          * Though this restriction is not documented for
  1481.                          * gen4, gen5, or byt, they also behave similarly
  1482.                          * and hang if the HWS is placed at the top of the
  1483.                          * GTT. To generalise, it appears that all !llc
  1484.                          * platforms have issues with us placing the HWS
  1485.                          * above the mappable region (even though we never
  1486.                          * actualy map it).
  1487.                          */
  1488.                         flags |= PIN_MAPPABLE;
  1489.                 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
  1490.                 if (ret) {
  1491. err_unref:
  1492.                         drm_gem_object_unreference(&obj->base);
  1493.                         return ret;
  1494.                 }
  1495.  
  1496.                 ring->status_page.obj = obj;
  1497.         }
  1498.  
  1499.         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
  1500.     ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW|0x100);
  1501.         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1502.  
  1503.         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  1504.                         ring->name, ring->status_page.gfx_addr);
  1505.  
  1506.         return 0;
  1507. }
  1508.  
  1509. static int init_phys_status_page(struct intel_engine_cs *ring)
  1510. {
  1511.     struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1512.  
  1513.     if (!dev_priv->status_page_dmah) {
  1514.         dev_priv->status_page_dmah =
  1515.             drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
  1516.         if (!dev_priv->status_page_dmah)
  1517.             return -ENOMEM;
  1518.     }
  1519.  
  1520.     ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
  1521.     memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1522.  
  1523.     return 0;
  1524. }
  1525.  
  1526. static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
  1527. {
  1528.         if (!ringbuf->obj)
  1529.                 return;
  1530.  
  1531.         iounmap(ringbuf->virtual_start);
  1532.         i915_gem_object_ggtt_unpin(ringbuf->obj);
  1533.         drm_gem_object_unreference(&ringbuf->obj->base);
  1534.         ringbuf->obj = NULL;
  1535. }
  1536.  
  1537. static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
  1538.                                       struct intel_ringbuffer *ringbuf)
  1539. {
  1540.         struct drm_i915_private *dev_priv = to_i915(dev);
  1541.         struct drm_i915_gem_object *obj;
  1542.         int ret;
  1543.  
  1544.         if (ringbuf->obj)
  1545.                 return 0;
  1546.  
  1547.         obj = NULL;
  1548.         if (!HAS_LLC(dev))
  1549.                 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
  1550.         if (obj == NULL)
  1551.                 obj = i915_gem_alloc_object(dev, ringbuf->size);
  1552.         if (obj == NULL)
  1553.                 return -ENOMEM;
  1554.  
  1555.         /* mark ring buffers as read-only from GPU side by default */
  1556.         obj->gt_ro = 1;
  1557.  
  1558.         ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
  1559.         if (ret)
  1560.                 goto err_unref;
  1561.  
  1562.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  1563.         if (ret)
  1564.                 goto err_unpin;
  1565.  
  1566.         ringbuf->virtual_start =
  1567.                 ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
  1568.                                 ringbuf->size);
  1569.         if (ringbuf->virtual_start == NULL) {
  1570.                 ret = -EINVAL;
  1571.                 goto err_unpin;
  1572.         }
  1573.  
  1574.         ringbuf->obj = obj;
  1575.         return 0;
  1576.  
  1577. err_unpin:
  1578.         i915_gem_object_ggtt_unpin(obj);
  1579. err_unref:
  1580.         drm_gem_object_unreference(&obj->base);
  1581.         return ret;
  1582. }
  1583.  
  1584. static int intel_init_ring_buffer(struct drm_device *dev,
  1585.                                   struct intel_engine_cs *ring)
  1586. {
  1587.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1588.         int ret;
  1589.  
  1590.         if (ringbuf == NULL) {
  1591.                 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
  1592.                 if (!ringbuf)
  1593.                         return -ENOMEM;
  1594.                 ring->buffer = ringbuf;
  1595.         }
  1596.  
  1597.         ring->dev = dev;
  1598.         INIT_LIST_HEAD(&ring->active_list);
  1599.         INIT_LIST_HEAD(&ring->request_list);
  1600.         ringbuf->size = 32 * PAGE_SIZE;
  1601.         memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
  1602.  
  1603.         init_waitqueue_head(&ring->irq_queue);
  1604.  
  1605.         if (I915_NEED_GFX_HWS(dev)) {
  1606.                 ret = init_status_page(ring);
  1607.                 if (ret)
  1608.                         goto error;
  1609.         } else {
  1610.                 BUG_ON(ring->id != RCS);
  1611.                 ret = init_phys_status_page(ring);
  1612.         if (ret)
  1613.                         goto error;
  1614.         }
  1615.  
  1616.         ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
  1617.         if (ret) {
  1618.                 DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret);
  1619.                 goto error;
  1620.         }
  1621.  
  1622.         /* Workaround an erratum on the i830 which causes a hang if
  1623.          * the TAIL pointer points to within the last 2 cachelines
  1624.          * of the buffer.
  1625.          */
  1626.         ringbuf->effective_size = ringbuf->size;
  1627.         if (IS_I830(dev) || IS_845G(dev))
  1628.                 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
  1629.  
  1630.         ret = i915_cmd_parser_init_ring(ring);
  1631.         if (ret)
  1632.                 goto error;
  1633.  
  1634.         ret = ring->init(ring);
  1635.         if (ret)
  1636.                 goto error;
  1637.  
  1638.         return 0;
  1639.  
  1640. error:
  1641.         kfree(ringbuf);
  1642.         ring->buffer = NULL;
  1643.         return ret;
  1644. }
  1645.  
  1646. void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
  1647. {
  1648.         struct drm_i915_private *dev_priv = to_i915(ring->dev);
  1649.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1650.  
  1651.         if (!intel_ring_initialized(ring))
  1652.                 return;
  1653.  
  1654.         intel_stop_ring_buffer(ring);
  1655.         WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
  1656.  
  1657.         intel_destroy_ringbuffer_obj(ringbuf);
  1658.         ring->preallocated_lazy_request = NULL;
  1659.         ring->outstanding_lazy_seqno = 0;
  1660.  
  1661.         if (ring->cleanup)
  1662.                 ring->cleanup(ring);
  1663.  
  1664. //      cleanup_status_page(ring);
  1665.  
  1666.         i915_cmd_parser_fini_ring(ring);
  1667.  
  1668.         kfree(ringbuf);
  1669.         ring->buffer = NULL;
  1670. }
  1671.  
  1672. static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
  1673. {
  1674.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1675.         struct drm_i915_gem_request *request;
  1676.         u32 seqno = 0;
  1677.         int ret;
  1678.  
  1679.         if (ringbuf->last_retired_head != -1) {
  1680.                 ringbuf->head = ringbuf->last_retired_head;
  1681.                 ringbuf->last_retired_head = -1;
  1682.  
  1683.                 ringbuf->space = ring_space(ringbuf);
  1684.                 if (ringbuf->space >= n)
  1685.                         return 0;
  1686.         }
  1687.  
  1688.         list_for_each_entry(request, &ring->request_list, list) {
  1689.                 if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) {
  1690.                         seqno = request->seqno;
  1691.                         break;
  1692.                 }
  1693.         }
  1694.  
  1695.         if (seqno == 0)
  1696.                 return -ENOSPC;
  1697.  
  1698.         ret = i915_wait_seqno(ring, seqno);
  1699.         if (ret)
  1700.                 return ret;
  1701.  
  1702.         i915_gem_retire_requests_ring(ring);
  1703.         ringbuf->head = ringbuf->last_retired_head;
  1704.         ringbuf->last_retired_head = -1;
  1705.  
  1706.         ringbuf->space = ring_space(ringbuf);
  1707.         return 0;
  1708. }
  1709.  
  1710. static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
  1711. {
  1712.         struct drm_device *dev = ring->dev;
  1713.         struct drm_i915_private *dev_priv = dev->dev_private;
  1714.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1715.         unsigned long end;
  1716.         int ret;
  1717.  
  1718.         ret = intel_ring_wait_request(ring, n);
  1719.         if (ret != -ENOSPC)
  1720.                 return ret;
  1721.  
  1722.         /* force the tail write in case we have been skipping them */
  1723.         __intel_ring_advance(ring);
  1724.  
  1725.         /* With GEM the hangcheck timer should kick us out of the loop,
  1726.          * leaving it early runs the risk of corrupting GEM state (due
  1727.          * to running on almost untested codepaths). But on resume
  1728.          * timers don't work yet, so prevent a complete hang in that
  1729.          * case by choosing an insanely large timeout. */
  1730.         end = jiffies + 60 * HZ;
  1731.  
  1732.         trace_i915_ring_wait_begin(ring);
  1733.         do {
  1734.                 ringbuf->head = I915_READ_HEAD(ring);
  1735.                 ringbuf->space = ring_space(ringbuf);
  1736.                 if (ringbuf->space >= n) {
  1737.                         ret = 0;
  1738.                         break;
  1739.                 }
  1740.  
  1741.  
  1742.                 msleep(1);
  1743.  
  1744.                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1745.                                            dev_priv->mm.interruptible);
  1746.                 if (ret)
  1747.                         break;
  1748.  
  1749.                 if (time_after(jiffies, end)) {
  1750.                         ret = -EBUSY;
  1751.                         break;
  1752.                 }
  1753.         } while (1);
  1754.         trace_i915_ring_wait_end(ring);
  1755.                         return ret;
  1756. }
  1757.  
  1758. static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
  1759. {
  1760.         uint32_t __iomem *virt;
  1761.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1762.         int rem = ringbuf->size - ringbuf->tail;
  1763.  
  1764.         if (ringbuf->space < rem) {
  1765.                 int ret = ring_wait_for_space(ring, rem);
  1766.                 if (ret)
  1767.                         return ret;
  1768.         }
  1769.  
  1770.         virt = ringbuf->virtual_start + ringbuf->tail;
  1771.         rem /= 4;
  1772.         while (rem--)
  1773.                 iowrite32(MI_NOOP, virt++);
  1774.  
  1775.         ringbuf->tail = 0;
  1776.         ringbuf->space = ring_space(ringbuf);
  1777.  
  1778.         return 0;
  1779. }
  1780.  
  1781. int intel_ring_idle(struct intel_engine_cs *ring)
  1782. {
  1783.         u32 seqno;
  1784.         int ret;
  1785.  
  1786.         /* We need to add any requests required to flush the objects and ring */
  1787.         if (ring->outstanding_lazy_seqno) {
  1788.                 ret = i915_add_request(ring, NULL);
  1789.                 if (ret)
  1790.                         return ret;
  1791.         }
  1792.  
  1793.         /* Wait upon the last request to be completed */
  1794.         if (list_empty(&ring->request_list))
  1795.                 return 0;
  1796.  
  1797.         seqno = list_entry(ring->request_list.prev,
  1798.                            struct drm_i915_gem_request,
  1799.                            list)->seqno;
  1800.  
  1801.         return i915_wait_seqno(ring, seqno);
  1802. }
  1803.  
  1804. static int
  1805. intel_ring_alloc_seqno(struct intel_engine_cs *ring)
  1806. {
  1807.         if (ring->outstanding_lazy_seqno)
  1808.                 return 0;
  1809.  
  1810.         if (ring->preallocated_lazy_request == NULL) {
  1811.                 struct drm_i915_gem_request *request;
  1812.  
  1813.                 request = kmalloc(sizeof(*request), GFP_KERNEL);
  1814.                 if (request == NULL)
  1815.                         return -ENOMEM;
  1816.  
  1817.                 ring->preallocated_lazy_request = request;
  1818.         }
  1819.  
  1820.         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
  1821. }
  1822.  
  1823. static int __intel_ring_prepare(struct intel_engine_cs *ring,
  1824.                               int bytes)
  1825. {
  1826.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1827.         int ret;
  1828.  
  1829.         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
  1830.                 ret = intel_wrap_ring_buffer(ring);
  1831.                 if (unlikely(ret))
  1832.                         return ret;
  1833.         }
  1834.  
  1835.         if (unlikely(ringbuf->space < bytes)) {
  1836.                 ret = ring_wait_for_space(ring, bytes);
  1837.                 if (unlikely(ret))
  1838.                         return ret;
  1839.         }
  1840.  
  1841.         return 0;
  1842. }
  1843.  
  1844. int intel_ring_begin(struct intel_engine_cs *ring,
  1845.                      int num_dwords)
  1846. {
  1847.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1848.         int ret;
  1849.  
  1850.         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1851.                                    dev_priv->mm.interruptible);
  1852.         if (ret)
  1853.                 return ret;
  1854.  
  1855.         ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
  1856.         if (ret)
  1857.                 return ret;
  1858.  
  1859.         /* Preallocate the olr before touching the ring */
  1860.         ret = intel_ring_alloc_seqno(ring);
  1861.         if (ret)
  1862.                 return ret;
  1863.  
  1864.         ring->buffer->space -= num_dwords * sizeof(uint32_t);
  1865.         return 0;
  1866. }
  1867.  
  1868. /* Align the ring tail to a cacheline boundary */
  1869. int intel_ring_cacheline_align(struct intel_engine_cs *ring)
  1870. {
  1871.         int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
  1872.         int ret;
  1873.  
  1874.         if (num_dwords == 0)
  1875.                 return 0;
  1876.  
  1877.         num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
  1878.         ret = intel_ring_begin(ring, num_dwords);
  1879.         if (ret)
  1880.                 return ret;
  1881.  
  1882.         while (num_dwords--)
  1883.                 intel_ring_emit(ring, MI_NOOP);
  1884.  
  1885.         intel_ring_advance(ring);
  1886.  
  1887.         return 0;
  1888. }
  1889.  
  1890. void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
  1891. {
  1892.         struct drm_device *dev = ring->dev;
  1893.         struct drm_i915_private *dev_priv = dev->dev_private;
  1894.  
  1895.         BUG_ON(ring->outstanding_lazy_seqno);
  1896.  
  1897.         if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
  1898.                 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
  1899.                 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
  1900.                 if (HAS_VEBOX(dev))
  1901.                         I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
  1902.         }
  1903.  
  1904.         ring->set_seqno(ring, seqno);
  1905.         ring->hangcheck.seqno = seqno;
  1906. }
  1907.  
  1908. static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
  1909.                                      u32 value)
  1910. {
  1911.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1912.  
  1913.        /* Every tail move must follow the sequence below */
  1914.  
  1915.         /* Disable notification that the ring is IDLE. The GT
  1916.          * will then assume that it is busy and bring it out of rc6.
  1917.          */
  1918.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1919.                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1920.  
  1921.         /* Clear the context id. Here be magic! */
  1922.         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
  1923.  
  1924.         /* Wait for the ring not to be idle, i.e. for it to wake up. */
  1925.        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
  1926.                       GEN6_BSD_SLEEP_INDICATOR) == 0,
  1927.                        50))
  1928.                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
  1929.  
  1930.         /* Now that the ring is fully powered up, update the tail */
  1931.        I915_WRITE_TAIL(ring, value);
  1932.         POSTING_READ(RING_TAIL(ring->mmio_base));
  1933.  
  1934.         /* Let the ring send IDLE messages to the GT again,
  1935.          * and so let it sleep to conserve power when idle.
  1936.          */
  1937.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1938.                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1939. }
  1940.  
  1941. static int gen6_bsd_ring_flush(struct intel_engine_cs *ring,
  1942.                            u32 invalidate, u32 flush)
  1943. {
  1944.         uint32_t cmd;
  1945.         int ret;
  1946.  
  1947.         ret = intel_ring_begin(ring, 4);
  1948.         if (ret)
  1949.                 return ret;
  1950.  
  1951.         cmd = MI_FLUSH_DW;
  1952.         if (INTEL_INFO(ring->dev)->gen >= 8)
  1953.                 cmd += 1;
  1954.         /*
  1955.          * Bspec vol 1c.5 - video engine command streamer:
  1956.          * "If ENABLED, all TLBs will be invalidated once the flush
  1957.          * operation is complete. This bit is only valid when the
  1958.          * Post-Sync Operation field is a value of 1h or 3h."
  1959.          */
  1960.         if (invalidate & I915_GEM_GPU_DOMAINS)
  1961.                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
  1962.                         MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
  1963.         intel_ring_emit(ring, cmd);
  1964.         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  1965.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  1966.                 intel_ring_emit(ring, 0); /* upper addr */
  1967.                 intel_ring_emit(ring, 0); /* value */
  1968.         } else  {
  1969.         intel_ring_emit(ring, 0);
  1970.         intel_ring_emit(ring, MI_NOOP);
  1971.         }
  1972.         intel_ring_advance(ring);
  1973.         return 0;
  1974. }
  1975.  
  1976. static int
  1977. gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
  1978.                               u64 offset, u32 len,
  1979.                               unsigned flags)
  1980. {
  1981.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1982.         bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
  1983.                 !(flags & I915_DISPATCH_SECURE);
  1984.         int ret;
  1985.  
  1986.         ret = intel_ring_begin(ring, 4);
  1987.         if (ret)
  1988.                 return ret;
  1989.  
  1990.         /* FIXME(BDW): Address space and security selectors. */
  1991.         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
  1992.         intel_ring_emit(ring, lower_32_bits(offset));
  1993.         intel_ring_emit(ring, upper_32_bits(offset));
  1994.         intel_ring_emit(ring, MI_NOOP);
  1995.         intel_ring_advance(ring);
  1996.  
  1997.         return 0;
  1998. }
  1999.  
  2000. static int
  2001. hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
  2002.                               u64 offset, u32 len,
  2003.                               unsigned flags)
  2004. {
  2005.         int ret;
  2006.  
  2007.         ret = intel_ring_begin(ring, 2);
  2008.         if (ret)
  2009.                 return ret;
  2010.  
  2011.         intel_ring_emit(ring,
  2012.                         MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
  2013.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
  2014.         /* bit0-7 is the length on GEN6+ */
  2015.         intel_ring_emit(ring, offset);
  2016.         intel_ring_advance(ring);
  2017.  
  2018.         return 0;
  2019. }
  2020.  
  2021. static int
  2022. gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
  2023.                               u64 offset, u32 len,
  2024.                               unsigned flags)
  2025. {
  2026.        int ret;
  2027.  
  2028.        ret = intel_ring_begin(ring, 2);
  2029.        if (ret)
  2030.                return ret;
  2031.  
  2032.         intel_ring_emit(ring,
  2033.                         MI_BATCH_BUFFER_START |
  2034.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  2035.        /* bit0-7 is the length on GEN6+ */
  2036.        intel_ring_emit(ring, offset);
  2037.        intel_ring_advance(ring);
  2038.  
  2039.        return 0;
  2040. }
  2041.  
  2042. /* Blitter support (SandyBridge+) */
  2043.  
  2044. static int gen6_ring_flush(struct intel_engine_cs *ring,
  2045.                           u32 invalidate, u32 flush)
  2046. {
  2047.         struct drm_device *dev = ring->dev;
  2048.         uint32_t cmd;
  2049.         int ret;
  2050.  
  2051.         ret = intel_ring_begin(ring, 4);
  2052.         if (ret)
  2053.                 return ret;
  2054.  
  2055.         cmd = MI_FLUSH_DW;
  2056.         if (INTEL_INFO(ring->dev)->gen >= 8)
  2057.                 cmd += 1;
  2058.         /*
  2059.          * Bspec vol 1c.3 - blitter engine command streamer:
  2060.          * "If ENABLED, all TLBs will be invalidated once the flush
  2061.          * operation is complete. This bit is only valid when the
  2062.          * Post-Sync Operation field is a value of 1h or 3h."
  2063.          */
  2064.         if (invalidate & I915_GEM_DOMAIN_RENDER)
  2065.                 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
  2066.                         MI_FLUSH_DW_OP_STOREDW;
  2067.         intel_ring_emit(ring, cmd);
  2068.         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  2069.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  2070.                 intel_ring_emit(ring, 0); /* upper addr */
  2071.                 intel_ring_emit(ring, 0); /* value */
  2072.         } else  {
  2073.         intel_ring_emit(ring, 0);
  2074.         intel_ring_emit(ring, MI_NOOP);
  2075.         }
  2076.         intel_ring_advance(ring);
  2077.  
  2078.         if (IS_GEN7(dev) && !invalidate && flush)
  2079.                 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
  2080.  
  2081.         return 0;
  2082. }
  2083.  
  2084. int intel_init_render_ring_buffer(struct drm_device *dev)
  2085. {
  2086.         struct drm_i915_private *dev_priv = dev->dev_private;
  2087.         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
  2088.         struct drm_i915_gem_object *obj;
  2089.         int ret;
  2090.  
  2091.         ring->name = "render ring";
  2092.         ring->id = RCS;
  2093.         ring->mmio_base = RENDER_RING_BASE;
  2094.  
  2095.         if (INTEL_INFO(dev)->gen >= 8) {
  2096.                 if (i915_semaphore_is_enabled(dev)) {
  2097.                         obj = i915_gem_alloc_object(dev, 4096);
  2098.                         if (obj == NULL) {
  2099.                                 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
  2100.                                 i915.semaphores = 0;
  2101.                         } else {
  2102.                                 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  2103.                                 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
  2104.                                 if (ret != 0) {
  2105.                                         drm_gem_object_unreference(&obj->base);
  2106.                                         DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
  2107.                                         i915.semaphores = 0;
  2108.                                 } else
  2109.                                         dev_priv->semaphore_obj = obj;
  2110.                         }
  2111.                 }
  2112.                 ring->add_request = gen6_add_request;
  2113.                 ring->flush = gen8_render_ring_flush;
  2114.                 ring->irq_get = gen8_ring_get_irq;
  2115.                 ring->irq_put = gen8_ring_put_irq;
  2116.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
  2117.                 ring->get_seqno = gen6_ring_get_seqno;
  2118.                 ring->set_seqno = ring_set_seqno;
  2119.                 if (i915_semaphore_is_enabled(dev)) {
  2120.                         WARN_ON(!dev_priv->semaphore_obj);
  2121.                         ring->semaphore.sync_to = gen8_ring_sync;
  2122.                         ring->semaphore.signal = gen8_rcs_signal;
  2123.                         GEN8_RING_SEMAPHORE_INIT;
  2124.                 }
  2125.         } else if (INTEL_INFO(dev)->gen >= 6) {
  2126.        ring->add_request = gen6_add_request;
  2127.                 ring->flush = gen7_render_ring_flush;
  2128.                 if (INTEL_INFO(dev)->gen == 6)
  2129.                 ring->flush = gen6_render_ring_flush;
  2130.                 ring->irq_get = gen6_ring_get_irq;
  2131.                 ring->irq_put = gen6_ring_put_irq;
  2132.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
  2133.                 ring->get_seqno = gen6_ring_get_seqno;
  2134.                 ring->set_seqno = ring_set_seqno;
  2135.                 if (i915_semaphore_is_enabled(dev)) {
  2136.                 ring->semaphore.sync_to = gen6_ring_sync;
  2137.                 ring->semaphore.signal = gen6_signal;
  2138.                 /*
  2139.                          * The current semaphore is only applied on pre-gen8
  2140.                          * platform.  And there is no VCS2 ring on the pre-gen8
  2141.                          * platform. So the semaphore between RCS and VCS2 is
  2142.                          * initialized as INVALID.  Gen8 will initialize the
  2143.                          * sema between VCS2 and RCS later.
  2144.                  */
  2145.                 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
  2146.                 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
  2147.                 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
  2148.                 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
  2149.                 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2150.                 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
  2151.                 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
  2152.                 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
  2153.                 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
  2154.                 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2155.                 }
  2156.         } else if (IS_GEN5(dev)) {
  2157.        ring->add_request = pc_render_add_request;
  2158.                 ring->flush = gen4_render_ring_flush;
  2159.                 ring->get_seqno = pc_render_get_seqno;
  2160.                 ring->set_seqno = pc_render_set_seqno;
  2161.                 ring->irq_get = gen5_ring_get_irq;
  2162.                 ring->irq_put = gen5_ring_put_irq;
  2163.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
  2164.                                         GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
  2165.         } else {
  2166.                 ring->add_request = i9xx_add_request;
  2167.                 if (INTEL_INFO(dev)->gen < 4)
  2168.                         ring->flush = gen2_render_ring_flush;
  2169.                 else
  2170.                         ring->flush = gen4_render_ring_flush;
  2171.                 ring->get_seqno = ring_get_seqno;
  2172.                 ring->set_seqno = ring_set_seqno;
  2173.                 if (IS_GEN2(dev)) {
  2174.                         ring->irq_get = i8xx_ring_get_irq;
  2175.                         ring->irq_put = i8xx_ring_put_irq;
  2176.                 } else {
  2177.                         ring->irq_get = i9xx_ring_get_irq;
  2178.                         ring->irq_put = i9xx_ring_put_irq;
  2179.                 }
  2180.                 ring->irq_enable_mask = I915_USER_INTERRUPT;
  2181.         }
  2182.         ring->write_tail = ring_write_tail;
  2183.  
  2184.         if (IS_HASWELL(dev))
  2185.                 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
  2186.         else if (IS_GEN8(dev))
  2187.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2188.         else if (INTEL_INFO(dev)->gen >= 6)
  2189.                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2190.         else if (INTEL_INFO(dev)->gen >= 4)
  2191.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2192.         else if (IS_I830(dev) || IS_845G(dev))
  2193.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  2194.         else
  2195.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  2196.         ring->init = init_render_ring;
  2197.         ring->cleanup = render_ring_cleanup;
  2198.  
  2199.         /* Workaround batchbuffer to combat CS tlb bug. */
  2200.         if (HAS_BROKEN_CS_TLB(dev)) {
  2201.                 obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
  2202.                 if (obj == NULL) {
  2203.                         DRM_ERROR("Failed to allocate batch bo\n");
  2204.                         return -ENOMEM;
  2205.                 }
  2206.  
  2207.                 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
  2208.                 if (ret != 0) {
  2209.                         drm_gem_object_unreference(&obj->base);
  2210.                         DRM_ERROR("Failed to ping batch bo\n");
  2211.                         return ret;
  2212.                 }
  2213.  
  2214.                 ring->scratch.obj = obj;
  2215.                 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
  2216.         }
  2217.  
  2218.         return intel_init_ring_buffer(dev, ring);
  2219. }
  2220.  
  2221. #if 0
  2222. int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
  2223. {
  2224.         struct drm_i915_private *dev_priv = dev->dev_private;
  2225.         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
  2226.         struct intel_ringbuffer *ringbuf = ring->buffer;
  2227.         int ret;
  2228.  
  2229.         if (ringbuf == NULL) {
  2230.                 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
  2231.                 if (!ringbuf)
  2232.                         return -ENOMEM;
  2233.                 ring->buffer = ringbuf;
  2234.         }
  2235.  
  2236.         ring->name = "render ring";
  2237.         ring->id = RCS;
  2238.         ring->mmio_base = RENDER_RING_BASE;
  2239.  
  2240.         if (INTEL_INFO(dev)->gen >= 6) {
  2241.                 /* non-kms not supported on gen6+ */
  2242.                 ret = -ENODEV;
  2243.                 goto err_ringbuf;
  2244.         }
  2245.  
  2246.         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
  2247.          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
  2248.          * the special gen5 functions. */
  2249.         ring->add_request = i9xx_add_request;
  2250.         if (INTEL_INFO(dev)->gen < 4)
  2251.                 ring->flush = gen2_render_ring_flush;
  2252.         else
  2253.                 ring->flush = gen4_render_ring_flush;
  2254.         ring->get_seqno = ring_get_seqno;
  2255.         ring->set_seqno = ring_set_seqno;
  2256.         if (IS_GEN2(dev)) {
  2257.                 ring->irq_get = i8xx_ring_get_irq;
  2258.                 ring->irq_put = i8xx_ring_put_irq;
  2259.         } else {
  2260.                 ring->irq_get = i9xx_ring_get_irq;
  2261.                 ring->irq_put = i9xx_ring_put_irq;
  2262.         }
  2263.         ring->irq_enable_mask = I915_USER_INTERRUPT;
  2264.         ring->write_tail = ring_write_tail;
  2265.         if (INTEL_INFO(dev)->gen >= 4)
  2266.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2267.         else if (IS_I830(dev) || IS_845G(dev))
  2268.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  2269.         else
  2270.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  2271.         ring->init = init_render_ring;
  2272.         ring->cleanup = render_ring_cleanup;
  2273.  
  2274.         ring->dev = dev;
  2275.         INIT_LIST_HEAD(&ring->active_list);
  2276.         INIT_LIST_HEAD(&ring->request_list);
  2277.  
  2278.         ringbuf->size = size;
  2279.         ringbuf->effective_size = ringbuf->size;
  2280.         if (IS_I830(ring->dev) || IS_845G(ring->dev))
  2281.                 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
  2282.  
  2283.         ringbuf->virtual_start = ioremap_wc(start, size);
  2284.         if (ringbuf->virtual_start == NULL) {
  2285.                 DRM_ERROR("can not ioremap virtual address for"
  2286.                           " ring buffer\n");
  2287.                 ret = -ENOMEM;
  2288.                 goto err_ringbuf;
  2289.         }
  2290.  
  2291.         if (!I915_NEED_GFX_HWS(dev)) {
  2292.                 ret = init_phys_status_page(ring);
  2293.                 if (ret)
  2294.                         goto err_vstart;
  2295.         }
  2296.  
  2297.         return 0;
  2298.  
  2299. err_vstart:
  2300.         iounmap(ringbuf->virtual_start);
  2301. err_ringbuf:
  2302.         kfree(ringbuf);
  2303.         ring->buffer = NULL;
  2304.         return ret;
  2305. }
  2306. #endif
  2307.  
  2308. int intel_init_bsd_ring_buffer(struct drm_device *dev)
  2309. {
  2310.         struct drm_i915_private *dev_priv = dev->dev_private;
  2311.         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
  2312.  
  2313.         ring->name = "bsd ring";
  2314.         ring->id = VCS;
  2315.  
  2316.         ring->write_tail = ring_write_tail;
  2317.         if (INTEL_INFO(dev)->gen >= 6) {
  2318.                 ring->mmio_base = GEN6_BSD_RING_BASE;
  2319.                 /* gen6 bsd needs a special wa for tail updates */
  2320.                 if (IS_GEN6(dev))
  2321.                         ring->write_tail = gen6_bsd_ring_write_tail;
  2322.                 ring->flush = gen6_bsd_ring_flush;
  2323.                 ring->add_request = gen6_add_request;
  2324.                 ring->get_seqno = gen6_ring_get_seqno;
  2325.                 ring->set_seqno = ring_set_seqno;
  2326.                 if (INTEL_INFO(dev)->gen >= 8) {
  2327.                         ring->irq_enable_mask =
  2328.                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
  2329.                         ring->irq_get = gen8_ring_get_irq;
  2330.                         ring->irq_put = gen8_ring_put_irq;
  2331.                         ring->dispatch_execbuffer =
  2332.                                 gen8_ring_dispatch_execbuffer;
  2333.                         if (i915_semaphore_is_enabled(dev)) {
  2334.                                 ring->semaphore.sync_to = gen8_ring_sync;
  2335.                                 ring->semaphore.signal = gen8_xcs_signal;
  2336.                                 GEN8_RING_SEMAPHORE_INIT;
  2337.                         }
  2338.                 } else {
  2339.                 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
  2340.                 ring->irq_get = gen6_ring_get_irq;
  2341.                 ring->irq_put = gen6_ring_put_irq;
  2342.                         ring->dispatch_execbuffer =
  2343.                                 gen6_ring_dispatch_execbuffer;
  2344.                         if (i915_semaphore_is_enabled(dev)) {
  2345.                 ring->semaphore.sync_to = gen6_ring_sync;
  2346.                 ring->semaphore.signal = gen6_signal;
  2347.                 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
  2348.                 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
  2349.                 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
  2350.                 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
  2351.                 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2352.                 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
  2353.                 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
  2354.                 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
  2355.                 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
  2356.                 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2357.                         }
  2358.                 }
  2359.         } else {
  2360.                 ring->mmio_base = BSD_RING_BASE;
  2361.                 ring->flush = bsd_ring_flush;
  2362.                 ring->add_request = i9xx_add_request;
  2363.                 ring->get_seqno = ring_get_seqno;
  2364.                 ring->set_seqno = ring_set_seqno;
  2365.                 if (IS_GEN5(dev)) {
  2366.                         ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
  2367.                         ring->irq_get = gen5_ring_get_irq;
  2368.                         ring->irq_put = gen5_ring_put_irq;
  2369.                 } else {
  2370.                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
  2371.                         ring->irq_get = i9xx_ring_get_irq;
  2372.                         ring->irq_put = i9xx_ring_put_irq;
  2373.                 }
  2374.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2375.         }
  2376.         ring->init = init_ring_common;
  2377.  
  2378.         return intel_init_ring_buffer(dev, ring);
  2379. }
  2380.  
  2381. /**
  2382.  * Initialize the second BSD ring for Broadwell GT3.
  2383.  * It is noted that this only exists on Broadwell GT3.
  2384.  */
  2385. int intel_init_bsd2_ring_buffer(struct drm_device *dev)
  2386. {
  2387.         struct drm_i915_private *dev_priv = dev->dev_private;
  2388.         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
  2389.  
  2390.         if ((INTEL_INFO(dev)->gen != 8)) {
  2391.                 DRM_ERROR("No dual-BSD ring on non-BDW machine\n");
  2392.                 return -EINVAL;
  2393.         }
  2394.  
  2395.         ring->name = "bsd2 ring";
  2396.         ring->id = VCS2;
  2397.  
  2398.         ring->write_tail = ring_write_tail;
  2399.         ring->mmio_base = GEN8_BSD2_RING_BASE;
  2400.         ring->flush = gen6_bsd_ring_flush;
  2401.         ring->add_request = gen6_add_request;
  2402.         ring->get_seqno = gen6_ring_get_seqno;
  2403.         ring->set_seqno = ring_set_seqno;
  2404.         ring->irq_enable_mask =
  2405.                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
  2406.         ring->irq_get = gen8_ring_get_irq;
  2407.         ring->irq_put = gen8_ring_put_irq;
  2408.         ring->dispatch_execbuffer =
  2409.                         gen8_ring_dispatch_execbuffer;
  2410.         if (i915_semaphore_is_enabled(dev)) {
  2411.                 ring->semaphore.sync_to = gen8_ring_sync;
  2412.                 ring->semaphore.signal = gen8_xcs_signal;
  2413.                 GEN8_RING_SEMAPHORE_INIT;
  2414.         }
  2415.         ring->init = init_ring_common;
  2416.  
  2417.         return intel_init_ring_buffer(dev, ring);
  2418. }
  2419.  
  2420. int intel_init_blt_ring_buffer(struct drm_device *dev)
  2421. {
  2422.         struct drm_i915_private *dev_priv = dev->dev_private;
  2423.         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
  2424.  
  2425.         ring->name = "blitter ring";
  2426.         ring->id = BCS;
  2427.  
  2428.         ring->mmio_base = BLT_RING_BASE;
  2429.         ring->write_tail = ring_write_tail;
  2430.         ring->flush = gen6_ring_flush;
  2431.         ring->add_request = gen6_add_request;
  2432.         ring->get_seqno = gen6_ring_get_seqno;
  2433.         ring->set_seqno = ring_set_seqno;
  2434.         if (INTEL_INFO(dev)->gen >= 8) {
  2435.                 ring->irq_enable_mask =
  2436.                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
  2437.                 ring->irq_get = gen8_ring_get_irq;
  2438.                 ring->irq_put = gen8_ring_put_irq;
  2439.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2440.                 if (i915_semaphore_is_enabled(dev)) {
  2441.                         ring->semaphore.sync_to = gen8_ring_sync;
  2442.                         ring->semaphore.signal = gen8_xcs_signal;
  2443.                         GEN8_RING_SEMAPHORE_INIT;
  2444.                 }
  2445.         } else {
  2446.         ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
  2447.         ring->irq_get = gen6_ring_get_irq;
  2448.         ring->irq_put = gen6_ring_put_irq;
  2449.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2450.                 if (i915_semaphore_is_enabled(dev)) {
  2451.                         ring->semaphore.signal = gen6_signal;
  2452.         ring->semaphore.sync_to = gen6_ring_sync;
  2453.         /*
  2454.                          * The current semaphore is only applied on pre-gen8
  2455.                          * platform.  And there is no VCS2 ring on the pre-gen8
  2456.                          * platform. So the semaphore between BCS and VCS2 is
  2457.                          * initialized as INVALID.  Gen8 will initialize the
  2458.                          * sema between BCS and VCS2 later.
  2459.          */
  2460.         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
  2461.         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
  2462.         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
  2463.         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
  2464.         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2465.         ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
  2466.         ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
  2467.         ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
  2468.         ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
  2469.         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2470.                 }
  2471.         }
  2472.         ring->init = init_ring_common;
  2473.  
  2474.         return intel_init_ring_buffer(dev, ring);
  2475. }
  2476.  
  2477. int intel_init_vebox_ring_buffer(struct drm_device *dev)
  2478. {
  2479.         struct drm_i915_private *dev_priv = dev->dev_private;
  2480.         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
  2481.  
  2482.         ring->name = "video enhancement ring";
  2483.         ring->id = VECS;
  2484.  
  2485.         ring->mmio_base = VEBOX_RING_BASE;
  2486.         ring->write_tail = ring_write_tail;
  2487.         ring->flush = gen6_ring_flush;
  2488.         ring->add_request = gen6_add_request;
  2489.         ring->get_seqno = gen6_ring_get_seqno;
  2490.         ring->set_seqno = ring_set_seqno;
  2491.  
  2492.         if (INTEL_INFO(dev)->gen >= 8) {
  2493.                 ring->irq_enable_mask =
  2494.                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
  2495.                 ring->irq_get = gen8_ring_get_irq;
  2496.                 ring->irq_put = gen8_ring_put_irq;
  2497.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2498.                 if (i915_semaphore_is_enabled(dev)) {
  2499.                         ring->semaphore.sync_to = gen8_ring_sync;
  2500.                         ring->semaphore.signal = gen8_xcs_signal;
  2501.                         GEN8_RING_SEMAPHORE_INIT;
  2502.                 }
  2503.         } else {
  2504.         ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
  2505.         ring->irq_get = hsw_vebox_get_irq;
  2506.         ring->irq_put = hsw_vebox_put_irq;
  2507.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2508.                 if (i915_semaphore_is_enabled(dev)) {
  2509.         ring->semaphore.sync_to = gen6_ring_sync;
  2510.         ring->semaphore.signal = gen6_signal;
  2511.         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
  2512.         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
  2513.         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
  2514.         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
  2515.         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2516.         ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
  2517.         ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
  2518.         ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
  2519.         ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
  2520.         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2521.                 }
  2522.         }
  2523.         ring->init = init_ring_common;
  2524.  
  2525.         return intel_init_ring_buffer(dev, ring);
  2526. }
  2527.  
  2528. int
  2529. intel_ring_flush_all_caches(struct intel_engine_cs *ring)
  2530. {
  2531.         int ret;
  2532.  
  2533.         if (!ring->gpu_caches_dirty)
  2534.                 return 0;
  2535.  
  2536.         ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
  2537.         if (ret)
  2538.                 return ret;
  2539.  
  2540.         trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
  2541.  
  2542.         ring->gpu_caches_dirty = false;
  2543.         return 0;
  2544. }
  2545.  
  2546. int
  2547. intel_ring_invalidate_all_caches(struct intel_engine_cs *ring)
  2548. {
  2549.         uint32_t flush_domains;
  2550.         int ret;
  2551.  
  2552.         flush_domains = 0;
  2553.         if (ring->gpu_caches_dirty)
  2554.                 flush_domains = I915_GEM_GPU_DOMAINS;
  2555.  
  2556.         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  2557.         if (ret)
  2558.                 return ret;
  2559.  
  2560.         trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  2561.  
  2562.         ring->gpu_caches_dirty = false;
  2563.         return 0;
  2564. }
  2565.  
  2566. void
  2567. intel_stop_ring_buffer(struct intel_engine_cs *ring)
  2568. {
  2569.         int ret;
  2570.  
  2571.         if (!intel_ring_initialized(ring))
  2572.                 return;
  2573.  
  2574.         ret = intel_ring_idle(ring);
  2575.         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
  2576.                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
  2577.                           ring->name, ret);
  2578.  
  2579.         stop_ring(ring);
  2580. }
  2581.