Subversion Repositories Kolibri OS

Rev

Rev 4560 | Rev 5128 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008-2010 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *    Zou Nan hai <nanhai.zou@intel.com>
  26.  *    Xiang Hai hao<haihao.xiang@intel.com>
  27.  *
  28.  */
  29.  
  30. #include <drm/drmP.h>
  31. #include "i915_drv.h"
  32. #include <drm/i915_drm.h>
  33. #include "i915_trace.h"
  34. #include "intel_drv.h"
  35.  
  36. /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
  37.  * but keeps the logic simple. Indeed, the whole purpose of this macro is just
  38.  * to give some inclination as to some of the magic values used in the various
  39.  * workarounds!
  40.  */
  41. #define CACHELINE_BYTES 64
  42.  
  43. static inline int __ring_space(int head, int tail, int size)
  44. {
  45.         int space = head - (tail + I915_RING_FREE_SPACE);
  46.         if (space < 0)
  47.                 space += size;
  48.         return space;
  49. }
  50.  
  51. static inline int ring_space(struct intel_ringbuffer *ringbuf)
  52. {
  53.         return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size);
  54. }
  55.  
  56. static bool intel_ring_stopped(struct intel_engine_cs *ring)
  57. {
  58.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  59.         return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
  60. }
  61.  
  62. void __intel_ring_advance(struct intel_engine_cs *ring)
  63. {
  64.         struct intel_ringbuffer *ringbuf = ring->buffer;
  65.         ringbuf->tail &= ringbuf->size - 1;
  66.         if (intel_ring_stopped(ring))
  67.                 return;
  68.         ring->write_tail(ring, ringbuf->tail);
  69. }
  70.  
  71. static int
  72. gen2_render_ring_flush(struct intel_engine_cs *ring,
  73.                        u32      invalidate_domains,
  74.                        u32      flush_domains)
  75. {
  76.         u32 cmd;
  77.         int ret;
  78.  
  79.         cmd = MI_FLUSH;
  80.         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  81.                 cmd |= MI_NO_WRITE_FLUSH;
  82.  
  83.         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  84.                 cmd |= MI_READ_FLUSH;
  85.  
  86.         ret = intel_ring_begin(ring, 2);
  87.         if (ret)
  88.                 return ret;
  89.  
  90.         intel_ring_emit(ring, cmd);
  91.         intel_ring_emit(ring, MI_NOOP);
  92.         intel_ring_advance(ring);
  93.  
  94.         return 0;
  95. }
  96.  
  97. static int
  98. gen4_render_ring_flush(struct intel_engine_cs *ring,
  99.                   u32   invalidate_domains,
  100.                   u32   flush_domains)
  101. {
  102.         struct drm_device *dev = ring->dev;
  103.         u32 cmd;
  104.         int ret;
  105.  
  106.         /*
  107.          * read/write caches:
  108.          *
  109.          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  110.          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  111.          * also flushed at 2d versus 3d pipeline switches.
  112.          *
  113.          * read-only caches:
  114.          *
  115.          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  116.          * MI_READ_FLUSH is set, and is always flushed on 965.
  117.          *
  118.          * I915_GEM_DOMAIN_COMMAND may not exist?
  119.          *
  120.          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  121.          * invalidated when MI_EXE_FLUSH is set.
  122.          *
  123.          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  124.          * invalidated with every MI_FLUSH.
  125.          *
  126.          * TLBs:
  127.          *
  128.          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  129.          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  130.          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  131.          * are flushed at any MI_FLUSH.
  132.          */
  133.  
  134.         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  135.         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
  136.                 cmd &= ~MI_NO_WRITE_FLUSH;
  137.         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  138.                 cmd |= MI_EXE_FLUSH;
  139.  
  140.         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
  141.             (IS_G4X(dev) || IS_GEN5(dev)))
  142.                 cmd |= MI_INVALIDATE_ISP;
  143.  
  144.         ret = intel_ring_begin(ring, 2);
  145.         if (ret)
  146.                 return ret;
  147.  
  148.         intel_ring_emit(ring, cmd);
  149.         intel_ring_emit(ring, MI_NOOP);
  150.         intel_ring_advance(ring);
  151.  
  152.         return 0;
  153. }
  154.  
  155. /**
  156.  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
  157.  * implementing two workarounds on gen6.  From section 1.4.7.1
  158.  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
  159.  *
  160.  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
  161.  * produced by non-pipelined state commands), software needs to first
  162.  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
  163.  * 0.
  164.  *
  165.  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
  166.  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
  167.  *
  168.  * And the workaround for these two requires this workaround first:
  169.  *
  170.  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
  171.  * BEFORE the pipe-control with a post-sync op and no write-cache
  172.  * flushes.
  173.  *
  174.  * And this last workaround is tricky because of the requirements on
  175.  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
  176.  * volume 2 part 1:
  177.  *
  178.  *     "1 of the following must also be set:
  179.  *      - Render Target Cache Flush Enable ([12] of DW1)
  180.  *      - Depth Cache Flush Enable ([0] of DW1)
  181.  *      - Stall at Pixel Scoreboard ([1] of DW1)
  182.  *      - Depth Stall ([13] of DW1)
  183.  *      - Post-Sync Operation ([13] of DW1)
  184.  *      - Notify Enable ([8] of DW1)"
  185.  *
  186.  * The cache flushes require the workaround flush that triggered this
  187.  * one, so we can't use it.  Depth stall would trigger the same.
  188.  * Post-sync nonzero is what triggered this second workaround, so we
  189.  * can't use that one either.  Notify enable is IRQs, which aren't
  190.  * really our business.  That leaves only stall at scoreboard.
  191.  */
  192. static int
  193. intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring)
  194. {
  195.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  196.         int ret;
  197.  
  198.  
  199.         ret = intel_ring_begin(ring, 6);
  200.         if (ret)
  201.                 return ret;
  202.  
  203.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  204.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  205.                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
  206.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  207.         intel_ring_emit(ring, 0); /* low dword */
  208.         intel_ring_emit(ring, 0); /* high dword */
  209.         intel_ring_emit(ring, MI_NOOP);
  210.         intel_ring_advance(ring);
  211.  
  212.         ret = intel_ring_begin(ring, 6);
  213.         if (ret)
  214.                 return ret;
  215.  
  216.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  217.         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
  218.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  219.         intel_ring_emit(ring, 0);
  220.         intel_ring_emit(ring, 0);
  221.         intel_ring_emit(ring, MI_NOOP);
  222.         intel_ring_advance(ring);
  223.  
  224.         return 0;
  225. }
  226.  
  227. static int
  228. gen6_render_ring_flush(struct intel_engine_cs *ring,
  229.                          u32 invalidate_domains, u32 flush_domains)
  230. {
  231.         u32 flags = 0;
  232.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  233.         int ret;
  234.  
  235.         /* Force SNB workarounds for PIPE_CONTROL flushes */
  236.         ret = intel_emit_post_sync_nonzero_flush(ring);
  237.         if (ret)
  238.                 return ret;
  239.  
  240.         /* Just flush everything.  Experiments have shown that reducing the
  241.          * number of bits based on the write domains has little performance
  242.          * impact.
  243.          */
  244.         if (flush_domains) {
  245.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  246.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  247.                 /*
  248.                  * Ensure that any following seqno writes only happen
  249.                  * when the render cache is indeed flushed.
  250.                  */
  251.                 flags |= PIPE_CONTROL_CS_STALL;
  252.         }
  253.         if (invalidate_domains) {
  254.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  255.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  256.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  257.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  258.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  259.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  260.                 /*
  261.                  * TLB invalidate requires a post-sync write.
  262.                  */
  263.                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
  264.         }
  265.  
  266.         ret = intel_ring_begin(ring, 4);
  267.         if (ret)
  268.                 return ret;
  269.  
  270.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  271.         intel_ring_emit(ring, flags);
  272.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
  273.         intel_ring_emit(ring, 0);
  274.         intel_ring_advance(ring);
  275.  
  276.         return 0;
  277. }
  278.  
  279. static int
  280. gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
  281. {
  282.         int ret;
  283.  
  284.         ret = intel_ring_begin(ring, 4);
  285.         if (ret)
  286.                 return ret;
  287.  
  288.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  289.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  290.                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
  291.         intel_ring_emit(ring, 0);
  292.         intel_ring_emit(ring, 0);
  293.         intel_ring_advance(ring);
  294.  
  295.         return 0;
  296. }
  297.  
  298. static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value)
  299. {
  300.         int ret;
  301.  
  302.         if (!ring->fbc_dirty)
  303.                 return 0;
  304.  
  305.         ret = intel_ring_begin(ring, 6);
  306.         if (ret)
  307.                 return ret;
  308.         /* WaFbcNukeOn3DBlt:ivb/hsw */
  309.         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  310.         intel_ring_emit(ring, MSG_FBC_REND_STATE);
  311.         intel_ring_emit(ring, value);
  312.         intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
  313.         intel_ring_emit(ring, MSG_FBC_REND_STATE);
  314.         intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
  315.         intel_ring_advance(ring);
  316.  
  317.         ring->fbc_dirty = false;
  318.         return 0;
  319. }
  320.  
  321. static int
  322. gen7_render_ring_flush(struct intel_engine_cs *ring,
  323.                        u32 invalidate_domains, u32 flush_domains)
  324. {
  325.         u32 flags = 0;
  326.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  327.         int ret;
  328.  
  329.         /*
  330.          * Ensure that any following seqno writes only happen when the render
  331.          * cache is indeed flushed.
  332.          *
  333.          * Workaround: 4th PIPE_CONTROL command (except the ones with only
  334.          * read-cache invalidate bits set) must have the CS_STALL bit set. We
  335.          * don't try to be clever and just set it unconditionally.
  336.          */
  337.         flags |= PIPE_CONTROL_CS_STALL;
  338.  
  339.         /* Just flush everything.  Experiments have shown that reducing the
  340.          * number of bits based on the write domains has little performance
  341.          * impact.
  342.          */
  343.         if (flush_domains) {
  344.         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  345.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  346.         }
  347.         if (invalidate_domains) {
  348.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  349.         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  350.         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  351.         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  352.         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  353.         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  354.                 /*
  355.                  * TLB invalidate requires a post-sync write.
  356.                  */
  357.                 flags |= PIPE_CONTROL_QW_WRITE;
  358.                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  359.  
  360.                 /* Workaround: we must issue a pipe_control with CS-stall bit
  361.                  * set before a pipe_control command that has the state cache
  362.                  * invalidate bit set. */
  363.                 gen7_render_ring_cs_stall_wa(ring);
  364.         }
  365.  
  366.         ret = intel_ring_begin(ring, 4);
  367.         if (ret)
  368.                 return ret;
  369.  
  370.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  371.         intel_ring_emit(ring, flags);
  372.         intel_ring_emit(ring, scratch_addr);
  373.         intel_ring_emit(ring, 0);
  374.         intel_ring_advance(ring);
  375.  
  376.         if (!invalidate_domains && flush_domains)
  377.                 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
  378.  
  379.         return 0;
  380. }
  381.  
  382. static int
  383. gen8_emit_pipe_control(struct intel_engine_cs *ring,
  384.                        u32 flags, u32 scratch_addr)
  385. {
  386.         int ret;
  387.  
  388.         ret = intel_ring_begin(ring, 6);
  389.         if (ret)
  390.                 return ret;
  391.  
  392.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
  393.         intel_ring_emit(ring, flags);
  394.         intel_ring_emit(ring, scratch_addr);
  395.         intel_ring_emit(ring, 0);
  396.         intel_ring_emit(ring, 0);
  397.         intel_ring_emit(ring, 0);
  398.         intel_ring_advance(ring);
  399.  
  400.         return 0;
  401. }
  402.  
  403. static int
  404. gen8_render_ring_flush(struct intel_engine_cs *ring,
  405.                        u32 invalidate_domains, u32 flush_domains)
  406. {
  407.         u32 flags = 0;
  408.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  409.         int ret;
  410.  
  411.         flags |= PIPE_CONTROL_CS_STALL;
  412.  
  413.         if (flush_domains) {
  414.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  415.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  416.         }
  417.         if (invalidate_domains) {
  418.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  419.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  420.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  421.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  422.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  423.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  424.                 flags |= PIPE_CONTROL_QW_WRITE;
  425.                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  426.  
  427.                 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
  428.                 ret = gen8_emit_pipe_control(ring,
  429.                                              PIPE_CONTROL_CS_STALL |
  430.                                              PIPE_CONTROL_STALL_AT_SCOREBOARD,
  431.                                              0);
  432.         if (ret)
  433.                 return ret;
  434.         }
  435.  
  436.         return gen8_emit_pipe_control(ring, flags, scratch_addr);
  437. }
  438.  
  439. static void ring_write_tail(struct intel_engine_cs *ring,
  440.                             u32 value)
  441. {
  442.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  443.         I915_WRITE_TAIL(ring, value);
  444. }
  445.  
  446. u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
  447. {
  448.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  449.         u64 acthd;
  450.  
  451.         if (INTEL_INFO(ring->dev)->gen >= 8)
  452.                 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
  453.                                          RING_ACTHD_UDW(ring->mmio_base));
  454.         else if (INTEL_INFO(ring->dev)->gen >= 4)
  455.                 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
  456.         else
  457.                 acthd = I915_READ(ACTHD);
  458.  
  459.         return acthd;
  460. }
  461.  
  462. static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
  463. {
  464.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  465.         u32 addr;
  466.  
  467.         addr = dev_priv->status_page_dmah->busaddr;
  468.         if (INTEL_INFO(ring->dev)->gen >= 4)
  469.                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
  470.         I915_WRITE(HWS_PGA, addr);
  471. }
  472.  
  473. static bool stop_ring(struct intel_engine_cs *ring)
  474. {
  475.         struct drm_i915_private *dev_priv = to_i915(ring->dev);
  476.  
  477.         if (!IS_GEN2(ring->dev)) {
  478.                 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
  479.                 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
  480.                         DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
  481.                         return false;
  482.                 }
  483.         }
  484.  
  485.         I915_WRITE_CTL(ring, 0);
  486.         I915_WRITE_HEAD(ring, 0);
  487.         ring->write_tail(ring, 0);
  488.  
  489.         if (!IS_GEN2(ring->dev)) {
  490.                 (void)I915_READ_CTL(ring);
  491.                 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
  492.         }
  493.  
  494.         return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
  495. }
  496.  
  497. static int init_ring_common(struct intel_engine_cs *ring)
  498. {
  499.         struct drm_device *dev = ring->dev;
  500.         struct drm_i915_private *dev_priv = dev->dev_private;
  501.         struct intel_ringbuffer *ringbuf = ring->buffer;
  502.         struct drm_i915_gem_object *obj = ringbuf->obj;
  503.         int ret = 0;
  504.  
  505.         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
  506.  
  507.         if (!stop_ring(ring)) {
  508.                 /* G45 ring initialization often fails to reset head to zero */
  509.                 DRM_DEBUG_KMS("%s head not reset to zero "
  510.                               "ctl %08x head %08x tail %08x start %08x\n",
  511.                               ring->name,
  512.                               I915_READ_CTL(ring),
  513.                               I915_READ_HEAD(ring),
  514.                               I915_READ_TAIL(ring),
  515.                               I915_READ_START(ring));
  516.  
  517.                 if (!stop_ring(ring)) {
  518.                         DRM_ERROR("failed to set %s head to zero "
  519.                                   "ctl %08x head %08x tail %08x start %08x\n",
  520.                                   ring->name,
  521.                                   I915_READ_CTL(ring),
  522.                                   I915_READ_HEAD(ring),
  523.                                   I915_READ_TAIL(ring),
  524.                                   I915_READ_START(ring));
  525.                         ret = -EIO;
  526.                         goto out;
  527.                 }
  528.         }
  529.  
  530.         if (I915_NEED_GFX_HWS(dev))
  531.                 intel_ring_setup_status_page(ring);
  532.         else
  533.                 ring_setup_phys_status_page(ring);
  534.  
  535.         /* Enforce ordering by reading HEAD register back */
  536.         I915_READ_HEAD(ring);
  537.  
  538.         /* Initialize the ring. This must happen _after_ we've cleared the ring
  539.          * registers with the above sequence (the readback of the HEAD registers
  540.          * also enforces ordering), otherwise the hw might lose the new ring
  541.          * register values. */
  542.         I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
  543.         I915_WRITE_CTL(ring,
  544.                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
  545.                         | RING_VALID);
  546.  
  547.         /* If the head is still not zero, the ring is dead */
  548.         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
  549.                      I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
  550.                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
  551.                 DRM_ERROR("%s initialization failed "
  552.                           "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
  553.                                 ring->name,
  554.                           I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
  555.                           I915_READ_HEAD(ring), I915_READ_TAIL(ring),
  556.                           I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
  557.                 ret = -EIO;
  558.                 goto out;
  559.         }
  560.  
  561.  
  562.                 ringbuf->head = I915_READ_HEAD(ring);
  563.                 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
  564.                 ringbuf->space = ring_space(ringbuf);
  565.                 ringbuf->last_retired_head = -1;
  566.  
  567.         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
  568.  
  569. out:
  570.         gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
  571.  
  572.         return ret;
  573. }
  574.  
  575. static int
  576. init_pipe_control(struct intel_engine_cs *ring)
  577. {
  578.         int ret;
  579.  
  580.         if (ring->scratch.obj)
  581.                 return 0;
  582.  
  583.         ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
  584.         if (ring->scratch.obj == NULL) {
  585.                 DRM_ERROR("Failed to allocate seqno page\n");
  586.                 ret = -ENOMEM;
  587.                 goto err;
  588.         }
  589.  
  590.         ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
  591.         if (ret)
  592.                 goto err_unref;
  593.  
  594.         ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
  595.         if (ret)
  596.                 goto err_unref;
  597.  
  598.         ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
  599.     ring->scratch.cpu_page = (void*)MapIoMem((addr_t)sg_page(ring->scratch.obj->pages->sgl),4096, PG_SW|0x100);
  600.         if (ring->scratch.cpu_page == NULL) {
  601.                 ret = -ENOMEM;
  602.                 goto err_unpin;
  603.         }
  604.  
  605.         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
  606.                          ring->name, ring->scratch.gtt_offset);
  607.         return 0;
  608.  
  609. err_unpin:
  610.         i915_gem_object_ggtt_unpin(ring->scratch.obj);
  611. err_unref:
  612.         drm_gem_object_unreference(&ring->scratch.obj->base);
  613. err:
  614.         return ret;
  615. }
  616.  
  617. static int init_render_ring(struct intel_engine_cs *ring)
  618. {
  619.         struct drm_device *dev = ring->dev;
  620.         struct drm_i915_private *dev_priv = dev->dev_private;
  621.         int ret = init_ring_common(ring);
  622.         if (ret)
  623.                 return ret;
  624.  
  625.         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
  626.         if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
  627.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
  628.  
  629.         /* We need to disable the AsyncFlip performance optimisations in order
  630.          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
  631.          * programmed to '1' on all products.
  632.          *
  633.          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
  634.          */
  635.         if (INTEL_INFO(dev)->gen >= 6)
  636.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
  637.  
  638.         /* Required for the hardware to program scanline values for waiting */
  639.         /* WaEnableFlushTlbInvalidationMode:snb */
  640.         if (INTEL_INFO(dev)->gen == 6)
  641.                 I915_WRITE(GFX_MODE,
  642.                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
  643.  
  644.         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
  645.                 if (IS_GEN7(dev))
  646.                         I915_WRITE(GFX_MODE_GEN7,
  647.                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
  648.                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
  649.  
  650.         if (INTEL_INFO(dev)->gen >= 5) {
  651.                 ret = init_pipe_control(ring);
  652.                 if (ret)
  653.                         return ret;
  654.         }
  655.  
  656.         if (IS_GEN6(dev)) {
  657.                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
  658.                  * "If this bit is set, STCunit will have LRA as replacement
  659.                  *  policy. [...] This bit must be reset.  LRA replacement
  660.                  *  policy is not supported."
  661.                  */
  662.                 I915_WRITE(CACHE_MODE_0,
  663.                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
  664.         }
  665.  
  666.         if (INTEL_INFO(dev)->gen >= 6)
  667.                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
  668.  
  669.         if (HAS_L3_DPF(dev))
  670.                 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
  671.  
  672.         return ret;
  673. }
  674.  
  675. static void render_ring_cleanup(struct intel_engine_cs *ring)
  676. {
  677.         struct drm_device *dev = ring->dev;
  678.  
  679.         if (ring->scratch.obj == NULL)
  680.                 return;
  681.  
  682.         if (INTEL_INFO(dev)->gen >= 5) {
  683. //              kunmap(sg_page(ring->scratch.obj->pages->sgl));
  684.                 i915_gem_object_ggtt_unpin(ring->scratch.obj);
  685.         }
  686.  
  687.         drm_gem_object_unreference(&ring->scratch.obj->base);
  688.         ring->scratch.obj = NULL;
  689. }
  690.  
  691. static int gen8_rcs_signal(struct intel_engine_cs *signaller,
  692.                            unsigned int num_dwords)
  693. {
  694. #define MBOX_UPDATE_DWORDS 8
  695.         struct drm_device *dev = signaller->dev;
  696.         struct drm_i915_private *dev_priv = dev->dev_private;
  697.         struct intel_engine_cs *waiter;
  698.         int i, ret, num_rings;
  699.  
  700.         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
  701.         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
  702. #undef MBOX_UPDATE_DWORDS
  703.  
  704.         ret = intel_ring_begin(signaller, num_dwords);
  705.         if (ret)
  706.                 return ret;
  707.  
  708.         for_each_ring(waiter, dev_priv, i) {
  709.                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
  710.                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
  711.                         continue;
  712.  
  713.                 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
  714.                 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
  715.                                            PIPE_CONTROL_QW_WRITE |
  716.                                            PIPE_CONTROL_FLUSH_ENABLE);
  717.                 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
  718.                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
  719.                 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
  720.                 intel_ring_emit(signaller, 0);
  721.                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
  722.                                            MI_SEMAPHORE_TARGET(waiter->id));
  723.                 intel_ring_emit(signaller, 0);
  724.         }
  725.  
  726.         return 0;
  727. }
  728.  
  729. static int gen8_xcs_signal(struct intel_engine_cs *signaller,
  730.                            unsigned int num_dwords)
  731. {
  732. #define MBOX_UPDATE_DWORDS 6
  733.         struct drm_device *dev = signaller->dev;
  734.         struct drm_i915_private *dev_priv = dev->dev_private;
  735.         struct intel_engine_cs *waiter;
  736.         int i, ret, num_rings;
  737.  
  738.         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
  739.         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
  740. #undef MBOX_UPDATE_DWORDS
  741.  
  742.         ret = intel_ring_begin(signaller, num_dwords);
  743.         if (ret)
  744.                 return ret;
  745.  
  746.         for_each_ring(waiter, dev_priv, i) {
  747.                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
  748.                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
  749.                         continue;
  750.  
  751.                 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
  752.                                            MI_FLUSH_DW_OP_STOREDW);
  753.                 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
  754.                                            MI_FLUSH_DW_USE_GTT);
  755.                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
  756.                 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
  757.                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
  758.                                            MI_SEMAPHORE_TARGET(waiter->id));
  759.                 intel_ring_emit(signaller, 0);
  760.         }
  761.  
  762.         return 0;
  763. }
  764.  
  765. static int gen6_signal(struct intel_engine_cs *signaller,
  766.                        unsigned int num_dwords)
  767. {
  768.         struct drm_device *dev = signaller->dev;
  769.         struct drm_i915_private *dev_priv = dev->dev_private;
  770.         struct intel_engine_cs *useless;
  771.         int i, ret, num_rings;
  772.  
  773. #define MBOX_UPDATE_DWORDS 3
  774.         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
  775.         num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
  776. #undef MBOX_UPDATE_DWORDS
  777.  
  778.         ret = intel_ring_begin(signaller, num_dwords);
  779.         if (ret)
  780.                 return ret;
  781.  
  782.         for_each_ring(useless, dev_priv, i) {
  783.                 u32 mbox_reg = signaller->semaphore.mbox.signal[i];
  784.                 if (mbox_reg != GEN6_NOSYNC) {
  785.                         intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
  786.                         intel_ring_emit(signaller, mbox_reg);
  787.                         intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
  788.                 }
  789.         }
  790.  
  791.         /* If num_dwords was rounded, make sure the tail pointer is correct */
  792.         if (num_rings % 2 == 0)
  793.                 intel_ring_emit(signaller, MI_NOOP);
  794.  
  795.         return 0;
  796. }
  797.  
  798. /**
  799.  * gen6_add_request - Update the semaphore mailbox registers
  800.  *
  801.  * @ring - ring that is adding a request
  802.  * @seqno - return seqno stuck into the ring
  803.  *
  804.  * Update the mailbox registers in the *other* rings with the current seqno.
  805.  * This acts like a signal in the canonical semaphore.
  806.  */
  807. static int
  808. gen6_add_request(struct intel_engine_cs *ring)
  809. {
  810.         int ret;
  811.  
  812.         if (ring->semaphore.signal)
  813.         ret = ring->semaphore.signal(ring, 4);
  814.         else
  815.                 ret = intel_ring_begin(ring, 4);
  816.  
  817.         if (ret)
  818.                 return ret;
  819.  
  820.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  821.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  822.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  823.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  824.         __intel_ring_advance(ring);
  825.  
  826.         return 0;
  827. }
  828.  
  829. static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
  830.                                               u32 seqno)
  831. {
  832.         struct drm_i915_private *dev_priv = dev->dev_private;
  833.         return dev_priv->last_seqno < seqno;
  834. }
  835.  
  836. /**
  837.  * intel_ring_sync - sync the waiter to the signaller on seqno
  838.  *
  839.  * @waiter - ring that is waiting
  840.  * @signaller - ring which has, or will signal
  841.  * @seqno - seqno which the waiter will block on
  842.  */
  843.  
  844. static int
  845. gen8_ring_sync(struct intel_engine_cs *waiter,
  846.                struct intel_engine_cs *signaller,
  847.                u32 seqno)
  848. {
  849.         struct drm_i915_private *dev_priv = waiter->dev->dev_private;
  850.         int ret;
  851.  
  852.         ret = intel_ring_begin(waiter, 4);
  853.         if (ret)
  854.                 return ret;
  855.  
  856.         intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
  857.                                 MI_SEMAPHORE_GLOBAL_GTT |
  858.                                 MI_SEMAPHORE_POLL |
  859.                                 MI_SEMAPHORE_SAD_GTE_SDD);
  860.         intel_ring_emit(waiter, seqno);
  861.         intel_ring_emit(waiter,
  862.                         lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
  863.         intel_ring_emit(waiter,
  864.                         upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
  865.         intel_ring_advance(waiter);
  866.         return 0;
  867. }
  868.  
  869. static int
  870. gen6_ring_sync(struct intel_engine_cs *waiter,
  871.                struct intel_engine_cs *signaller,
  872.                 u32 seqno)
  873. {
  874.         u32 dw1 = MI_SEMAPHORE_MBOX |
  875.                   MI_SEMAPHORE_COMPARE |
  876.                   MI_SEMAPHORE_REGISTER;
  877.         u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
  878.         int ret;
  879.  
  880.         /* Throughout all of the GEM code, seqno passed implies our current
  881.          * seqno is >= the last seqno executed. However for hardware the
  882.          * comparison is strictly greater than.
  883.          */
  884.         seqno -= 1;
  885.  
  886.         WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
  887.  
  888.         ret = intel_ring_begin(waiter, 4);
  889.         if (ret)
  890.                 return ret;
  891.  
  892.         /* If seqno wrap happened, omit the wait with no-ops */
  893.         if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
  894.                 intel_ring_emit(waiter, dw1 | wait_mbox);
  895.         intel_ring_emit(waiter, seqno);
  896.         intel_ring_emit(waiter, 0);
  897.         intel_ring_emit(waiter, MI_NOOP);
  898.         } else {
  899.                 intel_ring_emit(waiter, MI_NOOP);
  900.                 intel_ring_emit(waiter, MI_NOOP);
  901.                 intel_ring_emit(waiter, MI_NOOP);
  902.                 intel_ring_emit(waiter, MI_NOOP);
  903.         }
  904.         intel_ring_advance(waiter);
  905.  
  906.         return 0;
  907. }
  908.  
  909. #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
  910. do {                                                                    \
  911.         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
  912.                  PIPE_CONTROL_DEPTH_STALL);                             \
  913.         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
  914.         intel_ring_emit(ring__, 0);                                                     \
  915.         intel_ring_emit(ring__, 0);                                                     \
  916. } while (0)
  917.  
  918. static int
  919. pc_render_add_request(struct intel_engine_cs *ring)
  920. {
  921.         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
  922.         int ret;
  923.  
  924.         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
  925.          * incoherent with writes to memory, i.e. completely fubar,
  926.          * so we need to use PIPE_NOTIFY instead.
  927.          *
  928.          * However, we also need to workaround the qword write
  929.          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
  930.          * memory before requesting an interrupt.
  931.          */
  932.         ret = intel_ring_begin(ring, 32);
  933.         if (ret)
  934.                 return ret;
  935.  
  936.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  937.                         PIPE_CONTROL_WRITE_FLUSH |
  938.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
  939.         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  940.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  941.         intel_ring_emit(ring, 0);
  942.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  943.         scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
  944.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  945.         scratch_addr += 2 * CACHELINE_BYTES;
  946.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  947.         scratch_addr += 2 * CACHELINE_BYTES;
  948.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  949.         scratch_addr += 2 * CACHELINE_BYTES;
  950.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  951.         scratch_addr += 2 * CACHELINE_BYTES;
  952.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  953.  
  954.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  955.                         PIPE_CONTROL_WRITE_FLUSH |
  956.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
  957.                         PIPE_CONTROL_NOTIFY);
  958.         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  959.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  960.         intel_ring_emit(ring, 0);
  961.         __intel_ring_advance(ring);
  962.  
  963.         return 0;
  964. }
  965.  
  966. static u32
  967. gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
  968. {
  969.         /* Workaround to force correct ordering between irq and seqno writes on
  970.          * ivb (and maybe also on snb) by reading from a CS register (like
  971.          * ACTHD) before reading the status page. */
  972.         if (!lazy_coherency) {
  973.                 struct drm_i915_private *dev_priv = ring->dev->dev_private;
  974.                 POSTING_READ(RING_ACTHD(ring->mmio_base));
  975.         }
  976.  
  977.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  978. }
  979.  
  980. static u32
  981. ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
  982. {
  983.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  984. }
  985.  
  986. static void
  987. ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
  988. {
  989.         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
  990. }
  991.  
  992. static u32
  993. pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
  994. {
  995.         return ring->scratch.cpu_page[0];
  996. }
  997.  
  998. static void
  999. pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
  1000. {
  1001.         ring->scratch.cpu_page[0] = seqno;
  1002. }
  1003.  
  1004. static bool
  1005. gen5_ring_get_irq(struct intel_engine_cs *ring)
  1006. {
  1007.         struct drm_device *dev = ring->dev;
  1008.         struct drm_i915_private *dev_priv = dev->dev_private;
  1009.         unsigned long flags;
  1010.  
  1011.         if (!dev->irq_enabled)
  1012.                 return false;
  1013.  
  1014.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1015.         if (ring->irq_refcount++ == 0)
  1016.                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  1017.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1018.  
  1019.         return true;
  1020. }
  1021.  
  1022. static void
  1023. gen5_ring_put_irq(struct intel_engine_cs *ring)
  1024. {
  1025.         struct drm_device *dev = ring->dev;
  1026.         struct drm_i915_private *dev_priv = dev->dev_private;
  1027.         unsigned long flags;
  1028.  
  1029.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1030.         if (--ring->irq_refcount == 0)
  1031.                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  1032.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1033. }
  1034.  
  1035. static bool
  1036. i9xx_ring_get_irq(struct intel_engine_cs *ring)
  1037. {
  1038.         struct drm_device *dev = ring->dev;
  1039.         struct drm_i915_private *dev_priv = dev->dev_private;
  1040.         unsigned long flags;
  1041.  
  1042.         if (!dev->irq_enabled)
  1043.                 return false;
  1044.  
  1045.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1046.         if (ring->irq_refcount++ == 0) {
  1047.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  1048.         I915_WRITE(IMR, dev_priv->irq_mask);
  1049.         POSTING_READ(IMR);
  1050.         }
  1051.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1052.  
  1053.         return true;
  1054. }
  1055.  
  1056. static void
  1057. i9xx_ring_put_irq(struct intel_engine_cs *ring)
  1058. {
  1059.         struct drm_device *dev = ring->dev;
  1060.         struct drm_i915_private *dev_priv = dev->dev_private;
  1061.         unsigned long flags;
  1062.  
  1063.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1064.         if (--ring->irq_refcount == 0) {
  1065.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  1066.         I915_WRITE(IMR, dev_priv->irq_mask);
  1067.         POSTING_READ(IMR);
  1068.         }
  1069.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1070. }
  1071.  
  1072. static bool
  1073. i8xx_ring_get_irq(struct intel_engine_cs *ring)
  1074. {
  1075.         struct drm_device *dev = ring->dev;
  1076.         struct drm_i915_private *dev_priv = dev->dev_private;
  1077.         unsigned long flags;
  1078.  
  1079.         if (!dev->irq_enabled)
  1080.                 return false;
  1081.  
  1082.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1083.         if (ring->irq_refcount++ == 0) {
  1084.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  1085.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  1086.                 POSTING_READ16(IMR);
  1087.         }
  1088.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1089.  
  1090.         return true;
  1091. }
  1092.  
  1093. static void
  1094. i8xx_ring_put_irq(struct intel_engine_cs *ring)
  1095. {
  1096.         struct drm_device *dev = ring->dev;
  1097.         struct drm_i915_private *dev_priv = dev->dev_private;
  1098.         unsigned long flags;
  1099.  
  1100.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1101.         if (--ring->irq_refcount == 0) {
  1102.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  1103.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  1104.                 POSTING_READ16(IMR);
  1105.         }
  1106.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1107. }
  1108.  
  1109. void intel_ring_setup_status_page(struct intel_engine_cs *ring)
  1110. {
  1111.         struct drm_device *dev = ring->dev;
  1112.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1113.         u32 mmio = 0;
  1114.  
  1115.         /* The ring status page addresses are no longer next to the rest of
  1116.          * the ring registers as of gen7.
  1117.          */
  1118.         if (IS_GEN7(dev)) {
  1119.                 switch (ring->id) {
  1120.                 case RCS:
  1121.                         mmio = RENDER_HWS_PGA_GEN7;
  1122.                         break;
  1123.                 case BCS:
  1124.                         mmio = BLT_HWS_PGA_GEN7;
  1125.                         break;
  1126.                 /*
  1127.                  * VCS2 actually doesn't exist on Gen7. Only shut up
  1128.                  * gcc switch check warning
  1129.                  */
  1130.                 case VCS2:
  1131.                 case VCS:
  1132.                         mmio = BSD_HWS_PGA_GEN7;
  1133.                         break;
  1134.                 case VECS:
  1135.                         mmio = VEBOX_HWS_PGA_GEN7;
  1136.                         break;
  1137.                 }
  1138.         } else if (IS_GEN6(ring->dev)) {
  1139.                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
  1140.         } else {
  1141.                 /* XXX: gen8 returns to sanity */
  1142.                 mmio = RING_HWS_PGA(ring->mmio_base);
  1143.         }
  1144.  
  1145.         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
  1146.         POSTING_READ(mmio);
  1147.  
  1148.         /*
  1149.          * Flush the TLB for this page
  1150.          *
  1151.          * FIXME: These two bits have disappeared on gen8, so a question
  1152.          * arises: do we still need this and if so how should we go about
  1153.          * invalidating the TLB?
  1154.          */
  1155.         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
  1156.                 u32 reg = RING_INSTPM(ring->mmio_base);
  1157.  
  1158.                 /* ring should be idle before issuing a sync flush*/
  1159.                 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
  1160.  
  1161.                 I915_WRITE(reg,
  1162.                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
  1163.                                               INSTPM_SYNC_FLUSH));
  1164.                 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
  1165.                              1000))
  1166.                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
  1167.                                   ring->name);
  1168.         }
  1169. }
  1170.  
  1171. static int
  1172. bsd_ring_flush(struct intel_engine_cs *ring,
  1173.                u32     invalidate_domains,
  1174.                u32     flush_domains)
  1175. {
  1176.         int ret;
  1177.  
  1178.         ret = intel_ring_begin(ring, 2);
  1179.         if (ret)
  1180.                 return ret;
  1181.  
  1182.         intel_ring_emit(ring, MI_FLUSH);
  1183.         intel_ring_emit(ring, MI_NOOP);
  1184.         intel_ring_advance(ring);
  1185.         return 0;
  1186. }
  1187.  
  1188. static int
  1189. i9xx_add_request(struct intel_engine_cs *ring)
  1190. {
  1191.         int ret;
  1192.  
  1193.         ret = intel_ring_begin(ring, 4);
  1194.         if (ret)
  1195.                 return ret;
  1196.  
  1197.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  1198.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  1199.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  1200.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  1201.         __intel_ring_advance(ring);
  1202.  
  1203.         return 0;
  1204. }
  1205.  
  1206. static bool
  1207. gen6_ring_get_irq(struct intel_engine_cs *ring)
  1208. {
  1209.         struct drm_device *dev = ring->dev;
  1210.         struct drm_i915_private *dev_priv = dev->dev_private;
  1211.         unsigned long flags;
  1212.  
  1213.         if (!dev->irq_enabled)
  1214.                return false;
  1215.  
  1216.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1217.         if (ring->irq_refcount++ == 0) {
  1218.                 if (HAS_L3_DPF(dev) && ring->id == RCS)
  1219.                         I915_WRITE_IMR(ring,
  1220.                                        ~(ring->irq_enable_mask |
  1221.                                          GT_PARITY_ERROR(dev)));
  1222.                 else
  1223.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1224.                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  1225.         }
  1226.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1227.  
  1228.     return true;
  1229. }
  1230.  
  1231. static void
  1232. gen6_ring_put_irq(struct intel_engine_cs *ring)
  1233. {
  1234.         struct drm_device *dev = ring->dev;
  1235.         struct drm_i915_private *dev_priv = dev->dev_private;
  1236.         unsigned long flags;
  1237.  
  1238.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1239.         if (--ring->irq_refcount == 0) {
  1240.                 if (HAS_L3_DPF(dev) && ring->id == RCS)
  1241.                         I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
  1242.                 else
  1243.                         I915_WRITE_IMR(ring, ~0);
  1244.                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  1245.         }
  1246.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1247. }
  1248.  
  1249. static bool
  1250. hsw_vebox_get_irq(struct intel_engine_cs *ring)
  1251. {
  1252.         struct drm_device *dev = ring->dev;
  1253.         struct drm_i915_private *dev_priv = dev->dev_private;
  1254.         unsigned long flags;
  1255.  
  1256.         if (!dev->irq_enabled)
  1257.                 return false;
  1258.  
  1259.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1260.         if (ring->irq_refcount++ == 0) {
  1261.                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1262.                 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
  1263.         }
  1264.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1265.  
  1266.         return true;
  1267. }
  1268.  
  1269. static void
  1270. hsw_vebox_put_irq(struct intel_engine_cs *ring)
  1271. {
  1272.         struct drm_device *dev = ring->dev;
  1273.         struct drm_i915_private *dev_priv = dev->dev_private;
  1274.         unsigned long flags;
  1275.  
  1276.         if (!dev->irq_enabled)
  1277.                 return;
  1278.  
  1279.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1280.         if (--ring->irq_refcount == 0) {
  1281.                 I915_WRITE_IMR(ring, ~0);
  1282.                 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
  1283.         }
  1284.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1285. }
  1286.  
  1287. static bool
  1288. gen8_ring_get_irq(struct intel_engine_cs *ring)
  1289. {
  1290.         struct drm_device *dev = ring->dev;
  1291.         struct drm_i915_private *dev_priv = dev->dev_private;
  1292.         unsigned long flags;
  1293.  
  1294.         if (!dev->irq_enabled)
  1295.                 return false;
  1296.  
  1297.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1298.         if (ring->irq_refcount++ == 0) {
  1299.                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
  1300.                         I915_WRITE_IMR(ring,
  1301.                                        ~(ring->irq_enable_mask |
  1302.                                          GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
  1303.                 } else {
  1304.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1305.                 }
  1306.                 POSTING_READ(RING_IMR(ring->mmio_base));
  1307.         }
  1308.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1309.  
  1310.         return true;
  1311. }
  1312.  
  1313. static void
  1314. gen8_ring_put_irq(struct intel_engine_cs *ring)
  1315. {
  1316.         struct drm_device *dev = ring->dev;
  1317.         struct drm_i915_private *dev_priv = dev->dev_private;
  1318.         unsigned long flags;
  1319.  
  1320.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1321.         if (--ring->irq_refcount == 0) {
  1322.                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
  1323.                         I915_WRITE_IMR(ring,
  1324.                                        ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
  1325.                 } else {
  1326.                         I915_WRITE_IMR(ring, ~0);
  1327.                 }
  1328.                 POSTING_READ(RING_IMR(ring->mmio_base));
  1329.         }
  1330.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1331. }
  1332.  
  1333. static int
  1334. i965_dispatch_execbuffer(struct intel_engine_cs *ring,
  1335.                          u64 offset, u32 length,
  1336.                          unsigned flags)
  1337. {
  1338.         int ret;
  1339.  
  1340.         ret = intel_ring_begin(ring, 2);
  1341.         if (ret)
  1342.                 return ret;
  1343.  
  1344.         intel_ring_emit(ring,
  1345.                         MI_BATCH_BUFFER_START |
  1346.                         MI_BATCH_GTT |
  1347.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  1348.         intel_ring_emit(ring, offset);
  1349.         intel_ring_advance(ring);
  1350.  
  1351.         return 0;
  1352. }
  1353.  
  1354. /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
  1355. #define I830_BATCH_LIMIT (256*1024)
  1356. static int
  1357. i830_dispatch_execbuffer(struct intel_engine_cs *ring,
  1358.                                 u64 offset, u32 len,
  1359.                                 unsigned flags)
  1360. {
  1361.         int ret;
  1362.  
  1363.         if (flags & I915_DISPATCH_PINNED) {
  1364.                 ret = intel_ring_begin(ring, 4);
  1365.                 if (ret)
  1366.                         return ret;
  1367.  
  1368.                 intel_ring_emit(ring, MI_BATCH_BUFFER);
  1369.                 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1370.                 intel_ring_emit(ring, offset + len - 8);
  1371.                 intel_ring_emit(ring, MI_NOOP);
  1372.                 intel_ring_advance(ring);
  1373.         } else {
  1374.                 u32 cs_offset = ring->scratch.gtt_offset;
  1375.  
  1376.                 if (len > I830_BATCH_LIMIT)
  1377.                         return -ENOSPC;
  1378.  
  1379.                 ret = intel_ring_begin(ring, 9+3);
  1380.                 if (ret)
  1381.                         return ret;
  1382.                 /* Blit the batch (which has now all relocs applied) to the stable batch
  1383.                  * scratch bo area (so that the CS never stumbles over its tlb
  1384.                  * invalidation bug) ... */
  1385.                 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
  1386.                                 XY_SRC_COPY_BLT_WRITE_ALPHA |
  1387.                                 XY_SRC_COPY_BLT_WRITE_RGB);
  1388.                 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
  1389.                 intel_ring_emit(ring, 0);
  1390.                 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
  1391.                 intel_ring_emit(ring, cs_offset);
  1392.                 intel_ring_emit(ring, 0);
  1393.                 intel_ring_emit(ring, 4096);
  1394.                 intel_ring_emit(ring, offset);
  1395.                 intel_ring_emit(ring, MI_FLUSH);
  1396.  
  1397.                 /* ... and execute it. */
  1398.                 intel_ring_emit(ring, MI_BATCH_BUFFER);
  1399.                 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1400.                 intel_ring_emit(ring, cs_offset + len - 8);
  1401.         intel_ring_advance(ring);
  1402.         }
  1403.  
  1404.         return 0;
  1405. }
  1406.  
  1407. static int
  1408. i915_dispatch_execbuffer(struct intel_engine_cs *ring,
  1409.                          u64 offset, u32 len,
  1410.                          unsigned flags)
  1411. {
  1412.         int ret;
  1413.  
  1414.                 ret = intel_ring_begin(ring, 2);
  1415.                 if (ret)
  1416.                         return ret;
  1417.  
  1418.         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
  1419.         intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1420.         intel_ring_advance(ring);
  1421.  
  1422.         return 0;
  1423. }
  1424.  
  1425. static void cleanup_status_page(struct intel_engine_cs *ring)
  1426. {
  1427.         struct drm_i915_gem_object *obj;
  1428.  
  1429.         obj = ring->status_page.obj;
  1430.         if (obj == NULL)
  1431.                 return;
  1432.  
  1433. //      kunmap(sg_page(obj->pages->sgl));
  1434.         i915_gem_object_ggtt_unpin(obj);
  1435.         drm_gem_object_unreference(&obj->base);
  1436.         ring->status_page.obj = NULL;
  1437. }
  1438.  
  1439. static int init_status_page(struct intel_engine_cs *ring)
  1440. {
  1441.         struct drm_i915_gem_object *obj;
  1442.  
  1443.         if ((obj = ring->status_page.obj) == NULL) {
  1444.                 unsigned flags;
  1445.         int ret;
  1446.  
  1447.                 obj = i915_gem_alloc_object(ring->dev, 4096);
  1448.         if (obj == NULL) {
  1449.                 DRM_ERROR("Failed to allocate status page\n");
  1450.                         return -ENOMEM;
  1451.         }
  1452.  
  1453.         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  1454.         if (ret)
  1455.                 goto err_unref;
  1456.  
  1457.                 flags = 0;
  1458.                 if (!HAS_LLC(ring->dev))
  1459.                         /* On g33, we cannot place HWS above 256MiB, so
  1460.                          * restrict its pinning to the low mappable arena.
  1461.                          * Though this restriction is not documented for
  1462.                          * gen4, gen5, or byt, they also behave similarly
  1463.                          * and hang if the HWS is placed at the top of the
  1464.                          * GTT. To generalise, it appears that all !llc
  1465.                          * platforms have issues with us placing the HWS
  1466.                          * above the mappable region (even though we never
  1467.                          * actualy map it).
  1468.                          */
  1469.                         flags |= PIN_MAPPABLE;
  1470.                 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
  1471.                 if (ret) {
  1472. err_unref:
  1473.                         drm_gem_object_unreference(&obj->base);
  1474.                         return ret;
  1475.                 }
  1476.  
  1477.                 ring->status_page.obj = obj;
  1478.         }
  1479.  
  1480.         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
  1481.     ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW|0x100);
  1482.         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1483.  
  1484.         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  1485.                         ring->name, ring->status_page.gfx_addr);
  1486.  
  1487.         return 0;
  1488. }
  1489.  
  1490. static int init_phys_status_page(struct intel_engine_cs *ring)
  1491. {
  1492.     struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1493.  
  1494.     if (!dev_priv->status_page_dmah) {
  1495.         dev_priv->status_page_dmah =
  1496.             drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
  1497.         if (!dev_priv->status_page_dmah)
  1498.             return -ENOMEM;
  1499.     }
  1500.  
  1501.     ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
  1502.     memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1503.  
  1504.     return 0;
  1505. }
  1506.  
  1507. static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
  1508. {
  1509.         if (!ringbuf->obj)
  1510.                 return;
  1511.  
  1512.         iounmap(ringbuf->virtual_start);
  1513.         i915_gem_object_ggtt_unpin(ringbuf->obj);
  1514.         drm_gem_object_unreference(&ringbuf->obj->base);
  1515.         ringbuf->obj = NULL;
  1516. }
  1517.  
  1518. static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
  1519.                                       struct intel_ringbuffer *ringbuf)
  1520. {
  1521.         struct drm_i915_private *dev_priv = to_i915(dev);
  1522.         struct drm_i915_gem_object *obj;
  1523.         int ret;
  1524.  
  1525.         if (ringbuf->obj)
  1526.                 return 0;
  1527.  
  1528.         obj = NULL;
  1529.         if (!HAS_LLC(dev))
  1530.                 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
  1531.         if (obj == NULL)
  1532.                 obj = i915_gem_alloc_object(dev, ringbuf->size);
  1533.         if (obj == NULL)
  1534.                 return -ENOMEM;
  1535.  
  1536.         /* mark ring buffers as read-only from GPU side by default */
  1537.         obj->gt_ro = 1;
  1538.  
  1539.         ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
  1540.         if (ret)
  1541.                 goto err_unref;
  1542.  
  1543.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  1544.         if (ret)
  1545.                 goto err_unpin;
  1546.  
  1547.         ringbuf->virtual_start =
  1548.                 ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
  1549.                                 ringbuf->size);
  1550.         if (ringbuf->virtual_start == NULL) {
  1551.                 ret = -EINVAL;
  1552.                 goto err_unpin;
  1553.         }
  1554.  
  1555.         ringbuf->obj = obj;
  1556.         return 0;
  1557.  
  1558. err_unpin:
  1559.         i915_gem_object_ggtt_unpin(obj);
  1560. err_unref:
  1561.         drm_gem_object_unreference(&obj->base);
  1562.         return ret;
  1563. }
  1564.  
  1565. static int intel_init_ring_buffer(struct drm_device *dev,
  1566.                                   struct intel_engine_cs *ring)
  1567. {
  1568.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1569.         int ret;
  1570.  
  1571.         if (ringbuf == NULL) {
  1572.                 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
  1573.                 if (!ringbuf)
  1574.                         return -ENOMEM;
  1575.                 ring->buffer = ringbuf;
  1576.         }
  1577.  
  1578.         ring->dev = dev;
  1579.         INIT_LIST_HEAD(&ring->active_list);
  1580.         INIT_LIST_HEAD(&ring->request_list);
  1581.         ringbuf->size = 32 * PAGE_SIZE;
  1582.         memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
  1583.  
  1584.         init_waitqueue_head(&ring->irq_queue);
  1585.  
  1586.         if (I915_NEED_GFX_HWS(dev)) {
  1587.                 ret = init_status_page(ring);
  1588.                 if (ret)
  1589.                         goto error;
  1590.         } else {
  1591.                 BUG_ON(ring->id != RCS);
  1592.                 ret = init_phys_status_page(ring);
  1593.         if (ret)
  1594.                         goto error;
  1595.         }
  1596.  
  1597.         ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
  1598.         if (ret) {
  1599.                 DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret);
  1600.                 goto error;
  1601.         }
  1602.  
  1603.         /* Workaround an erratum on the i830 which causes a hang if
  1604.          * the TAIL pointer points to within the last 2 cachelines
  1605.          * of the buffer.
  1606.          */
  1607.         ringbuf->effective_size = ringbuf->size;
  1608.         if (IS_I830(dev) || IS_845G(dev))
  1609.                 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
  1610.  
  1611.         ret = i915_cmd_parser_init_ring(ring);
  1612.         if (ret)
  1613.                 goto error;
  1614.  
  1615.         ret = ring->init(ring);
  1616.         if (ret)
  1617.                 goto error;
  1618.  
  1619.         return 0;
  1620.  
  1621. error:
  1622.         kfree(ringbuf);
  1623.         ring->buffer = NULL;
  1624.         return ret;
  1625. }
  1626.  
  1627. void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
  1628. {
  1629.         struct drm_i915_private *dev_priv = to_i915(ring->dev);
  1630.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1631.  
  1632.         if (!intel_ring_initialized(ring))
  1633.                 return;
  1634.  
  1635.         intel_stop_ring_buffer(ring);
  1636.         WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
  1637.  
  1638.         intel_destroy_ringbuffer_obj(ringbuf);
  1639.         ring->preallocated_lazy_request = NULL;
  1640.         ring->outstanding_lazy_seqno = 0;
  1641.  
  1642.         if (ring->cleanup)
  1643.                 ring->cleanup(ring);
  1644.  
  1645. //      cleanup_status_page(ring);
  1646.  
  1647.         i915_cmd_parser_fini_ring(ring);
  1648.  
  1649.         kfree(ringbuf);
  1650.         ring->buffer = NULL;
  1651. }
  1652.  
  1653. static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
  1654. {
  1655.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1656.         struct drm_i915_gem_request *request;
  1657.         u32 seqno = 0;
  1658.         int ret;
  1659.  
  1660.         if (ringbuf->last_retired_head != -1) {
  1661.                 ringbuf->head = ringbuf->last_retired_head;
  1662.                 ringbuf->last_retired_head = -1;
  1663.  
  1664.                 ringbuf->space = ring_space(ringbuf);
  1665.                 if (ringbuf->space >= n)
  1666.                         return 0;
  1667.         }
  1668.  
  1669.         list_for_each_entry(request, &ring->request_list, list) {
  1670.                 if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) {
  1671.                         seqno = request->seqno;
  1672.                         break;
  1673.                 }
  1674.         }
  1675.  
  1676.         if (seqno == 0)
  1677.                 return -ENOSPC;
  1678.  
  1679.         ret = i915_wait_seqno(ring, seqno);
  1680.         if (ret)
  1681.                 return ret;
  1682.  
  1683.         i915_gem_retire_requests_ring(ring);
  1684.         ringbuf->head = ringbuf->last_retired_head;
  1685.         ringbuf->last_retired_head = -1;
  1686.  
  1687.         ringbuf->space = ring_space(ringbuf);
  1688.         return 0;
  1689. }
  1690.  
  1691. static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
  1692. {
  1693.         struct drm_device *dev = ring->dev;
  1694.         struct drm_i915_private *dev_priv = dev->dev_private;
  1695.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1696.         unsigned long end;
  1697.         int ret;
  1698.  
  1699.         ret = intel_ring_wait_request(ring, n);
  1700.         if (ret != -ENOSPC)
  1701.                 return ret;
  1702.  
  1703.         /* force the tail write in case we have been skipping them */
  1704.         __intel_ring_advance(ring);
  1705.  
  1706.         /* With GEM the hangcheck timer should kick us out of the loop,
  1707.          * leaving it early runs the risk of corrupting GEM state (due
  1708.          * to running on almost untested codepaths). But on resume
  1709.          * timers don't work yet, so prevent a complete hang in that
  1710.          * case by choosing an insanely large timeout. */
  1711.         end = jiffies + 60 * HZ;
  1712.  
  1713.         trace_i915_ring_wait_begin(ring);
  1714.         do {
  1715.                 ringbuf->head = I915_READ_HEAD(ring);
  1716.                 ringbuf->space = ring_space(ringbuf);
  1717.                 if (ringbuf->space >= n) {
  1718.                         ret = 0;
  1719.                         break;
  1720.                 }
  1721.  
  1722.  
  1723.                 msleep(1);
  1724.  
  1725.                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1726.                                            dev_priv->mm.interruptible);
  1727.                 if (ret)
  1728.                         break;
  1729.  
  1730.                 if (time_after(jiffies, end)) {
  1731.                         ret = -EBUSY;
  1732.                         break;
  1733.                 }
  1734.         } while (1);
  1735.         trace_i915_ring_wait_end(ring);
  1736.                         return ret;
  1737. }
  1738.  
  1739. static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
  1740. {
  1741.         uint32_t __iomem *virt;
  1742.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1743.         int rem = ringbuf->size - ringbuf->tail;
  1744.  
  1745.         if (ringbuf->space < rem) {
  1746.                 int ret = ring_wait_for_space(ring, rem);
  1747.                 if (ret)
  1748.                         return ret;
  1749.         }
  1750.  
  1751.         virt = ringbuf->virtual_start + ringbuf->tail;
  1752.         rem /= 4;
  1753.         while (rem--)
  1754.                 iowrite32(MI_NOOP, virt++);
  1755.  
  1756.         ringbuf->tail = 0;
  1757.         ringbuf->space = ring_space(ringbuf);
  1758.  
  1759.         return 0;
  1760. }
  1761.  
  1762. int intel_ring_idle(struct intel_engine_cs *ring)
  1763. {
  1764.         u32 seqno;
  1765.         int ret;
  1766.  
  1767.         /* We need to add any requests required to flush the objects and ring */
  1768.         if (ring->outstanding_lazy_seqno) {
  1769.                 ret = i915_add_request(ring, NULL);
  1770.                 if (ret)
  1771.                         return ret;
  1772.         }
  1773.  
  1774.         /* Wait upon the last request to be completed */
  1775.         if (list_empty(&ring->request_list))
  1776.                 return 0;
  1777.  
  1778.         seqno = list_entry(ring->request_list.prev,
  1779.                            struct drm_i915_gem_request,
  1780.                            list)->seqno;
  1781.  
  1782.         return i915_wait_seqno(ring, seqno);
  1783. }
  1784.  
  1785. static int
  1786. intel_ring_alloc_seqno(struct intel_engine_cs *ring)
  1787. {
  1788.         if (ring->outstanding_lazy_seqno)
  1789.                 return 0;
  1790.  
  1791.         if (ring->preallocated_lazy_request == NULL) {
  1792.                 struct drm_i915_gem_request *request;
  1793.  
  1794.                 request = kmalloc(sizeof(*request), GFP_KERNEL);
  1795.                 if (request == NULL)
  1796.                         return -ENOMEM;
  1797.  
  1798.                 ring->preallocated_lazy_request = request;
  1799.         }
  1800.  
  1801.         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
  1802. }
  1803.  
  1804. static int __intel_ring_prepare(struct intel_engine_cs *ring,
  1805.                               int bytes)
  1806. {
  1807.         struct intel_ringbuffer *ringbuf = ring->buffer;
  1808.         int ret;
  1809.  
  1810.         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
  1811.                 ret = intel_wrap_ring_buffer(ring);
  1812.                 if (unlikely(ret))
  1813.                         return ret;
  1814.         }
  1815.  
  1816.         if (unlikely(ringbuf->space < bytes)) {
  1817.                 ret = ring_wait_for_space(ring, bytes);
  1818.                 if (unlikely(ret))
  1819.                         return ret;
  1820.         }
  1821.  
  1822.         return 0;
  1823. }
  1824.  
  1825. int intel_ring_begin(struct intel_engine_cs *ring,
  1826.                      int num_dwords)
  1827. {
  1828.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1829.         int ret;
  1830.  
  1831.         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1832.                                    dev_priv->mm.interruptible);
  1833.         if (ret)
  1834.                 return ret;
  1835.  
  1836.         ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
  1837.         if (ret)
  1838.                 return ret;
  1839.  
  1840.         /* Preallocate the olr before touching the ring */
  1841.         ret = intel_ring_alloc_seqno(ring);
  1842.         if (ret)
  1843.                 return ret;
  1844.  
  1845.         ring->buffer->space -= num_dwords * sizeof(uint32_t);
  1846.         return 0;
  1847. }
  1848.  
  1849. /* Align the ring tail to a cacheline boundary */
  1850. int intel_ring_cacheline_align(struct intel_engine_cs *ring)
  1851. {
  1852.         int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
  1853.         int ret;
  1854.  
  1855.         if (num_dwords == 0)
  1856.                 return 0;
  1857.  
  1858.         num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
  1859.         ret = intel_ring_begin(ring, num_dwords);
  1860.         if (ret)
  1861.                 return ret;
  1862.  
  1863.         while (num_dwords--)
  1864.                 intel_ring_emit(ring, MI_NOOP);
  1865.  
  1866.         intel_ring_advance(ring);
  1867.  
  1868.         return 0;
  1869. }
  1870.  
  1871. void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
  1872. {
  1873.         struct drm_device *dev = ring->dev;
  1874.         struct drm_i915_private *dev_priv = dev->dev_private;
  1875.  
  1876.         BUG_ON(ring->outstanding_lazy_seqno);
  1877.  
  1878.         if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
  1879.                 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
  1880.                 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
  1881.                 if (HAS_VEBOX(dev))
  1882.                         I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
  1883.         }
  1884.  
  1885.         ring->set_seqno(ring, seqno);
  1886.         ring->hangcheck.seqno = seqno;
  1887. }
  1888.  
  1889. static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
  1890.                                      u32 value)
  1891. {
  1892.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1893.  
  1894.        /* Every tail move must follow the sequence below */
  1895.  
  1896.         /* Disable notification that the ring is IDLE. The GT
  1897.          * will then assume that it is busy and bring it out of rc6.
  1898.          */
  1899.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1900.                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1901.  
  1902.         /* Clear the context id. Here be magic! */
  1903.         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
  1904.  
  1905.         /* Wait for the ring not to be idle, i.e. for it to wake up. */
  1906.        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
  1907.                       GEN6_BSD_SLEEP_INDICATOR) == 0,
  1908.                        50))
  1909.                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
  1910.  
  1911.         /* Now that the ring is fully powered up, update the tail */
  1912.        I915_WRITE_TAIL(ring, value);
  1913.         POSTING_READ(RING_TAIL(ring->mmio_base));
  1914.  
  1915.         /* Let the ring send IDLE messages to the GT again,
  1916.          * and so let it sleep to conserve power when idle.
  1917.          */
  1918.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1919.                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1920. }
  1921.  
  1922. static int gen6_bsd_ring_flush(struct intel_engine_cs *ring,
  1923.                            u32 invalidate, u32 flush)
  1924. {
  1925.         uint32_t cmd;
  1926.         int ret;
  1927.  
  1928.         ret = intel_ring_begin(ring, 4);
  1929.         if (ret)
  1930.                 return ret;
  1931.  
  1932.         cmd = MI_FLUSH_DW;
  1933.         if (INTEL_INFO(ring->dev)->gen >= 8)
  1934.                 cmd += 1;
  1935.         /*
  1936.          * Bspec vol 1c.5 - video engine command streamer:
  1937.          * "If ENABLED, all TLBs will be invalidated once the flush
  1938.          * operation is complete. This bit is only valid when the
  1939.          * Post-Sync Operation field is a value of 1h or 3h."
  1940.          */
  1941.         if (invalidate & I915_GEM_GPU_DOMAINS)
  1942.                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
  1943.                         MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
  1944.         intel_ring_emit(ring, cmd);
  1945.         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  1946.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  1947.                 intel_ring_emit(ring, 0); /* upper addr */
  1948.                 intel_ring_emit(ring, 0); /* value */
  1949.         } else  {
  1950.         intel_ring_emit(ring, 0);
  1951.         intel_ring_emit(ring, MI_NOOP);
  1952.         }
  1953.         intel_ring_advance(ring);
  1954.         return 0;
  1955. }
  1956.  
  1957. static int
  1958. gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
  1959.                               u64 offset, u32 len,
  1960.                               unsigned flags)
  1961. {
  1962.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1963.         bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
  1964.                 !(flags & I915_DISPATCH_SECURE);
  1965.         int ret;
  1966.  
  1967.         ret = intel_ring_begin(ring, 4);
  1968.         if (ret)
  1969.                 return ret;
  1970.  
  1971.         /* FIXME(BDW): Address space and security selectors. */
  1972.         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
  1973.         intel_ring_emit(ring, lower_32_bits(offset));
  1974.         intel_ring_emit(ring, upper_32_bits(offset));
  1975.         intel_ring_emit(ring, MI_NOOP);
  1976.         intel_ring_advance(ring);
  1977.  
  1978.         return 0;
  1979. }
  1980.  
  1981. static int
  1982. hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
  1983.                               u64 offset, u32 len,
  1984.                               unsigned flags)
  1985. {
  1986.         int ret;
  1987.  
  1988.         ret = intel_ring_begin(ring, 2);
  1989.         if (ret)
  1990.                 return ret;
  1991.  
  1992.         intel_ring_emit(ring,
  1993.                         MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
  1994.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
  1995.         /* bit0-7 is the length on GEN6+ */
  1996.         intel_ring_emit(ring, offset);
  1997.         intel_ring_advance(ring);
  1998.  
  1999.         return 0;
  2000. }
  2001.  
  2002. static int
  2003. gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
  2004.                               u64 offset, u32 len,
  2005.                               unsigned flags)
  2006. {
  2007.        int ret;
  2008.  
  2009.        ret = intel_ring_begin(ring, 2);
  2010.        if (ret)
  2011.                return ret;
  2012.  
  2013.         intel_ring_emit(ring,
  2014.                         MI_BATCH_BUFFER_START |
  2015.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  2016.        /* bit0-7 is the length on GEN6+ */
  2017.        intel_ring_emit(ring, offset);
  2018.        intel_ring_advance(ring);
  2019.  
  2020.        return 0;
  2021. }
  2022.  
  2023. /* Blitter support (SandyBridge+) */
  2024.  
  2025. static int gen6_ring_flush(struct intel_engine_cs *ring,
  2026.                           u32 invalidate, u32 flush)
  2027. {
  2028.         struct drm_device *dev = ring->dev;
  2029.         uint32_t cmd;
  2030.         int ret;
  2031.  
  2032.         ret = intel_ring_begin(ring, 4);
  2033.         if (ret)
  2034.                 return ret;
  2035.  
  2036.         cmd = MI_FLUSH_DW;
  2037.         if (INTEL_INFO(ring->dev)->gen >= 8)
  2038.                 cmd += 1;
  2039.         /*
  2040.          * Bspec vol 1c.3 - blitter engine command streamer:
  2041.          * "If ENABLED, all TLBs will be invalidated once the flush
  2042.          * operation is complete. This bit is only valid when the
  2043.          * Post-Sync Operation field is a value of 1h or 3h."
  2044.          */
  2045.         if (invalidate & I915_GEM_DOMAIN_RENDER)
  2046.                 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
  2047.                         MI_FLUSH_DW_OP_STOREDW;
  2048.         intel_ring_emit(ring, cmd);
  2049.         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  2050.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  2051.                 intel_ring_emit(ring, 0); /* upper addr */
  2052.                 intel_ring_emit(ring, 0); /* value */
  2053.         } else  {
  2054.         intel_ring_emit(ring, 0);
  2055.         intel_ring_emit(ring, MI_NOOP);
  2056.         }
  2057.         intel_ring_advance(ring);
  2058.  
  2059.         if (IS_GEN7(dev) && !invalidate && flush)
  2060.                 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
  2061.  
  2062.         return 0;
  2063. }
  2064.  
  2065. int intel_init_render_ring_buffer(struct drm_device *dev)
  2066. {
  2067.         struct drm_i915_private *dev_priv = dev->dev_private;
  2068.         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
  2069.         struct drm_i915_gem_object *obj;
  2070.         int ret;
  2071.  
  2072.         ring->name = "render ring";
  2073.         ring->id = RCS;
  2074.         ring->mmio_base = RENDER_RING_BASE;
  2075.  
  2076.         if (INTEL_INFO(dev)->gen >= 8) {
  2077.                 if (i915_semaphore_is_enabled(dev)) {
  2078.                         obj = i915_gem_alloc_object(dev, 4096);
  2079.                         if (obj == NULL) {
  2080.                                 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
  2081.                                 i915.semaphores = 0;
  2082.                         } else {
  2083.                                 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  2084.                                 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
  2085.                                 if (ret != 0) {
  2086.                                         drm_gem_object_unreference(&obj->base);
  2087.                                         DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
  2088.                                         i915.semaphores = 0;
  2089.                                 } else
  2090.                                         dev_priv->semaphore_obj = obj;
  2091.                         }
  2092.                 }
  2093.                 ring->add_request = gen6_add_request;
  2094.                 ring->flush = gen8_render_ring_flush;
  2095.                 ring->irq_get = gen8_ring_get_irq;
  2096.                 ring->irq_put = gen8_ring_put_irq;
  2097.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
  2098.                 ring->get_seqno = gen6_ring_get_seqno;
  2099.                 ring->set_seqno = ring_set_seqno;
  2100.                 if (i915_semaphore_is_enabled(dev)) {
  2101.                         WARN_ON(!dev_priv->semaphore_obj);
  2102.                         ring->semaphore.sync_to = gen8_ring_sync;
  2103.                         ring->semaphore.signal = gen8_rcs_signal;
  2104.                         GEN8_RING_SEMAPHORE_INIT;
  2105.                 }
  2106.         } else if (INTEL_INFO(dev)->gen >= 6) {
  2107.        ring->add_request = gen6_add_request;
  2108.                 ring->flush = gen7_render_ring_flush;
  2109.                 if (INTEL_INFO(dev)->gen == 6)
  2110.                 ring->flush = gen6_render_ring_flush;
  2111.                 ring->irq_get = gen6_ring_get_irq;
  2112.                 ring->irq_put = gen6_ring_put_irq;
  2113.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
  2114.                 ring->get_seqno = gen6_ring_get_seqno;
  2115.                 ring->set_seqno = ring_set_seqno;
  2116.                 if (i915_semaphore_is_enabled(dev)) {
  2117.                 ring->semaphore.sync_to = gen6_ring_sync;
  2118.                 ring->semaphore.signal = gen6_signal;
  2119.                 /*
  2120.                          * The current semaphore is only applied on pre-gen8
  2121.                          * platform.  And there is no VCS2 ring on the pre-gen8
  2122.                          * platform. So the semaphore between RCS and VCS2 is
  2123.                          * initialized as INVALID.  Gen8 will initialize the
  2124.                          * sema between VCS2 and RCS later.
  2125.                  */
  2126.                 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
  2127.                 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
  2128.                 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
  2129.                 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
  2130.                 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2131.                 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
  2132.                 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
  2133.                 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
  2134.                 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
  2135.                 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2136.                 }
  2137.         } else if (IS_GEN5(dev)) {
  2138.        ring->add_request = pc_render_add_request;
  2139.                 ring->flush = gen4_render_ring_flush;
  2140.                 ring->get_seqno = pc_render_get_seqno;
  2141.                 ring->set_seqno = pc_render_set_seqno;
  2142.                 ring->irq_get = gen5_ring_get_irq;
  2143.                 ring->irq_put = gen5_ring_put_irq;
  2144.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
  2145.                                         GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
  2146.         } else {
  2147.                 ring->add_request = i9xx_add_request;
  2148.                 if (INTEL_INFO(dev)->gen < 4)
  2149.                         ring->flush = gen2_render_ring_flush;
  2150.                 else
  2151.                         ring->flush = gen4_render_ring_flush;
  2152.                 ring->get_seqno = ring_get_seqno;
  2153.                 ring->set_seqno = ring_set_seqno;
  2154.                 if (IS_GEN2(dev)) {
  2155.                         ring->irq_get = i8xx_ring_get_irq;
  2156.                         ring->irq_put = i8xx_ring_put_irq;
  2157.                 } else {
  2158.                         ring->irq_get = i9xx_ring_get_irq;
  2159.                         ring->irq_put = i9xx_ring_put_irq;
  2160.                 }
  2161.                 ring->irq_enable_mask = I915_USER_INTERRUPT;
  2162.         }
  2163.         ring->write_tail = ring_write_tail;
  2164.  
  2165.         if (IS_HASWELL(dev))
  2166.                 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
  2167.         else if (IS_GEN8(dev))
  2168.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2169.         else if (INTEL_INFO(dev)->gen >= 6)
  2170.                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2171.         else if (INTEL_INFO(dev)->gen >= 4)
  2172.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2173.         else if (IS_I830(dev) || IS_845G(dev))
  2174.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  2175.         else
  2176.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  2177.         ring->init = init_render_ring;
  2178.         ring->cleanup = render_ring_cleanup;
  2179.  
  2180.         /* Workaround batchbuffer to combat CS tlb bug. */
  2181.         if (HAS_BROKEN_CS_TLB(dev)) {
  2182.                 obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
  2183.                 if (obj == NULL) {
  2184.                         DRM_ERROR("Failed to allocate batch bo\n");
  2185.                         return -ENOMEM;
  2186.                 }
  2187.  
  2188.                 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
  2189.                 if (ret != 0) {
  2190.                         drm_gem_object_unreference(&obj->base);
  2191.                         DRM_ERROR("Failed to ping batch bo\n");
  2192.                         return ret;
  2193.                 }
  2194.  
  2195.                 ring->scratch.obj = obj;
  2196.                 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
  2197.         }
  2198.  
  2199.         return intel_init_ring_buffer(dev, ring);
  2200. }
  2201.  
  2202. #if 0
  2203. int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
  2204. {
  2205.         struct drm_i915_private *dev_priv = dev->dev_private;
  2206.         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
  2207.         struct intel_ringbuffer *ringbuf = ring->buffer;
  2208.         int ret;
  2209.  
  2210.         if (ringbuf == NULL) {
  2211.                 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
  2212.                 if (!ringbuf)
  2213.                         return -ENOMEM;
  2214.                 ring->buffer = ringbuf;
  2215.         }
  2216.  
  2217.         ring->name = "render ring";
  2218.         ring->id = RCS;
  2219.         ring->mmio_base = RENDER_RING_BASE;
  2220.  
  2221.         if (INTEL_INFO(dev)->gen >= 6) {
  2222.                 /* non-kms not supported on gen6+ */
  2223.                 ret = -ENODEV;
  2224.                 goto err_ringbuf;
  2225.         }
  2226.  
  2227.         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
  2228.          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
  2229.          * the special gen5 functions. */
  2230.         ring->add_request = i9xx_add_request;
  2231.         if (INTEL_INFO(dev)->gen < 4)
  2232.                 ring->flush = gen2_render_ring_flush;
  2233.         else
  2234.                 ring->flush = gen4_render_ring_flush;
  2235.         ring->get_seqno = ring_get_seqno;
  2236.         ring->set_seqno = ring_set_seqno;
  2237.         if (IS_GEN2(dev)) {
  2238.                 ring->irq_get = i8xx_ring_get_irq;
  2239.                 ring->irq_put = i8xx_ring_put_irq;
  2240.         } else {
  2241.                 ring->irq_get = i9xx_ring_get_irq;
  2242.                 ring->irq_put = i9xx_ring_put_irq;
  2243.         }
  2244.         ring->irq_enable_mask = I915_USER_INTERRUPT;
  2245.         ring->write_tail = ring_write_tail;
  2246.         if (INTEL_INFO(dev)->gen >= 4)
  2247.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2248.         else if (IS_I830(dev) || IS_845G(dev))
  2249.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  2250.         else
  2251.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  2252.         ring->init = init_render_ring;
  2253.         ring->cleanup = render_ring_cleanup;
  2254.  
  2255.         ring->dev = dev;
  2256.         INIT_LIST_HEAD(&ring->active_list);
  2257.         INIT_LIST_HEAD(&ring->request_list);
  2258.  
  2259.         ringbuf->size = size;
  2260.         ringbuf->effective_size = ringbuf->size;
  2261.         if (IS_I830(ring->dev) || IS_845G(ring->dev))
  2262.                 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
  2263.  
  2264.         ringbuf->virtual_start = ioremap_wc(start, size);
  2265.         if (ringbuf->virtual_start == NULL) {
  2266.                 DRM_ERROR("can not ioremap virtual address for"
  2267.                           " ring buffer\n");
  2268.                 ret = -ENOMEM;
  2269.                 goto err_ringbuf;
  2270.         }
  2271.  
  2272.         if (!I915_NEED_GFX_HWS(dev)) {
  2273.                 ret = init_phys_status_page(ring);
  2274.                 if (ret)
  2275.                         goto err_vstart;
  2276.         }
  2277.  
  2278.         return 0;
  2279.  
  2280. err_vstart:
  2281.         iounmap(ringbuf->virtual_start);
  2282. err_ringbuf:
  2283.         kfree(ringbuf);
  2284.         ring->buffer = NULL;
  2285.         return ret;
  2286. }
  2287. #endif
  2288.  
  2289. int intel_init_bsd_ring_buffer(struct drm_device *dev)
  2290. {
  2291.         struct drm_i915_private *dev_priv = dev->dev_private;
  2292.         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
  2293.  
  2294.         ring->name = "bsd ring";
  2295.         ring->id = VCS;
  2296.  
  2297.         ring->write_tail = ring_write_tail;
  2298.         if (INTEL_INFO(dev)->gen >= 6) {
  2299.                 ring->mmio_base = GEN6_BSD_RING_BASE;
  2300.                 /* gen6 bsd needs a special wa for tail updates */
  2301.                 if (IS_GEN6(dev))
  2302.                         ring->write_tail = gen6_bsd_ring_write_tail;
  2303.                 ring->flush = gen6_bsd_ring_flush;
  2304.                 ring->add_request = gen6_add_request;
  2305.                 ring->get_seqno = gen6_ring_get_seqno;
  2306.                 ring->set_seqno = ring_set_seqno;
  2307.                 if (INTEL_INFO(dev)->gen >= 8) {
  2308.                         ring->irq_enable_mask =
  2309.                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
  2310.                         ring->irq_get = gen8_ring_get_irq;
  2311.                         ring->irq_put = gen8_ring_put_irq;
  2312.                         ring->dispatch_execbuffer =
  2313.                                 gen8_ring_dispatch_execbuffer;
  2314.                         if (i915_semaphore_is_enabled(dev)) {
  2315.                                 ring->semaphore.sync_to = gen8_ring_sync;
  2316.                                 ring->semaphore.signal = gen8_xcs_signal;
  2317.                                 GEN8_RING_SEMAPHORE_INIT;
  2318.                         }
  2319.                 } else {
  2320.                 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
  2321.                 ring->irq_get = gen6_ring_get_irq;
  2322.                 ring->irq_put = gen6_ring_put_irq;
  2323.                         ring->dispatch_execbuffer =
  2324.                                 gen6_ring_dispatch_execbuffer;
  2325.                         if (i915_semaphore_is_enabled(dev)) {
  2326.                 ring->semaphore.sync_to = gen6_ring_sync;
  2327.                 ring->semaphore.signal = gen6_signal;
  2328.                 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
  2329.                 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
  2330.                 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
  2331.                 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
  2332.                 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2333.                 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
  2334.                 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
  2335.                 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
  2336.                 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
  2337.                 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2338.                         }
  2339.                 }
  2340.         } else {
  2341.                 ring->mmio_base = BSD_RING_BASE;
  2342.                 ring->flush = bsd_ring_flush;
  2343.                 ring->add_request = i9xx_add_request;
  2344.                 ring->get_seqno = ring_get_seqno;
  2345.                 ring->set_seqno = ring_set_seqno;
  2346.                 if (IS_GEN5(dev)) {
  2347.                         ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
  2348.                         ring->irq_get = gen5_ring_get_irq;
  2349.                         ring->irq_put = gen5_ring_put_irq;
  2350.                 } else {
  2351.                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
  2352.                         ring->irq_get = i9xx_ring_get_irq;
  2353.                         ring->irq_put = i9xx_ring_put_irq;
  2354.                 }
  2355.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2356.         }
  2357.         ring->init = init_ring_common;
  2358.  
  2359.         return intel_init_ring_buffer(dev, ring);
  2360. }
  2361.  
  2362. /**
  2363.  * Initialize the second BSD ring for Broadwell GT3.
  2364.  * It is noted that this only exists on Broadwell GT3.
  2365.  */
  2366. int intel_init_bsd2_ring_buffer(struct drm_device *dev)
  2367. {
  2368.         struct drm_i915_private *dev_priv = dev->dev_private;
  2369.         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
  2370.  
  2371.         if ((INTEL_INFO(dev)->gen != 8)) {
  2372.                 DRM_ERROR("No dual-BSD ring on non-BDW machine\n");
  2373.                 return -EINVAL;
  2374.         }
  2375.  
  2376.         ring->name = "bsd2 ring";
  2377.         ring->id = VCS2;
  2378.  
  2379.         ring->write_tail = ring_write_tail;
  2380.         ring->mmio_base = GEN8_BSD2_RING_BASE;
  2381.         ring->flush = gen6_bsd_ring_flush;
  2382.         ring->add_request = gen6_add_request;
  2383.         ring->get_seqno = gen6_ring_get_seqno;
  2384.         ring->set_seqno = ring_set_seqno;
  2385.         ring->irq_enable_mask =
  2386.                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
  2387.         ring->irq_get = gen8_ring_get_irq;
  2388.         ring->irq_put = gen8_ring_put_irq;
  2389.         ring->dispatch_execbuffer =
  2390.                         gen8_ring_dispatch_execbuffer;
  2391.         if (i915_semaphore_is_enabled(dev)) {
  2392.                 ring->semaphore.sync_to = gen8_ring_sync;
  2393.                 ring->semaphore.signal = gen8_xcs_signal;
  2394.                 GEN8_RING_SEMAPHORE_INIT;
  2395.         }
  2396.         ring->init = init_ring_common;
  2397.  
  2398.         return intel_init_ring_buffer(dev, ring);
  2399. }
  2400.  
  2401. int intel_init_blt_ring_buffer(struct drm_device *dev)
  2402. {
  2403.         struct drm_i915_private *dev_priv = dev->dev_private;
  2404.         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
  2405.  
  2406.         ring->name = "blitter ring";
  2407.         ring->id = BCS;
  2408.  
  2409.         ring->mmio_base = BLT_RING_BASE;
  2410.         ring->write_tail = ring_write_tail;
  2411.         ring->flush = gen6_ring_flush;
  2412.         ring->add_request = gen6_add_request;
  2413.         ring->get_seqno = gen6_ring_get_seqno;
  2414.         ring->set_seqno = ring_set_seqno;
  2415.         if (INTEL_INFO(dev)->gen >= 8) {
  2416.                 ring->irq_enable_mask =
  2417.                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
  2418.                 ring->irq_get = gen8_ring_get_irq;
  2419.                 ring->irq_put = gen8_ring_put_irq;
  2420.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2421.                 if (i915_semaphore_is_enabled(dev)) {
  2422.                         ring->semaphore.sync_to = gen8_ring_sync;
  2423.                         ring->semaphore.signal = gen8_xcs_signal;
  2424.                         GEN8_RING_SEMAPHORE_INIT;
  2425.                 }
  2426.         } else {
  2427.         ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
  2428.         ring->irq_get = gen6_ring_get_irq;
  2429.         ring->irq_put = gen6_ring_put_irq;
  2430.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2431.                 if (i915_semaphore_is_enabled(dev)) {
  2432.                         ring->semaphore.signal = gen6_signal;
  2433.         ring->semaphore.sync_to = gen6_ring_sync;
  2434.         /*
  2435.                          * The current semaphore is only applied on pre-gen8
  2436.                          * platform.  And there is no VCS2 ring on the pre-gen8
  2437.                          * platform. So the semaphore between BCS and VCS2 is
  2438.                          * initialized as INVALID.  Gen8 will initialize the
  2439.                          * sema between BCS and VCS2 later.
  2440.          */
  2441.         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
  2442.         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
  2443.         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
  2444.         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
  2445.         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2446.         ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
  2447.         ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
  2448.         ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
  2449.         ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
  2450.         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2451.                 }
  2452.         }
  2453.         ring->init = init_ring_common;
  2454.  
  2455.         return intel_init_ring_buffer(dev, ring);
  2456. }
  2457.  
  2458. int intel_init_vebox_ring_buffer(struct drm_device *dev)
  2459. {
  2460.         struct drm_i915_private *dev_priv = dev->dev_private;
  2461.         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
  2462.  
  2463.         ring->name = "video enhancement ring";
  2464.         ring->id = VECS;
  2465.  
  2466.         ring->mmio_base = VEBOX_RING_BASE;
  2467.         ring->write_tail = ring_write_tail;
  2468.         ring->flush = gen6_ring_flush;
  2469.         ring->add_request = gen6_add_request;
  2470.         ring->get_seqno = gen6_ring_get_seqno;
  2471.         ring->set_seqno = ring_set_seqno;
  2472.  
  2473.         if (INTEL_INFO(dev)->gen >= 8) {
  2474.                 ring->irq_enable_mask =
  2475.                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
  2476.                 ring->irq_get = gen8_ring_get_irq;
  2477.                 ring->irq_put = gen8_ring_put_irq;
  2478.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2479.                 if (i915_semaphore_is_enabled(dev)) {
  2480.                         ring->semaphore.sync_to = gen8_ring_sync;
  2481.                         ring->semaphore.signal = gen8_xcs_signal;
  2482.                         GEN8_RING_SEMAPHORE_INIT;
  2483.                 }
  2484.         } else {
  2485.         ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
  2486.         ring->irq_get = hsw_vebox_get_irq;
  2487.         ring->irq_put = hsw_vebox_put_irq;
  2488.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2489.                 if (i915_semaphore_is_enabled(dev)) {
  2490.         ring->semaphore.sync_to = gen6_ring_sync;
  2491.         ring->semaphore.signal = gen6_signal;
  2492.         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
  2493.         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
  2494.         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
  2495.         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
  2496.         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
  2497.         ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
  2498.         ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
  2499.         ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
  2500.         ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
  2501.         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
  2502.                 }
  2503.         }
  2504.         ring->init = init_ring_common;
  2505.  
  2506.         return intel_init_ring_buffer(dev, ring);
  2507. }
  2508.  
  2509. int
  2510. intel_ring_flush_all_caches(struct intel_engine_cs *ring)
  2511. {
  2512.         int ret;
  2513.  
  2514.         if (!ring->gpu_caches_dirty)
  2515.                 return 0;
  2516.  
  2517.         ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
  2518.         if (ret)
  2519.                 return ret;
  2520.  
  2521.         trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
  2522.  
  2523.         ring->gpu_caches_dirty = false;
  2524.         return 0;
  2525. }
  2526.  
  2527. int
  2528. intel_ring_invalidate_all_caches(struct intel_engine_cs *ring)
  2529. {
  2530.         uint32_t flush_domains;
  2531.         int ret;
  2532.  
  2533.         flush_domains = 0;
  2534.         if (ring->gpu_caches_dirty)
  2535.                 flush_domains = I915_GEM_GPU_DOMAINS;
  2536.  
  2537.         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  2538.         if (ret)
  2539.                 return ret;
  2540.  
  2541.         trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  2542.  
  2543.         ring->gpu_caches_dirty = false;
  2544.         return 0;
  2545. }
  2546.  
  2547. void
  2548. intel_stop_ring_buffer(struct intel_engine_cs *ring)
  2549. {
  2550.         int ret;
  2551.  
  2552.         if (!intel_ring_initialized(ring))
  2553.                 return;
  2554.  
  2555.         ret = intel_ring_idle(ring);
  2556.         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
  2557.                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
  2558.                           ring->name, ret);
  2559.  
  2560.         stop_ring(ring);
  2561. }
  2562.