Subversion Repositories Kolibri OS

Rev

Rev 4539 | Rev 5060 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008-2010 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *    Zou Nan hai <nanhai.zou@intel.com>
  26.  *    Xiang Hai hao<haihao.xiang@intel.com>
  27.  *
  28.  */
  29.  
  30. #include <drm/drmP.h>
  31. #include "i915_drv.h"
  32. #include <drm/i915_drm.h>
  33. #include "i915_trace.h"
  34. #include "intel_drv.h"
  35.  
  36. static inline int ring_space(struct intel_ring_buffer *ring)
  37. {
  38.         int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
  39.         if (space < 0)
  40.                 space += ring->size;
  41.         return space;
  42. }
  43.  
  44. void __intel_ring_advance(struct intel_ring_buffer *ring)
  45. {
  46.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  47.  
  48.         ring->tail &= ring->size - 1;
  49.         if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring))
  50.                 return;
  51.         ring->write_tail(ring, ring->tail);
  52. }
  53.  
  54. static int
  55. gen2_render_ring_flush(struct intel_ring_buffer *ring,
  56.                        u32      invalidate_domains,
  57.                        u32      flush_domains)
  58. {
  59.         u32 cmd;
  60.         int ret;
  61.  
  62.         cmd = MI_FLUSH;
  63.         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  64.                 cmd |= MI_NO_WRITE_FLUSH;
  65.  
  66.         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  67.                 cmd |= MI_READ_FLUSH;
  68.  
  69.         ret = intel_ring_begin(ring, 2);
  70.         if (ret)
  71.                 return ret;
  72.  
  73.         intel_ring_emit(ring, cmd);
  74.         intel_ring_emit(ring, MI_NOOP);
  75.         intel_ring_advance(ring);
  76.  
  77.         return 0;
  78. }
  79.  
  80. static int
  81. gen4_render_ring_flush(struct intel_ring_buffer *ring,
  82.                   u32   invalidate_domains,
  83.                   u32   flush_domains)
  84. {
  85.         struct drm_device *dev = ring->dev;
  86.         u32 cmd;
  87.         int ret;
  88.  
  89.         /*
  90.          * read/write caches:
  91.          *
  92.          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  93.          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  94.          * also flushed at 2d versus 3d pipeline switches.
  95.          *
  96.          * read-only caches:
  97.          *
  98.          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  99.          * MI_READ_FLUSH is set, and is always flushed on 965.
  100.          *
  101.          * I915_GEM_DOMAIN_COMMAND may not exist?
  102.          *
  103.          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  104.          * invalidated when MI_EXE_FLUSH is set.
  105.          *
  106.          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  107.          * invalidated with every MI_FLUSH.
  108.          *
  109.          * TLBs:
  110.          *
  111.          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  112.          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  113.          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  114.          * are flushed at any MI_FLUSH.
  115.          */
  116.  
  117.         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  118.         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
  119.                 cmd &= ~MI_NO_WRITE_FLUSH;
  120.         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  121.                 cmd |= MI_EXE_FLUSH;
  122.  
  123.         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
  124.             (IS_G4X(dev) || IS_GEN5(dev)))
  125.                 cmd |= MI_INVALIDATE_ISP;
  126.  
  127.         ret = intel_ring_begin(ring, 2);
  128.         if (ret)
  129.                 return ret;
  130.  
  131.         intel_ring_emit(ring, cmd);
  132.         intel_ring_emit(ring, MI_NOOP);
  133.         intel_ring_advance(ring);
  134.  
  135.         return 0;
  136. }
  137.  
  138. /**
  139.  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
  140.  * implementing two workarounds on gen6.  From section 1.4.7.1
  141.  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
  142.  *
  143.  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
  144.  * produced by non-pipelined state commands), software needs to first
  145.  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
  146.  * 0.
  147.  *
  148.  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
  149.  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
  150.  *
  151.  * And the workaround for these two requires this workaround first:
  152.  *
  153.  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
  154.  * BEFORE the pipe-control with a post-sync op and no write-cache
  155.  * flushes.
  156.  *
  157.  * And this last workaround is tricky because of the requirements on
  158.  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
  159.  * volume 2 part 1:
  160.  *
  161.  *     "1 of the following must also be set:
  162.  *      - Render Target Cache Flush Enable ([12] of DW1)
  163.  *      - Depth Cache Flush Enable ([0] of DW1)
  164.  *      - Stall at Pixel Scoreboard ([1] of DW1)
  165.  *      - Depth Stall ([13] of DW1)
  166.  *      - Post-Sync Operation ([13] of DW1)
  167.  *      - Notify Enable ([8] of DW1)"
  168.  *
  169.  * The cache flushes require the workaround flush that triggered this
  170.  * one, so we can't use it.  Depth stall would trigger the same.
  171.  * Post-sync nonzero is what triggered this second workaround, so we
  172.  * can't use that one either.  Notify enable is IRQs, which aren't
  173.  * really our business.  That leaves only stall at scoreboard.
  174.  */
  175. static int
  176. intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
  177. {
  178.         u32 scratch_addr = ring->scratch.gtt_offset + 128;
  179.         int ret;
  180.  
  181.  
  182.         ret = intel_ring_begin(ring, 6);
  183.         if (ret)
  184.                 return ret;
  185.  
  186.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  187.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  188.                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
  189.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  190.         intel_ring_emit(ring, 0); /* low dword */
  191.         intel_ring_emit(ring, 0); /* high dword */
  192.         intel_ring_emit(ring, MI_NOOP);
  193.         intel_ring_advance(ring);
  194.  
  195.         ret = intel_ring_begin(ring, 6);
  196.         if (ret)
  197.                 return ret;
  198.  
  199.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  200.         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
  201.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  202.         intel_ring_emit(ring, 0);
  203.         intel_ring_emit(ring, 0);
  204.         intel_ring_emit(ring, MI_NOOP);
  205.         intel_ring_advance(ring);
  206.  
  207.         return 0;
  208. }
  209.  
  210. static int
  211. gen6_render_ring_flush(struct intel_ring_buffer *ring,
  212.                          u32 invalidate_domains, u32 flush_domains)
  213. {
  214.         u32 flags = 0;
  215.         u32 scratch_addr = ring->scratch.gtt_offset + 128;
  216.         int ret;
  217.  
  218.         /* Force SNB workarounds for PIPE_CONTROL flushes */
  219.         ret = intel_emit_post_sync_nonzero_flush(ring);
  220.         if (ret)
  221.                 return ret;
  222.  
  223.         /* Just flush everything.  Experiments have shown that reducing the
  224.          * number of bits based on the write domains has little performance
  225.          * impact.
  226.          */
  227.         if (flush_domains) {
  228.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  229.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  230.                 /*
  231.                  * Ensure that any following seqno writes only happen
  232.                  * when the render cache is indeed flushed.
  233.                  */
  234.                 flags |= PIPE_CONTROL_CS_STALL;
  235.         }
  236.         if (invalidate_domains) {
  237.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  238.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  239.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  240.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  241.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  242.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  243.                 /*
  244.                  * TLB invalidate requires a post-sync write.
  245.                  */
  246.                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
  247.         }
  248.  
  249.         ret = intel_ring_begin(ring, 4);
  250.         if (ret)
  251.                 return ret;
  252.  
  253.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  254.         intel_ring_emit(ring, flags);
  255.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
  256.         intel_ring_emit(ring, 0);
  257.         intel_ring_advance(ring);
  258.  
  259.         return 0;
  260. }
  261.  
  262. static int
  263. gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
  264. {
  265.         int ret;
  266.  
  267.         ret = intel_ring_begin(ring, 4);
  268.         if (ret)
  269.                 return ret;
  270.  
  271.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  272.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  273.                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
  274.         intel_ring_emit(ring, 0);
  275.         intel_ring_emit(ring, 0);
  276.         intel_ring_advance(ring);
  277.  
  278.         return 0;
  279. }
  280.  
  281. static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value)
  282. {
  283.         int ret;
  284.  
  285.         if (!ring->fbc_dirty)
  286.                 return 0;
  287.  
  288.         ret = intel_ring_begin(ring, 6);
  289.         if (ret)
  290.                 return ret;
  291.         /* WaFbcNukeOn3DBlt:ivb/hsw */
  292.         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  293.         intel_ring_emit(ring, MSG_FBC_REND_STATE);
  294.         intel_ring_emit(ring, value);
  295.         intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
  296.         intel_ring_emit(ring, MSG_FBC_REND_STATE);
  297.         intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
  298.         intel_ring_advance(ring);
  299.  
  300.         ring->fbc_dirty = false;
  301.         return 0;
  302. }
  303.  
  304. static int
  305. gen7_render_ring_flush(struct intel_ring_buffer *ring,
  306.                        u32 invalidate_domains, u32 flush_domains)
  307. {
  308.         u32 flags = 0;
  309.         u32 scratch_addr = ring->scratch.gtt_offset + 128;
  310.         int ret;
  311.  
  312.         /*
  313.          * Ensure that any following seqno writes only happen when the render
  314.          * cache is indeed flushed.
  315.          *
  316.          * Workaround: 4th PIPE_CONTROL command (except the ones with only
  317.          * read-cache invalidate bits set) must have the CS_STALL bit set. We
  318.          * don't try to be clever and just set it unconditionally.
  319.          */
  320.         flags |= PIPE_CONTROL_CS_STALL;
  321.  
  322.         /* Just flush everything.  Experiments have shown that reducing the
  323.          * number of bits based on the write domains has little performance
  324.          * impact.
  325.          */
  326.         if (flush_domains) {
  327.         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  328.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  329.         }
  330.         if (invalidate_domains) {
  331.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  332.         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  333.         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  334.         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  335.         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  336.         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  337.                 /*
  338.                  * TLB invalidate requires a post-sync write.
  339.                  */
  340.                 flags |= PIPE_CONTROL_QW_WRITE;
  341.                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  342.  
  343.                 /* Workaround: we must issue a pipe_control with CS-stall bit
  344.                  * set before a pipe_control command that has the state cache
  345.                  * invalidate bit set. */
  346.                 gen7_render_ring_cs_stall_wa(ring);
  347.         }
  348.  
  349.         ret = intel_ring_begin(ring, 4);
  350.         if (ret)
  351.                 return ret;
  352.  
  353.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  354.         intel_ring_emit(ring, flags);
  355.         intel_ring_emit(ring, scratch_addr);
  356.         intel_ring_emit(ring, 0);
  357.         intel_ring_advance(ring);
  358.  
  359.         if (!invalidate_domains && flush_domains)
  360.                 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
  361.  
  362.         return 0;
  363. }
  364.  
  365. static int
  366. gen8_render_ring_flush(struct intel_ring_buffer *ring,
  367.                        u32 invalidate_domains, u32 flush_domains)
  368. {
  369.         u32 flags = 0;
  370.         u32 scratch_addr = ring->scratch.gtt_offset + 128;
  371.         int ret;
  372.  
  373.         flags |= PIPE_CONTROL_CS_STALL;
  374.  
  375.         if (flush_domains) {
  376.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  377.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  378.         }
  379.         if (invalidate_domains) {
  380.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  381.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  382.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  383.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  384.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  385.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  386.                 flags |= PIPE_CONTROL_QW_WRITE;
  387.                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
  388.         }
  389.  
  390.         ret = intel_ring_begin(ring, 6);
  391.         if (ret)
  392.                 return ret;
  393.  
  394.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
  395.         intel_ring_emit(ring, flags);
  396.         intel_ring_emit(ring, scratch_addr);
  397.         intel_ring_emit(ring, 0);
  398.         intel_ring_emit(ring, 0);
  399.         intel_ring_emit(ring, 0);
  400.         intel_ring_advance(ring);
  401.  
  402.         return 0;
  403.  
  404. }
  405.  
  406. static void ring_write_tail(struct intel_ring_buffer *ring,
  407.                             u32 value)
  408. {
  409.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  410.         I915_WRITE_TAIL(ring, value);
  411. }
  412.  
  413. u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
  414. {
  415.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  416.         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
  417.                         RING_ACTHD(ring->mmio_base) : ACTHD;
  418.  
  419.         return I915_READ(acthd_reg);
  420. }
  421.  
  422. static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
  423. {
  424.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  425.         u32 addr;
  426.  
  427.         addr = dev_priv->status_page_dmah->busaddr;
  428.         if (INTEL_INFO(ring->dev)->gen >= 4)
  429.                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
  430.         I915_WRITE(HWS_PGA, addr);
  431. }
  432.  
  433. static int init_ring_common(struct intel_ring_buffer *ring)
  434. {
  435.         struct drm_device *dev = ring->dev;
  436.         drm_i915_private_t *dev_priv = dev->dev_private;
  437.         struct drm_i915_gem_object *obj = ring->obj;
  438.         int ret = 0;
  439.         u32 head;
  440.  
  441.         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
  442.  
  443.         if (I915_NEED_GFX_HWS(dev))
  444.                 intel_ring_setup_status_page(ring);
  445.         else
  446.                 ring_setup_phys_status_page(ring);
  447.  
  448.         /* Stop the ring if it's running. */
  449.         I915_WRITE_CTL(ring, 0);
  450.         I915_WRITE_HEAD(ring, 0);
  451.         ring->write_tail(ring, 0);
  452.  
  453.         head = I915_READ_HEAD(ring) & HEAD_ADDR;
  454.  
  455.         /* G45 ring initialization fails to reset head to zero */
  456.         if (head != 0) {
  457.                 DRM_DEBUG_KMS("%s head not reset to zero "
  458.                               "ctl %08x head %08x tail %08x start %08x\n",
  459.                               ring->name,
  460.                               I915_READ_CTL(ring),
  461.                               I915_READ_HEAD(ring),
  462.                               I915_READ_TAIL(ring),
  463.                               I915_READ_START(ring));
  464.  
  465.                 I915_WRITE_HEAD(ring, 0);
  466.  
  467.                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
  468.                         DRM_ERROR("failed to set %s head to zero "
  469.                                   "ctl %08x head %08x tail %08x start %08x\n",
  470.                                   ring->name,
  471.                                   I915_READ_CTL(ring),
  472.                                   I915_READ_HEAD(ring),
  473.                                   I915_READ_TAIL(ring),
  474.                                   I915_READ_START(ring));
  475.                 }
  476.         }
  477.  
  478.         /* Initialize the ring. This must happen _after_ we've cleared the ring
  479.          * registers with the above sequence (the readback of the HEAD registers
  480.          * also enforces ordering), otherwise the hw might lose the new ring
  481.          * register values. */
  482.         I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
  483.         I915_WRITE_CTL(ring,
  484.                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
  485.                         | RING_VALID);
  486.  
  487.         /* If the head is still not zero, the ring is dead */
  488.         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
  489.                      I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
  490.                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
  491.                 DRM_ERROR("%s initialization failed "
  492.                                 "ctl %08x head %08x tail %08x start %08x\n",
  493.                                 ring->name,
  494.                                 I915_READ_CTL(ring),
  495.                                 I915_READ_HEAD(ring),
  496.                                 I915_READ_TAIL(ring),
  497.                                 I915_READ_START(ring));
  498.                 ret = -EIO;
  499.                 goto out;
  500.         }
  501.  
  502.                 ring->head = I915_READ_HEAD(ring);
  503.                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
  504.                 ring->space = ring_space(ring);
  505.                 ring->last_retired_head = -1;
  506.  
  507.         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
  508.  
  509. out:
  510.         gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
  511.  
  512.         return ret;
  513. }
  514.  
  515. static int
  516. init_pipe_control(struct intel_ring_buffer *ring)
  517. {
  518.         int ret;
  519.  
  520.         if (ring->scratch.obj)
  521.                 return 0;
  522.  
  523.         ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
  524.         if (ring->scratch.obj == NULL) {
  525.                 DRM_ERROR("Failed to allocate seqno page\n");
  526.                 ret = -ENOMEM;
  527.                 goto err;
  528.         }
  529.  
  530.         i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
  531.  
  532.         ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, true, false);
  533.         if (ret)
  534.                 goto err_unref;
  535.  
  536.         ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
  537.     ring->scratch.cpu_page = (void*)MapIoMem((addr_t)sg_page(ring->scratch.obj->pages->sgl),4096, PG_SW|0x100);
  538.         if (ring->scratch.cpu_page == NULL) {
  539.                 ret = -ENOMEM;
  540.                 goto err_unpin;
  541.         }
  542.  
  543.         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
  544.                          ring->name, ring->scratch.gtt_offset);
  545.         return 0;
  546.  
  547. err_unpin:
  548.         i915_gem_object_unpin(ring->scratch.obj);
  549. err_unref:
  550.         drm_gem_object_unreference(&ring->scratch.obj->base);
  551. err:
  552.         return ret;
  553. }
  554.  
  555. static int init_render_ring(struct intel_ring_buffer *ring)
  556. {
  557.         struct drm_device *dev = ring->dev;
  558.         struct drm_i915_private *dev_priv = dev->dev_private;
  559.         int ret = init_ring_common(ring);
  560.  
  561.         if (INTEL_INFO(dev)->gen > 3)
  562.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
  563.  
  564.         /* We need to disable the AsyncFlip performance optimisations in order
  565.          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
  566.          * programmed to '1' on all products.
  567.          *
  568.          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
  569.          */
  570.         if (INTEL_INFO(dev)->gen >= 6)
  571.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
  572.  
  573.         /* Required for the hardware to program scanline values for waiting */
  574.         if (INTEL_INFO(dev)->gen == 6)
  575.                 I915_WRITE(GFX_MODE,
  576.                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS));
  577.  
  578.                 if (IS_GEN7(dev))
  579.                         I915_WRITE(GFX_MODE_GEN7,
  580.                                    _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
  581.                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
  582.  
  583.         if (INTEL_INFO(dev)->gen >= 5) {
  584.                 ret = init_pipe_control(ring);
  585.                 if (ret)
  586.                         return ret;
  587.         }
  588.  
  589.         if (IS_GEN6(dev)) {
  590.                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
  591.                  * "If this bit is set, STCunit will have LRA as replacement
  592.                  *  policy. [...] This bit must be reset.  LRA replacement
  593.                  *  policy is not supported."
  594.                  */
  595.                 I915_WRITE(CACHE_MODE_0,
  596.                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
  597.  
  598.                 /* This is not explicitly set for GEN6, so read the register.
  599.                  * see intel_ring_mi_set_context() for why we care.
  600.                  * TODO: consider explicitly setting the bit for GEN5
  601.                  */
  602.                 ring->itlb_before_ctx_switch =
  603.                         !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
  604.         }
  605.  
  606.         if (INTEL_INFO(dev)->gen >= 6)
  607.                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
  608.  
  609.         if (HAS_L3_DPF(dev))
  610.                 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
  611.  
  612.         return ret;
  613. }
  614.  
  615. static void render_ring_cleanup(struct intel_ring_buffer *ring)
  616. {
  617.         struct drm_device *dev = ring->dev;
  618.  
  619.         if (ring->scratch.obj == NULL)
  620.                 return;
  621.  
  622.         if (INTEL_INFO(dev)->gen >= 5) {
  623. //       kunmap(sg_page(ring->scratch.obj->pages->sgl));
  624.                 i915_gem_object_unpin(ring->scratch.obj);
  625.         }
  626.  
  627.         drm_gem_object_unreference(&ring->scratch.obj->base);
  628.         ring->scratch.obj = NULL;
  629. }
  630.  
  631. static void
  632. update_mboxes(struct intel_ring_buffer *ring,
  633.             u32 mmio_offset)
  634. {
  635. /* NB: In order to be able to do semaphore MBOX updates for varying number
  636.  * of rings, it's easiest if we round up each individual update to a
  637.  * multiple of 2 (since ring updates must always be a multiple of 2)
  638.  * even though the actual update only requires 3 dwords.
  639.  */
  640. #define MBOX_UPDATE_DWORDS 4
  641.         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
  642.         intel_ring_emit(ring, mmio_offset);
  643.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  644.         intel_ring_emit(ring, MI_NOOP);
  645. }
  646.  
  647. /**
  648.  * gen6_add_request - Update the semaphore mailbox registers
  649.  *
  650.  * @ring - ring that is adding a request
  651.  * @seqno - return seqno stuck into the ring
  652.  *
  653.  * Update the mailbox registers in the *other* rings with the current seqno.
  654.  * This acts like a signal in the canonical semaphore.
  655.  */
  656. static int
  657. gen6_add_request(struct intel_ring_buffer *ring)
  658. {
  659.         struct drm_device *dev = ring->dev;
  660.         struct drm_i915_private *dev_priv = dev->dev_private;
  661.         struct intel_ring_buffer *useless;
  662.         int i, ret, num_dwords = 4;
  663.  
  664.         if (i915_semaphore_is_enabled(dev))
  665.                 num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS);
  666. #undef MBOX_UPDATE_DWORDS
  667.  
  668.         ret = intel_ring_begin(ring, num_dwords);
  669.         if (ret)
  670.                 return ret;
  671.  
  672.         if (i915_semaphore_is_enabled(dev)) {
  673.         for_each_ring(useless, dev_priv, i) {
  674.                 u32 mbox_reg = ring->signal_mbox[i];
  675.                 if (mbox_reg != GEN6_NOSYNC)
  676.                         update_mboxes(ring, mbox_reg);
  677.         }
  678.         }
  679.  
  680.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  681.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  682.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  683.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  684.         __intel_ring_advance(ring);
  685.  
  686.         return 0;
  687. }
  688.  
  689. static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
  690.                                               u32 seqno)
  691. {
  692.         struct drm_i915_private *dev_priv = dev->dev_private;
  693.         return dev_priv->last_seqno < seqno;
  694. }
  695.  
  696. /**
  697.  * intel_ring_sync - sync the waiter to the signaller on seqno
  698.  *
  699.  * @waiter - ring that is waiting
  700.  * @signaller - ring which has, or will signal
  701.  * @seqno - seqno which the waiter will block on
  702.  */
  703. static int
  704. gen6_ring_sync(struct intel_ring_buffer *waiter,
  705.                 struct intel_ring_buffer *signaller,
  706.                 u32 seqno)
  707. {
  708.         int ret;
  709.         u32 dw1 = MI_SEMAPHORE_MBOX |
  710.                   MI_SEMAPHORE_COMPARE |
  711.                   MI_SEMAPHORE_REGISTER;
  712.  
  713.         /* Throughout all of the GEM code, seqno passed implies our current
  714.          * seqno is >= the last seqno executed. However for hardware the
  715.          * comparison is strictly greater than.
  716.          */
  717.         seqno -= 1;
  718.  
  719.         WARN_ON(signaller->semaphore_register[waiter->id] ==
  720.                 MI_SEMAPHORE_SYNC_INVALID);
  721.  
  722.         ret = intel_ring_begin(waiter, 4);
  723.         if (ret)
  724.                 return ret;
  725.  
  726.         /* If seqno wrap happened, omit the wait with no-ops */
  727.         if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
  728.         intel_ring_emit(waiter,
  729.                                 dw1 |
  730.                                 signaller->semaphore_register[waiter->id]);
  731.         intel_ring_emit(waiter, seqno);
  732.         intel_ring_emit(waiter, 0);
  733.         intel_ring_emit(waiter, MI_NOOP);
  734.         } else {
  735.                 intel_ring_emit(waiter, MI_NOOP);
  736.                 intel_ring_emit(waiter, MI_NOOP);
  737.                 intel_ring_emit(waiter, MI_NOOP);
  738.                 intel_ring_emit(waiter, MI_NOOP);
  739.         }
  740.         intel_ring_advance(waiter);
  741.  
  742.         return 0;
  743. }
  744.  
  745. #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
  746. do {                                                                    \
  747.         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
  748.                  PIPE_CONTROL_DEPTH_STALL);                             \
  749.         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
  750.         intel_ring_emit(ring__, 0);                                                     \
  751.         intel_ring_emit(ring__, 0);                                                     \
  752. } while (0)
  753.  
  754. static int
  755. pc_render_add_request(struct intel_ring_buffer *ring)
  756. {
  757.         u32 scratch_addr = ring->scratch.gtt_offset + 128;
  758.         int ret;
  759.  
  760.         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
  761.          * incoherent with writes to memory, i.e. completely fubar,
  762.          * so we need to use PIPE_NOTIFY instead.
  763.          *
  764.          * However, we also need to workaround the qword write
  765.          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
  766.          * memory before requesting an interrupt.
  767.          */
  768.         ret = intel_ring_begin(ring, 32);
  769.         if (ret)
  770.                 return ret;
  771.  
  772.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  773.                         PIPE_CONTROL_WRITE_FLUSH |
  774.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
  775.         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  776.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  777.         intel_ring_emit(ring, 0);
  778.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  779.         scratch_addr += 128; /* write to separate cachelines */
  780.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  781.         scratch_addr += 128;
  782.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  783.         scratch_addr += 128;
  784.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  785.         scratch_addr += 128;
  786.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  787.         scratch_addr += 128;
  788.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  789.  
  790.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  791.                         PIPE_CONTROL_WRITE_FLUSH |
  792.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
  793.                         PIPE_CONTROL_NOTIFY);
  794.         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  795.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  796.         intel_ring_emit(ring, 0);
  797.         __intel_ring_advance(ring);
  798.  
  799.         return 0;
  800. }
  801.  
  802. static u32
  803. gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  804. {
  805.         /* Workaround to force correct ordering between irq and seqno writes on
  806.          * ivb (and maybe also on snb) by reading from a CS register (like
  807.          * ACTHD) before reading the status page. */
  808.         if (!lazy_coherency)
  809.                 intel_ring_get_active_head(ring);
  810.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  811. }
  812.  
  813. static u32
  814. ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  815. {
  816.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  817. }
  818.  
  819. static void
  820. ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
  821. {
  822.         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
  823. }
  824.  
  825. static u32
  826. pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  827. {
  828.         return ring->scratch.cpu_page[0];
  829. }
  830.  
  831. static void
  832. pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
  833. {
  834.         ring->scratch.cpu_page[0] = seqno;
  835. }
  836.  
  837. static bool
  838. gen5_ring_get_irq(struct intel_ring_buffer *ring)
  839. {
  840.         struct drm_device *dev = ring->dev;
  841.         drm_i915_private_t *dev_priv = dev->dev_private;
  842.         unsigned long flags;
  843.  
  844.         if (!dev->irq_enabled)
  845.                 return false;
  846.  
  847.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  848.         if (ring->irq_refcount++ == 0)
  849.                 ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  850.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  851.  
  852.         return true;
  853. }
  854.  
  855. static void
  856. gen5_ring_put_irq(struct intel_ring_buffer *ring)
  857. {
  858.         struct drm_device *dev = ring->dev;
  859.         drm_i915_private_t *dev_priv = dev->dev_private;
  860.         unsigned long flags;
  861.  
  862.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  863.         if (--ring->irq_refcount == 0)
  864.                 ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  865.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  866. }
  867.  
  868. static bool
  869. i9xx_ring_get_irq(struct intel_ring_buffer *ring)
  870. {
  871.         struct drm_device *dev = ring->dev;
  872.         drm_i915_private_t *dev_priv = dev->dev_private;
  873.         unsigned long flags;
  874.  
  875.         if (!dev->irq_enabled)
  876.                 return false;
  877.  
  878.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  879.         if (ring->irq_refcount++ == 0) {
  880.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  881.         I915_WRITE(IMR, dev_priv->irq_mask);
  882.         POSTING_READ(IMR);
  883.         }
  884.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  885.  
  886.         return true;
  887. }
  888.  
  889. static void
  890. i9xx_ring_put_irq(struct intel_ring_buffer *ring)
  891. {
  892.         struct drm_device *dev = ring->dev;
  893.         drm_i915_private_t *dev_priv = dev->dev_private;
  894.         unsigned long flags;
  895.  
  896.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  897.         if (--ring->irq_refcount == 0) {
  898.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  899.         I915_WRITE(IMR, dev_priv->irq_mask);
  900.         POSTING_READ(IMR);
  901.         }
  902.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  903. }
  904.  
  905. static bool
  906. i8xx_ring_get_irq(struct intel_ring_buffer *ring)
  907. {
  908.         struct drm_device *dev = ring->dev;
  909.         drm_i915_private_t *dev_priv = dev->dev_private;
  910.         unsigned long flags;
  911.  
  912.         if (!dev->irq_enabled)
  913.                 return false;
  914.  
  915.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  916.         if (ring->irq_refcount++ == 0) {
  917.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  918.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  919.                 POSTING_READ16(IMR);
  920.         }
  921.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  922.  
  923.         return true;
  924. }
  925.  
  926. static void
  927. i8xx_ring_put_irq(struct intel_ring_buffer *ring)
  928. {
  929.         struct drm_device *dev = ring->dev;
  930.         drm_i915_private_t *dev_priv = dev->dev_private;
  931.         unsigned long flags;
  932.  
  933.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  934.         if (--ring->irq_refcount == 0) {
  935.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  936.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  937.                 POSTING_READ16(IMR);
  938.         }
  939.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  940. }
  941.  
  942. void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
  943. {
  944.         struct drm_device *dev = ring->dev;
  945.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  946.         u32 mmio = 0;
  947.  
  948.         /* The ring status page addresses are no longer next to the rest of
  949.          * the ring registers as of gen7.
  950.          */
  951.         if (IS_GEN7(dev)) {
  952.                 switch (ring->id) {
  953.                 case RCS:
  954.                         mmio = RENDER_HWS_PGA_GEN7;
  955.                         break;
  956.                 case BCS:
  957.                         mmio = BLT_HWS_PGA_GEN7;
  958.                         break;
  959.                 case VCS:
  960.                         mmio = BSD_HWS_PGA_GEN7;
  961.                         break;
  962.                 case VECS:
  963.                         mmio = VEBOX_HWS_PGA_GEN7;
  964.                         break;
  965.                 }
  966.         } else if (IS_GEN6(ring->dev)) {
  967.                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
  968.         } else {
  969.                 /* XXX: gen8 returns to sanity */
  970.                 mmio = RING_HWS_PGA(ring->mmio_base);
  971.         }
  972.  
  973.         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
  974.         POSTING_READ(mmio);
  975.  
  976.         /* Flush the TLB for this page */
  977.         if (INTEL_INFO(dev)->gen >= 6) {
  978.                 u32 reg = RING_INSTPM(ring->mmio_base);
  979.                 I915_WRITE(reg,
  980.                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
  981.                                               INSTPM_SYNC_FLUSH));
  982.                 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
  983.                              1000))
  984.                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
  985.                                   ring->name);
  986.         }
  987. }
  988.  
  989. static int
  990. bsd_ring_flush(struct intel_ring_buffer *ring,
  991.                u32     invalidate_domains,
  992.                u32     flush_domains)
  993. {
  994.         int ret;
  995.  
  996.         ret = intel_ring_begin(ring, 2);
  997.         if (ret)
  998.                 return ret;
  999.  
  1000.         intel_ring_emit(ring, MI_FLUSH);
  1001.         intel_ring_emit(ring, MI_NOOP);
  1002.         intel_ring_advance(ring);
  1003.         return 0;
  1004. }
  1005.  
  1006. static int
  1007. i9xx_add_request(struct intel_ring_buffer *ring)
  1008. {
  1009.         int ret;
  1010.  
  1011.         ret = intel_ring_begin(ring, 4);
  1012.         if (ret)
  1013.                 return ret;
  1014.  
  1015.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  1016.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  1017.         intel_ring_emit(ring, ring->outstanding_lazy_seqno);
  1018.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  1019.         __intel_ring_advance(ring);
  1020.  
  1021.         return 0;
  1022. }
  1023.  
  1024. static bool
  1025. gen6_ring_get_irq(struct intel_ring_buffer *ring)
  1026. {
  1027.         struct drm_device *dev = ring->dev;
  1028.         drm_i915_private_t *dev_priv = dev->dev_private;
  1029.         unsigned long flags;
  1030.  
  1031.         if (!dev->irq_enabled)
  1032.                return false;
  1033.  
  1034.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1035.         if (ring->irq_refcount++ == 0) {
  1036.                 if (HAS_L3_DPF(dev) && ring->id == RCS)
  1037.                         I915_WRITE_IMR(ring,
  1038.                                        ~(ring->irq_enable_mask |
  1039.                                          GT_PARITY_ERROR(dev)));
  1040.                 else
  1041.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1042.                 ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
  1043.         }
  1044.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1045.  
  1046.     return true;
  1047. }
  1048.  
  1049. static void
  1050. gen6_ring_put_irq(struct intel_ring_buffer *ring)
  1051. {
  1052.         struct drm_device *dev = ring->dev;
  1053.         drm_i915_private_t *dev_priv = dev->dev_private;
  1054.         unsigned long flags;
  1055.  
  1056.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1057.         if (--ring->irq_refcount == 0) {
  1058.                 if (HAS_L3_DPF(dev) && ring->id == RCS)
  1059.                         I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
  1060.                 else
  1061.                         I915_WRITE_IMR(ring, ~0);
  1062.                 ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
  1063.         }
  1064.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1065. }
  1066.  
  1067. static bool
  1068. hsw_vebox_get_irq(struct intel_ring_buffer *ring)
  1069. {
  1070.         struct drm_device *dev = ring->dev;
  1071.         struct drm_i915_private *dev_priv = dev->dev_private;
  1072.         unsigned long flags;
  1073.  
  1074.         if (!dev->irq_enabled)
  1075.                 return false;
  1076.  
  1077.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1078.         if (ring->irq_refcount++ == 0) {
  1079.                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1080.                 snb_enable_pm_irq(dev_priv, ring->irq_enable_mask);
  1081.         }
  1082.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1083.  
  1084.         return true;
  1085. }
  1086.  
  1087. static void
  1088. hsw_vebox_put_irq(struct intel_ring_buffer *ring)
  1089. {
  1090.         struct drm_device *dev = ring->dev;
  1091.         struct drm_i915_private *dev_priv = dev->dev_private;
  1092.         unsigned long flags;
  1093.  
  1094.         if (!dev->irq_enabled)
  1095.                 return;
  1096.  
  1097.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1098.         if (--ring->irq_refcount == 0) {
  1099.                 I915_WRITE_IMR(ring, ~0);
  1100.                 snb_disable_pm_irq(dev_priv, ring->irq_enable_mask);
  1101.         }
  1102.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1103. }
  1104.  
  1105. static bool
  1106. gen8_ring_get_irq(struct intel_ring_buffer *ring)
  1107. {
  1108.         struct drm_device *dev = ring->dev;
  1109.         struct drm_i915_private *dev_priv = dev->dev_private;
  1110.         unsigned long flags;
  1111.  
  1112.         if (!dev->irq_enabled)
  1113.                 return false;
  1114.  
  1115.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1116.         if (ring->irq_refcount++ == 0) {
  1117.                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
  1118.                         I915_WRITE_IMR(ring,
  1119.                                        ~(ring->irq_enable_mask |
  1120.                                          GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
  1121.                 } else {
  1122.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  1123.                 }
  1124.                 POSTING_READ(RING_IMR(ring->mmio_base));
  1125.         }
  1126.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1127.  
  1128.         return true;
  1129. }
  1130.  
  1131. static void
  1132. gen8_ring_put_irq(struct intel_ring_buffer *ring)
  1133. {
  1134.         struct drm_device *dev = ring->dev;
  1135.         struct drm_i915_private *dev_priv = dev->dev_private;
  1136.         unsigned long flags;
  1137.  
  1138.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  1139.         if (--ring->irq_refcount == 0) {
  1140.                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
  1141.                         I915_WRITE_IMR(ring,
  1142.                                        ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
  1143.                 } else {
  1144.                         I915_WRITE_IMR(ring, ~0);
  1145.                 }
  1146.                 POSTING_READ(RING_IMR(ring->mmio_base));
  1147.         }
  1148.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  1149. }
  1150.  
  1151. static int
  1152. i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1153.                          u32 offset, u32 length,
  1154.                          unsigned flags)
  1155. {
  1156.         int ret;
  1157.  
  1158.         ret = intel_ring_begin(ring, 2);
  1159.         if (ret)
  1160.                 return ret;
  1161.  
  1162.         intel_ring_emit(ring,
  1163.                         MI_BATCH_BUFFER_START |
  1164.                         MI_BATCH_GTT |
  1165.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  1166.         intel_ring_emit(ring, offset);
  1167.         intel_ring_advance(ring);
  1168.  
  1169.         return 0;
  1170. }
  1171.  
  1172. /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
  1173. #define I830_BATCH_LIMIT (256*1024)
  1174. static int
  1175. i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1176.                                 u32 offset, u32 len,
  1177.                                 unsigned flags)
  1178. {
  1179.         int ret;
  1180.  
  1181.         if (flags & I915_DISPATCH_PINNED) {
  1182.                 ret = intel_ring_begin(ring, 4);
  1183.                 if (ret)
  1184.                         return ret;
  1185.  
  1186.                 intel_ring_emit(ring, MI_BATCH_BUFFER);
  1187.                 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1188.                 intel_ring_emit(ring, offset + len - 8);
  1189.                 intel_ring_emit(ring, MI_NOOP);
  1190.                 intel_ring_advance(ring);
  1191.         } else {
  1192.                 u32 cs_offset = ring->scratch.gtt_offset;
  1193.  
  1194.                 if (len > I830_BATCH_LIMIT)
  1195.                         return -ENOSPC;
  1196.  
  1197.                 ret = intel_ring_begin(ring, 9+3);
  1198.                 if (ret)
  1199.                         return ret;
  1200.                 /* Blit the batch (which has now all relocs applied) to the stable batch
  1201.                  * scratch bo area (so that the CS never stumbles over its tlb
  1202.                  * invalidation bug) ... */
  1203.                 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
  1204.                                 XY_SRC_COPY_BLT_WRITE_ALPHA |
  1205.                                 XY_SRC_COPY_BLT_WRITE_RGB);
  1206.                 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
  1207.                 intel_ring_emit(ring, 0);
  1208.                 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
  1209.                 intel_ring_emit(ring, cs_offset);
  1210.                 intel_ring_emit(ring, 0);
  1211.                 intel_ring_emit(ring, 4096);
  1212.                 intel_ring_emit(ring, offset);
  1213.                 intel_ring_emit(ring, MI_FLUSH);
  1214.  
  1215.                 /* ... and execute it. */
  1216.                 intel_ring_emit(ring, MI_BATCH_BUFFER);
  1217.                 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1218.                 intel_ring_emit(ring, cs_offset + len - 8);
  1219.         intel_ring_advance(ring);
  1220.         }
  1221.  
  1222.         return 0;
  1223. }
  1224.  
  1225. static int
  1226. i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1227.                          u32 offset, u32 len,
  1228.                          unsigned flags)
  1229. {
  1230.         int ret;
  1231.  
  1232.                 ret = intel_ring_begin(ring, 2);
  1233.                 if (ret)
  1234.                         return ret;
  1235.  
  1236.         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
  1237.         intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
  1238.         intel_ring_advance(ring);
  1239.  
  1240.         return 0;
  1241. }
  1242.  
  1243. static void cleanup_status_page(struct intel_ring_buffer *ring)
  1244. {
  1245.         struct drm_i915_gem_object *obj;
  1246.  
  1247.         obj = ring->status_page.obj;
  1248.         if (obj == NULL)
  1249.                 return;
  1250.  
  1251. //   kunmap(sg_page(obj->pages->sgl));
  1252.     i915_gem_object_unpin(obj);
  1253.         drm_gem_object_unreference(&obj->base);
  1254.         ring->status_page.obj = NULL;
  1255. }
  1256.  
  1257. static int init_status_page(struct intel_ring_buffer *ring)
  1258. {
  1259.         struct drm_device *dev = ring->dev;
  1260.         struct drm_i915_gem_object *obj;
  1261.         int ret;
  1262.  
  1263.         obj = i915_gem_alloc_object(dev, 4096);
  1264.         if (obj == NULL) {
  1265.                 DRM_ERROR("Failed to allocate status page\n");
  1266.                 ret = -ENOMEM;
  1267.                 goto err;
  1268.         }
  1269.  
  1270.         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  1271.  
  1272.         ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false);
  1273.         if (ret != 0) {
  1274.                 goto err_unref;
  1275.         }
  1276.  
  1277.         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
  1278.     ring->status_page.page_addr = (void*)MapIoMem((addr_t)sg_page(obj->pages->sgl),4096,PG_SW|0x100);
  1279.         if (ring->status_page.page_addr == NULL) {
  1280.                 ret = -ENOMEM;
  1281.                 goto err_unpin;
  1282.         }
  1283.         ring->status_page.obj = obj;
  1284.         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1285.  
  1286.         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  1287.                         ring->name, ring->status_page.gfx_addr);
  1288.  
  1289.         return 0;
  1290.  
  1291. err_unpin:
  1292.         i915_gem_object_unpin(obj);
  1293. err_unref:
  1294.         drm_gem_object_unreference(&obj->base);
  1295. err:
  1296.         return ret;
  1297. }
  1298.  
  1299. static int init_phys_status_page(struct intel_ring_buffer *ring)
  1300. {
  1301.     struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1302.  
  1303.     if (!dev_priv->status_page_dmah) {
  1304.         dev_priv->status_page_dmah =
  1305.             drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
  1306.         if (!dev_priv->status_page_dmah)
  1307.             return -ENOMEM;
  1308.     }
  1309.  
  1310.     ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
  1311.     memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1312.  
  1313.     return 0;
  1314. }
  1315.  
  1316. static int intel_init_ring_buffer(struct drm_device *dev,
  1317.                            struct intel_ring_buffer *ring)
  1318. {
  1319.         struct drm_i915_gem_object *obj;
  1320.         struct drm_i915_private *dev_priv = dev->dev_private;
  1321.         int ret;
  1322.  
  1323.         ring->dev = dev;
  1324.         INIT_LIST_HEAD(&ring->active_list);
  1325.         INIT_LIST_HEAD(&ring->request_list);
  1326.         ring->size = 32 * PAGE_SIZE;
  1327.         memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
  1328.  
  1329.         init_waitqueue_head(&ring->irq_queue);
  1330.  
  1331.         if (I915_NEED_GFX_HWS(dev)) {
  1332.        ret = init_status_page(ring);
  1333.        if (ret)
  1334.            return ret;
  1335.         } else {
  1336.                 BUG_ON(ring->id != RCS);
  1337.                 ret = init_phys_status_page(ring);
  1338.                 if (ret)
  1339.                         return ret;
  1340.         }
  1341.  
  1342.         obj = NULL;
  1343.         if (!HAS_LLC(dev))
  1344.                 obj = i915_gem_object_create_stolen(dev, ring->size);
  1345.         if (obj == NULL)
  1346.         obj = i915_gem_alloc_object(dev, ring->size);
  1347.         if (obj == NULL) {
  1348.                 DRM_ERROR("Failed to allocate ringbuffer\n");
  1349.                 ret = -ENOMEM;
  1350.                 goto err_hws;
  1351.         }
  1352.  
  1353.         ring->obj = obj;
  1354.  
  1355.         ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, true, false);
  1356.         if (ret)
  1357.                 goto err_unref;
  1358.  
  1359.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  1360.         if (ret)
  1361.                 goto err_unpin;
  1362.  
  1363.         ring->virtual_start =
  1364.                 ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
  1365.                            ring->size);
  1366.         if (ring->virtual_start == NULL) {
  1367.                 DRM_ERROR("Failed to map ringbuffer.\n");
  1368.                 ret = -EINVAL;
  1369.                 goto err_unpin;
  1370.         }
  1371.  
  1372.         ret = ring->init(ring);
  1373.         if (ret)
  1374.                 goto err_unmap;
  1375.  
  1376.         /* Workaround an erratum on the i830 which causes a hang if
  1377.          * the TAIL pointer points to within the last 2 cachelines
  1378.          * of the buffer.
  1379.          */
  1380.         ring->effective_size = ring->size;
  1381.         if (IS_I830(ring->dev) || IS_845G(ring->dev))
  1382.                 ring->effective_size -= 128;
  1383.  
  1384.         return 0;
  1385.  
  1386. err_unmap:
  1387.         iounmap(ring->virtual_start);
  1388. err_unpin:
  1389.         i915_gem_object_unpin(obj);
  1390. err_unref:
  1391.         drm_gem_object_unreference(&obj->base);
  1392.         ring->obj = NULL;
  1393. err_hws:
  1394. //   cleanup_status_page(ring);
  1395.         return ret;
  1396. }
  1397.  
  1398. void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
  1399. {
  1400.         struct drm_i915_private *dev_priv;
  1401.         int ret;
  1402.  
  1403.         if (ring->obj == NULL)
  1404.                 return;
  1405.  
  1406.         /* Disable the ring buffer. The ring must be idle at this point */
  1407.         dev_priv = ring->dev->dev_private;
  1408.         ret = intel_ring_idle(ring);
  1409.         if (ret && !i915_reset_in_progress(&dev_priv->gpu_error))
  1410.                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
  1411.                           ring->name, ret);
  1412.  
  1413.         I915_WRITE_CTL(ring, 0);
  1414.  
  1415.         iounmap(ring->virtual_start);
  1416.  
  1417.     i915_gem_object_unpin(ring->obj);
  1418.         drm_gem_object_unreference(&ring->obj->base);
  1419.         ring->obj = NULL;
  1420.         ring->preallocated_lazy_request = NULL;
  1421.         ring->outstanding_lazy_seqno = 0;
  1422.  
  1423.         if (ring->cleanup)
  1424.                 ring->cleanup(ring);
  1425.  
  1426. //   cleanup_status_page(ring);
  1427. }
  1428.  
  1429. static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1430. {
  1431.         int ret;
  1432.  
  1433.         ret = i915_wait_seqno(ring, seqno);
  1434.         if (!ret)
  1435.                 i915_gem_retire_requests_ring(ring);
  1436.  
  1437.         return ret;
  1438. }
  1439.  
  1440. static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
  1441. {
  1442.         struct drm_i915_gem_request *request;
  1443.         u32 seqno = 0;
  1444.         int ret;
  1445.  
  1446.         i915_gem_retire_requests_ring(ring);
  1447.  
  1448.         if (ring->last_retired_head != -1) {
  1449.                 ring->head = ring->last_retired_head;
  1450.                 ring->last_retired_head = -1;
  1451.                 ring->space = ring_space(ring);
  1452.                 if (ring->space >= n)
  1453.                         return 0;
  1454.         }
  1455.  
  1456.         list_for_each_entry(request, &ring->request_list, list) {
  1457.                 int space;
  1458.  
  1459.                 if (request->tail == -1)
  1460.                         continue;
  1461.  
  1462.                 space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
  1463.                 if (space < 0)
  1464.                         space += ring->size;
  1465.                 if (space >= n) {
  1466.                         seqno = request->seqno;
  1467.                         break;
  1468.                 }
  1469.  
  1470.                 /* Consume this request in case we need more space than
  1471.                  * is available and so need to prevent a race between
  1472.                  * updating last_retired_head and direct reads of
  1473.                  * I915_RING_HEAD. It also provides a nice sanity check.
  1474.                  */
  1475.                 request->tail = -1;
  1476.         }
  1477.  
  1478.         if (seqno == 0)
  1479.                 return -ENOSPC;
  1480.  
  1481.         ret = intel_ring_wait_seqno(ring, seqno);
  1482.         if (ret)
  1483.                 return ret;
  1484.  
  1485.         if (WARN_ON(ring->last_retired_head == -1))
  1486.                 return -ENOSPC;
  1487.  
  1488.         ring->head = ring->last_retired_head;
  1489.         ring->last_retired_head = -1;
  1490.         ring->space = ring_space(ring);
  1491.         if (WARN_ON(ring->space < n))
  1492.                 return -ENOSPC;
  1493.  
  1494.         return 0;
  1495. }
  1496.  
  1497. static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
  1498. {
  1499.         struct drm_device *dev = ring->dev;
  1500.         struct drm_i915_private *dev_priv = dev->dev_private;
  1501.         unsigned long end;
  1502.         int ret;
  1503.  
  1504.         ret = intel_ring_wait_request(ring, n);
  1505.         if (ret != -ENOSPC)
  1506.                 return ret;
  1507.  
  1508.         /* force the tail write in case we have been skipping them */
  1509.         __intel_ring_advance(ring);
  1510.  
  1511.         trace_i915_ring_wait_begin(ring);
  1512.         /* With GEM the hangcheck timer should kick us out of the loop,
  1513.          * leaving it early runs the risk of corrupting GEM state (due
  1514.          * to running on almost untested codepaths). But on resume
  1515.          * timers don't work yet, so prevent a complete hang in that
  1516.          * case by choosing an insanely large timeout. */
  1517.         end = GetTimerTicks() + 60 * HZ;
  1518.  
  1519.         do {
  1520.                 ring->head = I915_READ_HEAD(ring);
  1521.                 ring->space = ring_space(ring);
  1522.                 if (ring->space >= n) {
  1523.                         trace_i915_ring_wait_end(ring);
  1524.                         return 0;
  1525.                 }
  1526.  
  1527.                 msleep(1);
  1528.  
  1529.                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1530.                                            dev_priv->mm.interruptible);
  1531.                 if (ret)
  1532.                         return ret;
  1533.     } while (!time_after(GetTimerTicks(), end));
  1534.         trace_i915_ring_wait_end(ring);
  1535.         return -EBUSY;
  1536. }
  1537.  
  1538. static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
  1539. {
  1540.         uint32_t __iomem *virt;
  1541.         int rem = ring->size - ring->tail;
  1542.  
  1543.         if (ring->space < rem) {
  1544.                 int ret = ring_wait_for_space(ring, rem);
  1545.                 if (ret)
  1546.                         return ret;
  1547.         }
  1548.  
  1549.         virt = ring->virtual_start + ring->tail;
  1550.         rem /= 4;
  1551.         while (rem--)
  1552.                 iowrite32(MI_NOOP, virt++);
  1553.  
  1554.         ring->tail = 0;
  1555.         ring->space = ring_space(ring);
  1556.  
  1557.         return 0;
  1558. }
  1559.  
  1560. int intel_ring_idle(struct intel_ring_buffer *ring)
  1561. {
  1562.         u32 seqno;
  1563.         int ret;
  1564.  
  1565.         /* We need to add any requests required to flush the objects and ring */
  1566.         if (ring->outstanding_lazy_seqno) {
  1567.                 ret = i915_add_request(ring, NULL);
  1568.                 if (ret)
  1569.                         return ret;
  1570.         }
  1571.  
  1572.         /* Wait upon the last request to be completed */
  1573.         if (list_empty(&ring->request_list))
  1574.                 return 0;
  1575.  
  1576.         seqno = list_entry(ring->request_list.prev,
  1577.                            struct drm_i915_gem_request,
  1578.                            list)->seqno;
  1579.  
  1580.         return i915_wait_seqno(ring, seqno);
  1581. }
  1582.  
  1583. static int
  1584. intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
  1585. {
  1586.         if (ring->outstanding_lazy_seqno)
  1587.                 return 0;
  1588.  
  1589.         if (ring->preallocated_lazy_request == NULL) {
  1590.                 struct drm_i915_gem_request *request;
  1591.  
  1592.                 request = kmalloc(sizeof(*request), GFP_KERNEL);
  1593.                 if (request == NULL)
  1594.                         return -ENOMEM;
  1595.  
  1596.                 ring->preallocated_lazy_request = request;
  1597.         }
  1598.  
  1599.         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
  1600. }
  1601.  
  1602. static int __intel_ring_prepare(struct intel_ring_buffer *ring,
  1603.                               int bytes)
  1604. {
  1605.         int ret;
  1606.  
  1607.         if (unlikely(ring->tail + bytes > ring->effective_size)) {
  1608.                 ret = intel_wrap_ring_buffer(ring);
  1609.                 if (unlikely(ret))
  1610.                         return ret;
  1611.         }
  1612.  
  1613.         if (unlikely(ring->space < bytes)) {
  1614.                 ret = ring_wait_for_space(ring, bytes);
  1615.                 if (unlikely(ret))
  1616.                         return ret;
  1617.         }
  1618.  
  1619.         return 0;
  1620. }
  1621.  
  1622. int intel_ring_begin(struct intel_ring_buffer *ring,
  1623.                      int num_dwords)
  1624. {
  1625.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1626.         int ret;
  1627.  
  1628.         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
  1629.                                    dev_priv->mm.interruptible);
  1630.         if (ret)
  1631.                 return ret;
  1632.  
  1633.         ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
  1634.         if (ret)
  1635.                 return ret;
  1636.  
  1637.         /* Preallocate the olr before touching the ring */
  1638.         ret = intel_ring_alloc_seqno(ring);
  1639.         if (ret)
  1640.                 return ret;
  1641.  
  1642.         ring->space -= num_dwords * sizeof(uint32_t);
  1643.         return 0;
  1644. }
  1645.  
  1646. void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1647. {
  1648.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1649.  
  1650.         BUG_ON(ring->outstanding_lazy_seqno);
  1651.  
  1652.         if (INTEL_INFO(ring->dev)->gen >= 6) {
  1653.                 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
  1654.                 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
  1655.                 if (HAS_VEBOX(ring->dev))
  1656.                         I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
  1657.         }
  1658.  
  1659.         ring->set_seqno(ring, seqno);
  1660.         ring->hangcheck.seqno = seqno;
  1661. }
  1662.  
  1663. static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
  1664.                                      u32 value)
  1665. {
  1666.        drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1667.  
  1668.        /* Every tail move must follow the sequence below */
  1669.  
  1670.         /* Disable notification that the ring is IDLE. The GT
  1671.          * will then assume that it is busy and bring it out of rc6.
  1672.          */
  1673.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1674.                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1675.  
  1676.         /* Clear the context id. Here be magic! */
  1677.         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
  1678.  
  1679.         /* Wait for the ring not to be idle, i.e. for it to wake up. */
  1680.        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
  1681.                       GEN6_BSD_SLEEP_INDICATOR) == 0,
  1682.                        50))
  1683.                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
  1684.  
  1685.         /* Now that the ring is fully powered up, update the tail */
  1686.        I915_WRITE_TAIL(ring, value);
  1687.         POSTING_READ(RING_TAIL(ring->mmio_base));
  1688.  
  1689.         /* Let the ring send IDLE messages to the GT again,
  1690.          * and so let it sleep to conserve power when idle.
  1691.          */
  1692.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1693.                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1694. }
  1695.  
  1696. static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
  1697.                            u32 invalidate, u32 flush)
  1698. {
  1699.         uint32_t cmd;
  1700.         int ret;
  1701.  
  1702.         ret = intel_ring_begin(ring, 4);
  1703.         if (ret)
  1704.                 return ret;
  1705.  
  1706.         cmd = MI_FLUSH_DW;
  1707.         if (INTEL_INFO(ring->dev)->gen >= 8)
  1708.                 cmd += 1;
  1709.         /*
  1710.          * Bspec vol 1c.5 - video engine command streamer:
  1711.          * "If ENABLED, all TLBs will be invalidated once the flush
  1712.          * operation is complete. This bit is only valid when the
  1713.          * Post-Sync Operation field is a value of 1h or 3h."
  1714.          */
  1715.         if (invalidate & I915_GEM_GPU_DOMAINS)
  1716.                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
  1717.                         MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
  1718.         intel_ring_emit(ring, cmd);
  1719.         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  1720.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  1721.                 intel_ring_emit(ring, 0); /* upper addr */
  1722.                 intel_ring_emit(ring, 0); /* value */
  1723.         } else  {
  1724.         intel_ring_emit(ring, 0);
  1725.         intel_ring_emit(ring, MI_NOOP);
  1726.         }
  1727.         intel_ring_advance(ring);
  1728.         return 0;
  1729. }
  1730.  
  1731. static int
  1732. gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1733.                               u32 offset, u32 len,
  1734.                               unsigned flags)
  1735. {
  1736.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1737.         bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
  1738.                 !(flags & I915_DISPATCH_SECURE);
  1739.         int ret;
  1740.  
  1741.         ret = intel_ring_begin(ring, 4);
  1742.         if (ret)
  1743.                 return ret;
  1744.  
  1745.         /* FIXME(BDW): Address space and security selectors. */
  1746.         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
  1747.         intel_ring_emit(ring, offset);
  1748.         intel_ring_emit(ring, 0);
  1749.         intel_ring_emit(ring, MI_NOOP);
  1750.         intel_ring_advance(ring);
  1751.  
  1752.         return 0;
  1753. }
  1754.  
  1755. static int
  1756. hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1757.                               u32 offset, u32 len,
  1758.                               unsigned flags)
  1759. {
  1760.         int ret;
  1761.  
  1762.         ret = intel_ring_begin(ring, 2);
  1763.         if (ret)
  1764.                 return ret;
  1765.  
  1766.         intel_ring_emit(ring,
  1767.                         MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
  1768.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
  1769.         /* bit0-7 is the length on GEN6+ */
  1770.         intel_ring_emit(ring, offset);
  1771.         intel_ring_advance(ring);
  1772.  
  1773.         return 0;
  1774. }
  1775.  
  1776. static int
  1777. gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1778.                               u32 offset, u32 len,
  1779.                               unsigned flags)
  1780. {
  1781.        int ret;
  1782.  
  1783.        ret = intel_ring_begin(ring, 2);
  1784.        if (ret)
  1785.                return ret;
  1786.  
  1787.         intel_ring_emit(ring,
  1788.                         MI_BATCH_BUFFER_START |
  1789.                         (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
  1790.        /* bit0-7 is the length on GEN6+ */
  1791.        intel_ring_emit(ring, offset);
  1792.        intel_ring_advance(ring);
  1793.  
  1794.        return 0;
  1795. }
  1796.  
  1797. /* Blitter support (SandyBridge+) */
  1798.  
  1799. static int gen6_ring_flush(struct intel_ring_buffer *ring,
  1800.                           u32 invalidate, u32 flush)
  1801. {
  1802.         struct drm_device *dev = ring->dev;
  1803.         uint32_t cmd;
  1804.         int ret;
  1805.  
  1806.         ret = intel_ring_begin(ring, 4);
  1807.         if (ret)
  1808.                 return ret;
  1809.  
  1810.         cmd = MI_FLUSH_DW;
  1811.         if (INTEL_INFO(ring->dev)->gen >= 8)
  1812.                 cmd += 1;
  1813.         /*
  1814.          * Bspec vol 1c.3 - blitter engine command streamer:
  1815.          * "If ENABLED, all TLBs will be invalidated once the flush
  1816.          * operation is complete. This bit is only valid when the
  1817.          * Post-Sync Operation field is a value of 1h or 3h."
  1818.          */
  1819.         if (invalidate & I915_GEM_DOMAIN_RENDER)
  1820.                 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
  1821.                         MI_FLUSH_DW_OP_STOREDW;
  1822.         intel_ring_emit(ring, cmd);
  1823.         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
  1824.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  1825.                 intel_ring_emit(ring, 0); /* upper addr */
  1826.                 intel_ring_emit(ring, 0); /* value */
  1827.         } else  {
  1828.         intel_ring_emit(ring, 0);
  1829.         intel_ring_emit(ring, MI_NOOP);
  1830.         }
  1831.         intel_ring_advance(ring);
  1832.  
  1833.         if (IS_GEN7(dev) && !invalidate && flush)
  1834.                 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
  1835.  
  1836.         return 0;
  1837. }
  1838.  
  1839. int intel_init_render_ring_buffer(struct drm_device *dev)
  1840. {
  1841.         drm_i915_private_t *dev_priv = dev->dev_private;
  1842.         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1843.  
  1844.         ring->name = "render ring";
  1845.         ring->id = RCS;
  1846.         ring->mmio_base = RENDER_RING_BASE;
  1847.  
  1848.         if (INTEL_INFO(dev)->gen >= 6) {
  1849.        ring->add_request = gen6_add_request;
  1850.                 ring->flush = gen7_render_ring_flush;
  1851.                 if (INTEL_INFO(dev)->gen == 6)
  1852.                 ring->flush = gen6_render_ring_flush;
  1853.                 if (INTEL_INFO(dev)->gen >= 8) {
  1854.                         ring->flush = gen8_render_ring_flush;
  1855.                         ring->irq_get = gen8_ring_get_irq;
  1856.                         ring->irq_put = gen8_ring_put_irq;
  1857.                 } else {
  1858.                 ring->irq_get = gen6_ring_get_irq;
  1859.                 ring->irq_put = gen6_ring_put_irq;
  1860.                 }
  1861.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
  1862.                 ring->get_seqno = gen6_ring_get_seqno;
  1863.                 ring->set_seqno = ring_set_seqno;
  1864.                 ring->sync_to = gen6_ring_sync;
  1865.                 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
  1866.                 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
  1867.                 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
  1868.                 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE;
  1869.                 ring->signal_mbox[RCS] = GEN6_NOSYNC;
  1870.                 ring->signal_mbox[VCS] = GEN6_VRSYNC;
  1871.                 ring->signal_mbox[BCS] = GEN6_BRSYNC;
  1872.                 ring->signal_mbox[VECS] = GEN6_VERSYNC;
  1873.         } else if (IS_GEN5(dev)) {
  1874.        ring->add_request = pc_render_add_request;
  1875.                 ring->flush = gen4_render_ring_flush;
  1876.                 ring->get_seqno = pc_render_get_seqno;
  1877.                 ring->set_seqno = pc_render_set_seqno;
  1878.                 ring->irq_get = gen5_ring_get_irq;
  1879.                 ring->irq_put = gen5_ring_put_irq;
  1880.                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
  1881.                                         GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
  1882.         } else {
  1883.                 ring->add_request = i9xx_add_request;
  1884.                 if (INTEL_INFO(dev)->gen < 4)
  1885.                         ring->flush = gen2_render_ring_flush;
  1886.                 else
  1887.                         ring->flush = gen4_render_ring_flush;
  1888.                 ring->get_seqno = ring_get_seqno;
  1889.                 ring->set_seqno = ring_set_seqno;
  1890.                 if (IS_GEN2(dev)) {
  1891.                         ring->irq_get = i8xx_ring_get_irq;
  1892.                         ring->irq_put = i8xx_ring_put_irq;
  1893.                 } else {
  1894.                         ring->irq_get = i9xx_ring_get_irq;
  1895.                         ring->irq_put = i9xx_ring_put_irq;
  1896.                 }
  1897.                 ring->irq_enable_mask = I915_USER_INTERRUPT;
  1898.         }
  1899.         ring->write_tail = ring_write_tail;
  1900.         if (IS_HASWELL(dev))
  1901.                 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
  1902.         else if (IS_GEN8(dev))
  1903.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  1904.         else if (INTEL_INFO(dev)->gen >= 6)
  1905.                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1906.         else if (INTEL_INFO(dev)->gen >= 4)
  1907.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1908.         else if (IS_I830(dev) || IS_845G(dev))
  1909.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  1910.         else
  1911.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  1912.         ring->init = init_render_ring;
  1913.         ring->cleanup = render_ring_cleanup;
  1914.  
  1915.         /* Workaround batchbuffer to combat CS tlb bug. */
  1916.         if (HAS_BROKEN_CS_TLB(dev)) {
  1917.                 struct drm_i915_gem_object *obj;
  1918.                 int ret;
  1919.  
  1920.                 obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
  1921.                 if (obj == NULL) {
  1922.                         DRM_ERROR("Failed to allocate batch bo\n");
  1923.                         return -ENOMEM;
  1924.                 }
  1925.  
  1926.                 ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
  1927.                 if (ret != 0) {
  1928.                         drm_gem_object_unreference(&obj->base);
  1929.                         DRM_ERROR("Failed to ping batch bo\n");
  1930.                         return ret;
  1931.                 }
  1932.  
  1933.                 ring->scratch.obj = obj;
  1934.                 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
  1935.         }
  1936.  
  1937.         return intel_init_ring_buffer(dev, ring);
  1938. }
  1939.  
  1940. #if 0
  1941. int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
  1942. {
  1943.         drm_i915_private_t *dev_priv = dev->dev_private;
  1944.         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1945.         int ret;
  1946.  
  1947.         ring->name = "render ring";
  1948.         ring->id = RCS;
  1949.         ring->mmio_base = RENDER_RING_BASE;
  1950.  
  1951.         if (INTEL_INFO(dev)->gen >= 6) {
  1952.                 /* non-kms not supported on gen6+ */
  1953.                 return -ENODEV;
  1954.         }
  1955.  
  1956.         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
  1957.          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
  1958.          * the special gen5 functions. */
  1959.         ring->add_request = i9xx_add_request;
  1960.         if (INTEL_INFO(dev)->gen < 4)
  1961.                 ring->flush = gen2_render_ring_flush;
  1962.         else
  1963.                 ring->flush = gen4_render_ring_flush;
  1964.         ring->get_seqno = ring_get_seqno;
  1965.         ring->set_seqno = ring_set_seqno;
  1966.         if (IS_GEN2(dev)) {
  1967.                 ring->irq_get = i8xx_ring_get_irq;
  1968.                 ring->irq_put = i8xx_ring_put_irq;
  1969.         } else {
  1970.                 ring->irq_get = i9xx_ring_get_irq;
  1971.                 ring->irq_put = i9xx_ring_put_irq;
  1972.         }
  1973.         ring->irq_enable_mask = I915_USER_INTERRUPT;
  1974.         ring->write_tail = ring_write_tail;
  1975.         if (INTEL_INFO(dev)->gen >= 4)
  1976.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1977.         else if (IS_I830(dev) || IS_845G(dev))
  1978.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  1979.         else
  1980.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  1981.         ring->init = init_render_ring;
  1982.         ring->cleanup = render_ring_cleanup;
  1983.  
  1984.         ring->dev = dev;
  1985.         INIT_LIST_HEAD(&ring->active_list);
  1986.         INIT_LIST_HEAD(&ring->request_list);
  1987.  
  1988.         ring->size = size;
  1989.         ring->effective_size = ring->size;
  1990.         if (IS_I830(ring->dev) || IS_845G(ring->dev))
  1991.                 ring->effective_size -= 128;
  1992.  
  1993.         ring->virtual_start = ioremap_wc(start, size);
  1994.         if (ring->virtual_start == NULL) {
  1995.                 DRM_ERROR("can not ioremap virtual address for"
  1996.                           " ring buffer\n");
  1997.                 return -ENOMEM;
  1998.         }
  1999.  
  2000.         if (!I915_NEED_GFX_HWS(dev)) {
  2001.                 ret = init_phys_status_page(ring);
  2002.                 if (ret)
  2003.                         return ret;
  2004.         }
  2005.  
  2006.         return 0;
  2007. }
  2008. #endif
  2009.  
  2010. int intel_init_bsd_ring_buffer(struct drm_device *dev)
  2011. {
  2012.         drm_i915_private_t *dev_priv = dev->dev_private;
  2013.         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
  2014.  
  2015.         ring->name = "bsd ring";
  2016.         ring->id = VCS;
  2017.  
  2018.         ring->write_tail = ring_write_tail;
  2019.         if (INTEL_INFO(dev)->gen >= 6) {
  2020.                 ring->mmio_base = GEN6_BSD_RING_BASE;
  2021.                 /* gen6 bsd needs a special wa for tail updates */
  2022.                 if (IS_GEN6(dev))
  2023.                         ring->write_tail = gen6_bsd_ring_write_tail;
  2024.                 ring->flush = gen6_bsd_ring_flush;
  2025.                 ring->add_request = gen6_add_request;
  2026.                 ring->get_seqno = gen6_ring_get_seqno;
  2027.                 ring->set_seqno = ring_set_seqno;
  2028.                 if (INTEL_INFO(dev)->gen >= 8) {
  2029.                         ring->irq_enable_mask =
  2030.                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
  2031.                         ring->irq_get = gen8_ring_get_irq;
  2032.                         ring->irq_put = gen8_ring_put_irq;
  2033.                         ring->dispatch_execbuffer =
  2034.                                 gen8_ring_dispatch_execbuffer;
  2035.                 } else {
  2036.                 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
  2037.                 ring->irq_get = gen6_ring_get_irq;
  2038.                 ring->irq_put = gen6_ring_put_irq;
  2039.                         ring->dispatch_execbuffer =
  2040.                                 gen6_ring_dispatch_execbuffer;
  2041.                 }
  2042.                 ring->sync_to = gen6_ring_sync;
  2043.                 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
  2044.                 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
  2045.                 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
  2046.                 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE;
  2047.                 ring->signal_mbox[RCS] = GEN6_RVSYNC;
  2048.                 ring->signal_mbox[VCS] = GEN6_NOSYNC;
  2049.                 ring->signal_mbox[BCS] = GEN6_BVSYNC;
  2050.                 ring->signal_mbox[VECS] = GEN6_VEVSYNC;
  2051.         } else {
  2052.                 ring->mmio_base = BSD_RING_BASE;
  2053.                 ring->flush = bsd_ring_flush;
  2054.                 ring->add_request = i9xx_add_request;
  2055.                 ring->get_seqno = ring_get_seqno;
  2056.                 ring->set_seqno = ring_set_seqno;
  2057.                 if (IS_GEN5(dev)) {
  2058.                         ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
  2059.                         ring->irq_get = gen5_ring_get_irq;
  2060.                         ring->irq_put = gen5_ring_put_irq;
  2061.                 } else {
  2062.                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
  2063.                         ring->irq_get = i9xx_ring_get_irq;
  2064.                         ring->irq_put = i9xx_ring_put_irq;
  2065.                 }
  2066.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  2067.         }
  2068.         ring->init = init_ring_common;
  2069.  
  2070.         return intel_init_ring_buffer(dev, ring);
  2071. }
  2072.  
  2073. int intel_init_blt_ring_buffer(struct drm_device *dev)
  2074. {
  2075.         drm_i915_private_t *dev_priv = dev->dev_private;
  2076.         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
  2077.  
  2078.         ring->name = "blitter ring";
  2079.         ring->id = BCS;
  2080.  
  2081.         ring->mmio_base = BLT_RING_BASE;
  2082.         ring->write_tail = ring_write_tail;
  2083.         ring->flush = gen6_ring_flush;
  2084.         ring->add_request = gen6_add_request;
  2085.         ring->get_seqno = gen6_ring_get_seqno;
  2086.         ring->set_seqno = ring_set_seqno;
  2087.         if (INTEL_INFO(dev)->gen >= 8) {
  2088.                 ring->irq_enable_mask =
  2089.                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
  2090.                 ring->irq_get = gen8_ring_get_irq;
  2091.                 ring->irq_put = gen8_ring_put_irq;
  2092.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2093.         } else {
  2094.         ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
  2095.         ring->irq_get = gen6_ring_get_irq;
  2096.         ring->irq_put = gen6_ring_put_irq;
  2097.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2098.         }
  2099.         ring->sync_to = gen6_ring_sync;
  2100.         ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
  2101.         ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
  2102.         ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
  2103.         ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE;
  2104.         ring->signal_mbox[RCS] = GEN6_RBSYNC;
  2105.         ring->signal_mbox[VCS] = GEN6_VBSYNC;
  2106.         ring->signal_mbox[BCS] = GEN6_NOSYNC;
  2107.         ring->signal_mbox[VECS] = GEN6_VEBSYNC;
  2108.         ring->init = init_ring_common;
  2109.  
  2110.         return intel_init_ring_buffer(dev, ring);
  2111. }
  2112.  
  2113. int intel_init_vebox_ring_buffer(struct drm_device *dev)
  2114. {
  2115.         drm_i915_private_t *dev_priv = dev->dev_private;
  2116.         struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
  2117.  
  2118.         ring->name = "video enhancement ring";
  2119.         ring->id = VECS;
  2120.  
  2121.         ring->mmio_base = VEBOX_RING_BASE;
  2122.         ring->write_tail = ring_write_tail;
  2123.         ring->flush = gen6_ring_flush;
  2124.         ring->add_request = gen6_add_request;
  2125.         ring->get_seqno = gen6_ring_get_seqno;
  2126.         ring->set_seqno = ring_set_seqno;
  2127.  
  2128.         if (INTEL_INFO(dev)->gen >= 8) {
  2129.                 ring->irq_enable_mask =
  2130.                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
  2131.                 ring->irq_get = gen8_ring_get_irq;
  2132.                 ring->irq_put = gen8_ring_put_irq;
  2133.                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
  2134.         } else {
  2135.         ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
  2136.         ring->irq_get = hsw_vebox_get_irq;
  2137.         ring->irq_put = hsw_vebox_put_irq;
  2138.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  2139.         }
  2140.         ring->sync_to = gen6_ring_sync;
  2141.         ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
  2142.         ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
  2143.         ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
  2144.         ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID;
  2145.         ring->signal_mbox[RCS] = GEN6_RVESYNC;
  2146.         ring->signal_mbox[VCS] = GEN6_VVESYNC;
  2147.         ring->signal_mbox[BCS] = GEN6_BVESYNC;
  2148.         ring->signal_mbox[VECS] = GEN6_NOSYNC;
  2149.         ring->init = init_ring_common;
  2150.  
  2151.         return intel_init_ring_buffer(dev, ring);
  2152. }
  2153.  
  2154. int
  2155. intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
  2156. {
  2157.         int ret;
  2158.  
  2159.         if (!ring->gpu_caches_dirty)
  2160.                 return 0;
  2161.  
  2162.         ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
  2163.         if (ret)
  2164.                 return ret;
  2165.  
  2166.         trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
  2167.  
  2168.         ring->gpu_caches_dirty = false;
  2169.         return 0;
  2170. }
  2171.  
  2172. int
  2173. intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
  2174. {
  2175.         uint32_t flush_domains;
  2176.         int ret;
  2177.  
  2178.         flush_domains = 0;
  2179.         if (ring->gpu_caches_dirty)
  2180.                 flush_domains = I915_GEM_GPU_DOMAINS;
  2181.  
  2182.         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  2183.         if (ret)
  2184.                 return ret;
  2185.  
  2186.         trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  2187.  
  2188.         ring->gpu_caches_dirty = false;
  2189.         return 0;
  2190. }
  2191.