Subversion Repositories Kolibri OS

Rev

Rev 3031 | Rev 3243 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /*
  2.  * Copyright © 2008-2010 Intel Corporation
  3.  *
  4.  * Permission is hereby granted, free of charge, to any person obtaining a
  5.  * copy of this software and associated documentation files (the "Software"),
  6.  * to deal in the Software without restriction, including without limitation
  7.  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8.  * and/or sell copies of the Software, and to permit persons to whom the
  9.  * Software is furnished to do so, subject to the following conditions:
  10.  *
  11.  * The above copyright notice and this permission notice (including the next
  12.  * paragraph) shall be included in all copies or substantial portions of the
  13.  * Software.
  14.  *
  15.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16.  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17.  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18.  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19.  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20.  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21.  * IN THE SOFTWARE.
  22.  *
  23.  * Authors:
  24.  *    Eric Anholt <eric@anholt.net>
  25.  *    Zou Nan hai <nanhai.zou@intel.com>
  26.  *    Xiang Hai hao<haihao.xiang@intel.com>
  27.  *
  28.  */
  29. #define iowrite32(v, addr)      writel((v), (addr))
  30. #define ioread32(addr)          readl(addr)
  31.  
  32. #include <drm/drmP.h>
  33. #include "i915_drv.h"
  34. #include <drm/i915_drm.h>
  35. #include "i915_trace.h"
  36. #include "intel_drv.h"
  37.  
  38. /*
  39.  * 965+ support PIPE_CONTROL commands, which provide finer grained control
  40.  * over cache flushing.
  41.  */
  42. struct pipe_control {
  43.         struct drm_i915_gem_object *obj;
  44.         volatile u32 *cpu_page;
  45.         u32 gtt_offset;
  46. };
  47.  
  48. static inline int ring_space(struct intel_ring_buffer *ring)
  49. {
  50.         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  51.         if (space < 0)
  52.                 space += ring->size;
  53.         return space;
  54. }
  55.  
  56. static int
  57. gen2_render_ring_flush(struct intel_ring_buffer *ring,
  58.                        u32      invalidate_domains,
  59.                        u32      flush_domains)
  60. {
  61.         u32 cmd;
  62.         int ret;
  63.  
  64.         cmd = MI_FLUSH;
  65.         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  66.                 cmd |= MI_NO_WRITE_FLUSH;
  67.  
  68.         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  69.                 cmd |= MI_READ_FLUSH;
  70.  
  71.         ret = intel_ring_begin(ring, 2);
  72.         if (ret)
  73.                 return ret;
  74.  
  75.         intel_ring_emit(ring, cmd);
  76.         intel_ring_emit(ring, MI_NOOP);
  77.         intel_ring_advance(ring);
  78.  
  79.         return 0;
  80. }
  81.  
  82. static int
  83. gen4_render_ring_flush(struct intel_ring_buffer *ring,
  84.                   u32   invalidate_domains,
  85.                   u32   flush_domains)
  86. {
  87.         struct drm_device *dev = ring->dev;
  88.         u32 cmd;
  89.         int ret;
  90.  
  91.         /*
  92.          * read/write caches:
  93.          *
  94.          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  95.          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  96.          * also flushed at 2d versus 3d pipeline switches.
  97.          *
  98.          * read-only caches:
  99.          *
  100.          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  101.          * MI_READ_FLUSH is set, and is always flushed on 965.
  102.          *
  103.          * I915_GEM_DOMAIN_COMMAND may not exist?
  104.          *
  105.          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  106.          * invalidated when MI_EXE_FLUSH is set.
  107.          *
  108.          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  109.          * invalidated with every MI_FLUSH.
  110.          *
  111.          * TLBs:
  112.          *
  113.          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  114.          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  115.          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  116.          * are flushed at any MI_FLUSH.
  117.          */
  118.  
  119.         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  120.         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
  121.                 cmd &= ~MI_NO_WRITE_FLUSH;
  122.         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  123.                 cmd |= MI_EXE_FLUSH;
  124.  
  125.         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
  126.             (IS_G4X(dev) || IS_GEN5(dev)))
  127.                 cmd |= MI_INVALIDATE_ISP;
  128.  
  129.         ret = intel_ring_begin(ring, 2);
  130.         if (ret)
  131.                 return ret;
  132.  
  133.         intel_ring_emit(ring, cmd);
  134.         intel_ring_emit(ring, MI_NOOP);
  135.         intel_ring_advance(ring);
  136.  
  137.         return 0;
  138. }
  139.  
  140. /**
  141.  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
  142.  * implementing two workarounds on gen6.  From section 1.4.7.1
  143.  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
  144.  *
  145.  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
  146.  * produced by non-pipelined state commands), software needs to first
  147.  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
  148.  * 0.
  149.  *
  150.  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
  151.  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
  152.  *
  153.  * And the workaround for these two requires this workaround first:
  154.  *
  155.  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
  156.  * BEFORE the pipe-control with a post-sync op and no write-cache
  157.  * flushes.
  158.  *
  159.  * And this last workaround is tricky because of the requirements on
  160.  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
  161.  * volume 2 part 1:
  162.  *
  163.  *     "1 of the following must also be set:
  164.  *      - Render Target Cache Flush Enable ([12] of DW1)
  165.  *      - Depth Cache Flush Enable ([0] of DW1)
  166.  *      - Stall at Pixel Scoreboard ([1] of DW1)
  167.  *      - Depth Stall ([13] of DW1)
  168.  *      - Post-Sync Operation ([13] of DW1)
  169.  *      - Notify Enable ([8] of DW1)"
  170.  *
  171.  * The cache flushes require the workaround flush that triggered this
  172.  * one, so we can't use it.  Depth stall would trigger the same.
  173.  * Post-sync nonzero is what triggered this second workaround, so we
  174.  * can't use that one either.  Notify enable is IRQs, which aren't
  175.  * really our business.  That leaves only stall at scoreboard.
  176.  */
  177. static int
  178. intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
  179. {
  180.         struct pipe_control *pc = ring->private;
  181.         u32 scratch_addr = pc->gtt_offset + 128;
  182.         int ret;
  183.  
  184.  
  185.         ret = intel_ring_begin(ring, 6);
  186.         if (ret)
  187.                 return ret;
  188.  
  189.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  190.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  191.                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
  192.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  193.         intel_ring_emit(ring, 0); /* low dword */
  194.         intel_ring_emit(ring, 0); /* high dword */
  195.         intel_ring_emit(ring, MI_NOOP);
  196.         intel_ring_advance(ring);
  197.  
  198.         ret = intel_ring_begin(ring, 6);
  199.         if (ret)
  200.                 return ret;
  201.  
  202.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
  203.         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
  204.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
  205.         intel_ring_emit(ring, 0);
  206.         intel_ring_emit(ring, 0);
  207.         intel_ring_emit(ring, MI_NOOP);
  208.         intel_ring_advance(ring);
  209.  
  210.         return 0;
  211. }
  212.  
  213. static int
  214. gen6_render_ring_flush(struct intel_ring_buffer *ring,
  215.                          u32 invalidate_domains, u32 flush_domains)
  216. {
  217.         u32 flags = 0;
  218.         struct pipe_control *pc = ring->private;
  219.         u32 scratch_addr = pc->gtt_offset + 128;
  220.         int ret;
  221.  
  222.         /* Force SNB workarounds for PIPE_CONTROL flushes */
  223.         ret = intel_emit_post_sync_nonzero_flush(ring);
  224.         if (ret)
  225.                 return ret;
  226.  
  227.         /* Just flush everything.  Experiments have shown that reducing the
  228.          * number of bits based on the write domains has little performance
  229.          * impact.
  230.          */
  231.         if (flush_domains) {
  232.                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  233.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  234.                 /*
  235.                  * Ensure that any following seqno writes only happen
  236.                  * when the render cache is indeed flushed.
  237.                  */
  238.                 flags |= PIPE_CONTROL_CS_STALL;
  239.         }
  240.         if (invalidate_domains) {
  241.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  242.                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  243.                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  244.                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  245.                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  246.                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  247.                 /*
  248.                  * TLB invalidate requires a post-sync write.
  249.                  */
  250.                 flags |= PIPE_CONTROL_QW_WRITE;
  251.         }
  252.  
  253.         ret = intel_ring_begin(ring, 4);
  254.         if (ret)
  255.                 return ret;
  256.  
  257.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  258.         intel_ring_emit(ring, flags);
  259.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
  260.         intel_ring_emit(ring, 0);
  261.         intel_ring_advance(ring);
  262.  
  263.         return 0;
  264. }
  265.  
  266. static int
  267. gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
  268. {
  269.         int ret;
  270.  
  271.         ret = intel_ring_begin(ring, 4);
  272.         if (ret)
  273.                 return ret;
  274.  
  275.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  276.         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
  277.                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
  278.         intel_ring_emit(ring, 0);
  279.         intel_ring_emit(ring, 0);
  280.         intel_ring_advance(ring);
  281.  
  282.         return 0;
  283. }
  284.  
  285. static int
  286. gen7_render_ring_flush(struct intel_ring_buffer *ring,
  287.                        u32 invalidate_domains, u32 flush_domains)
  288. {
  289.         u32 flags = 0;
  290.         struct pipe_control *pc = ring->private;
  291.         u32 scratch_addr = pc->gtt_offset + 128;
  292.         int ret;
  293.  
  294.         /*
  295.          * Ensure that any following seqno writes only happen when the render
  296.          * cache is indeed flushed.
  297.          *
  298.          * Workaround: 4th PIPE_CONTROL command (except the ones with only
  299.          * read-cache invalidate bits set) must have the CS_STALL bit set. We
  300.          * don't try to be clever and just set it unconditionally.
  301.          */
  302.         flags |= PIPE_CONTROL_CS_STALL;
  303.  
  304.         /* Just flush everything.  Experiments have shown that reducing the
  305.          * number of bits based on the write domains has little performance
  306.          * impact.
  307.          */
  308.         if (flush_domains) {
  309.         flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
  310.                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
  311.         }
  312.         if (invalidate_domains) {
  313.                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
  314.         flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
  315.         flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
  316.         flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
  317.         flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
  318.         flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
  319.                 /*
  320.                  * TLB invalidate requires a post-sync write.
  321.                  */
  322.                 flags |= PIPE_CONTROL_QW_WRITE;
  323.  
  324.                 /* Workaround: we must issue a pipe_control with CS-stall bit
  325.                  * set before a pipe_control command that has the state cache
  326.                  * invalidate bit set. */
  327.                 gen7_render_ring_cs_stall_wa(ring);
  328.         }
  329.  
  330.         ret = intel_ring_begin(ring, 4);
  331.         if (ret)
  332.                 return ret;
  333.  
  334.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
  335.         intel_ring_emit(ring, flags);
  336.         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
  337.         intel_ring_emit(ring, 0);
  338.         intel_ring_advance(ring);
  339.  
  340.         return 0;
  341. }
  342.  
  343. static void ring_write_tail(struct intel_ring_buffer *ring,
  344.                             u32 value)
  345. {
  346.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  347.         I915_WRITE_TAIL(ring, value);
  348. }
  349.  
  350. u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
  351. {
  352.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  353.         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
  354.                         RING_ACTHD(ring->mmio_base) : ACTHD;
  355.  
  356.         return I915_READ(acthd_reg);
  357. }
  358.  
  359. static int init_ring_common(struct intel_ring_buffer *ring)
  360. {
  361.         struct drm_device *dev = ring->dev;
  362.         drm_i915_private_t *dev_priv = dev->dev_private;
  363.         struct drm_i915_gem_object *obj = ring->obj;
  364.         int ret = 0;
  365.         u32 head;
  366.  
  367.         if (HAS_FORCE_WAKE(dev))
  368.                 gen6_gt_force_wake_get(dev_priv);
  369.  
  370.         /* Stop the ring if it's running. */
  371.         I915_WRITE_CTL(ring, 0);
  372.         I915_WRITE_HEAD(ring, 0);
  373.         ring->write_tail(ring, 0);
  374.  
  375.         head = I915_READ_HEAD(ring) & HEAD_ADDR;
  376.  
  377.         /* G45 ring initialization fails to reset head to zero */
  378.         if (head != 0) {
  379.                 DRM_DEBUG_KMS("%s head not reset to zero "
  380.                               "ctl %08x head %08x tail %08x start %08x\n",
  381.                               ring->name,
  382.                               I915_READ_CTL(ring),
  383.                               I915_READ_HEAD(ring),
  384.                               I915_READ_TAIL(ring),
  385.                               I915_READ_START(ring));
  386.  
  387.                 I915_WRITE_HEAD(ring, 0);
  388.  
  389.                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
  390.                         DRM_ERROR("failed to set %s head to zero "
  391.                                   "ctl %08x head %08x tail %08x start %08x\n",
  392.                                   ring->name,
  393.                                   I915_READ_CTL(ring),
  394.                                   I915_READ_HEAD(ring),
  395.                                   I915_READ_TAIL(ring),
  396.                                   I915_READ_START(ring));
  397.                 }
  398.         }
  399.  
  400.         /* Initialize the ring. This must happen _after_ we've cleared the ring
  401.          * registers with the above sequence (the readback of the HEAD registers
  402.          * also enforces ordering), otherwise the hw might lose the new ring
  403.          * register values. */
  404.         I915_WRITE_START(ring, obj->gtt_offset);
  405.         I915_WRITE_CTL(ring,
  406.                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
  407.                         | RING_VALID);
  408.  
  409.         /* If the head is still not zero, the ring is dead */
  410.         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
  411.                      I915_READ_START(ring) == obj->gtt_offset &&
  412.                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
  413.                 DRM_ERROR("%s initialization failed "
  414.                                 "ctl %08x head %08x tail %08x start %08x\n",
  415.                                 ring->name,
  416.                                 I915_READ_CTL(ring),
  417.                                 I915_READ_HEAD(ring),
  418.                                 I915_READ_TAIL(ring),
  419.                                 I915_READ_START(ring));
  420.                 ret = -EIO;
  421.                 goto out;
  422.         }
  423.  
  424.                 ring->head = I915_READ_HEAD(ring);
  425.                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
  426.                 ring->space = ring_space(ring);
  427.                 ring->last_retired_head = -1;
  428.  
  429. out:
  430.         if (HAS_FORCE_WAKE(dev))
  431.                 gen6_gt_force_wake_put(dev_priv);
  432.  
  433.         return ret;
  434. }
  435.  
  436. static int
  437. init_pipe_control(struct intel_ring_buffer *ring)
  438. {
  439.         struct pipe_control *pc;
  440.         struct drm_i915_gem_object *obj;
  441.         int ret;
  442.  
  443.         if (ring->private)
  444.                 return 0;
  445.  
  446.         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
  447.         if (!pc)
  448.                 return -ENOMEM;
  449.  
  450.         obj = i915_gem_alloc_object(ring->dev, 4096);
  451.         if (obj == NULL) {
  452.                 DRM_ERROR("Failed to allocate seqno page\n");
  453.                 ret = -ENOMEM;
  454.                 goto err;
  455.         }
  456.  
  457.         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  458.  
  459.         ret = i915_gem_object_pin(obj, 4096, true, false);
  460.         if (ret)
  461.                 goto err_unref;
  462.  
  463.         pc->gtt_offset = obj->gtt_offset;
  464.     pc->cpu_page =  (void*)MapIoMem((addr_t)obj->pages.page[0], 4096, PG_SW);
  465.         if (pc->cpu_page == NULL)
  466.                 goto err_unpin;
  467.  
  468.         pc->obj = obj;
  469.         ring->private = pc;
  470.         return 0;
  471.  
  472. err_unpin:
  473.         i915_gem_object_unpin(obj);
  474. err_unref:
  475.         drm_gem_object_unreference(&obj->base);
  476. err:
  477.         kfree(pc);
  478.         return ret;
  479. }
  480.  
  481. static void
  482. cleanup_pipe_control(struct intel_ring_buffer *ring)
  483. {
  484.         struct pipe_control *pc = ring->private;
  485.         struct drm_i915_gem_object *obj;
  486.  
  487.         if (!ring->private)
  488.                 return;
  489.  
  490.         obj = pc->obj;
  491. //      kunmap(obj->pages[0]);
  492.         i915_gem_object_unpin(obj);
  493.         drm_gem_object_unreference(&obj->base);
  494.  
  495.         kfree(pc);
  496.         ring->private = NULL;
  497. }
  498.  
  499. static int init_render_ring(struct intel_ring_buffer *ring)
  500. {
  501.         struct drm_device *dev = ring->dev;
  502.         struct drm_i915_private *dev_priv = dev->dev_private;
  503.         int ret = init_ring_common(ring);
  504.  
  505.         if (INTEL_INFO(dev)->gen > 3) {
  506.                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
  507.                 if (IS_GEN7(dev))
  508.                         I915_WRITE(GFX_MODE_GEN7,
  509.                                    _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
  510.                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
  511.         }
  512.  
  513.         if (INTEL_INFO(dev)->gen >= 5) {
  514.                 ret = init_pipe_control(ring);
  515.                 if (ret)
  516.                         return ret;
  517.         }
  518.  
  519.         if (IS_GEN6(dev)) {
  520.                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
  521.                  * "If this bit is set, STCunit will have LRA as replacement
  522.                  *  policy. [...] This bit must be reset.  LRA replacement
  523.                  *  policy is not supported."
  524.                  */
  525.                 I915_WRITE(CACHE_MODE_0,
  526.                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
  527.  
  528.                 /* This is not explicitly set for GEN6, so read the register.
  529.                  * see intel_ring_mi_set_context() for why we care.
  530.                  * TODO: consider explicitly setting the bit for GEN5
  531.                  */
  532.                 ring->itlb_before_ctx_switch =
  533.                         !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
  534.         }
  535.  
  536.         if (INTEL_INFO(dev)->gen >= 6)
  537.                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
  538.  
  539.         if (HAS_L3_GPU_CACHE(dev))
  540.                 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
  541.  
  542.         return ret;
  543. }
  544.  
  545. static void render_ring_cleanup(struct intel_ring_buffer *ring)
  546. {
  547.         if (!ring->private)
  548.                 return;
  549.  
  550.         cleanup_pipe_control(ring);
  551. }
  552.  
  553. static void
  554. update_mboxes(struct intel_ring_buffer *ring,
  555.             u32 seqno,
  556.             u32 mmio_offset)
  557. {
  558.         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
  559.                               MI_SEMAPHORE_GLOBAL_GTT |
  560.                         MI_SEMAPHORE_REGISTER |
  561.                         MI_SEMAPHORE_UPDATE);
  562.         intel_ring_emit(ring, seqno);
  563.         intel_ring_emit(ring, mmio_offset);
  564. }
  565.  
  566. /**
  567.  * gen6_add_request - Update the semaphore mailbox registers
  568.  *
  569.  * @ring - ring that is adding a request
  570.  * @seqno - return seqno stuck into the ring
  571.  *
  572.  * Update the mailbox registers in the *other* rings with the current seqno.
  573.  * This acts like a signal in the canonical semaphore.
  574.  */
  575. static int
  576. gen6_add_request(struct intel_ring_buffer *ring,
  577.                  u32 *seqno)
  578. {
  579.         u32 mbox1_reg;
  580.         u32 mbox2_reg;
  581.         int ret;
  582.  
  583.         ret = intel_ring_begin(ring, 10);
  584.         if (ret)
  585.                 return ret;
  586.  
  587.         mbox1_reg = ring->signal_mbox[0];
  588.         mbox2_reg = ring->signal_mbox[1];
  589.  
  590.         *seqno = i915_gem_next_request_seqno(ring);
  591.  
  592.         update_mboxes(ring, *seqno, mbox1_reg);
  593.         update_mboxes(ring, *seqno, mbox2_reg);
  594.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  595.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  596.         intel_ring_emit(ring, *seqno);
  597.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  598.         intel_ring_advance(ring);
  599.  
  600.         return 0;
  601. }
  602.  
  603. /**
  604.  * intel_ring_sync - sync the waiter to the signaller on seqno
  605.  *
  606.  * @waiter - ring that is waiting
  607.  * @signaller - ring which has, or will signal
  608.  * @seqno - seqno which the waiter will block on
  609.  */
  610. static int
  611. gen6_ring_sync(struct intel_ring_buffer *waiter,
  612.                 struct intel_ring_buffer *signaller,
  613.                 u32 seqno)
  614. {
  615.         int ret;
  616.         u32 dw1 = MI_SEMAPHORE_MBOX |
  617.                   MI_SEMAPHORE_COMPARE |
  618.                   MI_SEMAPHORE_REGISTER;
  619.  
  620.         /* Throughout all of the GEM code, seqno passed implies our current
  621.          * seqno is >= the last seqno executed. However for hardware the
  622.          * comparison is strictly greater than.
  623.          */
  624.         seqno -= 1;
  625.  
  626.         WARN_ON(signaller->semaphore_register[waiter->id] ==
  627.                 MI_SEMAPHORE_SYNC_INVALID);
  628.  
  629.         ret = intel_ring_begin(waiter, 4);
  630.         if (ret)
  631.                 return ret;
  632.  
  633.         intel_ring_emit(waiter,
  634.                         dw1 | signaller->semaphore_register[waiter->id]);
  635.         intel_ring_emit(waiter, seqno);
  636.         intel_ring_emit(waiter, 0);
  637.         intel_ring_emit(waiter, MI_NOOP);
  638.         intel_ring_advance(waiter);
  639.  
  640.         return 0;
  641. }
  642.  
  643. #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
  644. do {                                                                    \
  645.         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
  646.                  PIPE_CONTROL_DEPTH_STALL);                             \
  647.         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
  648.         intel_ring_emit(ring__, 0);                                                     \
  649.         intel_ring_emit(ring__, 0);                                                     \
  650. } while (0)
  651.  
  652. static int
  653. pc_render_add_request(struct intel_ring_buffer *ring,
  654.                       u32 *result)
  655. {
  656.         u32 seqno = i915_gem_next_request_seqno(ring);
  657.         struct pipe_control *pc = ring->private;
  658.         u32 scratch_addr = pc->gtt_offset + 128;
  659.         int ret;
  660.  
  661.         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
  662.          * incoherent with writes to memory, i.e. completely fubar,
  663.          * so we need to use PIPE_NOTIFY instead.
  664.          *
  665.          * However, we also need to workaround the qword write
  666.          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
  667.          * memory before requesting an interrupt.
  668.          */
  669.         ret = intel_ring_begin(ring, 32);
  670.         if (ret)
  671.                 return ret;
  672.  
  673.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  674.                         PIPE_CONTROL_WRITE_FLUSH |
  675.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
  676.         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  677.         intel_ring_emit(ring, seqno);
  678.         intel_ring_emit(ring, 0);
  679.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  680.         scratch_addr += 128; /* write to separate cachelines */
  681.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  682.         scratch_addr += 128;
  683.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  684.         scratch_addr += 128;
  685.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  686.         scratch_addr += 128;
  687.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  688.         scratch_addr += 128;
  689.         PIPE_CONTROL_FLUSH(ring, scratch_addr);
  690.  
  691.         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
  692.                         PIPE_CONTROL_WRITE_FLUSH |
  693.                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
  694.                         PIPE_CONTROL_NOTIFY);
  695.         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
  696.         intel_ring_emit(ring, seqno);
  697.         intel_ring_emit(ring, 0);
  698.         intel_ring_advance(ring);
  699.  
  700.         *result = seqno;
  701.         return 0;
  702. }
  703.  
  704. static u32
  705. gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  706. {
  707.         /* Workaround to force correct ordering between irq and seqno writes on
  708.          * ivb (and maybe also on snb) by reading from a CS register (like
  709.          * ACTHD) before reading the status page. */
  710.         if (!lazy_coherency)
  711.                 intel_ring_get_active_head(ring);
  712.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  713. }
  714.  
  715. static u32
  716. ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  717. {
  718.         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
  719. }
  720.  
  721. static u32
  722. pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
  723. {
  724.         struct pipe_control *pc = ring->private;
  725.         return pc->cpu_page[0];
  726. }
  727.  
  728. static bool
  729. gen5_ring_get_irq(struct intel_ring_buffer *ring)
  730. {
  731.         struct drm_device *dev = ring->dev;
  732.         drm_i915_private_t *dev_priv = dev->dev_private;
  733.         unsigned long flags;
  734.  
  735.         if (!dev->irq_enabled)
  736.                 return false;
  737.  
  738.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  739.         if (ring->irq_refcount++ == 0) {
  740.                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
  741.         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  742.         POSTING_READ(GTIMR);
  743.         }
  744.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  745.  
  746.         return true;
  747. }
  748.  
  749. static void
  750. gen5_ring_put_irq(struct intel_ring_buffer *ring)
  751. {
  752.         struct drm_device *dev = ring->dev;
  753.         drm_i915_private_t *dev_priv = dev->dev_private;
  754.         unsigned long flags;
  755.  
  756.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  757.         if (--ring->irq_refcount == 0) {
  758.                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
  759.         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  760.         POSTING_READ(GTIMR);
  761.         }
  762.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  763. }
  764.  
  765. static bool
  766. i9xx_ring_get_irq(struct intel_ring_buffer *ring)
  767. {
  768.         struct drm_device *dev = ring->dev;
  769.         drm_i915_private_t *dev_priv = dev->dev_private;
  770.         unsigned long flags;
  771.  
  772.         if (!dev->irq_enabled)
  773.                 return false;
  774.  
  775.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  776.         if (ring->irq_refcount++ == 0) {
  777.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  778.         I915_WRITE(IMR, dev_priv->irq_mask);
  779.         POSTING_READ(IMR);
  780.         }
  781.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  782.  
  783.         return true;
  784. }
  785.  
  786. static void
  787. i9xx_ring_put_irq(struct intel_ring_buffer *ring)
  788. {
  789.         struct drm_device *dev = ring->dev;
  790.         drm_i915_private_t *dev_priv = dev->dev_private;
  791.         unsigned long flags;
  792.  
  793.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  794.         if (--ring->irq_refcount == 0) {
  795.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  796.         I915_WRITE(IMR, dev_priv->irq_mask);
  797.         POSTING_READ(IMR);
  798.         }
  799.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  800. }
  801.  
  802. static bool
  803. i8xx_ring_get_irq(struct intel_ring_buffer *ring)
  804. {
  805.         struct drm_device *dev = ring->dev;
  806.         drm_i915_private_t *dev_priv = dev->dev_private;
  807.         unsigned long flags;
  808.  
  809.         if (!dev->irq_enabled)
  810.                 return false;
  811.  
  812.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  813.         if (ring->irq_refcount++ == 0) {
  814.                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
  815.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  816.                 POSTING_READ16(IMR);
  817.         }
  818.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  819.  
  820.         return true;
  821. }
  822.  
  823. static void
  824. i8xx_ring_put_irq(struct intel_ring_buffer *ring)
  825. {
  826.         struct drm_device *dev = ring->dev;
  827.         drm_i915_private_t *dev_priv = dev->dev_private;
  828.         unsigned long flags;
  829.  
  830.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  831.         if (--ring->irq_refcount == 0) {
  832.                 dev_priv->irq_mask |= ring->irq_enable_mask;
  833.                 I915_WRITE16(IMR, dev_priv->irq_mask);
  834.                 POSTING_READ16(IMR);
  835.         }
  836.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  837. }
  838.  
  839. void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
  840. {
  841.         struct drm_device *dev = ring->dev;
  842.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  843.         u32 mmio = 0;
  844.  
  845.         /* The ring status page addresses are no longer next to the rest of
  846.          * the ring registers as of gen7.
  847.          */
  848.         if (IS_GEN7(dev)) {
  849.                 switch (ring->id) {
  850.                 case RCS:
  851.                         mmio = RENDER_HWS_PGA_GEN7;
  852.                         break;
  853.                 case BCS:
  854.                         mmio = BLT_HWS_PGA_GEN7;
  855.                         break;
  856.                 case VCS:
  857.                         mmio = BSD_HWS_PGA_GEN7;
  858.                         break;
  859.                 }
  860.         } else if (IS_GEN6(ring->dev)) {
  861.                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
  862.         } else {
  863.                 mmio = RING_HWS_PGA(ring->mmio_base);
  864.         }
  865.  
  866.         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
  867.         POSTING_READ(mmio);
  868. }
  869.  
  870. static int
  871. bsd_ring_flush(struct intel_ring_buffer *ring,
  872.                u32     invalidate_domains,
  873.                u32     flush_domains)
  874. {
  875.         int ret;
  876.  
  877.         ret = intel_ring_begin(ring, 2);
  878.         if (ret)
  879.                 return ret;
  880.  
  881.         intel_ring_emit(ring, MI_FLUSH);
  882.         intel_ring_emit(ring, MI_NOOP);
  883.         intel_ring_advance(ring);
  884.         return 0;
  885. }
  886.  
  887. static int
  888. i9xx_add_request(struct intel_ring_buffer *ring,
  889.                  u32 *result)
  890. {
  891.         u32 seqno;
  892.         int ret;
  893.  
  894.         ret = intel_ring_begin(ring, 4);
  895.         if (ret)
  896.                 return ret;
  897.  
  898.         seqno = i915_gem_next_request_seqno(ring);
  899.  
  900.         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
  901.         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
  902.         intel_ring_emit(ring, seqno);
  903.         intel_ring_emit(ring, MI_USER_INTERRUPT);
  904.         intel_ring_advance(ring);
  905.  
  906.         *result = seqno;
  907.         return 0;
  908. }
  909.  
  910. static bool
  911. gen6_ring_get_irq(struct intel_ring_buffer *ring)
  912. {
  913.         struct drm_device *dev = ring->dev;
  914.         drm_i915_private_t *dev_priv = dev->dev_private;
  915.         unsigned long flags;
  916.  
  917.         if (!dev->irq_enabled)
  918.                return false;
  919.  
  920.         /* It looks like we need to prevent the gt from suspending while waiting
  921.          * for an notifiy irq, otherwise irqs seem to get lost on at least the
  922.          * blt/bsd rings on ivb. */
  923.                 gen6_gt_force_wake_get(dev_priv);
  924.  
  925.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  926.         if (ring->irq_refcount++ == 0) {
  927.                 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
  928.                         I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
  929.                                                 GEN6_RENDER_L3_PARITY_ERROR));
  930.                 else
  931.                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
  932.                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
  933.                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  934.                 POSTING_READ(GTIMR);
  935.         }
  936.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  937.  
  938.     return true;
  939. }
  940.  
  941. static void
  942. gen6_ring_put_irq(struct intel_ring_buffer *ring)
  943. {
  944.         struct drm_device *dev = ring->dev;
  945.         drm_i915_private_t *dev_priv = dev->dev_private;
  946.         unsigned long flags;
  947.  
  948.         spin_lock_irqsave(&dev_priv->irq_lock, flags);
  949.         if (--ring->irq_refcount == 0) {
  950.                 if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
  951.                         I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
  952.                 else
  953.                         I915_WRITE_IMR(ring, ~0);
  954.                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
  955.                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  956.                 POSTING_READ(GTIMR);
  957.         }
  958.         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
  959.  
  960.                 gen6_gt_force_wake_put(dev_priv);
  961. }
  962.  
  963. static int
  964. i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
  965. {
  966.         int ret;
  967.  
  968.         ret = intel_ring_begin(ring, 2);
  969.         if (ret)
  970.                 return ret;
  971.  
  972.         intel_ring_emit(ring,
  973.                         MI_BATCH_BUFFER_START |
  974.                         MI_BATCH_GTT |
  975.                         MI_BATCH_NON_SECURE_I965);
  976.         intel_ring_emit(ring, offset);
  977.         intel_ring_advance(ring);
  978.  
  979.         return 0;
  980. }
  981.  
  982. static int
  983. i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
  984.                                 u32 offset, u32 len)
  985. {
  986.         int ret;
  987.  
  988.                 ret = intel_ring_begin(ring, 4);
  989.                 if (ret)
  990.                         return ret;
  991.  
  992.                 intel_ring_emit(ring, MI_BATCH_BUFFER);
  993.                 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
  994.                 intel_ring_emit(ring, offset + len - 8);
  995.                 intel_ring_emit(ring, 0);
  996.         intel_ring_advance(ring);
  997.  
  998.         return 0;
  999. }
  1000.  
  1001. static int
  1002. i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1003.                                 u32 offset, u32 len)
  1004. {
  1005.         int ret;
  1006.  
  1007.                 ret = intel_ring_begin(ring, 2);
  1008.                 if (ret)
  1009.                         return ret;
  1010.  
  1011.         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
  1012.                         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
  1013.         intel_ring_advance(ring);
  1014.  
  1015.         return 0;
  1016. }
  1017.  
  1018. static void cleanup_status_page(struct intel_ring_buffer *ring)
  1019. {
  1020.         struct drm_i915_gem_object *obj;
  1021.  
  1022.         obj = ring->status_page.obj;
  1023.         if (obj == NULL)
  1024.                 return;
  1025.  
  1026. //   kunmap(sg_page(obj->pages->sgl));
  1027.     i915_gem_object_unpin(obj);
  1028.         drm_gem_object_unreference(&obj->base);
  1029.         ring->status_page.obj = NULL;
  1030. }
  1031.  
  1032. static int init_status_page(struct intel_ring_buffer *ring)
  1033. {
  1034.         struct drm_device *dev = ring->dev;
  1035.         struct drm_i915_gem_object *obj;
  1036.         int ret;
  1037.  
  1038.         obj = i915_gem_alloc_object(dev, 4096);
  1039.         if (obj == NULL) {
  1040.                 DRM_ERROR("Failed to allocate status page\n");
  1041.                 ret = -ENOMEM;
  1042.                 goto err;
  1043.         }
  1044.  
  1045.         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
  1046.  
  1047.         ret = i915_gem_object_pin(obj, 4096, true, false);
  1048.         if (ret != 0) {
  1049.                 goto err_unref;
  1050.         }
  1051.  
  1052.         ring->status_page.gfx_addr = obj->gtt_offset;
  1053.     ring->status_page.page_addr = (void*)MapIoMem(obj->pages.page[0],4096,PG_SW);
  1054.         if (ring->status_page.page_addr == NULL) {
  1055.                 ret = -ENOMEM;
  1056.                 goto err_unpin;
  1057.         }
  1058.         ring->status_page.obj = obj;
  1059.         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1060.  
  1061.         intel_ring_setup_status_page(ring);
  1062.         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  1063.                         ring->name, ring->status_page.gfx_addr);
  1064.  
  1065.         return 0;
  1066.  
  1067. err_unpin:
  1068.         i915_gem_object_unpin(obj);
  1069. err_unref:
  1070.         drm_gem_object_unreference(&obj->base);
  1071. err:
  1072.         return ret;
  1073. }
  1074.  
  1075. static int intel_init_ring_buffer(struct drm_device *dev,
  1076.                            struct intel_ring_buffer *ring)
  1077. {
  1078.         struct drm_i915_gem_object *obj;
  1079.         struct drm_i915_private *dev_priv = dev->dev_private;
  1080.         int ret;
  1081.  
  1082.         ring->dev = dev;
  1083.         INIT_LIST_HEAD(&ring->active_list);
  1084.         INIT_LIST_HEAD(&ring->request_list);
  1085.         ring->size = 32 * PAGE_SIZE;
  1086.  
  1087.         init_waitqueue_head(&ring->irq_queue);
  1088.  
  1089.         if (I915_NEED_GFX_HWS(dev)) {
  1090.        ret = init_status_page(ring);
  1091.        if (ret)
  1092.            return ret;
  1093.         }
  1094.  
  1095.     obj = i915_gem_alloc_object(dev, ring->size);
  1096.         if (obj == NULL) {
  1097.                 DRM_ERROR("Failed to allocate ringbuffer\n");
  1098.                 ret = -ENOMEM;
  1099.                 goto err_hws;
  1100.         }
  1101.  
  1102.         ring->obj = obj;
  1103.  
  1104.         ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
  1105.         if (ret)
  1106.                 goto err_unref;
  1107.  
  1108.         ret = i915_gem_object_set_to_gtt_domain(obj, true);
  1109.         if (ret)
  1110.                 goto err_unpin;
  1111.  
  1112.         ring->virtual_start =
  1113.         ioremap(dev_priv->mm.gtt->gma_bus_addr + obj->gtt_offset,
  1114.                            ring->size);
  1115.         if (ring->virtual_start == NULL) {
  1116.                 DRM_ERROR("Failed to map ringbuffer.\n");
  1117.                 ret = -EINVAL;
  1118.                 goto err_unpin;
  1119.         }
  1120.  
  1121.         ret = ring->init(ring);
  1122.         if (ret)
  1123.                 goto err_unmap;
  1124.  
  1125.         /* Workaround an erratum on the i830 which causes a hang if
  1126.          * the TAIL pointer points to within the last 2 cachelines
  1127.          * of the buffer.
  1128.          */
  1129.         ring->effective_size = ring->size;
  1130.         if (IS_I830(ring->dev) || IS_845G(ring->dev))
  1131.                 ring->effective_size -= 128;
  1132.  
  1133.         return 0;
  1134.  
  1135. err_unmap:
  1136.     FreeKernelSpace(ring->virtual_start);
  1137. err_unpin:
  1138.         i915_gem_object_unpin(obj);
  1139. err_unref:
  1140.         drm_gem_object_unreference(&obj->base);
  1141.         ring->obj = NULL;
  1142. err_hws:
  1143. //   cleanup_status_page(ring);
  1144.         return ret;
  1145. }
  1146.  
  1147. void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
  1148. {
  1149.         struct drm_i915_private *dev_priv;
  1150.         int ret;
  1151.  
  1152.         if (ring->obj == NULL)
  1153.                 return;
  1154.  
  1155.         /* Disable the ring buffer. The ring must be idle at this point */
  1156.         dev_priv = ring->dev->dev_private;
  1157.         ret = intel_wait_ring_idle(ring);
  1158.         if (ret)
  1159.                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
  1160.                           ring->name, ret);
  1161.  
  1162.         I915_WRITE_CTL(ring, 0);
  1163.  
  1164. //   drm_core_ioremapfree(&ring->map, ring->dev);
  1165.  
  1166.     i915_gem_object_unpin(ring->obj);
  1167.         drm_gem_object_unreference(&ring->obj->base);
  1168.         ring->obj = NULL;
  1169.  
  1170.         if (ring->cleanup)
  1171.                 ring->cleanup(ring);
  1172.  
  1173. //   cleanup_status_page(ring);
  1174. }
  1175.  
  1176. static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
  1177. {
  1178.         uint32_t __iomem *virt;
  1179.         int rem = ring->size - ring->tail;
  1180.  
  1181.         if (ring->space < rem) {
  1182.                 int ret = intel_wait_ring_buffer(ring, rem);
  1183.                 if (ret)
  1184.                         return ret;
  1185.         }
  1186.  
  1187.         virt = ring->virtual_start + ring->tail;
  1188.         rem /= 4;
  1189.         while (rem--)
  1190.                 iowrite32(MI_NOOP, virt++);
  1191.  
  1192.         ring->tail = 0;
  1193.         ring->space = ring_space(ring);
  1194.  
  1195.     return 0;
  1196. }
  1197.  
  1198. static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
  1199. {
  1200.         int ret;
  1201.  
  1202.         ret = i915_wait_seqno(ring, seqno);
  1203.         if (!ret)
  1204.                 i915_gem_retire_requests_ring(ring);
  1205.  
  1206.         return ret;
  1207. }
  1208.  
  1209. static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
  1210. {
  1211.         struct drm_i915_gem_request *request;
  1212.         u32 seqno = 0;
  1213.         int ret;
  1214.  
  1215.         i915_gem_retire_requests_ring(ring);
  1216.  
  1217.         if (ring->last_retired_head != -1) {
  1218.                 ring->head = ring->last_retired_head;
  1219.                 ring->last_retired_head = -1;
  1220.                 ring->space = ring_space(ring);
  1221.                 if (ring->space >= n)
  1222.                         return 0;
  1223.         }
  1224.  
  1225.         list_for_each_entry(request, &ring->request_list, list) {
  1226.                 int space;
  1227.  
  1228.                 if (request->tail == -1)
  1229.                         continue;
  1230.  
  1231.                 space = request->tail - (ring->tail + 8);
  1232.                 if (space < 0)
  1233.                         space += ring->size;
  1234.                 if (space >= n) {
  1235.                         seqno = request->seqno;
  1236.                         break;
  1237.                 }
  1238.  
  1239.                 /* Consume this request in case we need more space than
  1240.                  * is available and so need to prevent a race between
  1241.                  * updating last_retired_head and direct reads of
  1242.                  * I915_RING_HEAD. It also provides a nice sanity check.
  1243.                  */
  1244.                 request->tail = -1;
  1245.         }
  1246.  
  1247.         if (seqno == 0)
  1248.                 return -ENOSPC;
  1249.  
  1250.         ret = intel_ring_wait_seqno(ring, seqno);
  1251.         if (ret)
  1252.                 return ret;
  1253.  
  1254.         if (WARN_ON(ring->last_retired_head == -1))
  1255.                 return -ENOSPC;
  1256.  
  1257.         ring->head = ring->last_retired_head;
  1258.         ring->last_retired_head = -1;
  1259.         ring->space = ring_space(ring);
  1260.         if (WARN_ON(ring->space < n))
  1261.                 return -ENOSPC;
  1262.  
  1263.         return 0;
  1264. }
  1265.  
  1266. int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
  1267. {
  1268.         struct drm_device *dev = ring->dev;
  1269.         struct drm_i915_private *dev_priv = dev->dev_private;
  1270.         unsigned long end;
  1271.         int ret;
  1272.  
  1273.         ret = intel_ring_wait_request(ring, n);
  1274.         if (ret != -ENOSPC)
  1275.                 return ret;
  1276.  
  1277.  
  1278.         /* With GEM the hangcheck timer should kick us out of the loop,
  1279.          * leaving it early runs the risk of corrupting GEM state (due
  1280.          * to running on almost untested codepaths). But on resume
  1281.          * timers don't work yet, so prevent a complete hang in that
  1282.          * case by choosing an insanely large timeout. */
  1283.         end = GetTimerTicks() + 60 * HZ;
  1284.  
  1285.         do {
  1286.                 ring->head = I915_READ_HEAD(ring);
  1287.                 ring->space = ring_space(ring);
  1288.                 if (ring->space >= n) {
  1289.                         trace_i915_ring_wait_end(ring);
  1290.                         return 0;
  1291.                 }
  1292.  
  1293.                 msleep(1);
  1294.  
  1295.                 ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
  1296.                 if (ret)
  1297.                         return ret;
  1298.     } while (!time_after(GetTimerTicks(), end));
  1299.         trace_i915_ring_wait_end(ring);
  1300.         return -EBUSY;
  1301. }
  1302.  
  1303. int intel_ring_begin(struct intel_ring_buffer *ring,
  1304.                      int num_dwords)
  1305. {
  1306.         drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1307.         int n = 4*num_dwords;
  1308.         int ret;
  1309.  
  1310.         ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
  1311.         if (ret)
  1312.                 return ret;
  1313.  
  1314.         if (unlikely(ring->tail + n > ring->effective_size)) {
  1315.                 ret = intel_wrap_ring_buffer(ring);
  1316.                 if (unlikely(ret))
  1317.                         return ret;
  1318.         }
  1319.  
  1320.         if (unlikely(ring->space < n)) {
  1321.                 ret = intel_wait_ring_buffer(ring, n);
  1322.                 if (unlikely(ret))
  1323.                         return ret;
  1324.         }
  1325.  
  1326.         ring->space -= n;
  1327.         return 0;
  1328. }
  1329.  
  1330. void intel_ring_advance(struct intel_ring_buffer *ring)
  1331. {
  1332.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  1333.  
  1334.         ring->tail &= ring->size - 1;
  1335.         if (dev_priv->stop_rings & intel_ring_flag(ring))
  1336.                 return;
  1337.         ring->write_tail(ring, ring->tail);
  1338. }
  1339.  
  1340.  
  1341. static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
  1342.                                      u32 value)
  1343. {
  1344.        drm_i915_private_t *dev_priv = ring->dev->dev_private;
  1345.  
  1346.        /* Every tail move must follow the sequence below */
  1347.  
  1348.         /* Disable notification that the ring is IDLE. The GT
  1349.          * will then assume that it is busy and bring it out of rc6.
  1350.          */
  1351.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1352.                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1353.  
  1354.         /* Clear the context id. Here be magic! */
  1355.         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
  1356.  
  1357.         /* Wait for the ring not to be idle, i.e. for it to wake up. */
  1358.        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
  1359.                       GEN6_BSD_SLEEP_INDICATOR) == 0,
  1360.                        50))
  1361.                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
  1362.  
  1363.         /* Now that the ring is fully powered up, update the tail */
  1364.        I915_WRITE_TAIL(ring, value);
  1365.         POSTING_READ(RING_TAIL(ring->mmio_base));
  1366.  
  1367.         /* Let the ring send IDLE messages to the GT again,
  1368.          * and so let it sleep to conserve power when idle.
  1369.          */
  1370.        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
  1371.                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
  1372. }
  1373.  
  1374. static int gen6_ring_flush(struct intel_ring_buffer *ring,
  1375.                            u32 invalidate, u32 flush)
  1376. {
  1377.         uint32_t cmd;
  1378.         int ret;
  1379.  
  1380.         ret = intel_ring_begin(ring, 4);
  1381.         if (ret)
  1382.                 return ret;
  1383.  
  1384.         cmd = MI_FLUSH_DW;
  1385.         if (invalidate & I915_GEM_GPU_DOMAINS)
  1386.                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
  1387.         intel_ring_emit(ring, cmd);
  1388.         intel_ring_emit(ring, 0);
  1389.         intel_ring_emit(ring, 0);
  1390.         intel_ring_emit(ring, MI_NOOP);
  1391.         intel_ring_advance(ring);
  1392.         return 0;
  1393. }
  1394.  
  1395. static int
  1396. gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
  1397.                               u32 offset, u32 len)
  1398. {
  1399.        int ret;
  1400.  
  1401.        ret = intel_ring_begin(ring, 2);
  1402.        if (ret)
  1403.                return ret;
  1404.  
  1405.        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
  1406.        /* bit0-7 is the length on GEN6+ */
  1407.        intel_ring_emit(ring, offset);
  1408.        intel_ring_advance(ring);
  1409.  
  1410.        return 0;
  1411. }
  1412.  
  1413. /* Blitter support (SandyBridge+) */
  1414.  
  1415. static int blt_ring_flush(struct intel_ring_buffer *ring,
  1416.                           u32 invalidate, u32 flush)
  1417. {
  1418.         uint32_t cmd;
  1419.         int ret;
  1420.  
  1421.         ret = intel_ring_begin(ring, 4);
  1422.         if (ret)
  1423.                 return ret;
  1424.  
  1425.         cmd = MI_FLUSH_DW;
  1426.         if (invalidate & I915_GEM_DOMAIN_RENDER)
  1427.                 cmd |= MI_INVALIDATE_TLB;
  1428.         intel_ring_emit(ring, cmd);
  1429.         intel_ring_emit(ring, 0);
  1430.         intel_ring_emit(ring, 0);
  1431.         intel_ring_emit(ring, MI_NOOP);
  1432.         intel_ring_advance(ring);
  1433.         return 0;
  1434. }
  1435.  
  1436. int intel_init_render_ring_buffer(struct drm_device *dev)
  1437. {
  1438.         drm_i915_private_t *dev_priv = dev->dev_private;
  1439.         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
  1440.  
  1441.         ring->name = "render ring";
  1442.         ring->id = RCS;
  1443.         ring->mmio_base = RENDER_RING_BASE;
  1444.  
  1445.         if (INTEL_INFO(dev)->gen >= 6) {
  1446.        ring->add_request = gen6_add_request;
  1447.                 ring->flush = gen7_render_ring_flush;
  1448.                 if (INTEL_INFO(dev)->gen == 6)
  1449.                 ring->flush = gen6_render_ring_flush;
  1450.                 ring->irq_get = gen6_ring_get_irq;
  1451.                 ring->irq_put = gen6_ring_put_irq;
  1452.                 ring->irq_enable_mask = GT_USER_INTERRUPT;
  1453.                 ring->get_seqno = gen6_ring_get_seqno;
  1454.                 ring->sync_to = gen6_ring_sync;
  1455.                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
  1456.                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
  1457.                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
  1458.                 ring->signal_mbox[0] = GEN6_VRSYNC;
  1459.                 ring->signal_mbox[1] = GEN6_BRSYNC;
  1460.         } else if (IS_GEN5(dev)) {
  1461.        ring->add_request = pc_render_add_request;
  1462.                 ring->flush = gen4_render_ring_flush;
  1463.                 ring->get_seqno = pc_render_get_seqno;
  1464.                 ring->irq_get = gen5_ring_get_irq;
  1465.                 ring->irq_put = gen5_ring_put_irq;
  1466.                 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
  1467.         } else {
  1468.                 ring->add_request = i9xx_add_request;
  1469.                 if (INTEL_INFO(dev)->gen < 4)
  1470.                         ring->flush = gen2_render_ring_flush;
  1471.                 else
  1472.                         ring->flush = gen4_render_ring_flush;
  1473.                 ring->get_seqno = ring_get_seqno;
  1474.                 if (IS_GEN2(dev)) {
  1475.                         ring->irq_get = i8xx_ring_get_irq;
  1476.                         ring->irq_put = i8xx_ring_put_irq;
  1477.                 } else {
  1478.                         ring->irq_get = i9xx_ring_get_irq;
  1479.                         ring->irq_put = i9xx_ring_put_irq;
  1480.                 }
  1481.                 ring->irq_enable_mask = I915_USER_INTERRUPT;
  1482.         }
  1483.         ring->write_tail = ring_write_tail;
  1484.         if (INTEL_INFO(dev)->gen >= 6)
  1485.                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1486.         else if (INTEL_INFO(dev)->gen >= 4)
  1487.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1488.         else if (IS_I830(dev) || IS_845G(dev))
  1489.                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
  1490.         else
  1491.                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
  1492.         ring->init = init_render_ring;
  1493.         ring->cleanup = render_ring_cleanup;
  1494.  
  1495.  
  1496.         if (!I915_NEED_GFX_HWS(dev)) {
  1497.                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
  1498.                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
  1499.         }
  1500.  
  1501.         return intel_init_ring_buffer(dev, ring);
  1502. }
  1503.  
  1504.  
  1505. int intel_init_bsd_ring_buffer(struct drm_device *dev)
  1506. {
  1507.         drm_i915_private_t *dev_priv = dev->dev_private;
  1508.         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
  1509.  
  1510.         ring->name = "bsd ring";
  1511.         ring->id = VCS;
  1512.  
  1513.         ring->write_tail = ring_write_tail;
  1514.         if (IS_GEN6(dev) || IS_GEN7(dev)) {
  1515.                 ring->mmio_base = GEN6_BSD_RING_BASE;
  1516.                 /* gen6 bsd needs a special wa for tail updates */
  1517.                 if (IS_GEN6(dev))
  1518.                         ring->write_tail = gen6_bsd_ring_write_tail;
  1519.                 ring->flush = gen6_ring_flush;
  1520.                 ring->add_request = gen6_add_request;
  1521.                 ring->get_seqno = gen6_ring_get_seqno;
  1522.                 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
  1523.                 ring->irq_get = gen6_ring_get_irq;
  1524.                 ring->irq_put = gen6_ring_put_irq;
  1525.                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1526.                 ring->sync_to = gen6_ring_sync;
  1527.                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
  1528.                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
  1529.                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
  1530.                 ring->signal_mbox[0] = GEN6_RVSYNC;
  1531.                 ring->signal_mbox[1] = GEN6_BVSYNC;
  1532.         } else {
  1533.                 ring->mmio_base = BSD_RING_BASE;
  1534.                 ring->flush = bsd_ring_flush;
  1535.                 ring->add_request = i9xx_add_request;
  1536.                 ring->get_seqno = ring_get_seqno;
  1537.                 if (IS_GEN5(dev)) {
  1538.                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
  1539.                         ring->irq_get = gen5_ring_get_irq;
  1540.                         ring->irq_put = gen5_ring_put_irq;
  1541.                 } else {
  1542.                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
  1543.                         ring->irq_get = i9xx_ring_get_irq;
  1544.                         ring->irq_put = i9xx_ring_put_irq;
  1545.                 }
  1546.                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
  1547.         }
  1548.         ring->init = init_ring_common;
  1549.  
  1550.  
  1551.         return intel_init_ring_buffer(dev, ring);
  1552. }
  1553.  
  1554. int intel_init_blt_ring_buffer(struct drm_device *dev)
  1555. {
  1556.         drm_i915_private_t *dev_priv = dev->dev_private;
  1557.         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
  1558.  
  1559.         ring->name = "blitter ring";
  1560.         ring->id = BCS;
  1561.  
  1562.         ring->mmio_base = BLT_RING_BASE;
  1563.         ring->write_tail = ring_write_tail;
  1564.         ring->flush = blt_ring_flush;
  1565.         ring->add_request = gen6_add_request;
  1566.         ring->get_seqno = gen6_ring_get_seqno;
  1567.         ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
  1568.         ring->irq_get = gen6_ring_get_irq;
  1569.         ring->irq_put = gen6_ring_put_irq;
  1570.         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
  1571.         ring->sync_to = gen6_ring_sync;
  1572.         ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
  1573.         ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
  1574.         ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
  1575.         ring->signal_mbox[0] = GEN6_RBSYNC;
  1576.         ring->signal_mbox[1] = GEN6_VBSYNC;
  1577.         ring->init = init_ring_common;
  1578.  
  1579.         return intel_init_ring_buffer(dev, ring);
  1580. }
  1581.  
  1582. int
  1583. intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
  1584. {
  1585.         int ret;
  1586.  
  1587.         if (!ring->gpu_caches_dirty)
  1588.                 return 0;
  1589.  
  1590.         ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
  1591.         if (ret)
  1592.                 return ret;
  1593.  
  1594.         trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
  1595.  
  1596.         ring->gpu_caches_dirty = false;
  1597.         return 0;
  1598. }
  1599.  
  1600. int
  1601. intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
  1602. {
  1603.         uint32_t flush_domains;
  1604.         int ret;
  1605.  
  1606.         flush_domains = 0;
  1607.         if (ring->gpu_caches_dirty)
  1608.                 flush_domains = I915_GEM_GPU_DOMAINS;
  1609.  
  1610.         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  1611.         if (ret)
  1612.                 return ret;
  1613.  
  1614.         trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
  1615.  
  1616.         ring->gpu_caches_dirty = false;
  1617.         return 0;
  1618. }
  1619.