Subversion Repositories Kolibri OS

Rev

Rev 6935 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
  2.  */
  3. /*
  4.  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
  5.  * All Rights Reserved.
  6.  *
  7.  * Permission is hereby granted, free of charge, to any person obtaining a
  8.  * copy of this software and associated documentation files (the
  9.  * "Software"), to deal in the Software without restriction, including
  10.  * without limitation the rights to use, copy, modify, merge, publish,
  11.  * distribute, sub license, and/or sell copies of the Software, and to
  12.  * permit persons to whom the Software is furnished to do so, subject to
  13.  * the following conditions:
  14.  *
  15.  * The above copyright notice and this permission notice (including the
  16.  * next paragraph) shall be included in all copies or substantial portions
  17.  * of the Software.
  18.  *
  19.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20.  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21.  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22.  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23.  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24.  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25.  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26.  *
  27.  */
  28.  
  29. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30.  
  31. #include <linux/sysrq.h>
  32. #include <linux/slab.h>
  33. #include <linux/circ_buf.h>
  34. #include <drm/drmP.h>
  35. #include <drm/i915_drm.h>
  36. #include "i915_drv.h"
  37. #include "i915_trace.h"
  38. #include "intel_drv.h"
  39.  
  40. /**
  41.  * DOC: interrupt handling
  42.  *
  43.  * These functions provide the basic support for enabling and disabling the
  44.  * interrupt handling support. There's a lot more functionality in i915_irq.c
  45.  * and related files, but that will be described in separate chapters.
  46.  */
  47.  
  48. static const u32 hpd_ilk[HPD_NUM_PINS] = {
  49.         [HPD_PORT_A] = DE_DP_A_HOTPLUG,
  50. };
  51.  
  52. static const u32 hpd_ivb[HPD_NUM_PINS] = {
  53.         [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
  54. };
  55.  
  56. static const u32 hpd_bdw[HPD_NUM_PINS] = {
  57.         [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
  58. };
  59.  
  60. static const u32 hpd_ibx[HPD_NUM_PINS] = {
  61.         [HPD_CRT] = SDE_CRT_HOTPLUG,
  62.         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  63.         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  64.         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  65.         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  66. };
  67.  
  68. static const u32 hpd_cpt[HPD_NUM_PINS] = {
  69.         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  70.         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  71.         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  72.         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  73.         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  74. };
  75.  
  76. static const u32 hpd_spt[HPD_NUM_PINS] = {
  77.         [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
  78.         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  79.         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  80.         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
  81.         [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
  82. };
  83.  
  84. static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
  85.         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  86.         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  87.         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  88.         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  89.         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  90.         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  91. };
  92.  
  93. static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
  94.         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  95.         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  96.         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  97.         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  98.         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  99.         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  100. };
  101.  
  102. static const u32 hpd_status_i915[HPD_NUM_PINS] = {
  103.         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  104.         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  105.         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  106.         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  107.         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  108.         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  109. };
  110.  
  111. /* BXT hpd list */
  112. static const u32 hpd_bxt[HPD_NUM_PINS] = {
  113.         [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
  114.         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
  115.         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
  116. };
  117.  
  118. /* IIR can theoretically queue up two events. Be paranoid. */
  119. #define GEN8_IRQ_RESET_NDX(type, which) do { \
  120.         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
  121.         POSTING_READ(GEN8_##type##_IMR(which)); \
  122.         I915_WRITE(GEN8_##type##_IER(which), 0); \
  123.         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
  124.         POSTING_READ(GEN8_##type##_IIR(which)); \
  125.         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
  126.         POSTING_READ(GEN8_##type##_IIR(which)); \
  127. } while (0)
  128.  
  129. #define GEN5_IRQ_RESET(type) do { \
  130.         I915_WRITE(type##IMR, 0xffffffff); \
  131.         POSTING_READ(type##IMR); \
  132.         I915_WRITE(type##IER, 0); \
  133.         I915_WRITE(type##IIR, 0xffffffff); \
  134.         POSTING_READ(type##IIR); \
  135.         I915_WRITE(type##IIR, 0xffffffff); \
  136.         POSTING_READ(type##IIR); \
  137. } while (0)
  138.  
  139. /*
  140.  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
  141.  */
  142. static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
  143.                                     i915_reg_t reg)
  144. {
  145.         u32 val = I915_READ(reg);
  146.  
  147.         if (val == 0)
  148.                 return;
  149.  
  150.         WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
  151.              i915_mmio_reg_offset(reg), val);
  152.         I915_WRITE(reg, 0xffffffff);
  153.         POSTING_READ(reg);
  154.         I915_WRITE(reg, 0xffffffff);
  155.         POSTING_READ(reg);
  156. }
  157.  
  158. #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
  159.         gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
  160.         I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
  161.         I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
  162.         POSTING_READ(GEN8_##type##_IMR(which)); \
  163. } while (0)
  164.  
  165. #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
  166.         gen5_assert_iir_is_zero(dev_priv, type##IIR); \
  167.         I915_WRITE(type##IER, (ier_val)); \
  168.         I915_WRITE(type##IMR, (imr_val)); \
  169.         POSTING_READ(type##IMR); \
  170. } while (0)
  171.  
  172. static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
  173.  
  174. /* For display hotplug interrupt */
  175. static inline void
  176. i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
  177.                                      uint32_t mask,
  178.                                      uint32_t bits)
  179. {
  180.         uint32_t val;
  181.  
  182.         assert_spin_locked(&dev_priv->irq_lock);
  183.         WARN_ON(bits & ~mask);
  184.  
  185.         val = I915_READ(PORT_HOTPLUG_EN);
  186.         val &= ~mask;
  187.         val |= bits;
  188.         I915_WRITE(PORT_HOTPLUG_EN, val);
  189. }
  190.  
  191. /**
  192.  * i915_hotplug_interrupt_update - update hotplug interrupt enable
  193.  * @dev_priv: driver private
  194.  * @mask: bits to update
  195.  * @bits: bits to enable
  196.  * NOTE: the HPD enable bits are modified both inside and outside
  197.  * of an interrupt context. To avoid that read-modify-write cycles
  198.  * interfer, these bits are protected by a spinlock. Since this
  199.  * function is usually not called from a context where the lock is
  200.  * held already, this function acquires the lock itself. A non-locking
  201.  * version is also available.
  202.  */
  203. void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
  204.                                    uint32_t mask,
  205.                                    uint32_t bits)
  206. {
  207.         spin_lock_irq(&dev_priv->irq_lock);
  208.         i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
  209.         spin_unlock_irq(&dev_priv->irq_lock);
  210. }
  211.  
  212. /**
  213.  * ilk_update_display_irq - update DEIMR
  214.  * @dev_priv: driver private
  215.  * @interrupt_mask: mask of interrupt bits to update
  216.  * @enabled_irq_mask: mask of interrupt bits to enable
  217.  */
  218. void ilk_update_display_irq(struct drm_i915_private *dev_priv,
  219.                                    uint32_t interrupt_mask,
  220.                                    uint32_t enabled_irq_mask)
  221. {
  222.         uint32_t new_val;
  223.  
  224.         assert_spin_locked(&dev_priv->irq_lock);
  225.  
  226.         WARN_ON(enabled_irq_mask & ~interrupt_mask);
  227.  
  228.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  229.                 return;
  230.  
  231.         new_val = dev_priv->irq_mask;
  232.         new_val &= ~interrupt_mask;
  233.         new_val |= (~enabled_irq_mask & interrupt_mask);
  234.  
  235.         if (new_val != dev_priv->irq_mask) {
  236.                 dev_priv->irq_mask = new_val;
  237.                 I915_WRITE(DEIMR, dev_priv->irq_mask);
  238.                 POSTING_READ(DEIMR);
  239.         }
  240. }
  241.  
  242. /**
  243.  * ilk_update_gt_irq - update GTIMR
  244.  * @dev_priv: driver private
  245.  * @interrupt_mask: mask of interrupt bits to update
  246.  * @enabled_irq_mask: mask of interrupt bits to enable
  247.  */
  248. static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
  249.                               uint32_t interrupt_mask,
  250.                               uint32_t enabled_irq_mask)
  251. {
  252.         assert_spin_locked(&dev_priv->irq_lock);
  253.  
  254.         WARN_ON(enabled_irq_mask & ~interrupt_mask);
  255.  
  256.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  257.                 return;
  258.  
  259.         dev_priv->gt_irq_mask &= ~interrupt_mask;
  260.         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
  261.         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
  262.         POSTING_READ(GTIMR);
  263. }
  264.  
  265. void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
  266. {
  267.         ilk_update_gt_irq(dev_priv, mask, mask);
  268. }
  269.  
  270. void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
  271. {
  272.         ilk_update_gt_irq(dev_priv, mask, 0);
  273. }
  274.  
  275. static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
  276. {
  277.         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
  278. }
  279.  
  280. static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
  281. {
  282.         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
  283. }
  284.  
  285. static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
  286. {
  287.         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
  288. }
  289.  
  290. /**
  291.   * snb_update_pm_irq - update GEN6_PMIMR
  292.   * @dev_priv: driver private
  293.   * @interrupt_mask: mask of interrupt bits to update
  294.   * @enabled_irq_mask: mask of interrupt bits to enable
  295.   */
  296. static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
  297.                               uint32_t interrupt_mask,
  298.                               uint32_t enabled_irq_mask)
  299. {
  300.         uint32_t new_val;
  301.  
  302.         WARN_ON(enabled_irq_mask & ~interrupt_mask);
  303.  
  304.         assert_spin_locked(&dev_priv->irq_lock);
  305.  
  306.         new_val = dev_priv->pm_irq_mask;
  307.         new_val &= ~interrupt_mask;
  308.         new_val |= (~enabled_irq_mask & interrupt_mask);
  309.  
  310.         if (new_val != dev_priv->pm_irq_mask) {
  311.                 dev_priv->pm_irq_mask = new_val;
  312.                 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
  313.                 POSTING_READ(gen6_pm_imr(dev_priv));
  314.         }
  315. }
  316.  
  317. void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
  318. {
  319.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  320.                 return;
  321.  
  322.         snb_update_pm_irq(dev_priv, mask, mask);
  323. }
  324.  
  325. static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
  326.                                   uint32_t mask)
  327. {
  328.         snb_update_pm_irq(dev_priv, mask, 0);
  329. }
  330.  
  331. void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
  332. {
  333.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  334.                 return;
  335.  
  336.         __gen6_disable_pm_irq(dev_priv, mask);
  337. }
  338.  
  339. void gen6_reset_rps_interrupts(struct drm_device *dev)
  340. {
  341.         struct drm_i915_private *dev_priv = dev->dev_private;
  342.         i915_reg_t reg = gen6_pm_iir(dev_priv);
  343.  
  344.         spin_lock_irq(&dev_priv->irq_lock);
  345.         I915_WRITE(reg, dev_priv->pm_rps_events);
  346.         I915_WRITE(reg, dev_priv->pm_rps_events);
  347.         POSTING_READ(reg);
  348.         dev_priv->rps.pm_iir = 0;
  349.         spin_unlock_irq(&dev_priv->irq_lock);
  350. }
  351.  
  352. void gen6_enable_rps_interrupts(struct drm_device *dev)
  353. {
  354.         struct drm_i915_private *dev_priv = dev->dev_private;
  355.  
  356.         spin_lock_irq(&dev_priv->irq_lock);
  357.  
  358.         WARN_ON(dev_priv->rps.pm_iir);
  359.         WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
  360.         dev_priv->rps.interrupts_enabled = true;
  361.         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
  362.                                 dev_priv->pm_rps_events);
  363.         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
  364.  
  365.         spin_unlock_irq(&dev_priv->irq_lock);
  366. }
  367.  
  368. u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
  369. {
  370.         /*
  371.          * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
  372.          * if GEN6_PM_UP_EI_EXPIRED is masked.
  373.          *
  374.          * TODO: verify if this can be reproduced on VLV,CHV.
  375.          */
  376.         if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
  377.                 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
  378.  
  379.         if (INTEL_INFO(dev_priv)->gen >= 8)
  380.                 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
  381.  
  382.         return mask;
  383. }
  384.  
  385. void gen6_disable_rps_interrupts(struct drm_device *dev)
  386. {
  387.         struct drm_i915_private *dev_priv = dev->dev_private;
  388.  
  389.         spin_lock_irq(&dev_priv->irq_lock);
  390.         dev_priv->rps.interrupts_enabled = false;
  391.         spin_unlock_irq(&dev_priv->irq_lock);
  392.  
  393.  
  394.         spin_lock_irq(&dev_priv->irq_lock);
  395.  
  396.         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
  397.  
  398.         __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
  399.         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
  400.                                 ~dev_priv->pm_rps_events);
  401.  
  402.         spin_unlock_irq(&dev_priv->irq_lock);
  403.  
  404. }
  405.  
  406. /**
  407.   * bdw_update_port_irq - update DE port interrupt
  408.   * @dev_priv: driver private
  409.   * @interrupt_mask: mask of interrupt bits to update
  410.   * @enabled_irq_mask: mask of interrupt bits to enable
  411.   */
  412. static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
  413.                                 uint32_t interrupt_mask,
  414.                                 uint32_t enabled_irq_mask)
  415. {
  416.         uint32_t new_val;
  417.         uint32_t old_val;
  418.  
  419.         assert_spin_locked(&dev_priv->irq_lock);
  420.  
  421.         WARN_ON(enabled_irq_mask & ~interrupt_mask);
  422.  
  423.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  424.                 return;
  425.  
  426.         old_val = I915_READ(GEN8_DE_PORT_IMR);
  427.  
  428.         new_val = old_val;
  429.         new_val &= ~interrupt_mask;
  430.         new_val |= (~enabled_irq_mask & interrupt_mask);
  431.  
  432.         if (new_val != old_val) {
  433.                 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
  434.                 POSTING_READ(GEN8_DE_PORT_IMR);
  435.         }
  436. }
  437.  
  438. /**
  439.  * bdw_update_pipe_irq - update DE pipe interrupt
  440.  * @dev_priv: driver private
  441.  * @pipe: pipe whose interrupt to update
  442.  * @interrupt_mask: mask of interrupt bits to update
  443.  * @enabled_irq_mask: mask of interrupt bits to enable
  444.  */
  445. void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
  446.                          enum pipe pipe,
  447.                          uint32_t interrupt_mask,
  448.                          uint32_t enabled_irq_mask)
  449. {
  450.         uint32_t new_val;
  451.  
  452.         assert_spin_locked(&dev_priv->irq_lock);
  453.  
  454.         WARN_ON(enabled_irq_mask & ~interrupt_mask);
  455.  
  456.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  457.                 return;
  458.  
  459.         new_val = dev_priv->de_irq_mask[pipe];
  460.         new_val &= ~interrupt_mask;
  461.         new_val |= (~enabled_irq_mask & interrupt_mask);
  462.  
  463.         if (new_val != dev_priv->de_irq_mask[pipe]) {
  464.                 dev_priv->de_irq_mask[pipe] = new_val;
  465.                 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
  466.                 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
  467.         }
  468. }
  469.  
  470. /**
  471.  * ibx_display_interrupt_update - update SDEIMR
  472.  * @dev_priv: driver private
  473.  * @interrupt_mask: mask of interrupt bits to update
  474.  * @enabled_irq_mask: mask of interrupt bits to enable
  475.  */
  476. void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
  477.                                   uint32_t interrupt_mask,
  478.                                   uint32_t enabled_irq_mask)
  479. {
  480.         uint32_t sdeimr = I915_READ(SDEIMR);
  481.         sdeimr &= ~interrupt_mask;
  482.         sdeimr |= (~enabled_irq_mask & interrupt_mask);
  483.  
  484.         WARN_ON(enabled_irq_mask & ~interrupt_mask);
  485.  
  486.         assert_spin_locked(&dev_priv->irq_lock);
  487.  
  488.         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
  489.                 return;
  490.  
  491.         I915_WRITE(SDEIMR, sdeimr);
  492.         POSTING_READ(SDEIMR);
  493. }
  494.  
  495. static void
  496. __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
  497.                        u32 enable_mask, u32 status_mask)
  498. {
  499.         i915_reg_t reg = PIPESTAT(pipe);
  500.         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
  501.  
  502.         assert_spin_locked(&dev_priv->irq_lock);
  503.         WARN_ON(!intel_irqs_enabled(dev_priv));
  504.  
  505.         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
  506.                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
  507.                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
  508.                       pipe_name(pipe), enable_mask, status_mask))
  509.                 return;
  510.  
  511.         if ((pipestat & enable_mask) == enable_mask)
  512.                 return;
  513.  
  514.         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
  515.  
  516.         /* Enable the interrupt, clear any pending status */
  517.         pipestat |= enable_mask | status_mask;
  518.         I915_WRITE(reg, pipestat);
  519.         POSTING_READ(reg);
  520. }
  521.  
  522. static void
  523. __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
  524.                         u32 enable_mask, u32 status_mask)
  525. {
  526.         i915_reg_t reg = PIPESTAT(pipe);
  527.         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
  528.  
  529.         assert_spin_locked(&dev_priv->irq_lock);
  530.         WARN_ON(!intel_irqs_enabled(dev_priv));
  531.  
  532.         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
  533.                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
  534.                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
  535.                       pipe_name(pipe), enable_mask, status_mask))
  536.                 return;
  537.  
  538.         if ((pipestat & enable_mask) == 0)
  539.                 return;
  540.  
  541.         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
  542.  
  543.         pipestat &= ~enable_mask;
  544.         I915_WRITE(reg, pipestat);
  545.         POSTING_READ(reg);
  546. }
  547.  
  548. static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
  549. {
  550.         u32 enable_mask = status_mask << 16;
  551.  
  552.         /*
  553.          * On pipe A we don't support the PSR interrupt yet,
  554.          * on pipe B and C the same bit MBZ.
  555.          */
  556.         if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
  557.                 return 0;
  558.         /*
  559.          * On pipe B and C we don't support the PSR interrupt yet, on pipe
  560.          * A the same bit is for perf counters which we don't use either.
  561.          */
  562.         if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
  563.                 return 0;
  564.  
  565.         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
  566.                          SPRITE0_FLIP_DONE_INT_EN_VLV |
  567.                          SPRITE1_FLIP_DONE_INT_EN_VLV);
  568.         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
  569.                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
  570.         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
  571.                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
  572.  
  573.         return enable_mask;
  574. }
  575.  
  576. void
  577. i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
  578.                      u32 status_mask)
  579. {
  580.         u32 enable_mask;
  581.  
  582.         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
  583.                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
  584.                                                            status_mask);
  585.         else
  586.                 enable_mask = status_mask << 16;
  587.         __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
  588. }
  589.  
  590. void
  591. i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
  592.                       u32 status_mask)
  593. {
  594.         u32 enable_mask;
  595.  
  596.         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
  597.                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
  598.                                                            status_mask);
  599.         else
  600.                 enable_mask = status_mask << 16;
  601.         __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
  602. }
  603.  
  604. /**
  605.  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
  606.  * @dev: drm device
  607.  */
  608. static void i915_enable_asle_pipestat(struct drm_device *dev)
  609. {
  610.         struct drm_i915_private *dev_priv = dev->dev_private;
  611.  
  612.         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
  613.                 return;
  614.  
  615.         spin_lock_irq(&dev_priv->irq_lock);
  616.  
  617.         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
  618.         if (INTEL_INFO(dev)->gen >= 4)
  619.                 i915_enable_pipestat(dev_priv, PIPE_A,
  620.                                      PIPE_LEGACY_BLC_EVENT_STATUS);
  621.  
  622.         spin_unlock_irq(&dev_priv->irq_lock);
  623. }
  624.  
  625. /*
  626.  * This timing diagram depicts the video signal in and
  627.  * around the vertical blanking period.
  628.  *
  629.  * Assumptions about the fictitious mode used in this example:
  630.  *  vblank_start >= 3
  631.  *  vsync_start = vblank_start + 1
  632.  *  vsync_end = vblank_start + 2
  633.  *  vtotal = vblank_start + 3
  634.  *
  635.  *           start of vblank:
  636.  *           latch double buffered registers
  637.  *           increment frame counter (ctg+)
  638.  *           generate start of vblank interrupt (gen4+)
  639.  *           |
  640.  *           |          frame start:
  641.  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
  642.  *           |          may be shifted forward 1-3 extra lines via PIPECONF
  643.  *           |          |
  644.  *           |          |  start of vsync:
  645.  *           |          |  generate vsync interrupt
  646.  *           |          |  |
  647.  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
  648.  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
  649.  * ----va---> <-----------------vb--------------------> <--------va-------------
  650.  *       |          |       <----vs----->                     |
  651.  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
  652.  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
  653.  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
  654.  *       |          |                                         |
  655.  *       last visible pixel                                   first visible pixel
  656.  *                  |                                         increment frame counter (gen3/4)
  657.  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
  658.  *
  659.  * x  = horizontal active
  660.  * _  = horizontal blanking
  661.  * hs = horizontal sync
  662.  * va = vertical active
  663.  * vb = vertical blanking
  664.  * vs = vertical sync
  665.  * vbs = vblank_start (number)
  666.  *
  667.  * Summary:
  668.  * - most events happen at the start of horizontal sync
  669.  * - frame start happens at the start of horizontal blank, 1-4 lines
  670.  *   (depending on PIPECONF settings) after the start of vblank
  671.  * - gen3/4 pixel and frame counter are synchronized with the start
  672.  *   of horizontal active on the first line of vertical active
  673.  */
  674.  
  675. static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
  676. {
  677.         /* Gen2 doesn't have a hardware frame counter */
  678.         return 0;
  679. }
  680.  
  681. /* Called from drm generic code, passed a 'crtc', which
  682.  * we use as a pipe index
  683.  */
  684. static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
  685. {
  686.         struct drm_i915_private *dev_priv = dev->dev_private;
  687.         i915_reg_t high_frame, low_frame;
  688.         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
  689.         struct intel_crtc *intel_crtc =
  690.                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
  691.         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
  692.  
  693.         htotal = mode->crtc_htotal;
  694.         hsync_start = mode->crtc_hsync_start;
  695.         vbl_start = mode->crtc_vblank_start;
  696.         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
  697.                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
  698.  
  699.         /* Convert to pixel count */
  700.         vbl_start *= htotal;
  701.  
  702.         /* Start of vblank event occurs at start of hsync */
  703.         vbl_start -= htotal - hsync_start;
  704.  
  705.         high_frame = PIPEFRAME(pipe);
  706.         low_frame = PIPEFRAMEPIXEL(pipe);
  707.  
  708.         /*
  709.          * High & low register fields aren't synchronized, so make sure
  710.          * we get a low value that's stable across two reads of the high
  711.          * register.
  712.          */
  713.         do {
  714.                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
  715.                 low   = I915_READ(low_frame);
  716.                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
  717.         } while (high1 != high2);
  718.  
  719.         high1 >>= PIPE_FRAME_HIGH_SHIFT;
  720.         pixel = low & PIPE_PIXEL_MASK;
  721.         low >>= PIPE_FRAME_LOW_SHIFT;
  722.  
  723.         /*
  724.          * The frame counter increments at beginning of active.
  725.          * Cook up a vblank counter by also checking the pixel
  726.          * counter against vblank start.
  727.          */
  728.         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
  729. }
  730.  
  731. static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
  732. {
  733.         struct drm_i915_private *dev_priv = dev->dev_private;
  734.  
  735.         return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
  736. }
  737.  
  738. /* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
  739. static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
  740. {
  741.         struct drm_device *dev = crtc->base.dev;
  742.         struct drm_i915_private *dev_priv = dev->dev_private;
  743.         const struct drm_display_mode *mode = &crtc->base.hwmode;
  744.         enum pipe pipe = crtc->pipe;
  745.         int position, vtotal;
  746.  
  747.         vtotal = mode->crtc_vtotal;
  748.         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
  749.                 vtotal /= 2;
  750.  
  751.         if (IS_GEN2(dev))
  752.                 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
  753.         else
  754.                 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
  755.  
  756.         /*
  757.          * On HSW, the DSL reg (0x70000) appears to return 0 if we
  758.          * read it just before the start of vblank.  So try it again
  759.          * so we don't accidentally end up spanning a vblank frame
  760.          * increment, causing the pipe_update_end() code to squak at us.
  761.          *
  762.          * The nature of this problem means we can't simply check the ISR
  763.          * bit and return the vblank start value; nor can we use the scanline
  764.          * debug register in the transcoder as it appears to have the same
  765.          * problem.  We may need to extend this to include other platforms,
  766.          * but so far testing only shows the problem on HSW.
  767.          */
  768.         if (HAS_DDI(dev) && !position) {
  769.                 int i, temp;
  770.  
  771.                 for (i = 0; i < 100; i++) {
  772.                         udelay(1);
  773.                         temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
  774.                                 DSL_LINEMASK_GEN3;
  775.                         if (temp != position) {
  776.                                 position = temp;
  777.                                 break;
  778.                         }
  779.                 }
  780.         }
  781.  
  782.         /*
  783.          * See update_scanline_offset() for the details on the
  784.          * scanline_offset adjustment.
  785.          */
  786.         return (position + crtc->scanline_offset) % vtotal;
  787. }
  788.  
  789. static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
  790.                                     unsigned int flags, int *vpos, int *hpos,
  791.                                     ktime_t *stime, ktime_t *etime,
  792.                                     const struct drm_display_mode *mode)
  793. {
  794.         struct drm_i915_private *dev_priv = dev->dev_private;
  795.         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
  796.         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
  797.         int position;
  798.         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
  799.         bool in_vbl = true;
  800.         int ret = 0;
  801.         unsigned long irqflags;
  802.  
  803.         if (WARN_ON(!mode->crtc_clock)) {
  804.                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
  805.                                  "pipe %c\n", pipe_name(pipe));
  806.                 return 0;
  807.         }
  808.  
  809.         htotal = mode->crtc_htotal;
  810.         hsync_start = mode->crtc_hsync_start;
  811.         vtotal = mode->crtc_vtotal;
  812.         vbl_start = mode->crtc_vblank_start;
  813.         vbl_end = mode->crtc_vblank_end;
  814.  
  815.         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
  816.                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
  817.                 vbl_end /= 2;
  818.                 vtotal /= 2;
  819.         }
  820.  
  821.         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
  822.  
  823.         /*
  824.          * Lock uncore.lock, as we will do multiple timing critical raw
  825.          * register reads, potentially with preemption disabled, so the
  826.          * following code must not block on uncore.lock.
  827.          */
  828.         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
  829.  
  830.         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
  831.  
  832.         /* Get optional system timestamp before query. */
  833.         if (stime)
  834.                 *stime = ktime_get();
  835.  
  836.         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
  837.                 /* No obvious pixelcount register. Only query vertical
  838.                  * scanout position from Display scan line register.
  839.                  */
  840.                 position = __intel_get_crtc_scanline(intel_crtc);
  841.         } else {
  842.                 /* Have access to pixelcount since start of frame.
  843.                  * We can split this into vertical and horizontal
  844.                  * scanout position.
  845.                  */
  846.                 position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
  847.  
  848.                 /* convert to pixel counts */
  849.                 vbl_start *= htotal;
  850.                 vbl_end *= htotal;
  851.                 vtotal *= htotal;
  852.  
  853.                 /*
  854.                  * In interlaced modes, the pixel counter counts all pixels,
  855.                  * so one field will have htotal more pixels. In order to avoid
  856.                  * the reported position from jumping backwards when the pixel
  857.                  * counter is beyond the length of the shorter field, just
  858.                  * clamp the position the length of the shorter field. This
  859.                  * matches how the scanline counter based position works since
  860.                  * the scanline counter doesn't count the two half lines.
  861.                  */
  862.                 if (position >= vtotal)
  863.                         position = vtotal - 1;
  864.  
  865.                 /*
  866.                  * Start of vblank interrupt is triggered at start of hsync,
  867.                  * just prior to the first active line of vblank. However we
  868.                  * consider lines to start at the leading edge of horizontal
  869.                  * active. So, should we get here before we've crossed into
  870.                  * the horizontal active of the first line in vblank, we would
  871.                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
  872.                  * always add htotal-hsync_start to the current pixel position.
  873.                  */
  874.                 position = (position + htotal - hsync_start) % vtotal;
  875.         }
  876.  
  877.         /* Get optional system timestamp after query. */
  878.         if (etime)
  879.                 *etime = ktime_get();
  880.  
  881.         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
  882.  
  883.         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
  884.  
  885.         in_vbl = position >= vbl_start && position < vbl_end;
  886.  
  887.         /*
  888.          * While in vblank, position will be negative
  889.          * counting up towards 0 at vbl_end. And outside
  890.          * vblank, position will be positive counting
  891.          * up since vbl_end.
  892.          */
  893.         if (position >= vbl_start)
  894.                 position -= vbl_end;
  895.         else
  896.                 position += vtotal - vbl_end;
  897.  
  898.         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
  899.                 *vpos = position;
  900.                 *hpos = 0;
  901.         } else {
  902.                 *vpos = position / htotal;
  903.                 *hpos = position - (*vpos * htotal);
  904.         }
  905.  
  906.         /* In vblank? */
  907.         if (in_vbl)
  908.                 ret |= DRM_SCANOUTPOS_IN_VBLANK;
  909.  
  910.         return ret;
  911. }
  912.  
  913. int intel_get_crtc_scanline(struct intel_crtc *crtc)
  914. {
  915.         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
  916.         unsigned long irqflags;
  917.         int position;
  918.  
  919.         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
  920.         position = __intel_get_crtc_scanline(crtc);
  921.         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
  922.  
  923.         return position;
  924. }
  925.  
  926. static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
  927.                               int *max_error,
  928.                               struct timeval *vblank_time,
  929.                               unsigned flags)
  930. {
  931.         struct drm_crtc *crtc;
  932.  
  933.         if (pipe >= INTEL_INFO(dev)->num_pipes) {
  934.                 DRM_ERROR("Invalid crtc %u\n", pipe);
  935.                 return -EINVAL;
  936.         }
  937.  
  938.         /* Get drm_crtc to timestamp: */
  939.         crtc = intel_get_crtc_for_pipe(dev, pipe);
  940.         if (crtc == NULL) {
  941.                 DRM_ERROR("Invalid crtc %u\n", pipe);
  942.                 return -EINVAL;
  943.         }
  944.  
  945.         if (!crtc->hwmode.crtc_clock) {
  946.                 DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
  947.                 return -EBUSY;
  948.         }
  949.  
  950.         /* Helper routine in DRM core does all the work: */
  951.         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
  952.                                                      vblank_time, flags,
  953.                                                      &crtc->hwmode);
  954. }
  955.  
  956. static void ironlake_rps_change_irq_handler(struct drm_device *dev)
  957. {
  958.         struct drm_i915_private *dev_priv = dev->dev_private;
  959.         u32 busy_up, busy_down, max_avg, min_avg;
  960.         u8 new_delay;
  961.  
  962.         spin_lock(&mchdev_lock);
  963.  
  964.         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
  965.  
  966.         new_delay = dev_priv->ips.cur_delay;
  967.  
  968.         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
  969.         busy_up = I915_READ(RCPREVBSYTUPAVG);
  970.         busy_down = I915_READ(RCPREVBSYTDNAVG);
  971.         max_avg = I915_READ(RCBMAXAVG);
  972.         min_avg = I915_READ(RCBMINAVG);
  973.  
  974.         /* Handle RCS change request from hw */
  975.         if (busy_up > max_avg) {
  976.                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
  977.                         new_delay = dev_priv->ips.cur_delay - 1;
  978.                 if (new_delay < dev_priv->ips.max_delay)
  979.                         new_delay = dev_priv->ips.max_delay;
  980.         } else if (busy_down < min_avg) {
  981.                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
  982.                         new_delay = dev_priv->ips.cur_delay + 1;
  983.                 if (new_delay > dev_priv->ips.min_delay)
  984.                         new_delay = dev_priv->ips.min_delay;
  985.         }
  986.  
  987.         if (ironlake_set_drps(dev, new_delay))
  988.                 dev_priv->ips.cur_delay = new_delay;
  989.  
  990.         spin_unlock(&mchdev_lock);
  991.  
  992.         return;
  993. }
  994.  
  995. static void notify_ring(struct intel_engine_cs *ring)
  996. {
  997.         if (!intel_ring_initialized(ring))
  998.                 return;
  999.  
  1000.         trace_i915_gem_request_notify(ring);
  1001.  
  1002.         wake_up_all(&ring->irq_queue);
  1003. }
  1004.  
  1005. static void vlv_c0_read(struct drm_i915_private *dev_priv,
  1006.                         struct intel_rps_ei *ei)
  1007. {
  1008.         ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
  1009.         ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
  1010.         ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
  1011. }
  1012.  
  1013. static bool vlv_c0_above(struct drm_i915_private *dev_priv,
  1014.                          const struct intel_rps_ei *old,
  1015.                          const struct intel_rps_ei *now,
  1016.                          int threshold)
  1017. {
  1018.         u64 time, c0;
  1019.         unsigned int mul = 100;
  1020.  
  1021.         if (old->cz_clock == 0)
  1022.                 return false;
  1023.  
  1024.         if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
  1025.                 mul <<= 8;
  1026.  
  1027.         time = now->cz_clock - old->cz_clock;
  1028.         time *= threshold * dev_priv->czclk_freq;
  1029.  
  1030.         /* Workload can be split between render + media, e.g. SwapBuffers
  1031.          * being blitted in X after being rendered in mesa. To account for
  1032.          * this we need to combine both engines into our activity counter.
  1033.          */
  1034.         c0 = now->render_c0 - old->render_c0;
  1035.         c0 += now->media_c0 - old->media_c0;
  1036.         c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
  1037.  
  1038.         return c0 >= time;
  1039. }
  1040.  
  1041. void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
  1042. {
  1043.         vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
  1044.         dev_priv->rps.up_ei = dev_priv->rps.down_ei;
  1045. }
  1046.  
  1047. static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
  1048. {
  1049.         struct intel_rps_ei now;
  1050.         u32 events = 0;
  1051.  
  1052.         if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
  1053.                 return 0;
  1054.  
  1055.         vlv_c0_read(dev_priv, &now);
  1056.         if (now.cz_clock == 0)
  1057.                 return 0;
  1058.  
  1059.         if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
  1060.                 if (!vlv_c0_above(dev_priv,
  1061.                                   &dev_priv->rps.down_ei, &now,
  1062.                                   dev_priv->rps.down_threshold))
  1063.                         events |= GEN6_PM_RP_DOWN_THRESHOLD;
  1064.                 dev_priv->rps.down_ei = now;
  1065.         }
  1066.  
  1067.         if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
  1068.                 if (vlv_c0_above(dev_priv,
  1069.                                  &dev_priv->rps.up_ei, &now,
  1070.                                  dev_priv->rps.up_threshold))
  1071.                         events |= GEN6_PM_RP_UP_THRESHOLD;
  1072.                 dev_priv->rps.up_ei = now;
  1073.         }
  1074.  
  1075.         return events;
  1076. }
  1077.  
  1078. static bool any_waiters(struct drm_i915_private *dev_priv)
  1079. {
  1080.         struct intel_engine_cs *ring;
  1081.         int i;
  1082.  
  1083.         for_each_ring(ring, dev_priv, i)
  1084.                 if (ring->irq_refcount)
  1085.                         return true;
  1086.  
  1087.         return false;
  1088. }
  1089.  
  1090. static void gen6_pm_rps_work(struct work_struct *work)
  1091. {
  1092.         struct drm_i915_private *dev_priv =
  1093.                 container_of(work, struct drm_i915_private, rps.work);
  1094.         bool client_boost;
  1095.         int new_delay, adj, min, max;
  1096.         u32 pm_iir;
  1097.  
  1098.         spin_lock_irq(&dev_priv->irq_lock);
  1099.         /* Speed up work cancelation during disabling rps interrupts. */
  1100.         if (!dev_priv->rps.interrupts_enabled) {
  1101.                 spin_unlock_irq(&dev_priv->irq_lock);
  1102.                 return;
  1103.         }
  1104.  
  1105.         /*
  1106.          * The RPS work is synced during runtime suspend, we don't require a
  1107.          * wakeref. TODO: instead of disabling the asserts make sure that we
  1108.          * always hold an RPM reference while the work is running.
  1109.          */
  1110.         DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);
  1111.  
  1112.         pm_iir = dev_priv->rps.pm_iir;
  1113.         dev_priv->rps.pm_iir = 0;
  1114.         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
  1115.         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
  1116.         client_boost = dev_priv->rps.client_boost;
  1117.         dev_priv->rps.client_boost = false;
  1118.         spin_unlock_irq(&dev_priv->irq_lock);
  1119.  
  1120.         /* Make sure we didn't queue anything we're not going to process. */
  1121.         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
  1122.  
  1123.         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
  1124.                 goto out;
  1125.  
  1126.         mutex_lock(&dev_priv->rps.hw_lock);
  1127.  
  1128.         pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
  1129.  
  1130.         adj = dev_priv->rps.last_adj;
  1131.         new_delay = dev_priv->rps.cur_freq;
  1132.         min = dev_priv->rps.min_freq_softlimit;
  1133.         max = dev_priv->rps.max_freq_softlimit;
  1134.  
  1135.         if (client_boost) {
  1136.                 new_delay = dev_priv->rps.max_freq_softlimit;
  1137.                 adj = 0;
  1138.         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
  1139.                 if (adj > 0)
  1140.                         adj *= 2;
  1141.                 else /* CHV needs even encode values */
  1142.                         adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
  1143.                 /*
  1144.                  * For better performance, jump directly
  1145.                  * to RPe if we're below it.
  1146.                  */
  1147.                 if (new_delay < dev_priv->rps.efficient_freq - adj) {
  1148.                         new_delay = dev_priv->rps.efficient_freq;
  1149.                         adj = 0;
  1150.                 }
  1151.         } else if (any_waiters(dev_priv)) {
  1152.                 adj = 0;
  1153.         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
  1154.                 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
  1155.                         new_delay = dev_priv->rps.efficient_freq;
  1156.                 else
  1157.                         new_delay = dev_priv->rps.min_freq_softlimit;
  1158.                 adj = 0;
  1159.         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
  1160.                 if (adj < 0)
  1161.                         adj *= 2;
  1162.                 else /* CHV needs even encode values */
  1163.                         adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
  1164.         } else { /* unknown event */
  1165.                 adj = 0;
  1166.         }
  1167.  
  1168.         dev_priv->rps.last_adj = adj;
  1169.  
  1170.         /* sysfs frequency interfaces may have snuck in while servicing the
  1171.          * interrupt
  1172.          */
  1173.         new_delay += adj;
  1174.         new_delay = clamp_t(int, new_delay, min, max);
  1175.  
  1176.         intel_set_rps(dev_priv->dev, new_delay);
  1177.  
  1178.         mutex_unlock(&dev_priv->rps.hw_lock);
  1179. out:
  1180.         ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
  1181. }
  1182.  
  1183.  
  1184. /**
  1185.  * ivybridge_parity_work - Workqueue called when a parity error interrupt
  1186.  * occurred.
  1187.  * @work: workqueue struct
  1188.  *
  1189.  * Doesn't actually do anything except notify userspace. As a consequence of
  1190.  * this event, userspace should try to remap the bad rows since statistically
  1191.  * it is likely the same row is more likely to go bad again.
  1192.  */
  1193. static void ivybridge_parity_work(struct work_struct *work)
  1194. {
  1195.         struct drm_i915_private *dev_priv =
  1196.                 container_of(work, struct drm_i915_private, l3_parity.error_work);
  1197.         u32 error_status, row, bank, subbank;
  1198.         char *parity_event[6];
  1199.         uint32_t misccpctl;
  1200.         uint8_t slice = 0;
  1201.  
  1202.         /* We must turn off DOP level clock gating to access the L3 registers.
  1203.          * In order to prevent a get/put style interface, acquire struct mutex
  1204.          * any time we access those registers.
  1205.          */
  1206.         mutex_lock(&dev_priv->dev->struct_mutex);
  1207.  
  1208.         /* If we've screwed up tracking, just let the interrupt fire again */
  1209.         if (WARN_ON(!dev_priv->l3_parity.which_slice))
  1210.                 goto out;
  1211.  
  1212.         misccpctl = I915_READ(GEN7_MISCCPCTL);
  1213.         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  1214.         POSTING_READ(GEN7_MISCCPCTL);
  1215.  
  1216.         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
  1217.                 i915_reg_t reg;
  1218.  
  1219.                 slice--;
  1220.                 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
  1221.                         break;
  1222.  
  1223.                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
  1224.  
  1225.                 reg = GEN7_L3CDERRST1(slice);
  1226.  
  1227.                 error_status = I915_READ(reg);
  1228.                 row = GEN7_PARITY_ERROR_ROW(error_status);
  1229.                 bank = GEN7_PARITY_ERROR_BANK(error_status);
  1230.                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
  1231.  
  1232.                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
  1233.                 POSTING_READ(reg);
  1234.  
  1235.                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
  1236.                           slice, row, bank, subbank);
  1237.  
  1238.         }
  1239.  
  1240.         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
  1241.  
  1242. out:
  1243.         WARN_ON(dev_priv->l3_parity.which_slice);
  1244.         spin_lock_irq(&dev_priv->irq_lock);
  1245.         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
  1246.         spin_unlock_irq(&dev_priv->irq_lock);
  1247.  
  1248.         mutex_unlock(&dev_priv->dev->struct_mutex);
  1249. }
  1250.  
  1251. static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
  1252. {
  1253.         struct drm_i915_private *dev_priv = dev->dev_private;
  1254.  
  1255.         if (!HAS_L3_DPF(dev))
  1256.                 return;
  1257.  
  1258.         spin_lock(&dev_priv->irq_lock);
  1259.         gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
  1260.         spin_unlock(&dev_priv->irq_lock);
  1261.  
  1262.         iir &= GT_PARITY_ERROR(dev);
  1263.         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
  1264.                 dev_priv->l3_parity.which_slice |= 1 << 1;
  1265.  
  1266.         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
  1267.                 dev_priv->l3_parity.which_slice |= 1 << 0;
  1268.  
  1269.         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
  1270. }
  1271.  
  1272. static void ilk_gt_irq_handler(struct drm_device *dev,
  1273.                                struct drm_i915_private *dev_priv,
  1274.                                u32 gt_iir)
  1275. {
  1276.         if (gt_iir &
  1277.             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
  1278.                 notify_ring(&dev_priv->ring[RCS]);
  1279.         if (gt_iir & ILK_BSD_USER_INTERRUPT)
  1280.                 notify_ring(&dev_priv->ring[VCS]);
  1281. }
  1282.  
  1283. static void snb_gt_irq_handler(struct drm_device *dev,
  1284.                                struct drm_i915_private *dev_priv,
  1285.                                u32 gt_iir)
  1286. {
  1287.  
  1288.         if (gt_iir &
  1289.             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
  1290.                 notify_ring(&dev_priv->ring[RCS]);
  1291.         if (gt_iir & GT_BSD_USER_INTERRUPT)
  1292.                 notify_ring(&dev_priv->ring[VCS]);
  1293.         if (gt_iir & GT_BLT_USER_INTERRUPT)
  1294.                 notify_ring(&dev_priv->ring[BCS]);
  1295.  
  1296.         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
  1297.                       GT_BSD_CS_ERROR_INTERRUPT |
  1298.                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
  1299.                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
  1300.  
  1301.         if (gt_iir & GT_PARITY_ERROR(dev))
  1302.                 ivybridge_parity_error_irq_handler(dev, gt_iir);
  1303. }
  1304.  
  1305. static __always_inline void
  1306. gen8_cs_irq_handler(struct intel_engine_cs *ring, u32 iir, int test_shift)
  1307. {
  1308.         if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
  1309.                 notify_ring(ring);
  1310.         if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift))
  1311.                 intel_lrc_irq_handler(ring);
  1312. }
  1313.  
  1314. static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
  1315.                                        u32 master_ctl)
  1316. {
  1317.         irqreturn_t ret = IRQ_NONE;
  1318.  
  1319.         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
  1320.                 u32 iir = I915_READ_FW(GEN8_GT_IIR(0));
  1321.                 if (iir) {
  1322.                         I915_WRITE_FW(GEN8_GT_IIR(0), iir);
  1323.                         ret = IRQ_HANDLED;
  1324.  
  1325.                         gen8_cs_irq_handler(&dev_priv->ring[RCS],
  1326.                                         iir, GEN8_RCS_IRQ_SHIFT);
  1327.  
  1328.                         gen8_cs_irq_handler(&dev_priv->ring[BCS],
  1329.                                         iir, GEN8_BCS_IRQ_SHIFT);
  1330.                 } else
  1331.                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
  1332.         }
  1333.  
  1334.         if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
  1335.                 u32 iir = I915_READ_FW(GEN8_GT_IIR(1));
  1336.                 if (iir) {
  1337.                         I915_WRITE_FW(GEN8_GT_IIR(1), iir);
  1338.                         ret = IRQ_HANDLED;
  1339.  
  1340.                         gen8_cs_irq_handler(&dev_priv->ring[VCS],
  1341.                                         iir, GEN8_VCS1_IRQ_SHIFT);
  1342.  
  1343.                         gen8_cs_irq_handler(&dev_priv->ring[VCS2],
  1344.                                         iir, GEN8_VCS2_IRQ_SHIFT);
  1345.                 } else
  1346.                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
  1347.         }
  1348.  
  1349.         if (master_ctl & GEN8_GT_VECS_IRQ) {
  1350.                 u32 iir = I915_READ_FW(GEN8_GT_IIR(3));
  1351.                 if (iir) {
  1352.                         I915_WRITE_FW(GEN8_GT_IIR(3), iir);
  1353.                         ret = IRQ_HANDLED;
  1354.  
  1355.                         gen8_cs_irq_handler(&dev_priv->ring[VECS],
  1356.                                         iir, GEN8_VECS_IRQ_SHIFT);
  1357.                 } else
  1358.                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
  1359.         }
  1360.  
  1361.         if (master_ctl & GEN8_GT_PM_IRQ) {
  1362.                 u32 iir = I915_READ_FW(GEN8_GT_IIR(2));
  1363.                 if (iir & dev_priv->pm_rps_events) {
  1364.                         I915_WRITE_FW(GEN8_GT_IIR(2),
  1365.                                       iir & dev_priv->pm_rps_events);
  1366.                         ret = IRQ_HANDLED;
  1367.                         gen6_rps_irq_handler(dev_priv, iir);
  1368.                 } else
  1369.                         DRM_ERROR("The master control interrupt lied (PM)!\n");
  1370.         }
  1371.  
  1372.         return ret;
  1373. }
  1374.  
  1375. static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
  1376. {
  1377.         switch (port) {
  1378.         case PORT_A:
  1379.                 return val & PORTA_HOTPLUG_LONG_DETECT;
  1380.         case PORT_B:
  1381.                 return val & PORTB_HOTPLUG_LONG_DETECT;
  1382.         case PORT_C:
  1383.                 return val & PORTC_HOTPLUG_LONG_DETECT;
  1384.         default:
  1385.                 return false;
  1386.         }
  1387. }
  1388.  
  1389. static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
  1390. {
  1391.         switch (port) {
  1392.         case PORT_E:
  1393.                 return val & PORTE_HOTPLUG_LONG_DETECT;
  1394.         default:
  1395.                 return false;
  1396.         }
  1397. }
  1398.  
  1399. static bool spt_port_hotplug_long_detect(enum port port, u32 val)
  1400. {
  1401.         switch (port) {
  1402.         case PORT_A:
  1403.                 return val & PORTA_HOTPLUG_LONG_DETECT;
  1404.         case PORT_B:
  1405.                 return val & PORTB_HOTPLUG_LONG_DETECT;
  1406.         case PORT_C:
  1407.                 return val & PORTC_HOTPLUG_LONG_DETECT;
  1408.         case PORT_D:
  1409.                 return val & PORTD_HOTPLUG_LONG_DETECT;
  1410.         default:
  1411.                 return false;
  1412.         }
  1413. }
  1414.  
  1415. static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
  1416. {
  1417.         switch (port) {
  1418.         case PORT_A:
  1419.                 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
  1420.         default:
  1421.                 return false;
  1422.         }
  1423. }
  1424.  
  1425. static bool pch_port_hotplug_long_detect(enum port port, u32 val)
  1426. {
  1427.         switch (port) {
  1428.         case PORT_B:
  1429.                 return val & PORTB_HOTPLUG_LONG_DETECT;
  1430.         case PORT_C:
  1431.                 return val & PORTC_HOTPLUG_LONG_DETECT;
  1432.         case PORT_D:
  1433.                 return val & PORTD_HOTPLUG_LONG_DETECT;
  1434.         default:
  1435.                 return false;
  1436.         }
  1437. }
  1438.  
  1439. static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
  1440. {
  1441.         switch (port) {
  1442.         case PORT_B:
  1443.                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
  1444.         case PORT_C:
  1445.                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
  1446.         case PORT_D:
  1447.                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
  1448.         default:
  1449.                 return false;
  1450.         }
  1451. }
  1452.  
  1453. /*
  1454.  * Get a bit mask of pins that have triggered, and which ones may be long.
  1455.  * This can be called multiple times with the same masks to accumulate
  1456.  * hotplug detection results from several registers.
  1457.  *
  1458.  * Note that the caller is expected to zero out the masks initially.
  1459.  */
  1460. static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
  1461.                              u32 hotplug_trigger, u32 dig_hotplug_reg,
  1462.                              const u32 hpd[HPD_NUM_PINS],
  1463.                              bool long_pulse_detect(enum port port, u32 val))
  1464. {
  1465.         enum port port;
  1466.         int i;
  1467.  
  1468.         for_each_hpd_pin(i) {
  1469.                 if ((hpd[i] & hotplug_trigger) == 0)
  1470.                         continue;
  1471.  
  1472.                 *pin_mask |= BIT(i);
  1473.  
  1474.                 if (!intel_hpd_pin_to_port(i, &port))
  1475.                         continue;
  1476.  
  1477.                 if (long_pulse_detect(port, dig_hotplug_reg))
  1478.                         *long_mask |= BIT(i);
  1479.         }
  1480.  
  1481.         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
  1482.                          hotplug_trigger, dig_hotplug_reg, *pin_mask);
  1483.  
  1484. }
  1485.  
  1486. static void gmbus_irq_handler(struct drm_device *dev)
  1487. {
  1488.         struct drm_i915_private *dev_priv = dev->dev_private;
  1489.  
  1490.         wake_up_all(&dev_priv->gmbus_wait_queue);
  1491. }
  1492.  
  1493. static void dp_aux_irq_handler(struct drm_device *dev)
  1494. {
  1495.         struct drm_i915_private *dev_priv = dev->dev_private;
  1496.  
  1497.         wake_up_all(&dev_priv->gmbus_wait_queue);
  1498. }
  1499.  
  1500. #if defined(CONFIG_DEBUG_FS)
  1501. static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
  1502.                                          uint32_t crc0, uint32_t crc1,
  1503.                                          uint32_t crc2, uint32_t crc3,
  1504.                                          uint32_t crc4)
  1505. {
  1506.         struct drm_i915_private *dev_priv = dev->dev_private;
  1507.         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
  1508.         struct intel_pipe_crc_entry *entry;
  1509.         int head, tail;
  1510.  
  1511.         spin_lock(&pipe_crc->lock);
  1512.  
  1513.         if (!pipe_crc->entries) {
  1514.                 spin_unlock(&pipe_crc->lock);
  1515.                 DRM_DEBUG_KMS("spurious interrupt\n");
  1516.                 return;
  1517.         }
  1518.  
  1519.         head = pipe_crc->head;
  1520.         tail = pipe_crc->tail;
  1521.  
  1522.         if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
  1523.                 spin_unlock(&pipe_crc->lock);
  1524.                 DRM_ERROR("CRC buffer overflowing\n");
  1525.                 return;
  1526.         }
  1527.  
  1528.         entry = &pipe_crc->entries[head];
  1529.  
  1530.         entry->frame = dev->driver->get_vblank_counter(dev, pipe);
  1531.         entry->crc[0] = crc0;
  1532.         entry->crc[1] = crc1;
  1533.         entry->crc[2] = crc2;
  1534.         entry->crc[3] = crc3;
  1535.         entry->crc[4] = crc4;
  1536.  
  1537.         head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
  1538.         pipe_crc->head = head;
  1539.  
  1540.         spin_unlock(&pipe_crc->lock);
  1541.  
  1542.         wake_up_interruptible(&pipe_crc->wq);
  1543. }
  1544. #else
  1545. static inline void
  1546. display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
  1547.                              uint32_t crc0, uint32_t crc1,
  1548.                              uint32_t crc2, uint32_t crc3,
  1549.                              uint32_t crc4) {}
  1550. #endif
  1551.  
  1552.  
  1553. static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
  1554. {
  1555.         struct drm_i915_private *dev_priv = dev->dev_private;
  1556.  
  1557.         display_pipe_crc_irq_handler(dev, pipe,
  1558.                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
  1559.                                      0, 0, 0, 0);
  1560. }
  1561.  
  1562. static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
  1563. {
  1564.         struct drm_i915_private *dev_priv = dev->dev_private;
  1565.  
  1566.         display_pipe_crc_irq_handler(dev, pipe,
  1567.                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
  1568.                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
  1569.                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
  1570.                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
  1571.                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
  1572. }
  1573.  
  1574. static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
  1575. {
  1576.         struct drm_i915_private *dev_priv = dev->dev_private;
  1577.         uint32_t res1, res2;
  1578.  
  1579.         if (INTEL_INFO(dev)->gen >= 3)
  1580.                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
  1581.         else
  1582.                 res1 = 0;
  1583.  
  1584.         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
  1585.                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
  1586.         else
  1587.                 res2 = 0;
  1588.  
  1589.         display_pipe_crc_irq_handler(dev, pipe,
  1590.                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
  1591.                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
  1592.                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
  1593.                                      res1, res2);
  1594. }
  1595.  
  1596. /* The RPS events need forcewake, so we add them to a work queue and mask their
  1597.  * IMR bits until the work is done. Other interrupts can be processed without
  1598.  * the work queue. */
  1599. static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
  1600. {
  1601.         if (pm_iir & dev_priv->pm_rps_events) {
  1602.                 spin_lock(&dev_priv->irq_lock);
  1603.                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
  1604.                 if (dev_priv->rps.interrupts_enabled) {
  1605.                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
  1606.                         queue_work(dev_priv->wq, &dev_priv->rps.work);
  1607.                 }
  1608.                 spin_unlock(&dev_priv->irq_lock);
  1609.         }
  1610.  
  1611.         if (INTEL_INFO(dev_priv)->gen >= 8)
  1612.                 return;
  1613.  
  1614.         if (HAS_VEBOX(dev_priv->dev)) {
  1615.                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
  1616.                         notify_ring(&dev_priv->ring[VECS]);
  1617.  
  1618.                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
  1619.                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
  1620.         }
  1621. }
  1622.  
  1623. static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
  1624. {
  1625.         if (!drm_handle_vblank(dev, pipe))
  1626.                 return false;
  1627.  
  1628.         return true;
  1629. }
  1630.  
  1631. static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
  1632. {
  1633.         struct drm_i915_private *dev_priv = dev->dev_private;
  1634.         u32 pipe_stats[I915_MAX_PIPES] = { };
  1635.         int pipe;
  1636.  
  1637.         spin_lock(&dev_priv->irq_lock);
  1638.         for_each_pipe(dev_priv, pipe) {
  1639.                 i915_reg_t reg;
  1640.                 u32 mask, iir_bit = 0;
  1641.  
  1642.                 /*
  1643.                  * PIPESTAT bits get signalled even when the interrupt is
  1644.                  * disabled with the mask bits, and some of the status bits do
  1645.                  * not generate interrupts at all (like the underrun bit). Hence
  1646.                  * we need to be careful that we only handle what we want to
  1647.                  * handle.
  1648.                  */
  1649.  
  1650.                 /* fifo underruns are filterered in the underrun handler. */
  1651.                 mask = PIPE_FIFO_UNDERRUN_STATUS;
  1652.  
  1653.                 switch (pipe) {
  1654.                 case PIPE_A:
  1655.                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
  1656.                         break;
  1657.                 case PIPE_B:
  1658.                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
  1659.                         break;
  1660.                 case PIPE_C:
  1661.                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
  1662.                         break;
  1663.                 }
  1664.                 if (iir & iir_bit)
  1665.                         mask |= dev_priv->pipestat_irq_mask[pipe];
  1666.  
  1667.                 if (!mask)
  1668.                         continue;
  1669.  
  1670.                 reg = PIPESTAT(pipe);
  1671.                 mask |= PIPESTAT_INT_ENABLE_MASK;
  1672.                 pipe_stats[pipe] = I915_READ(reg) & mask;
  1673.  
  1674.                 /*
  1675.                  * Clear the PIPE*STAT regs before the IIR
  1676.                  */
  1677.                 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
  1678.                                         PIPESTAT_INT_STATUS_MASK))
  1679.                         I915_WRITE(reg, pipe_stats[pipe]);
  1680.         }
  1681.         spin_unlock(&dev_priv->irq_lock);
  1682.  
  1683.         for_each_pipe(dev_priv, pipe) {
  1684.                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
  1685.                     intel_pipe_handle_vblank(dev, pipe))
  1686.                         intel_check_page_flip(dev, pipe);
  1687.  
  1688.                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
  1689.                         intel_prepare_page_flip(dev, pipe);
  1690.                         intel_finish_page_flip(dev, pipe);
  1691.                 }
  1692.  
  1693.                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
  1694.                         i9xx_pipe_crc_irq_handler(dev, pipe);
  1695.  
  1696.                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
  1697.                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
  1698.         }
  1699.  
  1700.         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
  1701.                 gmbus_irq_handler(dev);
  1702. }
  1703.  
  1704. static void i9xx_hpd_irq_handler(struct drm_device *dev)
  1705. {
  1706.         struct drm_i915_private *dev_priv = dev->dev_private;
  1707.         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
  1708.         u32 pin_mask = 0, long_mask = 0;
  1709.  
  1710.         if (!hotplug_status)
  1711.                 return;
  1712.  
  1713.         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
  1714.         /*
  1715.          * Make sure hotplug status is cleared before we clear IIR, or else we
  1716.          * may miss hotplug events.
  1717.          */
  1718.         POSTING_READ(PORT_HOTPLUG_STAT);
  1719.  
  1720.         if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
  1721.                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
  1722.  
  1723.                 if (hotplug_trigger) {
  1724.                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
  1725.                                            hotplug_trigger, hpd_status_g4x,
  1726.                                            i9xx_port_hotplug_long_detect);
  1727.  
  1728.                         intel_hpd_irq_handler(dev, pin_mask, long_mask);
  1729.                 }
  1730.  
  1731.                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
  1732.                         dp_aux_irq_handler(dev);
  1733.         } else {
  1734.                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
  1735.  
  1736.                 if (hotplug_trigger) {
  1737.                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
  1738.                                            hotplug_trigger, hpd_status_i915,
  1739.                                            i9xx_port_hotplug_long_detect);
  1740.                         intel_hpd_irq_handler(dev, pin_mask, long_mask);
  1741.                 }
  1742.         }
  1743. }
  1744.  
  1745. static irqreturn_t valleyview_irq_handler(int irq, void *arg)
  1746. {
  1747.         struct drm_device *dev = arg;
  1748.         struct drm_i915_private *dev_priv = dev->dev_private;
  1749.         u32 iir, gt_iir, pm_iir;
  1750.         irqreturn_t ret = IRQ_NONE;
  1751.  
  1752.         if (!intel_irqs_enabled(dev_priv))
  1753.                 return IRQ_NONE;
  1754.  
  1755.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  1756.         disable_rpm_wakeref_asserts(dev_priv);
  1757.  
  1758.         while (true) {
  1759.                 /* Find, clear, then process each source of interrupt */
  1760.  
  1761.                 gt_iir = I915_READ(GTIIR);
  1762.                 if (gt_iir)
  1763.                         I915_WRITE(GTIIR, gt_iir);
  1764.  
  1765.                 pm_iir = I915_READ(GEN6_PMIIR);
  1766.                 if (pm_iir)
  1767.                         I915_WRITE(GEN6_PMIIR, pm_iir);
  1768.  
  1769.                 iir = I915_READ(VLV_IIR);
  1770.                 if (iir) {
  1771.                         /* Consume port before clearing IIR or we'll miss events */
  1772.                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
  1773.                                 i9xx_hpd_irq_handler(dev);
  1774.                         I915_WRITE(VLV_IIR, iir);
  1775.                 }
  1776.  
  1777.                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
  1778.                         goto out;
  1779.  
  1780.                 ret = IRQ_HANDLED;
  1781.  
  1782.                 if (gt_iir)
  1783.                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
  1784.                 if (pm_iir)
  1785.                         gen6_rps_irq_handler(dev_priv, pm_iir);
  1786.                 /* Call regardless, as some status bits might not be
  1787.                  * signalled in iir */
  1788.                 valleyview_pipestat_irq_handler(dev, iir);
  1789.         }
  1790.  
  1791. out:
  1792.         enable_rpm_wakeref_asserts(dev_priv);
  1793.  
  1794.         return ret;
  1795. }
  1796.  
  1797. static irqreturn_t cherryview_irq_handler(int irq, void *arg)
  1798. {
  1799.         struct drm_device *dev = arg;
  1800.         struct drm_i915_private *dev_priv = dev->dev_private;
  1801.         u32 master_ctl, iir;
  1802.         irqreturn_t ret = IRQ_NONE;
  1803.  
  1804.         if (!intel_irqs_enabled(dev_priv))
  1805.                 return IRQ_NONE;
  1806.  
  1807.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  1808.         disable_rpm_wakeref_asserts(dev_priv);
  1809.  
  1810.         for (;;) {
  1811.                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
  1812.                 iir = I915_READ(VLV_IIR);
  1813.  
  1814.                 if (master_ctl == 0 && iir == 0)
  1815.                         break;
  1816.  
  1817.                 ret = IRQ_HANDLED;
  1818.  
  1819.                 I915_WRITE(GEN8_MASTER_IRQ, 0);
  1820.  
  1821.                 /* Find, clear, then process each source of interrupt */
  1822.  
  1823.                 if (iir) {
  1824.                         /* Consume port before clearing IIR or we'll miss events */
  1825.                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
  1826.                                 i9xx_hpd_irq_handler(dev);
  1827.                         I915_WRITE(VLV_IIR, iir);
  1828.                 }
  1829.  
  1830.                 gen8_gt_irq_handler(dev_priv, master_ctl);
  1831.  
  1832.                 /* Call regardless, as some status bits might not be
  1833.                  * signalled in iir */
  1834.                 valleyview_pipestat_irq_handler(dev, iir);
  1835.  
  1836.                 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
  1837.                 POSTING_READ(GEN8_MASTER_IRQ);
  1838.         }
  1839.  
  1840.         enable_rpm_wakeref_asserts(dev_priv);
  1841.  
  1842.         return ret;
  1843. }
  1844.  
  1845. static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
  1846.                                 const u32 hpd[HPD_NUM_PINS])
  1847. {
  1848.         struct drm_i915_private *dev_priv = to_i915(dev);
  1849.         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
  1850.  
  1851.         /*
  1852.          * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
  1853.          * unless we touch the hotplug register, even if hotplug_trigger is
  1854.          * zero. Not acking leads to "The master control interrupt lied (SDE)!"
  1855.          * errors.
  1856.          */
  1857.         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
  1858.         if (!hotplug_trigger) {
  1859.                 u32 mask = PORTA_HOTPLUG_STATUS_MASK |
  1860.                         PORTD_HOTPLUG_STATUS_MASK |
  1861.                         PORTC_HOTPLUG_STATUS_MASK |
  1862.                         PORTB_HOTPLUG_STATUS_MASK;
  1863.                 dig_hotplug_reg &= ~mask;
  1864.         }
  1865.  
  1866.         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
  1867.         if (!hotplug_trigger)
  1868.                 return;
  1869.  
  1870.         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
  1871.                            dig_hotplug_reg, hpd,
  1872.                            pch_port_hotplug_long_detect);
  1873.  
  1874.         intel_hpd_irq_handler(dev, pin_mask, long_mask);
  1875. }
  1876.  
  1877. static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
  1878. {
  1879.         struct drm_i915_private *dev_priv = dev->dev_private;
  1880.         int pipe;
  1881.         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
  1882.  
  1883.                 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
  1884.  
  1885.         if (pch_iir & SDE_AUDIO_POWER_MASK) {
  1886.                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
  1887.                                SDE_AUDIO_POWER_SHIFT);
  1888.                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
  1889.                                  port_name(port));
  1890.         }
  1891.  
  1892.         if (pch_iir & SDE_AUX_MASK)
  1893.                 dp_aux_irq_handler(dev);
  1894.  
  1895.         if (pch_iir & SDE_GMBUS)
  1896.                 gmbus_irq_handler(dev);
  1897.  
  1898.         if (pch_iir & SDE_AUDIO_HDCP_MASK)
  1899.                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
  1900.  
  1901.         if (pch_iir & SDE_AUDIO_TRANS_MASK)
  1902.                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
  1903.  
  1904.         if (pch_iir & SDE_POISON)
  1905.                 DRM_ERROR("PCH poison interrupt\n");
  1906.  
  1907.         if (pch_iir & SDE_FDI_MASK)
  1908.                 for_each_pipe(dev_priv, pipe)
  1909.                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
  1910.                                          pipe_name(pipe),
  1911.                                          I915_READ(FDI_RX_IIR(pipe)));
  1912.  
  1913.         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
  1914.                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
  1915.  
  1916.         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
  1917.                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
  1918.  
  1919.         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
  1920.                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
  1921.  
  1922.         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
  1923.                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
  1924. }
  1925.  
  1926. static void ivb_err_int_handler(struct drm_device *dev)
  1927. {
  1928.         struct drm_i915_private *dev_priv = dev->dev_private;
  1929.         u32 err_int = I915_READ(GEN7_ERR_INT);
  1930.         enum pipe pipe;
  1931.  
  1932.         if (err_int & ERR_INT_POISON)
  1933.                 DRM_ERROR("Poison interrupt\n");
  1934.  
  1935.         for_each_pipe(dev_priv, pipe) {
  1936.                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
  1937.                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
  1938.  
  1939.                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
  1940.                         if (IS_IVYBRIDGE(dev))
  1941.                                 ivb_pipe_crc_irq_handler(dev, pipe);
  1942.                         else
  1943.                                 hsw_pipe_crc_irq_handler(dev, pipe);
  1944.                 }
  1945.         }
  1946.  
  1947.         I915_WRITE(GEN7_ERR_INT, err_int);
  1948. }
  1949.  
  1950. static void cpt_serr_int_handler(struct drm_device *dev)
  1951. {
  1952.         struct drm_i915_private *dev_priv = dev->dev_private;
  1953.         u32 serr_int = I915_READ(SERR_INT);
  1954.  
  1955.         if (serr_int & SERR_INT_POISON)
  1956.                 DRM_ERROR("PCH poison interrupt\n");
  1957.  
  1958.         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
  1959.                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
  1960.  
  1961.         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
  1962.                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
  1963.  
  1964.         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
  1965.                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
  1966.  
  1967.         I915_WRITE(SERR_INT, serr_int);
  1968. }
  1969.  
  1970. static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
  1971. {
  1972.         struct drm_i915_private *dev_priv = dev->dev_private;
  1973.         int pipe;
  1974.         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
  1975.  
  1976.                 ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
  1977.  
  1978.         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
  1979.                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
  1980.                                SDE_AUDIO_POWER_SHIFT_CPT);
  1981.                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
  1982.                                  port_name(port));
  1983.         }
  1984.  
  1985.         if (pch_iir & SDE_AUX_MASK_CPT)
  1986.                 dp_aux_irq_handler(dev);
  1987.  
  1988.         if (pch_iir & SDE_GMBUS_CPT)
  1989.                 gmbus_irq_handler(dev);
  1990.  
  1991.         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
  1992.                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
  1993.  
  1994.         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
  1995.                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
  1996.  
  1997.         if (pch_iir & SDE_FDI_MASK_CPT)
  1998.                 for_each_pipe(dev_priv, pipe)
  1999.                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
  2000.                                          pipe_name(pipe),
  2001.                                          I915_READ(FDI_RX_IIR(pipe)));
  2002.  
  2003.         if (pch_iir & SDE_ERROR_CPT)
  2004.                 cpt_serr_int_handler(dev);
  2005. }
  2006.  
  2007. static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
  2008. {
  2009.         struct drm_i915_private *dev_priv = dev->dev_private;
  2010.         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
  2011.                 ~SDE_PORTE_HOTPLUG_SPT;
  2012.         u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
  2013.         u32 pin_mask = 0, long_mask = 0;
  2014.  
  2015.         if (hotplug_trigger) {
  2016.                 u32 dig_hotplug_reg;
  2017.  
  2018.                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
  2019.                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
  2020.  
  2021.                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
  2022.                                    dig_hotplug_reg, hpd_spt,
  2023.                                    spt_port_hotplug_long_detect);
  2024.         }
  2025.  
  2026.         if (hotplug2_trigger) {
  2027.                 u32 dig_hotplug_reg;
  2028.  
  2029.                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
  2030.                 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
  2031.  
  2032.                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
  2033.                                    dig_hotplug_reg, hpd_spt,
  2034.                                    spt_port_hotplug2_long_detect);
  2035.         }
  2036.  
  2037.         if (pin_mask)
  2038.                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
  2039.  
  2040.         if (pch_iir & SDE_GMBUS_CPT)
  2041.                 gmbus_irq_handler(dev);
  2042. }
  2043.  
  2044. static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
  2045.                                 const u32 hpd[HPD_NUM_PINS])
  2046. {
  2047.         struct drm_i915_private *dev_priv = to_i915(dev);
  2048.         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
  2049.  
  2050.         dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
  2051.         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
  2052.  
  2053.         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
  2054.                            dig_hotplug_reg, hpd,
  2055.                            ilk_port_hotplug_long_detect);
  2056.  
  2057.         intel_hpd_irq_handler(dev, pin_mask, long_mask);
  2058. }
  2059.  
  2060. static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
  2061. {
  2062.         struct drm_i915_private *dev_priv = dev->dev_private;
  2063.         enum pipe pipe;
  2064.         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
  2065.  
  2066.         if (hotplug_trigger)
  2067.                 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
  2068.  
  2069.         if (de_iir & DE_AUX_CHANNEL_A)
  2070.                 dp_aux_irq_handler(dev);
  2071.  
  2072.         if (de_iir & DE_GSE)
  2073.                 intel_opregion_asle_intr(dev);
  2074.  
  2075.         if (de_iir & DE_POISON)
  2076.                 DRM_ERROR("Poison interrupt\n");
  2077.  
  2078.         for_each_pipe(dev_priv, pipe) {
  2079.                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
  2080.                     intel_pipe_handle_vblank(dev, pipe))
  2081.                         intel_check_page_flip(dev, pipe);
  2082.  
  2083.                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
  2084.                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
  2085.  
  2086.                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
  2087.                         i9xx_pipe_crc_irq_handler(dev, pipe);
  2088.  
  2089.                 /* plane/pipes map 1:1 on ilk+ */
  2090.                 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
  2091.                         intel_prepare_page_flip(dev, pipe);
  2092.                         intel_finish_page_flip_plane(dev, pipe);
  2093.                 }
  2094.         }
  2095.  
  2096.         /* check event from PCH */
  2097.         if (de_iir & DE_PCH_EVENT) {
  2098.                 u32 pch_iir = I915_READ(SDEIIR);
  2099.  
  2100.                 if (HAS_PCH_CPT(dev))
  2101.                         cpt_irq_handler(dev, pch_iir);
  2102.                 else
  2103.                         ibx_irq_handler(dev, pch_iir);
  2104.  
  2105.                 /* should clear PCH hotplug event before clear CPU irq */
  2106.                 I915_WRITE(SDEIIR, pch_iir);
  2107.         }
  2108.  
  2109.         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
  2110.                 ironlake_rps_change_irq_handler(dev);
  2111. }
  2112.  
  2113. static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
  2114. {
  2115.         struct drm_i915_private *dev_priv = dev->dev_private;
  2116.         enum pipe pipe;
  2117.         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
  2118.  
  2119.         if (hotplug_trigger)
  2120.                 ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
  2121.  
  2122.         if (de_iir & DE_ERR_INT_IVB)
  2123.                 ivb_err_int_handler(dev);
  2124.  
  2125.         if (de_iir & DE_AUX_CHANNEL_A_IVB)
  2126.                 dp_aux_irq_handler(dev);
  2127.  
  2128.         if (de_iir & DE_GSE_IVB)
  2129.                 intel_opregion_asle_intr(dev);
  2130.  
  2131.         for_each_pipe(dev_priv, pipe) {
  2132.                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
  2133.                     intel_pipe_handle_vblank(dev, pipe))
  2134.                         intel_check_page_flip(dev, pipe);
  2135.  
  2136.                 /* plane/pipes map 1:1 on ilk+ */
  2137.                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
  2138.                         intel_prepare_page_flip(dev, pipe);
  2139.                         intel_finish_page_flip_plane(dev, pipe);
  2140.                 }
  2141.         }
  2142.  
  2143.         /* check event from PCH */
  2144.         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
  2145.                 u32 pch_iir = I915_READ(SDEIIR);
  2146.  
  2147.                 cpt_irq_handler(dev, pch_iir);
  2148.  
  2149.                 /* clear PCH hotplug event before clear CPU irq */
  2150.                 I915_WRITE(SDEIIR, pch_iir);
  2151.         }
  2152. }
  2153.  
  2154. /*
  2155.  * To handle irqs with the minimum potential races with fresh interrupts, we:
  2156.  * 1 - Disable Master Interrupt Control.
  2157.  * 2 - Find the source(s) of the interrupt.
  2158.  * 3 - Clear the Interrupt Identity bits (IIR).
  2159.  * 4 - Process the interrupt(s) that had bits set in the IIRs.
  2160.  * 5 - Re-enable Master Interrupt Control.
  2161.  */
  2162. static irqreturn_t ironlake_irq_handler(int irq, void *arg)
  2163. {
  2164.         struct drm_device *dev = arg;
  2165.         struct drm_i915_private *dev_priv = dev->dev_private;
  2166.         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
  2167.         irqreturn_t ret = IRQ_NONE;
  2168.  
  2169.         if (!intel_irqs_enabled(dev_priv))
  2170.                 return IRQ_NONE;
  2171.  
  2172.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  2173.         disable_rpm_wakeref_asserts(dev_priv);
  2174.  
  2175.         /* We get interrupts on unclaimed registers, so check for this before we
  2176.          * do any I915_{READ,WRITE}. */
  2177.         intel_uncore_check_errors(dev);
  2178.  
  2179.         /* disable master interrupt before clearing iir  */
  2180.         de_ier = I915_READ(DEIER);
  2181.         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
  2182.         POSTING_READ(DEIER);
  2183.  
  2184.         /* Disable south interrupts. We'll only write to SDEIIR once, so further
  2185.          * interrupts will will be stored on its back queue, and then we'll be
  2186.          * able to process them after we restore SDEIER (as soon as we restore
  2187.          * it, we'll get an interrupt if SDEIIR still has something to process
  2188.          * due to its back queue). */
  2189.         if (!HAS_PCH_NOP(dev)) {
  2190.                 sde_ier = I915_READ(SDEIER);
  2191.                 I915_WRITE(SDEIER, 0);
  2192.                 POSTING_READ(SDEIER);
  2193.         }
  2194.  
  2195.         /* Find, clear, then process each source of interrupt */
  2196.  
  2197.         gt_iir = I915_READ(GTIIR);
  2198.         if (gt_iir) {
  2199.                 I915_WRITE(GTIIR, gt_iir);
  2200.                 ret = IRQ_HANDLED;
  2201.                 if (INTEL_INFO(dev)->gen >= 6)
  2202.                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
  2203.                 else
  2204.                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
  2205.         }
  2206.  
  2207.         de_iir = I915_READ(DEIIR);
  2208.         if (de_iir) {
  2209.                 I915_WRITE(DEIIR, de_iir);
  2210.                 ret = IRQ_HANDLED;
  2211.                 if (INTEL_INFO(dev)->gen >= 7)
  2212.                         ivb_display_irq_handler(dev, de_iir);
  2213.                 else
  2214.                         ilk_display_irq_handler(dev, de_iir);
  2215.         }
  2216.  
  2217.         if (INTEL_INFO(dev)->gen >= 6) {
  2218.                 u32 pm_iir = I915_READ(GEN6_PMIIR);
  2219.                 if (pm_iir) {
  2220.                         I915_WRITE(GEN6_PMIIR, pm_iir);
  2221.                         ret = IRQ_HANDLED;
  2222.                         gen6_rps_irq_handler(dev_priv, pm_iir);
  2223.                 }
  2224.         }
  2225.  
  2226.         I915_WRITE(DEIER, de_ier);
  2227.         POSTING_READ(DEIER);
  2228.         if (!HAS_PCH_NOP(dev)) {
  2229.                 I915_WRITE(SDEIER, sde_ier);
  2230.                 POSTING_READ(SDEIER);
  2231.         }
  2232.  
  2233.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  2234.         enable_rpm_wakeref_asserts(dev_priv);
  2235.  
  2236.         return ret;
  2237. }
  2238.  
  2239. static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
  2240.                                 const u32 hpd[HPD_NUM_PINS])
  2241. {
  2242.         struct drm_i915_private *dev_priv = to_i915(dev);
  2243.         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
  2244.  
  2245.         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
  2246.         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
  2247.  
  2248.         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
  2249.                            dig_hotplug_reg, hpd,
  2250.                            bxt_port_hotplug_long_detect);
  2251.  
  2252.         intel_hpd_irq_handler(dev, pin_mask, long_mask);
  2253. }
  2254.  
  2255. static irqreturn_t gen8_irq_handler(int irq, void *arg)
  2256. {
  2257.         struct drm_device *dev = arg;
  2258.         struct drm_i915_private *dev_priv = dev->dev_private;
  2259.         u32 master_ctl;
  2260.         irqreturn_t ret = IRQ_NONE;
  2261.         uint32_t tmp = 0;
  2262.         enum pipe pipe;
  2263.         u32 aux_mask = GEN8_AUX_CHANNEL_A;
  2264.  
  2265.         if (!intel_irqs_enabled(dev_priv))
  2266.                 return IRQ_NONE;
  2267.  
  2268.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  2269.         disable_rpm_wakeref_asserts(dev_priv);
  2270.  
  2271.         if (INTEL_INFO(dev_priv)->gen >= 9)
  2272.                 aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
  2273.                         GEN9_AUX_CHANNEL_D;
  2274.  
  2275.         master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
  2276.         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
  2277.         if (!master_ctl)
  2278.                 goto out;
  2279.  
  2280.         I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
  2281.  
  2282.         /* Find, clear, then process each source of interrupt */
  2283.  
  2284.         ret = gen8_gt_irq_handler(dev_priv, master_ctl);
  2285.  
  2286.         if (master_ctl & GEN8_DE_MISC_IRQ) {
  2287.                 tmp = I915_READ(GEN8_DE_MISC_IIR);
  2288.                 if (tmp) {
  2289.                         I915_WRITE(GEN8_DE_MISC_IIR, tmp);
  2290.                         ret = IRQ_HANDLED;
  2291.                         if (tmp & GEN8_DE_MISC_GSE)
  2292.                                 intel_opregion_asle_intr(dev);
  2293.                         else
  2294.                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
  2295.                 }
  2296.                 else
  2297.                         DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
  2298.         }
  2299.  
  2300.         if (master_ctl & GEN8_DE_PORT_IRQ) {
  2301.                 tmp = I915_READ(GEN8_DE_PORT_IIR);
  2302.                 if (tmp) {
  2303.                         bool found = false;
  2304.                         u32 hotplug_trigger = 0;
  2305.  
  2306.                         if (IS_BROXTON(dev_priv))
  2307.                                 hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
  2308.                         else if (IS_BROADWELL(dev_priv))
  2309.                                 hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
  2310.  
  2311.                         I915_WRITE(GEN8_DE_PORT_IIR, tmp);
  2312.                         ret = IRQ_HANDLED;
  2313.  
  2314.                         if (tmp & aux_mask) {
  2315.                                 dp_aux_irq_handler(dev);
  2316.                                 found = true;
  2317.                         }
  2318.  
  2319.                         if (hotplug_trigger) {
  2320.                                 if (IS_BROXTON(dev))
  2321.                                         bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
  2322.                                 else
  2323.                                         ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
  2324.                                 found = true;
  2325.                         }
  2326.  
  2327.                         if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
  2328.                                 gmbus_irq_handler(dev);
  2329.                                 found = true;
  2330.                         }
  2331.  
  2332.                         if (!found)
  2333.                                 DRM_ERROR("Unexpected DE Port interrupt\n");
  2334.                 }
  2335.                 else
  2336.                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
  2337.         }
  2338.  
  2339.         for_each_pipe(dev_priv, pipe) {
  2340.                 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
  2341.  
  2342.                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
  2343.                         continue;
  2344.  
  2345.                 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
  2346.                 if (pipe_iir) {
  2347.                         ret = IRQ_HANDLED;
  2348.                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
  2349.  
  2350.                         if (pipe_iir & GEN8_PIPE_VBLANK &&
  2351.                             intel_pipe_handle_vblank(dev, pipe))
  2352.                                 intel_check_page_flip(dev, pipe);
  2353.  
  2354.                         if (INTEL_INFO(dev_priv)->gen >= 9)
  2355.                                 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
  2356.                         else
  2357.                                 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
  2358.  
  2359.                         if (flip_done) {
  2360.                                 intel_prepare_page_flip(dev, pipe);
  2361.                                 intel_finish_page_flip_plane(dev, pipe);
  2362.                         }
  2363.  
  2364.                         if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
  2365.                                 hsw_pipe_crc_irq_handler(dev, pipe);
  2366.  
  2367.                         if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
  2368.                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
  2369.                                                                     pipe);
  2370.  
  2371.  
  2372.                         if (INTEL_INFO(dev_priv)->gen >= 9)
  2373.                                 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
  2374.                         else
  2375.                                 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
  2376.  
  2377.                         if (fault_errors)
  2378.                                 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
  2379.                                           pipe_name(pipe),
  2380.                                           pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
  2381.                 } else
  2382.                         DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
  2383.         }
  2384.  
  2385.         if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
  2386.             master_ctl & GEN8_DE_PCH_IRQ) {
  2387.                 /*
  2388.                  * FIXME(BDW): Assume for now that the new interrupt handling
  2389.                  * scheme also closed the SDE interrupt handling race we've seen
  2390.                  * on older pch-split platforms. But this needs testing.
  2391.                  */
  2392.                 u32 pch_iir = I915_READ(SDEIIR);
  2393.                 if (pch_iir) {
  2394.                         I915_WRITE(SDEIIR, pch_iir);
  2395.                         ret = IRQ_HANDLED;
  2396.  
  2397.                         if (HAS_PCH_SPT(dev_priv))
  2398.                                 spt_irq_handler(dev, pch_iir);
  2399.                         else
  2400.                                 cpt_irq_handler(dev, pch_iir);
  2401.                 } else {
  2402.                         /*
  2403.                          * Like on previous PCH there seems to be something
  2404.                          * fishy going on with forwarding PCH interrupts.
  2405.                          */
  2406.                         DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
  2407.                 }
  2408.         }
  2409.  
  2410.         I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
  2411.         POSTING_READ_FW(GEN8_MASTER_IRQ);
  2412.  
  2413. out:
  2414.         enable_rpm_wakeref_asserts(dev_priv);
  2415.  
  2416.         return ret;
  2417. }
  2418.  
  2419. static void i915_error_wake_up(struct drm_i915_private *dev_priv,
  2420.                                bool reset_completed)
  2421. {
  2422.         struct intel_engine_cs *ring;
  2423.         int i;
  2424.  
  2425.         /*
  2426.          * Notify all waiters for GPU completion events that reset state has
  2427.          * been changed, and that they need to restart their wait after
  2428.          * checking for potential errors (and bail out to drop locks if there is
  2429.          * a gpu reset pending so that i915_error_work_func can acquire them).
  2430.          */
  2431.  
  2432.         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
  2433.         for_each_ring(ring, dev_priv, i)
  2434.                 wake_up_all(&ring->irq_queue);
  2435.  
  2436.         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
  2437.         wake_up_all(&dev_priv->pending_flip_queue);
  2438.  
  2439.         /*
  2440.          * Signal tasks blocked in i915_gem_wait_for_error that the pending
  2441.          * reset state is cleared.
  2442.          */
  2443.         if (reset_completed)
  2444.                 wake_up_all(&dev_priv->gpu_error.reset_queue);
  2445. }
  2446.  
  2447. /**
  2448.  * i915_reset_and_wakeup - do process context error handling work
  2449.  * @dev: drm device
  2450.  *
  2451.  * Fire an error uevent so userspace can see that a hang or error
  2452.  * was detected.
  2453.  */
  2454. static void i915_reset_and_wakeup(struct drm_device *dev)
  2455. {
  2456.         struct drm_i915_private *dev_priv = to_i915(dev);
  2457.         struct i915_gpu_error *error = &dev_priv->gpu_error;
  2458.         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
  2459.         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
  2460.         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
  2461.         int ret;
  2462.  
  2463.         /*
  2464.          * Note that there's only one work item which does gpu resets, so we
  2465.          * need not worry about concurrent gpu resets potentially incrementing
  2466.          * error->reset_counter twice. We only need to take care of another
  2467.          * racing irq/hangcheck declaring the gpu dead for a second time. A
  2468.          * quick check for that is good enough: schedule_work ensures the
  2469.          * correct ordering between hang detection and this work item, and since
  2470.          * the reset in-progress bit is only ever set by code outside of this
  2471.          * work we don't need to worry about any other races.
  2472.          */
  2473.         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
  2474.                 DRM_DEBUG_DRIVER("resetting chip\n");
  2475.                 /*
  2476.                  * In most cases it's guaranteed that we get here with an RPM
  2477.                  * reference held, for example because there is a pending GPU
  2478.                  * request that won't finish until the reset is done. This
  2479.                  * isn't the case at least when we get here by doing a
  2480.                  * simulated reset via debugs, so get an RPM reference.
  2481.                  */
  2482.                 intel_runtime_pm_get(dev_priv);
  2483.  
  2484.                 /*
  2485.                  * All state reset _must_ be completed before we update the
  2486.                  * reset counter, for otherwise waiters might miss the reset
  2487.                  * pending state and not properly drop locks, resulting in
  2488.                  * deadlocks with the reset work.
  2489.                  */
  2490. //              ret = i915_reset(dev);
  2491.  
  2492. //              intel_finish_reset(dev);
  2493.  
  2494.                 intel_runtime_pm_put(dev_priv);
  2495.  
  2496.                 if (ret == 0) {
  2497.                         /*
  2498.                          * After all the gem state is reset, increment the reset
  2499.                          * counter and wake up everyone waiting for the reset to
  2500.                          * complete.
  2501.                          *
  2502.                          * Since unlock operations are a one-sided barrier only,
  2503.                          * we need to insert a barrier here to order any seqno
  2504.                          * updates before
  2505.                          * the counter increment.
  2506.                          */
  2507.                         smp_mb__before_atomic();
  2508.                         atomic_inc(&dev_priv->gpu_error.reset_counter);
  2509.  
  2510.                 } else {
  2511.                         atomic_or(I915_WEDGED, &error->reset_counter);
  2512.                 }
  2513.  
  2514.                 /*
  2515.                  * Note: The wake_up also serves as a memory barrier so that
  2516.                  * waiters see the update value of the reset counter atomic_t.
  2517.                  */
  2518.                 i915_error_wake_up(dev_priv, true);
  2519.         }
  2520. }
  2521.  
  2522. static void i915_report_and_clear_eir(struct drm_device *dev)
  2523. {
  2524.         struct drm_i915_private *dev_priv = dev->dev_private;
  2525.         uint32_t instdone[I915_NUM_INSTDONE_REG];
  2526.         u32 eir = I915_READ(EIR);
  2527.         int pipe, i;
  2528.  
  2529.         if (!eir)
  2530.                 return;
  2531.  
  2532.         pr_err("render error detected, EIR: 0x%08x\n", eir);
  2533.  
  2534.         i915_get_extra_instdone(dev, instdone);
  2535.  
  2536.         if (IS_G4X(dev)) {
  2537.                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
  2538.                         u32 ipeir = I915_READ(IPEIR_I965);
  2539.  
  2540.                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
  2541.                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
  2542.                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
  2543.                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
  2544.                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
  2545.                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
  2546.                         I915_WRITE(IPEIR_I965, ipeir);
  2547.                         POSTING_READ(IPEIR_I965);
  2548.                 }
  2549.                 if (eir & GM45_ERROR_PAGE_TABLE) {
  2550.                         u32 pgtbl_err = I915_READ(PGTBL_ER);
  2551.                         pr_err("page table error\n");
  2552.                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
  2553.                         I915_WRITE(PGTBL_ER, pgtbl_err);
  2554.                         POSTING_READ(PGTBL_ER);
  2555.                 }
  2556.         }
  2557.  
  2558.         if (!IS_GEN2(dev)) {
  2559.                 if (eir & I915_ERROR_PAGE_TABLE) {
  2560.                         u32 pgtbl_err = I915_READ(PGTBL_ER);
  2561.                         pr_err("page table error\n");
  2562.                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
  2563.                         I915_WRITE(PGTBL_ER, pgtbl_err);
  2564.                         POSTING_READ(PGTBL_ER);
  2565.                 }
  2566.         }
  2567.  
  2568.         if (eir & I915_ERROR_MEMORY_REFRESH) {
  2569.                 pr_err("memory refresh error:\n");
  2570.                 for_each_pipe(dev_priv, pipe)
  2571.                         pr_err("pipe %c stat: 0x%08x\n",
  2572.                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
  2573.                 /* pipestat has already been acked */
  2574.         }
  2575.         if (eir & I915_ERROR_INSTRUCTION) {
  2576.                 pr_err("instruction error\n");
  2577.                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
  2578.                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
  2579.                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
  2580.                 if (INTEL_INFO(dev)->gen < 4) {
  2581.                         u32 ipeir = I915_READ(IPEIR);
  2582.  
  2583.                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
  2584.                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
  2585.                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
  2586.                         I915_WRITE(IPEIR, ipeir);
  2587.                         POSTING_READ(IPEIR);
  2588.                 } else {
  2589.                         u32 ipeir = I915_READ(IPEIR_I965);
  2590.  
  2591.                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
  2592.                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
  2593.                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
  2594.                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
  2595.                         I915_WRITE(IPEIR_I965, ipeir);
  2596.                         POSTING_READ(IPEIR_I965);
  2597.                 }
  2598.         }
  2599.  
  2600.         I915_WRITE(EIR, eir);
  2601.         POSTING_READ(EIR);
  2602.         eir = I915_READ(EIR);
  2603.         if (eir) {
  2604.                 /*
  2605.                  * some errors might have become stuck,
  2606.                  * mask them.
  2607.                  */
  2608.                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
  2609.                 I915_WRITE(EMR, I915_READ(EMR) | eir);
  2610.                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
  2611.         }
  2612. }
  2613.  
  2614. /**
  2615.  * i915_handle_error - handle a gpu error
  2616.  * @dev: drm device
  2617.  *
  2618.  * Do some basic checking of register state at error time and
  2619.  * dump it to the syslog.  Also call i915_capture_error_state() to make
  2620.  * sure we get a record and make it available in debugfs.  Fire a uevent
  2621.  * so userspace knows something bad happened (should trigger collection
  2622.  * of a ring dump etc.).
  2623.  */
  2624. void i915_handle_error(struct drm_device *dev, bool wedged,
  2625.                        const char *fmt, ...)
  2626. {
  2627.         struct drm_i915_private *dev_priv = dev->dev_private;
  2628.         va_list args;
  2629.         char error_msg[80];
  2630.  
  2631.         va_start(args, fmt);
  2632.         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
  2633.         va_end(args);
  2634.  
  2635. //      i915_capture_error_state(dev);
  2636.         i915_report_and_clear_eir(dev);
  2637.  
  2638.         if (wedged) {
  2639.                 atomic_or(I915_RESET_IN_PROGRESS_FLAG,
  2640.                                 &dev_priv->gpu_error.reset_counter);
  2641.  
  2642.                 /*
  2643.                  * Wakeup waiting processes so that the reset function
  2644.                  * i915_reset_and_wakeup doesn't deadlock trying to grab
  2645.                  * various locks. By bumping the reset counter first, the woken
  2646.                  * processes will see a reset in progress and back off,
  2647.                  * releasing their locks and then wait for the reset completion.
  2648.                  * We must do this for _all_ gpu waiters that might hold locks
  2649.                  * that the reset work needs to acquire.
  2650.                  *
  2651.                  * Note: The wake_up serves as the required memory barrier to
  2652.                  * ensure that the waiters see the updated value of the reset
  2653.                  * counter atomic_t.
  2654.                  */
  2655.                 i915_error_wake_up(dev_priv, false);
  2656.         }
  2657.  
  2658.         i915_reset_and_wakeup(dev);
  2659. }
  2660.  
  2661. /* Called from drm generic code, passed 'crtc' which
  2662.  * we use as a pipe index
  2663.  */
  2664. static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
  2665. {
  2666.         struct drm_i915_private *dev_priv = dev->dev_private;
  2667.         unsigned long irqflags;
  2668.  
  2669.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2670.         if (INTEL_INFO(dev)->gen >= 4)
  2671.                 i915_enable_pipestat(dev_priv, pipe,
  2672.                                      PIPE_START_VBLANK_INTERRUPT_STATUS);
  2673.         else
  2674.                 i915_enable_pipestat(dev_priv, pipe,
  2675.                                      PIPE_VBLANK_INTERRUPT_STATUS);
  2676.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2677.  
  2678.         return 0;
  2679. }
  2680.  
  2681. static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
  2682. {
  2683.         struct drm_i915_private *dev_priv = dev->dev_private;
  2684.         unsigned long irqflags;
  2685.         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
  2686.                                                      DE_PIPE_VBLANK(pipe);
  2687.  
  2688.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2689.         ilk_enable_display_irq(dev_priv, bit);
  2690.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2691.  
  2692.         return 0;
  2693. }
  2694.  
  2695. static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
  2696. {
  2697.         struct drm_i915_private *dev_priv = dev->dev_private;
  2698.         unsigned long irqflags;
  2699.  
  2700.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2701.         i915_enable_pipestat(dev_priv, pipe,
  2702.                              PIPE_START_VBLANK_INTERRUPT_STATUS);
  2703.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2704.  
  2705.         return 0;
  2706. }
  2707.  
  2708. static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
  2709. {
  2710.         struct drm_i915_private *dev_priv = dev->dev_private;
  2711.         unsigned long irqflags;
  2712.  
  2713.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2714.         bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
  2715.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2716.  
  2717.         return 0;
  2718. }
  2719.  
  2720. /* Called from drm generic code, passed 'crtc' which
  2721.  * we use as a pipe index
  2722.  */
  2723. static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
  2724. {
  2725.         struct drm_i915_private *dev_priv = dev->dev_private;
  2726.         unsigned long irqflags;
  2727.  
  2728.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2729.         i915_disable_pipestat(dev_priv, pipe,
  2730.                               PIPE_VBLANK_INTERRUPT_STATUS |
  2731.                               PIPE_START_VBLANK_INTERRUPT_STATUS);
  2732.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2733. }
  2734.  
  2735. static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
  2736. {
  2737.         struct drm_i915_private *dev_priv = dev->dev_private;
  2738.         unsigned long irqflags;
  2739.         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
  2740.                                                      DE_PIPE_VBLANK(pipe);
  2741.  
  2742.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2743.         ilk_disable_display_irq(dev_priv, bit);
  2744.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2745. }
  2746.  
  2747. static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
  2748. {
  2749.         struct drm_i915_private *dev_priv = dev->dev_private;
  2750.         unsigned long irqflags;
  2751.  
  2752.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2753.         i915_disable_pipestat(dev_priv, pipe,
  2754.                               PIPE_START_VBLANK_INTERRUPT_STATUS);
  2755.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2756. }
  2757.  
  2758. static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
  2759. {
  2760.         struct drm_i915_private *dev_priv = dev->dev_private;
  2761.         unsigned long irqflags;
  2762.  
  2763.         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
  2764.         bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
  2765.         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
  2766. }
  2767.  
  2768. static bool
  2769. ring_idle(struct intel_engine_cs *ring, u32 seqno)
  2770. {
  2771.         return (list_empty(&ring->request_list) ||
  2772.                 i915_seqno_passed(seqno, ring->last_submitted_seqno));
  2773. }
  2774.  
  2775. static bool
  2776. ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
  2777. {
  2778.         if (INTEL_INFO(dev)->gen >= 8) {
  2779.                 return (ipehr >> 23) == 0x1c;
  2780.         } else {
  2781.                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
  2782.                 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
  2783.                                  MI_SEMAPHORE_REGISTER);
  2784.         }
  2785. }
  2786.  
  2787. static struct intel_engine_cs *
  2788. semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
  2789. {
  2790.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  2791.         struct intel_engine_cs *signaller;
  2792.         int i;
  2793.  
  2794.         if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
  2795.                 for_each_ring(signaller, dev_priv, i) {
  2796.                         if (ring == signaller)
  2797.                                 continue;
  2798.  
  2799.                         if (offset == signaller->semaphore.signal_ggtt[ring->id])
  2800.                                 return signaller;
  2801.                 }
  2802.         } else {
  2803.                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
  2804.  
  2805.                 for_each_ring(signaller, dev_priv, i) {
  2806.                         if(ring == signaller)
  2807.                                 continue;
  2808.  
  2809.                         if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
  2810.                                 return signaller;
  2811.                 }
  2812.         }
  2813.  
  2814.         DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
  2815.                   ring->id, ipehr, offset);
  2816.  
  2817.         return NULL;
  2818. }
  2819.  
  2820. static struct intel_engine_cs *
  2821. semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
  2822. {
  2823.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  2824.         u32 cmd, ipehr, head;
  2825.         u64 offset = 0;
  2826.         int i, backwards;
  2827.  
  2828.         /*
  2829.          * This function does not support execlist mode - any attempt to
  2830.          * proceed further into this function will result in a kernel panic
  2831.          * when dereferencing ring->buffer, which is not set up in execlist
  2832.          * mode.
  2833.          *
  2834.          * The correct way of doing it would be to derive the currently
  2835.          * executing ring buffer from the current context, which is derived
  2836.          * from the currently running request. Unfortunately, to get the
  2837.          * current request we would have to grab the struct_mutex before doing
  2838.          * anything else, which would be ill-advised since some other thread
  2839.          * might have grabbed it already and managed to hang itself, causing
  2840.          * the hang checker to deadlock.
  2841.          *
  2842.          * Therefore, this function does not support execlist mode in its
  2843.          * current form. Just return NULL and move on.
  2844.          */
  2845.         if (ring->buffer == NULL)
  2846.                 return NULL;
  2847.  
  2848.         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
  2849.         if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
  2850.                 return NULL;
  2851.  
  2852.         /*
  2853.          * HEAD is likely pointing to the dword after the actual command,
  2854.          * so scan backwards until we find the MBOX. But limit it to just 3
  2855.          * or 4 dwords depending on the semaphore wait command size.
  2856.          * Note that we don't care about ACTHD here since that might
  2857.          * point at at batch, and semaphores are always emitted into the
  2858.          * ringbuffer itself.
  2859.          */
  2860.         head = I915_READ_HEAD(ring) & HEAD_ADDR;
  2861.         backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
  2862.  
  2863.         for (i = backwards; i; --i) {
  2864.                 /*
  2865.                  * Be paranoid and presume the hw has gone off into the wild -
  2866.                  * our ring is smaller than what the hardware (and hence
  2867.                  * HEAD_ADDR) allows. Also handles wrap-around.
  2868.                  */
  2869.                 head &= ring->buffer->size - 1;
  2870.  
  2871.                 /* This here seems to blow up */
  2872.                 cmd = ioread32(ring->buffer->virtual_start + head);
  2873.                 if (cmd == ipehr)
  2874.                         break;
  2875.  
  2876.                 head -= 4;
  2877.         }
  2878.  
  2879.         if (!i)
  2880.                 return NULL;
  2881.  
  2882.         *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
  2883.         if (INTEL_INFO(ring->dev)->gen >= 8) {
  2884.                 offset = ioread32(ring->buffer->virtual_start + head + 12);
  2885.                 offset <<= 32;
  2886.                 offset = ioread32(ring->buffer->virtual_start + head + 8);
  2887.         }
  2888.         return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
  2889. }
  2890.  
  2891. static int semaphore_passed(struct intel_engine_cs *ring)
  2892. {
  2893.         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  2894.         struct intel_engine_cs *signaller;
  2895.         u32 seqno;
  2896.  
  2897.         ring->hangcheck.deadlock++;
  2898.  
  2899.         signaller = semaphore_waits_for(ring, &seqno);
  2900.         if (signaller == NULL)
  2901.                 return -1;
  2902.  
  2903.         /* Prevent pathological recursion due to driver bugs */
  2904.         if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
  2905.                 return -1;
  2906.  
  2907.         if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
  2908.                 return 1;
  2909.  
  2910.         /* cursory check for an unkickable deadlock */
  2911.         if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
  2912.             semaphore_passed(signaller) < 0)
  2913.                 return -1;
  2914.  
  2915.         return 0;
  2916. }
  2917.  
  2918. static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
  2919. {
  2920.         struct intel_engine_cs *ring;
  2921.         int i;
  2922.  
  2923.         for_each_ring(ring, dev_priv, i)
  2924.                 ring->hangcheck.deadlock = 0;
  2925. }
  2926.  
  2927. static enum intel_ring_hangcheck_action
  2928. ring_stuck(struct intel_engine_cs *ring, u64 acthd)
  2929. {
  2930.         struct drm_device *dev = ring->dev;
  2931.         struct drm_i915_private *dev_priv = dev->dev_private;
  2932.         u32 tmp;
  2933.  
  2934.         if (acthd != ring->hangcheck.acthd) {
  2935.                 if (acthd > ring->hangcheck.max_acthd) {
  2936.                         ring->hangcheck.max_acthd = acthd;
  2937.                         return HANGCHECK_ACTIVE;
  2938.                 }
  2939.  
  2940.                 return HANGCHECK_ACTIVE_LOOP;
  2941.         }
  2942.  
  2943.         if (IS_GEN2(dev))
  2944.                 return HANGCHECK_HUNG;
  2945.  
  2946.         /* Is the chip hanging on a WAIT_FOR_EVENT?
  2947.          * If so we can simply poke the RB_WAIT bit
  2948.          * and break the hang. This should work on
  2949.          * all but the second generation chipsets.
  2950.          */
  2951.         tmp = I915_READ_CTL(ring);
  2952.         if (tmp & RING_WAIT) {
  2953.                 i915_handle_error(dev, false,
  2954.                                   "Kicking stuck wait on %s",
  2955.                                   ring->name);
  2956.                 I915_WRITE_CTL(ring, tmp);
  2957.                 return HANGCHECK_KICK;
  2958.         }
  2959.  
  2960.         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
  2961.                 switch (semaphore_passed(ring)) {
  2962.                 default:
  2963.                         return HANGCHECK_HUNG;
  2964.                 case 1:
  2965.                         i915_handle_error(dev, false,
  2966.                                           "Kicking stuck semaphore on %s",
  2967.                                           ring->name);
  2968.                         I915_WRITE_CTL(ring, tmp);
  2969.                         return HANGCHECK_KICK;
  2970.                 case 0:
  2971.                         return HANGCHECK_WAIT;
  2972.                 }
  2973.         }
  2974.  
  2975.         return HANGCHECK_HUNG;
  2976. }
  2977.  
  2978. /*
  2979.  * This is called when the chip hasn't reported back with completed
  2980.  * batchbuffers in a long time. We keep track per ring seqno progress and
  2981.  * if there are no progress, hangcheck score for that ring is increased.
  2982.  * Further, acthd is inspected to see if the ring is stuck. On stuck case
  2983.  * we kick the ring. If we see no progress on three subsequent calls
  2984.  * we assume chip is wedged and try to fix it by resetting the chip.
  2985.  */
  2986. static void i915_hangcheck_elapsed(struct work_struct *work)
  2987. {
  2988.         struct drm_i915_private *dev_priv =
  2989.                 container_of(work, typeof(*dev_priv),
  2990.                              gpu_error.hangcheck_work.work);
  2991.         struct drm_device *dev = dev_priv->dev;
  2992.         struct intel_engine_cs *ring;
  2993.         int i;
  2994.         int busy_count = 0, rings_hung = 0;
  2995.         bool stuck[I915_NUM_RINGS] = { 0 };
  2996. #define BUSY 1
  2997. #define KICK 5
  2998. #define HUNG 20
  2999.  
  3000.         if (!i915.enable_hangcheck)
  3001.                 return;
  3002.  
  3003.         /*
  3004.          * The hangcheck work is synced during runtime suspend, we don't
  3005.          * require a wakeref. TODO: instead of disabling the asserts make
  3006.          * sure that we hold a reference when this work is running.
  3007.          */
  3008.         DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);
  3009.  
  3010.         for_each_ring(ring, dev_priv, i) {
  3011.                 u64 acthd;
  3012.                 u32 seqno;
  3013.                 bool busy = true;
  3014.  
  3015.                 semaphore_clear_deadlocks(dev_priv);
  3016.  
  3017.                 seqno = ring->get_seqno(ring, false);
  3018.                 acthd = intel_ring_get_active_head(ring);
  3019.  
  3020.                 if (ring->hangcheck.seqno == seqno) {
  3021.                         if (ring_idle(ring, seqno)) {
  3022.                                 ring->hangcheck.action = HANGCHECK_IDLE;
  3023.  
  3024.                                 if (waitqueue_active(&ring->irq_queue)) {
  3025.                                         /* Issue a wake-up to catch stuck h/w. */
  3026.                                         if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
  3027.                                                 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
  3028.                                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
  3029.                                                                   ring->name);
  3030.                                                 else
  3031.                                                         DRM_INFO("Fake missed irq on %s\n",
  3032.                                                                  ring->name);
  3033.                                                 wake_up_all(&ring->irq_queue);
  3034.                                         }
  3035.                                         /* Safeguard against driver failure */
  3036.                                         ring->hangcheck.score += BUSY;
  3037.                                 } else
  3038.                                         busy = false;
  3039.                         } else {
  3040.                                 /* We always increment the hangcheck score
  3041.                                  * if the ring is busy and still processing
  3042.                                  * the same request, so that no single request
  3043.                                  * can run indefinitely (such as a chain of
  3044.                                  * batches). The only time we do not increment
  3045.                                  * the hangcheck score on this ring, if this
  3046.                                  * ring is in a legitimate wait for another
  3047.                                  * ring. In that case the waiting ring is a
  3048.                                  * victim and we want to be sure we catch the
  3049.                                  * right culprit. Then every time we do kick
  3050.                                  * the ring, add a small increment to the
  3051.                                  * score so that we can catch a batch that is
  3052.                                  * being repeatedly kicked and so responsible
  3053.                                  * for stalling the machine.
  3054.                                  */
  3055.                                 ring->hangcheck.action = ring_stuck(ring,
  3056.                                                                     acthd);
  3057.  
  3058.                                 switch (ring->hangcheck.action) {
  3059.                                 case HANGCHECK_IDLE:
  3060.                                 case HANGCHECK_WAIT:
  3061.                                 case HANGCHECK_ACTIVE:
  3062.                                         break;
  3063.                                 case HANGCHECK_ACTIVE_LOOP:
  3064.                                         ring->hangcheck.score += BUSY;
  3065.                                         break;
  3066.                                 case HANGCHECK_KICK:
  3067.                                         ring->hangcheck.score += KICK;
  3068.                                         break;
  3069.                                 case HANGCHECK_HUNG:
  3070.                                         ring->hangcheck.score += HUNG;
  3071.                                         stuck[i] = true;
  3072.                                         break;
  3073.                                 }
  3074.                         }
  3075.                 } else {
  3076.                         ring->hangcheck.action = HANGCHECK_ACTIVE;
  3077.  
  3078.                         /* Gradually reduce the count so that we catch DoS
  3079.                          * attempts across multiple batches.
  3080.                          */
  3081.                         if (ring->hangcheck.score > 0)
  3082.                                 ring->hangcheck.score--;
  3083.  
  3084.                         ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
  3085.                 }
  3086.  
  3087.                 ring->hangcheck.seqno = seqno;
  3088.                 ring->hangcheck.acthd = acthd;
  3089.                 busy_count += busy;
  3090.         }
  3091.  
  3092.         for_each_ring(ring, dev_priv, i) {
  3093.                 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
  3094.                         DRM_INFO("%s on %s\n",
  3095.                                  stuck[i] ? "stuck" : "no progress",
  3096.                                  ring->name);
  3097.                         rings_hung++;
  3098.                 }
  3099.         }
  3100.  
  3101. //   if (rings_hung)
  3102. //       return i915_handle_error(dev, true);
  3103.  
  3104. }
  3105.  
  3106. static void ibx_irq_reset(struct drm_device *dev)
  3107. {
  3108.         struct drm_i915_private *dev_priv = dev->dev_private;
  3109.  
  3110.         if (HAS_PCH_NOP(dev))
  3111.                 return;
  3112.  
  3113.         GEN5_IRQ_RESET(SDE);
  3114.  
  3115.         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
  3116.                 I915_WRITE(SERR_INT, 0xffffffff);
  3117. }
  3118.  
  3119. /*
  3120.  * SDEIER is also touched by the interrupt handler to work around missed PCH
  3121.  * interrupts. Hence we can't update it after the interrupt handler is enabled -
  3122.  * instead we unconditionally enable all PCH interrupt sources here, but then
  3123.  * only unmask them as needed with SDEIMR.
  3124.  *
  3125.  * This function needs to be called before interrupts are enabled.
  3126.  */
  3127. static void ibx_irq_pre_postinstall(struct drm_device *dev)
  3128. {
  3129.         struct drm_i915_private *dev_priv = dev->dev_private;
  3130.  
  3131.         if (HAS_PCH_NOP(dev))
  3132.                 return;
  3133.  
  3134.         WARN_ON(I915_READ(SDEIER) != 0);
  3135.         I915_WRITE(SDEIER, 0xffffffff);
  3136.         POSTING_READ(SDEIER);
  3137. }
  3138.  
  3139. static void gen5_gt_irq_reset(struct drm_device *dev)
  3140. {
  3141.         struct drm_i915_private *dev_priv = dev->dev_private;
  3142.  
  3143.         GEN5_IRQ_RESET(GT);
  3144.         if (INTEL_INFO(dev)->gen >= 6)
  3145.                 GEN5_IRQ_RESET(GEN6_PM);
  3146. }
  3147.  
  3148. /* drm_dma.h hooks
  3149. */
  3150. static void ironlake_irq_reset(struct drm_device *dev)
  3151. {
  3152.         struct drm_i915_private *dev_priv = dev->dev_private;
  3153.  
  3154.         I915_WRITE(HWSTAM, 0xffffffff);
  3155.  
  3156.         GEN5_IRQ_RESET(DE);
  3157.         if (IS_GEN7(dev))
  3158.                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
  3159.  
  3160.         gen5_gt_irq_reset(dev);
  3161.  
  3162.         ibx_irq_reset(dev);
  3163. }
  3164.  
  3165. static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
  3166. {
  3167.         enum pipe pipe;
  3168.  
  3169.         i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
  3170.         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
  3171.  
  3172.         for_each_pipe(dev_priv, pipe)
  3173.                 I915_WRITE(PIPESTAT(pipe), 0xffff);
  3174.  
  3175.         GEN5_IRQ_RESET(VLV_);
  3176. }
  3177.  
  3178. static void valleyview_irq_preinstall(struct drm_device *dev)
  3179. {
  3180.         struct drm_i915_private *dev_priv = dev->dev_private;
  3181.  
  3182.         /* VLV magic */
  3183.         I915_WRITE(VLV_IMR, 0);
  3184.         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
  3185.         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
  3186.         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
  3187.  
  3188.         gen5_gt_irq_reset(dev);
  3189.  
  3190.         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
  3191.  
  3192.         vlv_display_irq_reset(dev_priv);
  3193. }
  3194.  
  3195. static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
  3196. {
  3197.         GEN8_IRQ_RESET_NDX(GT, 0);
  3198.         GEN8_IRQ_RESET_NDX(GT, 1);
  3199.         GEN8_IRQ_RESET_NDX(GT, 2);
  3200.         GEN8_IRQ_RESET_NDX(GT, 3);
  3201. }
  3202.  
  3203. static void gen8_irq_reset(struct drm_device *dev)
  3204. {
  3205.         struct drm_i915_private *dev_priv = dev->dev_private;
  3206.         int pipe;
  3207.  
  3208.         I915_WRITE(GEN8_MASTER_IRQ, 0);
  3209.         POSTING_READ(GEN8_MASTER_IRQ);
  3210.  
  3211.         gen8_gt_irq_reset(dev_priv);
  3212.  
  3213.         for_each_pipe(dev_priv, pipe)
  3214.                 if (intel_display_power_is_enabled(dev_priv,
  3215.                                                    POWER_DOMAIN_PIPE(pipe)))
  3216.                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
  3217.  
  3218.         GEN5_IRQ_RESET(GEN8_DE_PORT_);
  3219.         GEN5_IRQ_RESET(GEN8_DE_MISC_);
  3220.         GEN5_IRQ_RESET(GEN8_PCU_);
  3221.  
  3222.         if (HAS_PCH_SPLIT(dev))
  3223.                 ibx_irq_reset(dev);
  3224. }
  3225.  
  3226. void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
  3227.                                      unsigned int pipe_mask)
  3228. {
  3229.         uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
  3230.  
  3231.         spin_lock_irq(&dev_priv->irq_lock);
  3232.         if (pipe_mask & 1 << PIPE_A)
  3233.                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
  3234.                                   dev_priv->de_irq_mask[PIPE_A],
  3235.                                   ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
  3236.         if (pipe_mask & 1 << PIPE_B)
  3237.                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
  3238.                                   dev_priv->de_irq_mask[PIPE_B],
  3239.                                   ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
  3240.         if (pipe_mask & 1 << PIPE_C)
  3241.                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
  3242.                                   dev_priv->de_irq_mask[PIPE_C],
  3243.                                   ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
  3244.         spin_unlock_irq(&dev_priv->irq_lock);
  3245. }
  3246.  
  3247. static void cherryview_irq_preinstall(struct drm_device *dev)
  3248. {
  3249.         struct drm_i915_private *dev_priv = dev->dev_private;
  3250.  
  3251.         I915_WRITE(GEN8_MASTER_IRQ, 0);
  3252.         POSTING_READ(GEN8_MASTER_IRQ);
  3253.  
  3254.         gen8_gt_irq_reset(dev_priv);
  3255.  
  3256.         GEN5_IRQ_RESET(GEN8_PCU_);
  3257.  
  3258.         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
  3259.  
  3260.         vlv_display_irq_reset(dev_priv);
  3261. }
  3262.  
  3263. static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
  3264.                                   const u32 hpd[HPD_NUM_PINS])
  3265. {
  3266.         struct drm_i915_private *dev_priv = to_i915(dev);
  3267.         struct intel_encoder *encoder;
  3268.         u32 enabled_irqs = 0;
  3269.  
  3270.         for_each_intel_encoder(dev, encoder)
  3271.                 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
  3272.                         enabled_irqs |= hpd[encoder->hpd_pin];
  3273.  
  3274.         return enabled_irqs;
  3275. }
  3276.  
  3277. static void ibx_hpd_irq_setup(struct drm_device *dev)
  3278. {
  3279.         struct drm_i915_private *dev_priv = dev->dev_private;
  3280.         u32 hotplug_irqs, hotplug, enabled_irqs;
  3281.  
  3282.         if (HAS_PCH_IBX(dev)) {
  3283.                 hotplug_irqs = SDE_HOTPLUG_MASK;
  3284.                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
  3285.         } else {
  3286.                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
  3287.                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
  3288.         }
  3289.  
  3290.         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
  3291.  
  3292.         /*
  3293.          * Enable digital hotplug on the PCH, and configure the DP short pulse
  3294.          * duration to 2ms (which is the minimum in the Display Port spec).
  3295.          * The pulse duration bits are reserved on LPT+.
  3296.          */
  3297.         hotplug = I915_READ(PCH_PORT_HOTPLUG);
  3298.         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
  3299.         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
  3300.         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
  3301.         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
  3302.         /*
  3303.          * When CPU and PCH are on the same package, port A
  3304.          * HPD must be enabled in both north and south.
  3305.          */
  3306.         if (HAS_PCH_LPT_LP(dev))
  3307.                 hotplug |= PORTA_HOTPLUG_ENABLE;
  3308.         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
  3309. }
  3310.  
  3311. static void spt_hpd_irq_setup(struct drm_device *dev)
  3312. {
  3313.         struct drm_i915_private *dev_priv = dev->dev_private;
  3314.         u32 hotplug_irqs, hotplug, enabled_irqs;
  3315.  
  3316.         hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
  3317.         enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
  3318.  
  3319.         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
  3320.  
  3321.         /* Enable digital hotplug on the PCH */
  3322.         hotplug = I915_READ(PCH_PORT_HOTPLUG);
  3323.         hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
  3324.                 PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
  3325.         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
  3326.  
  3327.         hotplug = I915_READ(PCH_PORT_HOTPLUG2);
  3328.         hotplug |= PORTE_HOTPLUG_ENABLE;
  3329.         I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
  3330. }
  3331.  
  3332. static void ilk_hpd_irq_setup(struct drm_device *dev)
  3333. {
  3334.         struct drm_i915_private *dev_priv = dev->dev_private;
  3335.         u32 hotplug_irqs, hotplug, enabled_irqs;
  3336.  
  3337.         if (INTEL_INFO(dev)->gen >= 8) {
  3338.                 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
  3339.                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
  3340.  
  3341.                 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
  3342.         } else if (INTEL_INFO(dev)->gen >= 7) {
  3343.                 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
  3344.                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
  3345.  
  3346.                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
  3347.         } else {
  3348.                 hotplug_irqs = DE_DP_A_HOTPLUG;
  3349.                 enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
  3350.  
  3351.                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
  3352.         }
  3353.  
  3354.         /*
  3355.          * Enable digital hotplug on the CPU, and configure the DP short pulse
  3356.          * duration to 2ms (which is the minimum in the Display Port spec)
  3357.          * The pulse duration bits are reserved on HSW+.
  3358.          */
  3359.         hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
  3360.         hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
  3361.         hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
  3362.         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
  3363.  
  3364.         ibx_hpd_irq_setup(dev);
  3365. }
  3366.  
  3367. static void bxt_hpd_irq_setup(struct drm_device *dev)
  3368. {
  3369.         struct drm_i915_private *dev_priv = dev->dev_private;
  3370.         u32 hotplug_irqs, hotplug, enabled_irqs;
  3371.  
  3372.         enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
  3373.         hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
  3374.  
  3375.         bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
  3376.  
  3377.         hotplug = I915_READ(PCH_PORT_HOTPLUG);
  3378.         hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
  3379.                 PORTA_HOTPLUG_ENABLE;
  3380.         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
  3381. }
  3382.  
  3383. static void ibx_irq_postinstall(struct drm_device *dev)
  3384. {
  3385.         struct drm_i915_private *dev_priv = dev->dev_private;
  3386.         u32 mask;
  3387.  
  3388.         if (HAS_PCH_NOP(dev))
  3389.                 return;
  3390.  
  3391.         if (HAS_PCH_IBX(dev))
  3392.                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
  3393.         else
  3394.                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
  3395.  
  3396.         gen5_assert_iir_is_zero(dev_priv, SDEIIR);
  3397.         I915_WRITE(SDEIMR, ~mask);
  3398. }
  3399.  
  3400. static void gen5_gt_irq_postinstall(struct drm_device *dev)
  3401. {
  3402.         struct drm_i915_private *dev_priv = dev->dev_private;
  3403.         u32 pm_irqs, gt_irqs;
  3404.  
  3405.         pm_irqs = gt_irqs = 0;
  3406.  
  3407.         dev_priv->gt_irq_mask = ~0;
  3408.         if (HAS_L3_DPF(dev)) {
  3409.                 /* L3 parity interrupt is always unmasked. */
  3410.                 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
  3411.                 gt_irqs |= GT_PARITY_ERROR(dev);
  3412.         }
  3413.  
  3414.         gt_irqs |= GT_RENDER_USER_INTERRUPT;
  3415.         if (IS_GEN5(dev)) {
  3416.                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
  3417.                            ILK_BSD_USER_INTERRUPT;
  3418.         } else {
  3419.                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
  3420.         }
  3421.  
  3422.         GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
  3423.  
  3424.         if (INTEL_INFO(dev)->gen >= 6) {
  3425.                 /*
  3426.                  * RPS interrupts will get enabled/disabled on demand when RPS
  3427.                  * itself is enabled/disabled.
  3428.                  */
  3429.                 if (HAS_VEBOX(dev))
  3430.                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
  3431.  
  3432.                 dev_priv->pm_irq_mask = 0xffffffff;
  3433.                 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
  3434.         }
  3435. }
  3436.  
  3437. static int ironlake_irq_postinstall(struct drm_device *dev)
  3438. {
  3439.         struct drm_i915_private *dev_priv = dev->dev_private;
  3440.         u32 display_mask, extra_mask;
  3441.  
  3442.         if (INTEL_INFO(dev)->gen >= 7) {
  3443.                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
  3444.                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
  3445.                                 DE_PLANEB_FLIP_DONE_IVB |
  3446.                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
  3447.                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
  3448.                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
  3449.                               DE_DP_A_HOTPLUG_IVB);
  3450.         } else {
  3451.                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
  3452.                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
  3453.                                 DE_AUX_CHANNEL_A |
  3454.                                 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
  3455.                                 DE_POISON);
  3456.                 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
  3457.                               DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
  3458.                               DE_DP_A_HOTPLUG);
  3459.         }
  3460.  
  3461.         dev_priv->irq_mask = ~display_mask;
  3462.  
  3463.         I915_WRITE(HWSTAM, 0xeffe);
  3464.  
  3465.         ibx_irq_pre_postinstall(dev);
  3466.  
  3467.         GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
  3468.  
  3469.         gen5_gt_irq_postinstall(dev);
  3470.  
  3471.         ibx_irq_postinstall(dev);
  3472.  
  3473.         if (IS_IRONLAKE_M(dev)) {
  3474.                 /* Enable PCU event interrupts
  3475.                  *
  3476.                  * spinlocking not required here for correctness since interrupt
  3477.                  * setup is guaranteed to run in single-threaded context. But we
  3478.                  * need it to make the assert_spin_locked happy. */
  3479.                 spin_lock_irq(&dev_priv->irq_lock);
  3480.                 ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
  3481.                 spin_unlock_irq(&dev_priv->irq_lock);
  3482.         }
  3483.  
  3484.         return 0;
  3485. }
  3486.  
  3487. static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
  3488. {
  3489.         u32 pipestat_mask;
  3490.         u32 iir_mask;
  3491.         enum pipe pipe;
  3492.  
  3493.         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
  3494.                         PIPE_FIFO_UNDERRUN_STATUS;
  3495.  
  3496.         for_each_pipe(dev_priv, pipe)
  3497.                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
  3498.         POSTING_READ(PIPESTAT(PIPE_A));
  3499.  
  3500.         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
  3501.                         PIPE_CRC_DONE_INTERRUPT_STATUS;
  3502.  
  3503.         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
  3504.         for_each_pipe(dev_priv, pipe)
  3505.                       i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
  3506.  
  3507.         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
  3508.                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  3509.                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
  3510.         if (IS_CHERRYVIEW(dev_priv))
  3511.                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
  3512.         dev_priv->irq_mask &= ~iir_mask;
  3513.  
  3514.         I915_WRITE(VLV_IIR, iir_mask);
  3515.         I915_WRITE(VLV_IIR, iir_mask);
  3516.         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
  3517.         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
  3518.         POSTING_READ(VLV_IMR);
  3519. }
  3520.  
  3521. static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
  3522. {
  3523.         u32 pipestat_mask;
  3524.         u32 iir_mask;
  3525.         enum pipe pipe;
  3526.  
  3527.         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
  3528.                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  3529.                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
  3530.         if (IS_CHERRYVIEW(dev_priv))
  3531.                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
  3532.  
  3533.         dev_priv->irq_mask |= iir_mask;
  3534.         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
  3535.         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
  3536.         I915_WRITE(VLV_IIR, iir_mask);
  3537.         I915_WRITE(VLV_IIR, iir_mask);
  3538.         POSTING_READ(VLV_IIR);
  3539.  
  3540.         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
  3541.                         PIPE_CRC_DONE_INTERRUPT_STATUS;
  3542.  
  3543.         i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
  3544.         for_each_pipe(dev_priv, pipe)
  3545.                 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
  3546.  
  3547.         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
  3548.                         PIPE_FIFO_UNDERRUN_STATUS;
  3549.  
  3550.         for_each_pipe(dev_priv, pipe)
  3551.                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
  3552.         POSTING_READ(PIPESTAT(PIPE_A));
  3553. }
  3554.  
  3555. void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
  3556. {
  3557.         assert_spin_locked(&dev_priv->irq_lock);
  3558.  
  3559.         if (dev_priv->display_irqs_enabled)
  3560.                 return;
  3561.  
  3562.         dev_priv->display_irqs_enabled = true;
  3563.  
  3564.         if (intel_irqs_enabled(dev_priv))
  3565.                 valleyview_display_irqs_install(dev_priv);
  3566. }
  3567.  
  3568. void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
  3569. {
  3570.         assert_spin_locked(&dev_priv->irq_lock);
  3571.  
  3572.         if (!dev_priv->display_irqs_enabled)
  3573.                 return;
  3574.  
  3575.         dev_priv->display_irqs_enabled = false;
  3576.  
  3577.         if (intel_irqs_enabled(dev_priv))
  3578.                 valleyview_display_irqs_uninstall(dev_priv);
  3579. }
  3580.  
  3581. static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
  3582. {
  3583.         dev_priv->irq_mask = ~0;
  3584.  
  3585.         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  3586.         POSTING_READ(PORT_HOTPLUG_EN);
  3587.  
  3588.         I915_WRITE(VLV_IIR, 0xffffffff);
  3589.         I915_WRITE(VLV_IIR, 0xffffffff);
  3590.         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
  3591.         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
  3592.         POSTING_READ(VLV_IMR);
  3593.  
  3594.         /* Interrupt setup is already guaranteed to be single-threaded, this is
  3595.          * just to make the assert_spin_locked check happy. */
  3596.         spin_lock_irq(&dev_priv->irq_lock);
  3597.         if (dev_priv->display_irqs_enabled)
  3598.                 valleyview_display_irqs_install(dev_priv);
  3599.         spin_unlock_irq(&dev_priv->irq_lock);
  3600. }
  3601.  
  3602. static int valleyview_irq_postinstall(struct drm_device *dev)
  3603. {
  3604.         struct drm_i915_private *dev_priv = dev->dev_private;
  3605.  
  3606.         vlv_display_irq_postinstall(dev_priv);
  3607.  
  3608.         gen5_gt_irq_postinstall(dev);
  3609.  
  3610.         /* ack & enable invalid PTE error interrupts */
  3611. #if 0 /* FIXME: add support to irq handler for checking these bits */
  3612.         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
  3613.         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
  3614. #endif
  3615.  
  3616.         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
  3617.  
  3618.         return 0;
  3619. }
  3620.  
  3621. static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
  3622. {
  3623.         /* These are interrupts we'll toggle with the ring mask register */
  3624.         uint32_t gt_interrupts[] = {
  3625.                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
  3626.                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
  3627.                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
  3628.                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
  3629.                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
  3630.                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
  3631.                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
  3632.                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
  3633.                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
  3634.                 0,
  3635.                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
  3636.                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
  3637.                 };
  3638.  
  3639.         dev_priv->pm_irq_mask = 0xffffffff;
  3640.         GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
  3641.         GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
  3642.         /*
  3643.          * RPS interrupts will get enabled/disabled on demand when RPS itself
  3644.          * is enabled/disabled.
  3645.          */
  3646.         GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
  3647.         GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
  3648. }
  3649.  
  3650. static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
  3651. {
  3652.         uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
  3653.         uint32_t de_pipe_enables;
  3654.         u32 de_port_masked = GEN8_AUX_CHANNEL_A;
  3655.         u32 de_port_enables;
  3656.         enum pipe pipe;
  3657.  
  3658.         if (INTEL_INFO(dev_priv)->gen >= 9) {
  3659.                 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
  3660.                                   GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
  3661.                 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
  3662.                                   GEN9_AUX_CHANNEL_D;
  3663.                 if (IS_BROXTON(dev_priv))
  3664.                         de_port_masked |= BXT_DE_PORT_GMBUS;
  3665.         } else {
  3666.                 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
  3667.                                   GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
  3668.         }
  3669.  
  3670.         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
  3671.                                            GEN8_PIPE_FIFO_UNDERRUN;
  3672.  
  3673.         de_port_enables = de_port_masked;
  3674.         if (IS_BROXTON(dev_priv))
  3675.                 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
  3676.         else if (IS_BROADWELL(dev_priv))
  3677.                 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
  3678.  
  3679.         dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
  3680.         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
  3681.         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
  3682.  
  3683.         for_each_pipe(dev_priv, pipe)
  3684.                 if (intel_display_power_is_enabled(dev_priv,
  3685.                                 POWER_DOMAIN_PIPE(pipe)))
  3686.                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
  3687.                                           dev_priv->de_irq_mask[pipe],
  3688.                                           de_pipe_enables);
  3689.  
  3690.         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
  3691. }
  3692.  
  3693. static int gen8_irq_postinstall(struct drm_device *dev)
  3694. {
  3695.         struct drm_i915_private *dev_priv = dev->dev_private;
  3696.  
  3697.         if (HAS_PCH_SPLIT(dev))
  3698.                 ibx_irq_pre_postinstall(dev);
  3699.  
  3700.         gen8_gt_irq_postinstall(dev_priv);
  3701.         gen8_de_irq_postinstall(dev_priv);
  3702.  
  3703.         if (HAS_PCH_SPLIT(dev))
  3704.                 ibx_irq_postinstall(dev);
  3705.  
  3706.         I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
  3707.         POSTING_READ(GEN8_MASTER_IRQ);
  3708.  
  3709.         return 0;
  3710. }
  3711.  
  3712. static int cherryview_irq_postinstall(struct drm_device *dev)
  3713. {
  3714.         struct drm_i915_private *dev_priv = dev->dev_private;
  3715.  
  3716.         vlv_display_irq_postinstall(dev_priv);
  3717.  
  3718.         gen8_gt_irq_postinstall(dev_priv);
  3719.  
  3720.         I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
  3721.         POSTING_READ(GEN8_MASTER_IRQ);
  3722.  
  3723.         return 0;
  3724. }
  3725.  
  3726. static void gen8_irq_uninstall(struct drm_device *dev)
  3727. {
  3728.         struct drm_i915_private *dev_priv = dev->dev_private;
  3729.  
  3730.         if (!dev_priv)
  3731.                 return;
  3732.  
  3733.         gen8_irq_reset(dev);
  3734. }
  3735.  
  3736. static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
  3737. {
  3738.         /* Interrupt setup is already guaranteed to be single-threaded, this is
  3739.          * just to make the assert_spin_locked check happy. */
  3740.         spin_lock_irq(&dev_priv->irq_lock);
  3741.         if (dev_priv->display_irqs_enabled)
  3742.                 valleyview_display_irqs_uninstall(dev_priv);
  3743.         spin_unlock_irq(&dev_priv->irq_lock);
  3744.  
  3745.         vlv_display_irq_reset(dev_priv);
  3746.  
  3747.         dev_priv->irq_mask = ~0;
  3748. }
  3749.  
  3750. static void valleyview_irq_uninstall(struct drm_device *dev)
  3751. {
  3752.         struct drm_i915_private *dev_priv = dev->dev_private;
  3753.  
  3754.         if (!dev_priv)
  3755.                 return;
  3756.  
  3757.         I915_WRITE(VLV_MASTER_IER, 0);
  3758.  
  3759.         gen5_gt_irq_reset(dev);
  3760.  
  3761.         I915_WRITE(HWSTAM, 0xffffffff);
  3762.  
  3763.         vlv_display_irq_uninstall(dev_priv);
  3764. }
  3765.  
  3766. static void cherryview_irq_uninstall(struct drm_device *dev)
  3767. {
  3768.         struct drm_i915_private *dev_priv = dev->dev_private;
  3769.  
  3770.         if (!dev_priv)
  3771.                 return;
  3772.  
  3773.         I915_WRITE(GEN8_MASTER_IRQ, 0);
  3774.         POSTING_READ(GEN8_MASTER_IRQ);
  3775.  
  3776.         gen8_gt_irq_reset(dev_priv);
  3777.  
  3778.         GEN5_IRQ_RESET(GEN8_PCU_);
  3779.  
  3780.         vlv_display_irq_uninstall(dev_priv);
  3781. }
  3782.  
  3783. static void ironlake_irq_uninstall(struct drm_device *dev)
  3784. {
  3785.         struct drm_i915_private *dev_priv = dev->dev_private;
  3786.  
  3787.         if (!dev_priv)
  3788.                 return;
  3789.  
  3790.         ironlake_irq_reset(dev);
  3791. }
  3792.  
  3793. #if 0
  3794. static void i8xx_irq_preinstall(struct drm_device * dev)
  3795. {
  3796.         struct drm_i915_private *dev_priv = dev->dev_private;
  3797.         int pipe;
  3798.  
  3799.         for_each_pipe(dev_priv, pipe)
  3800.                 I915_WRITE(PIPESTAT(pipe), 0);
  3801.         I915_WRITE16(IMR, 0xffff);
  3802.         I915_WRITE16(IER, 0x0);
  3803.         POSTING_READ16(IER);
  3804. }
  3805.  
  3806. static int i8xx_irq_postinstall(struct drm_device *dev)
  3807. {
  3808.         struct drm_i915_private *dev_priv = dev->dev_private;
  3809.  
  3810.         I915_WRITE16(EMR,
  3811.                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
  3812.  
  3813.         /* Unmask the interrupts that we always want on. */
  3814.         dev_priv->irq_mask =
  3815.                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  3816.                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
  3817.                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  3818.                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
  3819.         I915_WRITE16(IMR, dev_priv->irq_mask);
  3820.  
  3821.         I915_WRITE16(IER,
  3822.                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  3823.                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
  3824.                      I915_USER_INTERRUPT);
  3825.         POSTING_READ16(IER);
  3826.  
  3827.         /* Interrupt setup is already guaranteed to be single-threaded, this is
  3828.          * just to make the assert_spin_locked check happy. */
  3829.         spin_lock_irq(&dev_priv->irq_lock);
  3830.         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
  3831.         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
  3832.         spin_unlock_irq(&dev_priv->irq_lock);
  3833.  
  3834.         return 0;
  3835. }
  3836.  
  3837. /*
  3838.  * Returns true when a page flip has completed.
  3839.  */
  3840. static bool i8xx_handle_vblank(struct drm_device *dev,
  3841.                                int plane, int pipe, u32 iir)
  3842. {
  3843.         struct drm_i915_private *dev_priv = dev->dev_private;
  3844.         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
  3845.  
  3846.         if (!intel_pipe_handle_vblank(dev, pipe))
  3847.                 return false;
  3848.  
  3849.         if ((iir & flip_pending) == 0)
  3850.                 goto check_page_flip;
  3851.  
  3852.         /* We detect FlipDone by looking for the change in PendingFlip from '1'
  3853.          * to '0' on the following vblank, i.e. IIR has the Pendingflip
  3854.          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
  3855.          * the flip is completed (no longer pending). Since this doesn't raise
  3856.          * an interrupt per se, we watch for the change at vblank.
  3857.          */
  3858.         if (I915_READ16(ISR) & flip_pending)
  3859.                 goto check_page_flip;
  3860.  
  3861.         intel_prepare_page_flip(dev, plane);
  3862.         intel_finish_page_flip(dev, pipe);
  3863.         return true;
  3864.  
  3865. check_page_flip:
  3866.         intel_check_page_flip(dev, pipe);
  3867.         return false;
  3868. }
  3869.  
  3870. static irqreturn_t i8xx_irq_handler(int irq, void *arg)
  3871. {
  3872.         struct drm_device *dev = arg;
  3873.         struct drm_i915_private *dev_priv = dev->dev_private;
  3874.         u16 iir, new_iir;
  3875.         u32 pipe_stats[2];
  3876.         int pipe;
  3877.         u16 flip_mask =
  3878.                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  3879.                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
  3880.         irqreturn_t ret;
  3881.  
  3882.         if (!intel_irqs_enabled(dev_priv))
  3883.                 return IRQ_NONE;
  3884.  
  3885.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  3886.         disable_rpm_wakeref_asserts(dev_priv);
  3887.  
  3888.         ret = IRQ_NONE;
  3889.         iir = I915_READ16(IIR);
  3890.         if (iir == 0)
  3891.                 goto out;
  3892.  
  3893.         while (iir & ~flip_mask) {
  3894.                 /* Can't rely on pipestat interrupt bit in iir as it might
  3895.                  * have been cleared after the pipestat interrupt was received.
  3896.                  * It doesn't set the bit in iir again, but it still produces
  3897.                  * interrupts (for non-MSI).
  3898.                  */
  3899.                 spin_lock(&dev_priv->irq_lock);
  3900.                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
  3901.                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
  3902.  
  3903.                 for_each_pipe(dev_priv, pipe) {
  3904.                         i915_reg_t reg = PIPESTAT(pipe);
  3905.                         pipe_stats[pipe] = I915_READ(reg);
  3906.  
  3907.                         /*
  3908.                          * Clear the PIPE*STAT regs before the IIR
  3909.                          */
  3910.                         if (pipe_stats[pipe] & 0x8000ffff)
  3911.                                 I915_WRITE(reg, pipe_stats[pipe]);
  3912.                 }
  3913.                 spin_unlock(&dev_priv->irq_lock);
  3914.  
  3915.                 I915_WRITE16(IIR, iir & ~flip_mask);
  3916.                 new_iir = I915_READ16(IIR); /* Flush posted writes */
  3917.  
  3918.                 if (iir & I915_USER_INTERRUPT)
  3919.                         notify_ring(&dev_priv->ring[RCS]);
  3920.  
  3921.                 for_each_pipe(dev_priv, pipe) {
  3922.                         int plane = pipe;
  3923.                         if (HAS_FBC(dev))
  3924.                                 plane = !plane;
  3925.  
  3926.                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
  3927.                             i8xx_handle_vblank(dev, plane, pipe, iir))
  3928.                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
  3929.  
  3930.                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
  3931.                                 i9xx_pipe_crc_irq_handler(dev, pipe);
  3932.  
  3933.                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
  3934.                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
  3935.                                                                     pipe);
  3936.                 }
  3937.  
  3938.                 iir = new_iir;
  3939.         }
  3940.         ret = IRQ_HANDLED;
  3941.  
  3942. out:
  3943.         enable_rpm_wakeref_asserts(dev_priv);
  3944.  
  3945.         return ret;
  3946. }
  3947.  
  3948. static void i8xx_irq_uninstall(struct drm_device * dev)
  3949. {
  3950.         struct drm_i915_private *dev_priv = dev->dev_private;
  3951.         int pipe;
  3952.  
  3953.         for_each_pipe(dev_priv, pipe) {
  3954.                 /* Clear enable bits; then clear status bits */
  3955.                 I915_WRITE(PIPESTAT(pipe), 0);
  3956.                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
  3957.         }
  3958.         I915_WRITE16(IMR, 0xffff);
  3959.         I915_WRITE16(IER, 0x0);
  3960.         I915_WRITE16(IIR, I915_READ16(IIR));
  3961. }
  3962.  
  3963. #endif
  3964.  
  3965. static void i915_irq_preinstall(struct drm_device * dev)
  3966. {
  3967.         struct drm_i915_private *dev_priv = dev->dev_private;
  3968.         int pipe;
  3969.  
  3970.         if (I915_HAS_HOTPLUG(dev)) {
  3971.                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  3972.                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
  3973.         }
  3974.  
  3975.         I915_WRITE16(HWSTAM, 0xeffe);
  3976.         for_each_pipe(dev_priv, pipe)
  3977.                 I915_WRITE(PIPESTAT(pipe), 0);
  3978.         I915_WRITE(IMR, 0xffffffff);
  3979.         I915_WRITE(IER, 0x0);
  3980.         POSTING_READ(IER);
  3981. }
  3982.  
  3983. static int i915_irq_postinstall(struct drm_device *dev)
  3984. {
  3985.         struct drm_i915_private *dev_priv = dev->dev_private;
  3986.         u32 enable_mask;
  3987.  
  3988.         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
  3989.  
  3990.         /* Unmask the interrupts that we always want on. */
  3991.         dev_priv->irq_mask =
  3992.                 ~(I915_ASLE_INTERRUPT |
  3993.                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  3994.                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
  3995.                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  3996.                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
  3997.  
  3998.         enable_mask =
  3999.                 I915_ASLE_INTERRUPT |
  4000.                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  4001.                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
  4002.                 I915_USER_INTERRUPT;
  4003.  
  4004.         if (I915_HAS_HOTPLUG(dev)) {
  4005.                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  4006.                 POSTING_READ(PORT_HOTPLUG_EN);
  4007.  
  4008.                 /* Enable in IER... */
  4009.                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
  4010.                 /* and unmask in IMR */
  4011.                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
  4012.         }
  4013.  
  4014.         I915_WRITE(IMR, dev_priv->irq_mask);
  4015.         I915_WRITE(IER, enable_mask);
  4016.         POSTING_READ(IER);
  4017.  
  4018.         i915_enable_asle_pipestat(dev);
  4019.  
  4020.         /* Interrupt setup is already guaranteed to be single-threaded, this is
  4021.          * just to make the assert_spin_locked check happy. */
  4022.         spin_lock_irq(&dev_priv->irq_lock);
  4023.         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
  4024.         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
  4025.         spin_unlock_irq(&dev_priv->irq_lock);
  4026.  
  4027.         return 0;
  4028. }
  4029.  
  4030. /*
  4031.  * Returns true when a page flip has completed.
  4032.  */
  4033. static bool i915_handle_vblank(struct drm_device *dev,
  4034.                                int plane, int pipe, u32 iir)
  4035. {
  4036.         struct drm_i915_private *dev_priv = dev->dev_private;
  4037.         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
  4038.  
  4039.         if (!intel_pipe_handle_vblank(dev, pipe))
  4040.                 return false;
  4041.  
  4042.         if ((iir & flip_pending) == 0)
  4043.                 goto check_page_flip;
  4044.  
  4045.         /* We detect FlipDone by looking for the change in PendingFlip from '1'
  4046.          * to '0' on the following vblank, i.e. IIR has the Pendingflip
  4047.          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
  4048.          * the flip is completed (no longer pending). Since this doesn't raise
  4049.          * an interrupt per se, we watch for the change at vblank.
  4050.          */
  4051.         if (I915_READ(ISR) & flip_pending)
  4052.                 goto check_page_flip;
  4053.  
  4054.         intel_prepare_page_flip(dev, plane);
  4055.         intel_finish_page_flip(dev, pipe);
  4056.         return true;
  4057.  
  4058. check_page_flip:
  4059.         intel_check_page_flip(dev, pipe);
  4060.         return false;
  4061. }
  4062.  
  4063. static irqreturn_t i915_irq_handler(int irq, void *arg)
  4064. {
  4065.         struct drm_device *dev = arg;
  4066.         struct drm_i915_private *dev_priv = dev->dev_private;
  4067.         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
  4068.         u32 flip_mask =
  4069.                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  4070.                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
  4071.         int pipe, ret = IRQ_NONE;
  4072.  
  4073.         if (!intel_irqs_enabled(dev_priv))
  4074.                 return IRQ_NONE;
  4075.  
  4076.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  4077.         disable_rpm_wakeref_asserts(dev_priv);
  4078.  
  4079.         iir = I915_READ(IIR);
  4080.         do {
  4081.                 bool irq_received = (iir & ~flip_mask) != 0;
  4082.                 bool blc_event = false;
  4083.  
  4084.                 /* Can't rely on pipestat interrupt bit in iir as it might
  4085.                  * have been cleared after the pipestat interrupt was received.
  4086.                  * It doesn't set the bit in iir again, but it still produces
  4087.                  * interrupts (for non-MSI).
  4088.                  */
  4089.                 spin_lock(&dev_priv->irq_lock);
  4090.                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
  4091.                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
  4092.  
  4093.                 for_each_pipe(dev_priv, pipe) {
  4094.                         i915_reg_t reg = PIPESTAT(pipe);
  4095.                         pipe_stats[pipe] = I915_READ(reg);
  4096.  
  4097.                         /* Clear the PIPE*STAT regs before the IIR */
  4098.                         if (pipe_stats[pipe] & 0x8000ffff) {
  4099.                                 I915_WRITE(reg, pipe_stats[pipe]);
  4100.                                 irq_received = true;
  4101.                         }
  4102.                 }
  4103.                 spin_unlock(&dev_priv->irq_lock);
  4104.  
  4105.                 if (!irq_received)
  4106.                         break;
  4107.  
  4108.                 /* Consume port.  Then clear IIR or we'll miss events */
  4109.                 if (I915_HAS_HOTPLUG(dev) &&
  4110.                     iir & I915_DISPLAY_PORT_INTERRUPT)
  4111.                         i9xx_hpd_irq_handler(dev);
  4112.  
  4113.                 I915_WRITE(IIR, iir & ~flip_mask);
  4114.                 new_iir = I915_READ(IIR); /* Flush posted writes */
  4115.  
  4116.                 if (iir & I915_USER_INTERRUPT)
  4117.                         notify_ring(&dev_priv->ring[RCS]);
  4118.  
  4119.                 for_each_pipe(dev_priv, pipe) {
  4120.                         int plane = pipe;
  4121.                         if (HAS_FBC(dev))
  4122.                                 plane = !plane;
  4123.  
  4124.                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
  4125.                             i915_handle_vblank(dev, plane, pipe, iir))
  4126.                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
  4127.  
  4128.                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
  4129.                                 blc_event = true;
  4130.  
  4131.                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
  4132.                                 i9xx_pipe_crc_irq_handler(dev, pipe);
  4133.  
  4134.                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
  4135.                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
  4136.                                                                     pipe);
  4137.                 }
  4138.  
  4139.                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
  4140.                         intel_opregion_asle_intr(dev);
  4141.  
  4142.                 /* With MSI, interrupts are only generated when iir
  4143.                  * transitions from zero to nonzero.  If another bit got
  4144.                  * set while we were handling the existing iir bits, then
  4145.                  * we would never get another interrupt.
  4146.                  *
  4147.                  * This is fine on non-MSI as well, as if we hit this path
  4148.                  * we avoid exiting the interrupt handler only to generate
  4149.                  * another one.
  4150.                  *
  4151.                  * Note that for MSI this could cause a stray interrupt report
  4152.                  * if an interrupt landed in the time between writing IIR and
  4153.                  * the posting read.  This should be rare enough to never
  4154.                  * trigger the 99% of 100,000 interrupts test for disabling
  4155.                  * stray interrupts.
  4156.                  */
  4157.                 ret = IRQ_HANDLED;
  4158.                 iir = new_iir;
  4159.         } while (iir & ~flip_mask);
  4160.  
  4161.         enable_rpm_wakeref_asserts(dev_priv);
  4162.  
  4163.         return ret;
  4164. }
  4165.  
  4166. static void i915_irq_uninstall(struct drm_device * dev)
  4167. {
  4168.         struct drm_i915_private *dev_priv = dev->dev_private;
  4169.         int pipe;
  4170.  
  4171.         if (I915_HAS_HOTPLUG(dev)) {
  4172.                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  4173.                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
  4174.         }
  4175.  
  4176.         I915_WRITE16(HWSTAM, 0xffff);
  4177.         for_each_pipe(dev_priv, pipe) {
  4178.                 /* Clear enable bits; then clear status bits */
  4179.                 I915_WRITE(PIPESTAT(pipe), 0);
  4180.                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
  4181.         }
  4182.         I915_WRITE(IMR, 0xffffffff);
  4183.         I915_WRITE(IER, 0x0);
  4184.  
  4185.         I915_WRITE(IIR, I915_READ(IIR));
  4186. }
  4187.  
  4188. static void i965_irq_preinstall(struct drm_device * dev)
  4189. {
  4190.         struct drm_i915_private *dev_priv = dev->dev_private;
  4191.         int pipe;
  4192.  
  4193.         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  4194.         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
  4195.  
  4196.         I915_WRITE(HWSTAM, 0xeffe);
  4197.         for_each_pipe(dev_priv, pipe)
  4198.                 I915_WRITE(PIPESTAT(pipe), 0);
  4199.         I915_WRITE(IMR, 0xffffffff);
  4200.         I915_WRITE(IER, 0x0);
  4201.         POSTING_READ(IER);
  4202. }
  4203.  
  4204. static int i965_irq_postinstall(struct drm_device *dev)
  4205. {
  4206.         struct drm_i915_private *dev_priv = dev->dev_private;
  4207.         u32 enable_mask;
  4208.         u32 error_mask;
  4209.  
  4210.         /* Unmask the interrupts that we always want on. */
  4211.         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
  4212.                                I915_DISPLAY_PORT_INTERRUPT |
  4213.                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
  4214.                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
  4215.                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  4216.                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
  4217.                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
  4218.  
  4219.         enable_mask = ~dev_priv->irq_mask;
  4220.         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  4221.                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
  4222.         enable_mask |= I915_USER_INTERRUPT;
  4223.  
  4224.         if (IS_G4X(dev))
  4225.                 enable_mask |= I915_BSD_USER_INTERRUPT;
  4226.  
  4227.         /* Interrupt setup is already guaranteed to be single-threaded, this is
  4228.          * just to make the assert_spin_locked check happy. */
  4229.         spin_lock_irq(&dev_priv->irq_lock);
  4230.         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
  4231.         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
  4232.         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
  4233.         spin_unlock_irq(&dev_priv->irq_lock);
  4234.  
  4235.         /*
  4236.          * Enable some error detection, note the instruction error mask
  4237.          * bit is reserved, so we leave it masked.
  4238.          */
  4239.         if (IS_G4X(dev)) {
  4240.                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
  4241.                                GM45_ERROR_MEM_PRIV |
  4242.                                GM45_ERROR_CP_PRIV |
  4243.                                I915_ERROR_MEMORY_REFRESH);
  4244.         } else {
  4245.                 error_mask = ~(I915_ERROR_PAGE_TABLE |
  4246.                                I915_ERROR_MEMORY_REFRESH);
  4247.         }
  4248.         I915_WRITE(EMR, error_mask);
  4249.  
  4250.         I915_WRITE(IMR, dev_priv->irq_mask);
  4251.         I915_WRITE(IER, enable_mask);
  4252.         POSTING_READ(IER);
  4253.  
  4254.         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  4255.         POSTING_READ(PORT_HOTPLUG_EN);
  4256.  
  4257.         i915_enable_asle_pipestat(dev);
  4258.  
  4259.         return 0;
  4260. }
  4261.  
  4262. static void i915_hpd_irq_setup(struct drm_device *dev)
  4263. {
  4264.         struct drm_i915_private *dev_priv = dev->dev_private;
  4265.         u32 hotplug_en;
  4266.  
  4267.         assert_spin_locked(&dev_priv->irq_lock);
  4268.  
  4269.         /* Note HDMI and DP share hotplug bits */
  4270.         /* enable bits are the same for all generations */
  4271.         hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
  4272.         /* Programming the CRT detection parameters tends
  4273.            to generate a spurious hotplug event about three
  4274.            seconds later.  So just do it once.
  4275.         */
  4276.         if (IS_G4X(dev))
  4277.                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
  4278.         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
  4279.  
  4280.         /* Ignore TV since it's buggy */
  4281.         i915_hotplug_interrupt_update_locked(dev_priv,
  4282.                                              HOTPLUG_INT_EN_MASK |
  4283.                                              CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
  4284.                                              CRT_HOTPLUG_ACTIVATION_PERIOD_64,
  4285.                                              hotplug_en);
  4286. }
  4287.  
  4288. static irqreturn_t i965_irq_handler(int irq, void *arg)
  4289. {
  4290.         struct drm_device *dev = arg;
  4291.         struct drm_i915_private *dev_priv = dev->dev_private;
  4292.         u32 iir, new_iir;
  4293.         u32 pipe_stats[I915_MAX_PIPES];
  4294.         int ret = IRQ_NONE, pipe;
  4295.         u32 flip_mask =
  4296.                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
  4297.                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
  4298.  
  4299.         if (!intel_irqs_enabled(dev_priv))
  4300.                 return IRQ_NONE;
  4301.  
  4302.         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
  4303.         disable_rpm_wakeref_asserts(dev_priv);
  4304.  
  4305.         iir = I915_READ(IIR);
  4306.  
  4307.         for (;;) {
  4308.                 bool irq_received = (iir & ~flip_mask) != 0;
  4309.                 bool blc_event = false;
  4310.  
  4311.                 /* Can't rely on pipestat interrupt bit in iir as it might
  4312.                  * have been cleared after the pipestat interrupt was received.
  4313.                  * It doesn't set the bit in iir again, but it still produces
  4314.                  * interrupts (for non-MSI).
  4315.                  */
  4316.                 spin_lock(&dev_priv->irq_lock);
  4317.                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
  4318.                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
  4319.  
  4320.                 for_each_pipe(dev_priv, pipe) {
  4321.                         i915_reg_t reg = PIPESTAT(pipe);
  4322.                         pipe_stats[pipe] = I915_READ(reg);
  4323.  
  4324.                         /*
  4325.                          * Clear the PIPE*STAT regs before the IIR
  4326.                          */
  4327.                         if (pipe_stats[pipe] & 0x8000ffff) {
  4328.                                 I915_WRITE(reg, pipe_stats[pipe]);
  4329.                                 irq_received = true;
  4330.                         }
  4331.                 }
  4332.                 spin_unlock(&dev_priv->irq_lock);
  4333.  
  4334.                 if (!irq_received)
  4335.                         break;
  4336.  
  4337.                 ret = IRQ_HANDLED;
  4338.  
  4339.                 /* Consume port.  Then clear IIR or we'll miss events */
  4340.                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
  4341.                         i9xx_hpd_irq_handler(dev);
  4342.  
  4343.                 I915_WRITE(IIR, iir & ~flip_mask);
  4344.                 new_iir = I915_READ(IIR); /* Flush posted writes */
  4345.  
  4346.                 if (iir & I915_USER_INTERRUPT)
  4347.                         notify_ring(&dev_priv->ring[RCS]);
  4348.                 if (iir & I915_BSD_USER_INTERRUPT)
  4349.                         notify_ring(&dev_priv->ring[VCS]);
  4350.  
  4351.                 for_each_pipe(dev_priv, pipe) {
  4352.                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
  4353.                             i915_handle_vblank(dev, pipe, pipe, iir))
  4354.                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
  4355.  
  4356.                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
  4357.                                 blc_event = true;
  4358.  
  4359.                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
  4360.                                 i9xx_pipe_crc_irq_handler(dev, pipe);
  4361.  
  4362.                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
  4363.                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
  4364.                 }
  4365.  
  4366.                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
  4367.                         intel_opregion_asle_intr(dev);
  4368.  
  4369.                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
  4370.                         gmbus_irq_handler(dev);
  4371.  
  4372.                 /* With MSI, interrupts are only generated when iir
  4373.                  * transitions from zero to nonzero.  If another bit got
  4374.                  * set while we were handling the existing iir bits, then
  4375.                  * we would never get another interrupt.
  4376.                  *
  4377.                  * This is fine on non-MSI as well, as if we hit this path
  4378.                  * we avoid exiting the interrupt handler only to generate
  4379.                  * another one.
  4380.                  *
  4381.                  * Note that for MSI this could cause a stray interrupt report
  4382.                  * if an interrupt landed in the time between writing IIR and
  4383.                  * the posting read.  This should be rare enough to never
  4384.                  * trigger the 99% of 100,000 interrupts test for disabling
  4385.                  * stray interrupts.
  4386.                  */
  4387.                 iir = new_iir;
  4388.         }
  4389.  
  4390.         enable_rpm_wakeref_asserts(dev_priv);
  4391.  
  4392.         return ret;
  4393. }
  4394.  
  4395. static void i965_irq_uninstall(struct drm_device * dev)
  4396. {
  4397.         struct drm_i915_private *dev_priv = dev->dev_private;
  4398.         int pipe;
  4399.  
  4400.         if (!dev_priv)
  4401.                 return;
  4402.  
  4403.         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
  4404.         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
  4405.  
  4406.         I915_WRITE(HWSTAM, 0xffffffff);
  4407.         for_each_pipe(dev_priv, pipe)
  4408.                 I915_WRITE(PIPESTAT(pipe), 0);
  4409.         I915_WRITE(IMR, 0xffffffff);
  4410.         I915_WRITE(IER, 0x0);
  4411.  
  4412.         for_each_pipe(dev_priv, pipe)
  4413.                 I915_WRITE(PIPESTAT(pipe),
  4414.                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
  4415.         I915_WRITE(IIR, I915_READ(IIR));
  4416. }
  4417.  
  4418. /**
  4419.  * intel_irq_init - initializes irq support
  4420.  * @dev_priv: i915 device instance
  4421.  *
  4422.  * This function initializes all the irq support including work items, timers
  4423.  * and all the vtables. It does not setup the interrupt itself though.
  4424.  */
  4425. void intel_irq_init(struct drm_i915_private *dev_priv)
  4426. {
  4427.         struct drm_device *dev = dev_priv->dev;
  4428.  
  4429.         intel_hpd_init_work(dev_priv);
  4430.  
  4431.         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
  4432.         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
  4433.  
  4434.         /* Let's track the enabled rps events */
  4435.         if (IS_VALLEYVIEW(dev_priv))
  4436.                 /* WaGsvRC0ResidencyMethod:vlv */
  4437.                 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
  4438.         else
  4439.                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
  4440.  
  4441.         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
  4442.                           i915_hangcheck_elapsed);
  4443.  
  4444.  
  4445.         if (IS_GEN2(dev_priv)) {
  4446.                 dev->max_vblank_count = 0;
  4447.                 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
  4448.         } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
  4449.                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
  4450.                 dev->driver->get_vblank_counter = g4x_get_vblank_counter;
  4451.         } else {
  4452.                 dev->driver->get_vblank_counter = i915_get_vblank_counter;
  4453.                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
  4454.         }
  4455.  
  4456.         /*
  4457.          * Opt out of the vblank disable timer on everything except gen2.
  4458.          * Gen2 doesn't have a hardware frame counter and so depends on
  4459.          * vblank interrupts to produce sane vblank seuquence numbers.
  4460.          */
  4461.         if (!IS_GEN2(dev_priv))
  4462.                 dev->vblank_disable_immediate = true;
  4463.  
  4464.         dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
  4465.         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
  4466.  
  4467.         if (IS_CHERRYVIEW(dev_priv)) {
  4468.                 dev->driver->irq_handler = cherryview_irq_handler;
  4469.                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
  4470.                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
  4471.                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
  4472.                 dev->driver->enable_vblank = valleyview_enable_vblank;
  4473.                 dev->driver->disable_vblank = valleyview_disable_vblank;
  4474.                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
  4475.         } else if (IS_VALLEYVIEW(dev_priv)) {
  4476.                 dev->driver->irq_handler = valleyview_irq_handler;
  4477.                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
  4478.                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
  4479.                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
  4480.                 dev->driver->enable_vblank = valleyview_enable_vblank;
  4481.                 dev->driver->disable_vblank = valleyview_disable_vblank;
  4482.                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
  4483.         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
  4484.                 dev->driver->irq_handler = gen8_irq_handler;
  4485.                 dev->driver->irq_preinstall = gen8_irq_reset;
  4486.                 dev->driver->irq_postinstall = gen8_irq_postinstall;
  4487.                 dev->driver->irq_uninstall = gen8_irq_uninstall;
  4488.                 dev->driver->enable_vblank = gen8_enable_vblank;
  4489.                 dev->driver->disable_vblank = gen8_disable_vblank;
  4490.                 if (IS_BROXTON(dev))
  4491.                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
  4492.                 else if (HAS_PCH_SPT(dev))
  4493.                         dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
  4494.                 else
  4495.                         dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
  4496.         } else if (HAS_PCH_SPLIT(dev)) {
  4497.                 dev->driver->irq_handler = ironlake_irq_handler;
  4498.                 dev->driver->irq_preinstall = ironlake_irq_reset;
  4499.                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
  4500.                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
  4501.                 dev->driver->enable_vblank = ironlake_enable_vblank;
  4502.                 dev->driver->disable_vblank = ironlake_disable_vblank;
  4503.                 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
  4504.         } else {
  4505.                 if (INTEL_INFO(dev_priv)->gen == 2) {
  4506.                 } else if (INTEL_INFO(dev_priv)->gen == 3) {
  4507.                         dev->driver->irq_preinstall = i915_irq_preinstall;
  4508.                         dev->driver->irq_postinstall = i915_irq_postinstall;
  4509.                         dev->driver->irq_uninstall = i915_irq_uninstall;
  4510.                         dev->driver->irq_handler = i915_irq_handler;
  4511.                 } else {
  4512.                         dev->driver->irq_preinstall = i965_irq_preinstall;
  4513.                         dev->driver->irq_postinstall = i965_irq_postinstall;
  4514.                         dev->driver->irq_uninstall = i965_irq_uninstall;
  4515.                         dev->driver->irq_handler = i965_irq_handler;
  4516.                 }
  4517.                 if (I915_HAS_HOTPLUG(dev_priv))
  4518.                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
  4519.                 dev->driver->enable_vblank = i915_enable_vblank;
  4520.                 dev->driver->disable_vblank = i915_disable_vblank;
  4521.         }
  4522. }
  4523.  
  4524. /**
  4525.  * intel_irq_install - enables the hardware interrupt
  4526.  * @dev_priv: i915 device instance
  4527.  *
  4528.  * This function enables the hardware interrupt handling, but leaves the hotplug
  4529.  * handling still disabled. It is called after intel_irq_init().
  4530.  *
  4531.  * In the driver load and resume code we need working interrupts in a few places
  4532.  * but don't want to deal with the hassle of concurrent probe and hotplug
  4533.  * workers. Hence the split into this two-stage approach.
  4534.  */
  4535. int intel_irq_install(struct drm_i915_private *dev_priv)
  4536. {
  4537.         /*
  4538.          * We enable some interrupt sources in our postinstall hooks, so mark
  4539.          * interrupts as enabled _before_ actually enabling them to avoid
  4540.          * special cases in our ordering checks.
  4541.          */
  4542.         dev_priv->pm.irqs_enabled = true;
  4543.  
  4544.         return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
  4545. }
  4546.  
  4547. /**
  4548.  * intel_irq_uninstall - finilizes all irq handling
  4549.  * @dev_priv: i915 device instance
  4550.  *
  4551.  * This stops interrupt and hotplug handling and unregisters and frees all
  4552.  * resources acquired in the init functions.
  4553.  */
  4554. void intel_irq_uninstall(struct drm_i915_private *dev_priv)
  4555. {
  4556. //      drm_irq_uninstall(dev_priv->dev);
  4557.         intel_hpd_cancel_work(dev_priv);
  4558.         dev_priv->pm.irqs_enabled = false;
  4559. }
  4560.  
  4561. /**
  4562.  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
  4563.  * @dev_priv: i915 device instance
  4564.  *
  4565.  * This function is used to disable interrupts at runtime, both in the runtime
  4566.  * pm and the system suspend/resume code.
  4567.  */
  4568. void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
  4569. {
  4570.         dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
  4571.         dev_priv->pm.irqs_enabled = false;
  4572. }
  4573.  
  4574. /**
  4575.  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
  4576.  * @dev_priv: i915 device instance
  4577.  *
  4578.  * This function is used to enable interrupts at runtime, both in the runtime
  4579.  * pm and the system suspend/resume code.
  4580.  */
  4581. void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
  4582. {
  4583.         dev_priv->pm.irqs_enabled = true;
  4584.         dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
  4585.         dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
  4586. }
  4587.