WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_irq.c

Rev	Author	Line No.	Line
2351	Serge	1	/* i915_irq.c -- IRQ support for the I915 -- linux-c --
		2	*/
		3	/*
		4	* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
		5	* All Rights Reserved.
		6	*
		7	* Permission is hereby granted, free of charge, to any person obtaining a
		8	* copy of this software and associated documentation files (the
		9	* "Software"), to deal in the Software without restriction, including
		10	* without limitation the rights to use, copy, modify, merge, publish,
		11	* distribute, sub license, and/or sell copies of the Software, and to
		12	* permit persons to whom the Software is furnished to do so, subject to
		13	* the following conditions:
		14	*
		15	* The above copyright notice and this permission notice (including the
		16	* next paragraph) shall be included in all copies or substantial portions
		17	* of the Software.
		18	*
		19	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
		20	* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
		21	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
		22	* IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
		23	* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
		24	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
		25	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
		26	*
		27	*/
		28
3746	Serge	29	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3031	serge	30
6103	serge	31	#include
3031	serge	32	#include
6088	serge	33	#include
3031	serge	34	#include
		35	#include
2351	Serge	36	#include "i915_drv.h"
		37	#include "i915_trace.h"
		38	#include "intel_drv.h"
		39
5354	serge	40	/**
		41	* DOC: interrupt handling
		42	*
		43	* These functions provide the basic support for enabling and disabling the
		44	* interrupt handling support. There's a lot more functionality in i915_irq.c
		45	* and related files, but that will be described in separate chapters.
		46	*/
4104	Serge	47
6084	serge	48	static const u32 hpd_ilk[HPD_NUM_PINS] = {
		49	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
		50	};
		51
		52	static const u32 hpd_ivb[HPD_NUM_PINS] = {
		53	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
		54	};
		55
		56	static const u32 hpd_bdw[HPD_NUM_PINS] = {
		57	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
		58	};
		59
		60	static const u32 hpd_ibx[HPD_NUM_PINS] = {
3746	Serge	61	[HPD_CRT] = SDE_CRT_HOTPLUG,
		62	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
		63	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
		64	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
		65	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
		66	};
3031	serge	67
6084	serge	68	static const u32 hpd_cpt[HPD_NUM_PINS] = {
3746	Serge	69	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
		70	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
		71	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
		72	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
		73	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
		74	};
		75
6084	serge	76	static const u32 hpd_spt[HPD_NUM_PINS] = {
		77	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
		78	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
		79	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
		80	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
		81	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
		82	};
		83
		84	static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
3746	Serge	85	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
		86	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
		87	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
		88	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
		89	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
		90	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
		91	};
		92
6084	serge	93	static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
3746	Serge	94	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
		95	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
		96	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
		97	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
		98	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
		99	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
		100	};
		101
6084	serge	102	static const u32 hpd_status_i915[HPD_NUM_PINS] = {
3746	Serge	103	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
		104	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
		105	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
		106	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
		107	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
		108	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
		109	};
		110
6084	serge	111	/* BXT hpd list */
		112	static const u32 hpd_bxt[HPD_NUM_PINS] = {
		113	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
		114	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
		115	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
		116	};
		117
5060	serge	118	/* IIR can theoretically queue up two events. Be paranoid. */
		119	#define GEN8_IRQ_RESET_NDX(type, which) do { \
		120	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
		121	POSTING_READ(GEN8_##type##_IMR(which)); \
		122	I915_WRITE(GEN8_##type##_IER(which), 0); \
		123	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
		124	POSTING_READ(GEN8_##type##_IIR(which)); \
		125	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
		126	POSTING_READ(GEN8_##type##_IIR(which)); \
		127	} while (0)
3746	Serge	128
5060	serge	129	#define GEN5_IRQ_RESET(type) do { \
		130	I915_WRITE(type##IMR, 0xffffffff); \
		131	POSTING_READ(type##IMR); \
		132	I915_WRITE(type##IER, 0); \
		133	I915_WRITE(type##IIR, 0xffffffff); \
		134	POSTING_READ(type##IIR); \
		135	I915_WRITE(type##IIR, 0xffffffff); \
		136	POSTING_READ(type##IIR); \
		137	} while (0)
		138
		139	/*
		140	* We should clear IMR at preinstall/uninstall, and just check at postinstall.
		141	*/
6084	serge	142	static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
		143	{
		144	u32 val = I915_READ(reg);
5060	serge	145
6084	serge	146	if (val == 0)
		147	return;
		148
		149	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
		150	reg, val);
		151	I915_WRITE(reg, 0xffffffff);
		152	POSTING_READ(reg);
		153	I915_WRITE(reg, 0xffffffff);
		154	POSTING_READ(reg);
		155	}
		156
5060	serge	157	#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
6084	serge	158	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
5354	serge	159	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
5060	serge	160	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
5354	serge	161	POSTING_READ(GEN8_##type##_IMR(which)); \
5060	serge	162	} while (0)
		163
		164	#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
6084	serge	165	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
5354	serge	166	I915_WRITE(type##IER, (ier_val)); \
5060	serge	167	I915_WRITE(type##IMR, (imr_val)); \
5354	serge	168	POSTING_READ(type##IMR); \
5060	serge	169	} while (0)
		170
5354	serge	171	static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
3031	serge	172
2351	Serge	173	/* For display hotplug interrupt */
6084	serge	174	static inline void
		175	i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
		176	uint32_t mask,
		177	uint32_t bits)
2351	Serge	178	{
6084	serge	179	uint32_t val;
		180
4104	Serge	181	assert_spin_locked(&dev_priv->irq_lock);
6084	serge	182	WARN_ON(bits & ~mask);
4104	Serge	183
6084	serge	184	val = I915_READ(PORT_HOTPLUG_EN);
		185	val &= ~mask;
		186	val \|= bits;
		187	I915_WRITE(PORT_HOTPLUG_EN, val);
		188	}
4104	Serge	189
6084	serge	190	/**
		191	* i915_hotplug_interrupt_update - update hotplug interrupt enable
		192	* @dev_priv: driver private
		193	* @mask: bits to update
		194	* @bits: bits to enable
		195	* NOTE: the HPD enable bits are modified both inside and outside
		196	* of an interrupt context. To avoid that read-modify-write cycles
		197	* interfer, these bits are protected by a spinlock. Since this
		198	* function is usually not called from a context where the lock is
		199	* held already, this function acquires the lock itself. A non-locking
		200	* version is also available.
		201	*/
		202	void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
		203	uint32_t mask,
		204	uint32_t bits)
		205	{
		206	spin_lock_irq(&dev_priv->irq_lock);
		207	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
		208	spin_unlock_irq(&dev_priv->irq_lock);
2351	Serge	209	}
		210
6084	serge	211	/**
		212	* ilk_update_display_irq - update DEIMR
		213	* @dev_priv: driver private
		214	* @interrupt_mask: mask of interrupt bits to update
		215	* @enabled_irq_mask: mask of interrupt bits to enable
		216	*/
		217	static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
		218	uint32_t interrupt_mask,
		219	uint32_t enabled_irq_mask)
2351	Serge	220	{
6084	serge	221	uint32_t new_val;
		222
4104	Serge	223	assert_spin_locked(&dev_priv->irq_lock);
		224
6084	serge	225	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		226
5354	serge	227	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	228	return;
		229
6084	serge	230	new_val = dev_priv->irq_mask;
		231	new_val &= ~interrupt_mask;
		232	new_val \|= (~enabled_irq_mask & interrupt_mask);
		233
		234	if (new_val != dev_priv->irq_mask) {
		235	dev_priv->irq_mask = new_val;
		236	I915_WRITE(DEIMR, dev_priv->irq_mask);
		237	POSTING_READ(DEIMR);
		238	}
2351	Serge	239	}
3031	serge	240
6084	serge	241	void
		242	ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
		243	{
		244	ilk_update_display_irq(dev_priv, mask, mask);
		245	}
		246
		247	void
		248	ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
		249	{
		250	ilk_update_display_irq(dev_priv, mask, 0);
		251	}
		252
4104	Serge	253	/**
		254	* ilk_update_gt_irq - update GTIMR
		255	* @dev_priv: driver private
		256	* @interrupt_mask: mask of interrupt bits to update
		257	* @enabled_irq_mask: mask of interrupt bits to enable
		258	*/
		259	static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
		260	uint32_t interrupt_mask,
		261	uint32_t enabled_irq_mask)
		262	{
		263	assert_spin_locked(&dev_priv->irq_lock);
		264
6084	serge	265	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		266
5060	serge	267	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	268	return;
		269
		270	dev_priv->gt_irq_mask &= ~interrupt_mask;
		271	dev_priv->gt_irq_mask \|= (~enabled_irq_mask & interrupt_mask);
		272	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		273	POSTING_READ(GTIMR);
		274	}
		275
5060	serge	276	void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	277	{
		278	ilk_update_gt_irq(dev_priv, mask, mask);
		279	}
		280
5060	serge	281	void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	282	{
		283	ilk_update_gt_irq(dev_priv, mask, 0);
		284	}
		285
5354	serge	286	static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
4104	Serge	287	{
5354	serge	288	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
4104	Serge	289	}
		290
5354	serge	291	static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
4104	Serge	292	{
5354	serge	293	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
4104	Serge	294	}
		295
5354	serge	296	static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
4104	Serge	297	{
5354	serge	298	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
4104	Serge	299	}
		300
5060	serge	301	/**
5354	serge	302	* snb_update_pm_irq - update GEN6_PMIMR
5060	serge	303	* @dev_priv: driver private
		304	* @interrupt_mask: mask of interrupt bits to update
		305	* @enabled_irq_mask: mask of interrupt bits to enable
		306	*/
5354	serge	307	static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
5060	serge	308	uint32_t interrupt_mask,
		309	uint32_t enabled_irq_mask)
		310	{
		311	uint32_t new_val;
		312
6084	serge	313	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		314
5060	serge	315	assert_spin_locked(&dev_priv->irq_lock);
		316
		317	new_val = dev_priv->pm_irq_mask;
		318	new_val &= ~interrupt_mask;
		319	new_val \|= (~enabled_irq_mask & interrupt_mask);
		320
		321	if (new_val != dev_priv->pm_irq_mask) {
		322	dev_priv->pm_irq_mask = new_val;
5354	serge	323	I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		324	POSTING_READ(gen6_pm_imr(dev_priv));
5060	serge	325	}
		326	}
		327
5354	serge	328	void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
5060	serge	329	{
5354	serge	330	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		331	return;
		332
		333	snb_update_pm_irq(dev_priv, mask, mask);
5060	serge	334	}
		335
5354	serge	336	static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
		337	uint32_t mask)
5060	serge	338	{
5354	serge	339	snb_update_pm_irq(dev_priv, mask, 0);
5060	serge	340	}
		341
5354	serge	342	void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	343	{
5354	serge	344	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		345	return;
4104	Serge	346
5354	serge	347	__gen6_disable_pm_irq(dev_priv, mask);
4104	Serge	348	}
		349
5354	serge	350	void gen6_reset_rps_interrupts(struct drm_device *dev)
5060	serge	351	{
		352	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	353	uint32_t reg = gen6_pm_iir(dev_priv);
5060	serge	354
5354	serge	355	spin_lock_irq(&dev_priv->irq_lock);
		356	I915_WRITE(reg, dev_priv->pm_rps_events);
		357	I915_WRITE(reg, dev_priv->pm_rps_events);
6084	serge	358	POSTING_READ(reg);
		359	dev_priv->rps.pm_iir = 0;
5354	serge	360	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	361	}
		362
5354	serge	363	void gen6_enable_rps_interrupts(struct drm_device *dev)
5060	serge	364	{
		365	struct drm_i915_private *dev_priv = dev->dev_private;
		366
5354	serge	367	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	368
5354	serge	369	WARN_ON(dev_priv->rps.pm_iir);
		370	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
		371	dev_priv->rps.interrupts_enabled = true;
		372	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) \|
		373	dev_priv->pm_rps_events);
		374	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		375
		376	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	377	}
		378
6084	serge	379	u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
		380	{
		381	/*
		382	* SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
		383	* if GEN6_PM_UP_EI_EXPIRED is masked.
		384	*
		385	* TODO: verify if this can be reproduced on VLV,CHV.
		386	*/
		387	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		388	mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
		389
		390	if (INTEL_INFO(dev_priv)->gen >= 8)
		391	mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
		392
		393	return mask;
		394	}
		395
5354	serge	396	void gen6_disable_rps_interrupts(struct drm_device *dev)
4104	Serge	397	{
		398	struct drm_i915_private *dev_priv = dev->dev_private;
		399
5354	serge	400	spin_lock_irq(&dev_priv->irq_lock);
		401	dev_priv->rps.interrupts_enabled = false;
		402	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	403
5354	serge	404	cancel_work_sync(&dev_priv->rps.work);
4104	Serge	405
5354	serge	406	spin_lock_irq(&dev_priv->irq_lock);
4104	Serge	407
6084	serge	408	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
4104	Serge	409
5354	serge	410	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		411	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
		412	~dev_priv->pm_rps_events);
4104	Serge	413
6084	serge	414	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	415
		416	}
		417
4104	Serge	418	/**
6084	serge	419	* bdw_update_port_irq - update DE port interrupt
		420	* @dev_priv: driver private
		421	* @interrupt_mask: mask of interrupt bits to update
		422	* @enabled_irq_mask: mask of interrupt bits to enable
		423	*/
		424	static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
		425	uint32_t interrupt_mask,
		426	uint32_t enabled_irq_mask)
		427	{
		428	uint32_t new_val;
		429	uint32_t old_val;
		430
		431	assert_spin_locked(&dev_priv->irq_lock);
		432
		433	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		434
		435	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		436	return;
		437
		438	old_val = I915_READ(GEN8_DE_PORT_IMR);
		439
		440	new_val = old_val;
		441	new_val &= ~interrupt_mask;
		442	new_val \|= (~enabled_irq_mask & interrupt_mask);
		443
		444	if (new_val != old_val) {
		445	I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		446	POSTING_READ(GEN8_DE_PORT_IMR);
		447	}
		448	}
		449
		450	/**
4104	Serge	451	* ibx_display_interrupt_update - update SDEIMR
		452	* @dev_priv: driver private
		453	* @interrupt_mask: mask of interrupt bits to update
		454	* @enabled_irq_mask: mask of interrupt bits to enable
		455	*/
5354	serge	456	void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
6084	serge	457	uint32_t interrupt_mask,
		458	uint32_t enabled_irq_mask)
4104	Serge	459	{
		460	uint32_t sdeimr = I915_READ(SDEIMR);
		461	sdeimr &= ~interrupt_mask;
		462	sdeimr \|= (~enabled_irq_mask & interrupt_mask);
		463
6084	serge	464	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		465
4104	Serge	466	assert_spin_locked(&dev_priv->irq_lock);
		467
5060	serge	468	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	469	return;
		470
		471	I915_WRITE(SDEIMR, sdeimr);
		472	POSTING_READ(SDEIMR);
		473	}
		474
5060	serge	475	static void
		476	__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		477	u32 enable_mask, u32 status_mask)
3031	serge	478	{
6084	serge	479	u32 reg = PIPESTAT(pipe);
5060	serge	480	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
3031	serge	481
4104	Serge	482	assert_spin_locked(&dev_priv->irq_lock);
5354	serge	483	WARN_ON(!intel_irqs_enabled(dev_priv));
4104	Serge	484
5060	serge	485	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK \|\|
		486	status_mask & ~PIPESTAT_INT_STATUS_MASK,
		487	"pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		488	pipe_name(pipe), enable_mask, status_mask))
3746	Serge	489	return;
		490
5060	serge	491	if ((pipestat & enable_mask) == enable_mask)
		492	return;
		493
		494	dev_priv->pipestat_irq_mask[pipe] \|= status_mask;
		495
6084	serge	496	/* Enable the interrupt, clear any pending status */
5060	serge	497	pipestat \|= enable_mask \| status_mask;
3746	Serge	498	I915_WRITE(reg, pipestat);
6084	serge	499	POSTING_READ(reg);
3031	serge	500	}
		501
5060	serge	502	static void
		503	__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		504	u32 enable_mask, u32 status_mask)
3031	serge	505	{
6084	serge	506	u32 reg = PIPESTAT(pipe);
5060	serge	507	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
3031	serge	508
4104	Serge	509	assert_spin_locked(&dev_priv->irq_lock);
5354	serge	510	WARN_ON(!intel_irqs_enabled(dev_priv));
4104	Serge	511
5060	serge	512	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK \|\|
		513	status_mask & ~PIPESTAT_INT_STATUS_MASK,
		514	"pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		515	pipe_name(pipe), enable_mask, status_mask))
3746	Serge	516	return;
		517
5060	serge	518	if ((pipestat & enable_mask) == 0)
		519	return;
		520
		521	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
		522
		523	pipestat &= ~enable_mask;
3746	Serge	524	I915_WRITE(reg, pipestat);
6084	serge	525	POSTING_READ(reg);
3031	serge	526	}
		527
5060	serge	528	static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
		529	{
		530	u32 enable_mask = status_mask << 16;
		531
		532	/*
		533	* On pipe A we don't support the PSR interrupt yet,
		534	* on pipe B and C the same bit MBZ.
		535	*/
		536	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		537	return 0;
		538	/*
		539	* On pipe B and C we don't support the PSR interrupt yet, on pipe
		540	* A the same bit is for perf counters which we don't use either.
		541	*/
		542	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		543	return 0;
		544
		545	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS \|
		546	SPRITE0_FLIP_DONE_INT_EN_VLV \|
		547	SPRITE1_FLIP_DONE_INT_EN_VLV);
		548	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		549	enable_mask \|= SPRITE0_FLIP_DONE_INT_EN_VLV;
		550	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		551	enable_mask \|= SPRITE1_FLIP_DONE_INT_EN_VLV;
		552
		553	return enable_mask;
		554	}
		555
		556	void
		557	i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		558	u32 status_mask)
		559	{
		560	u32 enable_mask;
		561
		562	if (IS_VALLEYVIEW(dev_priv->dev))
		563	enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
		564	status_mask);
		565	else
		566	enable_mask = status_mask << 16;
		567	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
		568	}
		569
		570	void
		571	i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		572	u32 status_mask)
		573	{
		574	u32 enable_mask;
		575
		576	if (IS_VALLEYVIEW(dev_priv->dev))
		577	enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
		578	status_mask);
		579	else
		580	enable_mask = status_mask << 16;
		581	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
		582	}
		583
3031	serge	584	/**
4104	Serge	585	* i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
6084	serge	586	* @dev: drm device
3031	serge	587	*/
4104	Serge	588	static void i915_enable_asle_pipestat(struct drm_device *dev)
3031	serge	589	{
5060	serge	590	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	591
4104	Serge	592	if (!dev_priv->opregion.asle \|\| !IS_MOBILE(dev))
3031	serge	593	return;
		594
5354	serge	595	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	596
5060	serge	597	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
6084	serge	598	if (INTEL_INFO(dev)->gen >= 4)
4560	Serge	599	i915_enable_pipestat(dev_priv, PIPE_A,
5060	serge	600	PIPE_LEGACY_BLC_EVENT_STATUS);
3031	serge	601
5354	serge	602	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	603	}
		604
5060	serge	605	/*
		606	* This timing diagram depicts the video signal in and
		607	* around the vertical blanking period.
		608	*
		609	* Assumptions about the fictitious mode used in this example:
		610	* vblank_start >= 3
		611	* vsync_start = vblank_start + 1
		612	* vsync_end = vblank_start + 2
		613	* vtotal = vblank_start + 3
		614	*
		615	* start of vblank:
		616	* latch double buffered registers
		617	* increment frame counter (ctg+)
		618	* generate start of vblank interrupt (gen4+)
		619	* \|
		620	* \| frame start:
		621	* \| generate frame start interrupt (aka. vblank interrupt) (gmch)
		622	* \| may be shifted forward 1-3 extra lines via PIPECONF
		623	* \| \|
		624	* \| \| start of vsync:
		625	* \| \| generate vsync interrupt
		626	* \| \| \|
		627	* ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
		628	* . \hs/ . \hs/ \hs/ \hs/ . \hs/
		629	* ----va---> <-----------------vb--------------------> <--------va-------------
		630	* \| \| <----vs-----> \|
		631	* -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
		632	* -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
		633	* -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
		634	* \| \| \|
		635	* last visible pixel first visible pixel
		636	* \| increment frame counter (gen3/4)
		637	* pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
		638	*
		639	* x = horizontal active
		640	* _ = horizontal blanking
		641	* hs = horizontal sync
		642	* va = vertical active
		643	* vb = vertical blanking
		644	* vs = vertical sync
		645	* vbs = vblank_start (number)
		646	*
		647	* Summary:
		648	* - most events happen at the start of horizontal sync
		649	* - frame start happens at the start of horizontal blank, 1-4 lines
		650	* (depending on PIPECONF settings) after the start of vblank
		651	* - gen3/4 pixel and frame counter are synchronized with the start
		652	* of horizontal active on the first line of vertical active
		653	*/
		654
6084	serge	655	static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
4560	Serge	656	{
		657	/* Gen2 doesn't have a hardware frame counter */
		658	return 0;
		659	}
		660
3031	serge	661	/* Called from drm generic code, passed a 'crtc', which
		662	* we use as a pipe index
		663	*/
6084	serge	664	static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
3031	serge	665	{
5060	serge	666	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	667	unsigned long high_frame;
		668	unsigned long low_frame;
5060	serge	669	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
6084	serge	670	struct intel_crtc *intel_crtc =
		671	to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
		672	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
3031	serge	673
6084	serge	674	htotal = mode->crtc_htotal;
		675	hsync_start = mode->crtc_hsync_start;
		676	vbl_start = mode->crtc_vblank_start;
		677	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		678	vbl_start = DIV_ROUND_UP(vbl_start, 2);
3031	serge	679
5060	serge	680	/* Convert to pixel count */
6084	serge	681	vbl_start *= htotal;
4560	Serge	682
5060	serge	683	/* Start of vblank event occurs at start of hsync */
		684	vbl_start -= htotal - hsync_start;
		685
3031	serge	686	high_frame = PIPEFRAME(pipe);
		687	low_frame = PIPEFRAMEPIXEL(pipe);
		688
		689	/*
		690	* High & low register fields aren't synchronized, so make sure
		691	* we get a low value that's stable across two reads of the high
		692	* register.
		693	*/
		694	do {
		695	high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
4560	Serge	696	low = I915_READ(low_frame);
3031	serge	697	high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
		698	} while (high1 != high2);
		699
		700	high1 >>= PIPE_FRAME_HIGH_SHIFT;
4560	Serge	701	pixel = low & PIPE_PIXEL_MASK;
3031	serge	702	low >>= PIPE_FRAME_LOW_SHIFT;
4560	Serge	703
		704	/*
		705	* The frame counter increments at beginning of active.
		706	* Cook up a vblank counter by also checking the pixel
		707	* counter against vblank start.
		708	*/
		709	return (((high1 << 8) \| low) + (pixel >= vbl_start)) & 0xffffff;
3031	serge	710	}
		711
6084	serge	712	static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
3031	serge	713	{
5060	serge	714	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	715
6084	serge	716	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
3031	serge	717	}
		718
4560	Serge	719	/* raw reads, only for fast reads of display block, no need for forcewake etc. */
		720	#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
		721
5060	serge	722	static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
4560	Serge	723	{
5060	serge	724	struct drm_device *dev = crtc->base.dev;
4560	Serge	725	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	726	const struct drm_display_mode *mode = &crtc->base.hwmode;
5060	serge	727	enum pipe pipe = crtc->pipe;
		728	int position, vtotal;
4560	Serge	729
5060	serge	730	vtotal = mode->crtc_vtotal;
		731	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		732	vtotal /= 2;
4560	Serge	733
5060	serge	734	if (IS_GEN2(dev))
		735	position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
		736	else
		737	position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
		738
		739	/*
6084	serge	740	* On HSW, the DSL reg (0x70000) appears to return 0 if we
		741	* read it just before the start of vblank. So try it again
		742	* so we don't accidentally end up spanning a vblank frame
		743	* increment, causing the pipe_update_end() code to squak at us.
		744	*
		745	* The nature of this problem means we can't simply check the ISR
		746	* bit and return the vblank start value; nor can we use the scanline
		747	* debug register in the transcoder as it appears to have the same
		748	* problem. We may need to extend this to include other platforms,
		749	* but so far testing only shows the problem on HSW.
		750	*/
		751	if (HAS_DDI(dev) && !position) {
		752	int i, temp;
		753
		754	for (i = 0; i < 100; i++) {
		755	udelay(1);
		756	temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
		757	DSL_LINEMASK_GEN3;
		758	if (temp != position) {
		759	position = temp;
		760	break;
		761	}
		762	}
		763	}
		764
		765	/*
5060	serge	766	* See update_scanline_offset() for the details on the
		767	* scanline_offset adjustment.
		768	*/
		769	return (position + crtc->scanline_offset) % vtotal;
4560	Serge	770	}
		771
6084	serge	772	static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
4560	Serge	773	unsigned int flags, int vpos, int hpos,
6084	serge	774	ktime_t stime, ktime_t etime,
		775	const struct drm_display_mode *mode)
3746	Serge	776	{
4560	Serge	777	struct drm_i915_private *dev_priv = dev->dev_private;
		778	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		779	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		780	int position;
5060	serge	781	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
3746	Serge	782	bool in_vbl = true;
		783	int ret = 0;
4560	Serge	784	unsigned long irqflags;
3746	Serge	785
6084	serge	786	if (WARN_ON(!mode->crtc_clock)) {
3746	Serge	787	DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
		788	"pipe %c\n", pipe_name(pipe));
		789	return 0;
		790	}
		791
4560	Serge	792	htotal = mode->crtc_htotal;
5060	serge	793	hsync_start = mode->crtc_hsync_start;
4560	Serge	794	vtotal = mode->crtc_vtotal;
		795	vbl_start = mode->crtc_vblank_start;
		796	vbl_end = mode->crtc_vblank_end;
3746	Serge	797
4560	Serge	798	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		799	vbl_start = DIV_ROUND_UP(vbl_start, 2);
		800	vbl_end /= 2;
		801	vtotal /= 2;
		802	}
		803
		804	ret \|= DRM_SCANOUTPOS_VALID \| DRM_SCANOUTPOS_ACCURATE;
		805
		806	/*
		807	* Lock uncore.lock, as we will do multiple timing critical raw
		808	* register reads, potentially with preemption disabled, so the
		809	* following code must not block on uncore.lock.
		810	*/
		811	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
		812
		813	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
		814
6084	serge	815	/* Get optional system timestamp before query. */
		816	if (stime)
		817	*stime = ktime_get();
4560	Serge	818
		819	if (IS_GEN2(dev) \|\| IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
3746	Serge	820	/* No obvious pixelcount register. Only query vertical
		821	* scanout position from Display scan line register.
		822	*/
5060	serge	823	position = __intel_get_crtc_scanline(intel_crtc);
3746	Serge	824	} else {
		825	/* Have access to pixelcount since start of frame.
		826	* We can split this into vertical and horizontal
		827	* scanout position.
		828	*/
4560	Serge	829	position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
3746	Serge	830
4560	Serge	831	/* convert to pixel counts */
		832	vbl_start *= htotal;
		833	vbl_end *= htotal;
		834	vtotal *= htotal;
5060	serge	835
		836	/*
		837	* In interlaced modes, the pixel counter counts all pixels,
		838	* so one field will have htotal more pixels. In order to avoid
		839	* the reported position from jumping backwards when the pixel
		840	* counter is beyond the length of the shorter field, just
		841	* clamp the position the length of the shorter field. This
		842	* matches how the scanline counter based position works since
		843	* the scanline counter doesn't count the two half lines.
		844	*/
		845	if (position >= vtotal)
		846	position = vtotal - 1;
		847
		848	/*
		849	* Start of vblank interrupt is triggered at start of hsync,
		850	* just prior to the first active line of vblank. However we
		851	* consider lines to start at the leading edge of horizontal
		852	* active. So, should we get here before we've crossed into
		853	* the horizontal active of the first line in vblank, we would
		854	* not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		855	* always add htotal-hsync_start to the current pixel position.
		856	*/
		857	position = (position + htotal - hsync_start) % vtotal;
3746	Serge	858	}
		859
6084	serge	860	/* Get optional system timestamp after query. */
		861	if (etime)
		862	*etime = ktime_get();
3746	Serge	863
4560	Serge	864	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
3746	Serge	865
4560	Serge	866	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3746	Serge	867
4560	Serge	868	in_vbl = position >= vbl_start && position < vbl_end;
3746	Serge	869
4560	Serge	870	/*
		871	* While in vblank, position will be negative
		872	* counting up towards 0 at vbl_end. And outside
		873	* vblank, position will be positive counting
		874	* up since vbl_end.
		875	*/
		876	if (position >= vbl_start)
		877	position -= vbl_end;
		878	else
		879	position += vtotal - vbl_end;
3746	Serge	880
4560	Serge	881	if (IS_GEN2(dev) \|\| IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		882	*vpos = position;
		883	*hpos = 0;
		884	} else {
		885	*vpos = position / htotal;
		886	hpos = position - (vpos * htotal);
		887	}
		888
3746	Serge	889	/* In vblank? */
		890	if (in_vbl)
5354	serge	891	ret \|= DRM_SCANOUTPOS_IN_VBLANK;
3746	Serge	892
		893	return ret;
		894	}
		895
5060	serge	896	int intel_get_crtc_scanline(struct intel_crtc *crtc)
		897	{
		898	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
		899	unsigned long irqflags;
		900	int position;
		901
		902	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
		903	position = __intel_get_crtc_scanline(crtc);
		904	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
		905
		906	return position;
		907	}
		908
6084	serge	909	static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
3746	Serge	910	int *max_error,
		911	struct timeval *vblank_time,
		912	unsigned flags)
		913	{
		914	struct drm_crtc *crtc;
		915
6084	serge	916	if (pipe >= INTEL_INFO(dev)->num_pipes) {
		917	DRM_ERROR("Invalid crtc %u\n", pipe);
3746	Serge	918	return -EINVAL;
		919	}
		920
		921	/* Get drm_crtc to timestamp: */
		922	crtc = intel_get_crtc_for_pipe(dev, pipe);
		923	if (crtc == NULL) {
6084	serge	924	DRM_ERROR("Invalid crtc %u\n", pipe);
3746	Serge	925	return -EINVAL;
		926	}
		927
6084	serge	928	if (!crtc->hwmode.crtc_clock) {
		929	DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
3746	Serge	930	return -EBUSY;
		931	}
		932
		933	/* Helper routine in DRM core does all the work: */
		934	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
		935	vblank_time, flags,
6084	serge	936	&crtc->hwmode);
3746	Serge	937	}
		938
4104	Serge	939	static void ironlake_rps_change_irq_handler(struct drm_device *dev)
3746	Serge	940	{
5060	serge	941	struct drm_i915_private *dev_priv = dev->dev_private;
3746	Serge	942	u32 busy_up, busy_down, max_avg, min_avg;
		943	u8 new_delay;
		944
4104	Serge	945	spin_lock(&mchdev_lock);
3746	Serge	946
		947	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
		948
		949	new_delay = dev_priv->ips.cur_delay;
		950
		951	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
		952	busy_up = I915_READ(RCPREVBSYTUPAVG);
		953	busy_down = I915_READ(RCPREVBSYTDNAVG);
		954	max_avg = I915_READ(RCBMAXAVG);
		955	min_avg = I915_READ(RCBMINAVG);
		956
		957	/* Handle RCS change request from hw */
		958	if (busy_up > max_avg) {
		959	if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
		960	new_delay = dev_priv->ips.cur_delay - 1;
		961	if (new_delay < dev_priv->ips.max_delay)
		962	new_delay = dev_priv->ips.max_delay;
		963	} else if (busy_down < min_avg) {
		964	if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
		965	new_delay = dev_priv->ips.cur_delay + 1;
		966	if (new_delay > dev_priv->ips.min_delay)
		967	new_delay = dev_priv->ips.min_delay;
		968	}
		969
		970	if (ironlake_set_drps(dev, new_delay))
		971	dev_priv->ips.cur_delay = new_delay;
		972
4104	Serge	973	spin_unlock(&mchdev_lock);
3746	Serge	974
		975	return;
		976	}
		977
6084	serge	978	static void notify_ring(struct intel_engine_cs *ring)
2352	Serge	979	{
5060	serge	980	if (!intel_ring_initialized(ring))
2352	Serge	981	return;
2351	Serge	982
6084	serge	983	trace_i915_gem_request_notify(ring);
2351	Serge	984
2352	Serge	985	wake_up_all(&ring->irq_queue);
		986	}
		987
6084	serge	988	static void vlv_c0_read(struct drm_i915_private *dev_priv,
		989	struct intel_rps_ei *ei)
5060	serge	990	{
6084	serge	991	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
		992	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
		993	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
		994	}
5060	serge	995
6084	serge	996	void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
5060	serge	997	{
6935	serge	998	memset(&dev_priv->rps.ei, 0, sizeof(dev_priv->rps.ei));
6084	serge	999	}
5060	serge	1000
6084	serge	1001	static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
		1002	{
6935	serge	1003	const struct intel_rps_ei *prev = &dev_priv->rps.ei;
6084	serge	1004	struct intel_rps_ei now;
		1005	u32 events = 0;
5060	serge	1006
6935	serge	1007	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
6084	serge	1008	return 0;
5060	serge	1009
6084	serge	1010	vlv_c0_read(dev_priv, &now);
		1011	if (now.cz_clock == 0)
		1012	return 0;
5060	serge	1013
6935	serge	1014	if (prev->cz_clock) {
		1015	u64 time, c0;
		1016	unsigned int mul;
5060	serge	1017
6935	serge	1018	mul = VLV_CZ_CLOCK_TO_MILLI_SEC * 100; /* scale to threshold% */
		1019	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		1020	mul <<= 8;
		1021
		1022	time = now.cz_clock - prev->cz_clock;
		1023	time *= dev_priv->czclk_freq;
		1024
		1025	/* Workload can be split between render + media,
		1026	* e.g. SwapBuffers being blitted in X after being rendered in
		1027	* mesa. To account for this we need to combine both engines
		1028	* into our activity counter.
		1029	*/
		1030	c0 = now.render_c0 - prev->render_c0;
		1031	c0 += now.media_c0 - prev->media_c0;
		1032	c0 *= mul;
		1033
		1034	if (c0 > time * dev_priv->rps.up_threshold)
		1035	events = GEN6_PM_RP_UP_THRESHOLD;
		1036	else if (c0 < time * dev_priv->rps.down_threshold)
		1037	events = GEN6_PM_RP_DOWN_THRESHOLD;
5060	serge	1038	}
		1039
6935	serge	1040	dev_priv->rps.ei = now;
6084	serge	1041	return events;
		1042	}
5060	serge	1043
6084	serge	1044	static bool any_waiters(struct drm_i915_private *dev_priv)
		1045	{
		1046	struct intel_engine_cs *ring;
		1047	int i;
5060	serge	1048
6084	serge	1049	for_each_ring(ring, dev_priv, i)
		1050	if (ring->irq_refcount)
		1051	return true;
5060	serge	1052
6084	serge	1053	return false;
5060	serge	1054	}
		1055
3031	serge	1056	static void gen6_pm_rps_work(struct work_struct *work)
		1057	{
5060	serge	1058	struct drm_i915_private *dev_priv =
		1059	container_of(work, struct drm_i915_private, rps.work);
6084	serge	1060	bool client_boost;
		1061	int new_delay, adj, min, max;
4104	Serge	1062	u32 pm_iir;
2352	Serge	1063
4104	Serge	1064	spin_lock_irq(&dev_priv->irq_lock);
5354	serge	1065	/* Speed up work cancelation during disabling rps interrupts. */
		1066	if (!dev_priv->rps.interrupts_enabled) {
		1067	spin_unlock_irq(&dev_priv->irq_lock);
		1068	return;
		1069	}
3031	serge	1070	pm_iir = dev_priv->rps.pm_iir;
		1071	dev_priv->rps.pm_iir = 0;
5354	serge	1072	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
6084	serge	1073	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		1074	client_boost = dev_priv->rps.client_boost;
		1075	dev_priv->rps.client_boost = false;
4104	Serge	1076	spin_unlock_irq(&dev_priv->irq_lock);
2352	Serge	1077
4104	Serge	1078	/* Make sure we didn't queue anything we're not going to process. */
5060	serge	1079	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
4104	Serge	1080
6084	serge	1081	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
3031	serge	1082	return;
		1083
3243	Serge	1084	mutex_lock(&dev_priv->rps.hw_lock);
3031	serge	1085
6084	serge	1086	pm_iir \|= vlv_wa_c0_ei(dev_priv, pm_iir);
		1087
4560	Serge	1088	adj = dev_priv->rps.last_adj;
6084	serge	1089	new_delay = dev_priv->rps.cur_freq;
		1090	min = dev_priv->rps.min_freq_softlimit;
		1091	max = dev_priv->rps.max_freq_softlimit;
		1092
		1093	if (client_boost) {
		1094	new_delay = dev_priv->rps.max_freq_softlimit;
		1095	adj = 0;
		1096	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
4560	Serge	1097	if (adj > 0)
		1098	adj *= 2;
6084	serge	1099	else /* CHV needs even encode values */
		1100	adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
4104	Serge	1101	/*
		1102	* For better performance, jump directly
		1103	* to RPe if we're below it.
		1104	*/
6084	serge	1105	if (new_delay < dev_priv->rps.efficient_freq - adj) {
5060	serge	1106	new_delay = dev_priv->rps.efficient_freq;
6084	serge	1107	adj = 0;
		1108	}
		1109	} else if (any_waiters(dev_priv)) {
		1110	adj = 0;
4560	Serge	1111	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
5060	serge	1112	if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
		1113	new_delay = dev_priv->rps.efficient_freq;
4560	Serge	1114	else
5060	serge	1115	new_delay = dev_priv->rps.min_freq_softlimit;
4560	Serge	1116	adj = 0;
		1117	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		1118	if (adj < 0)
		1119	adj *= 2;
6084	serge	1120	else /* CHV needs even encode values */
		1121	adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
4560	Serge	1122	} else { /* unknown event */
6084	serge	1123	adj = 0;
4560	Serge	1124	}
3031	serge	1125
6084	serge	1126	dev_priv->rps.last_adj = adj;
		1127
3031	serge	1128	/* sysfs frequency interfaces may have snuck in while servicing the
		1129	* interrupt
		1130	*/
6084	serge	1131	new_delay += adj;
		1132	new_delay = clamp_t(int, new_delay, min, max);
4560	Serge	1133
6084	serge	1134	intel_set_rps(dev_priv->dev, new_delay);
5060	serge	1135
3243	Serge	1136	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	1137	}
		1138
		1139
		1140	/**
		1141	* ivybridge_parity_work - Workqueue called when a parity error interrupt
		1142	* occurred.
		1143	* @work: workqueue struct
		1144	*
		1145	* Doesn't actually do anything except notify userspace. As a consequence of
		1146	* this event, userspace should try to remap the bad rows since statistically
		1147	* it is likely the same row is more likely to go bad again.
		1148	*/
		1149	static void ivybridge_parity_work(struct work_struct *work)
2351	Serge	1150	{
5060	serge	1151	struct drm_i915_private *dev_priv =
		1152	container_of(work, struct drm_i915_private, l3_parity.error_work);
3031	serge	1153	u32 error_status, row, bank, subbank;
4560	Serge	1154	char *parity_event[6];
3031	serge	1155	uint32_t misccpctl;
4560	Serge	1156	uint8_t slice = 0;
3031	serge	1157
		1158	/* We must turn off DOP level clock gating to access the L3 registers.
		1159	* In order to prevent a get/put style interface, acquire struct mutex
		1160	* any time we access those registers.
		1161	*/
		1162	mutex_lock(&dev_priv->dev->struct_mutex);
		1163
4560	Serge	1164	/* If we've screwed up tracking, just let the interrupt fire again */
		1165	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		1166	goto out;
		1167
3031	serge	1168	misccpctl = I915_READ(GEN7_MISCCPCTL);
		1169	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		1170	POSTING_READ(GEN7_MISCCPCTL);
		1171
4560	Serge	1172	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		1173	u32 reg;
		1174
		1175	slice--;
		1176	if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
		1177	break;
		1178
		1179	dev_priv->l3_parity.which_slice &= ~(1<
		1180
		1181	reg = GEN7_L3CDERRST1 + (slice * 0x200);
		1182
		1183	error_status = I915_READ(reg);
6084	serge	1184	row = GEN7_PARITY_ERROR_ROW(error_status);
		1185	bank = GEN7_PARITY_ERROR_BANK(error_status);
		1186	subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
3031	serge	1187
4560	Serge	1188	I915_WRITE(reg, GEN7_PARITY_ERROR_VALID \| GEN7_L3CDERRST1_ENABLE);
		1189	POSTING_READ(reg);
3031	serge	1190
4560	Serge	1191	DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
		1192	slice, row, bank, subbank);
		1193
		1194	}
		1195
3031	serge	1196	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		1197
4560	Serge	1198	out:
		1199	WARN_ON(dev_priv->l3_parity.which_slice);
5354	serge	1200	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	1201	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
5354	serge	1202	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	1203
		1204	mutex_unlock(&dev_priv->dev->struct_mutex);
		1205	}
		1206
4560	Serge	1207	static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
3031	serge	1208	{
5060	serge	1209	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1210
4560	Serge	1211	if (!HAS_L3_DPF(dev))
3031	serge	1212	return;
		1213
4104	Serge	1214	spin_lock(&dev_priv->irq_lock);
5060	serge	1215	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
4104	Serge	1216	spin_unlock(&dev_priv->irq_lock);
3031	serge	1217
4560	Serge	1218	iir &= GT_PARITY_ERROR(dev);
		1219	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		1220	dev_priv->l3_parity.which_slice \|= 1 << 1;
		1221
		1222	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		1223	dev_priv->l3_parity.which_slice \|= 1 << 0;
		1224
3243	Serge	1225	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
3031	serge	1226	}
		1227
4104	Serge	1228	static void ilk_gt_irq_handler(struct drm_device *dev,
		1229	struct drm_i915_private *dev_priv,
		1230	u32 gt_iir)
		1231	{
		1232	if (gt_iir &
		1233	(GT_RENDER_USER_INTERRUPT \| GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
6084	serge	1234	notify_ring(&dev_priv->ring[RCS]);
4104	Serge	1235	if (gt_iir & ILK_BSD_USER_INTERRUPT)
6084	serge	1236	notify_ring(&dev_priv->ring[VCS]);
4104	Serge	1237	}
		1238
3031	serge	1239	static void snb_gt_irq_handler(struct drm_device *dev,
		1240	struct drm_i915_private *dev_priv,
		1241	u32 gt_iir)
		1242	{
		1243
4104	Serge	1244	if (gt_iir &
		1245	(GT_RENDER_USER_INTERRUPT \| GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
6084	serge	1246	notify_ring(&dev_priv->ring[RCS]);
4104	Serge	1247	if (gt_iir & GT_BSD_USER_INTERRUPT)
6084	serge	1248	notify_ring(&dev_priv->ring[VCS]);
4104	Serge	1249	if (gt_iir & GT_BLT_USER_INTERRUPT)
6084	serge	1250	notify_ring(&dev_priv->ring[BCS]);
3031	serge	1251
4104	Serge	1252	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT \|
		1253	GT_BSD_CS_ERROR_INTERRUPT \|
5354	serge	1254	GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		1255	DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
3031	serge	1256
4560	Serge	1257	if (gt_iir & GT_PARITY_ERROR(dev))
		1258	ivybridge_parity_error_irq_handler(dev, gt_iir);
3031	serge	1259	}
		1260
6084	serge	1261	static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
4560	Serge	1262	u32 master_ctl)
		1263	{
		1264	irqreturn_t ret = IRQ_NONE;
		1265
		1266	if (master_ctl & (GEN8_GT_RCS_IRQ \| GEN8_GT_BCS_IRQ)) {
6084	serge	1267	u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
4560	Serge	1268	if (tmp) {
6084	serge	1269	I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
4560	Serge	1270	ret = IRQ_HANDLED;
5354	serge	1271
6084	serge	1272	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
		1273	intel_lrc_irq_handler(&dev_priv->ring[RCS]);
		1274	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
		1275	notify_ring(&dev_priv->ring[RCS]);
5354	serge	1276
6084	serge	1277	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
		1278	intel_lrc_irq_handler(&dev_priv->ring[BCS]);
		1279	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
		1280	notify_ring(&dev_priv->ring[BCS]);
4560	Serge	1281	} else
		1282	DRM_ERROR("The master control interrupt lied (GT0)!\n");
		1283	}
		1284
5060	serge	1285	if (master_ctl & (GEN8_GT_VCS1_IRQ \| GEN8_GT_VCS2_IRQ)) {
6084	serge	1286	u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
4560	Serge	1287	if (tmp) {
6084	serge	1288	I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
4560	Serge	1289	ret = IRQ_HANDLED;
5354	serge	1290
6084	serge	1291	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
		1292	intel_lrc_irq_handler(&dev_priv->ring[VCS]);
		1293	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
		1294	notify_ring(&dev_priv->ring[VCS]);
5354	serge	1295
6084	serge	1296	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
		1297	intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
		1298	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
		1299	notify_ring(&dev_priv->ring[VCS2]);
4560	Serge	1300	} else
		1301	DRM_ERROR("The master control interrupt lied (GT1)!\n");
		1302	}
		1303
6084	serge	1304	if (master_ctl & GEN8_GT_VECS_IRQ) {
		1305	u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
		1306	if (tmp) {
		1307	I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
		1308	ret = IRQ_HANDLED;
		1309
		1310	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
		1311	intel_lrc_irq_handler(&dev_priv->ring[VECS]);
		1312	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
		1313	notify_ring(&dev_priv->ring[VECS]);
		1314	} else
		1315	DRM_ERROR("The master control interrupt lied (GT3)!\n");
		1316	}
		1317
5060	serge	1318	if (master_ctl & GEN8_GT_PM_IRQ) {
6084	serge	1319	u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
5060	serge	1320	if (tmp & dev_priv->pm_rps_events) {
6084	serge	1321	I915_WRITE_FW(GEN8_GT_IIR(2),
		1322	tmp & dev_priv->pm_rps_events);
5060	serge	1323	ret = IRQ_HANDLED;
5354	serge	1324	gen6_rps_irq_handler(dev_priv, tmp);
5060	serge	1325	} else
		1326	DRM_ERROR("The master control interrupt lied (PM)!\n");
		1327	}
		1328
6084	serge	1329	return ret;
		1330	}
5354	serge	1331
6084	serge	1332	static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
		1333	{
		1334	switch (port) {
		1335	case PORT_A:
		1336	return val & PORTA_HOTPLUG_LONG_DETECT;
		1337	case PORT_B:
		1338	return val & PORTB_HOTPLUG_LONG_DETECT;
		1339	case PORT_C:
		1340	return val & PORTC_HOTPLUG_LONG_DETECT;
		1341	default:
		1342	return false;
4560	Serge	1343	}
6084	serge	1344	}
4560	Serge	1345
6084	serge	1346	static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
		1347	{
		1348	switch (port) {
		1349	case PORT_E:
		1350	return val & PORTE_HOTPLUG_LONG_DETECT;
		1351	default:
		1352	return false;
		1353	}
4560	Serge	1354	}
		1355
6084	serge	1356	static bool spt_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1357	{
		1358	switch (port) {
		1359	case PORT_A:
6084	serge	1360	return val & PORTA_HOTPLUG_LONG_DETECT;
5060	serge	1361	case PORT_B:
6084	serge	1362	return val & PORTB_HOTPLUG_LONG_DETECT;
5060	serge	1363	case PORT_C:
6084	serge	1364	return val & PORTC_HOTPLUG_LONG_DETECT;
5060	serge	1365	case PORT_D:
6084	serge	1366	return val & PORTD_HOTPLUG_LONG_DETECT;
		1367	default:
		1368	return false;
5060	serge	1369	}
		1370	}
		1371
6084	serge	1372	static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1373	{
		1374	switch (port) {
		1375	case PORT_A:
6084	serge	1376	return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
5060	serge	1377	default:
6084	serge	1378	return false;
		1379	}
		1380	}
		1381
		1382	static bool pch_port_hotplug_long_detect(enum port port, u32 val)
		1383	{
		1384	switch (port) {
5060	serge	1385	case PORT_B:
6084	serge	1386	return val & PORTB_HOTPLUG_LONG_DETECT;
5060	serge	1387	case PORT_C:
6084	serge	1388	return val & PORTC_HOTPLUG_LONG_DETECT;
5060	serge	1389	case PORT_D:
6084	serge	1390	return val & PORTD_HOTPLUG_LONG_DETECT;
		1391	default:
		1392	return false;
5060	serge	1393	}
		1394	}
		1395
6084	serge	1396	static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1397	{
6084	serge	1398	switch (port) {
		1399	case PORT_B:
		1400	return val & PORTB_HOTPLUG_INT_LONG_PULSE;
		1401	case PORT_C:
		1402	return val & PORTC_HOTPLUG_INT_LONG_PULSE;
		1403	case PORT_D:
		1404	return val & PORTD_HOTPLUG_INT_LONG_PULSE;
5060	serge	1405	default:
6084	serge	1406	return false;
5060	serge	1407	}
		1408	}
		1409
6084	serge	1410	/*
		1411	* Get a bit mask of pins that have triggered, and which ones may be long.
		1412	* This can be called multiple times with the same masks to accumulate
		1413	* hotplug detection results from several registers.
		1414	*
		1415	* Note that the caller is expected to zero out the masks initially.
		1416	*/
		1417	static void intel_get_hpd_pins(u32 pin_mask, u32 long_mask,
		1418	u32 hotplug_trigger, u32 dig_hotplug_reg,
		1419	const u32 hpd[HPD_NUM_PINS],
		1420	bool long_pulse_detect(enum port port, u32 val))
3746	Serge	1421	{
6084	serge	1422	enum port port;
3746	Serge	1423	int i;
		1424
6084	serge	1425	for_each_hpd_pin(i) {
		1426	if ((hpd[i] & hotplug_trigger) == 0)
5060	serge	1427	continue;
3746	Serge	1428
6084	serge	1429	*pin_mask \|= BIT(i);
5060	serge	1430
6296	serge	1431	if (!intel_hpd_pin_to_port(i, &port))
6131	serge	1432	continue;
5060	serge	1433
6084	serge	1434	if (long_pulse_detect(port, dig_hotplug_reg))
		1435	*long_mask \|= BIT(i);
3746	Serge	1436	}
		1437
6084	serge	1438	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
		1439	hotplug_trigger, dig_hotplug_reg, *pin_mask);
3746	Serge	1440
		1441	}
		1442
3480	Serge	1443	static void gmbus_irq_handler(struct drm_device *dev)
		1444	{
5060	serge	1445	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1446
		1447	wake_up_all(&dev_priv->gmbus_wait_queue);
		1448	}
		1449
		1450	static void dp_aux_irq_handler(struct drm_device *dev)
		1451	{
5060	serge	1452	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1453
		1454	wake_up_all(&dev_priv->gmbus_wait_queue);
		1455	}
		1456
4560	Serge	1457	#if defined(CONFIG_DEBUG_FS)
		1458	static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
		1459	uint32_t crc0, uint32_t crc1,
		1460	uint32_t crc2, uint32_t crc3,
		1461	uint32_t crc4)
		1462	{
		1463	struct drm_i915_private *dev_priv = dev->dev_private;
		1464	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
		1465	struct intel_pipe_crc_entry *entry;
		1466	int head, tail;
		1467
		1468	spin_lock(&pipe_crc->lock);
		1469
		1470	if (!pipe_crc->entries) {
		1471	spin_unlock(&pipe_crc->lock);
5354	serge	1472	DRM_DEBUG_KMS("spurious interrupt\n");
4560	Serge	1473	return;
		1474	}
		1475
		1476	head = pipe_crc->head;
		1477	tail = pipe_crc->tail;
		1478
		1479	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
		1480	spin_unlock(&pipe_crc->lock);
		1481	DRM_ERROR("CRC buffer overflowing\n");
		1482	return;
		1483	}
		1484
		1485	entry = &pipe_crc->entries[head];
		1486
		1487	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
		1488	entry->crc[0] = crc0;
		1489	entry->crc[1] = crc1;
		1490	entry->crc[2] = crc2;
		1491	entry->crc[3] = crc3;
		1492	entry->crc[4] = crc4;
		1493
		1494	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		1495	pipe_crc->head = head;
		1496
		1497	spin_unlock(&pipe_crc->lock);
		1498
		1499	wake_up_interruptible(&pipe_crc->wq);
		1500	}
		1501	#else
		1502	static inline void
		1503	display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
		1504	uint32_t crc0, uint32_t crc1,
		1505	uint32_t crc2, uint32_t crc3,
		1506	uint32_t crc4) {}
		1507	#endif
		1508
		1509
		1510	static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1511	{
		1512	struct drm_i915_private *dev_priv = dev->dev_private;
		1513
		1514	display_pipe_crc_irq_handler(dev, pipe,
		1515	I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
		1516	0, 0, 0, 0);
		1517	}
		1518
		1519	static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1520	{
		1521	struct drm_i915_private *dev_priv = dev->dev_private;
		1522
		1523	display_pipe_crc_irq_handler(dev, pipe,
		1524	I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
		1525	I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
		1526	I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
		1527	I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
		1528	I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
		1529	}
		1530
		1531	static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1532	{
		1533	struct drm_i915_private *dev_priv = dev->dev_private;
		1534	uint32_t res1, res2;
		1535
		1536	if (INTEL_INFO(dev)->gen >= 3)
		1537	res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
		1538	else
		1539	res1 = 0;
		1540
		1541	if (INTEL_INFO(dev)->gen >= 5 \|\| IS_G4X(dev))
		1542	res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
		1543	else
		1544	res2 = 0;
		1545
		1546	display_pipe_crc_irq_handler(dev, pipe,
		1547	I915_READ(PIPE_CRC_RES_RED(pipe)),
		1548	I915_READ(PIPE_CRC_RES_GREEN(pipe)),
		1549	I915_READ(PIPE_CRC_RES_BLUE(pipe)),
		1550	res1, res2);
		1551	}
		1552
4104	Serge	1553	/* The RPS events need forcewake, so we add them to a work queue and mask their
		1554	* IMR bits until the work is done. Other interrupts can be processed without
		1555	* the work queue. */
		1556	static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
		1557	{
5060	serge	1558	if (pm_iir & dev_priv->pm_rps_events) {
4104	Serge	1559	spin_lock(&dev_priv->irq_lock);
5354	serge	1560	gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
		1561	if (dev_priv->rps.interrupts_enabled) {
6084	serge	1562	dev_priv->rps.pm_iir \|= pm_iir & dev_priv->pm_rps_events;
5354	serge	1563	queue_work(dev_priv->wq, &dev_priv->rps.work);
		1564	}
4104	Serge	1565	spin_unlock(&dev_priv->irq_lock);
		1566	}
		1567
5354	serge	1568	if (INTEL_INFO(dev_priv)->gen >= 8)
		1569	return;
		1570
4104	Serge	1571	if (HAS_VEBOX(dev_priv->dev)) {
		1572	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
6084	serge	1573	notify_ring(&dev_priv->ring[VECS]);
4104	Serge	1574
5354	serge	1575	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
		1576	DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
4104	Serge	1577	}
		1578	}
		1579
5354	serge	1580	static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
		1581	{
6088	serge	1582	if (!drm_handle_vblank(dev, pipe))
		1583	return false;
5354	serge	1584
		1585	return true;
		1586	}
		1587
5060	serge	1588	static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
3031	serge	1589	{
5060	serge	1590	struct drm_i915_private *dev_priv = dev->dev_private;
		1591	u32 pipe_stats[I915_MAX_PIPES] = { };
3031	serge	1592	int pipe;
		1593
5060	serge	1594	spin_lock(&dev_priv->irq_lock);
5354	serge	1595	for_each_pipe(dev_priv, pipe) {
5060	serge	1596	int reg;
		1597	u32 mask, iir_bit = 0;
3031	serge	1598
5060	serge	1599	/*
		1600	* PIPESTAT bits get signalled even when the interrupt is
		1601	* disabled with the mask bits, and some of the status bits do
		1602	* not generate interrupts at all (like the underrun bit). Hence
		1603	* we need to be careful that we only handle what we want to
		1604	* handle.
		1605	*/
3031	serge	1606
5354	serge	1607	/* fifo underruns are filterered in the underrun handler. */
		1608	mask = PIPE_FIFO_UNDERRUN_STATUS;
		1609
5060	serge	1610	switch (pipe) {
		1611	case PIPE_A:
		1612	iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
		1613	break;
		1614	case PIPE_B:
		1615	iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
		1616	break;
		1617	case PIPE_C:
		1618	iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
		1619	break;
		1620	}
		1621	if (iir & iir_bit)
		1622	mask \|= dev_priv->pipestat_irq_mask[pipe];
3031	serge	1623
5060	serge	1624	if (!mask)
		1625	continue;
3031	serge	1626
5060	serge	1627	reg = PIPESTAT(pipe);
		1628	mask \|= PIPESTAT_INT_ENABLE_MASK;
		1629	pipe_stats[pipe] = I915_READ(reg) & mask;
3031	serge	1630
6084	serge	1631	/*
		1632	* Clear the PIPE*STAT regs before the IIR
		1633	*/
5060	serge	1634	if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS \|
		1635	PIPESTAT_INT_STATUS_MASK))
6084	serge	1636	I915_WRITE(reg, pipe_stats[pipe]);
		1637	}
5060	serge	1638	spin_unlock(&dev_priv->irq_lock);
3031	serge	1639
5354	serge	1640	for_each_pipe(dev_priv, pipe) {
6084	serge	1641	if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		1642	intel_pipe_handle_vblank(dev, pipe))
6320	serge	1643	intel_check_page_flip(dev, pipe);
3031	serge	1644
6084	serge	1645	if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
6320	serge	1646	intel_prepare_page_flip(dev, pipe);
		1647	intel_finish_page_flip(dev, pipe);
6084	serge	1648	}
4560	Serge	1649
6084	serge	1650	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		1651	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	1652
5354	serge	1653	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		1654	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
6084	serge	1655	}
3031	serge	1656
5060	serge	1657	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		1658	gmbus_irq_handler(dev);
		1659	}
3031	serge	1660
5060	serge	1661	static void i9xx_hpd_irq_handler(struct drm_device *dev)
		1662	{
		1663	struct drm_i915_private *dev_priv = dev->dev_private;
		1664	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
6084	serge	1665	u32 pin_mask = 0, long_mask = 0;
4104	Serge	1666
6084	serge	1667	if (!hotplug_status)
		1668	return;
4104	Serge	1669
6084	serge	1670	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
		1671	/*
		1672	* Make sure hotplug status is cleared before we clear IIR, or else we
		1673	* may miss hotplug events.
		1674	*/
		1675	POSTING_READ(PORT_HOTPLUG_STAT);
		1676
		1677	if (IS_G4X(dev) \|\| IS_VALLEYVIEW(dev)) {
5060	serge	1678	u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
4560	Serge	1679
6084	serge	1680	if (hotplug_trigger) {
		1681	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1682	hotplug_trigger, hpd_status_g4x,
		1683	i9xx_port_hotplug_long_detect);
		1684
6296	serge	1685	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1686	}
		1687
		1688	if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
		1689	dp_aux_irq_handler(dev);
5060	serge	1690	} else {
		1691	u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
		1692
6084	serge	1693	if (hotplug_trigger) {
		1694	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1695	hotplug_trigger, hpd_status_i915,
		1696	i9xx_port_hotplug_long_detect);
6296	serge	1697	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1698	}
5060	serge	1699	}
		1700	}
		1701
		1702	static irqreturn_t valleyview_irq_handler(int irq, void *arg)
		1703	{
		1704	struct drm_device *dev = arg;
		1705	struct drm_i915_private *dev_priv = dev->dev_private;
		1706	u32 iir, gt_iir, pm_iir;
		1707	irqreturn_t ret = IRQ_NONE;
		1708
6084	serge	1709	if (!intel_irqs_enabled(dev_priv))
		1710	return IRQ_NONE;
		1711
5060	serge	1712	while (true) {
		1713	/* Find, clear, then process each source of interrupt */
		1714
		1715	gt_iir = I915_READ(GTIIR);
		1716	if (gt_iir)
		1717	I915_WRITE(GTIIR, gt_iir);
		1718
		1719	pm_iir = I915_READ(GEN6_PMIIR);
		1720	if (pm_iir)
		1721	I915_WRITE(GEN6_PMIIR, pm_iir);
		1722
		1723	iir = I915_READ(VLV_IIR);
		1724	if (iir) {
		1725	/* Consume port before clearing IIR or we'll miss events */
		1726	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		1727	i9xx_hpd_irq_handler(dev);
		1728	I915_WRITE(VLV_IIR, iir);
3031	serge	1729	}
		1730
5060	serge	1731	if (gt_iir == 0 && pm_iir == 0 && iir == 0)
		1732	goto out;
3031	serge	1733
5060	serge	1734	ret = IRQ_HANDLED;
		1735
		1736	if (gt_iir)
6084	serge	1737	snb_gt_irq_handler(dev, dev_priv, gt_iir);
4126	Serge	1738	if (pm_iir)
		1739	gen6_rps_irq_handler(dev_priv, pm_iir);
5060	serge	1740	/* Call regardless, as some status bits might not be
		1741	* signalled in iir */
		1742	valleyview_pipestat_irq_handler(dev, iir);
3031	serge	1743	}
		1744
		1745	out:
		1746	return ret;
		1747	}
		1748
5060	serge	1749	static irqreturn_t cherryview_irq_handler(int irq, void *arg)
		1750	{
		1751	struct drm_device *dev = arg;
		1752	struct drm_i915_private *dev_priv = dev->dev_private;
		1753	u32 master_ctl, iir;
		1754	irqreturn_t ret = IRQ_NONE;
		1755
6084	serge	1756	if (!intel_irqs_enabled(dev_priv))
		1757	return IRQ_NONE;
		1758
5060	serge	1759	for (;;) {
		1760	master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		1761	iir = I915_READ(VLV_IIR);
		1762
		1763	if (master_ctl == 0 && iir == 0)
		1764	break;
		1765
		1766	ret = IRQ_HANDLED;
		1767
		1768	I915_WRITE(GEN8_MASTER_IRQ, 0);
		1769
		1770	/* Find, clear, then process each source of interrupt */
		1771
		1772	if (iir) {
		1773	/* Consume port before clearing IIR or we'll miss events */
		1774	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		1775	i9xx_hpd_irq_handler(dev);
		1776	I915_WRITE(VLV_IIR, iir);
		1777	}
		1778
6084	serge	1779	gen8_gt_irq_handler(dev_priv, master_ctl);
5060	serge	1780
		1781	/* Call regardless, as some status bits might not be
		1782	* signalled in iir */
		1783	valleyview_pipestat_irq_handler(dev, iir);
		1784
		1785	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		1786	POSTING_READ(GEN8_MASTER_IRQ);
		1787	}
		1788
		1789	return ret;
		1790	}
		1791
6084	serge	1792	static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		1793	const u32 hpd[HPD_NUM_PINS])
		1794	{
		1795	struct drm_i915_private *dev_priv = to_i915(dev);
		1796	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		1797
		1798	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		1799	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		1800
		1801	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1802	dig_hotplug_reg, hpd,
		1803	pch_port_hotplug_long_detect);
		1804
6296	serge	1805	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1806	}
		1807
3031	serge	1808	static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
		1809	{
5060	serge	1810	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1811	int pipe;
3746	Serge	1812	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
3031	serge	1813
6084	serge	1814	if (hotplug_trigger)
		1815	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
4104	Serge	1816
		1817	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		1818	int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
		1819	SDE_AUDIO_POWER_SHIFT);
		1820	DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
		1821	port_name(port));
3746	Serge	1822	}
3031	serge	1823
3480	Serge	1824	if (pch_iir & SDE_AUX_MASK)
		1825	dp_aux_irq_handler(dev);
		1826
3031	serge	1827	if (pch_iir & SDE_GMBUS)
3480	Serge	1828	gmbus_irq_handler(dev);
3031	serge	1829
		1830	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		1831	DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
		1832
		1833	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		1834	DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
		1835
		1836	if (pch_iir & SDE_POISON)
		1837	DRM_ERROR("PCH poison interrupt\n");
		1838
		1839	if (pch_iir & SDE_FDI_MASK)
5354	serge	1840	for_each_pipe(dev_priv, pipe)
3031	serge	1841	DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
		1842	pipe_name(pipe),
		1843	I915_READ(FDI_RX_IIR(pipe)));
		1844
		1845	if (pch_iir & (SDE_TRANSB_CRC_DONE \| SDE_TRANSA_CRC_DONE))
		1846	DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
		1847
		1848	if (pch_iir & (SDE_TRANSB_CRC_ERR \| SDE_TRANSA_CRC_ERR))
		1849	DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
		1850
4104	Serge	1851	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
5354	serge	1852	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
4104	Serge	1853
3031	serge	1854	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
5354	serge	1855	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
3031	serge	1856	}
		1857
4104	Serge	1858	static void ivb_err_int_handler(struct drm_device *dev)
		1859	{
		1860	struct drm_i915_private *dev_priv = dev->dev_private;
		1861	u32 err_int = I915_READ(GEN7_ERR_INT);
4560	Serge	1862	enum pipe pipe;
4104	Serge	1863
		1864	if (err_int & ERR_INT_POISON)
		1865	DRM_ERROR("Poison interrupt\n");
		1866
5354	serge	1867	for_each_pipe(dev_priv, pipe) {
		1868	if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
		1869	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4104	Serge	1870
4560	Serge	1871	if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
		1872	if (IS_IVYBRIDGE(dev))
		1873	ivb_pipe_crc_irq_handler(dev, pipe);
		1874	else
		1875	hsw_pipe_crc_irq_handler(dev, pipe);
		1876	}
		1877	}
4104	Serge	1878
		1879	I915_WRITE(GEN7_ERR_INT, err_int);
		1880	}
		1881
		1882	static void cpt_serr_int_handler(struct drm_device *dev)
		1883	{
		1884	struct drm_i915_private *dev_priv = dev->dev_private;
		1885	u32 serr_int = I915_READ(SERR_INT);
		1886
		1887	if (serr_int & SERR_INT_POISON)
		1888	DRM_ERROR("PCH poison interrupt\n");
		1889
		1890	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
5354	serge	1891	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
4104	Serge	1892
		1893	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
5354	serge	1894	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
4104	Serge	1895
		1896	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
5354	serge	1897	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
4104	Serge	1898
		1899	I915_WRITE(SERR_INT, serr_int);
		1900	}
		1901
3031	serge	1902	static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
		1903	{
5060	serge	1904	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1905	int pipe;
3746	Serge	1906	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
3031	serge	1907
6084	serge	1908	if (hotplug_trigger)
		1909	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
4104	Serge	1910
		1911	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		1912	int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
		1913	SDE_AUDIO_POWER_SHIFT_CPT);
		1914	DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
		1915	port_name(port));
3746	Serge	1916	}
3031	serge	1917
		1918	if (pch_iir & SDE_AUX_MASK_CPT)
3480	Serge	1919	dp_aux_irq_handler(dev);
3031	serge	1920
		1921	if (pch_iir & SDE_GMBUS_CPT)
3480	Serge	1922	gmbus_irq_handler(dev);
3031	serge	1923
		1924	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		1925	DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
		1926
		1927	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		1928	DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
		1929
		1930	if (pch_iir & SDE_FDI_MASK_CPT)
5354	serge	1931	for_each_pipe(dev_priv, pipe)
3031	serge	1932	DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
		1933	pipe_name(pipe),
		1934	I915_READ(FDI_RX_IIR(pipe)));
		1935
4104	Serge	1936	if (pch_iir & SDE_ERROR_CPT)
		1937	cpt_serr_int_handler(dev);
4539	Serge	1938	}
3480	Serge	1939
6084	serge	1940	static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
		1941	{
		1942	struct drm_i915_private *dev_priv = dev->dev_private;
		1943	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		1944	~SDE_PORTE_HOTPLUG_SPT;
		1945	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
		1946	u32 pin_mask = 0, long_mask = 0;
		1947
		1948	if (hotplug_trigger) {
		1949	u32 dig_hotplug_reg;
		1950
		1951	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		1952	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		1953
		1954	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1955	dig_hotplug_reg, hpd_spt,
		1956	spt_port_hotplug_long_detect);
		1957	}
		1958
		1959	if (hotplug2_trigger) {
		1960	u32 dig_hotplug_reg;
		1961
		1962	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		1963	I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
		1964
		1965	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
		1966	dig_hotplug_reg, hpd_spt,
		1967	spt_port_hotplug2_long_detect);
		1968	}
		1969
6296	serge	1970	if (pin_mask)
		1971	intel_hpd_irq_handler(dev, pin_mask, long_mask);
		1972
6084	serge	1973	if (pch_iir & SDE_GMBUS_CPT)
		1974	gmbus_irq_handler(dev);
		1975	}
		1976
		1977	static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		1978	const u32 hpd[HPD_NUM_PINS])
		1979	{
		1980	struct drm_i915_private *dev_priv = to_i915(dev);
		1981	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		1982
		1983	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
		1984	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
		1985
		1986	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1987	dig_hotplug_reg, hpd,
		1988	ilk_port_hotplug_long_detect);
		1989
6296	serge	1990	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1991	}
		1992
4104	Serge	1993	static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
3031	serge	1994	{
4104	Serge	1995	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	1996	enum pipe pipe;
6084	serge	1997	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
3031	serge	1998
6296	serge	1999	if (hotplug_trigger)
		2000	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
6084	serge	2001
3480	Serge	2002	if (de_iir & DE_AUX_CHANNEL_A)
		2003	dp_aux_irq_handler(dev);
		2004
3031	serge	2005	if (de_iir & DE_GSE)
4104	Serge	2006	intel_opregion_asle_intr(dev);
2351	Serge	2007
4104	Serge	2008	if (de_iir & DE_POISON)
		2009	DRM_ERROR("Poison interrupt\n");
		2010
5354	serge	2011	for_each_pipe(dev_priv, pipe) {
6084	serge	2012	if (de_iir & DE_PIPE_VBLANK(pipe) &&
		2013	intel_pipe_handle_vblank(dev, pipe))
6320	serge	2014	intel_check_page_flip(dev, pipe);
4104	Serge	2015
4560	Serge	2016	if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
5354	serge	2017	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2351	Serge	2018
4560	Serge	2019	if (de_iir & DE_PIPE_CRC_DONE(pipe))
		2020	i9xx_pipe_crc_irq_handler(dev, pipe);
		2021
		2022	/* plane/pipes map 1:1 on ilk+ */
		2023	if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
6320	serge	2024	intel_prepare_page_flip(dev, pipe);
		2025	intel_finish_page_flip_plane(dev, pipe);
4560	Serge	2026	}
3031	serge	2027	}
2351	Serge	2028
3031	serge	2029	/* check event from PCH */
		2030	if (de_iir & DE_PCH_EVENT) {
3480	Serge	2031	u32 pch_iir = I915_READ(SDEIIR);
		2032
3031	serge	2033	if (HAS_PCH_CPT(dev))
		2034	cpt_irq_handler(dev, pch_iir);
		2035	else
		2036	ibx_irq_handler(dev, pch_iir);
3480	Serge	2037
		2038	/* should clear PCH hotplug event before clear CPU irq */
		2039	I915_WRITE(SDEIIR, pch_iir);
3031	serge	2040	}
4104	Serge	2041
6084	serge	2042	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
4104	Serge	2043	ironlake_rps_change_irq_handler(dev);
2351	Serge	2044	}
		2045
4104	Serge	2046	static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
3031	serge	2047	{
		2048	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2049	enum pipe pipe;
6084	serge	2050	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2351	Serge	2051
6084	serge	2052	if (hotplug_trigger)
		2053	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
		2054
4126	Serge	2055	if (de_iir & DE_ERR_INT_IVB)
		2056	ivb_err_int_handler(dev);
2351	Serge	2057
4104	Serge	2058	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		2059	dp_aux_irq_handler(dev);
3031	serge	2060
4104	Serge	2061	if (de_iir & DE_GSE_IVB)
		2062	intel_opregion_asle_intr(dev);
4560	Serge	2063
5354	serge	2064	for_each_pipe(dev_priv, pipe) {
6084	serge	2065	if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		2066	intel_pipe_handle_vblank(dev, pipe))
6320	serge	2067	intel_check_page_flip(dev, pipe);
4560	Serge	2068
		2069	/* plane/pipes map 1:1 on ilk+ */
5060	serge	2070	if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
6320	serge	2071	intel_prepare_page_flip(dev, pipe);
		2072	intel_finish_page_flip_plane(dev, pipe);
3031	serge	2073	}
		2074	}
		2075
4104	Serge	2076	/* check event from PCH */
		2077	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		2078	u32 pch_iir = I915_READ(SDEIIR);
3031	serge	2079
4104	Serge	2080	cpt_irq_handler(dev, pch_iir);
3031	serge	2081
4104	Serge	2082	/* clear PCH hotplug event before clear CPU irq */
		2083	I915_WRITE(SDEIIR, pch_iir);
4539	Serge	2084	}
3031	serge	2085	}
		2086
5060	serge	2087	/*
		2088	* To handle irqs with the minimum potential races with fresh interrupts, we:
		2089	* 1 - Disable Master Interrupt Control.
		2090	* 2 - Find the source(s) of the interrupt.
		2091	* 3 - Clear the Interrupt Identity bits (IIR).
		2092	* 4 - Process the interrupt(s) that had bits set in the IIRs.
		2093	* 5 - Re-enable Master Interrupt Control.
		2094	*/
4104	Serge	2095	static irqreturn_t ironlake_irq_handler(int irq, void *arg)
3031	serge	2096	{
5060	serge	2097	struct drm_device *dev = arg;
		2098	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2099	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
		2100	irqreturn_t ret = IRQ_NONE;
3031	serge	2101
6084	serge	2102	if (!intel_irqs_enabled(dev_priv))
		2103	return IRQ_NONE;
		2104
4104	Serge	2105	/* We get interrupts on unclaimed registers, so check for this before we
		2106	* do any I915_{READ,WRITE}. */
		2107	intel_uncore_check_errors(dev);
3031	serge	2108
4104	Serge	2109	/* disable master interrupt before clearing iir */
		2110	de_ier = I915_READ(DEIER);
		2111	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
		2112	POSTING_READ(DEIER);
3031	serge	2113
4104	Serge	2114	/* Disable south interrupts. We'll only write to SDEIIR once, so further
		2115	* interrupts will will be stored on its back queue, and then we'll be
		2116	* able to process them after we restore SDEIER (as soon as we restore
		2117	* it, we'll get an interrupt if SDEIIR still has something to process
		2118	* due to its back queue). */
		2119	if (!HAS_PCH_NOP(dev)) {
		2120	sde_ier = I915_READ(SDEIER);
		2121	I915_WRITE(SDEIER, 0);
		2122	POSTING_READ(SDEIER);
3031	serge	2123	}
		2124
5060	serge	2125	/* Find, clear, then process each source of interrupt */
		2126
4104	Serge	2127	gt_iir = I915_READ(GTIIR);
		2128	if (gt_iir) {
5060	serge	2129	I915_WRITE(GTIIR, gt_iir);
		2130	ret = IRQ_HANDLED;
4104	Serge	2131	if (INTEL_INFO(dev)->gen >= 6)
		2132	snb_gt_irq_handler(dev, dev_priv, gt_iir);
		2133	else
		2134	ilk_gt_irq_handler(dev, dev_priv, gt_iir);
4539	Serge	2135	}
3031	serge	2136
4104	Serge	2137	de_iir = I915_READ(DEIIR);
		2138	if (de_iir) {
5060	serge	2139	I915_WRITE(DEIIR, de_iir);
		2140	ret = IRQ_HANDLED;
4104	Serge	2141	if (INTEL_INFO(dev)->gen >= 7)
		2142	ivb_display_irq_handler(dev, de_iir);
		2143	else
		2144	ilk_display_irq_handler(dev, de_iir);
3480	Serge	2145	}
		2146
4104	Serge	2147	if (INTEL_INFO(dev)->gen >= 6) {
		2148	u32 pm_iir = I915_READ(GEN6_PMIIR);
		2149	if (pm_iir) {
		2150	I915_WRITE(GEN6_PMIIR, pm_iir);
		2151	ret = IRQ_HANDLED;
5060	serge	2152	gen6_rps_irq_handler(dev_priv, pm_iir);
4560	Serge	2153	}
3031	serge	2154	}
		2155
4104	Serge	2156	I915_WRITE(DEIER, de_ier);
		2157	POSTING_READ(DEIER);
		2158	if (!HAS_PCH_NOP(dev)) {
		2159	I915_WRITE(SDEIER, sde_ier);
		2160	POSTING_READ(SDEIER);
3031	serge	2161	}
		2162
4104	Serge	2163	return ret;
3031	serge	2164	}
		2165
6084	serge	2166	static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		2167	const u32 hpd[HPD_NUM_PINS])
		2168	{
		2169	struct drm_i915_private *dev_priv = to_i915(dev);
		2170	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		2171
		2172	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		2173	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		2174
		2175	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		2176	dig_hotplug_reg, hpd,
		2177	bxt_port_hotplug_long_detect);
		2178
6296	serge	2179	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	2180	}
		2181
4560	Serge	2182	static irqreturn_t gen8_irq_handler(int irq, void *arg)
		2183	{
		2184	struct drm_device *dev = arg;
		2185	struct drm_i915_private *dev_priv = dev->dev_private;
		2186	u32 master_ctl;
		2187	irqreturn_t ret = IRQ_NONE;
		2188	uint32_t tmp = 0;
		2189	enum pipe pipe;
5354	serge	2190	u32 aux_mask = GEN8_AUX_CHANNEL_A;
4560	Serge	2191
6084	serge	2192	if (!intel_irqs_enabled(dev_priv))
		2193	return IRQ_NONE;
		2194
		2195	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2196	aux_mask \|= GEN9_AUX_CHANNEL_B \| GEN9_AUX_CHANNEL_C \|
		2197	GEN9_AUX_CHANNEL_D;
		2198
6084	serge	2199	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
4560	Serge	2200	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
		2201	if (!master_ctl)
		2202	return IRQ_NONE;
		2203
6084	serge	2204	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
4560	Serge	2205
5060	serge	2206	/* Find, clear, then process each source of interrupt */
		2207
6084	serge	2208	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
4560	Serge	2209
		2210	if (master_ctl & GEN8_DE_MISC_IRQ) {
		2211	tmp = I915_READ(GEN8_DE_MISC_IIR);
5060	serge	2212	if (tmp) {
		2213	I915_WRITE(GEN8_DE_MISC_IIR, tmp);
		2214	ret = IRQ_HANDLED;
6084	serge	2215	if (tmp & GEN8_DE_MISC_GSE)
		2216	intel_opregion_asle_intr(dev);
5060	serge	2217	else
6084	serge	2218	DRM_ERROR("Unexpected DE Misc interrupt\n");
5060	serge	2219	}
4560	Serge	2220	else
		2221	DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
		2222	}
		2223
		2224	if (master_ctl & GEN8_DE_PORT_IRQ) {
		2225	tmp = I915_READ(GEN8_DE_PORT_IIR);
5060	serge	2226	if (tmp) {
6084	serge	2227	bool found = false;
		2228	u32 hotplug_trigger = 0;
		2229
		2230	if (IS_BROXTON(dev_priv))
		2231	hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
		2232	else if (IS_BROADWELL(dev_priv))
		2233	hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
		2234
5060	serge	2235	I915_WRITE(GEN8_DE_PORT_IIR, tmp);
		2236	ret = IRQ_HANDLED;
5354	serge	2237
6084	serge	2238	if (tmp & aux_mask) {
		2239	dp_aux_irq_handler(dev);
		2240	found = true;
		2241	}
		2242
		2243	if (hotplug_trigger) {
		2244	if (IS_BROXTON(dev))
		2245	bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
		2246	else
		2247	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
		2248	found = true;
		2249	}
		2250
		2251	if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
		2252	gmbus_irq_handler(dev);
		2253	found = true;
		2254	}
		2255
		2256	if (!found)
		2257	DRM_ERROR("Unexpected DE Port interrupt\n");
5060	serge	2258	}
4560	Serge	2259	else
		2260	DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
		2261	}
		2262
5354	serge	2263	for_each_pipe(dev_priv, pipe) {
		2264	uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
4560	Serge	2265
		2266	if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
		2267	continue;
		2268
		2269	pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
5060	serge	2270	if (pipe_iir) {
		2271	ret = IRQ_HANDLED;
		2272	I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
4560	Serge	2273
6088	serge	2274	if (pipe_iir & GEN8_PIPE_VBLANK &&
		2275	intel_pipe_handle_vblank(dev, pipe))
6320	serge	2276	intel_check_page_flip(dev, pipe);
4560	Serge	2277
6084	serge	2278	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2279	flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
		2280	else
		2281	flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
		2282
6320	serge	2283	if (flip_done) {
		2284	intel_prepare_page_flip(dev, pipe);
		2285	intel_finish_page_flip_plane(dev, pipe);
		2286	}
5354	serge	2287
6084	serge	2288	if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
		2289	hsw_pipe_crc_irq_handler(dev, pipe);
4560	Serge	2290
5354	serge	2291	if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
		2292	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		2293	pipe);
4560	Serge	2294
5354	serge	2295
6084	serge	2296	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2297	fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		2298	else
		2299	fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
		2300
		2301	if (fault_errors)
6084	serge	2302	DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
		2303	pipe_name(pipe),
		2304	pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
4560	Serge	2305	} else
		2306	DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
		2307	}
		2308
6084	serge	2309	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
		2310	master_ctl & GEN8_DE_PCH_IRQ) {
4560	Serge	2311	/*
		2312	* FIXME(BDW): Assume for now that the new interrupt handling
		2313	* scheme also closed the SDE interrupt handling race we've seen
		2314	* on older pch-split platforms. But this needs testing.
		2315	*/
		2316	u32 pch_iir = I915_READ(SDEIIR);
		2317	if (pch_iir) {
		2318	I915_WRITE(SDEIIR, pch_iir);
		2319	ret = IRQ_HANDLED;
6084	serge	2320
		2321	if (HAS_PCH_SPT(dev_priv))
		2322	spt_irq_handler(dev, pch_iir);
		2323	else
		2324	cpt_irq_handler(dev, pch_iir);
6320	serge	2325	} else {
		2326	/*
		2327	* Like on previous PCH there seems to be something
		2328	* fishy going on with forwarding PCH interrupts.
		2329	*/
		2330	DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		2331	}
4560	Serge	2332	}
		2333
6084	serge	2334	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
		2335	POSTING_READ_FW(GEN8_MASTER_IRQ);
4560	Serge	2336
		2337	return ret;
		2338	}
		2339
4104	Serge	2340	static void i915_error_wake_up(struct drm_i915_private *dev_priv,
		2341	bool reset_completed)
3746	Serge	2342	{
5060	serge	2343	struct intel_engine_cs *ring;
4104	Serge	2344	int i;
3031	serge	2345
4104	Serge	2346	/*
		2347	* Notify all waiters for GPU completion events that reset state has
		2348	* been changed, and that they need to restart their wait after
		2349	* checking for potential errors (and bail out to drop locks if there is
		2350	* a gpu reset pending so that i915_error_work_func can acquire them).
		2351	*/
3031	serge	2352
4104	Serge	2353	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
		2354	for_each_ring(ring, dev_priv, i)
		2355	wake_up_all(&ring->irq_queue);
3031	serge	2356
6320	serge	2357	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
		2358	wake_up_all(&dev_priv->pending_flip_queue);
3031	serge	2359
4104	Serge	2360	/*
		2361	* Signal tasks blocked in i915_gem_wait_for_error that the pending
		2362	* reset state is cleared.
		2363	*/
		2364	if (reset_completed)
		2365	wake_up_all(&dev_priv->gpu_error.reset_queue);
3031	serge	2366	}
		2367
		2368	/**
6084	serge	2369	* i915_reset_and_wakeup - do process context error handling work
		2370	* @dev: drm device
3031	serge	2371	*
4104	Serge	2372	* Fire an error uevent so userspace can see that a hang or error
		2373	* was detected.
3031	serge	2374	*/
6084	serge	2375	static void i915_reset_and_wakeup(struct drm_device *dev)
3031	serge	2376	{
6084	serge	2377	struct drm_i915_private *dev_priv = to_i915(dev);
		2378	struct i915_gpu_error *error = &dev_priv->gpu_error;
4104	Serge	2379	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
		2380	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
		2381	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
		2382	int ret;
3031	serge	2383
4104	Serge	2384	/*
		2385	* Note that there's only one work item which does gpu resets, so we
		2386	* need not worry about concurrent gpu resets potentially incrementing
		2387	* error->reset_counter twice. We only need to take care of another
		2388	* racing irq/hangcheck declaring the gpu dead for a second time. A
		2389	* quick check for that is good enough: schedule_work ensures the
		2390	* correct ordering between hang detection and this work item, and since
		2391	* the reset in-progress bit is only ever set by code outside of this
		2392	* work we don't need to worry about any other races.
		2393	*/
		2394	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
		2395	DRM_DEBUG_DRIVER("resetting chip\n");
6084	serge	2396	intel_runtime_pm_get(dev_priv);
3031	serge	2397
4104	Serge	2398	/*
		2399	* All state reset _must_ be completed before we update the
		2400	* reset counter, for otherwise waiters might miss the reset
		2401	* pending state and not properly drop locks, resulting in
		2402	* deadlocks with the reset work.
		2403	*/
4560	Serge	2404	// ret = i915_reset(dev);
3031	serge	2405
6084	serge	2406	// intel_finish_reset(dev);
3031	serge	2407
6084	serge	2408	intel_runtime_pm_put(dev_priv);
		2409
4104	Serge	2410	if (ret == 0) {
		2411	/*
		2412	* After all the gem state is reset, increment the reset
		2413	* counter and wake up everyone waiting for the reset to
		2414	* complete.
		2415	*
		2416	* Since unlock operations are a one-sided barrier only,
		2417	* we need to insert a barrier here to order any seqno
		2418	* updates before
		2419	* the counter increment.
		2420	*/
6084	serge	2421	smp_mb__before_atomic();
4104	Serge	2422	atomic_inc(&dev_priv->gpu_error.reset_counter);
3031	serge	2423
4104	Serge	2424	} else {
6088	serge	2425	atomic_or(I915_WEDGED, &error->reset_counter);
		2426	}
3031	serge	2427
4104	Serge	2428	/*
		2429	* Note: The wake_up also serves as a memory barrier so that
		2430	* waiters see the update value of the reset counter atomic_t.
		2431	*/
		2432	i915_error_wake_up(dev_priv, true);
3031	serge	2433	}
		2434	}
		2435
		2436	static void i915_report_and_clear_eir(struct drm_device *dev)
		2437	{
		2438	struct drm_i915_private *dev_priv = dev->dev_private;
		2439	uint32_t instdone[I915_NUM_INSTDONE_REG];
		2440	u32 eir = I915_READ(EIR);
		2441	int pipe, i;
		2442
		2443	if (!eir)
		2444	return;
		2445
		2446	pr_err("render error detected, EIR: 0x%08x\n", eir);
		2447
		2448	i915_get_extra_instdone(dev, instdone);
		2449
		2450	if (IS_G4X(dev)) {
		2451	if (eir & (GM45_ERROR_MEM_PRIV \| GM45_ERROR_CP_PRIV)) {
		2452	u32 ipeir = I915_READ(IPEIR_I965);
		2453
		2454	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
		2455	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
		2456	for (i = 0; i < ARRAY_SIZE(instdone); i++)
		2457	pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
		2458	pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
		2459	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
		2460	I915_WRITE(IPEIR_I965, ipeir);
		2461	POSTING_READ(IPEIR_I965);
		2462	}
		2463	if (eir & GM45_ERROR_PAGE_TABLE) {
		2464	u32 pgtbl_err = I915_READ(PGTBL_ER);
		2465	pr_err("page table error\n");
		2466	pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
		2467	I915_WRITE(PGTBL_ER, pgtbl_err);
		2468	POSTING_READ(PGTBL_ER);
		2469	}
		2470	}
		2471
		2472	if (!IS_GEN2(dev)) {
		2473	if (eir & I915_ERROR_PAGE_TABLE) {
		2474	u32 pgtbl_err = I915_READ(PGTBL_ER);
		2475	pr_err("page table error\n");
		2476	pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
		2477	I915_WRITE(PGTBL_ER, pgtbl_err);
		2478	POSTING_READ(PGTBL_ER);
		2479	}
		2480	}
		2481
		2482	if (eir & I915_ERROR_MEMORY_REFRESH) {
		2483	pr_err("memory refresh error:\n");
5354	serge	2484	for_each_pipe(dev_priv, pipe)
3031	serge	2485	pr_err("pipe %c stat: 0x%08x\n",
		2486	pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
		2487	/* pipestat has already been acked */
		2488	}
		2489	if (eir & I915_ERROR_INSTRUCTION) {
		2490	pr_err("instruction error\n");
		2491	pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
		2492	for (i = 0; i < ARRAY_SIZE(instdone); i++)
		2493	pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
		2494	if (INTEL_INFO(dev)->gen < 4) {
		2495	u32 ipeir = I915_READ(IPEIR);
		2496
		2497	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
		2498	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
		2499	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
		2500	I915_WRITE(IPEIR, ipeir);
		2501	POSTING_READ(IPEIR);
		2502	} else {
		2503	u32 ipeir = I915_READ(IPEIR_I965);
		2504
		2505	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
		2506	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
		2507	pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
		2508	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
		2509	I915_WRITE(IPEIR_I965, ipeir);
		2510	POSTING_READ(IPEIR_I965);
		2511	}
		2512	}
		2513
		2514	I915_WRITE(EIR, eir);
		2515	POSTING_READ(EIR);
		2516	eir = I915_READ(EIR);
		2517	if (eir) {
		2518	/*
		2519	* some errors might have become stuck,
		2520	* mask them.
		2521	*/
		2522	DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		2523	I915_WRITE(EMR, I915_READ(EMR) \| eir);
		2524	I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
		2525	}
		2526	}
		2527
		2528	/**
6084	serge	2529	* i915_handle_error - handle a gpu error
3031	serge	2530	* @dev: drm device
		2531	*
6084	serge	2532	* Do some basic checking of register state at error time and
3031	serge	2533	* dump it to the syslog. Also call i915_capture_error_state() to make
		2534	* sure we get a record and make it available in debugfs. Fire a uevent
		2535	* so userspace knows something bad happened (should trigger collection
		2536	* of a ring dump etc.).
		2537	*/
5060	serge	2538	void i915_handle_error(struct drm_device *dev, bool wedged,
		2539	const char *fmt, ...)
3031	serge	2540	{
		2541	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2542	va_list args;
		2543	char error_msg[80];
3031	serge	2544
5060	serge	2545	va_start(args, fmt);
		2546	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
		2547	va_end(args);
		2548
4560	Serge	2549	// i915_capture_error_state(dev);
3031	serge	2550	i915_report_and_clear_eir(dev);
		2551
		2552	if (wedged) {
6084	serge	2553	atomic_or(I915_RESET_IN_PROGRESS_FLAG,
3480	Serge	2554	&dev_priv->gpu_error.reset_counter);
3031	serge	2555
		2556	/*
6084	serge	2557	* Wakeup waiting processes so that the reset function
		2558	* i915_reset_and_wakeup doesn't deadlock trying to grab
		2559	* various locks. By bumping the reset counter first, the woken
4104	Serge	2560	* processes will see a reset in progress and back off,
		2561	* releasing their locks and then wait for the reset completion.
		2562	* We must do this for _all_ gpu waiters that might hold locks
		2563	* that the reset work needs to acquire.
		2564	*
		2565	* Note: The wake_up serves as the required memory barrier to
		2566	* ensure that the waiters see the updated value of the reset
		2567	* counter atomic_t.
3031	serge	2568	*/
4104	Serge	2569	i915_error_wake_up(dev_priv, false);
3031	serge	2570	}
		2571
6084	serge	2572	i915_reset_and_wakeup(dev);
3031	serge	2573	}
		2574
		2575	/* Called from drm generic code, passed 'crtc' which
		2576	* we use as a pipe index
		2577	*/
6084	serge	2578	static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2579	{
5060	serge	2580	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2581	unsigned long irqflags;
		2582
		2583	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2584	if (INTEL_INFO(dev)->gen >= 4)
		2585	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2586	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2587	else
		2588	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2589	PIPE_VBLANK_INTERRUPT_STATUS);
3031	serge	2590	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2591
		2592	return 0;
		2593	}
		2594
6084	serge	2595	static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2596	{
5060	serge	2597	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2598	unsigned long irqflags;
4104	Serge	2599	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
4560	Serge	2600	DE_PIPE_VBLANK(pipe);
3031	serge	2601
		2602	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4104	Serge	2603	ironlake_enable_display_irq(dev_priv, bit);
3031	serge	2604	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2605
		2606	return 0;
		2607	}
		2608
6084	serge	2609	static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2610	{
5060	serge	2611	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2612	unsigned long irqflags;
		2613
		2614	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2615	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2616	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2617	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2618
		2619	return 0;
		2620	}
		2621
6084	serge	2622	static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
4560	Serge	2623	{
		2624	struct drm_i915_private *dev_priv = dev->dev_private;
		2625	unsigned long irqflags;
		2626
		2627	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2628	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
		2629	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		2630	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
		2631	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2632	return 0;
		2633	}
		2634
3031	serge	2635	/* Called from drm generic code, passed 'crtc' which
		2636	* we use as a pipe index
		2637	*/
6084	serge	2638	static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2639	{
5060	serge	2640	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2641	unsigned long irqflags;
		2642
		2643	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2644	i915_disable_pipestat(dev_priv, pipe,
5060	serge	2645	PIPE_VBLANK_INTERRUPT_STATUS \|
		2646	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2647	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2648	}
		2649
6084	serge	2650	static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2651	{
5060	serge	2652	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2653	unsigned long irqflags;
4104	Serge	2654	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
4560	Serge	2655	DE_PIPE_VBLANK(pipe);
3031	serge	2656
		2657	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4104	Serge	2658	ironlake_disable_display_irq(dev_priv, bit);
3031	serge	2659	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2660	}
		2661
6084	serge	2662	static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2663	{
5060	serge	2664	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2665	unsigned long irqflags;
		2666
		2667	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2668	i915_disable_pipestat(dev_priv, pipe,
5060	serge	2669	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2670	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2671	}
		2672
6084	serge	2673	static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
4560	Serge	2674	{
		2675	struct drm_i915_private *dev_priv = dev->dev_private;
		2676	unsigned long irqflags;
		2677
		2678	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2679	dev_priv->de_irq_mask[pipe] \|= GEN8_PIPE_VBLANK;
		2680	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		2681	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
		2682	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2683	}
		2684
4104	Serge	2685	static bool
5060	serge	2686	ring_idle(struct intel_engine_cs *ring, u32 seqno)
2351	Serge	2687	{
4104	Serge	2688	return (list_empty(&ring->request_list) \|\|
6084	serge	2689	i915_seqno_passed(seqno, ring->last_submitted_seqno));
4104	Serge	2690	}
2351	Serge	2691
5060	serge	2692	static bool
		2693	ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
4104	Serge	2694	{
5060	serge	2695	if (INTEL_INFO(dev)->gen >= 8) {
		2696	return (ipehr >> 23) == 0x1c;
		2697	} else {
		2698	ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		2699	return ipehr == (MI_SEMAPHORE_MBOX \| MI_SEMAPHORE_COMPARE \|
		2700	MI_SEMAPHORE_REGISTER);
		2701	}
		2702	}
		2703
		2704	static struct intel_engine_cs *
		2705	semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
		2706	{
4104	Serge	2707	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	2708	struct intel_engine_cs *signaller;
		2709	int i;
2351	Serge	2710
5060	serge	2711	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
		2712	for_each_ring(signaller, dev_priv, i) {
		2713	if (ring == signaller)
		2714	continue;
		2715
		2716	if (offset == signaller->semaphore.signal_ggtt[ring->id])
		2717	return signaller;
		2718	}
		2719	} else {
		2720	u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
		2721
		2722	for_each_ring(signaller, dev_priv, i) {
		2723	if(ring == signaller)
		2724	continue;
		2725
		2726	if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
		2727	return signaller;
		2728	}
		2729	}
		2730
		2731	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		2732	ring->id, ipehr, offset);
		2733
		2734	return NULL;
		2735	}
		2736
		2737	static struct intel_engine_cs *
		2738	semaphore_waits_for(struct intel_engine_cs ring, u32 seqno)
		2739	{
		2740	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		2741	u32 cmd, ipehr, head;
		2742	u64 offset = 0;
		2743	int i, backwards;
		2744
6084	serge	2745	/*
		2746	* This function does not support execlist mode - any attempt to
		2747	* proceed further into this function will result in a kernel panic
		2748	* when dereferencing ring->buffer, which is not set up in execlist
		2749	* mode.
		2750	*
		2751	* The correct way of doing it would be to derive the currently
		2752	* executing ring buffer from the current context, which is derived
		2753	* from the currently running request. Unfortunately, to get the
		2754	* current request we would have to grab the struct_mutex before doing
		2755	* anything else, which would be ill-advised since some other thread
		2756	* might have grabbed it already and managed to hang itself, causing
		2757	* the hang checker to deadlock.
		2758	*
		2759	* Therefore, this function does not support execlist mode in its
		2760	* current form. Just return NULL and move on.
		2761	*/
		2762	if (ring->buffer == NULL)
		2763	return NULL;
		2764
4104	Serge	2765	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
5060	serge	2766	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
4104	Serge	2767	return NULL;
2351	Serge	2768
5060	serge	2769	/*
		2770	* HEAD is likely pointing to the dword after the actual command,
		2771	* so scan backwards until we find the MBOX. But limit it to just 3
		2772	* or 4 dwords depending on the semaphore wait command size.
		2773	* Note that we don't care about ACTHD here since that might
		2774	* point at at batch, and semaphores are always emitted into the
		2775	* ringbuffer itself.
4104	Serge	2776	*/
5060	serge	2777	head = I915_READ_HEAD(ring) & HEAD_ADDR;
		2778	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
		2779
		2780	for (i = backwards; i; --i) {
		2781	/*
		2782	* Be paranoid and presume the hw has gone off into the wild -
		2783	* our ring is smaller than what the hardware (and hence
		2784	* HEAD_ADDR) allows. Also handles wrap-around.
		2785	*/
		2786	head &= ring->buffer->size - 1;
		2787
		2788	/* This here seems to blow up */
		2789	cmd = ioread32(ring->buffer->virtual_start + head);
4104	Serge	2790	if (cmd == ipehr)
		2791	break;
2351	Serge	2792
5060	serge	2793	head -= 4;
		2794	}
		2795
		2796	if (!i)
6084	serge	2797	return NULL;
2351	Serge	2798
5060	serge	2799	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
		2800	if (INTEL_INFO(ring->dev)->gen >= 8) {
		2801	offset = ioread32(ring->buffer->virtual_start + head + 12);
		2802	offset <<= 32;
		2803	offset = ioread32(ring->buffer->virtual_start + head + 8);
		2804	}
		2805	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
4104	Serge	2806	}
2351	Serge	2807
5060	serge	2808	static int semaphore_passed(struct intel_engine_cs *ring)
4104	Serge	2809	{
		2810	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	2811	struct intel_engine_cs *signaller;
		2812	u32 seqno;
4104	Serge	2813
5060	serge	2814	ring->hangcheck.deadlock++;
4104	Serge	2815
		2816	signaller = semaphore_waits_for(ring, &seqno);
5060	serge	2817	if (signaller == NULL)
4104	Serge	2818	return -1;
		2819
5060	serge	2820	/* Prevent pathological recursion due to driver bugs */
		2821	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
		2822	return -1;
		2823
		2824	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		2825	return 1;
		2826
4104	Serge	2827	/* cursory check for an unkickable deadlock */
5060	serge	2828	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
		2829	semaphore_passed(signaller) < 0)
4104	Serge	2830	return -1;
		2831
5060	serge	2832	return 0;
4104	Serge	2833	}
		2834
		2835	static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
		2836	{
5060	serge	2837	struct intel_engine_cs *ring;
4104	Serge	2838	int i;
		2839
		2840	for_each_ring(ring, dev_priv, i)
5060	serge	2841	ring->hangcheck.deadlock = 0;
4104	Serge	2842	}
		2843
		2844	static enum intel_ring_hangcheck_action
5060	serge	2845	ring_stuck(struct intel_engine_cs *ring, u64 acthd)
4104	Serge	2846	{
		2847	struct drm_device *dev = ring->dev;
		2848	struct drm_i915_private *dev_priv = dev->dev_private;
		2849	u32 tmp;
		2850
5060	serge	2851	if (acthd != ring->hangcheck.acthd) {
		2852	if (acthd > ring->hangcheck.max_acthd) {
		2853	ring->hangcheck.max_acthd = acthd;
6084	serge	2854	return HANGCHECK_ACTIVE;
5060	serge	2855	}
4104	Serge	2856
5060	serge	2857	return HANGCHECK_ACTIVE_LOOP;
		2858	}
		2859
4104	Serge	2860	if (IS_GEN2(dev))
		2861	return HANGCHECK_HUNG;
		2862
		2863	/* Is the chip hanging on a WAIT_FOR_EVENT?
		2864	* If so we can simply poke the RB_WAIT bit
		2865	* and break the hang. This should work on
		2866	* all but the second generation chipsets.
		2867	*/
		2868	tmp = I915_READ_CTL(ring);
		2869	if (tmp & RING_WAIT) {
5060	serge	2870	i915_handle_error(dev, false,
		2871	"Kicking stuck wait on %s",
6084	serge	2872	ring->name);
4104	Serge	2873	I915_WRITE_CTL(ring, tmp);
		2874	return HANGCHECK_KICK;
		2875	}
		2876
		2877	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		2878	switch (semaphore_passed(ring)) {
		2879	default:
		2880	return HANGCHECK_HUNG;
		2881	case 1:
5060	serge	2882	i915_handle_error(dev, false,
		2883	"Kicking stuck semaphore on %s",
6084	serge	2884	ring->name);
4104	Serge	2885	I915_WRITE_CTL(ring, tmp);
		2886	return HANGCHECK_KICK;
		2887	case 0:
		2888	return HANGCHECK_WAIT;
		2889	}
		2890	}
		2891
		2892	return HANGCHECK_HUNG;
		2893	}
		2894
6084	serge	2895	/*
4104	Serge	2896	* This is called when the chip hasn't reported back with completed
		2897	* batchbuffers in a long time. We keep track per ring seqno progress and
		2898	* if there are no progress, hangcheck score for that ring is increased.
		2899	* Further, acthd is inspected to see if the ring is stuck. On stuck case
		2900	* we kick the ring. If we see no progress on three subsequent calls
		2901	* we assume chip is wedged and try to fix it by resetting the chip.
		2902	*/
6084	serge	2903	static void i915_hangcheck_elapsed(struct work_struct *work)
4104	Serge	2904	{
6084	serge	2905	struct drm_i915_private *dev_priv =
		2906	container_of(work, typeof(*dev_priv),
		2907	gpu_error.hangcheck_work.work);
		2908	struct drm_device *dev = dev_priv->dev;
5060	serge	2909	struct intel_engine_cs *ring;
4104	Serge	2910	int i;
		2911	int busy_count = 0, rings_hung = 0;
		2912	bool stuck[I915_NUM_RINGS] = { 0 };
		2913	#define BUSY 1
		2914	#define KICK 5
		2915	#define HUNG 20
		2916
5060	serge	2917	if (!i915.enable_hangcheck)
4104	Serge	2918	return;
		2919
		2920	for_each_ring(ring, dev_priv, i) {
5060	serge	2921	u64 acthd;
		2922	u32 seqno;
4104	Serge	2923	bool busy = true;
		2924
		2925	semaphore_clear_deadlocks(dev_priv);
		2926
		2927	seqno = ring->get_seqno(ring, false);
		2928	acthd = intel_ring_get_active_head(ring);
		2929
		2930	if (ring->hangcheck.seqno == seqno) {
		2931	if (ring_idle(ring, seqno)) {
5060	serge	2932	ring->hangcheck.action = HANGCHECK_IDLE;
		2933
6084	serge	2934	if (waitqueue_active(&ring->irq_queue)) {
4104	Serge	2935	/* Issue a wake-up to catch stuck h/w. */
6084	serge	2936	if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
		2937	if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
		2938	DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
		2939	ring->name);
		2940	else
		2941	DRM_INFO("Fake missed irq on %s\n",
		2942	ring->name);
		2943	wake_up_all(&ring->irq_queue);
		2944	}
		2945	/* Safeguard against driver failure */
		2946	ring->hangcheck.score += BUSY;
		2947	} else
4104	Serge	2948	busy = false;
		2949	} else {
		2950	/* We always increment the hangcheck score
		2951	* if the ring is busy and still processing
		2952	* the same request, so that no single request
		2953	* can run indefinitely (such as a chain of
		2954	* batches). The only time we do not increment
		2955	* the hangcheck score on this ring, if this
		2956	* ring is in a legitimate wait for another
		2957	* ring. In that case the waiting ring is a
		2958	* victim and we want to be sure we catch the
		2959	* right culprit. Then every time we do kick
		2960	* the ring, add a small increment to the
		2961	* score so that we can catch a batch that is
		2962	* being repeatedly kicked and so responsible
		2963	* for stalling the machine.
		2964	*/
		2965	ring->hangcheck.action = ring_stuck(ring,
		2966	acthd);
		2967
		2968	switch (ring->hangcheck.action) {
4560	Serge	2969	case HANGCHECK_IDLE:
4104	Serge	2970	case HANGCHECK_WAIT:
5060	serge	2971	case HANGCHECK_ACTIVE:
4104	Serge	2972	break;
5060	serge	2973	case HANGCHECK_ACTIVE_LOOP:
4104	Serge	2974	ring->hangcheck.score += BUSY;
		2975	break;
		2976	case HANGCHECK_KICK:
		2977	ring->hangcheck.score += KICK;
		2978	break;
		2979	case HANGCHECK_HUNG:
		2980	ring->hangcheck.score += HUNG;
		2981	stuck[i] = true;
		2982	break;
		2983	}
		2984	}
		2985	} else {
4560	Serge	2986	ring->hangcheck.action = HANGCHECK_ACTIVE;
		2987
4104	Serge	2988	/* Gradually reduce the count so that we catch DoS
		2989	* attempts across multiple batches.
		2990	*/
		2991	if (ring->hangcheck.score > 0)
		2992	ring->hangcheck.score--;
5060	serge	2993
		2994	ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
4104	Serge	2995	}
		2996
		2997	ring->hangcheck.seqno = seqno;
		2998	ring->hangcheck.acthd = acthd;
		2999	busy_count += busy;
		3000	}
		3001
		3002	for_each_ring(ring, dev_priv, i) {
5060	serge	3003	if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
4104	Serge	3004	DRM_INFO("%s on %s\n",
6084	serge	3005	stuck[i] ? "stuck" : "no progress",
		3006	ring->name);
4104	Serge	3007	rings_hung++;
		3008	}
		3009	}
		3010
		3011	// if (rings_hung)
		3012	// return i915_handle_error(dev, true);
		3013
		3014	}
6088	serge	3015
5060	serge	3016	static void ibx_irq_reset(struct drm_device *dev)
		3017	{
		3018	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3019
5060	serge	3020	if (HAS_PCH_NOP(dev))
		3021	return;
		3022
		3023	GEN5_IRQ_RESET(SDE);
		3024
		3025	if (HAS_PCH_CPT(dev) \|\| HAS_PCH_LPT(dev))
		3026	I915_WRITE(SERR_INT, 0xffffffff);
		3027	}
		3028
		3029	/*
		3030	* SDEIER is also touched by the interrupt handler to work around missed PCH
		3031	* interrupts. Hence we can't update it after the interrupt handler is enabled -
		3032	* instead we unconditionally enable all PCH interrupt sources here, but then
		3033	* only unmask them as needed with SDEIMR.
		3034	*
		3035	* This function needs to be called before interrupts are enabled.
		3036	*/
		3037	static void ibx_irq_pre_postinstall(struct drm_device *dev)
4104	Serge	3038	{
		3039	struct drm_i915_private *dev_priv = dev->dev_private;
		3040
3746	Serge	3041	if (HAS_PCH_NOP(dev))
		3042	return;
		3043
5060	serge	3044	WARN_ON(I915_READ(SDEIER) != 0);
3746	Serge	3045	I915_WRITE(SDEIER, 0xffffffff);
4104	Serge	3046	POSTING_READ(SDEIER);
2351	Serge	3047	}
		3048
5060	serge	3049	static void gen5_gt_irq_reset(struct drm_device *dev)
4104	Serge	3050	{
		3051	struct drm_i915_private *dev_priv = dev->dev_private;
		3052
5060	serge	3053	GEN5_IRQ_RESET(GT);
		3054	if (INTEL_INFO(dev)->gen >= 6)
		3055	GEN5_IRQ_RESET(GEN6_PM);
4104	Serge	3056	}
		3057
		3058	/* drm_dma.h hooks
		3059	*/
5060	serge	3060	static void ironlake_irq_reset(struct drm_device *dev)
4104	Serge	3061	{
5060	serge	3062	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3063
5060	serge	3064	I915_WRITE(HWSTAM, 0xffffffff);
4104	Serge	3065
5060	serge	3066	GEN5_IRQ_RESET(DE);
		3067	if (IS_GEN7(dev))
		3068	I915_WRITE(GEN7_ERR_INT, 0xffffffff);
4104	Serge	3069
5060	serge	3070	gen5_gt_irq_reset(dev);
4104	Serge	3071
5060	serge	3072	ibx_irq_reset(dev);
4104	Serge	3073	}
		3074
5354	serge	3075	static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
		3076	{
		3077	enum pipe pipe;
		3078
6084	serge	3079	i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
5354	serge	3080	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		3081
		3082	for_each_pipe(dev_priv, pipe)
		3083	I915_WRITE(PIPESTAT(pipe), 0xffff);
		3084
		3085	GEN5_IRQ_RESET(VLV_);
		3086	}
		3087
3031	serge	3088	static void valleyview_irq_preinstall(struct drm_device *dev)
		3089	{
5060	serge	3090	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3091
		3092	/* VLV magic */
		3093	I915_WRITE(VLV_IMR, 0);
		3094	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
		3095	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
		3096	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
		3097
5060	serge	3098	gen5_gt_irq_reset(dev);
4104	Serge	3099
5354	serge	3100	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3031	serge	3101
5354	serge	3102	vlv_display_irq_reset(dev_priv);
3031	serge	3103	}
		3104
5060	serge	3105	static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
4560	Serge	3106	{
5060	serge	3107	GEN8_IRQ_RESET_NDX(GT, 0);
		3108	GEN8_IRQ_RESET_NDX(GT, 1);
		3109	GEN8_IRQ_RESET_NDX(GT, 2);
		3110	GEN8_IRQ_RESET_NDX(GT, 3);
		3111	}
		3112
		3113	static void gen8_irq_reset(struct drm_device *dev)
		3114	{
4560	Serge	3115	struct drm_i915_private *dev_priv = dev->dev_private;
		3116	int pipe;
		3117
		3118	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3119	POSTING_READ(GEN8_MASTER_IRQ);
		3120
5060	serge	3121	gen8_gt_irq_reset(dev_priv);
4560	Serge	3122
5354	serge	3123	for_each_pipe(dev_priv, pipe)
		3124	if (intel_display_power_is_enabled(dev_priv,
6084	serge	3125	POWER_DOMAIN_PIPE(pipe)))
		3126	GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
4560	Serge	3127
5060	serge	3128	GEN5_IRQ_RESET(GEN8_DE_PORT_);
		3129	GEN5_IRQ_RESET(GEN8_DE_MISC_);
		3130	GEN5_IRQ_RESET(GEN8_PCU_);
4560	Serge	3131
6084	serge	3132	if (HAS_PCH_SPLIT(dev))
		3133	ibx_irq_reset(dev);
5060	serge	3134	}
4560	Serge	3135
6084	serge	3136	void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
		3137	unsigned int pipe_mask)
5060	serge	3138	{
5354	serge	3139	uint32_t extra_ier = GEN8_PIPE_VBLANK \| GEN8_PIPE_FIFO_UNDERRUN;
4560	Serge	3140
5354	serge	3141	spin_lock_irq(&dev_priv->irq_lock);
6084	serge	3142	if (pipe_mask & 1 << PIPE_A)
		3143	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
		3144	dev_priv->de_irq_mask[PIPE_A],
		3145	~dev_priv->de_irq_mask[PIPE_A] \| extra_ier);
		3146	if (pipe_mask & 1 << PIPE_B)
		3147	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
		3148	dev_priv->de_irq_mask[PIPE_B],
		3149	~dev_priv->de_irq_mask[PIPE_B] \| extra_ier);
		3150	if (pipe_mask & 1 << PIPE_C)
		3151	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
		3152	dev_priv->de_irq_mask[PIPE_C],
		3153	~dev_priv->de_irq_mask[PIPE_C] \| extra_ier);
5354	serge	3154	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	3155	}
		3156
		3157	static void cherryview_irq_preinstall(struct drm_device *dev)
		3158	{
		3159	struct drm_i915_private *dev_priv = dev->dev_private;
		3160
		3161	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3162	POSTING_READ(GEN8_MASTER_IRQ);
		3163
		3164	gen8_gt_irq_reset(dev_priv);
		3165
		3166	GEN5_IRQ_RESET(GEN8_PCU_);
		3167
		3168	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
		3169
5354	serge	3170	vlv_display_irq_reset(dev_priv);
4560	Serge	3171	}
		3172
6084	serge	3173	static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
		3174	const u32 hpd[HPD_NUM_PINS])
		3175	{
		3176	struct drm_i915_private *dev_priv = to_i915(dev);
		3177	struct intel_encoder *encoder;
		3178	u32 enabled_irqs = 0;
		3179
		3180	for_each_intel_encoder(dev, encoder)
		3181	if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
		3182	enabled_irqs \|= hpd[encoder->hpd_pin];
		3183
		3184	return enabled_irqs;
		3185	}
		3186
3746	Serge	3187	static void ibx_hpd_irq_setup(struct drm_device *dev)
		3188	{
5060	serge	3189	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	3190	u32 hotplug_irqs, hotplug, enabled_irqs;
3746	Serge	3191
		3192	if (HAS_PCH_IBX(dev)) {
4104	Serge	3193	hotplug_irqs = SDE_HOTPLUG_MASK;
6084	serge	3194	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3746	Serge	3195	} else {
4104	Serge	3196	hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
6084	serge	3197	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3746	Serge	3198	}
		3199
4104	Serge	3200	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3746	Serge	3201
		3202	/*
6084	serge	3203	* Enable digital hotplug on the PCH, and configure the DP short pulse
		3204	* duration to 2ms (which is the minimum in the Display Port spec).
		3205	* The pulse duration bits are reserved on LPT+.
		3206	*/
2351	Serge	3207	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3208	hotplug &= ~(PORTD_PULSE_DURATION_MASK\|PORTC_PULSE_DURATION_MASK\|PORTB_PULSE_DURATION_MASK);
		3209	hotplug \|= PORTD_HOTPLUG_ENABLE \| PORTD_PULSE_DURATION_2ms;
		3210	hotplug \|= PORTC_HOTPLUG_ENABLE \| PORTC_PULSE_DURATION_2ms;
		3211	hotplug \|= PORTB_HOTPLUG_ENABLE \| PORTB_PULSE_DURATION_2ms;
6084	serge	3212	/*
		3213	* When CPU and PCH are on the same package, port A
		3214	* HPD must be enabled in both north and south.
		3215	*/
		3216	if (HAS_PCH_LPT_LP(dev))
		3217	hotplug \|= PORTA_HOTPLUG_ENABLE;
2351	Serge	3218	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3219	}
		3220
6084	serge	3221	static void spt_hpd_irq_setup(struct drm_device *dev)
		3222	{
		3223	struct drm_i915_private *dev_priv = dev->dev_private;
		3224	u32 hotplug_irqs, hotplug, enabled_irqs;
		3225
		3226	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
		3227	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
		3228
		3229	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
		3230
		3231	/* Enable digital hotplug on the PCH */
		3232	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3233	hotplug \|= PORTD_HOTPLUG_ENABLE \| PORTC_HOTPLUG_ENABLE \|
		3234	PORTB_HOTPLUG_ENABLE \| PORTA_HOTPLUG_ENABLE;
		3235	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3236
		3237	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
		3238	hotplug \|= PORTE_HOTPLUG_ENABLE;
		3239	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
		3240	}
		3241
		3242	static void ilk_hpd_irq_setup(struct drm_device *dev)
		3243	{
		3244	struct drm_i915_private *dev_priv = dev->dev_private;
		3245	u32 hotplug_irqs, hotplug, enabled_irqs;
		3246
		3247	if (INTEL_INFO(dev)->gen >= 8) {
		3248	hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		3249	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
		3250
		3251	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3252	} else if (INTEL_INFO(dev)->gen >= 7) {
		3253	hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		3254	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
		3255
		3256	ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3257	} else {
		3258	hotplug_irqs = DE_DP_A_HOTPLUG;
		3259	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
		3260
		3261	ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3262	}
		3263
		3264	/*
		3265	* Enable digital hotplug on the CPU, and configure the DP short pulse
		3266	* duration to 2ms (which is the minimum in the Display Port spec)
		3267	* The pulse duration bits are reserved on HSW+.
		3268	*/
		3269	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
		3270	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
		3271	hotplug \|= DIGITAL_PORTA_HOTPLUG_ENABLE \| DIGITAL_PORTA_PULSE_DURATION_2ms;
		3272	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
		3273
		3274	ibx_hpd_irq_setup(dev);
		3275	}
		3276
		3277	static void bxt_hpd_irq_setup(struct drm_device *dev)
		3278	{
		3279	struct drm_i915_private *dev_priv = dev->dev_private;
		3280	u32 hotplug_irqs, hotplug, enabled_irqs;
		3281
		3282	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
		3283	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
		3284
		3285	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3286
		3287	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3288	hotplug \|= PORTC_HOTPLUG_ENABLE \| PORTB_HOTPLUG_ENABLE \|
		3289	PORTA_HOTPLUG_ENABLE;
		3290	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3291	}
		3292
3480	Serge	3293	static void ibx_irq_postinstall(struct drm_device *dev)
		3294	{
5060	serge	3295	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	3296	u32 mask;
		3297
3746	Serge	3298	if (HAS_PCH_NOP(dev))
		3299	return;
		3300
5060	serge	3301	if (HAS_PCH_IBX(dev))
		3302	mask = SDE_GMBUS \| SDE_AUX_MASK \| SDE_POISON;
		3303	else
		3304	mask = SDE_GMBUS_CPT \| SDE_AUX_MASK_CPT;
4104	Serge	3305
6084	serge	3306	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
3480	Serge	3307	I915_WRITE(SDEIMR, ~mask);
		3308	}
		3309
4104	Serge	3310	static void gen5_gt_irq_postinstall(struct drm_device *dev)
2351	Serge	3311	{
4104	Serge	3312	struct drm_i915_private *dev_priv = dev->dev_private;
		3313	u32 pm_irqs, gt_irqs;
2351	Serge	3314
4104	Serge	3315	pm_irqs = gt_irqs = 0;
2351	Serge	3316
		3317	dev_priv->gt_irq_mask = ~0;
4560	Serge	3318	if (HAS_L3_DPF(dev)) {
4104	Serge	3319	/* L3 parity interrupt is always unmasked. */
4560	Serge	3320	dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		3321	gt_irqs \|= GT_PARITY_ERROR(dev);
4104	Serge	3322	}
2351	Serge	3323
4104	Serge	3324	gt_irqs \|= GT_RENDER_USER_INTERRUPT;
		3325	if (IS_GEN5(dev)) {
		3326	gt_irqs \|= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT \|
		3327	ILK_BSD_USER_INTERRUPT;
		3328	} else {
		3329	gt_irqs \|= GT_BLT_USER_INTERRUPT \| GT_BSD_USER_INTERRUPT;
		3330	}
2351	Serge	3331
5060	serge	3332	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
2351	Serge	3333
4104	Serge	3334	if (INTEL_INFO(dev)->gen >= 6) {
5354	serge	3335	/*
		3336	* RPS interrupts will get enabled/disabled on demand when RPS
		3337	* itself is enabled/disabled.
		3338	*/
4104	Serge	3339	if (HAS_VEBOX(dev))
		3340	pm_irqs \|= PM_VEBOX_USER_INTERRUPT;
		3341
		3342	dev_priv->pm_irq_mask = 0xffffffff;
5060	serge	3343	GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
6084	serge	3344	}
2351	Serge	3345	}
		3346
4104	Serge	3347	static int ironlake_irq_postinstall(struct drm_device *dev)
3031	serge	3348	{
5060	serge	3349	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3350	u32 display_mask, extra_mask;
		3351
		3352	if (INTEL_INFO(dev)->gen >= 7) {
		3353	display_mask = (DE_MASTER_IRQ_CONTROL \| DE_GSE_IVB \|
		3354	DE_PCH_EVENT_IVB \| DE_PLANEC_FLIP_DONE_IVB \|
6084	serge	3355	DE_PLANEB_FLIP_DONE_IVB \|
5060	serge	3356	DE_PLANEA_FLIP_DONE_IVB \| DE_AUX_CHANNEL_A_IVB);
4104	Serge	3357	extra_mask = (DE_PIPEC_VBLANK_IVB \| DE_PIPEB_VBLANK_IVB \|
6084	serge	3358	DE_PIPEA_VBLANK_IVB \| DE_ERR_INT_IVB \|
		3359	DE_DP_A_HOTPLUG_IVB);
4104	Serge	3360	} else {
		3361	display_mask = (DE_MASTER_IRQ_CONTROL \| DE_GSE \| DE_PCH_EVENT \|
		3362	DE_PLANEA_FLIP_DONE \| DE_PLANEB_FLIP_DONE \|
4560	Serge	3363	DE_AUX_CHANNEL_A \|
		3364	DE_PIPEB_CRC_DONE \| DE_PIPEA_CRC_DONE \|
		3365	DE_POISON);
6084	serge	3366	extra_mask = (DE_PIPEA_VBLANK \| DE_PIPEB_VBLANK \| DE_PCU_EVENT \|
		3367	DE_PIPEB_FIFO_UNDERRUN \| DE_PIPEA_FIFO_UNDERRUN \|
		3368	DE_DP_A_HOTPLUG);
4104	Serge	3369	}
		3370
3031	serge	3371	dev_priv->irq_mask = ~display_mask;
		3372
5060	serge	3373	I915_WRITE(HWSTAM, 0xeffe);
3031	serge	3374
5060	serge	3375	ibx_irq_pre_postinstall(dev);
		3376
		3377	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask \| extra_mask);
		3378
4104	Serge	3379	gen5_gt_irq_postinstall(dev);
3031	serge	3380
4104	Serge	3381	ibx_irq_postinstall(dev);
3031	serge	3382
4104	Serge	3383	if (IS_IRONLAKE_M(dev)) {
		3384	/* Enable PCU event interrupts
		3385	*
		3386	* spinlocking not required here for correctness since interrupt
		3387	* setup is guaranteed to run in single-threaded context. But we
		3388	* need it to make the assert_spin_locked happy. */
5354	serge	3389	spin_lock_irq(&dev_priv->irq_lock);
4104	Serge	3390	ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
5354	serge	3391	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	3392	}
3031	serge	3393
		3394	return 0;
		3395	}
		3396
5060	serge	3397	static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
		3398	{
		3399	u32 pipestat_mask;
		3400	u32 iir_mask;
5354	serge	3401	enum pipe pipe;
5060	serge	3402
		3403	pipestat_mask = PIPESTAT_INT_STATUS_MASK \|
		3404	PIPE_FIFO_UNDERRUN_STATUS;
		3405
5354	serge	3406	for_each_pipe(dev_priv, pipe)
		3407	I915_WRITE(PIPESTAT(pipe), pipestat_mask);
5060	serge	3408	POSTING_READ(PIPESTAT(PIPE_A));
		3409
		3410	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV \|
		3411	PIPE_CRC_DONE_INTERRUPT_STATUS;
		3412
5354	serge	3413	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		3414	for_each_pipe(dev_priv, pipe)
		3415	i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
5060	serge	3416
		3417	iir_mask = I915_DISPLAY_PORT_INTERRUPT \|
		3418	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3419	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
5354	serge	3420	if (IS_CHERRYVIEW(dev_priv))
		3421	iir_mask \|= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
5060	serge	3422	dev_priv->irq_mask &= ~iir_mask;
		3423
		3424	I915_WRITE(VLV_IIR, iir_mask);
		3425	I915_WRITE(VLV_IIR, iir_mask);
5354	serge	3426	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
5060	serge	3427	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5354	serge	3428	POSTING_READ(VLV_IMR);
5060	serge	3429	}
		3430
		3431	static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
		3432	{
		3433	u32 pipestat_mask;
		3434	u32 iir_mask;
5354	serge	3435	enum pipe pipe;
5060	serge	3436
		3437	iir_mask = I915_DISPLAY_PORT_INTERRUPT \|
		3438	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3439	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
5354	serge	3440	if (IS_CHERRYVIEW(dev_priv))
		3441	iir_mask \|= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
5060	serge	3442
		3443	dev_priv->irq_mask \|= iir_mask;
5354	serge	3444	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5060	serge	3445	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
		3446	I915_WRITE(VLV_IIR, iir_mask);
		3447	I915_WRITE(VLV_IIR, iir_mask);
		3448	POSTING_READ(VLV_IIR);
		3449
		3450	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV \|
		3451	PIPE_CRC_DONE_INTERRUPT_STATUS;
		3452
5354	serge	3453	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		3454	for_each_pipe(dev_priv, pipe)
		3455	i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
5060	serge	3456
		3457	pipestat_mask = PIPESTAT_INT_STATUS_MASK \|
		3458	PIPE_FIFO_UNDERRUN_STATUS;
5354	serge	3459
		3460	for_each_pipe(dev_priv, pipe)
		3461	I915_WRITE(PIPESTAT(pipe), pipestat_mask);
5060	serge	3462	POSTING_READ(PIPESTAT(PIPE_A));
		3463	}
		3464
		3465	void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
		3466	{
		3467	assert_spin_locked(&dev_priv->irq_lock);
		3468
		3469	if (dev_priv->display_irqs_enabled)
		3470	return;
		3471
		3472	dev_priv->display_irqs_enabled = true;
		3473
5354	serge	3474	if (intel_irqs_enabled(dev_priv))
5060	serge	3475	valleyview_display_irqs_install(dev_priv);
		3476	}
		3477
		3478	void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
		3479	{
		3480	assert_spin_locked(&dev_priv->irq_lock);
		3481
		3482	if (!dev_priv->display_irqs_enabled)
		3483	return;
		3484
		3485	dev_priv->display_irqs_enabled = false;
		3486
5354	serge	3487	if (intel_irqs_enabled(dev_priv))
5060	serge	3488	valleyview_display_irqs_uninstall(dev_priv);
		3489	}
		3490
5354	serge	3491	static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3031	serge	3492	{
5060	serge	3493	dev_priv->irq_mask = ~0;
3031	serge	3494
6084	serge	3495	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	3496	POSTING_READ(PORT_HOTPLUG_EN);
		3497
5354	serge	3498	I915_WRITE(VLV_IIR, 0xffffffff);
		3499	I915_WRITE(VLV_IIR, 0xffffffff);
		3500	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3031	serge	3501	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5354	serge	3502	POSTING_READ(VLV_IMR);
3031	serge	3503
4104	Serge	3504	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3505	* just to make the assert_spin_locked check happy. */
5354	serge	3506	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3507	if (dev_priv->display_irqs_enabled)
		3508	valleyview_display_irqs_install(dev_priv);
5354	serge	3509	spin_unlock_irq(&dev_priv->irq_lock);
		3510	}
3031	serge	3511
5354	serge	3512	static int valleyview_irq_postinstall(struct drm_device *dev)
		3513	{
		3514	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3515
5354	serge	3516	vlv_display_irq_postinstall(dev_priv);
		3517
4104	Serge	3518	gen5_gt_irq_postinstall(dev);
3243	Serge	3519
3031	serge	3520	/* ack & enable invalid PTE error interrupts */
		3521	#if 0 /* FIXME: add support to irq handler for checking these bits */
		3522	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
		3523	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
		3524	#endif
		3525
		3526	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3480	Serge	3527
		3528	return 0;
		3529	}
		3530
4560	Serge	3531	static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
		3532	{
		3533	/* These are interrupts we'll toggle with the ring mask register */
		3534	uint32_t gt_interrupts[] = {
		3535	GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT \|
5354	serge	3536	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT \|
4560	Serge	3537	GT_RENDER_L3_PARITY_ERROR_INTERRUPT \|
5354	serge	3538	GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT \|
		3539	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
4560	Serge	3540	GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT \|
5354	serge	3541	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT \|
		3542	GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT \|
		3543	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
4560	Serge	3544	0,
5354	serge	3545	GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT \|
		3546	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
4560	Serge	3547	};
		3548
5060	serge	3549	dev_priv->pm_irq_mask = 0xffffffff;
5354	serge	3550	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
		3551	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
		3552	/*
		3553	* RPS interrupts will get enabled/disabled on demand when RPS itself
		3554	* is enabled/disabled.
		3555	*/
		3556	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
		3557	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
4560	Serge	3558	}
		3559
		3560	static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
		3561	{
5354	serge	3562	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
		3563	uint32_t de_pipe_enables;
6084	serge	3564	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
		3565	u32 de_port_enables;
		3566	enum pipe pipe;
5354	serge	3567
6084	serge	3568	if (INTEL_INFO(dev_priv)->gen >= 9) {
5354	serge	3569	de_pipe_masked \|= GEN9_PIPE_PLANE1_FLIP_DONE \|
		3570	GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
6084	serge	3571	de_port_masked \|= GEN9_AUX_CHANNEL_B \| GEN9_AUX_CHANNEL_C \|
		3572	GEN9_AUX_CHANNEL_D;
		3573	if (IS_BROXTON(dev_priv))
		3574	de_port_masked \|= BXT_DE_PORT_GMBUS;
		3575	} else {
5354	serge	3576	de_pipe_masked \|= GEN8_PIPE_PRIMARY_FLIP_DONE \|
6084	serge	3577	GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
		3578	}
5354	serge	3579
		3580	de_pipe_enables = de_pipe_masked \| GEN8_PIPE_VBLANK \|
6084	serge	3581	GEN8_PIPE_FIFO_UNDERRUN;
5354	serge	3582
6084	serge	3583	de_port_enables = de_port_masked;
		3584	if (IS_BROXTON(dev_priv))
		3585	de_port_enables \|= BXT_DE_PORT_HOTPLUG_MASK;
		3586	else if (IS_BROADWELL(dev_priv))
		3587	de_port_enables \|= GEN8_PORT_DP_A_HOTPLUG;
		3588
4560	Serge	3589	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
		3590	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
		3591	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
		3592
5354	serge	3593	for_each_pipe(dev_priv, pipe)
		3594	if (intel_display_power_is_enabled(dev_priv,
5060	serge	3595	POWER_DOMAIN_PIPE(pipe)))
		3596	GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
		3597	dev_priv->de_irq_mask[pipe],
6084	serge	3598	de_pipe_enables);
4560	Serge	3599
6084	serge	3600	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
4560	Serge	3601	}
		3602
		3603	static int gen8_irq_postinstall(struct drm_device *dev)
		3604	{
		3605	struct drm_i915_private *dev_priv = dev->dev_private;
		3606
6084	serge	3607	if (HAS_PCH_SPLIT(dev))
		3608	ibx_irq_pre_postinstall(dev);
5060	serge	3609
4560	Serge	3610	gen8_gt_irq_postinstall(dev_priv);
		3611	gen8_de_irq_postinstall(dev_priv);
		3612
6084	serge	3613	if (HAS_PCH_SPLIT(dev))
		3614	ibx_irq_postinstall(dev);
4560	Serge	3615
		3616	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		3617	POSTING_READ(GEN8_MASTER_IRQ);
		3618
		3619	return 0;
		3620	}
		3621
5060	serge	3622	static int cherryview_irq_postinstall(struct drm_device *dev)
4560	Serge	3623	{
		3624	struct drm_i915_private *dev_priv = dev->dev_private;
		3625
5354	serge	3626	vlv_display_irq_postinstall(dev_priv);
4560	Serge	3627
5060	serge	3628	gen8_gt_irq_postinstall(dev_priv);
4560	Serge	3629
5060	serge	3630	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
		3631	POSTING_READ(GEN8_MASTER_IRQ);
4560	Serge	3632
5060	serge	3633	return 0;
		3634	}
4560	Serge	3635
5060	serge	3636	static void gen8_irq_uninstall(struct drm_device *dev)
		3637	{
		3638	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3639
5060	serge	3640	if (!dev_priv)
		3641	return;
		3642
		3643	gen8_irq_reset(dev);
4560	Serge	3644	}
		3645
5354	serge	3646	static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
		3647	{
		3648	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3649	* just to make the assert_spin_locked check happy. */
		3650	spin_lock_irq(&dev_priv->irq_lock);
		3651	if (dev_priv->display_irqs_enabled)
		3652	valleyview_display_irqs_uninstall(dev_priv);
		3653	spin_unlock_irq(&dev_priv->irq_lock);
		3654
		3655	vlv_display_irq_reset(dev_priv);
		3656
		3657	dev_priv->irq_mask = ~0;
		3658	}
		3659
3031	serge	3660	static void valleyview_irq_uninstall(struct drm_device *dev)
		3661	{
5060	serge	3662	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3663
		3664	if (!dev_priv)
		3665	return;
		3666
5060	serge	3667	I915_WRITE(VLV_MASTER_IER, 0);
4293	Serge	3668
5354	serge	3669	gen5_gt_irq_reset(dev);
3031	serge	3670
		3671	I915_WRITE(HWSTAM, 0xffffffff);
5060	serge	3672
5354	serge	3673	vlv_display_irq_uninstall(dev_priv);
3031	serge	3674	}
		3675
5060	serge	3676	static void cherryview_irq_uninstall(struct drm_device *dev)
3031	serge	3677	{
5060	serge	3678	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3679
		3680	if (!dev_priv)
		3681	return;
		3682
5060	serge	3683	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3684	POSTING_READ(GEN8_MASTER_IRQ);
4293	Serge	3685
5354	serge	3686	gen8_gt_irq_reset(dev_priv);
3031	serge	3687
5354	serge	3688	GEN5_IRQ_RESET(GEN8_PCU_);
3031	serge	3689
5354	serge	3690	vlv_display_irq_uninstall(dev_priv);
5060	serge	3691	}
		3692
		3693	static void ironlake_irq_uninstall(struct drm_device *dev)
		3694	{
		3695	struct drm_i915_private *dev_priv = dev->dev_private;
		3696
		3697	if (!dev_priv)
3746	Serge	3698	return;
		3699
5060	serge	3700	ironlake_irq_reset(dev);
3031	serge	3701	}
		3702
		3703	#if 0
		3704	static void i8xx_irq_preinstall(struct drm_device * dev)
		3705	{
5060	serge	3706	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3707	int pipe;
		3708
5354	serge	3709	for_each_pipe(dev_priv, pipe)
3031	serge	3710	I915_WRITE(PIPESTAT(pipe), 0);
		3711	I915_WRITE16(IMR, 0xffff);
		3712	I915_WRITE16(IER, 0x0);
		3713	POSTING_READ16(IER);
		3714	}
		3715
		3716	static int i8xx_irq_postinstall(struct drm_device *dev)
		3717	{
5060	serge	3718	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3719
		3720	I915_WRITE16(EMR,
		3721	~(I915_ERROR_PAGE_TABLE \| I915_ERROR_MEMORY_REFRESH));
		3722
		3723	/* Unmask the interrupts that we always want on. */
		3724	dev_priv->irq_mask =
		3725	~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3726	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3727	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
6084	serge	3728	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	3729	I915_WRITE16(IMR, dev_priv->irq_mask);
		3730
		3731	I915_WRITE16(IER,
		3732	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3733	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3734	I915_USER_INTERRUPT);
		3735	POSTING_READ16(IER);
		3736
4560	Serge	3737	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3738	* just to make the assert_spin_locked check happy. */
5354	serge	3739	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3740	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		3741	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	3742	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	3743
3031	serge	3744	return 0;
		3745	}
		3746
3746	Serge	3747	/*
		3748	* Returns true when a page flip has completed.
		3749	*/
		3750	static bool i8xx_handle_vblank(struct drm_device *dev,
4560	Serge	3751	int plane, int pipe, u32 iir)
3746	Serge	3752	{
5060	serge	3753	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3754	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3746	Serge	3755
6084	serge	3756	if (!intel_pipe_handle_vblank(dev, pipe))
		3757	return false;
3746	Serge	3758
		3759	if ((iir & flip_pending) == 0)
5354	serge	3760	goto check_page_flip;
3746	Serge	3761
		3762	/* We detect FlipDone by looking for the change in PendingFlip from '1'
		3763	* to '0' on the following vblank, i.e. IIR has the Pendingflip
		3764	* asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
		3765	* the flip is completed (no longer pending). Since this doesn't raise
		3766	* an interrupt per se, we watch for the change at vblank.
		3767	*/
		3768	if (I915_READ16(ISR) & flip_pending)
5354	serge	3769	goto check_page_flip;
3746	Serge	3770
6320	serge	3771	intel_prepare_page_flip(dev, plane);
		3772	intel_finish_page_flip(dev, pipe);
5354	serge	3773	return true;
3746	Serge	3774
5354	serge	3775	check_page_flip:
6320	serge	3776	intel_check_page_flip(dev, pipe);
5354	serge	3777	return false;
3746	Serge	3778	}
		3779
3243	Serge	3780	static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3031	serge	3781	{
5060	serge	3782	struct drm_device *dev = arg;
		3783	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3784	u16 iir, new_iir;
		3785	u32 pipe_stats[2];
		3786	int pipe;
		3787	u16 flip_mask =
		3788	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		3789	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
		3790
6084	serge	3791	if (!intel_irqs_enabled(dev_priv))
		3792	return IRQ_NONE;
		3793
3031	serge	3794	iir = I915_READ16(IIR);
		3795	if (iir == 0)
		3796	return IRQ_NONE;
		3797
		3798	while (iir & ~flip_mask) {
		3799	/* Can't rely on pipestat interrupt bit in iir as it might
		3800	* have been cleared after the pipestat interrupt was received.
		3801	* It doesn't set the bit in iir again, but it still produces
		3802	* interrupts (for non-MSI).
		3803	*/
5354	serge	3804	spin_lock(&dev_priv->irq_lock);
4126	Serge	3805	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	3806	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	3807
5354	serge	3808	for_each_pipe(dev_priv, pipe) {
3031	serge	3809	int reg = PIPESTAT(pipe);
		3810	pipe_stats[pipe] = I915_READ(reg);
		3811
		3812	/*
		3813	* Clear the PIPE*STAT regs before the IIR
		3814	*/
5060	serge	3815	if (pipe_stats[pipe] & 0x8000ffff)
3031	serge	3816	I915_WRITE(reg, pipe_stats[pipe]);
6084	serge	3817	}
5354	serge	3818	spin_unlock(&dev_priv->irq_lock);
3031	serge	3819
		3820	I915_WRITE16(IIR, iir & ~flip_mask);
		3821	new_iir = I915_READ16(IIR); /* Flush posted writes */
		3822
		3823	if (iir & I915_USER_INTERRUPT)
6084	serge	3824	notify_ring(&dev_priv->ring[RCS]);
3031	serge	3825
5354	serge	3826	for_each_pipe(dev_priv, pipe) {
4560	Serge	3827	int plane = pipe;
		3828	if (HAS_FBC(dev))
		3829	plane = !plane;
3031	serge	3830
4560	Serge	3831	if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
		3832	i8xx_handle_vblank(dev, plane, pipe, iir))
		3833	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3031	serge	3834
4560	Serge	3835	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		3836	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	3837
5354	serge	3838	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		3839	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		3840	pipe);
4560	Serge	3841	}
		3842
3031	serge	3843	iir = new_iir;
		3844	}
		3845
		3846	return IRQ_HANDLED;
		3847	}
		3848
		3849	static void i8xx_irq_uninstall(struct drm_device * dev)
		3850	{
5060	serge	3851	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3852	int pipe;
		3853
5354	serge	3854	for_each_pipe(dev_priv, pipe) {
3031	serge	3855	/* Clear enable bits; then clear status bits */
		3856	I915_WRITE(PIPESTAT(pipe), 0);
		3857	I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
		3858	}
		3859	I915_WRITE16(IMR, 0xffff);
		3860	I915_WRITE16(IER, 0x0);
		3861	I915_WRITE16(IIR, I915_READ16(IIR));
		3862	}
		3863
		3864	#endif
		3865
		3866	static void i915_irq_preinstall(struct drm_device * dev)
		3867	{
5060	serge	3868	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3869	int pipe;
		3870
		3871	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	3872	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	3873	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		3874	}
		3875
		3876	I915_WRITE16(HWSTAM, 0xeffe);
5354	serge	3877	for_each_pipe(dev_priv, pipe)
3031	serge	3878	I915_WRITE(PIPESTAT(pipe), 0);
		3879	I915_WRITE(IMR, 0xffffffff);
		3880	I915_WRITE(IER, 0x0);
		3881	POSTING_READ(IER);
		3882	}
		3883
		3884	static int i915_irq_postinstall(struct drm_device *dev)
		3885	{
5060	serge	3886	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3887	u32 enable_mask;
		3888
		3889	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE \| I915_ERROR_MEMORY_REFRESH));
		3890
		3891	/* Unmask the interrupts that we always want on. */
		3892	dev_priv->irq_mask =
		3893	~(I915_ASLE_INTERRUPT \|
		3894	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3895	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3896	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
6084	serge	3897	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	3898
		3899	enable_mask =
		3900	I915_ASLE_INTERRUPT \|
		3901	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3902	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3903	I915_USER_INTERRUPT;
3480	Serge	3904
3031	serge	3905	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	3906	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	3907	POSTING_READ(PORT_HOTPLUG_EN);
		3908
3031	serge	3909	/* Enable in IER... */
		3910	enable_mask \|= I915_DISPLAY_PORT_INTERRUPT;
		3911	/* and unmask in IMR */
		3912	dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
		3913	}
		3914
		3915	I915_WRITE(IMR, dev_priv->irq_mask);
		3916	I915_WRITE(IER, enable_mask);
		3917	POSTING_READ(IER);
		3918
4126	Serge	3919	i915_enable_asle_pipestat(dev);
3480	Serge	3920
4560	Serge	3921	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3922	* just to make the assert_spin_locked check happy. */
5354	serge	3923	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3924	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		3925	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	3926	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	3927
3480	Serge	3928	return 0;
		3929	}
		3930
3746	Serge	3931	/*
		3932	* Returns true when a page flip has completed.
		3933	*/
		3934	static bool i915_handle_vblank(struct drm_device *dev,
		3935	int plane, int pipe, u32 iir)
3480	Serge	3936	{
5060	serge	3937	struct drm_i915_private *dev_priv = dev->dev_private;
3746	Serge	3938	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3480	Serge	3939
6088	serge	3940	if (!intel_pipe_handle_vblank(dev, pipe))
		3941	return false;
3480	Serge	3942
3746	Serge	3943	if ((iir & flip_pending) == 0)
5354	serge	3944	goto check_page_flip;
3480	Serge	3945
3746	Serge	3946	/* We detect FlipDone by looking for the change in PendingFlip from '1'
		3947	* to '0' on the following vblank, i.e. IIR has the Pendingflip
		3948	* asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
		3949	* the flip is completed (no longer pending). Since this doesn't raise
		3950	* an interrupt per se, we watch for the change at vblank.
		3951	*/
		3952	if (I915_READ(ISR) & flip_pending)
5354	serge	3953	goto check_page_flip;
3746	Serge	3954
6320	serge	3955	intel_prepare_page_flip(dev, plane);
		3956	intel_finish_page_flip(dev, pipe);
5354	serge	3957	return true;
3746	Serge	3958
5354	serge	3959	check_page_flip:
6320	serge	3960	intel_check_page_flip(dev, pipe);
5354	serge	3961	return false;
3031	serge	3962	}
		3963
3243	Serge	3964	static irqreturn_t i915_irq_handler(int irq, void *arg)
3031	serge	3965	{
5060	serge	3966	struct drm_device *dev = arg;
		3967	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3968	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
		3969	u32 flip_mask =
		3970	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		3971	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
		3972	int pipe, ret = IRQ_NONE;
		3973
6084	serge	3974	if (!intel_irqs_enabled(dev_priv))
		3975	return IRQ_NONE;
		3976
3031	serge	3977	iir = I915_READ(IIR);
		3978	do {
		3979	bool irq_received = (iir & ~flip_mask) != 0;
		3980	bool blc_event = false;
		3981
		3982	/* Can't rely on pipestat interrupt bit in iir as it might
		3983	* have been cleared after the pipestat interrupt was received.
		3984	* It doesn't set the bit in iir again, but it still produces
		3985	* interrupts (for non-MSI).
		3986	*/
5354	serge	3987	spin_lock(&dev_priv->irq_lock);
4126	Serge	3988	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	3989	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	3990
5354	serge	3991	for_each_pipe(dev_priv, pipe) {
3031	serge	3992	int reg = PIPESTAT(pipe);
		3993	pipe_stats[pipe] = I915_READ(reg);
		3994
		3995	/* Clear the PIPESTAT regs before the IIR /
		3996	if (pipe_stats[pipe] & 0x8000ffff) {
		3997	I915_WRITE(reg, pipe_stats[pipe]);
		3998	irq_received = true;
		3999	}
		4000	}
5354	serge	4001	spin_unlock(&dev_priv->irq_lock);
3031	serge	4002
		4003	if (!irq_received)
		4004	break;
		4005
		4006	/* Consume port. Then clear IIR or we'll miss events */
5060	serge	4007	if (I915_HAS_HOTPLUG(dev) &&
		4008	iir & I915_DISPLAY_PORT_INTERRUPT)
		4009	i9xx_hpd_irq_handler(dev);
3031	serge	4010
		4011	I915_WRITE(IIR, iir & ~flip_mask);
		4012	new_iir = I915_READ(IIR); /* Flush posted writes */
		4013
		4014	if (iir & I915_USER_INTERRUPT)
6084	serge	4015	notify_ring(&dev_priv->ring[RCS]);
3031	serge	4016
5354	serge	4017	for_each_pipe(dev_priv, pipe) {
3031	serge	4018	int plane = pipe;
4560	Serge	4019	if (HAS_FBC(dev))
3031	serge	4020	plane = !plane;
		4021
3746	Serge	4022	if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
		4023	i915_handle_vblank(dev, plane, pipe, iir))
		4024	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
		4025
3031	serge	4026	if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
		4027	blc_event = true;
4560	Serge	4028
		4029	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		4030	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	4031
5354	serge	4032	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		4033	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		4034	pipe);
3031	serge	4035	}
		4036
4126	Serge	4037	if (blc_event \|\| (iir & I915_ASLE_INTERRUPT))
		4038	intel_opregion_asle_intr(dev);
3031	serge	4039
		4040	/* With MSI, interrupts are only generated when iir
		4041	* transitions from zero to nonzero. If another bit got
		4042	* set while we were handling the existing iir bits, then
		4043	* we would never get another interrupt.
		4044	*
		4045	* This is fine on non-MSI as well, as if we hit this path
		4046	* we avoid exiting the interrupt handler only to generate
		4047	* another one.
		4048	*
		4049	* Note that for MSI this could cause a stray interrupt report
		4050	* if an interrupt landed in the time between writing IIR and
		4051	* the posting read. This should be rare enough to never
		4052	* trigger the 99% of 100,000 interrupts test for disabling
		4053	* stray interrupts.
		4054	*/
		4055	ret = IRQ_HANDLED;
		4056	iir = new_iir;
		4057	} while (iir & ~flip_mask);
		4058
		4059	return ret;
		4060	}
		4061
		4062	static void i915_irq_uninstall(struct drm_device * dev)
		4063	{
5060	serge	4064	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4065	int pipe;
		4066
		4067	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	4068	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4069	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4070	}
		4071
		4072	I915_WRITE16(HWSTAM, 0xffff);
5354	serge	4073	for_each_pipe(dev_priv, pipe) {
3031	serge	4074	/* Clear enable bits; then clear status bits */
		4075	I915_WRITE(PIPESTAT(pipe), 0);
		4076	I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
		4077	}
		4078	I915_WRITE(IMR, 0xffffffff);
		4079	I915_WRITE(IER, 0x0);
		4080
		4081	I915_WRITE(IIR, I915_READ(IIR));
		4082	}
		4083
		4084	static void i965_irq_preinstall(struct drm_device * dev)
		4085	{
5060	serge	4086	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4087	int pipe;
		4088
6084	serge	4089	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4090	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4091
		4092	I915_WRITE(HWSTAM, 0xeffe);
5354	serge	4093	for_each_pipe(dev_priv, pipe)
3031	serge	4094	I915_WRITE(PIPESTAT(pipe), 0);
		4095	I915_WRITE(IMR, 0xffffffff);
		4096	I915_WRITE(IER, 0x0);
		4097	POSTING_READ(IER);
		4098	}
		4099
		4100	static int i965_irq_postinstall(struct drm_device *dev)
		4101	{
5060	serge	4102	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4103	u32 enable_mask;
		4104	u32 error_mask;
		4105
		4106	/* Unmask the interrupts that we always want on. */
		4107	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT \|
		4108	I915_DISPLAY_PORT_INTERRUPT \|
		4109	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		4110	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		4111	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4112	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT \|
		4113	I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
		4114
		4115	enable_mask = ~dev_priv->irq_mask;
3746	Serge	4116	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4117	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	4118	enable_mask \|= I915_USER_INTERRUPT;
		4119
		4120	if (IS_G4X(dev))
		4121	enable_mask \|= I915_BSD_USER_INTERRUPT;
		4122
4104	Serge	4123	/* Interrupt setup is already guaranteed to be single-threaded, this is
		4124	* just to make the assert_spin_locked check happy. */
5354	serge	4125	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	4126	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		4127	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		4128	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	4129	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	4130
		4131	/*
		4132	* Enable some error detection, note the instruction error mask
		4133	* bit is reserved, so we leave it masked.
		4134	*/
		4135	if (IS_G4X(dev)) {
		4136	error_mask = ~(GM45_ERROR_PAGE_TABLE \|
		4137	GM45_ERROR_MEM_PRIV \|
		4138	GM45_ERROR_CP_PRIV \|
		4139	I915_ERROR_MEMORY_REFRESH);
		4140	} else {
		4141	error_mask = ~(I915_ERROR_PAGE_TABLE \|
		4142	I915_ERROR_MEMORY_REFRESH);
		4143	}
		4144	I915_WRITE(EMR, error_mask);
		4145
		4146	I915_WRITE(IMR, dev_priv->irq_mask);
		4147	I915_WRITE(IER, enable_mask);
		4148	POSTING_READ(IER);
		4149
6084	serge	4150	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	4151	POSTING_READ(PORT_HOTPLUG_EN);
		4152
4126	Serge	4153	i915_enable_asle_pipestat(dev);
3480	Serge	4154
		4155	return 0;
		4156	}
		4157
3746	Serge	4158	static void i915_hpd_irq_setup(struct drm_device *dev)
3480	Serge	4159	{
5060	serge	4160	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	4161	u32 hotplug_en;
		4162
4104	Serge	4163	assert_spin_locked(&dev_priv->irq_lock);
		4164
3031	serge	4165	/* Note HDMI and DP share hotplug bits */
6084	serge	4166	/* enable bits are the same for all generations */
		4167	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
		4168	/* Programming the CRT detection parameters tends
		4169	to generate a spurious hotplug event about three
		4170	seconds later. So just do it once.
		4171	*/
		4172	if (IS_G4X(dev))
		4173	hotplug_en \|= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
		4174	hotplug_en \|= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3480	Serge	4175
3031	serge	4176	/* Ignore TV since it's buggy */
6084	serge	4177	i915_hotplug_interrupt_update_locked(dev_priv,
		4178	HOTPLUG_INT_EN_MASK \|
		4179	CRT_HOTPLUG_VOLTAGE_COMPARE_MASK \|
		4180	CRT_HOTPLUG_ACTIVATION_PERIOD_64,
		4181	hotplug_en);
3031	serge	4182	}
		4183
3243	Serge	4184	static irqreturn_t i965_irq_handler(int irq, void *arg)
3031	serge	4185	{
5060	serge	4186	struct drm_device *dev = arg;
		4187	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4188	u32 iir, new_iir;
		4189	u32 pipe_stats[I915_MAX_PIPES];
		4190	int ret = IRQ_NONE, pipe;
3746	Serge	4191	u32 flip_mask =
		4192	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4193	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3031	serge	4194
6084	serge	4195	if (!intel_irqs_enabled(dev_priv))
		4196	return IRQ_NONE;
		4197
3031	serge	4198	iir = I915_READ(IIR);
		4199
		4200	for (;;) {
5060	serge	4201	bool irq_received = (iir & ~flip_mask) != 0;
3031	serge	4202	bool blc_event = false;
		4203
		4204	/* Can't rely on pipestat interrupt bit in iir as it might
		4205	* have been cleared after the pipestat interrupt was received.
		4206	* It doesn't set the bit in iir again, but it still produces
		4207	* interrupts (for non-MSI).
		4208	*/
5354	serge	4209	spin_lock(&dev_priv->irq_lock);
4126	Serge	4210	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	4211	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	4212
5354	serge	4213	for_each_pipe(dev_priv, pipe) {
3031	serge	4214	int reg = PIPESTAT(pipe);
		4215	pipe_stats[pipe] = I915_READ(reg);
		4216
		4217	/*
		4218	* Clear the PIPE*STAT regs before the IIR
		4219	*/
		4220	if (pipe_stats[pipe] & 0x8000ffff) {
		4221	I915_WRITE(reg, pipe_stats[pipe]);
5060	serge	4222	irq_received = true;
3031	serge	4223	}
		4224	}
5354	serge	4225	spin_unlock(&dev_priv->irq_lock);
3031	serge	4226
		4227	if (!irq_received)
		4228	break;
		4229
		4230	ret = IRQ_HANDLED;
		4231
		4232	/* Consume port. Then clear IIR or we'll miss events */
5060	serge	4233	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		4234	i9xx_hpd_irq_handler(dev);
3031	serge	4235
3746	Serge	4236	I915_WRITE(IIR, iir & ~flip_mask);
3031	serge	4237	new_iir = I915_READ(IIR); /* Flush posted writes */
		4238
		4239	if (iir & I915_USER_INTERRUPT)
6084	serge	4240	notify_ring(&dev_priv->ring[RCS]);
3031	serge	4241	if (iir & I915_BSD_USER_INTERRUPT)
6084	serge	4242	notify_ring(&dev_priv->ring[VCS]);
3031	serge	4243
5354	serge	4244	for_each_pipe(dev_priv, pipe) {
3746	Serge	4245	if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		4246	i915_handle_vblank(dev, pipe, pipe, iir))
		4247	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3031	serge	4248
		4249	if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
		4250	blc_event = true;
4560	Serge	4251
		4252	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		4253	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	4254
5354	serge	4255	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		4256	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
3031	serge	4257	}
		4258
4126	Serge	4259	if (blc_event \|\| (iir & I915_ASLE_INTERRUPT))
		4260	intel_opregion_asle_intr(dev);
3031	serge	4261
3480	Serge	4262	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		4263	gmbus_irq_handler(dev);
		4264
3031	serge	4265	/* With MSI, interrupts are only generated when iir
		4266	* transitions from zero to nonzero. If another bit got
		4267	* set while we were handling the existing iir bits, then
		4268	* we would never get another interrupt.
		4269	*
		4270	* This is fine on non-MSI as well, as if we hit this path
		4271	* we avoid exiting the interrupt handler only to generate
		4272	* another one.
		4273	*
		4274	* Note that for MSI this could cause a stray interrupt report
		4275	* if an interrupt landed in the time between writing IIR and
		4276	* the posting read. This should be rare enough to never
		4277	* trigger the 99% of 100,000 interrupts test for disabling
		4278	* stray interrupts.
		4279	*/
		4280	iir = new_iir;
		4281	}
		4282
		4283	return ret;
		4284	}
		4285
		4286	static void i965_irq_uninstall(struct drm_device * dev)
		4287	{
5060	serge	4288	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4289	int pipe;
		4290
		4291	if (!dev_priv)
		4292	return;
		4293
6084	serge	4294	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4295	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4296
		4297	I915_WRITE(HWSTAM, 0xffffffff);
5354	serge	4298	for_each_pipe(dev_priv, pipe)
3031	serge	4299	I915_WRITE(PIPESTAT(pipe), 0);
		4300	I915_WRITE(IMR, 0xffffffff);
		4301	I915_WRITE(IER, 0x0);
		4302
5354	serge	4303	for_each_pipe(dev_priv, pipe)
3031	serge	4304	I915_WRITE(PIPESTAT(pipe),
		4305	I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
		4306	I915_WRITE(IIR, I915_READ(IIR));
		4307	}
		4308
5354	serge	4309	/**
		4310	* intel_irq_init - initializes irq support
		4311	* @dev_priv: i915 device instance
		4312	*
		4313	* This function initializes all the irq support including work items, timers
		4314	* and all the vtables. It does not setup the interrupt itself though.
		4315	*/
		4316	void intel_irq_init(struct drm_i915_private *dev_priv)
2351	Serge	4317	{
5354	serge	4318	struct drm_device *dev = dev_priv->dev;
3031	serge	4319
6296	serge	4320	intel_hpd_init_work(dev_priv);
6084	serge	4321
4126	Serge	4322	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
		4323	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3480	Serge	4324
5060	serge	4325	/* Let's track the enabled rps events */
5354	serge	4326	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
		4327	/* WaGsvRC0ResidencyMethod:vlv */
6935	serge	4328	dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
5060	serge	4329	else
6084	serge	4330	dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
3480	Serge	4331
6084	serge	4332	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
		4333	i915_hangcheck_elapsed);
4560	Serge	4334
5354	serge	4335
		4336	if (IS_GEN2(dev_priv)) {
4560	Serge	4337	dev->max_vblank_count = 0;
		4338	dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
5354	serge	4339	} else if (IS_G4X(dev_priv) \|\| INTEL_INFO(dev_priv)->gen >= 5) {
4560	Serge	4340	dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
6084	serge	4341	dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4560	Serge	4342	} else {
6084	serge	4343	dev->driver->get_vblank_counter = i915_get_vblank_counter;
		4344	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4293	Serge	4345	}
3480	Serge	4346
5354	serge	4347	/*
		4348	* Opt out of the vblank disable timer on everything except gen2.
		4349	* Gen2 doesn't have a hardware frame counter and so depends on
		4350	* vblank interrupts to produce sane vblank seuquence numbers.
		4351	*/
		4352	if (!IS_GEN2(dev_priv))
		4353	dev->vblank_disable_immediate = true;
		4354
6084	serge	4355	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4293	Serge	4356	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3480	Serge	4357
5354	serge	4358	if (IS_CHERRYVIEW(dev_priv)) {
5060	serge	4359	dev->driver->irq_handler = cherryview_irq_handler;
		4360	dev->driver->irq_preinstall = cherryview_irq_preinstall;
		4361	dev->driver->irq_postinstall = cherryview_irq_postinstall;
		4362	dev->driver->irq_uninstall = cherryview_irq_uninstall;
		4363	dev->driver->enable_vblank = valleyview_enable_vblank;
		4364	dev->driver->disable_vblank = valleyview_disable_vblank;
		4365	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
5354	serge	4366	} else if (IS_VALLEYVIEW(dev_priv)) {
3243	Serge	4367	dev->driver->irq_handler = valleyview_irq_handler;
		4368	dev->driver->irq_preinstall = valleyview_irq_preinstall;
		4369	dev->driver->irq_postinstall = valleyview_irq_postinstall;
4293	Serge	4370	dev->driver->irq_uninstall = valleyview_irq_uninstall;
		4371	dev->driver->enable_vblank = valleyview_enable_vblank;
		4372	dev->driver->disable_vblank = valleyview_disable_vblank;
3746	Serge	4373	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
5354	serge	4374	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4560	Serge	4375	dev->driver->irq_handler = gen8_irq_handler;
5060	serge	4376	dev->driver->irq_preinstall = gen8_irq_reset;
4560	Serge	4377	dev->driver->irq_postinstall = gen8_irq_postinstall;
		4378	dev->driver->irq_uninstall = gen8_irq_uninstall;
		4379	dev->driver->enable_vblank = gen8_enable_vblank;
		4380	dev->driver->disable_vblank = gen8_disable_vblank;
6084	serge	4381	if (IS_BROXTON(dev))
		4382	dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
		4383	else if (HAS_PCH_SPT(dev))
		4384	dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		4385	else
		4386	dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
2351	Serge	4387	} else if (HAS_PCH_SPLIT(dev)) {
3243	Serge	4388	dev->driver->irq_handler = ironlake_irq_handler;
5060	serge	4389	dev->driver->irq_preinstall = ironlake_irq_reset;
3243	Serge	4390	dev->driver->irq_postinstall = ironlake_irq_postinstall;
4293	Serge	4391	dev->driver->irq_uninstall = ironlake_irq_uninstall;
		4392	dev->driver->enable_vblank = ironlake_enable_vblank;
		4393	dev->driver->disable_vblank = ironlake_disable_vblank;
6084	serge	4394	dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
2351	Serge	4395	} else {
5354	serge	4396	if (INTEL_INFO(dev_priv)->gen == 2) {
		4397	} else if (INTEL_INFO(dev_priv)->gen == 3) {
3243	Serge	4398	dev->driver->irq_preinstall = i915_irq_preinstall;
		4399	dev->driver->irq_postinstall = i915_irq_postinstall;
4293	Serge	4400	dev->driver->irq_uninstall = i915_irq_uninstall;
3243	Serge	4401	dev->driver->irq_handler = i915_irq_handler;
3031	serge	4402	} else {
3243	Serge	4403	dev->driver->irq_preinstall = i965_irq_preinstall;
		4404	dev->driver->irq_postinstall = i965_irq_postinstall;
4293	Serge	4405	dev->driver->irq_uninstall = i965_irq_uninstall;
3243	Serge	4406	dev->driver->irq_handler = i965_irq_handler;
6084	serge	4407	}
		4408	if (I915_HAS_HOTPLUG(dev_priv))
3746	Serge	4409	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4293	Serge	4410	dev->driver->enable_vblank = i915_enable_vblank;
		4411	dev->driver->disable_vblank = i915_disable_vblank;
2351	Serge	4412	}
3480	Serge	4413	}
3243	Serge	4414
5354	serge	4415	/**
		4416	* intel_irq_install - enables the hardware interrupt
		4417	* @dev_priv: i915 device instance
		4418	*
		4419	* This function enables the hardware interrupt handling, but leaves the hotplug
		4420	* handling still disabled. It is called after intel_irq_init().
		4421	*
		4422	* In the driver load and resume code we need working interrupts in a few places
		4423	* but don't want to deal with the hassle of concurrent probe and hotplug
		4424	* workers. Hence the split into this two-stage approach.
		4425	*/
		4426	int intel_irq_install(struct drm_i915_private *dev_priv)
3243	Serge	4427	{
5354	serge	4428	/*
		4429	* We enable some interrupt sources in our postinstall hooks, so mark
		4430	* interrupts as enabled _before_ actually enabling them to avoid
		4431	* special cases in our ordering checks.
		4432	*/
		4433	dev_priv->pm.irqs_enabled = true;
2351	Serge	4434
5354	serge	4435	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
3243	Serge	4436	}
		4437
5354	serge	4438	/**
		4439	* intel_irq_uninstall - finilizes all irq handling
		4440	* @dev_priv: i915 device instance
		4441	*
		4442	* This stops interrupt and hotplug handling and unregisters and frees all
		4443	* resources acquired in the init functions.
		4444	*/
		4445	void intel_irq_uninstall(struct drm_i915_private *dev_priv)
2351	Serge	4446	{
5354	serge	4447	// drm_irq_uninstall(dev_priv->dev);
6320	serge	4448	intel_hpd_cancel_work(dev_priv);
5354	serge	4449	dev_priv->pm.irqs_enabled = false;
		4450	}
2351	Serge	4451
5354	serge	4452	/**
		4453	* intel_runtime_pm_disable_interrupts - runtime interrupt disabling
		4454	* @dev_priv: i915 device instance
		4455	*
		4456	* This function is used to disable interrupts at runtime, both in the runtime
		4457	* pm and the system suspend/resume code.
		4458	*/
		4459	void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
		4460	{
		4461	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
		4462	dev_priv->pm.irqs_enabled = false;
4104	Serge	4463	}
2351	Serge	4464
5354	serge	4465	/**
		4466	* intel_runtime_pm_enable_interrupts - runtime interrupt enabling
		4467	* @dev_priv: i915 device instance
		4468	*
		4469	* This function is used to enable interrupts at runtime, both in the runtime
		4470	* pm and the system suspend/resume code.
		4471	*/
		4472	void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
		4473	{
		4474	dev_priv->pm.irqs_enabled = true;
		4475	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
		4476	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
		4477	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_irq.c – Rev 6935