WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_irq.c

Rev	Author	Line No.	Line
2351	Serge	1	/* i915_irq.c -- IRQ support for the I915 -- linux-c --
		2	*/
		3	/*
		4	* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
		5	* All Rights Reserved.
		6	*
		7	* Permission is hereby granted, free of charge, to any person obtaining a
		8	* copy of this software and associated documentation files (the
		9	* "Software"), to deal in the Software without restriction, including
		10	* without limitation the rights to use, copy, modify, merge, publish,
		11	* distribute, sub license, and/or sell copies of the Software, and to
		12	* permit persons to whom the Software is furnished to do so, subject to
		13	* the following conditions:
		14	*
		15	* The above copyright notice and this permission notice (including the
		16	* next paragraph) shall be included in all copies or substantial portions
		17	* of the Software.
		18	*
		19	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
		20	* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
		21	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
		22	* IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
		23	* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
		24	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
		25	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
		26	*
		27	*/
		28
3746	Serge	29	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3031	serge	30
6103	serge	31	#include
3031	serge	32	#include
6088	serge	33	#include
3031	serge	34	#include
		35	#include
2351	Serge	36	#include "i915_drv.h"
		37	#include "i915_trace.h"
		38	#include "intel_drv.h"
		39
5354	serge	40	/**
		41	* DOC: interrupt handling
		42	*
		43	* These functions provide the basic support for enabling and disabling the
		44	* interrupt handling support. There's a lot more functionality in i915_irq.c
		45	* and related files, but that will be described in separate chapters.
		46	*/
4104	Serge	47
6084	serge	48	static const u32 hpd_ilk[HPD_NUM_PINS] = {
		49	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
		50	};
		51
		52	static const u32 hpd_ivb[HPD_NUM_PINS] = {
		53	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
		54	};
		55
		56	static const u32 hpd_bdw[HPD_NUM_PINS] = {
		57	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
		58	};
		59
		60	static const u32 hpd_ibx[HPD_NUM_PINS] = {
3746	Serge	61	[HPD_CRT] = SDE_CRT_HOTPLUG,
		62	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
		63	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
		64	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
		65	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
		66	};
3031	serge	67
6084	serge	68	static const u32 hpd_cpt[HPD_NUM_PINS] = {
3746	Serge	69	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
		70	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
		71	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
		72	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
		73	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
		74	};
		75
6084	serge	76	static const u32 hpd_spt[HPD_NUM_PINS] = {
		77	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
		78	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
		79	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
		80	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
		81	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
		82	};
		83
		84	static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
3746	Serge	85	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
		86	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
		87	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
		88	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
		89	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
		90	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
		91	};
		92
6084	serge	93	static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
3746	Serge	94	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
		95	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
		96	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
		97	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
		98	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
		99	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
		100	};
		101
6084	serge	102	static const u32 hpd_status_i915[HPD_NUM_PINS] = {
3746	Serge	103	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
		104	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
		105	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
		106	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
		107	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
		108	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
		109	};
		110
6084	serge	111	/* BXT hpd list */
		112	static const u32 hpd_bxt[HPD_NUM_PINS] = {
		113	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
		114	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
		115	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
		116	};
		117
5060	serge	118	/* IIR can theoretically queue up two events. Be paranoid. */
		119	#define GEN8_IRQ_RESET_NDX(type, which) do { \
		120	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
		121	POSTING_READ(GEN8_##type##_IMR(which)); \
		122	I915_WRITE(GEN8_##type##_IER(which), 0); \
		123	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
		124	POSTING_READ(GEN8_##type##_IIR(which)); \
		125	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
		126	POSTING_READ(GEN8_##type##_IIR(which)); \
		127	} while (0)
3746	Serge	128
5060	serge	129	#define GEN5_IRQ_RESET(type) do { \
		130	I915_WRITE(type##IMR, 0xffffffff); \
		131	POSTING_READ(type##IMR); \
		132	I915_WRITE(type##IER, 0); \
		133	I915_WRITE(type##IIR, 0xffffffff); \
		134	POSTING_READ(type##IIR); \
		135	I915_WRITE(type##IIR, 0xffffffff); \
		136	POSTING_READ(type##IIR); \
		137	} while (0)
		138
		139	/*
		140	* We should clear IMR at preinstall/uninstall, and just check at postinstall.
		141	*/
6084	serge	142	static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
		143	{
		144	u32 val = I915_READ(reg);
5060	serge	145
6084	serge	146	if (val == 0)
		147	return;
		148
		149	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
		150	reg, val);
		151	I915_WRITE(reg, 0xffffffff);
		152	POSTING_READ(reg);
		153	I915_WRITE(reg, 0xffffffff);
		154	POSTING_READ(reg);
		155	}
		156
5060	serge	157	#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
6084	serge	158	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
5354	serge	159	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
5060	serge	160	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
5354	serge	161	POSTING_READ(GEN8_##type##_IMR(which)); \
5060	serge	162	} while (0)
		163
		164	#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
6084	serge	165	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
5354	serge	166	I915_WRITE(type##IER, (ier_val)); \
5060	serge	167	I915_WRITE(type##IMR, (imr_val)); \
5354	serge	168	POSTING_READ(type##IMR); \
5060	serge	169	} while (0)
		170
5354	serge	171	static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
3031	serge	172
2351	Serge	173	/* For display hotplug interrupt */
6084	serge	174	static inline void
		175	i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
		176	uint32_t mask,
		177	uint32_t bits)
2351	Serge	178	{
6084	serge	179	uint32_t val;
		180
4104	Serge	181	assert_spin_locked(&dev_priv->irq_lock);
6084	serge	182	WARN_ON(bits & ~mask);
4104	Serge	183
6084	serge	184	val = I915_READ(PORT_HOTPLUG_EN);
		185	val &= ~mask;
		186	val \|= bits;
		187	I915_WRITE(PORT_HOTPLUG_EN, val);
		188	}
4104	Serge	189
6084	serge	190	/**
		191	* i915_hotplug_interrupt_update - update hotplug interrupt enable
		192	* @dev_priv: driver private
		193	* @mask: bits to update
		194	* @bits: bits to enable
		195	* NOTE: the HPD enable bits are modified both inside and outside
		196	* of an interrupt context. To avoid that read-modify-write cycles
		197	* interfer, these bits are protected by a spinlock. Since this
		198	* function is usually not called from a context where the lock is
		199	* held already, this function acquires the lock itself. A non-locking
		200	* version is also available.
		201	*/
		202	void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
		203	uint32_t mask,
		204	uint32_t bits)
		205	{
		206	spin_lock_irq(&dev_priv->irq_lock);
		207	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
		208	spin_unlock_irq(&dev_priv->irq_lock);
2351	Serge	209	}
		210
6084	serge	211	/**
		212	* ilk_update_display_irq - update DEIMR
		213	* @dev_priv: driver private
		214	* @interrupt_mask: mask of interrupt bits to update
		215	* @enabled_irq_mask: mask of interrupt bits to enable
		216	*/
		217	static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
		218	uint32_t interrupt_mask,
		219	uint32_t enabled_irq_mask)
2351	Serge	220	{
6084	serge	221	uint32_t new_val;
		222
4104	Serge	223	assert_spin_locked(&dev_priv->irq_lock);
		224
6084	serge	225	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		226
5354	serge	227	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	228	return;
		229
6084	serge	230	new_val = dev_priv->irq_mask;
		231	new_val &= ~interrupt_mask;
		232	new_val \|= (~enabled_irq_mask & interrupt_mask);
		233
		234	if (new_val != dev_priv->irq_mask) {
		235	dev_priv->irq_mask = new_val;
		236	I915_WRITE(DEIMR, dev_priv->irq_mask);
		237	POSTING_READ(DEIMR);
		238	}
2351	Serge	239	}
3031	serge	240
6084	serge	241	void
		242	ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
		243	{
		244	ilk_update_display_irq(dev_priv, mask, mask);
		245	}
		246
		247	void
		248	ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
		249	{
		250	ilk_update_display_irq(dev_priv, mask, 0);
		251	}
		252
4104	Serge	253	/**
		254	* ilk_update_gt_irq - update GTIMR
		255	* @dev_priv: driver private
		256	* @interrupt_mask: mask of interrupt bits to update
		257	* @enabled_irq_mask: mask of interrupt bits to enable
		258	*/
		259	static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
		260	uint32_t interrupt_mask,
		261	uint32_t enabled_irq_mask)
		262	{
		263	assert_spin_locked(&dev_priv->irq_lock);
		264
6084	serge	265	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		266
5060	serge	267	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	268	return;
		269
		270	dev_priv->gt_irq_mask &= ~interrupt_mask;
		271	dev_priv->gt_irq_mask \|= (~enabled_irq_mask & interrupt_mask);
		272	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		273	POSTING_READ(GTIMR);
		274	}
		275
5060	serge	276	void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	277	{
		278	ilk_update_gt_irq(dev_priv, mask, mask);
		279	}
		280
5060	serge	281	void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	282	{
		283	ilk_update_gt_irq(dev_priv, mask, 0);
		284	}
		285
5354	serge	286	static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
4104	Serge	287	{
5354	serge	288	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
4104	Serge	289	}
		290
5354	serge	291	static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
4104	Serge	292	{
5354	serge	293	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
4104	Serge	294	}
		295
5354	serge	296	static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
4104	Serge	297	{
5354	serge	298	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
4104	Serge	299	}
		300
5060	serge	301	/**
5354	serge	302	* snb_update_pm_irq - update GEN6_PMIMR
5060	serge	303	* @dev_priv: driver private
		304	* @interrupt_mask: mask of interrupt bits to update
		305	* @enabled_irq_mask: mask of interrupt bits to enable
		306	*/
5354	serge	307	static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
5060	serge	308	uint32_t interrupt_mask,
		309	uint32_t enabled_irq_mask)
		310	{
		311	uint32_t new_val;
		312
6084	serge	313	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		314
5060	serge	315	assert_spin_locked(&dev_priv->irq_lock);
		316
		317	new_val = dev_priv->pm_irq_mask;
		318	new_val &= ~interrupt_mask;
		319	new_val \|= (~enabled_irq_mask & interrupt_mask);
		320
		321	if (new_val != dev_priv->pm_irq_mask) {
		322	dev_priv->pm_irq_mask = new_val;
5354	serge	323	I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		324	POSTING_READ(gen6_pm_imr(dev_priv));
5060	serge	325	}
		326	}
		327
5354	serge	328	void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
5060	serge	329	{
5354	serge	330	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		331	return;
		332
		333	snb_update_pm_irq(dev_priv, mask, mask);
5060	serge	334	}
		335
5354	serge	336	static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
		337	uint32_t mask)
5060	serge	338	{
5354	serge	339	snb_update_pm_irq(dev_priv, mask, 0);
5060	serge	340	}
		341
5354	serge	342	void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	343	{
5354	serge	344	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		345	return;
4104	Serge	346
5354	serge	347	__gen6_disable_pm_irq(dev_priv, mask);
4104	Serge	348	}
		349
5354	serge	350	void gen6_reset_rps_interrupts(struct drm_device *dev)
5060	serge	351	{
		352	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	353	uint32_t reg = gen6_pm_iir(dev_priv);
5060	serge	354
5354	serge	355	spin_lock_irq(&dev_priv->irq_lock);
		356	I915_WRITE(reg, dev_priv->pm_rps_events);
		357	I915_WRITE(reg, dev_priv->pm_rps_events);
6084	serge	358	POSTING_READ(reg);
		359	dev_priv->rps.pm_iir = 0;
5354	serge	360	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	361	}
		362
5354	serge	363	void gen6_enable_rps_interrupts(struct drm_device *dev)
5060	serge	364	{
		365	struct drm_i915_private *dev_priv = dev->dev_private;
		366
5354	serge	367	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	368
5354	serge	369	WARN_ON(dev_priv->rps.pm_iir);
		370	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
		371	dev_priv->rps.interrupts_enabled = true;
		372	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) \|
		373	dev_priv->pm_rps_events);
		374	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		375
		376	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	377	}
		378
6084	serge	379	u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
		380	{
		381	/*
		382	* SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
		383	* if GEN6_PM_UP_EI_EXPIRED is masked.
		384	*
		385	* TODO: verify if this can be reproduced on VLV,CHV.
		386	*/
		387	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		388	mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
		389
		390	if (INTEL_INFO(dev_priv)->gen >= 8)
		391	mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
		392
		393	return mask;
		394	}
		395
5354	serge	396	void gen6_disable_rps_interrupts(struct drm_device *dev)
4104	Serge	397	{
		398	struct drm_i915_private *dev_priv = dev->dev_private;
		399
5354	serge	400	spin_lock_irq(&dev_priv->irq_lock);
		401	dev_priv->rps.interrupts_enabled = false;
		402	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	403
5354	serge	404	cancel_work_sync(&dev_priv->rps.work);
4104	Serge	405
5354	serge	406	spin_lock_irq(&dev_priv->irq_lock);
4104	Serge	407
6084	serge	408	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
4104	Serge	409
5354	serge	410	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		411	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
		412	~dev_priv->pm_rps_events);
4104	Serge	413
6084	serge	414	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	415
		416	}
		417
4104	Serge	418	/**
6084	serge	419	* bdw_update_port_irq - update DE port interrupt
		420	* @dev_priv: driver private
		421	* @interrupt_mask: mask of interrupt bits to update
		422	* @enabled_irq_mask: mask of interrupt bits to enable
		423	*/
		424	static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
		425	uint32_t interrupt_mask,
		426	uint32_t enabled_irq_mask)
		427	{
		428	uint32_t new_val;
		429	uint32_t old_val;
		430
		431	assert_spin_locked(&dev_priv->irq_lock);
		432
		433	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		434
		435	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		436	return;
		437
		438	old_val = I915_READ(GEN8_DE_PORT_IMR);
		439
		440	new_val = old_val;
		441	new_val &= ~interrupt_mask;
		442	new_val \|= (~enabled_irq_mask & interrupt_mask);
		443
		444	if (new_val != old_val) {
		445	I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		446	POSTING_READ(GEN8_DE_PORT_IMR);
		447	}
		448	}
		449
		450	/**
4104	Serge	451	* ibx_display_interrupt_update - update SDEIMR
		452	* @dev_priv: driver private
		453	* @interrupt_mask: mask of interrupt bits to update
		454	* @enabled_irq_mask: mask of interrupt bits to enable
		455	*/
5354	serge	456	void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
6084	serge	457	uint32_t interrupt_mask,
		458	uint32_t enabled_irq_mask)
4104	Serge	459	{
		460	uint32_t sdeimr = I915_READ(SDEIMR);
		461	sdeimr &= ~interrupt_mask;
		462	sdeimr \|= (~enabled_irq_mask & interrupt_mask);
		463
6084	serge	464	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		465
4104	Serge	466	assert_spin_locked(&dev_priv->irq_lock);
		467
5060	serge	468	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	469	return;
		470
		471	I915_WRITE(SDEIMR, sdeimr);
		472	POSTING_READ(SDEIMR);
		473	}
		474
5060	serge	475	static void
		476	__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		477	u32 enable_mask, u32 status_mask)
3031	serge	478	{
6084	serge	479	u32 reg = PIPESTAT(pipe);
5060	serge	480	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
3031	serge	481
4104	Serge	482	assert_spin_locked(&dev_priv->irq_lock);
5354	serge	483	WARN_ON(!intel_irqs_enabled(dev_priv));
4104	Serge	484
5060	serge	485	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK \|\|
		486	status_mask & ~PIPESTAT_INT_STATUS_MASK,
		487	"pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		488	pipe_name(pipe), enable_mask, status_mask))
3746	Serge	489	return;
		490
5060	serge	491	if ((pipestat & enable_mask) == enable_mask)
		492	return;
		493
		494	dev_priv->pipestat_irq_mask[pipe] \|= status_mask;
		495
6084	serge	496	/* Enable the interrupt, clear any pending status */
5060	serge	497	pipestat \|= enable_mask \| status_mask;
3746	Serge	498	I915_WRITE(reg, pipestat);
6084	serge	499	POSTING_READ(reg);
3031	serge	500	}
		501
5060	serge	502	static void
		503	__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		504	u32 enable_mask, u32 status_mask)
3031	serge	505	{
6084	serge	506	u32 reg = PIPESTAT(pipe);
5060	serge	507	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
3031	serge	508
4104	Serge	509	assert_spin_locked(&dev_priv->irq_lock);
5354	serge	510	WARN_ON(!intel_irqs_enabled(dev_priv));
4104	Serge	511
5060	serge	512	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK \|\|
		513	status_mask & ~PIPESTAT_INT_STATUS_MASK,
		514	"pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		515	pipe_name(pipe), enable_mask, status_mask))
3746	Serge	516	return;
		517
5060	serge	518	if ((pipestat & enable_mask) == 0)
		519	return;
		520
		521	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
		522
		523	pipestat &= ~enable_mask;
3746	Serge	524	I915_WRITE(reg, pipestat);
6084	serge	525	POSTING_READ(reg);
3031	serge	526	}
		527
5060	serge	528	static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
		529	{
		530	u32 enable_mask = status_mask << 16;
		531
		532	/*
		533	* On pipe A we don't support the PSR interrupt yet,
		534	* on pipe B and C the same bit MBZ.
		535	*/
		536	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		537	return 0;
		538	/*
		539	* On pipe B and C we don't support the PSR interrupt yet, on pipe
		540	* A the same bit is for perf counters which we don't use either.
		541	*/
		542	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		543	return 0;
		544
		545	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS \|
		546	SPRITE0_FLIP_DONE_INT_EN_VLV \|
		547	SPRITE1_FLIP_DONE_INT_EN_VLV);
		548	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		549	enable_mask \|= SPRITE0_FLIP_DONE_INT_EN_VLV;
		550	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		551	enable_mask \|= SPRITE1_FLIP_DONE_INT_EN_VLV;
		552
		553	return enable_mask;
		554	}
		555
		556	void
		557	i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		558	u32 status_mask)
		559	{
		560	u32 enable_mask;
		561
		562	if (IS_VALLEYVIEW(dev_priv->dev))
		563	enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
		564	status_mask);
		565	else
		566	enable_mask = status_mask << 16;
		567	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
		568	}
		569
		570	void
		571	i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		572	u32 status_mask)
		573	{
		574	u32 enable_mask;
		575
		576	if (IS_VALLEYVIEW(dev_priv->dev))
		577	enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
		578	status_mask);
		579	else
		580	enable_mask = status_mask << 16;
		581	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
		582	}
		583
3031	serge	584	/**
4104	Serge	585	* i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
6084	serge	586	* @dev: drm device
3031	serge	587	*/
4104	Serge	588	static void i915_enable_asle_pipestat(struct drm_device *dev)
3031	serge	589	{
5060	serge	590	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	591
4104	Serge	592	if (!dev_priv->opregion.asle \|\| !IS_MOBILE(dev))
3031	serge	593	return;
		594
5354	serge	595	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	596
5060	serge	597	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
6084	serge	598	if (INTEL_INFO(dev)->gen >= 4)
4560	Serge	599	i915_enable_pipestat(dev_priv, PIPE_A,
5060	serge	600	PIPE_LEGACY_BLC_EVENT_STATUS);
3031	serge	601
5354	serge	602	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	603	}
		604
5060	serge	605	/*
		606	* This timing diagram depicts the video signal in and
		607	* around the vertical blanking period.
		608	*
		609	* Assumptions about the fictitious mode used in this example:
		610	* vblank_start >= 3
		611	* vsync_start = vblank_start + 1
		612	* vsync_end = vblank_start + 2
		613	* vtotal = vblank_start + 3
		614	*
		615	* start of vblank:
		616	* latch double buffered registers
		617	* increment frame counter (ctg+)
		618	* generate start of vblank interrupt (gen4+)
		619	* \|
		620	* \| frame start:
		621	* \| generate frame start interrupt (aka. vblank interrupt) (gmch)
		622	* \| may be shifted forward 1-3 extra lines via PIPECONF
		623	* \| \|
		624	* \| \| start of vsync:
		625	* \| \| generate vsync interrupt
		626	* \| \| \|
		627	* ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
		628	* . \hs/ . \hs/ \hs/ \hs/ . \hs/
		629	* ----va---> <-----------------vb--------------------> <--------va-------------
		630	* \| \| <----vs-----> \|
		631	* -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
		632	* -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
		633	* -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
		634	* \| \| \|
		635	* last visible pixel first visible pixel
		636	* \| increment frame counter (gen3/4)
		637	* pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
		638	*
		639	* x = horizontal active
		640	* _ = horizontal blanking
		641	* hs = horizontal sync
		642	* va = vertical active
		643	* vb = vertical blanking
		644	* vs = vertical sync
		645	* vbs = vblank_start (number)
		646	*
		647	* Summary:
		648	* - most events happen at the start of horizontal sync
		649	* - frame start happens at the start of horizontal blank, 1-4 lines
		650	* (depending on PIPECONF settings) after the start of vblank
		651	* - gen3/4 pixel and frame counter are synchronized with the start
		652	* of horizontal active on the first line of vertical active
		653	*/
		654
6084	serge	655	static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
4560	Serge	656	{
		657	/* Gen2 doesn't have a hardware frame counter */
		658	return 0;
		659	}
		660
3031	serge	661	/* Called from drm generic code, passed a 'crtc', which
		662	* we use as a pipe index
		663	*/
6084	serge	664	static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
3031	serge	665	{
5060	serge	666	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	667	unsigned long high_frame;
		668	unsigned long low_frame;
5060	serge	669	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
6084	serge	670	struct intel_crtc *intel_crtc =
		671	to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
		672	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
3031	serge	673
6084	serge	674	htotal = mode->crtc_htotal;
		675	hsync_start = mode->crtc_hsync_start;
		676	vbl_start = mode->crtc_vblank_start;
		677	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		678	vbl_start = DIV_ROUND_UP(vbl_start, 2);
3031	serge	679
5060	serge	680	/* Convert to pixel count */
6084	serge	681	vbl_start *= htotal;
4560	Serge	682
5060	serge	683	/* Start of vblank event occurs at start of hsync */
		684	vbl_start -= htotal - hsync_start;
		685
3031	serge	686	high_frame = PIPEFRAME(pipe);
		687	low_frame = PIPEFRAMEPIXEL(pipe);
		688
		689	/*
		690	* High & low register fields aren't synchronized, so make sure
		691	* we get a low value that's stable across two reads of the high
		692	* register.
		693	*/
		694	do {
		695	high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
4560	Serge	696	low = I915_READ(low_frame);
3031	serge	697	high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
		698	} while (high1 != high2);
		699
		700	high1 >>= PIPE_FRAME_HIGH_SHIFT;
4560	Serge	701	pixel = low & PIPE_PIXEL_MASK;
3031	serge	702	low >>= PIPE_FRAME_LOW_SHIFT;
4560	Serge	703
		704	/*
		705	* The frame counter increments at beginning of active.
		706	* Cook up a vblank counter by also checking the pixel
		707	* counter against vblank start.
		708	*/
		709	return (((high1 << 8) \| low) + (pixel >= vbl_start)) & 0xffffff;
3031	serge	710	}
		711
6084	serge	712	static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
3031	serge	713	{
5060	serge	714	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	715
6084	serge	716	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
3031	serge	717	}
		718
4560	Serge	719	/* raw reads, only for fast reads of display block, no need for forcewake etc. */
		720	#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
		721
5060	serge	722	static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
4560	Serge	723	{
5060	serge	724	struct drm_device *dev = crtc->base.dev;
4560	Serge	725	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	726	const struct drm_display_mode *mode = &crtc->base.hwmode;
5060	serge	727	enum pipe pipe = crtc->pipe;
		728	int position, vtotal;
4560	Serge	729
5060	serge	730	vtotal = mode->crtc_vtotal;
		731	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		732	vtotal /= 2;
4560	Serge	733
5060	serge	734	if (IS_GEN2(dev))
		735	position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
		736	else
		737	position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
		738
		739	/*
6084	serge	740	* On HSW, the DSL reg (0x70000) appears to return 0 if we
		741	* read it just before the start of vblank. So try it again
		742	* so we don't accidentally end up spanning a vblank frame
		743	* increment, causing the pipe_update_end() code to squak at us.
		744	*
		745	* The nature of this problem means we can't simply check the ISR
		746	* bit and return the vblank start value; nor can we use the scanline
		747	* debug register in the transcoder as it appears to have the same
		748	* problem. We may need to extend this to include other platforms,
		749	* but so far testing only shows the problem on HSW.
		750	*/
		751	if (HAS_DDI(dev) && !position) {
		752	int i, temp;
		753
		754	for (i = 0; i < 100; i++) {
		755	udelay(1);
		756	temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
		757	DSL_LINEMASK_GEN3;
		758	if (temp != position) {
		759	position = temp;
		760	break;
		761	}
		762	}
		763	}
		764
		765	/*
5060	serge	766	* See update_scanline_offset() for the details on the
		767	* scanline_offset adjustment.
		768	*/
		769	return (position + crtc->scanline_offset) % vtotal;
4560	Serge	770	}
		771
6084	serge	772	static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
4560	Serge	773	unsigned int flags, int vpos, int hpos,
6084	serge	774	ktime_t stime, ktime_t etime,
		775	const struct drm_display_mode *mode)
3746	Serge	776	{
4560	Serge	777	struct drm_i915_private *dev_priv = dev->dev_private;
		778	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		779	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		780	int position;
5060	serge	781	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
3746	Serge	782	bool in_vbl = true;
		783	int ret = 0;
4560	Serge	784	unsigned long irqflags;
3746	Serge	785
6084	serge	786	if (WARN_ON(!mode->crtc_clock)) {
3746	Serge	787	DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
		788	"pipe %c\n", pipe_name(pipe));
		789	return 0;
		790	}
		791
4560	Serge	792	htotal = mode->crtc_htotal;
5060	serge	793	hsync_start = mode->crtc_hsync_start;
4560	Serge	794	vtotal = mode->crtc_vtotal;
		795	vbl_start = mode->crtc_vblank_start;
		796	vbl_end = mode->crtc_vblank_end;
3746	Serge	797
4560	Serge	798	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		799	vbl_start = DIV_ROUND_UP(vbl_start, 2);
		800	vbl_end /= 2;
		801	vtotal /= 2;
		802	}
		803
		804	ret \|= DRM_SCANOUTPOS_VALID \| DRM_SCANOUTPOS_ACCURATE;
		805
		806	/*
		807	* Lock uncore.lock, as we will do multiple timing critical raw
		808	* register reads, potentially with preemption disabled, so the
		809	* following code must not block on uncore.lock.
		810	*/
		811	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
		812
		813	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
		814
6084	serge	815	/* Get optional system timestamp before query. */
		816	if (stime)
		817	*stime = ktime_get();
4560	Serge	818
		819	if (IS_GEN2(dev) \|\| IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
3746	Serge	820	/* No obvious pixelcount register. Only query vertical
		821	* scanout position from Display scan line register.
		822	*/
5060	serge	823	position = __intel_get_crtc_scanline(intel_crtc);
3746	Serge	824	} else {
		825	/* Have access to pixelcount since start of frame.
		826	* We can split this into vertical and horizontal
		827	* scanout position.
		828	*/
4560	Serge	829	position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
3746	Serge	830
4560	Serge	831	/* convert to pixel counts */
		832	vbl_start *= htotal;
		833	vbl_end *= htotal;
		834	vtotal *= htotal;
5060	serge	835
		836	/*
		837	* In interlaced modes, the pixel counter counts all pixels,
		838	* so one field will have htotal more pixels. In order to avoid
		839	* the reported position from jumping backwards when the pixel
		840	* counter is beyond the length of the shorter field, just
		841	* clamp the position the length of the shorter field. This
		842	* matches how the scanline counter based position works since
		843	* the scanline counter doesn't count the two half lines.
		844	*/
		845	if (position >= vtotal)
		846	position = vtotal - 1;
		847
		848	/*
		849	* Start of vblank interrupt is triggered at start of hsync,
		850	* just prior to the first active line of vblank. However we
		851	* consider lines to start at the leading edge of horizontal
		852	* active. So, should we get here before we've crossed into
		853	* the horizontal active of the first line in vblank, we would
		854	* not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		855	* always add htotal-hsync_start to the current pixel position.
		856	*/
		857	position = (position + htotal - hsync_start) % vtotal;
3746	Serge	858	}
		859
6084	serge	860	/* Get optional system timestamp after query. */
		861	if (etime)
		862	*etime = ktime_get();
3746	Serge	863
4560	Serge	864	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
3746	Serge	865
4560	Serge	866	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3746	Serge	867
4560	Serge	868	in_vbl = position >= vbl_start && position < vbl_end;
3746	Serge	869
4560	Serge	870	/*
		871	* While in vblank, position will be negative
		872	* counting up towards 0 at vbl_end. And outside
		873	* vblank, position will be positive counting
		874	* up since vbl_end.
		875	*/
		876	if (position >= vbl_start)
		877	position -= vbl_end;
		878	else
		879	position += vtotal - vbl_end;
3746	Serge	880
4560	Serge	881	if (IS_GEN2(dev) \|\| IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		882	*vpos = position;
		883	*hpos = 0;
		884	} else {
		885	*vpos = position / htotal;
		886	hpos = position - (vpos * htotal);
		887	}
		888
3746	Serge	889	/* In vblank? */
		890	if (in_vbl)
5354	serge	891	ret \|= DRM_SCANOUTPOS_IN_VBLANK;
3746	Serge	892
		893	return ret;
		894	}
		895
5060	serge	896	int intel_get_crtc_scanline(struct intel_crtc *crtc)
		897	{
		898	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
		899	unsigned long irqflags;
		900	int position;
		901
		902	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
		903	position = __intel_get_crtc_scanline(crtc);
		904	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
		905
		906	return position;
		907	}
		908
6084	serge	909	static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
3746	Serge	910	int *max_error,
		911	struct timeval *vblank_time,
		912	unsigned flags)
		913	{
		914	struct drm_crtc *crtc;
		915
6084	serge	916	if (pipe >= INTEL_INFO(dev)->num_pipes) {
		917	DRM_ERROR("Invalid crtc %u\n", pipe);
3746	Serge	918	return -EINVAL;
		919	}
		920
		921	/* Get drm_crtc to timestamp: */
		922	crtc = intel_get_crtc_for_pipe(dev, pipe);
		923	if (crtc == NULL) {
6084	serge	924	DRM_ERROR("Invalid crtc %u\n", pipe);
3746	Serge	925	return -EINVAL;
		926	}
		927
6084	serge	928	if (!crtc->hwmode.crtc_clock) {
		929	DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
3746	Serge	930	return -EBUSY;
		931	}
		932
		933	/* Helper routine in DRM core does all the work: */
		934	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
		935	vblank_time, flags,
6084	serge	936	&crtc->hwmode);
3746	Serge	937	}
		938
4104	Serge	939	static void ironlake_rps_change_irq_handler(struct drm_device *dev)
3746	Serge	940	{
5060	serge	941	struct drm_i915_private *dev_priv = dev->dev_private;
3746	Serge	942	u32 busy_up, busy_down, max_avg, min_avg;
		943	u8 new_delay;
		944
4104	Serge	945	spin_lock(&mchdev_lock);
3746	Serge	946
		947	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
		948
		949	new_delay = dev_priv->ips.cur_delay;
		950
		951	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
		952	busy_up = I915_READ(RCPREVBSYTUPAVG);
		953	busy_down = I915_READ(RCPREVBSYTDNAVG);
		954	max_avg = I915_READ(RCBMAXAVG);
		955	min_avg = I915_READ(RCBMINAVG);
		956
		957	/* Handle RCS change request from hw */
		958	if (busy_up > max_avg) {
		959	if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
		960	new_delay = dev_priv->ips.cur_delay - 1;
		961	if (new_delay < dev_priv->ips.max_delay)
		962	new_delay = dev_priv->ips.max_delay;
		963	} else if (busy_down < min_avg) {
		964	if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
		965	new_delay = dev_priv->ips.cur_delay + 1;
		966	if (new_delay > dev_priv->ips.min_delay)
		967	new_delay = dev_priv->ips.min_delay;
		968	}
		969
		970	if (ironlake_set_drps(dev, new_delay))
		971	dev_priv->ips.cur_delay = new_delay;
		972
4104	Serge	973	spin_unlock(&mchdev_lock);
3746	Serge	974
		975	return;
		976	}
		977
6084	serge	978	static void notify_ring(struct intel_engine_cs *ring)
2352	Serge	979	{
5060	serge	980	if (!intel_ring_initialized(ring))
2352	Serge	981	return;
2351	Serge	982
6084	serge	983	trace_i915_gem_request_notify(ring);
2351	Serge	984
2352	Serge	985	wake_up_all(&ring->irq_queue);
		986	}
		987
6084	serge	988	static void vlv_c0_read(struct drm_i915_private *dev_priv,
		989	struct intel_rps_ei *ei)
5060	serge	990	{
6084	serge	991	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
		992	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
		993	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
		994	}
5060	serge	995
6084	serge	996	static bool vlv_c0_above(struct drm_i915_private *dev_priv,
		997	const struct intel_rps_ei *old,
		998	const struct intel_rps_ei *now,
		999	int threshold)
		1000	{
		1001	u64 time, c0;
		1002	unsigned int mul = 100;
5060	serge	1003
6084	serge	1004	if (old->cz_clock == 0)
		1005	return false;
5060	serge	1006
6084	serge	1007	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		1008	mul <<= 8;
5060	serge	1009
6084	serge	1010	time = now->cz_clock - old->cz_clock;
		1011	time = threshold dev_priv->czclk_freq;
5060	serge	1012
6084	serge	1013	/* Workload can be split between render + media, e.g. SwapBuffers
		1014	* being blitted in X after being rendered in mesa. To account for
		1015	* this we need to combine both engines into our activity counter.
5060	serge	1016	*/
6084	serge	1017	c0 = now->render_c0 - old->render_c0;
		1018	c0 += now->media_c0 - old->media_c0;
		1019	c0 = mul VLV_CZ_CLOCK_TO_MILLI_SEC;
5060	serge	1020
6084	serge	1021	return c0 >= time;
5060	serge	1022	}
		1023
6084	serge	1024	void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
5060	serge	1025	{
6084	serge	1026	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
		1027	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
		1028	}
5060	serge	1029
6084	serge	1030	static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
		1031	{
		1032	struct intel_rps_ei now;
		1033	u32 events = 0;
5060	serge	1034
6084	serge	1035	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
		1036	return 0;
5060	serge	1037
6084	serge	1038	vlv_c0_read(dev_priv, &now);
		1039	if (now.cz_clock == 0)
		1040	return 0;
5060	serge	1041
6084	serge	1042	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		1043	if (!vlv_c0_above(dev_priv,
		1044	&dev_priv->rps.down_ei, &now,
		1045	dev_priv->rps.down_threshold))
		1046	events \|= GEN6_PM_RP_DOWN_THRESHOLD;
		1047	dev_priv->rps.down_ei = now;
5060	serge	1048	}
		1049
6084	serge	1050	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		1051	if (vlv_c0_above(dev_priv,
		1052	&dev_priv->rps.up_ei, &now,
		1053	dev_priv->rps.up_threshold))
		1054	events \|= GEN6_PM_RP_UP_THRESHOLD;
		1055	dev_priv->rps.up_ei = now;
5060	serge	1056	}
		1057
6084	serge	1058	return events;
		1059	}
5060	serge	1060
6084	serge	1061	static bool any_waiters(struct drm_i915_private *dev_priv)
		1062	{
		1063	struct intel_engine_cs *ring;
		1064	int i;
5060	serge	1065
6084	serge	1066	for_each_ring(ring, dev_priv, i)
		1067	if (ring->irq_refcount)
		1068	return true;
5060	serge	1069
6084	serge	1070	return false;
5060	serge	1071	}
		1072
3031	serge	1073	static void gen6_pm_rps_work(struct work_struct *work)
		1074	{
5060	serge	1075	struct drm_i915_private *dev_priv =
		1076	container_of(work, struct drm_i915_private, rps.work);
6084	serge	1077	bool client_boost;
		1078	int new_delay, adj, min, max;
4104	Serge	1079	u32 pm_iir;
2352	Serge	1080
4104	Serge	1081	spin_lock_irq(&dev_priv->irq_lock);
5354	serge	1082	/* Speed up work cancelation during disabling rps interrupts. */
		1083	if (!dev_priv->rps.interrupts_enabled) {
		1084	spin_unlock_irq(&dev_priv->irq_lock);
		1085	return;
		1086	}
3031	serge	1087	pm_iir = dev_priv->rps.pm_iir;
		1088	dev_priv->rps.pm_iir = 0;
5354	serge	1089	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
6084	serge	1090	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		1091	client_boost = dev_priv->rps.client_boost;
		1092	dev_priv->rps.client_boost = false;
4104	Serge	1093	spin_unlock_irq(&dev_priv->irq_lock);
2352	Serge	1094
4104	Serge	1095	/* Make sure we didn't queue anything we're not going to process. */
5060	serge	1096	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
4104	Serge	1097
6084	serge	1098	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
3031	serge	1099	return;
		1100
3243	Serge	1101	mutex_lock(&dev_priv->rps.hw_lock);
3031	serge	1102
6084	serge	1103	pm_iir \|= vlv_wa_c0_ei(dev_priv, pm_iir);
		1104
4560	Serge	1105	adj = dev_priv->rps.last_adj;
6084	serge	1106	new_delay = dev_priv->rps.cur_freq;
		1107	min = dev_priv->rps.min_freq_softlimit;
		1108	max = dev_priv->rps.max_freq_softlimit;
		1109
		1110	if (client_boost) {
		1111	new_delay = dev_priv->rps.max_freq_softlimit;
		1112	adj = 0;
		1113	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
4560	Serge	1114	if (adj > 0)
		1115	adj *= 2;
6084	serge	1116	else /* CHV needs even encode values */
		1117	adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
4104	Serge	1118	/*
		1119	* For better performance, jump directly
		1120	* to RPe if we're below it.
		1121	*/
6084	serge	1122	if (new_delay < dev_priv->rps.efficient_freq - adj) {
5060	serge	1123	new_delay = dev_priv->rps.efficient_freq;
6084	serge	1124	adj = 0;
		1125	}
		1126	} else if (any_waiters(dev_priv)) {
		1127	adj = 0;
4560	Serge	1128	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
5060	serge	1129	if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
		1130	new_delay = dev_priv->rps.efficient_freq;
4560	Serge	1131	else
5060	serge	1132	new_delay = dev_priv->rps.min_freq_softlimit;
4560	Serge	1133	adj = 0;
		1134	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		1135	if (adj < 0)
		1136	adj *= 2;
6084	serge	1137	else /* CHV needs even encode values */
		1138	adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
4560	Serge	1139	} else { /* unknown event */
6084	serge	1140	adj = 0;
4560	Serge	1141	}
3031	serge	1142
6084	serge	1143	dev_priv->rps.last_adj = adj;
		1144
3031	serge	1145	/* sysfs frequency interfaces may have snuck in while servicing the
		1146	* interrupt
		1147	*/
6084	serge	1148	new_delay += adj;
		1149	new_delay = clamp_t(int, new_delay, min, max);
4560	Serge	1150
6084	serge	1151	intel_set_rps(dev_priv->dev, new_delay);
5060	serge	1152
3243	Serge	1153	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	1154	}
		1155
		1156
		1157	/**
		1158	* ivybridge_parity_work - Workqueue called when a parity error interrupt
		1159	* occurred.
		1160	* @work: workqueue struct
		1161	*
		1162	* Doesn't actually do anything except notify userspace. As a consequence of
		1163	* this event, userspace should try to remap the bad rows since statistically
		1164	* it is likely the same row is more likely to go bad again.
		1165	*/
		1166	static void ivybridge_parity_work(struct work_struct *work)
2351	Serge	1167	{
5060	serge	1168	struct drm_i915_private *dev_priv =
		1169	container_of(work, struct drm_i915_private, l3_parity.error_work);
3031	serge	1170	u32 error_status, row, bank, subbank;
4560	Serge	1171	char *parity_event[6];
3031	serge	1172	uint32_t misccpctl;
4560	Serge	1173	uint8_t slice = 0;
3031	serge	1174
		1175	/* We must turn off DOP level clock gating to access the L3 registers.
		1176	* In order to prevent a get/put style interface, acquire struct mutex
		1177	* any time we access those registers.
		1178	*/
		1179	mutex_lock(&dev_priv->dev->struct_mutex);
		1180
4560	Serge	1181	/* If we've screwed up tracking, just let the interrupt fire again */
		1182	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		1183	goto out;
		1184
3031	serge	1185	misccpctl = I915_READ(GEN7_MISCCPCTL);
		1186	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		1187	POSTING_READ(GEN7_MISCCPCTL);
		1188
4560	Serge	1189	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		1190	u32 reg;
		1191
		1192	slice--;
		1193	if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
		1194	break;
		1195
		1196	dev_priv->l3_parity.which_slice &= ~(1<
		1197
		1198	reg = GEN7_L3CDERRST1 + (slice * 0x200);
		1199
		1200	error_status = I915_READ(reg);
6084	serge	1201	row = GEN7_PARITY_ERROR_ROW(error_status);
		1202	bank = GEN7_PARITY_ERROR_BANK(error_status);
		1203	subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
3031	serge	1204
4560	Serge	1205	I915_WRITE(reg, GEN7_PARITY_ERROR_VALID \| GEN7_L3CDERRST1_ENABLE);
		1206	POSTING_READ(reg);
3031	serge	1207
4560	Serge	1208	DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
		1209	slice, row, bank, subbank);
		1210
		1211	}
		1212
3031	serge	1213	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		1214
4560	Serge	1215	out:
		1216	WARN_ON(dev_priv->l3_parity.which_slice);
5354	serge	1217	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	1218	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
5354	serge	1219	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	1220
		1221	mutex_unlock(&dev_priv->dev->struct_mutex);
		1222	}
		1223
4560	Serge	1224	static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
3031	serge	1225	{
5060	serge	1226	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1227
4560	Serge	1228	if (!HAS_L3_DPF(dev))
3031	serge	1229	return;
		1230
4104	Serge	1231	spin_lock(&dev_priv->irq_lock);
5060	serge	1232	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
4104	Serge	1233	spin_unlock(&dev_priv->irq_lock);
3031	serge	1234
4560	Serge	1235	iir &= GT_PARITY_ERROR(dev);
		1236	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		1237	dev_priv->l3_parity.which_slice \|= 1 << 1;
		1238
		1239	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		1240	dev_priv->l3_parity.which_slice \|= 1 << 0;
		1241
3243	Serge	1242	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
3031	serge	1243	}
		1244
4104	Serge	1245	static void ilk_gt_irq_handler(struct drm_device *dev,
		1246	struct drm_i915_private *dev_priv,
		1247	u32 gt_iir)
		1248	{
		1249	if (gt_iir &
		1250	(GT_RENDER_USER_INTERRUPT \| GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
6084	serge	1251	notify_ring(&dev_priv->ring[RCS]);
4104	Serge	1252	if (gt_iir & ILK_BSD_USER_INTERRUPT)
6084	serge	1253	notify_ring(&dev_priv->ring[VCS]);
4104	Serge	1254	}
		1255
3031	serge	1256	static void snb_gt_irq_handler(struct drm_device *dev,
		1257	struct drm_i915_private *dev_priv,
		1258	u32 gt_iir)
		1259	{
		1260
4104	Serge	1261	if (gt_iir &
		1262	(GT_RENDER_USER_INTERRUPT \| GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
6084	serge	1263	notify_ring(&dev_priv->ring[RCS]);
4104	Serge	1264	if (gt_iir & GT_BSD_USER_INTERRUPT)
6084	serge	1265	notify_ring(&dev_priv->ring[VCS]);
4104	Serge	1266	if (gt_iir & GT_BLT_USER_INTERRUPT)
6084	serge	1267	notify_ring(&dev_priv->ring[BCS]);
3031	serge	1268
4104	Serge	1269	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT \|
		1270	GT_BSD_CS_ERROR_INTERRUPT \|
5354	serge	1271	GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		1272	DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
3031	serge	1273
4560	Serge	1274	if (gt_iir & GT_PARITY_ERROR(dev))
		1275	ivybridge_parity_error_irq_handler(dev, gt_iir);
3031	serge	1276	}
		1277
6084	serge	1278	static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
4560	Serge	1279	u32 master_ctl)
		1280	{
		1281	irqreturn_t ret = IRQ_NONE;
		1282
		1283	if (master_ctl & (GEN8_GT_RCS_IRQ \| GEN8_GT_BCS_IRQ)) {
6084	serge	1284	u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
4560	Serge	1285	if (tmp) {
6084	serge	1286	I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
4560	Serge	1287	ret = IRQ_HANDLED;
5354	serge	1288
6084	serge	1289	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
		1290	intel_lrc_irq_handler(&dev_priv->ring[RCS]);
		1291	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
		1292	notify_ring(&dev_priv->ring[RCS]);
5354	serge	1293
6084	serge	1294	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
		1295	intel_lrc_irq_handler(&dev_priv->ring[BCS]);
		1296	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
		1297	notify_ring(&dev_priv->ring[BCS]);
4560	Serge	1298	} else
		1299	DRM_ERROR("The master control interrupt lied (GT0)!\n");
		1300	}
		1301
5060	serge	1302	if (master_ctl & (GEN8_GT_VCS1_IRQ \| GEN8_GT_VCS2_IRQ)) {
6084	serge	1303	u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
4560	Serge	1304	if (tmp) {
6084	serge	1305	I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
4560	Serge	1306	ret = IRQ_HANDLED;
5354	serge	1307
6084	serge	1308	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
		1309	intel_lrc_irq_handler(&dev_priv->ring[VCS]);
		1310	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
		1311	notify_ring(&dev_priv->ring[VCS]);
5354	serge	1312
6084	serge	1313	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
		1314	intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
		1315	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
		1316	notify_ring(&dev_priv->ring[VCS2]);
4560	Serge	1317	} else
		1318	DRM_ERROR("The master control interrupt lied (GT1)!\n");
		1319	}
		1320
6084	serge	1321	if (master_ctl & GEN8_GT_VECS_IRQ) {
		1322	u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
		1323	if (tmp) {
		1324	I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
		1325	ret = IRQ_HANDLED;
		1326
		1327	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
		1328	intel_lrc_irq_handler(&dev_priv->ring[VECS]);
		1329	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
		1330	notify_ring(&dev_priv->ring[VECS]);
		1331	} else
		1332	DRM_ERROR("The master control interrupt lied (GT3)!\n");
		1333	}
		1334
5060	serge	1335	if (master_ctl & GEN8_GT_PM_IRQ) {
6084	serge	1336	u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
5060	serge	1337	if (tmp & dev_priv->pm_rps_events) {
6084	serge	1338	I915_WRITE_FW(GEN8_GT_IIR(2),
		1339	tmp & dev_priv->pm_rps_events);
5060	serge	1340	ret = IRQ_HANDLED;
5354	serge	1341	gen6_rps_irq_handler(dev_priv, tmp);
5060	serge	1342	} else
		1343	DRM_ERROR("The master control interrupt lied (PM)!\n");
		1344	}
		1345
6084	serge	1346	return ret;
		1347	}
5354	serge	1348
6084	serge	1349	static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
		1350	{
		1351	switch (port) {
		1352	case PORT_A:
		1353	return val & PORTA_HOTPLUG_LONG_DETECT;
		1354	case PORT_B:
		1355	return val & PORTB_HOTPLUG_LONG_DETECT;
		1356	case PORT_C:
		1357	return val & PORTC_HOTPLUG_LONG_DETECT;
		1358	default:
		1359	return false;
4560	Serge	1360	}
6084	serge	1361	}
4560	Serge	1362
6084	serge	1363	static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
		1364	{
		1365	switch (port) {
		1366	case PORT_E:
		1367	return val & PORTE_HOTPLUG_LONG_DETECT;
		1368	default:
		1369	return false;
		1370	}
4560	Serge	1371	}
		1372
6084	serge	1373	static bool spt_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1374	{
		1375	switch (port) {
		1376	case PORT_A:
6084	serge	1377	return val & PORTA_HOTPLUG_LONG_DETECT;
5060	serge	1378	case PORT_B:
6084	serge	1379	return val & PORTB_HOTPLUG_LONG_DETECT;
5060	serge	1380	case PORT_C:
6084	serge	1381	return val & PORTC_HOTPLUG_LONG_DETECT;
5060	serge	1382	case PORT_D:
6084	serge	1383	return val & PORTD_HOTPLUG_LONG_DETECT;
		1384	default:
		1385	return false;
5060	serge	1386	}
		1387	}
		1388
6084	serge	1389	static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1390	{
		1391	switch (port) {
		1392	case PORT_A:
6084	serge	1393	return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
5060	serge	1394	default:
6084	serge	1395	return false;
		1396	}
		1397	}
		1398
		1399	static bool pch_port_hotplug_long_detect(enum port port, u32 val)
		1400	{
		1401	switch (port) {
5060	serge	1402	case PORT_B:
6084	serge	1403	return val & PORTB_HOTPLUG_LONG_DETECT;
5060	serge	1404	case PORT_C:
6084	serge	1405	return val & PORTC_HOTPLUG_LONG_DETECT;
5060	serge	1406	case PORT_D:
6084	serge	1407	return val & PORTD_HOTPLUG_LONG_DETECT;
		1408	default:
		1409	return false;
5060	serge	1410	}
		1411	}
		1412
6084	serge	1413	static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1414	{
6084	serge	1415	switch (port) {
		1416	case PORT_B:
		1417	return val & PORTB_HOTPLUG_INT_LONG_PULSE;
		1418	case PORT_C:
		1419	return val & PORTC_HOTPLUG_INT_LONG_PULSE;
		1420	case PORT_D:
		1421	return val & PORTD_HOTPLUG_INT_LONG_PULSE;
5060	serge	1422	default:
6084	serge	1423	return false;
5060	serge	1424	}
		1425	}
		1426
6084	serge	1427	/*
		1428	* Get a bit mask of pins that have triggered, and which ones may be long.
		1429	* This can be called multiple times with the same masks to accumulate
		1430	* hotplug detection results from several registers.
		1431	*
		1432	* Note that the caller is expected to zero out the masks initially.
		1433	*/
		1434	static void intel_get_hpd_pins(u32 pin_mask, u32 long_mask,
		1435	u32 hotplug_trigger, u32 dig_hotplug_reg,
		1436	const u32 hpd[HPD_NUM_PINS],
		1437	bool long_pulse_detect(enum port port, u32 val))
3746	Serge	1438	{
6084	serge	1439	enum port port;
3746	Serge	1440	int i;
		1441
6084	serge	1442	for_each_hpd_pin(i) {
		1443	if ((hpd[i] & hotplug_trigger) == 0)
5060	serge	1444	continue;
3746	Serge	1445
6084	serge	1446	*pin_mask \|= BIT(i);
5060	serge	1447
6296	serge	1448	if (!intel_hpd_pin_to_port(i, &port))
6131	serge	1449	continue;
5060	serge	1450
6084	serge	1451	if (long_pulse_detect(port, dig_hotplug_reg))
		1452	*long_mask \|= BIT(i);
3746	Serge	1453	}
		1454
6084	serge	1455	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
		1456	hotplug_trigger, dig_hotplug_reg, *pin_mask);
3746	Serge	1457
		1458	}
		1459
3480	Serge	1460	static void gmbus_irq_handler(struct drm_device *dev)
		1461	{
5060	serge	1462	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1463
		1464	wake_up_all(&dev_priv->gmbus_wait_queue);
		1465	}
		1466
		1467	static void dp_aux_irq_handler(struct drm_device *dev)
		1468	{
5060	serge	1469	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1470
		1471	wake_up_all(&dev_priv->gmbus_wait_queue);
		1472	}
		1473
4560	Serge	1474	#if defined(CONFIG_DEBUG_FS)
		1475	static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
		1476	uint32_t crc0, uint32_t crc1,
		1477	uint32_t crc2, uint32_t crc3,
		1478	uint32_t crc4)
		1479	{
		1480	struct drm_i915_private *dev_priv = dev->dev_private;
		1481	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
		1482	struct intel_pipe_crc_entry *entry;
		1483	int head, tail;
		1484
		1485	spin_lock(&pipe_crc->lock);
		1486
		1487	if (!pipe_crc->entries) {
		1488	spin_unlock(&pipe_crc->lock);
5354	serge	1489	DRM_DEBUG_KMS("spurious interrupt\n");
4560	Serge	1490	return;
		1491	}
		1492
		1493	head = pipe_crc->head;
		1494	tail = pipe_crc->tail;
		1495
		1496	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
		1497	spin_unlock(&pipe_crc->lock);
		1498	DRM_ERROR("CRC buffer overflowing\n");
		1499	return;
		1500	}
		1501
		1502	entry = &pipe_crc->entries[head];
		1503
		1504	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
		1505	entry->crc[0] = crc0;
		1506	entry->crc[1] = crc1;
		1507	entry->crc[2] = crc2;
		1508	entry->crc[3] = crc3;
		1509	entry->crc[4] = crc4;
		1510
		1511	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		1512	pipe_crc->head = head;
		1513
		1514	spin_unlock(&pipe_crc->lock);
		1515
		1516	wake_up_interruptible(&pipe_crc->wq);
		1517	}
		1518	#else
		1519	static inline void
		1520	display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
		1521	uint32_t crc0, uint32_t crc1,
		1522	uint32_t crc2, uint32_t crc3,
		1523	uint32_t crc4) {}
		1524	#endif
		1525
		1526
		1527	static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1528	{
		1529	struct drm_i915_private *dev_priv = dev->dev_private;
		1530
		1531	display_pipe_crc_irq_handler(dev, pipe,
		1532	I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
		1533	0, 0, 0, 0);
		1534	}
		1535
		1536	static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1537	{
		1538	struct drm_i915_private *dev_priv = dev->dev_private;
		1539
		1540	display_pipe_crc_irq_handler(dev, pipe,
		1541	I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
		1542	I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
		1543	I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
		1544	I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
		1545	I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
		1546	}
		1547
		1548	static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1549	{
		1550	struct drm_i915_private *dev_priv = dev->dev_private;
		1551	uint32_t res1, res2;
		1552
		1553	if (INTEL_INFO(dev)->gen >= 3)
		1554	res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
		1555	else
		1556	res1 = 0;
		1557
		1558	if (INTEL_INFO(dev)->gen >= 5 \|\| IS_G4X(dev))
		1559	res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
		1560	else
		1561	res2 = 0;
		1562
		1563	display_pipe_crc_irq_handler(dev, pipe,
		1564	I915_READ(PIPE_CRC_RES_RED(pipe)),
		1565	I915_READ(PIPE_CRC_RES_GREEN(pipe)),
		1566	I915_READ(PIPE_CRC_RES_BLUE(pipe)),
		1567	res1, res2);
		1568	}
		1569
4104	Serge	1570	/* The RPS events need forcewake, so we add them to a work queue and mask their
		1571	* IMR bits until the work is done. Other interrupts can be processed without
		1572	* the work queue. */
		1573	static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
		1574	{
5060	serge	1575	if (pm_iir & dev_priv->pm_rps_events) {
4104	Serge	1576	spin_lock(&dev_priv->irq_lock);
5354	serge	1577	gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
		1578	if (dev_priv->rps.interrupts_enabled) {
6084	serge	1579	dev_priv->rps.pm_iir \|= pm_iir & dev_priv->pm_rps_events;
5354	serge	1580	queue_work(dev_priv->wq, &dev_priv->rps.work);
		1581	}
4104	Serge	1582	spin_unlock(&dev_priv->irq_lock);
		1583	}
		1584
5354	serge	1585	if (INTEL_INFO(dev_priv)->gen >= 8)
		1586	return;
		1587
4104	Serge	1588	if (HAS_VEBOX(dev_priv->dev)) {
		1589	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
6084	serge	1590	notify_ring(&dev_priv->ring[VECS]);
4104	Serge	1591
5354	serge	1592	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
		1593	DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
4104	Serge	1594	}
		1595	}
		1596
5354	serge	1597	static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
		1598	{
6088	serge	1599	if (!drm_handle_vblank(dev, pipe))
		1600	return false;
5354	serge	1601
		1602	return true;
		1603	}
		1604
5060	serge	1605	static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
3031	serge	1606	{
5060	serge	1607	struct drm_i915_private *dev_priv = dev->dev_private;
		1608	u32 pipe_stats[I915_MAX_PIPES] = { };
3031	serge	1609	int pipe;
		1610
5060	serge	1611	spin_lock(&dev_priv->irq_lock);
5354	serge	1612	for_each_pipe(dev_priv, pipe) {
5060	serge	1613	int reg;
		1614	u32 mask, iir_bit = 0;
3031	serge	1615
5060	serge	1616	/*
		1617	* PIPESTAT bits get signalled even when the interrupt is
		1618	* disabled with the mask bits, and some of the status bits do
		1619	* not generate interrupts at all (like the underrun bit). Hence
		1620	* we need to be careful that we only handle what we want to
		1621	* handle.
		1622	*/
3031	serge	1623
5354	serge	1624	/* fifo underruns are filterered in the underrun handler. */
		1625	mask = PIPE_FIFO_UNDERRUN_STATUS;
		1626
5060	serge	1627	switch (pipe) {
		1628	case PIPE_A:
		1629	iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
		1630	break;
		1631	case PIPE_B:
		1632	iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
		1633	break;
		1634	case PIPE_C:
		1635	iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
		1636	break;
		1637	}
		1638	if (iir & iir_bit)
		1639	mask \|= dev_priv->pipestat_irq_mask[pipe];
3031	serge	1640
5060	serge	1641	if (!mask)
		1642	continue;
3031	serge	1643
5060	serge	1644	reg = PIPESTAT(pipe);
		1645	mask \|= PIPESTAT_INT_ENABLE_MASK;
		1646	pipe_stats[pipe] = I915_READ(reg) & mask;
3031	serge	1647
6084	serge	1648	/*
		1649	* Clear the PIPE*STAT regs before the IIR
		1650	*/
5060	serge	1651	if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS \|
		1652	PIPESTAT_INT_STATUS_MASK))
6084	serge	1653	I915_WRITE(reg, pipe_stats[pipe]);
		1654	}
5060	serge	1655	spin_unlock(&dev_priv->irq_lock);
3031	serge	1656
5354	serge	1657	for_each_pipe(dev_priv, pipe) {
6084	serge	1658	if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		1659	intel_pipe_handle_vblank(dev, pipe))
6320	serge	1660	intel_check_page_flip(dev, pipe);
3031	serge	1661
6084	serge	1662	if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
6320	serge	1663	intel_prepare_page_flip(dev, pipe);
		1664	intel_finish_page_flip(dev, pipe);
6084	serge	1665	}
4560	Serge	1666
6084	serge	1667	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		1668	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	1669
5354	serge	1670	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		1671	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
6084	serge	1672	}
3031	serge	1673
5060	serge	1674	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		1675	gmbus_irq_handler(dev);
		1676	}
3031	serge	1677
5060	serge	1678	static void i9xx_hpd_irq_handler(struct drm_device *dev)
		1679	{
		1680	struct drm_i915_private *dev_priv = dev->dev_private;
		1681	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
6084	serge	1682	u32 pin_mask = 0, long_mask = 0;
4104	Serge	1683
6084	serge	1684	if (!hotplug_status)
		1685	return;
4104	Serge	1686
6084	serge	1687	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
		1688	/*
		1689	* Make sure hotplug status is cleared before we clear IIR, or else we
		1690	* may miss hotplug events.
		1691	*/
		1692	POSTING_READ(PORT_HOTPLUG_STAT);
		1693
		1694	if (IS_G4X(dev) \|\| IS_VALLEYVIEW(dev)) {
5060	serge	1695	u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
4560	Serge	1696
6084	serge	1697	if (hotplug_trigger) {
		1698	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1699	hotplug_trigger, hpd_status_g4x,
		1700	i9xx_port_hotplug_long_detect);
		1701
6296	serge	1702	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1703	}
		1704
		1705	if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
		1706	dp_aux_irq_handler(dev);
5060	serge	1707	} else {
		1708	u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
		1709
6084	serge	1710	if (hotplug_trigger) {
		1711	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1712	hotplug_trigger, hpd_status_i915,
		1713	i9xx_port_hotplug_long_detect);
6296	serge	1714	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1715	}
5060	serge	1716	}
		1717	}
		1718
		1719	static irqreturn_t valleyview_irq_handler(int irq, void *arg)
		1720	{
		1721	struct drm_device *dev = arg;
		1722	struct drm_i915_private *dev_priv = dev->dev_private;
		1723	u32 iir, gt_iir, pm_iir;
		1724	irqreturn_t ret = IRQ_NONE;
		1725
6084	serge	1726	if (!intel_irqs_enabled(dev_priv))
		1727	return IRQ_NONE;
		1728
5060	serge	1729	while (true) {
		1730	/* Find, clear, then process each source of interrupt */
		1731
		1732	gt_iir = I915_READ(GTIIR);
		1733	if (gt_iir)
		1734	I915_WRITE(GTIIR, gt_iir);
		1735
		1736	pm_iir = I915_READ(GEN6_PMIIR);
		1737	if (pm_iir)
		1738	I915_WRITE(GEN6_PMIIR, pm_iir);
		1739
		1740	iir = I915_READ(VLV_IIR);
		1741	if (iir) {
		1742	/* Consume port before clearing IIR or we'll miss events */
		1743	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		1744	i9xx_hpd_irq_handler(dev);
		1745	I915_WRITE(VLV_IIR, iir);
3031	serge	1746	}
		1747
5060	serge	1748	if (gt_iir == 0 && pm_iir == 0 && iir == 0)
		1749	goto out;
3031	serge	1750
5060	serge	1751	ret = IRQ_HANDLED;
		1752
		1753	if (gt_iir)
6084	serge	1754	snb_gt_irq_handler(dev, dev_priv, gt_iir);
4126	Serge	1755	if (pm_iir)
		1756	gen6_rps_irq_handler(dev_priv, pm_iir);
5060	serge	1757	/* Call regardless, as some status bits might not be
		1758	* signalled in iir */
		1759	valleyview_pipestat_irq_handler(dev, iir);
3031	serge	1760	}
		1761
		1762	out:
		1763	return ret;
		1764	}
		1765
5060	serge	1766	static irqreturn_t cherryview_irq_handler(int irq, void *arg)
		1767	{
		1768	struct drm_device *dev = arg;
		1769	struct drm_i915_private *dev_priv = dev->dev_private;
		1770	u32 master_ctl, iir;
		1771	irqreturn_t ret = IRQ_NONE;
		1772
6084	serge	1773	if (!intel_irqs_enabled(dev_priv))
		1774	return IRQ_NONE;
		1775
5060	serge	1776	for (;;) {
		1777	master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		1778	iir = I915_READ(VLV_IIR);
		1779
		1780	if (master_ctl == 0 && iir == 0)
		1781	break;
		1782
		1783	ret = IRQ_HANDLED;
		1784
		1785	I915_WRITE(GEN8_MASTER_IRQ, 0);
		1786
		1787	/* Find, clear, then process each source of interrupt */
		1788
		1789	if (iir) {
		1790	/* Consume port before clearing IIR or we'll miss events */
		1791	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		1792	i9xx_hpd_irq_handler(dev);
		1793	I915_WRITE(VLV_IIR, iir);
		1794	}
		1795
6084	serge	1796	gen8_gt_irq_handler(dev_priv, master_ctl);
5060	serge	1797
		1798	/* Call regardless, as some status bits might not be
		1799	* signalled in iir */
		1800	valleyview_pipestat_irq_handler(dev, iir);
		1801
		1802	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		1803	POSTING_READ(GEN8_MASTER_IRQ);
		1804	}
		1805
		1806	return ret;
		1807	}
		1808
6084	serge	1809	static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		1810	const u32 hpd[HPD_NUM_PINS])
		1811	{
		1812	struct drm_i915_private *dev_priv = to_i915(dev);
		1813	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		1814
		1815	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		1816	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		1817
		1818	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1819	dig_hotplug_reg, hpd,
		1820	pch_port_hotplug_long_detect);
		1821
6296	serge	1822	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	1823	}
		1824
3031	serge	1825	static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
		1826	{
5060	serge	1827	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1828	int pipe;
3746	Serge	1829	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
3031	serge	1830
6084	serge	1831	if (hotplug_trigger)
		1832	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
4104	Serge	1833
		1834	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		1835	int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
		1836	SDE_AUDIO_POWER_SHIFT);
		1837	DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
		1838	port_name(port));
3746	Serge	1839	}
3031	serge	1840
3480	Serge	1841	if (pch_iir & SDE_AUX_MASK)
		1842	dp_aux_irq_handler(dev);
		1843
3031	serge	1844	if (pch_iir & SDE_GMBUS)
3480	Serge	1845	gmbus_irq_handler(dev);
3031	serge	1846
		1847	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		1848	DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
		1849
		1850	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		1851	DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
		1852
		1853	if (pch_iir & SDE_POISON)
		1854	DRM_ERROR("PCH poison interrupt\n");
		1855
		1856	if (pch_iir & SDE_FDI_MASK)
5354	serge	1857	for_each_pipe(dev_priv, pipe)
3031	serge	1858	DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
		1859	pipe_name(pipe),
		1860	I915_READ(FDI_RX_IIR(pipe)));
		1861
		1862	if (pch_iir & (SDE_TRANSB_CRC_DONE \| SDE_TRANSA_CRC_DONE))
		1863	DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
		1864
		1865	if (pch_iir & (SDE_TRANSB_CRC_ERR \| SDE_TRANSA_CRC_ERR))
		1866	DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
		1867
4104	Serge	1868	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
5354	serge	1869	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
4104	Serge	1870
3031	serge	1871	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
5354	serge	1872	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
3031	serge	1873	}
		1874
4104	Serge	1875	static void ivb_err_int_handler(struct drm_device *dev)
		1876	{
		1877	struct drm_i915_private *dev_priv = dev->dev_private;
		1878	u32 err_int = I915_READ(GEN7_ERR_INT);
4560	Serge	1879	enum pipe pipe;
4104	Serge	1880
		1881	if (err_int & ERR_INT_POISON)
		1882	DRM_ERROR("Poison interrupt\n");
		1883
5354	serge	1884	for_each_pipe(dev_priv, pipe) {
		1885	if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
		1886	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4104	Serge	1887
4560	Serge	1888	if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
		1889	if (IS_IVYBRIDGE(dev))
		1890	ivb_pipe_crc_irq_handler(dev, pipe);
		1891	else
		1892	hsw_pipe_crc_irq_handler(dev, pipe);
		1893	}
		1894	}
4104	Serge	1895
		1896	I915_WRITE(GEN7_ERR_INT, err_int);
		1897	}
		1898
		1899	static void cpt_serr_int_handler(struct drm_device *dev)
		1900	{
		1901	struct drm_i915_private *dev_priv = dev->dev_private;
		1902	u32 serr_int = I915_READ(SERR_INT);
		1903
		1904	if (serr_int & SERR_INT_POISON)
		1905	DRM_ERROR("PCH poison interrupt\n");
		1906
		1907	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
5354	serge	1908	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
4104	Serge	1909
		1910	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
5354	serge	1911	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
4104	Serge	1912
		1913	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
5354	serge	1914	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
4104	Serge	1915
		1916	I915_WRITE(SERR_INT, serr_int);
		1917	}
		1918
3031	serge	1919	static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
		1920	{
5060	serge	1921	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1922	int pipe;
3746	Serge	1923	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
3031	serge	1924
6084	serge	1925	if (hotplug_trigger)
		1926	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
4104	Serge	1927
		1928	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		1929	int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
		1930	SDE_AUDIO_POWER_SHIFT_CPT);
		1931	DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
		1932	port_name(port));
3746	Serge	1933	}
3031	serge	1934
		1935	if (pch_iir & SDE_AUX_MASK_CPT)
3480	Serge	1936	dp_aux_irq_handler(dev);
3031	serge	1937
		1938	if (pch_iir & SDE_GMBUS_CPT)
3480	Serge	1939	gmbus_irq_handler(dev);
3031	serge	1940
		1941	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		1942	DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
		1943
		1944	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		1945	DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
		1946
		1947	if (pch_iir & SDE_FDI_MASK_CPT)
5354	serge	1948	for_each_pipe(dev_priv, pipe)
3031	serge	1949	DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
		1950	pipe_name(pipe),
		1951	I915_READ(FDI_RX_IIR(pipe)));
		1952
4104	Serge	1953	if (pch_iir & SDE_ERROR_CPT)
		1954	cpt_serr_int_handler(dev);
4539	Serge	1955	}
3480	Serge	1956
6084	serge	1957	static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
		1958	{
		1959	struct drm_i915_private *dev_priv = dev->dev_private;
		1960	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		1961	~SDE_PORTE_HOTPLUG_SPT;
		1962	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
		1963	u32 pin_mask = 0, long_mask = 0;
		1964
		1965	if (hotplug_trigger) {
		1966	u32 dig_hotplug_reg;
		1967
		1968	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		1969	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		1970
		1971	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1972	dig_hotplug_reg, hpd_spt,
		1973	spt_port_hotplug_long_detect);
		1974	}
		1975
		1976	if (hotplug2_trigger) {
		1977	u32 dig_hotplug_reg;
		1978
		1979	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		1980	I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
		1981
		1982	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
		1983	dig_hotplug_reg, hpd_spt,
		1984	spt_port_hotplug2_long_detect);
		1985	}
		1986
6296	serge	1987	if (pin_mask)
		1988	intel_hpd_irq_handler(dev, pin_mask, long_mask);
		1989
6084	serge	1990	if (pch_iir & SDE_GMBUS_CPT)
		1991	gmbus_irq_handler(dev);
		1992	}
		1993
		1994	static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		1995	const u32 hpd[HPD_NUM_PINS])
		1996	{
		1997	struct drm_i915_private *dev_priv = to_i915(dev);
		1998	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		1999
		2000	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
		2001	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
		2002
		2003	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		2004	dig_hotplug_reg, hpd,
		2005	ilk_port_hotplug_long_detect);
		2006
6296	serge	2007	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	2008	}
		2009
4104	Serge	2010	static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
3031	serge	2011	{
4104	Serge	2012	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2013	enum pipe pipe;
6084	serge	2014	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
3031	serge	2015
6296	serge	2016	if (hotplug_trigger)
		2017	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
6084	serge	2018
3480	Serge	2019	if (de_iir & DE_AUX_CHANNEL_A)
		2020	dp_aux_irq_handler(dev);
		2021
3031	serge	2022	if (de_iir & DE_GSE)
4104	Serge	2023	intel_opregion_asle_intr(dev);
2351	Serge	2024
4104	Serge	2025	if (de_iir & DE_POISON)
		2026	DRM_ERROR("Poison interrupt\n");
		2027
5354	serge	2028	for_each_pipe(dev_priv, pipe) {
6084	serge	2029	if (de_iir & DE_PIPE_VBLANK(pipe) &&
		2030	intel_pipe_handle_vblank(dev, pipe))
6320	serge	2031	intel_check_page_flip(dev, pipe);
4104	Serge	2032
4560	Serge	2033	if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
5354	serge	2034	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2351	Serge	2035
4560	Serge	2036	if (de_iir & DE_PIPE_CRC_DONE(pipe))
		2037	i9xx_pipe_crc_irq_handler(dev, pipe);
		2038
		2039	/* plane/pipes map 1:1 on ilk+ */
		2040	if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
6320	serge	2041	intel_prepare_page_flip(dev, pipe);
		2042	intel_finish_page_flip_plane(dev, pipe);
4560	Serge	2043	}
3031	serge	2044	}
2351	Serge	2045
3031	serge	2046	/* check event from PCH */
		2047	if (de_iir & DE_PCH_EVENT) {
3480	Serge	2048	u32 pch_iir = I915_READ(SDEIIR);
		2049
3031	serge	2050	if (HAS_PCH_CPT(dev))
		2051	cpt_irq_handler(dev, pch_iir);
		2052	else
		2053	ibx_irq_handler(dev, pch_iir);
3480	Serge	2054
		2055	/* should clear PCH hotplug event before clear CPU irq */
		2056	I915_WRITE(SDEIIR, pch_iir);
3031	serge	2057	}
4104	Serge	2058
6084	serge	2059	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
4104	Serge	2060	ironlake_rps_change_irq_handler(dev);
2351	Serge	2061	}
		2062
4104	Serge	2063	static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
3031	serge	2064	{
		2065	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2066	enum pipe pipe;
6084	serge	2067	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2351	Serge	2068
6084	serge	2069	if (hotplug_trigger)
		2070	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
		2071
4126	Serge	2072	if (de_iir & DE_ERR_INT_IVB)
		2073	ivb_err_int_handler(dev);
2351	Serge	2074
4104	Serge	2075	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		2076	dp_aux_irq_handler(dev);
3031	serge	2077
4104	Serge	2078	if (de_iir & DE_GSE_IVB)
		2079	intel_opregion_asle_intr(dev);
4560	Serge	2080
5354	serge	2081	for_each_pipe(dev_priv, pipe) {
6084	serge	2082	if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		2083	intel_pipe_handle_vblank(dev, pipe))
6320	serge	2084	intel_check_page_flip(dev, pipe);
4560	Serge	2085
		2086	/* plane/pipes map 1:1 on ilk+ */
5060	serge	2087	if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
6320	serge	2088	intel_prepare_page_flip(dev, pipe);
		2089	intel_finish_page_flip_plane(dev, pipe);
3031	serge	2090	}
		2091	}
		2092
4104	Serge	2093	/* check event from PCH */
		2094	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		2095	u32 pch_iir = I915_READ(SDEIIR);
3031	serge	2096
4104	Serge	2097	cpt_irq_handler(dev, pch_iir);
3031	serge	2098
4104	Serge	2099	/* clear PCH hotplug event before clear CPU irq */
		2100	I915_WRITE(SDEIIR, pch_iir);
4539	Serge	2101	}
3031	serge	2102	}
		2103
5060	serge	2104	/*
		2105	* To handle irqs with the minimum potential races with fresh interrupts, we:
		2106	* 1 - Disable Master Interrupt Control.
		2107	* 2 - Find the source(s) of the interrupt.
		2108	* 3 - Clear the Interrupt Identity bits (IIR).
		2109	* 4 - Process the interrupt(s) that had bits set in the IIRs.
		2110	* 5 - Re-enable Master Interrupt Control.
		2111	*/
4104	Serge	2112	static irqreturn_t ironlake_irq_handler(int irq, void *arg)
3031	serge	2113	{
5060	serge	2114	struct drm_device *dev = arg;
		2115	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2116	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
		2117	irqreturn_t ret = IRQ_NONE;
3031	serge	2118
6084	serge	2119	if (!intel_irqs_enabled(dev_priv))
		2120	return IRQ_NONE;
		2121
4104	Serge	2122	/* We get interrupts on unclaimed registers, so check for this before we
		2123	* do any I915_{READ,WRITE}. */
		2124	intel_uncore_check_errors(dev);
3031	serge	2125
4104	Serge	2126	/* disable master interrupt before clearing iir */
		2127	de_ier = I915_READ(DEIER);
		2128	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
		2129	POSTING_READ(DEIER);
3031	serge	2130
4104	Serge	2131	/* Disable south interrupts. We'll only write to SDEIIR once, so further
		2132	* interrupts will will be stored on its back queue, and then we'll be
		2133	* able to process them after we restore SDEIER (as soon as we restore
		2134	* it, we'll get an interrupt if SDEIIR still has something to process
		2135	* due to its back queue). */
		2136	if (!HAS_PCH_NOP(dev)) {
		2137	sde_ier = I915_READ(SDEIER);
		2138	I915_WRITE(SDEIER, 0);
		2139	POSTING_READ(SDEIER);
3031	serge	2140	}
		2141
5060	serge	2142	/* Find, clear, then process each source of interrupt */
		2143
4104	Serge	2144	gt_iir = I915_READ(GTIIR);
		2145	if (gt_iir) {
5060	serge	2146	I915_WRITE(GTIIR, gt_iir);
		2147	ret = IRQ_HANDLED;
4104	Serge	2148	if (INTEL_INFO(dev)->gen >= 6)
		2149	snb_gt_irq_handler(dev, dev_priv, gt_iir);
		2150	else
		2151	ilk_gt_irq_handler(dev, dev_priv, gt_iir);
4539	Serge	2152	}
3031	serge	2153
4104	Serge	2154	de_iir = I915_READ(DEIIR);
		2155	if (de_iir) {
5060	serge	2156	I915_WRITE(DEIIR, de_iir);
		2157	ret = IRQ_HANDLED;
4104	Serge	2158	if (INTEL_INFO(dev)->gen >= 7)
		2159	ivb_display_irq_handler(dev, de_iir);
		2160	else
		2161	ilk_display_irq_handler(dev, de_iir);
3480	Serge	2162	}
		2163
4104	Serge	2164	if (INTEL_INFO(dev)->gen >= 6) {
		2165	u32 pm_iir = I915_READ(GEN6_PMIIR);
		2166	if (pm_iir) {
		2167	I915_WRITE(GEN6_PMIIR, pm_iir);
		2168	ret = IRQ_HANDLED;
5060	serge	2169	gen6_rps_irq_handler(dev_priv, pm_iir);
4560	Serge	2170	}
3031	serge	2171	}
		2172
4104	Serge	2173	I915_WRITE(DEIER, de_ier);
		2174	POSTING_READ(DEIER);
		2175	if (!HAS_PCH_NOP(dev)) {
		2176	I915_WRITE(SDEIER, sde_ier);
		2177	POSTING_READ(SDEIER);
3031	serge	2178	}
		2179
4104	Serge	2180	return ret;
3031	serge	2181	}
		2182
6084	serge	2183	static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		2184	const u32 hpd[HPD_NUM_PINS])
		2185	{
		2186	struct drm_i915_private *dev_priv = to_i915(dev);
		2187	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		2188
		2189	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		2190	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		2191
		2192	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		2193	dig_hotplug_reg, hpd,
		2194	bxt_port_hotplug_long_detect);
		2195
6296	serge	2196	intel_hpd_irq_handler(dev, pin_mask, long_mask);
6084	serge	2197	}
		2198
4560	Serge	2199	static irqreturn_t gen8_irq_handler(int irq, void *arg)
		2200	{
		2201	struct drm_device *dev = arg;
		2202	struct drm_i915_private *dev_priv = dev->dev_private;
		2203	u32 master_ctl;
		2204	irqreturn_t ret = IRQ_NONE;
		2205	uint32_t tmp = 0;
		2206	enum pipe pipe;
5354	serge	2207	u32 aux_mask = GEN8_AUX_CHANNEL_A;
4560	Serge	2208
6084	serge	2209	if (!intel_irqs_enabled(dev_priv))
		2210	return IRQ_NONE;
		2211
		2212	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2213	aux_mask \|= GEN9_AUX_CHANNEL_B \| GEN9_AUX_CHANNEL_C \|
		2214	GEN9_AUX_CHANNEL_D;
		2215
6084	serge	2216	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
4560	Serge	2217	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
		2218	if (!master_ctl)
		2219	return IRQ_NONE;
		2220
6084	serge	2221	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
4560	Serge	2222
5060	serge	2223	/* Find, clear, then process each source of interrupt */
		2224
6084	serge	2225	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
4560	Serge	2226
		2227	if (master_ctl & GEN8_DE_MISC_IRQ) {
		2228	tmp = I915_READ(GEN8_DE_MISC_IIR);
5060	serge	2229	if (tmp) {
		2230	I915_WRITE(GEN8_DE_MISC_IIR, tmp);
		2231	ret = IRQ_HANDLED;
6084	serge	2232	if (tmp & GEN8_DE_MISC_GSE)
		2233	intel_opregion_asle_intr(dev);
5060	serge	2234	else
6084	serge	2235	DRM_ERROR("Unexpected DE Misc interrupt\n");
5060	serge	2236	}
4560	Serge	2237	else
		2238	DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
		2239	}
		2240
		2241	if (master_ctl & GEN8_DE_PORT_IRQ) {
		2242	tmp = I915_READ(GEN8_DE_PORT_IIR);
5060	serge	2243	if (tmp) {
6084	serge	2244	bool found = false;
		2245	u32 hotplug_trigger = 0;
		2246
		2247	if (IS_BROXTON(dev_priv))
		2248	hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
		2249	else if (IS_BROADWELL(dev_priv))
		2250	hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
		2251
5060	serge	2252	I915_WRITE(GEN8_DE_PORT_IIR, tmp);
		2253	ret = IRQ_HANDLED;
5354	serge	2254
6084	serge	2255	if (tmp & aux_mask) {
		2256	dp_aux_irq_handler(dev);
		2257	found = true;
		2258	}
		2259
		2260	if (hotplug_trigger) {
		2261	if (IS_BROXTON(dev))
		2262	bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
		2263	else
		2264	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
		2265	found = true;
		2266	}
		2267
		2268	if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
		2269	gmbus_irq_handler(dev);
		2270	found = true;
		2271	}
		2272
		2273	if (!found)
		2274	DRM_ERROR("Unexpected DE Port interrupt\n");
5060	serge	2275	}
4560	Serge	2276	else
		2277	DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
		2278	}
		2279
5354	serge	2280	for_each_pipe(dev_priv, pipe) {
		2281	uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
4560	Serge	2282
		2283	if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
		2284	continue;
		2285
		2286	pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
5060	serge	2287	if (pipe_iir) {
		2288	ret = IRQ_HANDLED;
		2289	I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
4560	Serge	2290
6088	serge	2291	if (pipe_iir & GEN8_PIPE_VBLANK &&
		2292	intel_pipe_handle_vblank(dev, pipe))
6320	serge	2293	intel_check_page_flip(dev, pipe);
4560	Serge	2294
6084	serge	2295	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2296	flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
		2297	else
		2298	flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
		2299
6320	serge	2300	if (flip_done) {
		2301	intel_prepare_page_flip(dev, pipe);
		2302	intel_finish_page_flip_plane(dev, pipe);
		2303	}
5354	serge	2304
6084	serge	2305	if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
		2306	hsw_pipe_crc_irq_handler(dev, pipe);
4560	Serge	2307
5354	serge	2308	if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
		2309	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		2310	pipe);
4560	Serge	2311
5354	serge	2312
6084	serge	2313	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2314	fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		2315	else
		2316	fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
		2317
		2318	if (fault_errors)
6084	serge	2319	DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
		2320	pipe_name(pipe),
		2321	pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
4560	Serge	2322	} else
		2323	DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
		2324	}
		2325
6084	serge	2326	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
		2327	master_ctl & GEN8_DE_PCH_IRQ) {
4560	Serge	2328	/*
		2329	* FIXME(BDW): Assume for now that the new interrupt handling
		2330	* scheme also closed the SDE interrupt handling race we've seen
		2331	* on older pch-split platforms. But this needs testing.
		2332	*/
		2333	u32 pch_iir = I915_READ(SDEIIR);
		2334	if (pch_iir) {
		2335	I915_WRITE(SDEIIR, pch_iir);
		2336	ret = IRQ_HANDLED;
6084	serge	2337
		2338	if (HAS_PCH_SPT(dev_priv))
		2339	spt_irq_handler(dev, pch_iir);
		2340	else
		2341	cpt_irq_handler(dev, pch_iir);
6320	serge	2342	} else {
		2343	/*
		2344	* Like on previous PCH there seems to be something
		2345	* fishy going on with forwarding PCH interrupts.
		2346	*/
		2347	DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		2348	}
4560	Serge	2349	}
		2350
6084	serge	2351	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
		2352	POSTING_READ_FW(GEN8_MASTER_IRQ);
4560	Serge	2353
		2354	return ret;
		2355	}
		2356
4104	Serge	2357	static void i915_error_wake_up(struct drm_i915_private *dev_priv,
		2358	bool reset_completed)
3746	Serge	2359	{
5060	serge	2360	struct intel_engine_cs *ring;
4104	Serge	2361	int i;
3031	serge	2362
4104	Serge	2363	/*
		2364	* Notify all waiters for GPU completion events that reset state has
		2365	* been changed, and that they need to restart their wait after
		2366	* checking for potential errors (and bail out to drop locks if there is
		2367	* a gpu reset pending so that i915_error_work_func can acquire them).
		2368	*/
3031	serge	2369
4104	Serge	2370	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
		2371	for_each_ring(ring, dev_priv, i)
		2372	wake_up_all(&ring->irq_queue);
3031	serge	2373
6320	serge	2374	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
		2375	wake_up_all(&dev_priv->pending_flip_queue);
3031	serge	2376
4104	Serge	2377	/*
		2378	* Signal tasks blocked in i915_gem_wait_for_error that the pending
		2379	* reset state is cleared.
		2380	*/
		2381	if (reset_completed)
		2382	wake_up_all(&dev_priv->gpu_error.reset_queue);
3031	serge	2383	}
		2384
		2385	/**
6084	serge	2386	* i915_reset_and_wakeup - do process context error handling work
		2387	* @dev: drm device
3031	serge	2388	*
4104	Serge	2389	* Fire an error uevent so userspace can see that a hang or error
		2390	* was detected.
3031	serge	2391	*/
6084	serge	2392	static void i915_reset_and_wakeup(struct drm_device *dev)
3031	serge	2393	{
6084	serge	2394	struct drm_i915_private *dev_priv = to_i915(dev);
		2395	struct i915_gpu_error *error = &dev_priv->gpu_error;
4104	Serge	2396	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
		2397	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
		2398	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
		2399	int ret;
3031	serge	2400
4104	Serge	2401	/*
		2402	* Note that there's only one work item which does gpu resets, so we
		2403	* need not worry about concurrent gpu resets potentially incrementing
		2404	* error->reset_counter twice. We only need to take care of another
		2405	* racing irq/hangcheck declaring the gpu dead for a second time. A
		2406	* quick check for that is good enough: schedule_work ensures the
		2407	* correct ordering between hang detection and this work item, and since
		2408	* the reset in-progress bit is only ever set by code outside of this
		2409	* work we don't need to worry about any other races.
		2410	*/
		2411	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
		2412	DRM_DEBUG_DRIVER("resetting chip\n");
6084	serge	2413	intel_runtime_pm_get(dev_priv);
3031	serge	2414
4104	Serge	2415	/*
		2416	* All state reset _must_ be completed before we update the
		2417	* reset counter, for otherwise waiters might miss the reset
		2418	* pending state and not properly drop locks, resulting in
		2419	* deadlocks with the reset work.
		2420	*/
4560	Serge	2421	// ret = i915_reset(dev);
3031	serge	2422
6084	serge	2423	// intel_finish_reset(dev);
3031	serge	2424
6084	serge	2425	intel_runtime_pm_put(dev_priv);
		2426
4104	Serge	2427	if (ret == 0) {
		2428	/*
		2429	* After all the gem state is reset, increment the reset
		2430	* counter and wake up everyone waiting for the reset to
		2431	* complete.
		2432	*
		2433	* Since unlock operations are a one-sided barrier only,
		2434	* we need to insert a barrier here to order any seqno
		2435	* updates before
		2436	* the counter increment.
		2437	*/
6084	serge	2438	smp_mb__before_atomic();
4104	Serge	2439	atomic_inc(&dev_priv->gpu_error.reset_counter);
3031	serge	2440
4104	Serge	2441	} else {
6088	serge	2442	atomic_or(I915_WEDGED, &error->reset_counter);
		2443	}
3031	serge	2444
4104	Serge	2445	/*
		2446	* Note: The wake_up also serves as a memory barrier so that
		2447	* waiters see the update value of the reset counter atomic_t.
		2448	*/
		2449	i915_error_wake_up(dev_priv, true);
3031	serge	2450	}
		2451	}
		2452
		2453	static void i915_report_and_clear_eir(struct drm_device *dev)
		2454	{
		2455	struct drm_i915_private *dev_priv = dev->dev_private;
		2456	uint32_t instdone[I915_NUM_INSTDONE_REG];
		2457	u32 eir = I915_READ(EIR);
		2458	int pipe, i;
		2459
		2460	if (!eir)
		2461	return;
		2462
		2463	pr_err("render error detected, EIR: 0x%08x\n", eir);
		2464
		2465	i915_get_extra_instdone(dev, instdone);
		2466
		2467	if (IS_G4X(dev)) {
		2468	if (eir & (GM45_ERROR_MEM_PRIV \| GM45_ERROR_CP_PRIV)) {
		2469	u32 ipeir = I915_READ(IPEIR_I965);
		2470
		2471	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
		2472	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
		2473	for (i = 0; i < ARRAY_SIZE(instdone); i++)
		2474	pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
		2475	pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
		2476	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
		2477	I915_WRITE(IPEIR_I965, ipeir);
		2478	POSTING_READ(IPEIR_I965);
		2479	}
		2480	if (eir & GM45_ERROR_PAGE_TABLE) {
		2481	u32 pgtbl_err = I915_READ(PGTBL_ER);
		2482	pr_err("page table error\n");
		2483	pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
		2484	I915_WRITE(PGTBL_ER, pgtbl_err);
		2485	POSTING_READ(PGTBL_ER);
		2486	}
		2487	}
		2488
		2489	if (!IS_GEN2(dev)) {
		2490	if (eir & I915_ERROR_PAGE_TABLE) {
		2491	u32 pgtbl_err = I915_READ(PGTBL_ER);
		2492	pr_err("page table error\n");
		2493	pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
		2494	I915_WRITE(PGTBL_ER, pgtbl_err);
		2495	POSTING_READ(PGTBL_ER);
		2496	}
		2497	}
		2498
		2499	if (eir & I915_ERROR_MEMORY_REFRESH) {
		2500	pr_err("memory refresh error:\n");
5354	serge	2501	for_each_pipe(dev_priv, pipe)
3031	serge	2502	pr_err("pipe %c stat: 0x%08x\n",
		2503	pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
		2504	/* pipestat has already been acked */
		2505	}
		2506	if (eir & I915_ERROR_INSTRUCTION) {
		2507	pr_err("instruction error\n");
		2508	pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
		2509	for (i = 0; i < ARRAY_SIZE(instdone); i++)
		2510	pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
		2511	if (INTEL_INFO(dev)->gen < 4) {
		2512	u32 ipeir = I915_READ(IPEIR);
		2513
		2514	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
		2515	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
		2516	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
		2517	I915_WRITE(IPEIR, ipeir);
		2518	POSTING_READ(IPEIR);
		2519	} else {
		2520	u32 ipeir = I915_READ(IPEIR_I965);
		2521
		2522	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
		2523	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
		2524	pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
		2525	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
		2526	I915_WRITE(IPEIR_I965, ipeir);
		2527	POSTING_READ(IPEIR_I965);
		2528	}
		2529	}
		2530
		2531	I915_WRITE(EIR, eir);
		2532	POSTING_READ(EIR);
		2533	eir = I915_READ(EIR);
		2534	if (eir) {
		2535	/*
		2536	* some errors might have become stuck,
		2537	* mask them.
		2538	*/
		2539	DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		2540	I915_WRITE(EMR, I915_READ(EMR) \| eir);
		2541	I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
		2542	}
		2543	}
		2544
		2545	/**
6084	serge	2546	* i915_handle_error - handle a gpu error
3031	serge	2547	* @dev: drm device
		2548	*
6084	serge	2549	* Do some basic checking of register state at error time and
3031	serge	2550	* dump it to the syslog. Also call i915_capture_error_state() to make
		2551	* sure we get a record and make it available in debugfs. Fire a uevent
		2552	* so userspace knows something bad happened (should trigger collection
		2553	* of a ring dump etc.).
		2554	*/
5060	serge	2555	void i915_handle_error(struct drm_device *dev, bool wedged,
		2556	const char *fmt, ...)
3031	serge	2557	{
		2558	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2559	va_list args;
		2560	char error_msg[80];
3031	serge	2561
5060	serge	2562	va_start(args, fmt);
		2563	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
		2564	va_end(args);
		2565
4560	Serge	2566	// i915_capture_error_state(dev);
3031	serge	2567	i915_report_and_clear_eir(dev);
		2568
		2569	if (wedged) {
6084	serge	2570	atomic_or(I915_RESET_IN_PROGRESS_FLAG,
3480	Serge	2571	&dev_priv->gpu_error.reset_counter);
3031	serge	2572
		2573	/*
6084	serge	2574	* Wakeup waiting processes so that the reset function
		2575	* i915_reset_and_wakeup doesn't deadlock trying to grab
		2576	* various locks. By bumping the reset counter first, the woken
4104	Serge	2577	* processes will see a reset in progress and back off,
		2578	* releasing their locks and then wait for the reset completion.
		2579	* We must do this for _all_ gpu waiters that might hold locks
		2580	* that the reset work needs to acquire.
		2581	*
		2582	* Note: The wake_up serves as the required memory barrier to
		2583	* ensure that the waiters see the updated value of the reset
		2584	* counter atomic_t.
3031	serge	2585	*/
4104	Serge	2586	i915_error_wake_up(dev_priv, false);
3031	serge	2587	}
		2588
6084	serge	2589	i915_reset_and_wakeup(dev);
3031	serge	2590	}
		2591
		2592	/* Called from drm generic code, passed 'crtc' which
		2593	* we use as a pipe index
		2594	*/
6084	serge	2595	static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2596	{
5060	serge	2597	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2598	unsigned long irqflags;
		2599
		2600	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2601	if (INTEL_INFO(dev)->gen >= 4)
		2602	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2603	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2604	else
		2605	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2606	PIPE_VBLANK_INTERRUPT_STATUS);
3031	serge	2607	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2608
		2609	return 0;
		2610	}
		2611
6084	serge	2612	static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2613	{
5060	serge	2614	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2615	unsigned long irqflags;
4104	Serge	2616	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
4560	Serge	2617	DE_PIPE_VBLANK(pipe);
3031	serge	2618
		2619	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4104	Serge	2620	ironlake_enable_display_irq(dev_priv, bit);
3031	serge	2621	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2622
		2623	return 0;
		2624	}
		2625
6084	serge	2626	static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2627	{
5060	serge	2628	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2629	unsigned long irqflags;
		2630
		2631	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2632	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2633	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2634	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2635
		2636	return 0;
		2637	}
		2638
6084	serge	2639	static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
4560	Serge	2640	{
		2641	struct drm_i915_private *dev_priv = dev->dev_private;
		2642	unsigned long irqflags;
		2643
		2644	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2645	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
		2646	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		2647	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
		2648	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2649	return 0;
		2650	}
		2651
3031	serge	2652	/* Called from drm generic code, passed 'crtc' which
		2653	* we use as a pipe index
		2654	*/
6084	serge	2655	static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2656	{
5060	serge	2657	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2658	unsigned long irqflags;
		2659
		2660	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2661	i915_disable_pipestat(dev_priv, pipe,
5060	serge	2662	PIPE_VBLANK_INTERRUPT_STATUS \|
		2663	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2664	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2665	}
		2666
6084	serge	2667	static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2668	{
5060	serge	2669	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2670	unsigned long irqflags;
4104	Serge	2671	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
4560	Serge	2672	DE_PIPE_VBLANK(pipe);
3031	serge	2673
		2674	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4104	Serge	2675	ironlake_disable_display_irq(dev_priv, bit);
3031	serge	2676	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2677	}
		2678
6084	serge	2679	static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2680	{
5060	serge	2681	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2682	unsigned long irqflags;
		2683
		2684	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2685	i915_disable_pipestat(dev_priv, pipe,
5060	serge	2686	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2687	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2688	}
		2689
6084	serge	2690	static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
4560	Serge	2691	{
		2692	struct drm_i915_private *dev_priv = dev->dev_private;
		2693	unsigned long irqflags;
		2694
		2695	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2696	dev_priv->de_irq_mask[pipe] \|= GEN8_PIPE_VBLANK;
		2697	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		2698	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
		2699	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2700	}
		2701
4104	Serge	2702	static bool
5060	serge	2703	ring_idle(struct intel_engine_cs *ring, u32 seqno)
2351	Serge	2704	{
4104	Serge	2705	return (list_empty(&ring->request_list) \|\|
6084	serge	2706	i915_seqno_passed(seqno, ring->last_submitted_seqno));
4104	Serge	2707	}
2351	Serge	2708
5060	serge	2709	static bool
		2710	ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
4104	Serge	2711	{
5060	serge	2712	if (INTEL_INFO(dev)->gen >= 8) {
		2713	return (ipehr >> 23) == 0x1c;
		2714	} else {
		2715	ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		2716	return ipehr == (MI_SEMAPHORE_MBOX \| MI_SEMAPHORE_COMPARE \|
		2717	MI_SEMAPHORE_REGISTER);
		2718	}
		2719	}
		2720
		2721	static struct intel_engine_cs *
		2722	semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
		2723	{
4104	Serge	2724	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	2725	struct intel_engine_cs *signaller;
		2726	int i;
2351	Serge	2727
5060	serge	2728	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
		2729	for_each_ring(signaller, dev_priv, i) {
		2730	if (ring == signaller)
		2731	continue;
		2732
		2733	if (offset == signaller->semaphore.signal_ggtt[ring->id])
		2734	return signaller;
		2735	}
		2736	} else {
		2737	u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
		2738
		2739	for_each_ring(signaller, dev_priv, i) {
		2740	if(ring == signaller)
		2741	continue;
		2742
		2743	if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
		2744	return signaller;
		2745	}
		2746	}
		2747
		2748	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		2749	ring->id, ipehr, offset);
		2750
		2751	return NULL;
		2752	}
		2753
		2754	static struct intel_engine_cs *
		2755	semaphore_waits_for(struct intel_engine_cs ring, u32 seqno)
		2756	{
		2757	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		2758	u32 cmd, ipehr, head;
		2759	u64 offset = 0;
		2760	int i, backwards;
		2761
6084	serge	2762	/*
		2763	* This function does not support execlist mode - any attempt to
		2764	* proceed further into this function will result in a kernel panic
		2765	* when dereferencing ring->buffer, which is not set up in execlist
		2766	* mode.
		2767	*
		2768	* The correct way of doing it would be to derive the currently
		2769	* executing ring buffer from the current context, which is derived
		2770	* from the currently running request. Unfortunately, to get the
		2771	* current request we would have to grab the struct_mutex before doing
		2772	* anything else, which would be ill-advised since some other thread
		2773	* might have grabbed it already and managed to hang itself, causing
		2774	* the hang checker to deadlock.
		2775	*
		2776	* Therefore, this function does not support execlist mode in its
		2777	* current form. Just return NULL and move on.
		2778	*/
		2779	if (ring->buffer == NULL)
		2780	return NULL;
		2781
4104	Serge	2782	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
5060	serge	2783	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
4104	Serge	2784	return NULL;
2351	Serge	2785
5060	serge	2786	/*
		2787	* HEAD is likely pointing to the dword after the actual command,
		2788	* so scan backwards until we find the MBOX. But limit it to just 3
		2789	* or 4 dwords depending on the semaphore wait command size.
		2790	* Note that we don't care about ACTHD here since that might
		2791	* point at at batch, and semaphores are always emitted into the
		2792	* ringbuffer itself.
4104	Serge	2793	*/
5060	serge	2794	head = I915_READ_HEAD(ring) & HEAD_ADDR;
		2795	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
		2796
		2797	for (i = backwards; i; --i) {
		2798	/*
		2799	* Be paranoid and presume the hw has gone off into the wild -
		2800	* our ring is smaller than what the hardware (and hence
		2801	* HEAD_ADDR) allows. Also handles wrap-around.
		2802	*/
		2803	head &= ring->buffer->size - 1;
		2804
		2805	/* This here seems to blow up */
		2806	cmd = ioread32(ring->buffer->virtual_start + head);
4104	Serge	2807	if (cmd == ipehr)
		2808	break;
2351	Serge	2809
5060	serge	2810	head -= 4;
		2811	}
		2812
		2813	if (!i)
6084	serge	2814	return NULL;
2351	Serge	2815
5060	serge	2816	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
		2817	if (INTEL_INFO(ring->dev)->gen >= 8) {
		2818	offset = ioread32(ring->buffer->virtual_start + head + 12);
		2819	offset <<= 32;
		2820	offset = ioread32(ring->buffer->virtual_start + head + 8);
		2821	}
		2822	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
4104	Serge	2823	}
2351	Serge	2824
5060	serge	2825	static int semaphore_passed(struct intel_engine_cs *ring)
4104	Serge	2826	{
		2827	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	2828	struct intel_engine_cs *signaller;
		2829	u32 seqno;
4104	Serge	2830
5060	serge	2831	ring->hangcheck.deadlock++;
4104	Serge	2832
		2833	signaller = semaphore_waits_for(ring, &seqno);
5060	serge	2834	if (signaller == NULL)
4104	Serge	2835	return -1;
		2836
5060	serge	2837	/* Prevent pathological recursion due to driver bugs */
		2838	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
		2839	return -1;
		2840
		2841	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		2842	return 1;
		2843
4104	Serge	2844	/* cursory check for an unkickable deadlock */
5060	serge	2845	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
		2846	semaphore_passed(signaller) < 0)
4104	Serge	2847	return -1;
		2848
5060	serge	2849	return 0;
4104	Serge	2850	}
		2851
		2852	static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
		2853	{
5060	serge	2854	struct intel_engine_cs *ring;
4104	Serge	2855	int i;
		2856
		2857	for_each_ring(ring, dev_priv, i)
5060	serge	2858	ring->hangcheck.deadlock = 0;
4104	Serge	2859	}
		2860
		2861	static enum intel_ring_hangcheck_action
5060	serge	2862	ring_stuck(struct intel_engine_cs *ring, u64 acthd)
4104	Serge	2863	{
		2864	struct drm_device *dev = ring->dev;
		2865	struct drm_i915_private *dev_priv = dev->dev_private;
		2866	u32 tmp;
		2867
5060	serge	2868	if (acthd != ring->hangcheck.acthd) {
		2869	if (acthd > ring->hangcheck.max_acthd) {
		2870	ring->hangcheck.max_acthd = acthd;
6084	serge	2871	return HANGCHECK_ACTIVE;
5060	serge	2872	}
4104	Serge	2873
5060	serge	2874	return HANGCHECK_ACTIVE_LOOP;
		2875	}
		2876
4104	Serge	2877	if (IS_GEN2(dev))
		2878	return HANGCHECK_HUNG;
		2879
		2880	/* Is the chip hanging on a WAIT_FOR_EVENT?
		2881	* If so we can simply poke the RB_WAIT bit
		2882	* and break the hang. This should work on
		2883	* all but the second generation chipsets.
		2884	*/
		2885	tmp = I915_READ_CTL(ring);
		2886	if (tmp & RING_WAIT) {
5060	serge	2887	i915_handle_error(dev, false,
		2888	"Kicking stuck wait on %s",
6084	serge	2889	ring->name);
4104	Serge	2890	I915_WRITE_CTL(ring, tmp);
		2891	return HANGCHECK_KICK;
		2892	}
		2893
		2894	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		2895	switch (semaphore_passed(ring)) {
		2896	default:
		2897	return HANGCHECK_HUNG;
		2898	case 1:
5060	serge	2899	i915_handle_error(dev, false,
		2900	"Kicking stuck semaphore on %s",
6084	serge	2901	ring->name);
4104	Serge	2902	I915_WRITE_CTL(ring, tmp);
		2903	return HANGCHECK_KICK;
		2904	case 0:
		2905	return HANGCHECK_WAIT;
		2906	}
		2907	}
		2908
		2909	return HANGCHECK_HUNG;
		2910	}
		2911
6084	serge	2912	/*
4104	Serge	2913	* This is called when the chip hasn't reported back with completed
		2914	* batchbuffers in a long time. We keep track per ring seqno progress and
		2915	* if there are no progress, hangcheck score for that ring is increased.
		2916	* Further, acthd is inspected to see if the ring is stuck. On stuck case
		2917	* we kick the ring. If we see no progress on three subsequent calls
		2918	* we assume chip is wedged and try to fix it by resetting the chip.
		2919	*/
6084	serge	2920	static void i915_hangcheck_elapsed(struct work_struct *work)
4104	Serge	2921	{
6084	serge	2922	struct drm_i915_private *dev_priv =
		2923	container_of(work, typeof(*dev_priv),
		2924	gpu_error.hangcheck_work.work);
		2925	struct drm_device *dev = dev_priv->dev;
5060	serge	2926	struct intel_engine_cs *ring;
4104	Serge	2927	int i;
		2928	int busy_count = 0, rings_hung = 0;
		2929	bool stuck[I915_NUM_RINGS] = { 0 };
		2930	#define BUSY 1
		2931	#define KICK 5
		2932	#define HUNG 20
		2933
5060	serge	2934	if (!i915.enable_hangcheck)
4104	Serge	2935	return;
		2936
		2937	for_each_ring(ring, dev_priv, i) {
5060	serge	2938	u64 acthd;
		2939	u32 seqno;
4104	Serge	2940	bool busy = true;
		2941
		2942	semaphore_clear_deadlocks(dev_priv);
		2943
		2944	seqno = ring->get_seqno(ring, false);
		2945	acthd = intel_ring_get_active_head(ring);
		2946
		2947	if (ring->hangcheck.seqno == seqno) {
		2948	if (ring_idle(ring, seqno)) {
5060	serge	2949	ring->hangcheck.action = HANGCHECK_IDLE;
		2950
6084	serge	2951	if (waitqueue_active(&ring->irq_queue)) {
4104	Serge	2952	/* Issue a wake-up to catch stuck h/w. */
6084	serge	2953	if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
		2954	if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
		2955	DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
		2956	ring->name);
		2957	else
		2958	DRM_INFO("Fake missed irq on %s\n",
		2959	ring->name);
		2960	wake_up_all(&ring->irq_queue);
		2961	}
		2962	/* Safeguard against driver failure */
		2963	ring->hangcheck.score += BUSY;
		2964	} else
4104	Serge	2965	busy = false;
		2966	} else {
		2967	/* We always increment the hangcheck score
		2968	* if the ring is busy and still processing
		2969	* the same request, so that no single request
		2970	* can run indefinitely (such as a chain of
		2971	* batches). The only time we do not increment
		2972	* the hangcheck score on this ring, if this
		2973	* ring is in a legitimate wait for another
		2974	* ring. In that case the waiting ring is a
		2975	* victim and we want to be sure we catch the
		2976	* right culprit. Then every time we do kick
		2977	* the ring, add a small increment to the
		2978	* score so that we can catch a batch that is
		2979	* being repeatedly kicked and so responsible
		2980	* for stalling the machine.
		2981	*/
		2982	ring->hangcheck.action = ring_stuck(ring,
		2983	acthd);
		2984
		2985	switch (ring->hangcheck.action) {
4560	Serge	2986	case HANGCHECK_IDLE:
4104	Serge	2987	case HANGCHECK_WAIT:
5060	serge	2988	case HANGCHECK_ACTIVE:
4104	Serge	2989	break;
5060	serge	2990	case HANGCHECK_ACTIVE_LOOP:
4104	Serge	2991	ring->hangcheck.score += BUSY;
		2992	break;
		2993	case HANGCHECK_KICK:
		2994	ring->hangcheck.score += KICK;
		2995	break;
		2996	case HANGCHECK_HUNG:
		2997	ring->hangcheck.score += HUNG;
		2998	stuck[i] = true;
		2999	break;
		3000	}
		3001	}
		3002	} else {
4560	Serge	3003	ring->hangcheck.action = HANGCHECK_ACTIVE;
		3004
4104	Serge	3005	/* Gradually reduce the count so that we catch DoS
		3006	* attempts across multiple batches.
		3007	*/
		3008	if (ring->hangcheck.score > 0)
		3009	ring->hangcheck.score--;
5060	serge	3010
		3011	ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
4104	Serge	3012	}
		3013
		3014	ring->hangcheck.seqno = seqno;
		3015	ring->hangcheck.acthd = acthd;
		3016	busy_count += busy;
		3017	}
		3018
		3019	for_each_ring(ring, dev_priv, i) {
5060	serge	3020	if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
4104	Serge	3021	DRM_INFO("%s on %s\n",
6084	serge	3022	stuck[i] ? "stuck" : "no progress",
		3023	ring->name);
4104	Serge	3024	rings_hung++;
		3025	}
		3026	}
		3027
		3028	// if (rings_hung)
		3029	// return i915_handle_error(dev, true);
		3030
		3031	}
6088	serge	3032
5060	serge	3033	static void ibx_irq_reset(struct drm_device *dev)
		3034	{
		3035	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3036
5060	serge	3037	if (HAS_PCH_NOP(dev))
		3038	return;
		3039
		3040	GEN5_IRQ_RESET(SDE);
		3041
		3042	if (HAS_PCH_CPT(dev) \|\| HAS_PCH_LPT(dev))
		3043	I915_WRITE(SERR_INT, 0xffffffff);
		3044	}
		3045
		3046	/*
		3047	* SDEIER is also touched by the interrupt handler to work around missed PCH
		3048	* interrupts. Hence we can't update it after the interrupt handler is enabled -
		3049	* instead we unconditionally enable all PCH interrupt sources here, but then
		3050	* only unmask them as needed with SDEIMR.
		3051	*
		3052	* This function needs to be called before interrupts are enabled.
		3053	*/
		3054	static void ibx_irq_pre_postinstall(struct drm_device *dev)
4104	Serge	3055	{
		3056	struct drm_i915_private *dev_priv = dev->dev_private;
		3057
3746	Serge	3058	if (HAS_PCH_NOP(dev))
		3059	return;
		3060
5060	serge	3061	WARN_ON(I915_READ(SDEIER) != 0);
3746	Serge	3062	I915_WRITE(SDEIER, 0xffffffff);
4104	Serge	3063	POSTING_READ(SDEIER);
2351	Serge	3064	}
		3065
5060	serge	3066	static void gen5_gt_irq_reset(struct drm_device *dev)
4104	Serge	3067	{
		3068	struct drm_i915_private *dev_priv = dev->dev_private;
		3069
5060	serge	3070	GEN5_IRQ_RESET(GT);
		3071	if (INTEL_INFO(dev)->gen >= 6)
		3072	GEN5_IRQ_RESET(GEN6_PM);
4104	Serge	3073	}
		3074
		3075	/* drm_dma.h hooks
		3076	*/
5060	serge	3077	static void ironlake_irq_reset(struct drm_device *dev)
4104	Serge	3078	{
5060	serge	3079	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3080
5060	serge	3081	I915_WRITE(HWSTAM, 0xffffffff);
4104	Serge	3082
5060	serge	3083	GEN5_IRQ_RESET(DE);
		3084	if (IS_GEN7(dev))
		3085	I915_WRITE(GEN7_ERR_INT, 0xffffffff);
4104	Serge	3086
5060	serge	3087	gen5_gt_irq_reset(dev);
4104	Serge	3088
5060	serge	3089	ibx_irq_reset(dev);
4104	Serge	3090	}
		3091
5354	serge	3092	static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
		3093	{
		3094	enum pipe pipe;
		3095
6084	serge	3096	i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
5354	serge	3097	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		3098
		3099	for_each_pipe(dev_priv, pipe)
		3100	I915_WRITE(PIPESTAT(pipe), 0xffff);
		3101
		3102	GEN5_IRQ_RESET(VLV_);
		3103	}
		3104
3031	serge	3105	static void valleyview_irq_preinstall(struct drm_device *dev)
		3106	{
5060	serge	3107	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3108
		3109	/* VLV magic */
		3110	I915_WRITE(VLV_IMR, 0);
		3111	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
		3112	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
		3113	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
		3114
5060	serge	3115	gen5_gt_irq_reset(dev);
4104	Serge	3116
5354	serge	3117	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3031	serge	3118
5354	serge	3119	vlv_display_irq_reset(dev_priv);
3031	serge	3120	}
		3121
5060	serge	3122	static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
4560	Serge	3123	{
5060	serge	3124	GEN8_IRQ_RESET_NDX(GT, 0);
		3125	GEN8_IRQ_RESET_NDX(GT, 1);
		3126	GEN8_IRQ_RESET_NDX(GT, 2);
		3127	GEN8_IRQ_RESET_NDX(GT, 3);
		3128	}
		3129
		3130	static void gen8_irq_reset(struct drm_device *dev)
		3131	{
4560	Serge	3132	struct drm_i915_private *dev_priv = dev->dev_private;
		3133	int pipe;
		3134
		3135	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3136	POSTING_READ(GEN8_MASTER_IRQ);
		3137
5060	serge	3138	gen8_gt_irq_reset(dev_priv);
4560	Serge	3139
5354	serge	3140	for_each_pipe(dev_priv, pipe)
		3141	if (intel_display_power_is_enabled(dev_priv,
6084	serge	3142	POWER_DOMAIN_PIPE(pipe)))
		3143	GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
4560	Serge	3144
5060	serge	3145	GEN5_IRQ_RESET(GEN8_DE_PORT_);
		3146	GEN5_IRQ_RESET(GEN8_DE_MISC_);
		3147	GEN5_IRQ_RESET(GEN8_PCU_);
4560	Serge	3148
6084	serge	3149	if (HAS_PCH_SPLIT(dev))
		3150	ibx_irq_reset(dev);
5060	serge	3151	}
4560	Serge	3152
6084	serge	3153	void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
		3154	unsigned int pipe_mask)
5060	serge	3155	{
5354	serge	3156	uint32_t extra_ier = GEN8_PIPE_VBLANK \| GEN8_PIPE_FIFO_UNDERRUN;
4560	Serge	3157
5354	serge	3158	spin_lock_irq(&dev_priv->irq_lock);
6084	serge	3159	if (pipe_mask & 1 << PIPE_A)
		3160	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
		3161	dev_priv->de_irq_mask[PIPE_A],
		3162	~dev_priv->de_irq_mask[PIPE_A] \| extra_ier);
		3163	if (pipe_mask & 1 << PIPE_B)
		3164	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
		3165	dev_priv->de_irq_mask[PIPE_B],
		3166	~dev_priv->de_irq_mask[PIPE_B] \| extra_ier);
		3167	if (pipe_mask & 1 << PIPE_C)
		3168	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
		3169	dev_priv->de_irq_mask[PIPE_C],
		3170	~dev_priv->de_irq_mask[PIPE_C] \| extra_ier);
5354	serge	3171	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	3172	}
		3173
		3174	static void cherryview_irq_preinstall(struct drm_device *dev)
		3175	{
		3176	struct drm_i915_private *dev_priv = dev->dev_private;
		3177
		3178	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3179	POSTING_READ(GEN8_MASTER_IRQ);
		3180
		3181	gen8_gt_irq_reset(dev_priv);
		3182
		3183	GEN5_IRQ_RESET(GEN8_PCU_);
		3184
		3185	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
		3186
5354	serge	3187	vlv_display_irq_reset(dev_priv);
4560	Serge	3188	}
		3189
6084	serge	3190	static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
		3191	const u32 hpd[HPD_NUM_PINS])
		3192	{
		3193	struct drm_i915_private *dev_priv = to_i915(dev);
		3194	struct intel_encoder *encoder;
		3195	u32 enabled_irqs = 0;
		3196
		3197	for_each_intel_encoder(dev, encoder)
		3198	if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
		3199	enabled_irqs \|= hpd[encoder->hpd_pin];
		3200
		3201	return enabled_irqs;
		3202	}
		3203
3746	Serge	3204	static void ibx_hpd_irq_setup(struct drm_device *dev)
		3205	{
5060	serge	3206	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	3207	u32 hotplug_irqs, hotplug, enabled_irqs;
3746	Serge	3208
		3209	if (HAS_PCH_IBX(dev)) {
4104	Serge	3210	hotplug_irqs = SDE_HOTPLUG_MASK;
6084	serge	3211	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3746	Serge	3212	} else {
4104	Serge	3213	hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
6084	serge	3214	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3746	Serge	3215	}
		3216
4104	Serge	3217	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3746	Serge	3218
		3219	/*
6084	serge	3220	* Enable digital hotplug on the PCH, and configure the DP short pulse
		3221	* duration to 2ms (which is the minimum in the Display Port spec).
		3222	* The pulse duration bits are reserved on LPT+.
		3223	*/
2351	Serge	3224	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3225	hotplug &= ~(PORTD_PULSE_DURATION_MASK\|PORTC_PULSE_DURATION_MASK\|PORTB_PULSE_DURATION_MASK);
		3226	hotplug \|= PORTD_HOTPLUG_ENABLE \| PORTD_PULSE_DURATION_2ms;
		3227	hotplug \|= PORTC_HOTPLUG_ENABLE \| PORTC_PULSE_DURATION_2ms;
		3228	hotplug \|= PORTB_HOTPLUG_ENABLE \| PORTB_PULSE_DURATION_2ms;
6084	serge	3229	/*
		3230	* When CPU and PCH are on the same package, port A
		3231	* HPD must be enabled in both north and south.
		3232	*/
		3233	if (HAS_PCH_LPT_LP(dev))
		3234	hotplug \|= PORTA_HOTPLUG_ENABLE;
2351	Serge	3235	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3236	}
		3237
6084	serge	3238	static void spt_hpd_irq_setup(struct drm_device *dev)
		3239	{
		3240	struct drm_i915_private *dev_priv = dev->dev_private;
		3241	u32 hotplug_irqs, hotplug, enabled_irqs;
		3242
		3243	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
		3244	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
		3245
		3246	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
		3247
		3248	/* Enable digital hotplug on the PCH */
		3249	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3250	hotplug \|= PORTD_HOTPLUG_ENABLE \| PORTC_HOTPLUG_ENABLE \|
		3251	PORTB_HOTPLUG_ENABLE \| PORTA_HOTPLUG_ENABLE;
		3252	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3253
		3254	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
		3255	hotplug \|= PORTE_HOTPLUG_ENABLE;
		3256	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
		3257	}
		3258
		3259	static void ilk_hpd_irq_setup(struct drm_device *dev)
		3260	{
		3261	struct drm_i915_private *dev_priv = dev->dev_private;
		3262	u32 hotplug_irqs, hotplug, enabled_irqs;
		3263
		3264	if (INTEL_INFO(dev)->gen >= 8) {
		3265	hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		3266	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
		3267
		3268	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3269	} else if (INTEL_INFO(dev)->gen >= 7) {
		3270	hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		3271	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
		3272
		3273	ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3274	} else {
		3275	hotplug_irqs = DE_DP_A_HOTPLUG;
		3276	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
		3277
		3278	ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3279	}
		3280
		3281	/*
		3282	* Enable digital hotplug on the CPU, and configure the DP short pulse
		3283	* duration to 2ms (which is the minimum in the Display Port spec)
		3284	* The pulse duration bits are reserved on HSW+.
		3285	*/
		3286	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
		3287	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
		3288	hotplug \|= DIGITAL_PORTA_HOTPLUG_ENABLE \| DIGITAL_PORTA_PULSE_DURATION_2ms;
		3289	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
		3290
		3291	ibx_hpd_irq_setup(dev);
		3292	}
		3293
		3294	static void bxt_hpd_irq_setup(struct drm_device *dev)
		3295	{
		3296	struct drm_i915_private *dev_priv = dev->dev_private;
		3297	u32 hotplug_irqs, hotplug, enabled_irqs;
		3298
		3299	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
		3300	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
		3301
		3302	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3303
		3304	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3305	hotplug \|= PORTC_HOTPLUG_ENABLE \| PORTB_HOTPLUG_ENABLE \|
		3306	PORTA_HOTPLUG_ENABLE;
		3307	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3308	}
		3309
3480	Serge	3310	static void ibx_irq_postinstall(struct drm_device *dev)
		3311	{
5060	serge	3312	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	3313	u32 mask;
		3314
3746	Serge	3315	if (HAS_PCH_NOP(dev))
		3316	return;
		3317
5060	serge	3318	if (HAS_PCH_IBX(dev))
		3319	mask = SDE_GMBUS \| SDE_AUX_MASK \| SDE_POISON;
		3320	else
		3321	mask = SDE_GMBUS_CPT \| SDE_AUX_MASK_CPT;
4104	Serge	3322
6084	serge	3323	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
3480	Serge	3324	I915_WRITE(SDEIMR, ~mask);
		3325	}
		3326
4104	Serge	3327	static void gen5_gt_irq_postinstall(struct drm_device *dev)
2351	Serge	3328	{
4104	Serge	3329	struct drm_i915_private *dev_priv = dev->dev_private;
		3330	u32 pm_irqs, gt_irqs;
2351	Serge	3331
4104	Serge	3332	pm_irqs = gt_irqs = 0;
2351	Serge	3333
		3334	dev_priv->gt_irq_mask = ~0;
4560	Serge	3335	if (HAS_L3_DPF(dev)) {
4104	Serge	3336	/* L3 parity interrupt is always unmasked. */
4560	Serge	3337	dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		3338	gt_irqs \|= GT_PARITY_ERROR(dev);
4104	Serge	3339	}
2351	Serge	3340
4104	Serge	3341	gt_irqs \|= GT_RENDER_USER_INTERRUPT;
		3342	if (IS_GEN5(dev)) {
		3343	gt_irqs \|= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT \|
		3344	ILK_BSD_USER_INTERRUPT;
		3345	} else {
		3346	gt_irqs \|= GT_BLT_USER_INTERRUPT \| GT_BSD_USER_INTERRUPT;
		3347	}
2351	Serge	3348
5060	serge	3349	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
2351	Serge	3350
4104	Serge	3351	if (INTEL_INFO(dev)->gen >= 6) {
5354	serge	3352	/*
		3353	* RPS interrupts will get enabled/disabled on demand when RPS
		3354	* itself is enabled/disabled.
		3355	*/
4104	Serge	3356	if (HAS_VEBOX(dev))
		3357	pm_irqs \|= PM_VEBOX_USER_INTERRUPT;
		3358
		3359	dev_priv->pm_irq_mask = 0xffffffff;
5060	serge	3360	GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
6084	serge	3361	}
2351	Serge	3362	}
		3363
4104	Serge	3364	static int ironlake_irq_postinstall(struct drm_device *dev)
3031	serge	3365	{
5060	serge	3366	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3367	u32 display_mask, extra_mask;
		3368
		3369	if (INTEL_INFO(dev)->gen >= 7) {
		3370	display_mask = (DE_MASTER_IRQ_CONTROL \| DE_GSE_IVB \|
		3371	DE_PCH_EVENT_IVB \| DE_PLANEC_FLIP_DONE_IVB \|
6084	serge	3372	DE_PLANEB_FLIP_DONE_IVB \|
5060	serge	3373	DE_PLANEA_FLIP_DONE_IVB \| DE_AUX_CHANNEL_A_IVB);
4104	Serge	3374	extra_mask = (DE_PIPEC_VBLANK_IVB \| DE_PIPEB_VBLANK_IVB \|
6084	serge	3375	DE_PIPEA_VBLANK_IVB \| DE_ERR_INT_IVB \|
		3376	DE_DP_A_HOTPLUG_IVB);
4104	Serge	3377	} else {
		3378	display_mask = (DE_MASTER_IRQ_CONTROL \| DE_GSE \| DE_PCH_EVENT \|
		3379	DE_PLANEA_FLIP_DONE \| DE_PLANEB_FLIP_DONE \|
4560	Serge	3380	DE_AUX_CHANNEL_A \|
		3381	DE_PIPEB_CRC_DONE \| DE_PIPEA_CRC_DONE \|
		3382	DE_POISON);
6084	serge	3383	extra_mask = (DE_PIPEA_VBLANK \| DE_PIPEB_VBLANK \| DE_PCU_EVENT \|
		3384	DE_PIPEB_FIFO_UNDERRUN \| DE_PIPEA_FIFO_UNDERRUN \|
		3385	DE_DP_A_HOTPLUG);
4104	Serge	3386	}
		3387
3031	serge	3388	dev_priv->irq_mask = ~display_mask;
		3389
5060	serge	3390	I915_WRITE(HWSTAM, 0xeffe);
3031	serge	3391
5060	serge	3392	ibx_irq_pre_postinstall(dev);
		3393
		3394	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask \| extra_mask);
		3395
4104	Serge	3396	gen5_gt_irq_postinstall(dev);
3031	serge	3397
4104	Serge	3398	ibx_irq_postinstall(dev);
3031	serge	3399
4104	Serge	3400	if (IS_IRONLAKE_M(dev)) {
		3401	/* Enable PCU event interrupts
		3402	*
		3403	* spinlocking not required here for correctness since interrupt
		3404	* setup is guaranteed to run in single-threaded context. But we
		3405	* need it to make the assert_spin_locked happy. */
5354	serge	3406	spin_lock_irq(&dev_priv->irq_lock);
4104	Serge	3407	ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
5354	serge	3408	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	3409	}
3031	serge	3410
		3411	return 0;
		3412	}
		3413
5060	serge	3414	static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
		3415	{
		3416	u32 pipestat_mask;
		3417	u32 iir_mask;
5354	serge	3418	enum pipe pipe;
5060	serge	3419
		3420	pipestat_mask = PIPESTAT_INT_STATUS_MASK \|
		3421	PIPE_FIFO_UNDERRUN_STATUS;
		3422
5354	serge	3423	for_each_pipe(dev_priv, pipe)
		3424	I915_WRITE(PIPESTAT(pipe), pipestat_mask);
5060	serge	3425	POSTING_READ(PIPESTAT(PIPE_A));
		3426
		3427	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV \|
		3428	PIPE_CRC_DONE_INTERRUPT_STATUS;
		3429
5354	serge	3430	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		3431	for_each_pipe(dev_priv, pipe)
		3432	i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
5060	serge	3433
		3434	iir_mask = I915_DISPLAY_PORT_INTERRUPT \|
		3435	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3436	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
5354	serge	3437	if (IS_CHERRYVIEW(dev_priv))
		3438	iir_mask \|= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
5060	serge	3439	dev_priv->irq_mask &= ~iir_mask;
		3440
		3441	I915_WRITE(VLV_IIR, iir_mask);
		3442	I915_WRITE(VLV_IIR, iir_mask);
5354	serge	3443	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
5060	serge	3444	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5354	serge	3445	POSTING_READ(VLV_IMR);
5060	serge	3446	}
		3447
		3448	static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
		3449	{
		3450	u32 pipestat_mask;
		3451	u32 iir_mask;
5354	serge	3452	enum pipe pipe;
5060	serge	3453
		3454	iir_mask = I915_DISPLAY_PORT_INTERRUPT \|
		3455	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3456	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
5354	serge	3457	if (IS_CHERRYVIEW(dev_priv))
		3458	iir_mask \|= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
5060	serge	3459
		3460	dev_priv->irq_mask \|= iir_mask;
5354	serge	3461	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5060	serge	3462	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
		3463	I915_WRITE(VLV_IIR, iir_mask);
		3464	I915_WRITE(VLV_IIR, iir_mask);
		3465	POSTING_READ(VLV_IIR);
		3466
		3467	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV \|
		3468	PIPE_CRC_DONE_INTERRUPT_STATUS;
		3469
5354	serge	3470	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		3471	for_each_pipe(dev_priv, pipe)
		3472	i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
5060	serge	3473
		3474	pipestat_mask = PIPESTAT_INT_STATUS_MASK \|
		3475	PIPE_FIFO_UNDERRUN_STATUS;
5354	serge	3476
		3477	for_each_pipe(dev_priv, pipe)
		3478	I915_WRITE(PIPESTAT(pipe), pipestat_mask);
5060	serge	3479	POSTING_READ(PIPESTAT(PIPE_A));
		3480	}
		3481
		3482	void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
		3483	{
		3484	assert_spin_locked(&dev_priv->irq_lock);
		3485
		3486	if (dev_priv->display_irqs_enabled)
		3487	return;
		3488
		3489	dev_priv->display_irqs_enabled = true;
		3490
5354	serge	3491	if (intel_irqs_enabled(dev_priv))
5060	serge	3492	valleyview_display_irqs_install(dev_priv);
		3493	}
		3494
		3495	void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
		3496	{
		3497	assert_spin_locked(&dev_priv->irq_lock);
		3498
		3499	if (!dev_priv->display_irqs_enabled)
		3500	return;
		3501
		3502	dev_priv->display_irqs_enabled = false;
		3503
5354	serge	3504	if (intel_irqs_enabled(dev_priv))
5060	serge	3505	valleyview_display_irqs_uninstall(dev_priv);
		3506	}
		3507
5354	serge	3508	static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3031	serge	3509	{
5060	serge	3510	dev_priv->irq_mask = ~0;
3031	serge	3511
6084	serge	3512	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	3513	POSTING_READ(PORT_HOTPLUG_EN);
		3514
5354	serge	3515	I915_WRITE(VLV_IIR, 0xffffffff);
		3516	I915_WRITE(VLV_IIR, 0xffffffff);
		3517	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3031	serge	3518	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5354	serge	3519	POSTING_READ(VLV_IMR);
3031	serge	3520
4104	Serge	3521	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3522	* just to make the assert_spin_locked check happy. */
5354	serge	3523	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3524	if (dev_priv->display_irqs_enabled)
		3525	valleyview_display_irqs_install(dev_priv);
5354	serge	3526	spin_unlock_irq(&dev_priv->irq_lock);
		3527	}
3031	serge	3528
5354	serge	3529	static int valleyview_irq_postinstall(struct drm_device *dev)
		3530	{
		3531	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3532
5354	serge	3533	vlv_display_irq_postinstall(dev_priv);
		3534
4104	Serge	3535	gen5_gt_irq_postinstall(dev);
3243	Serge	3536
3031	serge	3537	/* ack & enable invalid PTE error interrupts */
		3538	#if 0 /* FIXME: add support to irq handler for checking these bits */
		3539	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
		3540	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
		3541	#endif
		3542
		3543	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3480	Serge	3544
		3545	return 0;
		3546	}
		3547
4560	Serge	3548	static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
		3549	{
		3550	/* These are interrupts we'll toggle with the ring mask register */
		3551	uint32_t gt_interrupts[] = {
		3552	GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT \|
5354	serge	3553	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT \|
4560	Serge	3554	GT_RENDER_L3_PARITY_ERROR_INTERRUPT \|
5354	serge	3555	GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT \|
		3556	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
4560	Serge	3557	GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT \|
5354	serge	3558	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT \|
		3559	GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT \|
		3560	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
4560	Serge	3561	0,
5354	serge	3562	GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT \|
		3563	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
4560	Serge	3564	};
		3565
5060	serge	3566	dev_priv->pm_irq_mask = 0xffffffff;
5354	serge	3567	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
		3568	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
		3569	/*
		3570	* RPS interrupts will get enabled/disabled on demand when RPS itself
		3571	* is enabled/disabled.
		3572	*/
		3573	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
		3574	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
4560	Serge	3575	}
		3576
		3577	static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
		3578	{
5354	serge	3579	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
		3580	uint32_t de_pipe_enables;
6084	serge	3581	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
		3582	u32 de_port_enables;
		3583	enum pipe pipe;
5354	serge	3584
6084	serge	3585	if (INTEL_INFO(dev_priv)->gen >= 9) {
5354	serge	3586	de_pipe_masked \|= GEN9_PIPE_PLANE1_FLIP_DONE \|
		3587	GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
6084	serge	3588	de_port_masked \|= GEN9_AUX_CHANNEL_B \| GEN9_AUX_CHANNEL_C \|
		3589	GEN9_AUX_CHANNEL_D;
		3590	if (IS_BROXTON(dev_priv))
		3591	de_port_masked \|= BXT_DE_PORT_GMBUS;
		3592	} else {
5354	serge	3593	de_pipe_masked \|= GEN8_PIPE_PRIMARY_FLIP_DONE \|
6084	serge	3594	GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
		3595	}
5354	serge	3596
		3597	de_pipe_enables = de_pipe_masked \| GEN8_PIPE_VBLANK \|
6084	serge	3598	GEN8_PIPE_FIFO_UNDERRUN;
5354	serge	3599
6084	serge	3600	de_port_enables = de_port_masked;
		3601	if (IS_BROXTON(dev_priv))
		3602	de_port_enables \|= BXT_DE_PORT_HOTPLUG_MASK;
		3603	else if (IS_BROADWELL(dev_priv))
		3604	de_port_enables \|= GEN8_PORT_DP_A_HOTPLUG;
		3605
4560	Serge	3606	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
		3607	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
		3608	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
		3609
5354	serge	3610	for_each_pipe(dev_priv, pipe)
		3611	if (intel_display_power_is_enabled(dev_priv,
5060	serge	3612	POWER_DOMAIN_PIPE(pipe)))
		3613	GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
		3614	dev_priv->de_irq_mask[pipe],
6084	serge	3615	de_pipe_enables);
4560	Serge	3616
6084	serge	3617	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
4560	Serge	3618	}
		3619
		3620	static int gen8_irq_postinstall(struct drm_device *dev)
		3621	{
		3622	struct drm_i915_private *dev_priv = dev->dev_private;
		3623
6084	serge	3624	if (HAS_PCH_SPLIT(dev))
		3625	ibx_irq_pre_postinstall(dev);
5060	serge	3626
4560	Serge	3627	gen8_gt_irq_postinstall(dev_priv);
		3628	gen8_de_irq_postinstall(dev_priv);
		3629
6084	serge	3630	if (HAS_PCH_SPLIT(dev))
		3631	ibx_irq_postinstall(dev);
4560	Serge	3632
		3633	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		3634	POSTING_READ(GEN8_MASTER_IRQ);
		3635
		3636	return 0;
		3637	}
		3638
5060	serge	3639	static int cherryview_irq_postinstall(struct drm_device *dev)
4560	Serge	3640	{
		3641	struct drm_i915_private *dev_priv = dev->dev_private;
		3642
5354	serge	3643	vlv_display_irq_postinstall(dev_priv);
4560	Serge	3644
5060	serge	3645	gen8_gt_irq_postinstall(dev_priv);
4560	Serge	3646
5060	serge	3647	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
		3648	POSTING_READ(GEN8_MASTER_IRQ);
4560	Serge	3649
5060	serge	3650	return 0;
		3651	}
4560	Serge	3652
5060	serge	3653	static void gen8_irq_uninstall(struct drm_device *dev)
		3654	{
		3655	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3656
5060	serge	3657	if (!dev_priv)
		3658	return;
		3659
		3660	gen8_irq_reset(dev);
4560	Serge	3661	}
		3662
5354	serge	3663	static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
		3664	{
		3665	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3666	* just to make the assert_spin_locked check happy. */
		3667	spin_lock_irq(&dev_priv->irq_lock);
		3668	if (dev_priv->display_irqs_enabled)
		3669	valleyview_display_irqs_uninstall(dev_priv);
		3670	spin_unlock_irq(&dev_priv->irq_lock);
		3671
		3672	vlv_display_irq_reset(dev_priv);
		3673
		3674	dev_priv->irq_mask = ~0;
		3675	}
		3676
3031	serge	3677	static void valleyview_irq_uninstall(struct drm_device *dev)
		3678	{
5060	serge	3679	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3680
		3681	if (!dev_priv)
		3682	return;
		3683
5060	serge	3684	I915_WRITE(VLV_MASTER_IER, 0);
4293	Serge	3685
5354	serge	3686	gen5_gt_irq_reset(dev);
3031	serge	3687
		3688	I915_WRITE(HWSTAM, 0xffffffff);
5060	serge	3689
5354	serge	3690	vlv_display_irq_uninstall(dev_priv);
3031	serge	3691	}
		3692
5060	serge	3693	static void cherryview_irq_uninstall(struct drm_device *dev)
3031	serge	3694	{
5060	serge	3695	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3696
		3697	if (!dev_priv)
		3698	return;
		3699
5060	serge	3700	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3701	POSTING_READ(GEN8_MASTER_IRQ);
4293	Serge	3702
5354	serge	3703	gen8_gt_irq_reset(dev_priv);
3031	serge	3704
5354	serge	3705	GEN5_IRQ_RESET(GEN8_PCU_);
3031	serge	3706
5354	serge	3707	vlv_display_irq_uninstall(dev_priv);
5060	serge	3708	}
		3709
		3710	static void ironlake_irq_uninstall(struct drm_device *dev)
		3711	{
		3712	struct drm_i915_private *dev_priv = dev->dev_private;
		3713
		3714	if (!dev_priv)
3746	Serge	3715	return;
		3716
5060	serge	3717	ironlake_irq_reset(dev);
3031	serge	3718	}
		3719
		3720	#if 0
		3721	static void i8xx_irq_preinstall(struct drm_device * dev)
		3722	{
5060	serge	3723	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3724	int pipe;
		3725
5354	serge	3726	for_each_pipe(dev_priv, pipe)
3031	serge	3727	I915_WRITE(PIPESTAT(pipe), 0);
		3728	I915_WRITE16(IMR, 0xffff);
		3729	I915_WRITE16(IER, 0x0);
		3730	POSTING_READ16(IER);
		3731	}
		3732
		3733	static int i8xx_irq_postinstall(struct drm_device *dev)
		3734	{
5060	serge	3735	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3736
		3737	I915_WRITE16(EMR,
		3738	~(I915_ERROR_PAGE_TABLE \| I915_ERROR_MEMORY_REFRESH));
		3739
		3740	/* Unmask the interrupts that we always want on. */
		3741	dev_priv->irq_mask =
		3742	~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3743	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3744	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
6084	serge	3745	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	3746	I915_WRITE16(IMR, dev_priv->irq_mask);
		3747
		3748	I915_WRITE16(IER,
		3749	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3750	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3751	I915_USER_INTERRUPT);
		3752	POSTING_READ16(IER);
		3753
4560	Serge	3754	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3755	* just to make the assert_spin_locked check happy. */
5354	serge	3756	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3757	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		3758	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	3759	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	3760
3031	serge	3761	return 0;
		3762	}
		3763
3746	Serge	3764	/*
		3765	* Returns true when a page flip has completed.
		3766	*/
		3767	static bool i8xx_handle_vblank(struct drm_device *dev,
4560	Serge	3768	int plane, int pipe, u32 iir)
3746	Serge	3769	{
5060	serge	3770	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3771	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3746	Serge	3772
6084	serge	3773	if (!intel_pipe_handle_vblank(dev, pipe))
		3774	return false;
3746	Serge	3775
		3776	if ((iir & flip_pending) == 0)
5354	serge	3777	goto check_page_flip;
3746	Serge	3778
		3779	/* We detect FlipDone by looking for the change in PendingFlip from '1'
		3780	* to '0' on the following vblank, i.e. IIR has the Pendingflip
		3781	* asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
		3782	* the flip is completed (no longer pending). Since this doesn't raise
		3783	* an interrupt per se, we watch for the change at vblank.
		3784	*/
		3785	if (I915_READ16(ISR) & flip_pending)
5354	serge	3786	goto check_page_flip;
3746	Serge	3787
6320	serge	3788	intel_prepare_page_flip(dev, plane);
		3789	intel_finish_page_flip(dev, pipe);
5354	serge	3790	return true;
3746	Serge	3791
5354	serge	3792	check_page_flip:
6320	serge	3793	intel_check_page_flip(dev, pipe);
5354	serge	3794	return false;
3746	Serge	3795	}
		3796
3243	Serge	3797	static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3031	serge	3798	{
5060	serge	3799	struct drm_device *dev = arg;
		3800	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3801	u16 iir, new_iir;
		3802	u32 pipe_stats[2];
		3803	int pipe;
		3804	u16 flip_mask =
		3805	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		3806	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
		3807
6084	serge	3808	if (!intel_irqs_enabled(dev_priv))
		3809	return IRQ_NONE;
		3810
3031	serge	3811	iir = I915_READ16(IIR);
		3812	if (iir == 0)
		3813	return IRQ_NONE;
		3814
		3815	while (iir & ~flip_mask) {
		3816	/* Can't rely on pipestat interrupt bit in iir as it might
		3817	* have been cleared after the pipestat interrupt was received.
		3818	* It doesn't set the bit in iir again, but it still produces
		3819	* interrupts (for non-MSI).
		3820	*/
5354	serge	3821	spin_lock(&dev_priv->irq_lock);
4126	Serge	3822	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	3823	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	3824
5354	serge	3825	for_each_pipe(dev_priv, pipe) {
3031	serge	3826	int reg = PIPESTAT(pipe);
		3827	pipe_stats[pipe] = I915_READ(reg);
		3828
		3829	/*
		3830	* Clear the PIPE*STAT regs before the IIR
		3831	*/
5060	serge	3832	if (pipe_stats[pipe] & 0x8000ffff)
3031	serge	3833	I915_WRITE(reg, pipe_stats[pipe]);
6084	serge	3834	}
5354	serge	3835	spin_unlock(&dev_priv->irq_lock);
3031	serge	3836
		3837	I915_WRITE16(IIR, iir & ~flip_mask);
		3838	new_iir = I915_READ16(IIR); /* Flush posted writes */
		3839
		3840	if (iir & I915_USER_INTERRUPT)
6084	serge	3841	notify_ring(&dev_priv->ring[RCS]);
3031	serge	3842
5354	serge	3843	for_each_pipe(dev_priv, pipe) {
4560	Serge	3844	int plane = pipe;
		3845	if (HAS_FBC(dev))
		3846	plane = !plane;
3031	serge	3847
4560	Serge	3848	if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
		3849	i8xx_handle_vblank(dev, plane, pipe, iir))
		3850	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3031	serge	3851
4560	Serge	3852	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		3853	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	3854
5354	serge	3855	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		3856	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		3857	pipe);
4560	Serge	3858	}
		3859
3031	serge	3860	iir = new_iir;
		3861	}
		3862
		3863	return IRQ_HANDLED;
		3864	}
		3865
		3866	static void i8xx_irq_uninstall(struct drm_device * dev)
		3867	{
5060	serge	3868	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3869	int pipe;
		3870
5354	serge	3871	for_each_pipe(dev_priv, pipe) {
3031	serge	3872	/* Clear enable bits; then clear status bits */
		3873	I915_WRITE(PIPESTAT(pipe), 0);
		3874	I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
		3875	}
		3876	I915_WRITE16(IMR, 0xffff);
		3877	I915_WRITE16(IER, 0x0);
		3878	I915_WRITE16(IIR, I915_READ16(IIR));
		3879	}
		3880
		3881	#endif
		3882
		3883	static void i915_irq_preinstall(struct drm_device * dev)
		3884	{
5060	serge	3885	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3886	int pipe;
		3887
		3888	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	3889	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	3890	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		3891	}
		3892
		3893	I915_WRITE16(HWSTAM, 0xeffe);
5354	serge	3894	for_each_pipe(dev_priv, pipe)
3031	serge	3895	I915_WRITE(PIPESTAT(pipe), 0);
		3896	I915_WRITE(IMR, 0xffffffff);
		3897	I915_WRITE(IER, 0x0);
		3898	POSTING_READ(IER);
		3899	}
		3900
		3901	static int i915_irq_postinstall(struct drm_device *dev)
		3902	{
5060	serge	3903	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3904	u32 enable_mask;
		3905
		3906	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE \| I915_ERROR_MEMORY_REFRESH));
		3907
		3908	/* Unmask the interrupts that we always want on. */
		3909	dev_priv->irq_mask =
		3910	~(I915_ASLE_INTERRUPT \|
		3911	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3912	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3913	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
6084	serge	3914	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	3915
		3916	enable_mask =
		3917	I915_ASLE_INTERRUPT \|
		3918	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3919	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3920	I915_USER_INTERRUPT;
3480	Serge	3921
3031	serge	3922	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	3923	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	3924	POSTING_READ(PORT_HOTPLUG_EN);
		3925
3031	serge	3926	/* Enable in IER... */
		3927	enable_mask \|= I915_DISPLAY_PORT_INTERRUPT;
		3928	/* and unmask in IMR */
		3929	dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
		3930	}
		3931
		3932	I915_WRITE(IMR, dev_priv->irq_mask);
		3933	I915_WRITE(IER, enable_mask);
		3934	POSTING_READ(IER);
		3935
4126	Serge	3936	i915_enable_asle_pipestat(dev);
3480	Serge	3937
4560	Serge	3938	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3939	* just to make the assert_spin_locked check happy. */
5354	serge	3940	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3941	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		3942	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	3943	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	3944
3480	Serge	3945	return 0;
		3946	}
		3947
3746	Serge	3948	/*
		3949	* Returns true when a page flip has completed.
		3950	*/
		3951	static bool i915_handle_vblank(struct drm_device *dev,
		3952	int plane, int pipe, u32 iir)
3480	Serge	3953	{
5060	serge	3954	struct drm_i915_private *dev_priv = dev->dev_private;
3746	Serge	3955	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3480	Serge	3956
6088	serge	3957	if (!intel_pipe_handle_vblank(dev, pipe))
		3958	return false;
3480	Serge	3959
3746	Serge	3960	if ((iir & flip_pending) == 0)
5354	serge	3961	goto check_page_flip;
3480	Serge	3962
3746	Serge	3963	/* We detect FlipDone by looking for the change in PendingFlip from '1'
		3964	* to '0' on the following vblank, i.e. IIR has the Pendingflip
		3965	* asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
		3966	* the flip is completed (no longer pending). Since this doesn't raise
		3967	* an interrupt per se, we watch for the change at vblank.
		3968	*/
		3969	if (I915_READ(ISR) & flip_pending)
5354	serge	3970	goto check_page_flip;
3746	Serge	3971
6320	serge	3972	intel_prepare_page_flip(dev, plane);
		3973	intel_finish_page_flip(dev, pipe);
5354	serge	3974	return true;
3746	Serge	3975
5354	serge	3976	check_page_flip:
6320	serge	3977	intel_check_page_flip(dev, pipe);
5354	serge	3978	return false;
3031	serge	3979	}
		3980
3243	Serge	3981	static irqreturn_t i915_irq_handler(int irq, void *arg)
3031	serge	3982	{
5060	serge	3983	struct drm_device *dev = arg;
		3984	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3985	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
		3986	u32 flip_mask =
		3987	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		3988	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
		3989	int pipe, ret = IRQ_NONE;
		3990
6084	serge	3991	if (!intel_irqs_enabled(dev_priv))
		3992	return IRQ_NONE;
		3993
3031	serge	3994	iir = I915_READ(IIR);
		3995	do {
		3996	bool irq_received = (iir & ~flip_mask) != 0;
		3997	bool blc_event = false;
		3998
		3999	/* Can't rely on pipestat interrupt bit in iir as it might
		4000	* have been cleared after the pipestat interrupt was received.
		4001	* It doesn't set the bit in iir again, but it still produces
		4002	* interrupts (for non-MSI).
		4003	*/
5354	serge	4004	spin_lock(&dev_priv->irq_lock);
4126	Serge	4005	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	4006	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	4007
5354	serge	4008	for_each_pipe(dev_priv, pipe) {
3031	serge	4009	int reg = PIPESTAT(pipe);
		4010	pipe_stats[pipe] = I915_READ(reg);
		4011
		4012	/* Clear the PIPESTAT regs before the IIR /
		4013	if (pipe_stats[pipe] & 0x8000ffff) {
		4014	I915_WRITE(reg, pipe_stats[pipe]);
		4015	irq_received = true;
		4016	}
		4017	}
5354	serge	4018	spin_unlock(&dev_priv->irq_lock);
3031	serge	4019
		4020	if (!irq_received)
		4021	break;
		4022
		4023	/* Consume port. Then clear IIR or we'll miss events */
5060	serge	4024	if (I915_HAS_HOTPLUG(dev) &&
		4025	iir & I915_DISPLAY_PORT_INTERRUPT)
		4026	i9xx_hpd_irq_handler(dev);
3031	serge	4027
		4028	I915_WRITE(IIR, iir & ~flip_mask);
		4029	new_iir = I915_READ(IIR); /* Flush posted writes */
		4030
		4031	if (iir & I915_USER_INTERRUPT)
6084	serge	4032	notify_ring(&dev_priv->ring[RCS]);
3031	serge	4033
5354	serge	4034	for_each_pipe(dev_priv, pipe) {
3031	serge	4035	int plane = pipe;
4560	Serge	4036	if (HAS_FBC(dev))
3031	serge	4037	plane = !plane;
		4038
3746	Serge	4039	if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
		4040	i915_handle_vblank(dev, plane, pipe, iir))
		4041	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
		4042
3031	serge	4043	if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
		4044	blc_event = true;
4560	Serge	4045
		4046	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		4047	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	4048
5354	serge	4049	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		4050	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		4051	pipe);
3031	serge	4052	}
		4053
4126	Serge	4054	if (blc_event \|\| (iir & I915_ASLE_INTERRUPT))
		4055	intel_opregion_asle_intr(dev);
3031	serge	4056
		4057	/* With MSI, interrupts are only generated when iir
		4058	* transitions from zero to nonzero. If another bit got
		4059	* set while we were handling the existing iir bits, then
		4060	* we would never get another interrupt.
		4061	*
		4062	* This is fine on non-MSI as well, as if we hit this path
		4063	* we avoid exiting the interrupt handler only to generate
		4064	* another one.
		4065	*
		4066	* Note that for MSI this could cause a stray interrupt report
		4067	* if an interrupt landed in the time between writing IIR and
		4068	* the posting read. This should be rare enough to never
		4069	* trigger the 99% of 100,000 interrupts test for disabling
		4070	* stray interrupts.
		4071	*/
		4072	ret = IRQ_HANDLED;
		4073	iir = new_iir;
		4074	} while (iir & ~flip_mask);
		4075
		4076	return ret;
		4077	}
		4078
		4079	static void i915_irq_uninstall(struct drm_device * dev)
		4080	{
5060	serge	4081	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4082	int pipe;
		4083
		4084	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	4085	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4086	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4087	}
		4088
		4089	I915_WRITE16(HWSTAM, 0xffff);
5354	serge	4090	for_each_pipe(dev_priv, pipe) {
3031	serge	4091	/* Clear enable bits; then clear status bits */
		4092	I915_WRITE(PIPESTAT(pipe), 0);
		4093	I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
		4094	}
		4095	I915_WRITE(IMR, 0xffffffff);
		4096	I915_WRITE(IER, 0x0);
		4097
		4098	I915_WRITE(IIR, I915_READ(IIR));
		4099	}
		4100
		4101	static void i965_irq_preinstall(struct drm_device * dev)
		4102	{
5060	serge	4103	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4104	int pipe;
		4105
6084	serge	4106	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4107	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4108
		4109	I915_WRITE(HWSTAM, 0xeffe);
5354	serge	4110	for_each_pipe(dev_priv, pipe)
3031	serge	4111	I915_WRITE(PIPESTAT(pipe), 0);
		4112	I915_WRITE(IMR, 0xffffffff);
		4113	I915_WRITE(IER, 0x0);
		4114	POSTING_READ(IER);
		4115	}
		4116
		4117	static int i965_irq_postinstall(struct drm_device *dev)
		4118	{
5060	serge	4119	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4120	u32 enable_mask;
		4121	u32 error_mask;
		4122
		4123	/* Unmask the interrupts that we always want on. */
		4124	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT \|
		4125	I915_DISPLAY_PORT_INTERRUPT \|
		4126	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		4127	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		4128	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4129	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT \|
		4130	I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
		4131
		4132	enable_mask = ~dev_priv->irq_mask;
3746	Serge	4133	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4134	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	4135	enable_mask \|= I915_USER_INTERRUPT;
		4136
		4137	if (IS_G4X(dev))
		4138	enable_mask \|= I915_BSD_USER_INTERRUPT;
		4139
4104	Serge	4140	/* Interrupt setup is already guaranteed to be single-threaded, this is
		4141	* just to make the assert_spin_locked check happy. */
5354	serge	4142	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	4143	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		4144	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		4145	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	4146	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	4147
		4148	/*
		4149	* Enable some error detection, note the instruction error mask
		4150	* bit is reserved, so we leave it masked.
		4151	*/
		4152	if (IS_G4X(dev)) {
		4153	error_mask = ~(GM45_ERROR_PAGE_TABLE \|
		4154	GM45_ERROR_MEM_PRIV \|
		4155	GM45_ERROR_CP_PRIV \|
		4156	I915_ERROR_MEMORY_REFRESH);
		4157	} else {
		4158	error_mask = ~(I915_ERROR_PAGE_TABLE \|
		4159	I915_ERROR_MEMORY_REFRESH);
		4160	}
		4161	I915_WRITE(EMR, error_mask);
		4162
		4163	I915_WRITE(IMR, dev_priv->irq_mask);
		4164	I915_WRITE(IER, enable_mask);
		4165	POSTING_READ(IER);
		4166
6084	serge	4167	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	4168	POSTING_READ(PORT_HOTPLUG_EN);
		4169
4126	Serge	4170	i915_enable_asle_pipestat(dev);
3480	Serge	4171
		4172	return 0;
		4173	}
		4174
3746	Serge	4175	static void i915_hpd_irq_setup(struct drm_device *dev)
3480	Serge	4176	{
5060	serge	4177	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	4178	u32 hotplug_en;
		4179
4104	Serge	4180	assert_spin_locked(&dev_priv->irq_lock);
		4181
3031	serge	4182	/* Note HDMI and DP share hotplug bits */
6084	serge	4183	/* enable bits are the same for all generations */
		4184	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
		4185	/* Programming the CRT detection parameters tends
		4186	to generate a spurious hotplug event about three
		4187	seconds later. So just do it once.
		4188	*/
		4189	if (IS_G4X(dev))
		4190	hotplug_en \|= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
		4191	hotplug_en \|= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3480	Serge	4192
3031	serge	4193	/* Ignore TV since it's buggy */
6084	serge	4194	i915_hotplug_interrupt_update_locked(dev_priv,
		4195	HOTPLUG_INT_EN_MASK \|
		4196	CRT_HOTPLUG_VOLTAGE_COMPARE_MASK \|
		4197	CRT_HOTPLUG_ACTIVATION_PERIOD_64,
		4198	hotplug_en);
3031	serge	4199	}
		4200
3243	Serge	4201	static irqreturn_t i965_irq_handler(int irq, void *arg)
3031	serge	4202	{
5060	serge	4203	struct drm_device *dev = arg;
		4204	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4205	u32 iir, new_iir;
		4206	u32 pipe_stats[I915_MAX_PIPES];
		4207	int ret = IRQ_NONE, pipe;
3746	Serge	4208	u32 flip_mask =
		4209	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4210	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3031	serge	4211
6084	serge	4212	if (!intel_irqs_enabled(dev_priv))
		4213	return IRQ_NONE;
		4214
3031	serge	4215	iir = I915_READ(IIR);
		4216
		4217	for (;;) {
5060	serge	4218	bool irq_received = (iir & ~flip_mask) != 0;
3031	serge	4219	bool blc_event = false;
		4220
		4221	/* Can't rely on pipestat interrupt bit in iir as it might
		4222	* have been cleared after the pipestat interrupt was received.
		4223	* It doesn't set the bit in iir again, but it still produces
		4224	* interrupts (for non-MSI).
		4225	*/
5354	serge	4226	spin_lock(&dev_priv->irq_lock);
4126	Serge	4227	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	4228	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	4229
5354	serge	4230	for_each_pipe(dev_priv, pipe) {
3031	serge	4231	int reg = PIPESTAT(pipe);
		4232	pipe_stats[pipe] = I915_READ(reg);
		4233
		4234	/*
		4235	* Clear the PIPE*STAT regs before the IIR
		4236	*/
		4237	if (pipe_stats[pipe] & 0x8000ffff) {
		4238	I915_WRITE(reg, pipe_stats[pipe]);
5060	serge	4239	irq_received = true;
3031	serge	4240	}
		4241	}
5354	serge	4242	spin_unlock(&dev_priv->irq_lock);
3031	serge	4243
		4244	if (!irq_received)
		4245	break;
		4246
		4247	ret = IRQ_HANDLED;
		4248
		4249	/* Consume port. Then clear IIR or we'll miss events */
5060	serge	4250	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		4251	i9xx_hpd_irq_handler(dev);
3031	serge	4252
3746	Serge	4253	I915_WRITE(IIR, iir & ~flip_mask);
3031	serge	4254	new_iir = I915_READ(IIR); /* Flush posted writes */
		4255
		4256	if (iir & I915_USER_INTERRUPT)
6084	serge	4257	notify_ring(&dev_priv->ring[RCS]);
3031	serge	4258	if (iir & I915_BSD_USER_INTERRUPT)
6084	serge	4259	notify_ring(&dev_priv->ring[VCS]);
3031	serge	4260
5354	serge	4261	for_each_pipe(dev_priv, pipe) {
3746	Serge	4262	if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		4263	i915_handle_vblank(dev, pipe, pipe, iir))
		4264	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3031	serge	4265
		4266	if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
		4267	blc_event = true;
4560	Serge	4268
		4269	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		4270	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	4271
5354	serge	4272	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		4273	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
3031	serge	4274	}
		4275
4126	Serge	4276	if (blc_event \|\| (iir & I915_ASLE_INTERRUPT))
		4277	intel_opregion_asle_intr(dev);
3031	serge	4278
3480	Serge	4279	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		4280	gmbus_irq_handler(dev);
		4281
3031	serge	4282	/* With MSI, interrupts are only generated when iir
		4283	* transitions from zero to nonzero. If another bit got
		4284	* set while we were handling the existing iir bits, then
		4285	* we would never get another interrupt.
		4286	*
		4287	* This is fine on non-MSI as well, as if we hit this path
		4288	* we avoid exiting the interrupt handler only to generate
		4289	* another one.
		4290	*
		4291	* Note that for MSI this could cause a stray interrupt report
		4292	* if an interrupt landed in the time between writing IIR and
		4293	* the posting read. This should be rare enough to never
		4294	* trigger the 99% of 100,000 interrupts test for disabling
		4295	* stray interrupts.
		4296	*/
		4297	iir = new_iir;
		4298	}
		4299
		4300	return ret;
		4301	}
		4302
		4303	static void i965_irq_uninstall(struct drm_device * dev)
		4304	{
5060	serge	4305	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4306	int pipe;
		4307
		4308	if (!dev_priv)
		4309	return;
		4310
6084	serge	4311	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4312	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4313
		4314	I915_WRITE(HWSTAM, 0xffffffff);
5354	serge	4315	for_each_pipe(dev_priv, pipe)
3031	serge	4316	I915_WRITE(PIPESTAT(pipe), 0);
		4317	I915_WRITE(IMR, 0xffffffff);
		4318	I915_WRITE(IER, 0x0);
		4319
5354	serge	4320	for_each_pipe(dev_priv, pipe)
3031	serge	4321	I915_WRITE(PIPESTAT(pipe),
		4322	I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
		4323	I915_WRITE(IIR, I915_READ(IIR));
		4324	}
		4325
5354	serge	4326	/**
		4327	* intel_irq_init - initializes irq support
		4328	* @dev_priv: i915 device instance
		4329	*
		4330	* This function initializes all the irq support including work items, timers
		4331	* and all the vtables. It does not setup the interrupt itself though.
		4332	*/
		4333	void intel_irq_init(struct drm_i915_private *dev_priv)
2351	Serge	4334	{
5354	serge	4335	struct drm_device *dev = dev_priv->dev;
3031	serge	4336
6296	serge	4337	intel_hpd_init_work(dev_priv);
6084	serge	4338
4126	Serge	4339	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
		4340	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3480	Serge	4341
5060	serge	4342	/* Let's track the enabled rps events */
5354	serge	4343	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
		4344	/* WaGsvRC0ResidencyMethod:vlv */
6084	serge	4345	dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED;
5060	serge	4346	else
6084	serge	4347	dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
3480	Serge	4348
6084	serge	4349	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
		4350	i915_hangcheck_elapsed);
4560	Serge	4351
5354	serge	4352
		4353	if (IS_GEN2(dev_priv)) {
4560	Serge	4354	dev->max_vblank_count = 0;
		4355	dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
5354	serge	4356	} else if (IS_G4X(dev_priv) \|\| INTEL_INFO(dev_priv)->gen >= 5) {
4560	Serge	4357	dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
6084	serge	4358	dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4560	Serge	4359	} else {
6084	serge	4360	dev->driver->get_vblank_counter = i915_get_vblank_counter;
		4361	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4293	Serge	4362	}
3480	Serge	4363
5354	serge	4364	/*
		4365	* Opt out of the vblank disable timer on everything except gen2.
		4366	* Gen2 doesn't have a hardware frame counter and so depends on
		4367	* vblank interrupts to produce sane vblank seuquence numbers.
		4368	*/
		4369	if (!IS_GEN2(dev_priv))
		4370	dev->vblank_disable_immediate = true;
		4371
6084	serge	4372	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4293	Serge	4373	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3480	Serge	4374
5354	serge	4375	if (IS_CHERRYVIEW(dev_priv)) {
5060	serge	4376	dev->driver->irq_handler = cherryview_irq_handler;
		4377	dev->driver->irq_preinstall = cherryview_irq_preinstall;
		4378	dev->driver->irq_postinstall = cherryview_irq_postinstall;
		4379	dev->driver->irq_uninstall = cherryview_irq_uninstall;
		4380	dev->driver->enable_vblank = valleyview_enable_vblank;
		4381	dev->driver->disable_vblank = valleyview_disable_vblank;
		4382	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
5354	serge	4383	} else if (IS_VALLEYVIEW(dev_priv)) {
3243	Serge	4384	dev->driver->irq_handler = valleyview_irq_handler;
		4385	dev->driver->irq_preinstall = valleyview_irq_preinstall;
		4386	dev->driver->irq_postinstall = valleyview_irq_postinstall;
4293	Serge	4387	dev->driver->irq_uninstall = valleyview_irq_uninstall;
		4388	dev->driver->enable_vblank = valleyview_enable_vblank;
		4389	dev->driver->disable_vblank = valleyview_disable_vblank;
3746	Serge	4390	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
5354	serge	4391	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4560	Serge	4392	dev->driver->irq_handler = gen8_irq_handler;
5060	serge	4393	dev->driver->irq_preinstall = gen8_irq_reset;
4560	Serge	4394	dev->driver->irq_postinstall = gen8_irq_postinstall;
		4395	dev->driver->irq_uninstall = gen8_irq_uninstall;
		4396	dev->driver->enable_vblank = gen8_enable_vblank;
		4397	dev->driver->disable_vblank = gen8_disable_vblank;
6084	serge	4398	if (IS_BROXTON(dev))
		4399	dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
		4400	else if (HAS_PCH_SPT(dev))
		4401	dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		4402	else
		4403	dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
2351	Serge	4404	} else if (HAS_PCH_SPLIT(dev)) {
3243	Serge	4405	dev->driver->irq_handler = ironlake_irq_handler;
5060	serge	4406	dev->driver->irq_preinstall = ironlake_irq_reset;
3243	Serge	4407	dev->driver->irq_postinstall = ironlake_irq_postinstall;
4293	Serge	4408	dev->driver->irq_uninstall = ironlake_irq_uninstall;
		4409	dev->driver->enable_vblank = ironlake_enable_vblank;
		4410	dev->driver->disable_vblank = ironlake_disable_vblank;
6084	serge	4411	dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
2351	Serge	4412	} else {
5354	serge	4413	if (INTEL_INFO(dev_priv)->gen == 2) {
		4414	} else if (INTEL_INFO(dev_priv)->gen == 3) {
3243	Serge	4415	dev->driver->irq_preinstall = i915_irq_preinstall;
		4416	dev->driver->irq_postinstall = i915_irq_postinstall;
4293	Serge	4417	dev->driver->irq_uninstall = i915_irq_uninstall;
3243	Serge	4418	dev->driver->irq_handler = i915_irq_handler;
3031	serge	4419	} else {
3243	Serge	4420	dev->driver->irq_preinstall = i965_irq_preinstall;
		4421	dev->driver->irq_postinstall = i965_irq_postinstall;
4293	Serge	4422	dev->driver->irq_uninstall = i965_irq_uninstall;
3243	Serge	4423	dev->driver->irq_handler = i965_irq_handler;
6084	serge	4424	}
		4425	if (I915_HAS_HOTPLUG(dev_priv))
3746	Serge	4426	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4293	Serge	4427	dev->driver->enable_vblank = i915_enable_vblank;
		4428	dev->driver->disable_vblank = i915_disable_vblank;
2351	Serge	4429	}
3480	Serge	4430	}
3243	Serge	4431
5354	serge	4432	/**
		4433	* intel_irq_install - enables the hardware interrupt
		4434	* @dev_priv: i915 device instance
		4435	*
		4436	* This function enables the hardware interrupt handling, but leaves the hotplug
		4437	* handling still disabled. It is called after intel_irq_init().
		4438	*
		4439	* In the driver load and resume code we need working interrupts in a few places
		4440	* but don't want to deal with the hassle of concurrent probe and hotplug
		4441	* workers. Hence the split into this two-stage approach.
		4442	*/
		4443	int intel_irq_install(struct drm_i915_private *dev_priv)
3243	Serge	4444	{
5354	serge	4445	/*
		4446	* We enable some interrupt sources in our postinstall hooks, so mark
		4447	* interrupts as enabled _before_ actually enabling them to avoid
		4448	* special cases in our ordering checks.
		4449	*/
		4450	dev_priv->pm.irqs_enabled = true;
2351	Serge	4451
5354	serge	4452	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
3243	Serge	4453	}
		4454
5354	serge	4455	/**
		4456	* intel_irq_uninstall - finilizes all irq handling
		4457	* @dev_priv: i915 device instance
		4458	*
		4459	* This stops interrupt and hotplug handling and unregisters and frees all
		4460	* resources acquired in the init functions.
		4461	*/
		4462	void intel_irq_uninstall(struct drm_i915_private *dev_priv)
2351	Serge	4463	{
5354	serge	4464	// drm_irq_uninstall(dev_priv->dev);
6320	serge	4465	intel_hpd_cancel_work(dev_priv);
5354	serge	4466	dev_priv->pm.irqs_enabled = false;
		4467	}
2351	Serge	4468
5354	serge	4469	/**
		4470	* intel_runtime_pm_disable_interrupts - runtime interrupt disabling
		4471	* @dev_priv: i915 device instance
		4472	*
		4473	* This function is used to disable interrupts at runtime, both in the runtime
		4474	* pm and the system suspend/resume code.
		4475	*/
		4476	void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
		4477	{
		4478	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
		4479	dev_priv->pm.irqs_enabled = false;
4104	Serge	4480	}
2351	Serge	4481
5354	serge	4482	/**
		4483	* intel_runtime_pm_enable_interrupts - runtime interrupt enabling
		4484	* @dev_priv: i915 device instance
		4485	*
		4486	* This function is used to enable interrupts at runtime, both in the runtime
		4487	* pm and the system suspend/resume code.
		4488	*/
		4489	void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
		4490	{
		4491	dev_priv->pm.irqs_enabled = true;
		4492	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
		4493	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
		4494	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_irq.c – Rev 6320