WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/i915_irq.c

Rev	Author	Line No.	Line
2351	Serge	1	/* i915_irq.c -- IRQ support for the I915 -- linux-c --
		2	*/
		3	/*
		4	* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
		5	* All Rights Reserved.
		6	*
		7	* Permission is hereby granted, free of charge, to any person obtaining a
		8	* copy of this software and associated documentation files (the
		9	* "Software"), to deal in the Software without restriction, including
		10	* without limitation the rights to use, copy, modify, merge, publish,
		11	* distribute, sub license, and/or sell copies of the Software, and to
		12	* permit persons to whom the Software is furnished to do so, subject to
		13	* the following conditions:
		14	*
		15	* The above copyright notice and this permission notice (including the
		16	* next paragraph) shall be included in all copies or substantial portions
		17	* of the Software.
		18	*
		19	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
		20	* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
		21	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
		22	* IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
		23	* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
		24	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
		25	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
		26	*
		27	*/
		28
3746	Serge	29	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3031	serge	30
6103	serge	31	#include
3031	serge	32	#include
6088	serge	33	#include
3031	serge	34	#include
		35	#include
2351	Serge	36	#include "i915_drv.h"
		37	#include "i915_trace.h"
		38	#include "intel_drv.h"
		39
5354	serge	40	/**
		41	* DOC: interrupt handling
		42	*
		43	* These functions provide the basic support for enabling and disabling the
		44	* interrupt handling support. There's a lot more functionality in i915_irq.c
		45	* and related files, but that will be described in separate chapters.
		46	*/
4104	Serge	47
6084	serge	48	static const u32 hpd_ilk[HPD_NUM_PINS] = {
		49	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
		50	};
		51
		52	static const u32 hpd_ivb[HPD_NUM_PINS] = {
		53	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
		54	};
		55
		56	static const u32 hpd_bdw[HPD_NUM_PINS] = {
		57	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
		58	};
		59
		60	static const u32 hpd_ibx[HPD_NUM_PINS] = {
3746	Serge	61	[HPD_CRT] = SDE_CRT_HOTPLUG,
		62	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
		63	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
		64	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
		65	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
		66	};
3031	serge	67
6084	serge	68	static const u32 hpd_cpt[HPD_NUM_PINS] = {
3746	Serge	69	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
		70	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
		71	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
		72	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
		73	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
		74	};
		75
6084	serge	76	static const u32 hpd_spt[HPD_NUM_PINS] = {
		77	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
		78	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
		79	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
		80	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
		81	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
		82	};
		83
		84	static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
3746	Serge	85	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
		86	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
		87	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
		88	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
		89	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
		90	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
		91	};
		92
6084	serge	93	static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
3746	Serge	94	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
		95	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
		96	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
		97	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
		98	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
		99	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
		100	};
		101
6084	serge	102	static const u32 hpd_status_i915[HPD_NUM_PINS] = {
3746	Serge	103	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
		104	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
		105	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
		106	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
		107	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
		108	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
		109	};
		110
6084	serge	111	/* BXT hpd list */
		112	static const u32 hpd_bxt[HPD_NUM_PINS] = {
		113	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
		114	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
		115	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
		116	};
		117
5060	serge	118	/* IIR can theoretically queue up two events. Be paranoid. */
		119	#define GEN8_IRQ_RESET_NDX(type, which) do { \
		120	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
		121	POSTING_READ(GEN8_##type##_IMR(which)); \
		122	I915_WRITE(GEN8_##type##_IER(which), 0); \
		123	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
		124	POSTING_READ(GEN8_##type##_IIR(which)); \
		125	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
		126	POSTING_READ(GEN8_##type##_IIR(which)); \
		127	} while (0)
3746	Serge	128
5060	serge	129	#define GEN5_IRQ_RESET(type) do { \
		130	I915_WRITE(type##IMR, 0xffffffff); \
		131	POSTING_READ(type##IMR); \
		132	I915_WRITE(type##IER, 0); \
		133	I915_WRITE(type##IIR, 0xffffffff); \
		134	POSTING_READ(type##IIR); \
		135	I915_WRITE(type##IIR, 0xffffffff); \
		136	POSTING_READ(type##IIR); \
		137	} while (0)
		138
		139	/*
		140	* We should clear IMR at preinstall/uninstall, and just check at postinstall.
		141	*/
6084	serge	142	static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
		143	{
		144	u32 val = I915_READ(reg);
5060	serge	145
6084	serge	146	if (val == 0)
		147	return;
		148
		149	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
		150	reg, val);
		151	I915_WRITE(reg, 0xffffffff);
		152	POSTING_READ(reg);
		153	I915_WRITE(reg, 0xffffffff);
		154	POSTING_READ(reg);
		155	}
		156
5060	serge	157	#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
6084	serge	158	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
5354	serge	159	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
5060	serge	160	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
5354	serge	161	POSTING_READ(GEN8_##type##_IMR(which)); \
5060	serge	162	} while (0)
		163
		164	#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
6084	serge	165	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
5354	serge	166	I915_WRITE(type##IER, (ier_val)); \
5060	serge	167	I915_WRITE(type##IMR, (imr_val)); \
5354	serge	168	POSTING_READ(type##IMR); \
5060	serge	169	} while (0)
		170
5354	serge	171	static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
3031	serge	172
2351	Serge	173	/* For display hotplug interrupt */
6084	serge	174	static inline void
		175	i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
		176	uint32_t mask,
		177	uint32_t bits)
2351	Serge	178	{
6084	serge	179	uint32_t val;
		180
4104	Serge	181	assert_spin_locked(&dev_priv->irq_lock);
6084	serge	182	WARN_ON(bits & ~mask);
4104	Serge	183
6084	serge	184	val = I915_READ(PORT_HOTPLUG_EN);
		185	val &= ~mask;
		186	val \|= bits;
		187	I915_WRITE(PORT_HOTPLUG_EN, val);
		188	}
4104	Serge	189
6084	serge	190	/**
		191	* i915_hotplug_interrupt_update - update hotplug interrupt enable
		192	* @dev_priv: driver private
		193	* @mask: bits to update
		194	* @bits: bits to enable
		195	* NOTE: the HPD enable bits are modified both inside and outside
		196	* of an interrupt context. To avoid that read-modify-write cycles
		197	* interfer, these bits are protected by a spinlock. Since this
		198	* function is usually not called from a context where the lock is
		199	* held already, this function acquires the lock itself. A non-locking
		200	* version is also available.
		201	*/
		202	void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
		203	uint32_t mask,
		204	uint32_t bits)
		205	{
		206	spin_lock_irq(&dev_priv->irq_lock);
		207	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
		208	spin_unlock_irq(&dev_priv->irq_lock);
2351	Serge	209	}
		210
6084	serge	211	/**
		212	* ilk_update_display_irq - update DEIMR
		213	* @dev_priv: driver private
		214	* @interrupt_mask: mask of interrupt bits to update
		215	* @enabled_irq_mask: mask of interrupt bits to enable
		216	*/
		217	static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
		218	uint32_t interrupt_mask,
		219	uint32_t enabled_irq_mask)
2351	Serge	220	{
6084	serge	221	uint32_t new_val;
		222
4104	Serge	223	assert_spin_locked(&dev_priv->irq_lock);
		224
6084	serge	225	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		226
5354	serge	227	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	228	return;
		229
6084	serge	230	new_val = dev_priv->irq_mask;
		231	new_val &= ~interrupt_mask;
		232	new_val \|= (~enabled_irq_mask & interrupt_mask);
		233
		234	if (new_val != dev_priv->irq_mask) {
		235	dev_priv->irq_mask = new_val;
		236	I915_WRITE(DEIMR, dev_priv->irq_mask);
		237	POSTING_READ(DEIMR);
		238	}
2351	Serge	239	}
3031	serge	240
6084	serge	241	void
		242	ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
		243	{
		244	ilk_update_display_irq(dev_priv, mask, mask);
		245	}
		246
		247	void
		248	ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
		249	{
		250	ilk_update_display_irq(dev_priv, mask, 0);
		251	}
		252
4104	Serge	253	/**
		254	* ilk_update_gt_irq - update GTIMR
		255	* @dev_priv: driver private
		256	* @interrupt_mask: mask of interrupt bits to update
		257	* @enabled_irq_mask: mask of interrupt bits to enable
		258	*/
		259	static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
		260	uint32_t interrupt_mask,
		261	uint32_t enabled_irq_mask)
		262	{
		263	assert_spin_locked(&dev_priv->irq_lock);
		264
6084	serge	265	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		266
5060	serge	267	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	268	return;
		269
		270	dev_priv->gt_irq_mask &= ~interrupt_mask;
		271	dev_priv->gt_irq_mask \|= (~enabled_irq_mask & interrupt_mask);
		272	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		273	POSTING_READ(GTIMR);
		274	}
		275
5060	serge	276	void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	277	{
		278	ilk_update_gt_irq(dev_priv, mask, mask);
		279	}
		280
5060	serge	281	void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	282	{
		283	ilk_update_gt_irq(dev_priv, mask, 0);
		284	}
		285
5354	serge	286	static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
4104	Serge	287	{
5354	serge	288	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
4104	Serge	289	}
		290
5354	serge	291	static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
4104	Serge	292	{
5354	serge	293	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
4104	Serge	294	}
		295
5354	serge	296	static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
4104	Serge	297	{
5354	serge	298	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
4104	Serge	299	}
		300
5060	serge	301	/**
5354	serge	302	* snb_update_pm_irq - update GEN6_PMIMR
5060	serge	303	* @dev_priv: driver private
		304	* @interrupt_mask: mask of interrupt bits to update
		305	* @enabled_irq_mask: mask of interrupt bits to enable
		306	*/
5354	serge	307	static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
5060	serge	308	uint32_t interrupt_mask,
		309	uint32_t enabled_irq_mask)
		310	{
		311	uint32_t new_val;
		312
6084	serge	313	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		314
5060	serge	315	assert_spin_locked(&dev_priv->irq_lock);
		316
		317	new_val = dev_priv->pm_irq_mask;
		318	new_val &= ~interrupt_mask;
		319	new_val \|= (~enabled_irq_mask & interrupt_mask);
		320
		321	if (new_val != dev_priv->pm_irq_mask) {
		322	dev_priv->pm_irq_mask = new_val;
5354	serge	323	I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		324	POSTING_READ(gen6_pm_imr(dev_priv));
5060	serge	325	}
		326	}
		327
5354	serge	328	void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
5060	serge	329	{
5354	serge	330	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		331	return;
		332
		333	snb_update_pm_irq(dev_priv, mask, mask);
5060	serge	334	}
		335
5354	serge	336	static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
		337	uint32_t mask)
5060	serge	338	{
5354	serge	339	snb_update_pm_irq(dev_priv, mask, 0);
5060	serge	340	}
		341
5354	serge	342	void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
4104	Serge	343	{
5354	serge	344	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		345	return;
4104	Serge	346
5354	serge	347	__gen6_disable_pm_irq(dev_priv, mask);
4104	Serge	348	}
		349
5354	serge	350	void gen6_reset_rps_interrupts(struct drm_device *dev)
5060	serge	351	{
		352	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	353	uint32_t reg = gen6_pm_iir(dev_priv);
5060	serge	354
5354	serge	355	spin_lock_irq(&dev_priv->irq_lock);
		356	I915_WRITE(reg, dev_priv->pm_rps_events);
		357	I915_WRITE(reg, dev_priv->pm_rps_events);
6084	serge	358	POSTING_READ(reg);
		359	dev_priv->rps.pm_iir = 0;
5354	serge	360	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	361	}
		362
5354	serge	363	void gen6_enable_rps_interrupts(struct drm_device *dev)
5060	serge	364	{
		365	struct drm_i915_private *dev_priv = dev->dev_private;
		366
5354	serge	367	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	368
5354	serge	369	WARN_ON(dev_priv->rps.pm_iir);
		370	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
		371	dev_priv->rps.interrupts_enabled = true;
		372	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) \|
		373	dev_priv->pm_rps_events);
		374	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		375
		376	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	377	}
		378
6084	serge	379	u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
		380	{
		381	/*
		382	* SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
		383	* if GEN6_PM_UP_EI_EXPIRED is masked.
		384	*
		385	* TODO: verify if this can be reproduced on VLV,CHV.
		386	*/
		387	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		388	mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
		389
		390	if (INTEL_INFO(dev_priv)->gen >= 8)
		391	mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
		392
		393	return mask;
		394	}
		395
5354	serge	396	void gen6_disable_rps_interrupts(struct drm_device *dev)
4104	Serge	397	{
		398	struct drm_i915_private *dev_priv = dev->dev_private;
		399
5354	serge	400	spin_lock_irq(&dev_priv->irq_lock);
		401	dev_priv->rps.interrupts_enabled = false;
		402	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	403
5354	serge	404	cancel_work_sync(&dev_priv->rps.work);
4104	Serge	405
5354	serge	406	spin_lock_irq(&dev_priv->irq_lock);
4104	Serge	407
6084	serge	408	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
4104	Serge	409
5354	serge	410	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		411	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
		412	~dev_priv->pm_rps_events);
4104	Serge	413
6084	serge	414	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	415
		416	}
		417
4104	Serge	418	/**
6084	serge	419	* bdw_update_port_irq - update DE port interrupt
		420	* @dev_priv: driver private
		421	* @interrupt_mask: mask of interrupt bits to update
		422	* @enabled_irq_mask: mask of interrupt bits to enable
		423	*/
		424	static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
		425	uint32_t interrupt_mask,
		426	uint32_t enabled_irq_mask)
		427	{
		428	uint32_t new_val;
		429	uint32_t old_val;
		430
		431	assert_spin_locked(&dev_priv->irq_lock);
		432
		433	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		434
		435	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		436	return;
		437
		438	old_val = I915_READ(GEN8_DE_PORT_IMR);
		439
		440	new_val = old_val;
		441	new_val &= ~interrupt_mask;
		442	new_val \|= (~enabled_irq_mask & interrupt_mask);
		443
		444	if (new_val != old_val) {
		445	I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		446	POSTING_READ(GEN8_DE_PORT_IMR);
		447	}
		448	}
		449
		450	/**
4104	Serge	451	* ibx_display_interrupt_update - update SDEIMR
		452	* @dev_priv: driver private
		453	* @interrupt_mask: mask of interrupt bits to update
		454	* @enabled_irq_mask: mask of interrupt bits to enable
		455	*/
5354	serge	456	void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
6084	serge	457	uint32_t interrupt_mask,
		458	uint32_t enabled_irq_mask)
4104	Serge	459	{
		460	uint32_t sdeimr = I915_READ(SDEIMR);
		461	sdeimr &= ~interrupt_mask;
		462	sdeimr \|= (~enabled_irq_mask & interrupt_mask);
		463
6084	serge	464	WARN_ON(enabled_irq_mask & ~interrupt_mask);
		465
4104	Serge	466	assert_spin_locked(&dev_priv->irq_lock);
		467
5060	serge	468	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
4104	Serge	469	return;
		470
		471	I915_WRITE(SDEIMR, sdeimr);
		472	POSTING_READ(SDEIMR);
		473	}
		474
5060	serge	475	static void
		476	__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		477	u32 enable_mask, u32 status_mask)
3031	serge	478	{
6084	serge	479	u32 reg = PIPESTAT(pipe);
5060	serge	480	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
3031	serge	481
4104	Serge	482	assert_spin_locked(&dev_priv->irq_lock);
5354	serge	483	WARN_ON(!intel_irqs_enabled(dev_priv));
4104	Serge	484
5060	serge	485	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK \|\|
		486	status_mask & ~PIPESTAT_INT_STATUS_MASK,
		487	"pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		488	pipe_name(pipe), enable_mask, status_mask))
3746	Serge	489	return;
		490
5060	serge	491	if ((pipestat & enable_mask) == enable_mask)
		492	return;
		493
		494	dev_priv->pipestat_irq_mask[pipe] \|= status_mask;
		495
6084	serge	496	/* Enable the interrupt, clear any pending status */
5060	serge	497	pipestat \|= enable_mask \| status_mask;
3746	Serge	498	I915_WRITE(reg, pipestat);
6084	serge	499	POSTING_READ(reg);
3031	serge	500	}
		501
5060	serge	502	static void
		503	__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		504	u32 enable_mask, u32 status_mask)
3031	serge	505	{
6084	serge	506	u32 reg = PIPESTAT(pipe);
5060	serge	507	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
3031	serge	508
4104	Serge	509	assert_spin_locked(&dev_priv->irq_lock);
5354	serge	510	WARN_ON(!intel_irqs_enabled(dev_priv));
4104	Serge	511
5060	serge	512	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK \|\|
		513	status_mask & ~PIPESTAT_INT_STATUS_MASK,
		514	"pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		515	pipe_name(pipe), enable_mask, status_mask))
3746	Serge	516	return;
		517
5060	serge	518	if ((pipestat & enable_mask) == 0)
		519	return;
		520
		521	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
		522
		523	pipestat &= ~enable_mask;
3746	Serge	524	I915_WRITE(reg, pipestat);
6084	serge	525	POSTING_READ(reg);
3031	serge	526	}
		527
5060	serge	528	static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
		529	{
		530	u32 enable_mask = status_mask << 16;
		531
		532	/*
		533	* On pipe A we don't support the PSR interrupt yet,
		534	* on pipe B and C the same bit MBZ.
		535	*/
		536	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		537	return 0;
		538	/*
		539	* On pipe B and C we don't support the PSR interrupt yet, on pipe
		540	* A the same bit is for perf counters which we don't use either.
		541	*/
		542	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		543	return 0;
		544
		545	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS \|
		546	SPRITE0_FLIP_DONE_INT_EN_VLV \|
		547	SPRITE1_FLIP_DONE_INT_EN_VLV);
		548	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		549	enable_mask \|= SPRITE0_FLIP_DONE_INT_EN_VLV;
		550	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		551	enable_mask \|= SPRITE1_FLIP_DONE_INT_EN_VLV;
		552
		553	return enable_mask;
		554	}
		555
		556	void
		557	i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		558	u32 status_mask)
		559	{
		560	u32 enable_mask;
		561
		562	if (IS_VALLEYVIEW(dev_priv->dev))
		563	enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
		564	status_mask);
		565	else
		566	enable_mask = status_mask << 16;
		567	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
		568	}
		569
		570	void
		571	i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		572	u32 status_mask)
		573	{
		574	u32 enable_mask;
		575
		576	if (IS_VALLEYVIEW(dev_priv->dev))
		577	enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
		578	status_mask);
		579	else
		580	enable_mask = status_mask << 16;
		581	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
		582	}
		583
3031	serge	584	/**
4104	Serge	585	* i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
6084	serge	586	* @dev: drm device
3031	serge	587	*/
4104	Serge	588	static void i915_enable_asle_pipestat(struct drm_device *dev)
3031	serge	589	{
5060	serge	590	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	591
4104	Serge	592	if (!dev_priv->opregion.asle \|\| !IS_MOBILE(dev))
3031	serge	593	return;
		594
5354	serge	595	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	596
5060	serge	597	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
6084	serge	598	if (INTEL_INFO(dev)->gen >= 4)
4560	Serge	599	i915_enable_pipestat(dev_priv, PIPE_A,
5060	serge	600	PIPE_LEGACY_BLC_EVENT_STATUS);
3031	serge	601
5354	serge	602	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	603	}
		604
5060	serge	605	/*
		606	* This timing diagram depicts the video signal in and
		607	* around the vertical blanking period.
		608	*
		609	* Assumptions about the fictitious mode used in this example:
		610	* vblank_start >= 3
		611	* vsync_start = vblank_start + 1
		612	* vsync_end = vblank_start + 2
		613	* vtotal = vblank_start + 3
		614	*
		615	* start of vblank:
		616	* latch double buffered registers
		617	* increment frame counter (ctg+)
		618	* generate start of vblank interrupt (gen4+)
		619	* \|
		620	* \| frame start:
		621	* \| generate frame start interrupt (aka. vblank interrupt) (gmch)
		622	* \| may be shifted forward 1-3 extra lines via PIPECONF
		623	* \| \|
		624	* \| \| start of vsync:
		625	* \| \| generate vsync interrupt
		626	* \| \| \|
		627	* ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
		628	* . \hs/ . \hs/ \hs/ \hs/ . \hs/
		629	* ----va---> <-----------------vb--------------------> <--------va-------------
		630	* \| \| <----vs-----> \|
		631	* -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
		632	* -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
		633	* -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
		634	* \| \| \|
		635	* last visible pixel first visible pixel
		636	* \| increment frame counter (gen3/4)
		637	* pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
		638	*
		639	* x = horizontal active
		640	* _ = horizontal blanking
		641	* hs = horizontal sync
		642	* va = vertical active
		643	* vb = vertical blanking
		644	* vs = vertical sync
		645	* vbs = vblank_start (number)
		646	*
		647	* Summary:
		648	* - most events happen at the start of horizontal sync
		649	* - frame start happens at the start of horizontal blank, 1-4 lines
		650	* (depending on PIPECONF settings) after the start of vblank
		651	* - gen3/4 pixel and frame counter are synchronized with the start
		652	* of horizontal active on the first line of vertical active
		653	*/
		654
6084	serge	655	static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
4560	Serge	656	{
		657	/* Gen2 doesn't have a hardware frame counter */
		658	return 0;
		659	}
		660
3031	serge	661	/* Called from drm generic code, passed a 'crtc', which
		662	* we use as a pipe index
		663	*/
6084	serge	664	static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
3031	serge	665	{
5060	serge	666	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	667	unsigned long high_frame;
		668	unsigned long low_frame;
5060	serge	669	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
6084	serge	670	struct intel_crtc *intel_crtc =
		671	to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
		672	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
3031	serge	673
6084	serge	674	htotal = mode->crtc_htotal;
		675	hsync_start = mode->crtc_hsync_start;
		676	vbl_start = mode->crtc_vblank_start;
		677	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		678	vbl_start = DIV_ROUND_UP(vbl_start, 2);
3031	serge	679
5060	serge	680	/* Convert to pixel count */
6084	serge	681	vbl_start *= htotal;
4560	Serge	682
5060	serge	683	/* Start of vblank event occurs at start of hsync */
		684	vbl_start -= htotal - hsync_start;
		685
3031	serge	686	high_frame = PIPEFRAME(pipe);
		687	low_frame = PIPEFRAMEPIXEL(pipe);
		688
		689	/*
		690	* High & low register fields aren't synchronized, so make sure
		691	* we get a low value that's stable across two reads of the high
		692	* register.
		693	*/
		694	do {
		695	high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
4560	Serge	696	low = I915_READ(low_frame);
3031	serge	697	high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
		698	} while (high1 != high2);
		699
		700	high1 >>= PIPE_FRAME_HIGH_SHIFT;
4560	Serge	701	pixel = low & PIPE_PIXEL_MASK;
3031	serge	702	low >>= PIPE_FRAME_LOW_SHIFT;
4560	Serge	703
		704	/*
		705	* The frame counter increments at beginning of active.
		706	* Cook up a vblank counter by also checking the pixel
		707	* counter against vblank start.
		708	*/
		709	return (((high1 << 8) \| low) + (pixel >= vbl_start)) & 0xffffff;
3031	serge	710	}
		711
6084	serge	712	static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
3031	serge	713	{
5060	serge	714	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	715
6084	serge	716	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
3031	serge	717	}
		718
4560	Serge	719	/* raw reads, only for fast reads of display block, no need for forcewake etc. */
		720	#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
		721
5060	serge	722	static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
4560	Serge	723	{
5060	serge	724	struct drm_device *dev = crtc->base.dev;
4560	Serge	725	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	726	const struct drm_display_mode *mode = &crtc->base.hwmode;
5060	serge	727	enum pipe pipe = crtc->pipe;
		728	int position, vtotal;
4560	Serge	729
5060	serge	730	vtotal = mode->crtc_vtotal;
		731	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		732	vtotal /= 2;
4560	Serge	733
5060	serge	734	if (IS_GEN2(dev))
		735	position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
		736	else
		737	position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
		738
		739	/*
6084	serge	740	* On HSW, the DSL reg (0x70000) appears to return 0 if we
		741	* read it just before the start of vblank. So try it again
		742	* so we don't accidentally end up spanning a vblank frame
		743	* increment, causing the pipe_update_end() code to squak at us.
		744	*
		745	* The nature of this problem means we can't simply check the ISR
		746	* bit and return the vblank start value; nor can we use the scanline
		747	* debug register in the transcoder as it appears to have the same
		748	* problem. We may need to extend this to include other platforms,
		749	* but so far testing only shows the problem on HSW.
		750	*/
		751	if (HAS_DDI(dev) && !position) {
		752	int i, temp;
		753
		754	for (i = 0; i < 100; i++) {
		755	udelay(1);
		756	temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
		757	DSL_LINEMASK_GEN3;
		758	if (temp != position) {
		759	position = temp;
		760	break;
		761	}
		762	}
		763	}
		764
		765	/*
5060	serge	766	* See update_scanline_offset() for the details on the
		767	* scanline_offset adjustment.
		768	*/
		769	return (position + crtc->scanline_offset) % vtotal;
4560	Serge	770	}
		771
6084	serge	772	static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
4560	Serge	773	unsigned int flags, int vpos, int hpos,
6084	serge	774	ktime_t stime, ktime_t etime,
		775	const struct drm_display_mode *mode)
3746	Serge	776	{
4560	Serge	777	struct drm_i915_private *dev_priv = dev->dev_private;
		778	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		779	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		780	int position;
5060	serge	781	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
3746	Serge	782	bool in_vbl = true;
		783	int ret = 0;
4560	Serge	784	unsigned long irqflags;
3746	Serge	785
6084	serge	786	if (WARN_ON(!mode->crtc_clock)) {
3746	Serge	787	DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
		788	"pipe %c\n", pipe_name(pipe));
		789	return 0;
		790	}
		791
4560	Serge	792	htotal = mode->crtc_htotal;
5060	serge	793	hsync_start = mode->crtc_hsync_start;
4560	Serge	794	vtotal = mode->crtc_vtotal;
		795	vbl_start = mode->crtc_vblank_start;
		796	vbl_end = mode->crtc_vblank_end;
3746	Serge	797
4560	Serge	798	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		799	vbl_start = DIV_ROUND_UP(vbl_start, 2);
		800	vbl_end /= 2;
		801	vtotal /= 2;
		802	}
		803
		804	ret \|= DRM_SCANOUTPOS_VALID \| DRM_SCANOUTPOS_ACCURATE;
		805
		806	/*
		807	* Lock uncore.lock, as we will do multiple timing critical raw
		808	* register reads, potentially with preemption disabled, so the
		809	* following code must not block on uncore.lock.
		810	*/
		811	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
		812
		813	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
		814
6084	serge	815	/* Get optional system timestamp before query. */
		816	if (stime)
		817	*stime = ktime_get();
4560	Serge	818
		819	if (IS_GEN2(dev) \|\| IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
3746	Serge	820	/* No obvious pixelcount register. Only query vertical
		821	* scanout position from Display scan line register.
		822	*/
5060	serge	823	position = __intel_get_crtc_scanline(intel_crtc);
3746	Serge	824	} else {
		825	/* Have access to pixelcount since start of frame.
		826	* We can split this into vertical and horizontal
		827	* scanout position.
		828	*/
4560	Serge	829	position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
3746	Serge	830
4560	Serge	831	/* convert to pixel counts */
		832	vbl_start *= htotal;
		833	vbl_end *= htotal;
		834	vtotal *= htotal;
5060	serge	835
		836	/*
		837	* In interlaced modes, the pixel counter counts all pixels,
		838	* so one field will have htotal more pixels. In order to avoid
		839	* the reported position from jumping backwards when the pixel
		840	* counter is beyond the length of the shorter field, just
		841	* clamp the position the length of the shorter field. This
		842	* matches how the scanline counter based position works since
		843	* the scanline counter doesn't count the two half lines.
		844	*/
		845	if (position >= vtotal)
		846	position = vtotal - 1;
		847
		848	/*
		849	* Start of vblank interrupt is triggered at start of hsync,
		850	* just prior to the first active line of vblank. However we
		851	* consider lines to start at the leading edge of horizontal
		852	* active. So, should we get here before we've crossed into
		853	* the horizontal active of the first line in vblank, we would
		854	* not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		855	* always add htotal-hsync_start to the current pixel position.
		856	*/
		857	position = (position + htotal - hsync_start) % vtotal;
3746	Serge	858	}
		859
6084	serge	860	/* Get optional system timestamp after query. */
		861	if (etime)
		862	*etime = ktime_get();
3746	Serge	863
4560	Serge	864	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
3746	Serge	865
4560	Serge	866	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3746	Serge	867
4560	Serge	868	in_vbl = position >= vbl_start && position < vbl_end;
3746	Serge	869
4560	Serge	870	/*
		871	* While in vblank, position will be negative
		872	* counting up towards 0 at vbl_end. And outside
		873	* vblank, position will be positive counting
		874	* up since vbl_end.
		875	*/
		876	if (position >= vbl_start)
		877	position -= vbl_end;
		878	else
		879	position += vtotal - vbl_end;
3746	Serge	880
4560	Serge	881	if (IS_GEN2(dev) \|\| IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		882	*vpos = position;
		883	*hpos = 0;
		884	} else {
		885	*vpos = position / htotal;
		886	hpos = position - (vpos * htotal);
		887	}
		888
3746	Serge	889	/* In vblank? */
		890	if (in_vbl)
5354	serge	891	ret \|= DRM_SCANOUTPOS_IN_VBLANK;
3746	Serge	892
		893	return ret;
		894	}
		895
5060	serge	896	int intel_get_crtc_scanline(struct intel_crtc *crtc)
		897	{
		898	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
		899	unsigned long irqflags;
		900	int position;
		901
		902	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
		903	position = __intel_get_crtc_scanline(crtc);
		904	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
		905
		906	return position;
		907	}
		908
6084	serge	909	static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
3746	Serge	910	int *max_error,
		911	struct timeval *vblank_time,
		912	unsigned flags)
		913	{
		914	struct drm_crtc *crtc;
		915
6084	serge	916	if (pipe >= INTEL_INFO(dev)->num_pipes) {
		917	DRM_ERROR("Invalid crtc %u\n", pipe);
3746	Serge	918	return -EINVAL;
		919	}
		920
		921	/* Get drm_crtc to timestamp: */
		922	crtc = intel_get_crtc_for_pipe(dev, pipe);
		923	if (crtc == NULL) {
6084	serge	924	DRM_ERROR("Invalid crtc %u\n", pipe);
3746	Serge	925	return -EINVAL;
		926	}
		927
6084	serge	928	if (!crtc->hwmode.crtc_clock) {
		929	DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
3746	Serge	930	return -EBUSY;
		931	}
		932
		933	/* Helper routine in DRM core does all the work: */
		934	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
		935	vblank_time, flags,
6084	serge	936	&crtc->hwmode);
3746	Serge	937	}
		938
4104	Serge	939	static void ironlake_rps_change_irq_handler(struct drm_device *dev)
3746	Serge	940	{
5060	serge	941	struct drm_i915_private *dev_priv = dev->dev_private;
3746	Serge	942	u32 busy_up, busy_down, max_avg, min_avg;
		943	u8 new_delay;
		944
4104	Serge	945	spin_lock(&mchdev_lock);
3746	Serge	946
		947	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
		948
		949	new_delay = dev_priv->ips.cur_delay;
		950
		951	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
		952	busy_up = I915_READ(RCPREVBSYTUPAVG);
		953	busy_down = I915_READ(RCPREVBSYTDNAVG);
		954	max_avg = I915_READ(RCBMAXAVG);
		955	min_avg = I915_READ(RCBMINAVG);
		956
		957	/* Handle RCS change request from hw */
		958	if (busy_up > max_avg) {
		959	if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
		960	new_delay = dev_priv->ips.cur_delay - 1;
		961	if (new_delay < dev_priv->ips.max_delay)
		962	new_delay = dev_priv->ips.max_delay;
		963	} else if (busy_down < min_avg) {
		964	if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
		965	new_delay = dev_priv->ips.cur_delay + 1;
		966	if (new_delay > dev_priv->ips.min_delay)
		967	new_delay = dev_priv->ips.min_delay;
		968	}
		969
		970	if (ironlake_set_drps(dev, new_delay))
		971	dev_priv->ips.cur_delay = new_delay;
		972
4104	Serge	973	spin_unlock(&mchdev_lock);
3746	Serge	974
		975	return;
		976	}
		977
6084	serge	978	static void notify_ring(struct intel_engine_cs *ring)
2352	Serge	979	{
5060	serge	980	if (!intel_ring_initialized(ring))
2352	Serge	981	return;
2351	Serge	982
6084	serge	983	trace_i915_gem_request_notify(ring);
2351	Serge	984
2352	Serge	985	wake_up_all(&ring->irq_queue);
		986	}
		987
6084	serge	988	static void vlv_c0_read(struct drm_i915_private *dev_priv,
		989	struct intel_rps_ei *ei)
5060	serge	990	{
6084	serge	991	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
		992	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
		993	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
		994	}
5060	serge	995
6084	serge	996	static bool vlv_c0_above(struct drm_i915_private *dev_priv,
		997	const struct intel_rps_ei *old,
		998	const struct intel_rps_ei *now,
		999	int threshold)
		1000	{
		1001	u64 time, c0;
		1002	unsigned int mul = 100;
5060	serge	1003
6084	serge	1004	if (old->cz_clock == 0)
		1005	return false;
5060	serge	1006
6084	serge	1007	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		1008	mul <<= 8;
5060	serge	1009
6084	serge	1010	time = now->cz_clock - old->cz_clock;
		1011	time = threshold dev_priv->czclk_freq;
5060	serge	1012
6084	serge	1013	/* Workload can be split between render + media, e.g. SwapBuffers
		1014	* being blitted in X after being rendered in mesa. To account for
		1015	* this we need to combine both engines into our activity counter.
5060	serge	1016	*/
6084	serge	1017	c0 = now->render_c0 - old->render_c0;
		1018	c0 += now->media_c0 - old->media_c0;
		1019	c0 = mul VLV_CZ_CLOCK_TO_MILLI_SEC;
5060	serge	1020
6084	serge	1021	return c0 >= time;
5060	serge	1022	}
		1023
6084	serge	1024	void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
5060	serge	1025	{
6084	serge	1026	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
		1027	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
		1028	}
5060	serge	1029
6084	serge	1030	static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
		1031	{
		1032	struct intel_rps_ei now;
		1033	u32 events = 0;
5060	serge	1034
6084	serge	1035	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
		1036	return 0;
5060	serge	1037
6084	serge	1038	vlv_c0_read(dev_priv, &now);
		1039	if (now.cz_clock == 0)
		1040	return 0;
5060	serge	1041
6084	serge	1042	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		1043	if (!vlv_c0_above(dev_priv,
		1044	&dev_priv->rps.down_ei, &now,
		1045	dev_priv->rps.down_threshold))
		1046	events \|= GEN6_PM_RP_DOWN_THRESHOLD;
		1047	dev_priv->rps.down_ei = now;
5060	serge	1048	}
		1049
6084	serge	1050	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		1051	if (vlv_c0_above(dev_priv,
		1052	&dev_priv->rps.up_ei, &now,
		1053	dev_priv->rps.up_threshold))
		1054	events \|= GEN6_PM_RP_UP_THRESHOLD;
		1055	dev_priv->rps.up_ei = now;
5060	serge	1056	}
		1057
6084	serge	1058	return events;
		1059	}
5060	serge	1060
6084	serge	1061	static bool any_waiters(struct drm_i915_private *dev_priv)
		1062	{
		1063	struct intel_engine_cs *ring;
		1064	int i;
5060	serge	1065
6084	serge	1066	for_each_ring(ring, dev_priv, i)
		1067	if (ring->irq_refcount)
		1068	return true;
5060	serge	1069
6084	serge	1070	return false;
5060	serge	1071	}
		1072
3031	serge	1073	static void gen6_pm_rps_work(struct work_struct *work)
		1074	{
5060	serge	1075	struct drm_i915_private *dev_priv =
		1076	container_of(work, struct drm_i915_private, rps.work);
6084	serge	1077	bool client_boost;
		1078	int new_delay, adj, min, max;
4104	Serge	1079	u32 pm_iir;
2352	Serge	1080
4104	Serge	1081	spin_lock_irq(&dev_priv->irq_lock);
5354	serge	1082	/* Speed up work cancelation during disabling rps interrupts. */
		1083	if (!dev_priv->rps.interrupts_enabled) {
		1084	spin_unlock_irq(&dev_priv->irq_lock);
		1085	return;
		1086	}
3031	serge	1087	pm_iir = dev_priv->rps.pm_iir;
		1088	dev_priv->rps.pm_iir = 0;
5354	serge	1089	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
6084	serge	1090	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		1091	client_boost = dev_priv->rps.client_boost;
		1092	dev_priv->rps.client_boost = false;
4104	Serge	1093	spin_unlock_irq(&dev_priv->irq_lock);
2352	Serge	1094
4104	Serge	1095	/* Make sure we didn't queue anything we're not going to process. */
5060	serge	1096	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
4104	Serge	1097
6084	serge	1098	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
3031	serge	1099	return;
		1100
3243	Serge	1101	mutex_lock(&dev_priv->rps.hw_lock);
3031	serge	1102
6084	serge	1103	pm_iir \|= vlv_wa_c0_ei(dev_priv, pm_iir);
		1104
4560	Serge	1105	adj = dev_priv->rps.last_adj;
6084	serge	1106	new_delay = dev_priv->rps.cur_freq;
		1107	min = dev_priv->rps.min_freq_softlimit;
		1108	max = dev_priv->rps.max_freq_softlimit;
		1109
		1110	if (client_boost) {
		1111	new_delay = dev_priv->rps.max_freq_softlimit;
		1112	adj = 0;
		1113	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
4560	Serge	1114	if (adj > 0)
		1115	adj *= 2;
6084	serge	1116	else /* CHV needs even encode values */
		1117	adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
4104	Serge	1118	/*
		1119	* For better performance, jump directly
		1120	* to RPe if we're below it.
		1121	*/
6084	serge	1122	if (new_delay < dev_priv->rps.efficient_freq - adj) {
5060	serge	1123	new_delay = dev_priv->rps.efficient_freq;
6084	serge	1124	adj = 0;
		1125	}
		1126	} else if (any_waiters(dev_priv)) {
		1127	adj = 0;
4560	Serge	1128	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
5060	serge	1129	if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
		1130	new_delay = dev_priv->rps.efficient_freq;
4560	Serge	1131	else
5060	serge	1132	new_delay = dev_priv->rps.min_freq_softlimit;
4560	Serge	1133	adj = 0;
		1134	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		1135	if (adj < 0)
		1136	adj *= 2;
6084	serge	1137	else /* CHV needs even encode values */
		1138	adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
4560	Serge	1139	} else { /* unknown event */
6084	serge	1140	adj = 0;
4560	Serge	1141	}
3031	serge	1142
6084	serge	1143	dev_priv->rps.last_adj = adj;
		1144
3031	serge	1145	/* sysfs frequency interfaces may have snuck in while servicing the
		1146	* interrupt
		1147	*/
6084	serge	1148	new_delay += adj;
		1149	new_delay = clamp_t(int, new_delay, min, max);
4560	Serge	1150
6084	serge	1151	intel_set_rps(dev_priv->dev, new_delay);
5060	serge	1152
3243	Serge	1153	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	1154	}
		1155
		1156
		1157	/**
		1158	* ivybridge_parity_work - Workqueue called when a parity error interrupt
		1159	* occurred.
		1160	* @work: workqueue struct
		1161	*
		1162	* Doesn't actually do anything except notify userspace. As a consequence of
		1163	* this event, userspace should try to remap the bad rows since statistically
		1164	* it is likely the same row is more likely to go bad again.
		1165	*/
		1166	static void ivybridge_parity_work(struct work_struct *work)
2351	Serge	1167	{
5060	serge	1168	struct drm_i915_private *dev_priv =
		1169	container_of(work, struct drm_i915_private, l3_parity.error_work);
3031	serge	1170	u32 error_status, row, bank, subbank;
4560	Serge	1171	char *parity_event[6];
3031	serge	1172	uint32_t misccpctl;
4560	Serge	1173	uint8_t slice = 0;
3031	serge	1174
		1175	/* We must turn off DOP level clock gating to access the L3 registers.
		1176	* In order to prevent a get/put style interface, acquire struct mutex
		1177	* any time we access those registers.
		1178	*/
		1179	mutex_lock(&dev_priv->dev->struct_mutex);
		1180
4560	Serge	1181	/* If we've screwed up tracking, just let the interrupt fire again */
		1182	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		1183	goto out;
		1184
3031	serge	1185	misccpctl = I915_READ(GEN7_MISCCPCTL);
		1186	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		1187	POSTING_READ(GEN7_MISCCPCTL);
		1188
4560	Serge	1189	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		1190	u32 reg;
		1191
		1192	slice--;
		1193	if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
		1194	break;
		1195
		1196	dev_priv->l3_parity.which_slice &= ~(1<
		1197
		1198	reg = GEN7_L3CDERRST1 + (slice * 0x200);
		1199
		1200	error_status = I915_READ(reg);
6084	serge	1201	row = GEN7_PARITY_ERROR_ROW(error_status);
		1202	bank = GEN7_PARITY_ERROR_BANK(error_status);
		1203	subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
3031	serge	1204
4560	Serge	1205	I915_WRITE(reg, GEN7_PARITY_ERROR_VALID \| GEN7_L3CDERRST1_ENABLE);
		1206	POSTING_READ(reg);
3031	serge	1207
4560	Serge	1208	DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
		1209	slice, row, bank, subbank);
		1210
		1211	}
		1212
3031	serge	1213	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		1214
4560	Serge	1215	out:
		1216	WARN_ON(dev_priv->l3_parity.which_slice);
5354	serge	1217	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	1218	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
5354	serge	1219	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	1220
		1221	mutex_unlock(&dev_priv->dev->struct_mutex);
		1222	}
		1223
4560	Serge	1224	static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
3031	serge	1225	{
5060	serge	1226	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1227
4560	Serge	1228	if (!HAS_L3_DPF(dev))
3031	serge	1229	return;
		1230
4104	Serge	1231	spin_lock(&dev_priv->irq_lock);
5060	serge	1232	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
4104	Serge	1233	spin_unlock(&dev_priv->irq_lock);
3031	serge	1234
4560	Serge	1235	iir &= GT_PARITY_ERROR(dev);
		1236	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		1237	dev_priv->l3_parity.which_slice \|= 1 << 1;
		1238
		1239	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		1240	dev_priv->l3_parity.which_slice \|= 1 << 0;
		1241
3243	Serge	1242	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
3031	serge	1243	}
		1244
4104	Serge	1245	static void ilk_gt_irq_handler(struct drm_device *dev,
		1246	struct drm_i915_private *dev_priv,
		1247	u32 gt_iir)
		1248	{
		1249	if (gt_iir &
		1250	(GT_RENDER_USER_INTERRUPT \| GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
6084	serge	1251	notify_ring(&dev_priv->ring[RCS]);
4104	Serge	1252	if (gt_iir & ILK_BSD_USER_INTERRUPT)
6084	serge	1253	notify_ring(&dev_priv->ring[VCS]);
4104	Serge	1254	}
		1255
3031	serge	1256	static void snb_gt_irq_handler(struct drm_device *dev,
		1257	struct drm_i915_private *dev_priv,
		1258	u32 gt_iir)
		1259	{
		1260
4104	Serge	1261	if (gt_iir &
		1262	(GT_RENDER_USER_INTERRUPT \| GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
6084	serge	1263	notify_ring(&dev_priv->ring[RCS]);
4104	Serge	1264	if (gt_iir & GT_BSD_USER_INTERRUPT)
6084	serge	1265	notify_ring(&dev_priv->ring[VCS]);
4104	Serge	1266	if (gt_iir & GT_BLT_USER_INTERRUPT)
6084	serge	1267	notify_ring(&dev_priv->ring[BCS]);
3031	serge	1268
4104	Serge	1269	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT \|
		1270	GT_BSD_CS_ERROR_INTERRUPT \|
5354	serge	1271	GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		1272	DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
3031	serge	1273
4560	Serge	1274	if (gt_iir & GT_PARITY_ERROR(dev))
		1275	ivybridge_parity_error_irq_handler(dev, gt_iir);
3031	serge	1276	}
		1277
6084	serge	1278	static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
4560	Serge	1279	u32 master_ctl)
		1280	{
		1281	irqreturn_t ret = IRQ_NONE;
		1282
		1283	if (master_ctl & (GEN8_GT_RCS_IRQ \| GEN8_GT_BCS_IRQ)) {
6084	serge	1284	u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
4560	Serge	1285	if (tmp) {
6084	serge	1286	I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
4560	Serge	1287	ret = IRQ_HANDLED;
5354	serge	1288
6084	serge	1289	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
		1290	intel_lrc_irq_handler(&dev_priv->ring[RCS]);
		1291	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
		1292	notify_ring(&dev_priv->ring[RCS]);
5354	serge	1293
6084	serge	1294	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
		1295	intel_lrc_irq_handler(&dev_priv->ring[BCS]);
		1296	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
		1297	notify_ring(&dev_priv->ring[BCS]);
4560	Serge	1298	} else
		1299	DRM_ERROR("The master control interrupt lied (GT0)!\n");
		1300	}
		1301
5060	serge	1302	if (master_ctl & (GEN8_GT_VCS1_IRQ \| GEN8_GT_VCS2_IRQ)) {
6084	serge	1303	u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
4560	Serge	1304	if (tmp) {
6084	serge	1305	I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
4560	Serge	1306	ret = IRQ_HANDLED;
5354	serge	1307
6084	serge	1308	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
		1309	intel_lrc_irq_handler(&dev_priv->ring[VCS]);
		1310	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
		1311	notify_ring(&dev_priv->ring[VCS]);
5354	serge	1312
6084	serge	1313	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
		1314	intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
		1315	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
		1316	notify_ring(&dev_priv->ring[VCS2]);
4560	Serge	1317	} else
		1318	DRM_ERROR("The master control interrupt lied (GT1)!\n");
		1319	}
		1320
6084	serge	1321	if (master_ctl & GEN8_GT_VECS_IRQ) {
		1322	u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
		1323	if (tmp) {
		1324	I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
		1325	ret = IRQ_HANDLED;
		1326
		1327	if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
		1328	intel_lrc_irq_handler(&dev_priv->ring[VECS]);
		1329	if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
		1330	notify_ring(&dev_priv->ring[VECS]);
		1331	} else
		1332	DRM_ERROR("The master control interrupt lied (GT3)!\n");
		1333	}
		1334
5060	serge	1335	if (master_ctl & GEN8_GT_PM_IRQ) {
6084	serge	1336	u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
5060	serge	1337	if (tmp & dev_priv->pm_rps_events) {
6084	serge	1338	I915_WRITE_FW(GEN8_GT_IIR(2),
		1339	tmp & dev_priv->pm_rps_events);
5060	serge	1340	ret = IRQ_HANDLED;
5354	serge	1341	gen6_rps_irq_handler(dev_priv, tmp);
5060	serge	1342	} else
		1343	DRM_ERROR("The master control interrupt lied (PM)!\n");
		1344	}
		1345
6084	serge	1346	return ret;
		1347	}
5354	serge	1348
6084	serge	1349	static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
		1350	{
		1351	switch (port) {
		1352	case PORT_A:
		1353	return val & PORTA_HOTPLUG_LONG_DETECT;
		1354	case PORT_B:
		1355	return val & PORTB_HOTPLUG_LONG_DETECT;
		1356	case PORT_C:
		1357	return val & PORTC_HOTPLUG_LONG_DETECT;
		1358	default:
		1359	return false;
4560	Serge	1360	}
6084	serge	1361	}
4560	Serge	1362
6084	serge	1363	static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
		1364	{
		1365	switch (port) {
		1366	case PORT_E:
		1367	return val & PORTE_HOTPLUG_LONG_DETECT;
		1368	default:
		1369	return false;
		1370	}
4560	Serge	1371	}
		1372
6084	serge	1373	static bool spt_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1374	{
		1375	switch (port) {
		1376	case PORT_A:
6084	serge	1377	return val & PORTA_HOTPLUG_LONG_DETECT;
5060	serge	1378	case PORT_B:
6084	serge	1379	return val & PORTB_HOTPLUG_LONG_DETECT;
5060	serge	1380	case PORT_C:
6084	serge	1381	return val & PORTC_HOTPLUG_LONG_DETECT;
5060	serge	1382	case PORT_D:
6084	serge	1383	return val & PORTD_HOTPLUG_LONG_DETECT;
		1384	default:
		1385	return false;
5060	serge	1386	}
		1387	}
		1388
6084	serge	1389	static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1390	{
		1391	switch (port) {
		1392	case PORT_A:
6084	serge	1393	return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
5060	serge	1394	default:
6084	serge	1395	return false;
		1396	}
		1397	}
		1398
		1399	static bool pch_port_hotplug_long_detect(enum port port, u32 val)
		1400	{
		1401	switch (port) {
5060	serge	1402	case PORT_B:
6084	serge	1403	return val & PORTB_HOTPLUG_LONG_DETECT;
5060	serge	1404	case PORT_C:
6084	serge	1405	return val & PORTC_HOTPLUG_LONG_DETECT;
5060	serge	1406	case PORT_D:
6084	serge	1407	return val & PORTD_HOTPLUG_LONG_DETECT;
		1408	default:
		1409	return false;
5060	serge	1410	}
		1411	}
		1412
6084	serge	1413	static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
5060	serge	1414	{
6084	serge	1415	switch (port) {
		1416	case PORT_B:
		1417	return val & PORTB_HOTPLUG_INT_LONG_PULSE;
		1418	case PORT_C:
		1419	return val & PORTC_HOTPLUG_INT_LONG_PULSE;
		1420	case PORT_D:
		1421	return val & PORTD_HOTPLUG_INT_LONG_PULSE;
5060	serge	1422	default:
6084	serge	1423	return false;
5060	serge	1424	}
		1425	}
		1426
6084	serge	1427	/*
		1428	* Get a bit mask of pins that have triggered, and which ones may be long.
		1429	* This can be called multiple times with the same masks to accumulate
		1430	* hotplug detection results from several registers.
		1431	*
		1432	* Note that the caller is expected to zero out the masks initially.
		1433	*/
		1434	static void intel_get_hpd_pins(u32 pin_mask, u32 long_mask,
		1435	u32 hotplug_trigger, u32 dig_hotplug_reg,
		1436	const u32 hpd[HPD_NUM_PINS],
		1437	bool long_pulse_detect(enum port port, u32 val))
3746	Serge	1438	{
6084	serge	1439	enum port port;
3746	Serge	1440	int i;
		1441
6084	serge	1442	for_each_hpd_pin(i) {
		1443	if ((hpd[i] & hotplug_trigger) == 0)
5060	serge	1444	continue;
3746	Serge	1445
6084	serge	1446	*pin_mask \|= BIT(i);
5060	serge	1447
6131	serge	1448	// if (!intel_hpd_pin_to_port(i, &port))
		1449	continue;
5060	serge	1450
6084	serge	1451	if (long_pulse_detect(port, dig_hotplug_reg))
		1452	*long_mask \|= BIT(i);
3746	Serge	1453	}
		1454
6084	serge	1455	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
		1456	hotplug_trigger, dig_hotplug_reg, *pin_mask);
3746	Serge	1457
		1458	}
		1459
3480	Serge	1460	static void gmbus_irq_handler(struct drm_device *dev)
		1461	{
5060	serge	1462	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1463
		1464	wake_up_all(&dev_priv->gmbus_wait_queue);
		1465	}
		1466
		1467	static void dp_aux_irq_handler(struct drm_device *dev)
		1468	{
5060	serge	1469	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	1470
		1471	wake_up_all(&dev_priv->gmbus_wait_queue);
		1472	}
		1473
4560	Serge	1474	#if defined(CONFIG_DEBUG_FS)
		1475	static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
		1476	uint32_t crc0, uint32_t crc1,
		1477	uint32_t crc2, uint32_t crc3,
		1478	uint32_t crc4)
		1479	{
		1480	struct drm_i915_private *dev_priv = dev->dev_private;
		1481	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
		1482	struct intel_pipe_crc_entry *entry;
		1483	int head, tail;
		1484
		1485	spin_lock(&pipe_crc->lock);
		1486
		1487	if (!pipe_crc->entries) {
		1488	spin_unlock(&pipe_crc->lock);
5354	serge	1489	DRM_DEBUG_KMS("spurious interrupt\n");
4560	Serge	1490	return;
		1491	}
		1492
		1493	head = pipe_crc->head;
		1494	tail = pipe_crc->tail;
		1495
		1496	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
		1497	spin_unlock(&pipe_crc->lock);
		1498	DRM_ERROR("CRC buffer overflowing\n");
		1499	return;
		1500	}
		1501
		1502	entry = &pipe_crc->entries[head];
		1503
		1504	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
		1505	entry->crc[0] = crc0;
		1506	entry->crc[1] = crc1;
		1507	entry->crc[2] = crc2;
		1508	entry->crc[3] = crc3;
		1509	entry->crc[4] = crc4;
		1510
		1511	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		1512	pipe_crc->head = head;
		1513
		1514	spin_unlock(&pipe_crc->lock);
		1515
		1516	wake_up_interruptible(&pipe_crc->wq);
		1517	}
		1518	#else
		1519	static inline void
		1520	display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
		1521	uint32_t crc0, uint32_t crc1,
		1522	uint32_t crc2, uint32_t crc3,
		1523	uint32_t crc4) {}
		1524	#endif
		1525
		1526
		1527	static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1528	{
		1529	struct drm_i915_private *dev_priv = dev->dev_private;
		1530
		1531	display_pipe_crc_irq_handler(dev, pipe,
		1532	I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
		1533	0, 0, 0, 0);
		1534	}
		1535
		1536	static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1537	{
		1538	struct drm_i915_private *dev_priv = dev->dev_private;
		1539
		1540	display_pipe_crc_irq_handler(dev, pipe,
		1541	I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
		1542	I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
		1543	I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
		1544	I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
		1545	I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
		1546	}
		1547
		1548	static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
		1549	{
		1550	struct drm_i915_private *dev_priv = dev->dev_private;
		1551	uint32_t res1, res2;
		1552
		1553	if (INTEL_INFO(dev)->gen >= 3)
		1554	res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
		1555	else
		1556	res1 = 0;
		1557
		1558	if (INTEL_INFO(dev)->gen >= 5 \|\| IS_G4X(dev))
		1559	res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
		1560	else
		1561	res2 = 0;
		1562
		1563	display_pipe_crc_irq_handler(dev, pipe,
		1564	I915_READ(PIPE_CRC_RES_RED(pipe)),
		1565	I915_READ(PIPE_CRC_RES_GREEN(pipe)),
		1566	I915_READ(PIPE_CRC_RES_BLUE(pipe)),
		1567	res1, res2);
		1568	}
		1569
4104	Serge	1570	/* The RPS events need forcewake, so we add them to a work queue and mask their
		1571	* IMR bits until the work is done. Other interrupts can be processed without
		1572	* the work queue. */
		1573	static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
		1574	{
5060	serge	1575	if (pm_iir & dev_priv->pm_rps_events) {
4104	Serge	1576	spin_lock(&dev_priv->irq_lock);
5354	serge	1577	gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
		1578	if (dev_priv->rps.interrupts_enabled) {
6084	serge	1579	dev_priv->rps.pm_iir \|= pm_iir & dev_priv->pm_rps_events;
5354	serge	1580	queue_work(dev_priv->wq, &dev_priv->rps.work);
		1581	}
4104	Serge	1582	spin_unlock(&dev_priv->irq_lock);
		1583	}
		1584
5354	serge	1585	if (INTEL_INFO(dev_priv)->gen >= 8)
		1586	return;
		1587
4104	Serge	1588	if (HAS_VEBOX(dev_priv->dev)) {
		1589	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
6084	serge	1590	notify_ring(&dev_priv->ring[VECS]);
4104	Serge	1591
5354	serge	1592	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
		1593	DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
4104	Serge	1594	}
		1595	}
		1596
5354	serge	1597	static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
		1598	{
6088	serge	1599	if (!drm_handle_vblank(dev, pipe))
		1600	return false;
5354	serge	1601
		1602	return true;
		1603	}
		1604
5060	serge	1605	static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
3031	serge	1606	{
5060	serge	1607	struct drm_i915_private *dev_priv = dev->dev_private;
		1608	u32 pipe_stats[I915_MAX_PIPES] = { };
3031	serge	1609	int pipe;
		1610
5060	serge	1611	spin_lock(&dev_priv->irq_lock);
5354	serge	1612	for_each_pipe(dev_priv, pipe) {
5060	serge	1613	int reg;
		1614	u32 mask, iir_bit = 0;
3031	serge	1615
5060	serge	1616	/*
		1617	* PIPESTAT bits get signalled even when the interrupt is
		1618	* disabled with the mask bits, and some of the status bits do
		1619	* not generate interrupts at all (like the underrun bit). Hence
		1620	* we need to be careful that we only handle what we want to
		1621	* handle.
		1622	*/
3031	serge	1623
5354	serge	1624	/* fifo underruns are filterered in the underrun handler. */
		1625	mask = PIPE_FIFO_UNDERRUN_STATUS;
		1626
5060	serge	1627	switch (pipe) {
		1628	case PIPE_A:
		1629	iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
		1630	break;
		1631	case PIPE_B:
		1632	iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
		1633	break;
		1634	case PIPE_C:
		1635	iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
		1636	break;
		1637	}
		1638	if (iir & iir_bit)
		1639	mask \|= dev_priv->pipestat_irq_mask[pipe];
3031	serge	1640
5060	serge	1641	if (!mask)
		1642	continue;
3031	serge	1643
5060	serge	1644	reg = PIPESTAT(pipe);
		1645	mask \|= PIPESTAT_INT_ENABLE_MASK;
		1646	pipe_stats[pipe] = I915_READ(reg) & mask;
3031	serge	1647
6084	serge	1648	/*
		1649	* Clear the PIPE*STAT regs before the IIR
		1650	*/
5060	serge	1651	if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS \|
		1652	PIPESTAT_INT_STATUS_MASK))
6084	serge	1653	I915_WRITE(reg, pipe_stats[pipe]);
		1654	}
5060	serge	1655	spin_unlock(&dev_priv->irq_lock);
3031	serge	1656
5354	serge	1657	for_each_pipe(dev_priv, pipe) {
6084	serge	1658	if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		1659	intel_pipe_handle_vblank(dev, pipe))
		1660	/intel_check_page_flip(dev, pipe)/;
3031	serge	1661
6084	serge	1662	if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
		1663	// intel_prepare_page_flip(dev, pipe);
		1664	// intel_finish_page_flip(dev, pipe);
		1665	}
4560	Serge	1666
6084	serge	1667	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		1668	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	1669
5354	serge	1670	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		1671	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
6084	serge	1672	}
3031	serge	1673
5060	serge	1674	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		1675	gmbus_irq_handler(dev);
		1676	}
3031	serge	1677
5060	serge	1678	static void i9xx_hpd_irq_handler(struct drm_device *dev)
		1679	{
		1680	struct drm_i915_private *dev_priv = dev->dev_private;
		1681	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
6084	serge	1682	u32 pin_mask = 0, long_mask = 0;
4104	Serge	1683
6084	serge	1684	if (!hotplug_status)
		1685	return;
4104	Serge	1686
6084	serge	1687	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
		1688	/*
		1689	* Make sure hotplug status is cleared before we clear IIR, or else we
		1690	* may miss hotplug events.
		1691	*/
		1692	POSTING_READ(PORT_HOTPLUG_STAT);
		1693
		1694	if (IS_G4X(dev) \|\| IS_VALLEYVIEW(dev)) {
5060	serge	1695	u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
4560	Serge	1696
6084	serge	1697	if (hotplug_trigger) {
		1698	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1699	hotplug_trigger, hpd_status_g4x,
		1700	i9xx_port_hotplug_long_detect);
		1701
		1702	// intel_hpd_irq_handler(dev, pin_mask, long_mask);
		1703	}
		1704
		1705	if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
		1706	dp_aux_irq_handler(dev);
5060	serge	1707	} else {
		1708	u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
		1709
6084	serge	1710	if (hotplug_trigger) {
		1711	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1712	hotplug_trigger, hpd_status_i915,
		1713	i9xx_port_hotplug_long_detect);
		1714	// intel_hpd_irq_handler(dev, pin_mask, long_mask);
		1715	}
5060	serge	1716	}
		1717	}
		1718
		1719	static irqreturn_t valleyview_irq_handler(int irq, void *arg)
		1720	{
		1721	struct drm_device *dev = arg;
		1722	struct drm_i915_private *dev_priv = dev->dev_private;
		1723	u32 iir, gt_iir, pm_iir;
		1724	irqreturn_t ret = IRQ_NONE;
		1725
6084	serge	1726	if (!intel_irqs_enabled(dev_priv))
		1727	return IRQ_NONE;
		1728
5060	serge	1729	while (true) {
		1730	/* Find, clear, then process each source of interrupt */
		1731
		1732	gt_iir = I915_READ(GTIIR);
		1733	if (gt_iir)
		1734	I915_WRITE(GTIIR, gt_iir);
		1735
		1736	pm_iir = I915_READ(GEN6_PMIIR);
		1737	if (pm_iir)
		1738	I915_WRITE(GEN6_PMIIR, pm_iir);
		1739
		1740	iir = I915_READ(VLV_IIR);
		1741	if (iir) {
		1742	/* Consume port before clearing IIR or we'll miss events */
		1743	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		1744	i9xx_hpd_irq_handler(dev);
		1745	I915_WRITE(VLV_IIR, iir);
3031	serge	1746	}
		1747
5060	serge	1748	if (gt_iir == 0 && pm_iir == 0 && iir == 0)
		1749	goto out;
3031	serge	1750
5060	serge	1751	ret = IRQ_HANDLED;
		1752
		1753	if (gt_iir)
6084	serge	1754	snb_gt_irq_handler(dev, dev_priv, gt_iir);
4126	Serge	1755	if (pm_iir)
		1756	gen6_rps_irq_handler(dev_priv, pm_iir);
5060	serge	1757	/* Call regardless, as some status bits might not be
		1758	* signalled in iir */
		1759	valleyview_pipestat_irq_handler(dev, iir);
3031	serge	1760	}
		1761
		1762	out:
		1763	return ret;
		1764	}
		1765
5060	serge	1766	static irqreturn_t cherryview_irq_handler(int irq, void *arg)
		1767	{
		1768	struct drm_device *dev = arg;
		1769	struct drm_i915_private *dev_priv = dev->dev_private;
		1770	u32 master_ctl, iir;
		1771	irqreturn_t ret = IRQ_NONE;
		1772
6084	serge	1773	if (!intel_irqs_enabled(dev_priv))
		1774	return IRQ_NONE;
		1775
5060	serge	1776	for (;;) {
		1777	master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		1778	iir = I915_READ(VLV_IIR);
		1779
		1780	if (master_ctl == 0 && iir == 0)
		1781	break;
		1782
		1783	ret = IRQ_HANDLED;
		1784
		1785	I915_WRITE(GEN8_MASTER_IRQ, 0);
		1786
		1787	/* Find, clear, then process each source of interrupt */
		1788
		1789	if (iir) {
		1790	/* Consume port before clearing IIR or we'll miss events */
		1791	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		1792	i9xx_hpd_irq_handler(dev);
		1793	I915_WRITE(VLV_IIR, iir);
		1794	}
		1795
6084	serge	1796	gen8_gt_irq_handler(dev_priv, master_ctl);
5060	serge	1797
		1798	/* Call regardless, as some status bits might not be
		1799	* signalled in iir */
		1800	valleyview_pipestat_irq_handler(dev, iir);
		1801
		1802	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		1803	POSTING_READ(GEN8_MASTER_IRQ);
		1804	}
		1805
		1806	return ret;
		1807	}
		1808
6084	serge	1809	static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		1810	const u32 hpd[HPD_NUM_PINS])
		1811	{
		1812	struct drm_i915_private *dev_priv = to_i915(dev);
		1813	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		1814
		1815	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		1816	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		1817
		1818	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1819	dig_hotplug_reg, hpd,
		1820	pch_port_hotplug_long_detect);
		1821
		1822	// intel_hpd_irq_handler(dev, pin_mask, long_mask);
		1823	}
		1824
3031	serge	1825	static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
		1826	{
5060	serge	1827	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1828	int pipe;
3746	Serge	1829	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
3031	serge	1830
6084	serge	1831	if (hotplug_trigger)
		1832	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
4104	Serge	1833
		1834	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		1835	int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
		1836	SDE_AUDIO_POWER_SHIFT);
		1837	DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
		1838	port_name(port));
3746	Serge	1839	}
3031	serge	1840
3480	Serge	1841	if (pch_iir & SDE_AUX_MASK)
		1842	dp_aux_irq_handler(dev);
		1843
3031	serge	1844	if (pch_iir & SDE_GMBUS)
3480	Serge	1845	gmbus_irq_handler(dev);
3031	serge	1846
		1847	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		1848	DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
		1849
		1850	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		1851	DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
		1852
		1853	if (pch_iir & SDE_POISON)
		1854	DRM_ERROR("PCH poison interrupt\n");
		1855
		1856	if (pch_iir & SDE_FDI_MASK)
5354	serge	1857	for_each_pipe(dev_priv, pipe)
3031	serge	1858	DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
		1859	pipe_name(pipe),
		1860	I915_READ(FDI_RX_IIR(pipe)));
		1861
		1862	if (pch_iir & (SDE_TRANSB_CRC_DONE \| SDE_TRANSA_CRC_DONE))
		1863	DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
		1864
		1865	if (pch_iir & (SDE_TRANSB_CRC_ERR \| SDE_TRANSA_CRC_ERR))
		1866	DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
		1867
4104	Serge	1868	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
5354	serge	1869	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
4104	Serge	1870
3031	serge	1871	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
5354	serge	1872	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
3031	serge	1873	}
		1874
4104	Serge	1875	static void ivb_err_int_handler(struct drm_device *dev)
		1876	{
		1877	struct drm_i915_private *dev_priv = dev->dev_private;
		1878	u32 err_int = I915_READ(GEN7_ERR_INT);
4560	Serge	1879	enum pipe pipe;
4104	Serge	1880
		1881	if (err_int & ERR_INT_POISON)
		1882	DRM_ERROR("Poison interrupt\n");
		1883
5354	serge	1884	for_each_pipe(dev_priv, pipe) {
		1885	if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
		1886	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4104	Serge	1887
4560	Serge	1888	if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
		1889	if (IS_IVYBRIDGE(dev))
		1890	ivb_pipe_crc_irq_handler(dev, pipe);
		1891	else
		1892	hsw_pipe_crc_irq_handler(dev, pipe);
		1893	}
		1894	}
4104	Serge	1895
		1896	I915_WRITE(GEN7_ERR_INT, err_int);
		1897	}
		1898
		1899	static void cpt_serr_int_handler(struct drm_device *dev)
		1900	{
		1901	struct drm_i915_private *dev_priv = dev->dev_private;
		1902	u32 serr_int = I915_READ(SERR_INT);
		1903
		1904	if (serr_int & SERR_INT_POISON)
		1905	DRM_ERROR("PCH poison interrupt\n");
		1906
		1907	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
5354	serge	1908	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
4104	Serge	1909
		1910	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
5354	serge	1911	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
4104	Serge	1912
		1913	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
5354	serge	1914	intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
4104	Serge	1915
		1916	I915_WRITE(SERR_INT, serr_int);
		1917	}
		1918
3031	serge	1919	static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
		1920	{
5060	serge	1921	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	1922	int pipe;
3746	Serge	1923	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
3031	serge	1924
6084	serge	1925	if (hotplug_trigger)
		1926	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
4104	Serge	1927
		1928	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		1929	int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
		1930	SDE_AUDIO_POWER_SHIFT_CPT);
		1931	DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
		1932	port_name(port));
3746	Serge	1933	}
3031	serge	1934
		1935	if (pch_iir & SDE_AUX_MASK_CPT)
3480	Serge	1936	dp_aux_irq_handler(dev);
3031	serge	1937
		1938	if (pch_iir & SDE_GMBUS_CPT)
3480	Serge	1939	gmbus_irq_handler(dev);
3031	serge	1940
		1941	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		1942	DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
		1943
		1944	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		1945	DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
		1946
		1947	if (pch_iir & SDE_FDI_MASK_CPT)
5354	serge	1948	for_each_pipe(dev_priv, pipe)
3031	serge	1949	DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
		1950	pipe_name(pipe),
		1951	I915_READ(FDI_RX_IIR(pipe)));
		1952
4104	Serge	1953	if (pch_iir & SDE_ERROR_CPT)
		1954	cpt_serr_int_handler(dev);
4539	Serge	1955	}
3480	Serge	1956
6084	serge	1957	static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
		1958	{
		1959	struct drm_i915_private *dev_priv = dev->dev_private;
		1960	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		1961	~SDE_PORTE_HOTPLUG_SPT;
		1962	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
		1963	u32 pin_mask = 0, long_mask = 0;
		1964
		1965	if (hotplug_trigger) {
		1966	u32 dig_hotplug_reg;
		1967
		1968	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		1969	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		1970
		1971	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		1972	dig_hotplug_reg, hpd_spt,
		1973	spt_port_hotplug_long_detect);
		1974	}
		1975
		1976	if (hotplug2_trigger) {
		1977	u32 dig_hotplug_reg;
		1978
		1979	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		1980	I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
		1981
		1982	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
		1983	dig_hotplug_reg, hpd_spt,
		1984	spt_port_hotplug2_long_detect);
		1985	}
		1986
		1987	if (pch_iir & SDE_GMBUS_CPT)
		1988	gmbus_irq_handler(dev);
		1989	}
		1990
		1991	static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		1992	const u32 hpd[HPD_NUM_PINS])
		1993	{
		1994	struct drm_i915_private *dev_priv = to_i915(dev);
		1995	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		1996
		1997	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
		1998	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
		1999
		2000	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		2001	dig_hotplug_reg, hpd,
		2002	ilk_port_hotplug_long_detect);
		2003
		2004	}
		2005
4104	Serge	2006	static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
3031	serge	2007	{
4104	Serge	2008	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2009	enum pipe pipe;
6084	serge	2010	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
3031	serge	2011
6131	serge	2012	// if (hotplug_trigger)
		2013	// ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
6084	serge	2014
3480	Serge	2015	if (de_iir & DE_AUX_CHANNEL_A)
		2016	dp_aux_irq_handler(dev);
		2017
3031	serge	2018	if (de_iir & DE_GSE)
4104	Serge	2019	intel_opregion_asle_intr(dev);
2351	Serge	2020
4104	Serge	2021	if (de_iir & DE_POISON)
		2022	DRM_ERROR("Poison interrupt\n");
		2023
5354	serge	2024	for_each_pipe(dev_priv, pipe) {
6084	serge	2025	if (de_iir & DE_PIPE_VBLANK(pipe) &&
		2026	intel_pipe_handle_vblank(dev, pipe))
		2027	/intel_check_page_flip(dev, pipe)/;
4104	Serge	2028
4560	Serge	2029	if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
5354	serge	2030	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2351	Serge	2031
4560	Serge	2032	if (de_iir & DE_PIPE_CRC_DONE(pipe))
		2033	i9xx_pipe_crc_irq_handler(dev, pipe);
		2034
		2035	/* plane/pipes map 1:1 on ilk+ */
		2036	if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
		2037	// intel_prepare_page_flip(dev, pipe);
		2038	// intel_finish_page_flip_plane(dev, pipe);
		2039	}
3031	serge	2040	}
2351	Serge	2041
3031	serge	2042	/* check event from PCH */
		2043	if (de_iir & DE_PCH_EVENT) {
3480	Serge	2044	u32 pch_iir = I915_READ(SDEIIR);
		2045
3031	serge	2046	if (HAS_PCH_CPT(dev))
		2047	cpt_irq_handler(dev, pch_iir);
		2048	else
		2049	ibx_irq_handler(dev, pch_iir);
3480	Serge	2050
		2051	/* should clear PCH hotplug event before clear CPU irq */
		2052	I915_WRITE(SDEIIR, pch_iir);
3031	serge	2053	}
4104	Serge	2054
6084	serge	2055	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
4104	Serge	2056	ironlake_rps_change_irq_handler(dev);
2351	Serge	2057	}
		2058
4104	Serge	2059	static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
3031	serge	2060	{
		2061	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2062	enum pipe pipe;
6084	serge	2063	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2351	Serge	2064
6084	serge	2065	if (hotplug_trigger)
		2066	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
		2067
4126	Serge	2068	if (de_iir & DE_ERR_INT_IVB)
		2069	ivb_err_int_handler(dev);
2351	Serge	2070
4104	Serge	2071	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		2072	dp_aux_irq_handler(dev);
3031	serge	2073
4104	Serge	2074	if (de_iir & DE_GSE_IVB)
		2075	intel_opregion_asle_intr(dev);
4560	Serge	2076
5354	serge	2077	for_each_pipe(dev_priv, pipe) {
6084	serge	2078	if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		2079	intel_pipe_handle_vblank(dev, pipe))
		2080	/intel_check_page_flip(dev, pipe)/;
4560	Serge	2081
		2082	/* plane/pipes map 1:1 on ilk+ */
5060	serge	2083	if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
		2084	// intel_prepare_page_flip(dev, pipe);
		2085	// intel_finish_page_flip_plane(dev, pipe);
3031	serge	2086	}
		2087	}
		2088
4104	Serge	2089	/* check event from PCH */
		2090	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		2091	u32 pch_iir = I915_READ(SDEIIR);
3031	serge	2092
4104	Serge	2093	cpt_irq_handler(dev, pch_iir);
3031	serge	2094
4104	Serge	2095	/* clear PCH hotplug event before clear CPU irq */
		2096	I915_WRITE(SDEIIR, pch_iir);
4539	Serge	2097	}
3031	serge	2098	}
		2099
5060	serge	2100	/*
		2101	* To handle irqs with the minimum potential races with fresh interrupts, we:
		2102	* 1 - Disable Master Interrupt Control.
		2103	* 2 - Find the source(s) of the interrupt.
		2104	* 3 - Clear the Interrupt Identity bits (IIR).
		2105	* 4 - Process the interrupt(s) that had bits set in the IIRs.
		2106	* 5 - Re-enable Master Interrupt Control.
		2107	*/
4104	Serge	2108	static irqreturn_t ironlake_irq_handler(int irq, void *arg)
3031	serge	2109	{
5060	serge	2110	struct drm_device *dev = arg;
		2111	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2112	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
		2113	irqreturn_t ret = IRQ_NONE;
3031	serge	2114
6084	serge	2115	if (!intel_irqs_enabled(dev_priv))
		2116	return IRQ_NONE;
		2117
4104	Serge	2118	/* We get interrupts on unclaimed registers, so check for this before we
		2119	* do any I915_{READ,WRITE}. */
		2120	intel_uncore_check_errors(dev);
3031	serge	2121
4104	Serge	2122	/* disable master interrupt before clearing iir */
		2123	de_ier = I915_READ(DEIER);
		2124	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
		2125	POSTING_READ(DEIER);
3031	serge	2126
4104	Serge	2127	/* Disable south interrupts. We'll only write to SDEIIR once, so further
		2128	* interrupts will will be stored on its back queue, and then we'll be
		2129	* able to process them after we restore SDEIER (as soon as we restore
		2130	* it, we'll get an interrupt if SDEIIR still has something to process
		2131	* due to its back queue). */
		2132	if (!HAS_PCH_NOP(dev)) {
		2133	sde_ier = I915_READ(SDEIER);
		2134	I915_WRITE(SDEIER, 0);
		2135	POSTING_READ(SDEIER);
3031	serge	2136	}
		2137
5060	serge	2138	/* Find, clear, then process each source of interrupt */
		2139
4104	Serge	2140	gt_iir = I915_READ(GTIIR);
		2141	if (gt_iir) {
5060	serge	2142	I915_WRITE(GTIIR, gt_iir);
		2143	ret = IRQ_HANDLED;
4104	Serge	2144	if (INTEL_INFO(dev)->gen >= 6)
		2145	snb_gt_irq_handler(dev, dev_priv, gt_iir);
		2146	else
		2147	ilk_gt_irq_handler(dev, dev_priv, gt_iir);
4539	Serge	2148	}
3031	serge	2149
4104	Serge	2150	de_iir = I915_READ(DEIIR);
		2151	if (de_iir) {
5060	serge	2152	I915_WRITE(DEIIR, de_iir);
		2153	ret = IRQ_HANDLED;
4104	Serge	2154	if (INTEL_INFO(dev)->gen >= 7)
		2155	ivb_display_irq_handler(dev, de_iir);
		2156	else
		2157	ilk_display_irq_handler(dev, de_iir);
3480	Serge	2158	}
		2159
4104	Serge	2160	if (INTEL_INFO(dev)->gen >= 6) {
		2161	u32 pm_iir = I915_READ(GEN6_PMIIR);
		2162	if (pm_iir) {
		2163	I915_WRITE(GEN6_PMIIR, pm_iir);
		2164	ret = IRQ_HANDLED;
5060	serge	2165	gen6_rps_irq_handler(dev_priv, pm_iir);
4560	Serge	2166	}
3031	serge	2167	}
		2168
4104	Serge	2169	I915_WRITE(DEIER, de_ier);
		2170	POSTING_READ(DEIER);
		2171	if (!HAS_PCH_NOP(dev)) {
		2172	I915_WRITE(SDEIER, sde_ier);
		2173	POSTING_READ(SDEIER);
3031	serge	2174	}
		2175
4104	Serge	2176	return ret;
3031	serge	2177	}
		2178
6084	serge	2179	static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
		2180	const u32 hpd[HPD_NUM_PINS])
		2181	{
		2182	struct drm_i915_private *dev_priv = to_i915(dev);
		2183	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
		2184
		2185	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		2186	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
		2187
		2188	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
		2189	dig_hotplug_reg, hpd,
		2190	bxt_port_hotplug_long_detect);
		2191
		2192	}
		2193
4560	Serge	2194	static irqreturn_t gen8_irq_handler(int irq, void *arg)
		2195	{
		2196	struct drm_device *dev = arg;
		2197	struct drm_i915_private *dev_priv = dev->dev_private;
		2198	u32 master_ctl;
		2199	irqreturn_t ret = IRQ_NONE;
		2200	uint32_t tmp = 0;
		2201	enum pipe pipe;
5354	serge	2202	u32 aux_mask = GEN8_AUX_CHANNEL_A;
4560	Serge	2203
6084	serge	2204	if (!intel_irqs_enabled(dev_priv))
		2205	return IRQ_NONE;
		2206
		2207	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2208	aux_mask \|= GEN9_AUX_CHANNEL_B \| GEN9_AUX_CHANNEL_C \|
		2209	GEN9_AUX_CHANNEL_D;
		2210
6084	serge	2211	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
4560	Serge	2212	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
		2213	if (!master_ctl)
		2214	return IRQ_NONE;
		2215
6084	serge	2216	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
4560	Serge	2217
5060	serge	2218	/* Find, clear, then process each source of interrupt */
		2219
6084	serge	2220	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
4560	Serge	2221
		2222	if (master_ctl & GEN8_DE_MISC_IRQ) {
		2223	tmp = I915_READ(GEN8_DE_MISC_IIR);
5060	serge	2224	if (tmp) {
		2225	I915_WRITE(GEN8_DE_MISC_IIR, tmp);
		2226	ret = IRQ_HANDLED;
6084	serge	2227	if (tmp & GEN8_DE_MISC_GSE)
		2228	intel_opregion_asle_intr(dev);
5060	serge	2229	else
6084	serge	2230	DRM_ERROR("Unexpected DE Misc interrupt\n");
5060	serge	2231	}
4560	Serge	2232	else
		2233	DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
		2234	}
		2235
		2236	if (master_ctl & GEN8_DE_PORT_IRQ) {
		2237	tmp = I915_READ(GEN8_DE_PORT_IIR);
5060	serge	2238	if (tmp) {
6084	serge	2239	bool found = false;
		2240	u32 hotplug_trigger = 0;
		2241
		2242	if (IS_BROXTON(dev_priv))
		2243	hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
		2244	else if (IS_BROADWELL(dev_priv))
		2245	hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
		2246
5060	serge	2247	I915_WRITE(GEN8_DE_PORT_IIR, tmp);
		2248	ret = IRQ_HANDLED;
5354	serge	2249
6084	serge	2250	if (tmp & aux_mask) {
		2251	dp_aux_irq_handler(dev);
		2252	found = true;
		2253	}
		2254
		2255	if (hotplug_trigger) {
		2256	if (IS_BROXTON(dev))
		2257	bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
		2258	else
		2259	ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
		2260	found = true;
		2261	}
		2262
		2263	if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
		2264	gmbus_irq_handler(dev);
		2265	found = true;
		2266	}
		2267
		2268	if (!found)
		2269	DRM_ERROR("Unexpected DE Port interrupt\n");
5060	serge	2270	}
4560	Serge	2271	else
		2272	DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
		2273	}
		2274
5354	serge	2275	for_each_pipe(dev_priv, pipe) {
		2276	uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
4560	Serge	2277
		2278	if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
		2279	continue;
		2280
		2281	pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
5060	serge	2282	if (pipe_iir) {
		2283	ret = IRQ_HANDLED;
		2284	I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
4560	Serge	2285
6088	serge	2286	if (pipe_iir & GEN8_PIPE_VBLANK &&
		2287	intel_pipe_handle_vblank(dev, pipe))
		2288	/* intel_check_page_flip(dev, pipe)*/;
4560	Serge	2289
6084	serge	2290	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2291	flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
		2292	else
		2293	flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
		2294
		2295
6084	serge	2296	if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
		2297	hsw_pipe_crc_irq_handler(dev, pipe);
4560	Serge	2298
5354	serge	2299	if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
		2300	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		2301	pipe);
4560	Serge	2302
5354	serge	2303
6084	serge	2304	if (INTEL_INFO(dev_priv)->gen >= 9)
5354	serge	2305	fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		2306	else
		2307	fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
		2308
		2309	if (fault_errors)
6084	serge	2310	DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
		2311	pipe_name(pipe),
		2312	pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
4560	Serge	2313	} else
		2314	DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
		2315	}
		2316
6084	serge	2317	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
		2318	master_ctl & GEN8_DE_PCH_IRQ) {
4560	Serge	2319	/*
		2320	* FIXME(BDW): Assume for now that the new interrupt handling
		2321	* scheme also closed the SDE interrupt handling race we've seen
		2322	* on older pch-split platforms. But this needs testing.
		2323	*/
		2324	u32 pch_iir = I915_READ(SDEIIR);
		2325	if (pch_iir) {
		2326	I915_WRITE(SDEIIR, pch_iir);
		2327	ret = IRQ_HANDLED;
6084	serge	2328
		2329	if (HAS_PCH_SPT(dev_priv))
		2330	spt_irq_handler(dev, pch_iir);
		2331	else
		2332	cpt_irq_handler(dev, pch_iir);
5060	serge	2333	} else
		2334	DRM_ERROR("The master control interrupt lied (SDE)!\n");
		2335
4560	Serge	2336	}
		2337
6084	serge	2338	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
		2339	POSTING_READ_FW(GEN8_MASTER_IRQ);
4560	Serge	2340
		2341	return ret;
		2342	}
		2343
4104	Serge	2344	static void i915_error_wake_up(struct drm_i915_private *dev_priv,
		2345	bool reset_completed)
3746	Serge	2346	{
5060	serge	2347	struct intel_engine_cs *ring;
4104	Serge	2348	int i;
3031	serge	2349
4104	Serge	2350	/*
		2351	* Notify all waiters for GPU completion events that reset state has
		2352	* been changed, and that they need to restart their wait after
		2353	* checking for potential errors (and bail out to drop locks if there is
		2354	* a gpu reset pending so that i915_error_work_func can acquire them).
		2355	*/
3031	serge	2356
4104	Serge	2357	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
		2358	for_each_ring(ring, dev_priv, i)
		2359	wake_up_all(&ring->irq_queue);
3031	serge	2360
		2361
4104	Serge	2362	/*
		2363	* Signal tasks blocked in i915_gem_wait_for_error that the pending
		2364	* reset state is cleared.
		2365	*/
		2366	if (reset_completed)
		2367	wake_up_all(&dev_priv->gpu_error.reset_queue);
3031	serge	2368	}
		2369
		2370	/**
6084	serge	2371	* i915_reset_and_wakeup - do process context error handling work
		2372	* @dev: drm device
3031	serge	2373	*
4104	Serge	2374	* Fire an error uevent so userspace can see that a hang or error
		2375	* was detected.
3031	serge	2376	*/
6084	serge	2377	static void i915_reset_and_wakeup(struct drm_device *dev)
3031	serge	2378	{
6084	serge	2379	struct drm_i915_private *dev_priv = to_i915(dev);
		2380	struct i915_gpu_error *error = &dev_priv->gpu_error;
4104	Serge	2381	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
		2382	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
		2383	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
		2384	int ret;
3031	serge	2385
4104	Serge	2386	/*
		2387	* Note that there's only one work item which does gpu resets, so we
		2388	* need not worry about concurrent gpu resets potentially incrementing
		2389	* error->reset_counter twice. We only need to take care of another
		2390	* racing irq/hangcheck declaring the gpu dead for a second time. A
		2391	* quick check for that is good enough: schedule_work ensures the
		2392	* correct ordering between hang detection and this work item, and since
		2393	* the reset in-progress bit is only ever set by code outside of this
		2394	* work we don't need to worry about any other races.
		2395	*/
		2396	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
		2397	DRM_DEBUG_DRIVER("resetting chip\n");
6084	serge	2398	intel_runtime_pm_get(dev_priv);
3031	serge	2399
4104	Serge	2400	/*
		2401	* All state reset _must_ be completed before we update the
		2402	* reset counter, for otherwise waiters might miss the reset
		2403	* pending state and not properly drop locks, resulting in
		2404	* deadlocks with the reset work.
		2405	*/
4560	Serge	2406	// ret = i915_reset(dev);
3031	serge	2407
6084	serge	2408	// intel_finish_reset(dev);
3031	serge	2409
6084	serge	2410	intel_runtime_pm_put(dev_priv);
		2411
4104	Serge	2412	if (ret == 0) {
		2413	/*
		2414	* After all the gem state is reset, increment the reset
		2415	* counter and wake up everyone waiting for the reset to
		2416	* complete.
		2417	*
		2418	* Since unlock operations are a one-sided barrier only,
		2419	* we need to insert a barrier here to order any seqno
		2420	* updates before
		2421	* the counter increment.
		2422	*/
6084	serge	2423	smp_mb__before_atomic();
4104	Serge	2424	atomic_inc(&dev_priv->gpu_error.reset_counter);
3031	serge	2425
4104	Serge	2426	} else {
6088	serge	2427	atomic_or(I915_WEDGED, &error->reset_counter);
		2428	}
3031	serge	2429
4104	Serge	2430	/*
		2431	* Note: The wake_up also serves as a memory barrier so that
		2432	* waiters see the update value of the reset counter atomic_t.
		2433	*/
		2434	i915_error_wake_up(dev_priv, true);
3031	serge	2435	}
		2436	}
		2437
		2438	static void i915_report_and_clear_eir(struct drm_device *dev)
		2439	{
		2440	struct drm_i915_private *dev_priv = dev->dev_private;
		2441	uint32_t instdone[I915_NUM_INSTDONE_REG];
		2442	u32 eir = I915_READ(EIR);
		2443	int pipe, i;
		2444
		2445	if (!eir)
		2446	return;
		2447
		2448	pr_err("render error detected, EIR: 0x%08x\n", eir);
		2449
		2450	i915_get_extra_instdone(dev, instdone);
		2451
		2452	if (IS_G4X(dev)) {
		2453	if (eir & (GM45_ERROR_MEM_PRIV \| GM45_ERROR_CP_PRIV)) {
		2454	u32 ipeir = I915_READ(IPEIR_I965);
		2455
		2456	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
		2457	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
		2458	for (i = 0; i < ARRAY_SIZE(instdone); i++)
		2459	pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
		2460	pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
		2461	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
		2462	I915_WRITE(IPEIR_I965, ipeir);
		2463	POSTING_READ(IPEIR_I965);
		2464	}
		2465	if (eir & GM45_ERROR_PAGE_TABLE) {
		2466	u32 pgtbl_err = I915_READ(PGTBL_ER);
		2467	pr_err("page table error\n");
		2468	pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
		2469	I915_WRITE(PGTBL_ER, pgtbl_err);
		2470	POSTING_READ(PGTBL_ER);
		2471	}
		2472	}
		2473
		2474	if (!IS_GEN2(dev)) {
		2475	if (eir & I915_ERROR_PAGE_TABLE) {
		2476	u32 pgtbl_err = I915_READ(PGTBL_ER);
		2477	pr_err("page table error\n");
		2478	pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
		2479	I915_WRITE(PGTBL_ER, pgtbl_err);
		2480	POSTING_READ(PGTBL_ER);
		2481	}
		2482	}
		2483
		2484	if (eir & I915_ERROR_MEMORY_REFRESH) {
		2485	pr_err("memory refresh error:\n");
5354	serge	2486	for_each_pipe(dev_priv, pipe)
3031	serge	2487	pr_err("pipe %c stat: 0x%08x\n",
		2488	pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
		2489	/* pipestat has already been acked */
		2490	}
		2491	if (eir & I915_ERROR_INSTRUCTION) {
		2492	pr_err("instruction error\n");
		2493	pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
		2494	for (i = 0; i < ARRAY_SIZE(instdone); i++)
		2495	pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
		2496	if (INTEL_INFO(dev)->gen < 4) {
		2497	u32 ipeir = I915_READ(IPEIR);
		2498
		2499	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
		2500	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
		2501	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
		2502	I915_WRITE(IPEIR, ipeir);
		2503	POSTING_READ(IPEIR);
		2504	} else {
		2505	u32 ipeir = I915_READ(IPEIR_I965);
		2506
		2507	pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
		2508	pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
		2509	pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
		2510	pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
		2511	I915_WRITE(IPEIR_I965, ipeir);
		2512	POSTING_READ(IPEIR_I965);
		2513	}
		2514	}
		2515
		2516	I915_WRITE(EIR, eir);
		2517	POSTING_READ(EIR);
		2518	eir = I915_READ(EIR);
		2519	if (eir) {
		2520	/*
		2521	* some errors might have become stuck,
		2522	* mask them.
		2523	*/
		2524	DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		2525	I915_WRITE(EMR, I915_READ(EMR) \| eir);
		2526	I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
		2527	}
		2528	}
		2529
		2530	/**
6084	serge	2531	* i915_handle_error - handle a gpu error
3031	serge	2532	* @dev: drm device
		2533	*
6084	serge	2534	* Do some basic checking of register state at error time and
3031	serge	2535	* dump it to the syslog. Also call i915_capture_error_state() to make
		2536	* sure we get a record and make it available in debugfs. Fire a uevent
		2537	* so userspace knows something bad happened (should trigger collection
		2538	* of a ring dump etc.).
		2539	*/
5060	serge	2540	void i915_handle_error(struct drm_device *dev, bool wedged,
		2541	const char *fmt, ...)
3031	serge	2542	{
		2543	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2544	va_list args;
		2545	char error_msg[80];
3031	serge	2546
5060	serge	2547	va_start(args, fmt);
		2548	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
		2549	va_end(args);
		2550
4560	Serge	2551	// i915_capture_error_state(dev);
3031	serge	2552	i915_report_and_clear_eir(dev);
		2553
		2554	if (wedged) {
6084	serge	2555	atomic_or(I915_RESET_IN_PROGRESS_FLAG,
3480	Serge	2556	&dev_priv->gpu_error.reset_counter);
3031	serge	2557
		2558	/*
6084	serge	2559	* Wakeup waiting processes so that the reset function
		2560	* i915_reset_and_wakeup doesn't deadlock trying to grab
		2561	* various locks. By bumping the reset counter first, the woken
4104	Serge	2562	* processes will see a reset in progress and back off,
		2563	* releasing their locks and then wait for the reset completion.
		2564	* We must do this for _all_ gpu waiters that might hold locks
		2565	* that the reset work needs to acquire.
		2566	*
		2567	* Note: The wake_up serves as the required memory barrier to
		2568	* ensure that the waiters see the updated value of the reset
		2569	* counter atomic_t.
3031	serge	2570	*/
4104	Serge	2571	i915_error_wake_up(dev_priv, false);
3031	serge	2572	}
		2573
6084	serge	2574	i915_reset_and_wakeup(dev);
3031	serge	2575	}
		2576
		2577	/* Called from drm generic code, passed 'crtc' which
		2578	* we use as a pipe index
		2579	*/
6084	serge	2580	static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2581	{
5060	serge	2582	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2583	unsigned long irqflags;
		2584
		2585	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2586	if (INTEL_INFO(dev)->gen >= 4)
		2587	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2588	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2589	else
		2590	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2591	PIPE_VBLANK_INTERRUPT_STATUS);
3031	serge	2592	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2593
		2594	return 0;
		2595	}
		2596
6084	serge	2597	static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2598	{
5060	serge	2599	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2600	unsigned long irqflags;
4104	Serge	2601	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
4560	Serge	2602	DE_PIPE_VBLANK(pipe);
3031	serge	2603
		2604	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4104	Serge	2605	ironlake_enable_display_irq(dev_priv, bit);
3031	serge	2606	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2607
		2608	return 0;
		2609	}
		2610
6084	serge	2611	static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2612	{
5060	serge	2613	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2614	unsigned long irqflags;
		2615
		2616	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2617	i915_enable_pipestat(dev_priv, pipe,
5060	serge	2618	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2619	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2620
		2621	return 0;
		2622	}
		2623
6084	serge	2624	static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
4560	Serge	2625	{
		2626	struct drm_i915_private *dev_priv = dev->dev_private;
		2627	unsigned long irqflags;
		2628
		2629	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2630	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
		2631	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		2632	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
		2633	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2634	return 0;
		2635	}
		2636
3031	serge	2637	/* Called from drm generic code, passed 'crtc' which
		2638	* we use as a pipe index
		2639	*/
6084	serge	2640	static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2641	{
5060	serge	2642	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2643	unsigned long irqflags;
		2644
		2645	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2646	i915_disable_pipestat(dev_priv, pipe,
5060	serge	2647	PIPE_VBLANK_INTERRUPT_STATUS \|
		2648	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2649	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2650	}
		2651
6084	serge	2652	static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2653	{
5060	serge	2654	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2655	unsigned long irqflags;
4104	Serge	2656	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
4560	Serge	2657	DE_PIPE_VBLANK(pipe);
3031	serge	2658
		2659	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4104	Serge	2660	ironlake_disable_display_irq(dev_priv, bit);
3031	serge	2661	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2662	}
		2663
6084	serge	2664	static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
3031	serge	2665	{
5060	serge	2666	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	2667	unsigned long irqflags;
		2668
		2669	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2670	i915_disable_pipestat(dev_priv, pipe,
5060	serge	2671	PIPE_START_VBLANK_INTERRUPT_STATUS);
3031	serge	2672	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2673	}
		2674
6084	serge	2675	static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
4560	Serge	2676	{
		2677	struct drm_i915_private *dev_priv = dev->dev_private;
		2678	unsigned long irqflags;
		2679
		2680	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		2681	dev_priv->de_irq_mask[pipe] \|= GEN8_PIPE_VBLANK;
		2682	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		2683	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
		2684	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		2685	}
		2686
4104	Serge	2687	static bool
5060	serge	2688	ring_idle(struct intel_engine_cs *ring, u32 seqno)
2351	Serge	2689	{
4104	Serge	2690	return (list_empty(&ring->request_list) \|\|
6084	serge	2691	i915_seqno_passed(seqno, ring->last_submitted_seqno));
4104	Serge	2692	}
2351	Serge	2693
5060	serge	2694	static bool
		2695	ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
4104	Serge	2696	{
5060	serge	2697	if (INTEL_INFO(dev)->gen >= 8) {
		2698	return (ipehr >> 23) == 0x1c;
		2699	} else {
		2700	ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		2701	return ipehr == (MI_SEMAPHORE_MBOX \| MI_SEMAPHORE_COMPARE \|
		2702	MI_SEMAPHORE_REGISTER);
		2703	}
		2704	}
		2705
		2706	static struct intel_engine_cs *
		2707	semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
		2708	{
4104	Serge	2709	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	2710	struct intel_engine_cs *signaller;
		2711	int i;
2351	Serge	2712
5060	serge	2713	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
		2714	for_each_ring(signaller, dev_priv, i) {
		2715	if (ring == signaller)
		2716	continue;
		2717
		2718	if (offset == signaller->semaphore.signal_ggtt[ring->id])
		2719	return signaller;
		2720	}
		2721	} else {
		2722	u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
		2723
		2724	for_each_ring(signaller, dev_priv, i) {
		2725	if(ring == signaller)
		2726	continue;
		2727
		2728	if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
		2729	return signaller;
		2730	}
		2731	}
		2732
		2733	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
		2734	ring->id, ipehr, offset);
		2735
		2736	return NULL;
		2737	}
		2738
		2739	static struct intel_engine_cs *
		2740	semaphore_waits_for(struct intel_engine_cs ring, u32 seqno)
		2741	{
		2742	struct drm_i915_private *dev_priv = ring->dev->dev_private;
		2743	u32 cmd, ipehr, head;
		2744	u64 offset = 0;
		2745	int i, backwards;
		2746
6084	serge	2747	/*
		2748	* This function does not support execlist mode - any attempt to
		2749	* proceed further into this function will result in a kernel panic
		2750	* when dereferencing ring->buffer, which is not set up in execlist
		2751	* mode.
		2752	*
		2753	* The correct way of doing it would be to derive the currently
		2754	* executing ring buffer from the current context, which is derived
		2755	* from the currently running request. Unfortunately, to get the
		2756	* current request we would have to grab the struct_mutex before doing
		2757	* anything else, which would be ill-advised since some other thread
		2758	* might have grabbed it already and managed to hang itself, causing
		2759	* the hang checker to deadlock.
		2760	*
		2761	* Therefore, this function does not support execlist mode in its
		2762	* current form. Just return NULL and move on.
		2763	*/
		2764	if (ring->buffer == NULL)
		2765	return NULL;
		2766
4104	Serge	2767	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
5060	serge	2768	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
4104	Serge	2769	return NULL;
2351	Serge	2770
5060	serge	2771	/*
		2772	* HEAD is likely pointing to the dword after the actual command,
		2773	* so scan backwards until we find the MBOX. But limit it to just 3
		2774	* or 4 dwords depending on the semaphore wait command size.
		2775	* Note that we don't care about ACTHD here since that might
		2776	* point at at batch, and semaphores are always emitted into the
		2777	* ringbuffer itself.
4104	Serge	2778	*/
5060	serge	2779	head = I915_READ_HEAD(ring) & HEAD_ADDR;
		2780	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
		2781
		2782	for (i = backwards; i; --i) {
		2783	/*
		2784	* Be paranoid and presume the hw has gone off into the wild -
		2785	* our ring is smaller than what the hardware (and hence
		2786	* HEAD_ADDR) allows. Also handles wrap-around.
		2787	*/
		2788	head &= ring->buffer->size - 1;
		2789
		2790	/* This here seems to blow up */
		2791	cmd = ioread32(ring->buffer->virtual_start + head);
4104	Serge	2792	if (cmd == ipehr)
		2793	break;
2351	Serge	2794
5060	serge	2795	head -= 4;
		2796	}
		2797
		2798	if (!i)
6084	serge	2799	return NULL;
2351	Serge	2800
5060	serge	2801	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
		2802	if (INTEL_INFO(ring->dev)->gen >= 8) {
		2803	offset = ioread32(ring->buffer->virtual_start + head + 12);
		2804	offset <<= 32;
		2805	offset = ioread32(ring->buffer->virtual_start + head + 8);
		2806	}
		2807	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
4104	Serge	2808	}
2351	Serge	2809
5060	serge	2810	static int semaphore_passed(struct intel_engine_cs *ring)
4104	Serge	2811	{
		2812	struct drm_i915_private *dev_priv = ring->dev->dev_private;
5060	serge	2813	struct intel_engine_cs *signaller;
		2814	u32 seqno;
4104	Serge	2815
5060	serge	2816	ring->hangcheck.deadlock++;
4104	Serge	2817
		2818	signaller = semaphore_waits_for(ring, &seqno);
5060	serge	2819	if (signaller == NULL)
4104	Serge	2820	return -1;
		2821
5060	serge	2822	/* Prevent pathological recursion due to driver bugs */
		2823	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
		2824	return -1;
		2825
		2826	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
		2827	return 1;
		2828
4104	Serge	2829	/* cursory check for an unkickable deadlock */
5060	serge	2830	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
		2831	semaphore_passed(signaller) < 0)
4104	Serge	2832	return -1;
		2833
5060	serge	2834	return 0;
4104	Serge	2835	}
		2836
		2837	static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
		2838	{
5060	serge	2839	struct intel_engine_cs *ring;
4104	Serge	2840	int i;
		2841
		2842	for_each_ring(ring, dev_priv, i)
5060	serge	2843	ring->hangcheck.deadlock = 0;
4104	Serge	2844	}
		2845
		2846	static enum intel_ring_hangcheck_action
5060	serge	2847	ring_stuck(struct intel_engine_cs *ring, u64 acthd)
4104	Serge	2848	{
		2849	struct drm_device *dev = ring->dev;
		2850	struct drm_i915_private *dev_priv = dev->dev_private;
		2851	u32 tmp;
		2852
5060	serge	2853	if (acthd != ring->hangcheck.acthd) {
		2854	if (acthd > ring->hangcheck.max_acthd) {
		2855	ring->hangcheck.max_acthd = acthd;
6084	serge	2856	return HANGCHECK_ACTIVE;
5060	serge	2857	}
4104	Serge	2858
5060	serge	2859	return HANGCHECK_ACTIVE_LOOP;
		2860	}
		2861
4104	Serge	2862	if (IS_GEN2(dev))
		2863	return HANGCHECK_HUNG;
		2864
		2865	/* Is the chip hanging on a WAIT_FOR_EVENT?
		2866	* If so we can simply poke the RB_WAIT bit
		2867	* and break the hang. This should work on
		2868	* all but the second generation chipsets.
		2869	*/
		2870	tmp = I915_READ_CTL(ring);
		2871	if (tmp & RING_WAIT) {
5060	serge	2872	i915_handle_error(dev, false,
		2873	"Kicking stuck wait on %s",
6084	serge	2874	ring->name);
4104	Serge	2875	I915_WRITE_CTL(ring, tmp);
		2876	return HANGCHECK_KICK;
		2877	}
		2878
		2879	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		2880	switch (semaphore_passed(ring)) {
		2881	default:
		2882	return HANGCHECK_HUNG;
		2883	case 1:
5060	serge	2884	i915_handle_error(dev, false,
		2885	"Kicking stuck semaphore on %s",
6084	serge	2886	ring->name);
4104	Serge	2887	I915_WRITE_CTL(ring, tmp);
		2888	return HANGCHECK_KICK;
		2889	case 0:
		2890	return HANGCHECK_WAIT;
		2891	}
		2892	}
		2893
		2894	return HANGCHECK_HUNG;
		2895	}
		2896
6084	serge	2897	/*
4104	Serge	2898	* This is called when the chip hasn't reported back with completed
		2899	* batchbuffers in a long time. We keep track per ring seqno progress and
		2900	* if there are no progress, hangcheck score for that ring is increased.
		2901	* Further, acthd is inspected to see if the ring is stuck. On stuck case
		2902	* we kick the ring. If we see no progress on three subsequent calls
		2903	* we assume chip is wedged and try to fix it by resetting the chip.
		2904	*/
6084	serge	2905	static void i915_hangcheck_elapsed(struct work_struct *work)
4104	Serge	2906	{
6084	serge	2907	struct drm_i915_private *dev_priv =
		2908	container_of(work, typeof(*dev_priv),
		2909	gpu_error.hangcheck_work.work);
		2910	struct drm_device *dev = dev_priv->dev;
5060	serge	2911	struct intel_engine_cs *ring;
4104	Serge	2912	int i;
		2913	int busy_count = 0, rings_hung = 0;
		2914	bool stuck[I915_NUM_RINGS] = { 0 };
		2915	#define BUSY 1
		2916	#define KICK 5
		2917	#define HUNG 20
		2918
5060	serge	2919	if (!i915.enable_hangcheck)
4104	Serge	2920	return;
		2921
		2922	for_each_ring(ring, dev_priv, i) {
5060	serge	2923	u64 acthd;
		2924	u32 seqno;
4104	Serge	2925	bool busy = true;
		2926
		2927	semaphore_clear_deadlocks(dev_priv);
		2928
		2929	seqno = ring->get_seqno(ring, false);
		2930	acthd = intel_ring_get_active_head(ring);
		2931
		2932	if (ring->hangcheck.seqno == seqno) {
		2933	if (ring_idle(ring, seqno)) {
5060	serge	2934	ring->hangcheck.action = HANGCHECK_IDLE;
		2935
6084	serge	2936	if (waitqueue_active(&ring->irq_queue)) {
4104	Serge	2937	/* Issue a wake-up to catch stuck h/w. */
6084	serge	2938	if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
		2939	if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
		2940	DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
		2941	ring->name);
		2942	else
		2943	DRM_INFO("Fake missed irq on %s\n",
		2944	ring->name);
		2945	wake_up_all(&ring->irq_queue);
		2946	}
		2947	/* Safeguard against driver failure */
		2948	ring->hangcheck.score += BUSY;
		2949	} else
4104	Serge	2950	busy = false;
		2951	} else {
		2952	/* We always increment the hangcheck score
		2953	* if the ring is busy and still processing
		2954	* the same request, so that no single request
		2955	* can run indefinitely (such as a chain of
		2956	* batches). The only time we do not increment
		2957	* the hangcheck score on this ring, if this
		2958	* ring is in a legitimate wait for another
		2959	* ring. In that case the waiting ring is a
		2960	* victim and we want to be sure we catch the
		2961	* right culprit. Then every time we do kick
		2962	* the ring, add a small increment to the
		2963	* score so that we can catch a batch that is
		2964	* being repeatedly kicked and so responsible
		2965	* for stalling the machine.
		2966	*/
		2967	ring->hangcheck.action = ring_stuck(ring,
		2968	acthd);
		2969
		2970	switch (ring->hangcheck.action) {
4560	Serge	2971	case HANGCHECK_IDLE:
4104	Serge	2972	case HANGCHECK_WAIT:
5060	serge	2973	case HANGCHECK_ACTIVE:
4104	Serge	2974	break;
5060	serge	2975	case HANGCHECK_ACTIVE_LOOP:
4104	Serge	2976	ring->hangcheck.score += BUSY;
		2977	break;
		2978	case HANGCHECK_KICK:
		2979	ring->hangcheck.score += KICK;
		2980	break;
		2981	case HANGCHECK_HUNG:
		2982	ring->hangcheck.score += HUNG;
		2983	stuck[i] = true;
		2984	break;
		2985	}
		2986	}
		2987	} else {
4560	Serge	2988	ring->hangcheck.action = HANGCHECK_ACTIVE;
		2989
4104	Serge	2990	/* Gradually reduce the count so that we catch DoS
		2991	* attempts across multiple batches.
		2992	*/
		2993	if (ring->hangcheck.score > 0)
		2994	ring->hangcheck.score--;
5060	serge	2995
		2996	ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
4104	Serge	2997	}
		2998
		2999	ring->hangcheck.seqno = seqno;
		3000	ring->hangcheck.acthd = acthd;
		3001	busy_count += busy;
		3002	}
		3003
		3004	for_each_ring(ring, dev_priv, i) {
5060	serge	3005	if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
4104	Serge	3006	DRM_INFO("%s on %s\n",
6084	serge	3007	stuck[i] ? "stuck" : "no progress",
		3008	ring->name);
4104	Serge	3009	rings_hung++;
		3010	}
		3011	}
		3012
		3013	// if (rings_hung)
		3014	// return i915_handle_error(dev, true);
		3015
		3016	}
6088	serge	3017
5060	serge	3018	static void ibx_irq_reset(struct drm_device *dev)
		3019	{
		3020	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3021
5060	serge	3022	if (HAS_PCH_NOP(dev))
		3023	return;
		3024
		3025	GEN5_IRQ_RESET(SDE);
		3026
		3027	if (HAS_PCH_CPT(dev) \|\| HAS_PCH_LPT(dev))
		3028	I915_WRITE(SERR_INT, 0xffffffff);
		3029	}
		3030
		3031	/*
		3032	* SDEIER is also touched by the interrupt handler to work around missed PCH
		3033	* interrupts. Hence we can't update it after the interrupt handler is enabled -
		3034	* instead we unconditionally enable all PCH interrupt sources here, but then
		3035	* only unmask them as needed with SDEIMR.
		3036	*
		3037	* This function needs to be called before interrupts are enabled.
		3038	*/
		3039	static void ibx_irq_pre_postinstall(struct drm_device *dev)
4104	Serge	3040	{
		3041	struct drm_i915_private *dev_priv = dev->dev_private;
		3042
3746	Serge	3043	if (HAS_PCH_NOP(dev))
		3044	return;
		3045
5060	serge	3046	WARN_ON(I915_READ(SDEIER) != 0);
3746	Serge	3047	I915_WRITE(SDEIER, 0xffffffff);
4104	Serge	3048	POSTING_READ(SDEIER);
2351	Serge	3049	}
		3050
5060	serge	3051	static void gen5_gt_irq_reset(struct drm_device *dev)
4104	Serge	3052	{
		3053	struct drm_i915_private *dev_priv = dev->dev_private;
		3054
5060	serge	3055	GEN5_IRQ_RESET(GT);
		3056	if (INTEL_INFO(dev)->gen >= 6)
		3057	GEN5_IRQ_RESET(GEN6_PM);
4104	Serge	3058	}
		3059
		3060	/* drm_dma.h hooks
		3061	*/
5060	serge	3062	static void ironlake_irq_reset(struct drm_device *dev)
4104	Serge	3063	{
5060	serge	3064	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3065
5060	serge	3066	I915_WRITE(HWSTAM, 0xffffffff);
4104	Serge	3067
5060	serge	3068	GEN5_IRQ_RESET(DE);
		3069	if (IS_GEN7(dev))
		3070	I915_WRITE(GEN7_ERR_INT, 0xffffffff);
4104	Serge	3071
5060	serge	3072	gen5_gt_irq_reset(dev);
4104	Serge	3073
5060	serge	3074	ibx_irq_reset(dev);
4104	Serge	3075	}
		3076
5354	serge	3077	static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
		3078	{
		3079	enum pipe pipe;
		3080
6084	serge	3081	i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
5354	serge	3082	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		3083
		3084	for_each_pipe(dev_priv, pipe)
		3085	I915_WRITE(PIPESTAT(pipe), 0xffff);
		3086
		3087	GEN5_IRQ_RESET(VLV_);
		3088	}
		3089
3031	serge	3090	static void valleyview_irq_preinstall(struct drm_device *dev)
		3091	{
5060	serge	3092	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3093
		3094	/* VLV magic */
		3095	I915_WRITE(VLV_IMR, 0);
		3096	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
		3097	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
		3098	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
		3099
5060	serge	3100	gen5_gt_irq_reset(dev);
4104	Serge	3101
5354	serge	3102	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3031	serge	3103
5354	serge	3104	vlv_display_irq_reset(dev_priv);
3031	serge	3105	}
		3106
5060	serge	3107	static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
4560	Serge	3108	{
5060	serge	3109	GEN8_IRQ_RESET_NDX(GT, 0);
		3110	GEN8_IRQ_RESET_NDX(GT, 1);
		3111	GEN8_IRQ_RESET_NDX(GT, 2);
		3112	GEN8_IRQ_RESET_NDX(GT, 3);
		3113	}
		3114
		3115	static void gen8_irq_reset(struct drm_device *dev)
		3116	{
4560	Serge	3117	struct drm_i915_private *dev_priv = dev->dev_private;
		3118	int pipe;
		3119
		3120	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3121	POSTING_READ(GEN8_MASTER_IRQ);
		3122
5060	serge	3123	gen8_gt_irq_reset(dev_priv);
4560	Serge	3124
5354	serge	3125	for_each_pipe(dev_priv, pipe)
		3126	if (intel_display_power_is_enabled(dev_priv,
6084	serge	3127	POWER_DOMAIN_PIPE(pipe)))
		3128	GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
4560	Serge	3129
5060	serge	3130	GEN5_IRQ_RESET(GEN8_DE_PORT_);
		3131	GEN5_IRQ_RESET(GEN8_DE_MISC_);
		3132	GEN5_IRQ_RESET(GEN8_PCU_);
4560	Serge	3133
6084	serge	3134	if (HAS_PCH_SPLIT(dev))
		3135	ibx_irq_reset(dev);
5060	serge	3136	}
4560	Serge	3137
6084	serge	3138	void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
		3139	unsigned int pipe_mask)
5060	serge	3140	{
5354	serge	3141	uint32_t extra_ier = GEN8_PIPE_VBLANK \| GEN8_PIPE_FIFO_UNDERRUN;
4560	Serge	3142
5354	serge	3143	spin_lock_irq(&dev_priv->irq_lock);
6084	serge	3144	if (pipe_mask & 1 << PIPE_A)
		3145	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
		3146	dev_priv->de_irq_mask[PIPE_A],
		3147	~dev_priv->de_irq_mask[PIPE_A] \| extra_ier);
		3148	if (pipe_mask & 1 << PIPE_B)
		3149	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
		3150	dev_priv->de_irq_mask[PIPE_B],
		3151	~dev_priv->de_irq_mask[PIPE_B] \| extra_ier);
		3152	if (pipe_mask & 1 << PIPE_C)
		3153	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
		3154	dev_priv->de_irq_mask[PIPE_C],
		3155	~dev_priv->de_irq_mask[PIPE_C] \| extra_ier);
5354	serge	3156	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	3157	}
		3158
		3159	static void cherryview_irq_preinstall(struct drm_device *dev)
		3160	{
		3161	struct drm_i915_private *dev_priv = dev->dev_private;
		3162
		3163	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3164	POSTING_READ(GEN8_MASTER_IRQ);
		3165
		3166	gen8_gt_irq_reset(dev_priv);
		3167
		3168	GEN5_IRQ_RESET(GEN8_PCU_);
		3169
		3170	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
		3171
5354	serge	3172	vlv_display_irq_reset(dev_priv);
4560	Serge	3173	}
		3174
6084	serge	3175	static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
		3176	const u32 hpd[HPD_NUM_PINS])
		3177	{
		3178	struct drm_i915_private *dev_priv = to_i915(dev);
		3179	struct intel_encoder *encoder;
		3180	u32 enabled_irqs = 0;
		3181
		3182	for_each_intel_encoder(dev, encoder)
		3183	if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
		3184	enabled_irqs \|= hpd[encoder->hpd_pin];
		3185
		3186	return enabled_irqs;
		3187	}
		3188
3746	Serge	3189	static void ibx_hpd_irq_setup(struct drm_device *dev)
		3190	{
5060	serge	3191	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	3192	u32 hotplug_irqs, hotplug, enabled_irqs;
3746	Serge	3193
		3194	if (HAS_PCH_IBX(dev)) {
4104	Serge	3195	hotplug_irqs = SDE_HOTPLUG_MASK;
6084	serge	3196	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3746	Serge	3197	} else {
4104	Serge	3198	hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
6084	serge	3199	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3746	Serge	3200	}
		3201
4104	Serge	3202	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3746	Serge	3203
		3204	/*
6084	serge	3205	* Enable digital hotplug on the PCH, and configure the DP short pulse
		3206	* duration to 2ms (which is the minimum in the Display Port spec).
		3207	* The pulse duration bits are reserved on LPT+.
		3208	*/
2351	Serge	3209	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3210	hotplug &= ~(PORTD_PULSE_DURATION_MASK\|PORTC_PULSE_DURATION_MASK\|PORTB_PULSE_DURATION_MASK);
		3211	hotplug \|= PORTD_HOTPLUG_ENABLE \| PORTD_PULSE_DURATION_2ms;
		3212	hotplug \|= PORTC_HOTPLUG_ENABLE \| PORTC_PULSE_DURATION_2ms;
		3213	hotplug \|= PORTB_HOTPLUG_ENABLE \| PORTB_PULSE_DURATION_2ms;
6084	serge	3214	/*
		3215	* When CPU and PCH are on the same package, port A
		3216	* HPD must be enabled in both north and south.
		3217	*/
		3218	if (HAS_PCH_LPT_LP(dev))
		3219	hotplug \|= PORTA_HOTPLUG_ENABLE;
2351	Serge	3220	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3221	}
		3222
6084	serge	3223	static void spt_hpd_irq_setup(struct drm_device *dev)
		3224	{
		3225	struct drm_i915_private *dev_priv = dev->dev_private;
		3226	u32 hotplug_irqs, hotplug, enabled_irqs;
		3227
		3228	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
		3229	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
		3230
		3231	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
		3232
		3233	/* Enable digital hotplug on the PCH */
		3234	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3235	hotplug \|= PORTD_HOTPLUG_ENABLE \| PORTC_HOTPLUG_ENABLE \|
		3236	PORTB_HOTPLUG_ENABLE \| PORTA_HOTPLUG_ENABLE;
		3237	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3238
		3239	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
		3240	hotplug \|= PORTE_HOTPLUG_ENABLE;
		3241	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
		3242	}
		3243
		3244	static void ilk_hpd_irq_setup(struct drm_device *dev)
		3245	{
		3246	struct drm_i915_private *dev_priv = dev->dev_private;
		3247	u32 hotplug_irqs, hotplug, enabled_irqs;
		3248
		3249	if (INTEL_INFO(dev)->gen >= 8) {
		3250	hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		3251	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
		3252
		3253	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3254	} else if (INTEL_INFO(dev)->gen >= 7) {
		3255	hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		3256	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
		3257
		3258	ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3259	} else {
		3260	hotplug_irqs = DE_DP_A_HOTPLUG;
		3261	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
		3262
		3263	ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3264	}
		3265
		3266	/*
		3267	* Enable digital hotplug on the CPU, and configure the DP short pulse
		3268	* duration to 2ms (which is the minimum in the Display Port spec)
		3269	* The pulse duration bits are reserved on HSW+.
		3270	*/
		3271	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
		3272	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
		3273	hotplug \|= DIGITAL_PORTA_HOTPLUG_ENABLE \| DIGITAL_PORTA_PULSE_DURATION_2ms;
		3274	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
		3275
		3276	ibx_hpd_irq_setup(dev);
		3277	}
		3278
		3279	static void bxt_hpd_irq_setup(struct drm_device *dev)
		3280	{
		3281	struct drm_i915_private *dev_priv = dev->dev_private;
		3282	u32 hotplug_irqs, hotplug, enabled_irqs;
		3283
		3284	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
		3285	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
		3286
		3287	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
		3288
		3289	hotplug = I915_READ(PCH_PORT_HOTPLUG);
		3290	hotplug \|= PORTC_HOTPLUG_ENABLE \| PORTB_HOTPLUG_ENABLE \|
		3291	PORTA_HOTPLUG_ENABLE;
		3292	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
		3293	}
		3294
3480	Serge	3295	static void ibx_irq_postinstall(struct drm_device *dev)
		3296	{
5060	serge	3297	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	3298	u32 mask;
		3299
3746	Serge	3300	if (HAS_PCH_NOP(dev))
		3301	return;
		3302
5060	serge	3303	if (HAS_PCH_IBX(dev))
		3304	mask = SDE_GMBUS \| SDE_AUX_MASK \| SDE_POISON;
		3305	else
		3306	mask = SDE_GMBUS_CPT \| SDE_AUX_MASK_CPT;
4104	Serge	3307
6084	serge	3308	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
3480	Serge	3309	I915_WRITE(SDEIMR, ~mask);
		3310	}
		3311
4104	Serge	3312	static void gen5_gt_irq_postinstall(struct drm_device *dev)
2351	Serge	3313	{
4104	Serge	3314	struct drm_i915_private *dev_priv = dev->dev_private;
		3315	u32 pm_irqs, gt_irqs;
2351	Serge	3316
4104	Serge	3317	pm_irqs = gt_irqs = 0;
2351	Serge	3318
		3319	dev_priv->gt_irq_mask = ~0;
4560	Serge	3320	if (HAS_L3_DPF(dev)) {
4104	Serge	3321	/* L3 parity interrupt is always unmasked. */
4560	Serge	3322	dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		3323	gt_irqs \|= GT_PARITY_ERROR(dev);
4104	Serge	3324	}
2351	Serge	3325
4104	Serge	3326	gt_irqs \|= GT_RENDER_USER_INTERRUPT;
		3327	if (IS_GEN5(dev)) {
		3328	gt_irqs \|= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT \|
		3329	ILK_BSD_USER_INTERRUPT;
		3330	} else {
		3331	gt_irqs \|= GT_BLT_USER_INTERRUPT \| GT_BSD_USER_INTERRUPT;
		3332	}
2351	Serge	3333
5060	serge	3334	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
2351	Serge	3335
4104	Serge	3336	if (INTEL_INFO(dev)->gen >= 6) {
5354	serge	3337	/*
		3338	* RPS interrupts will get enabled/disabled on demand when RPS
		3339	* itself is enabled/disabled.
		3340	*/
4104	Serge	3341	if (HAS_VEBOX(dev))
		3342	pm_irqs \|= PM_VEBOX_USER_INTERRUPT;
		3343
		3344	dev_priv->pm_irq_mask = 0xffffffff;
5060	serge	3345	GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
6084	serge	3346	}
2351	Serge	3347	}
		3348
4104	Serge	3349	static int ironlake_irq_postinstall(struct drm_device *dev)
3031	serge	3350	{
5060	serge	3351	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	3352	u32 display_mask, extra_mask;
		3353
		3354	if (INTEL_INFO(dev)->gen >= 7) {
		3355	display_mask = (DE_MASTER_IRQ_CONTROL \| DE_GSE_IVB \|
		3356	DE_PCH_EVENT_IVB \| DE_PLANEC_FLIP_DONE_IVB \|
6084	serge	3357	DE_PLANEB_FLIP_DONE_IVB \|
5060	serge	3358	DE_PLANEA_FLIP_DONE_IVB \| DE_AUX_CHANNEL_A_IVB);
4104	Serge	3359	extra_mask = (DE_PIPEC_VBLANK_IVB \| DE_PIPEB_VBLANK_IVB \|
6084	serge	3360	DE_PIPEA_VBLANK_IVB \| DE_ERR_INT_IVB \|
		3361	DE_DP_A_HOTPLUG_IVB);
4104	Serge	3362	} else {
		3363	display_mask = (DE_MASTER_IRQ_CONTROL \| DE_GSE \| DE_PCH_EVENT \|
		3364	DE_PLANEA_FLIP_DONE \| DE_PLANEB_FLIP_DONE \|
4560	Serge	3365	DE_AUX_CHANNEL_A \|
		3366	DE_PIPEB_CRC_DONE \| DE_PIPEA_CRC_DONE \|
		3367	DE_POISON);
6084	serge	3368	extra_mask = (DE_PIPEA_VBLANK \| DE_PIPEB_VBLANK \| DE_PCU_EVENT \|
		3369	DE_PIPEB_FIFO_UNDERRUN \| DE_PIPEA_FIFO_UNDERRUN \|
		3370	DE_DP_A_HOTPLUG);
4104	Serge	3371	}
		3372
3031	serge	3373	dev_priv->irq_mask = ~display_mask;
		3374
5060	serge	3375	I915_WRITE(HWSTAM, 0xeffe);
3031	serge	3376
5060	serge	3377	ibx_irq_pre_postinstall(dev);
		3378
		3379	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask \| extra_mask);
		3380
4104	Serge	3381	gen5_gt_irq_postinstall(dev);
3031	serge	3382
4104	Serge	3383	ibx_irq_postinstall(dev);
3031	serge	3384
4104	Serge	3385	if (IS_IRONLAKE_M(dev)) {
		3386	/* Enable PCU event interrupts
		3387	*
		3388	* spinlocking not required here for correctness since interrupt
		3389	* setup is guaranteed to run in single-threaded context. But we
		3390	* need it to make the assert_spin_locked happy. */
5354	serge	3391	spin_lock_irq(&dev_priv->irq_lock);
4104	Serge	3392	ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
5354	serge	3393	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	3394	}
3031	serge	3395
		3396	return 0;
		3397	}
		3398
5060	serge	3399	static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
		3400	{
		3401	u32 pipestat_mask;
		3402	u32 iir_mask;
5354	serge	3403	enum pipe pipe;
5060	serge	3404
		3405	pipestat_mask = PIPESTAT_INT_STATUS_MASK \|
		3406	PIPE_FIFO_UNDERRUN_STATUS;
		3407
5354	serge	3408	for_each_pipe(dev_priv, pipe)
		3409	I915_WRITE(PIPESTAT(pipe), pipestat_mask);
5060	serge	3410	POSTING_READ(PIPESTAT(PIPE_A));
		3411
		3412	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV \|
		3413	PIPE_CRC_DONE_INTERRUPT_STATUS;
		3414
5354	serge	3415	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		3416	for_each_pipe(dev_priv, pipe)
		3417	i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
5060	serge	3418
		3419	iir_mask = I915_DISPLAY_PORT_INTERRUPT \|
		3420	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3421	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
5354	serge	3422	if (IS_CHERRYVIEW(dev_priv))
		3423	iir_mask \|= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
5060	serge	3424	dev_priv->irq_mask &= ~iir_mask;
		3425
		3426	I915_WRITE(VLV_IIR, iir_mask);
		3427	I915_WRITE(VLV_IIR, iir_mask);
5354	serge	3428	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
5060	serge	3429	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5354	serge	3430	POSTING_READ(VLV_IMR);
5060	serge	3431	}
		3432
		3433	static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
		3434	{
		3435	u32 pipestat_mask;
		3436	u32 iir_mask;
5354	serge	3437	enum pipe pipe;
5060	serge	3438
		3439	iir_mask = I915_DISPLAY_PORT_INTERRUPT \|
		3440	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3441	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
5354	serge	3442	if (IS_CHERRYVIEW(dev_priv))
		3443	iir_mask \|= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
5060	serge	3444
		3445	dev_priv->irq_mask \|= iir_mask;
5354	serge	3446	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5060	serge	3447	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
		3448	I915_WRITE(VLV_IIR, iir_mask);
		3449	I915_WRITE(VLV_IIR, iir_mask);
		3450	POSTING_READ(VLV_IIR);
		3451
		3452	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV \|
		3453	PIPE_CRC_DONE_INTERRUPT_STATUS;
		3454
5354	serge	3455	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		3456	for_each_pipe(dev_priv, pipe)
		3457	i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
5060	serge	3458
		3459	pipestat_mask = PIPESTAT_INT_STATUS_MASK \|
		3460	PIPE_FIFO_UNDERRUN_STATUS;
5354	serge	3461
		3462	for_each_pipe(dev_priv, pipe)
		3463	I915_WRITE(PIPESTAT(pipe), pipestat_mask);
5060	serge	3464	POSTING_READ(PIPESTAT(PIPE_A));
		3465	}
		3466
		3467	void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
		3468	{
		3469	assert_spin_locked(&dev_priv->irq_lock);
		3470
		3471	if (dev_priv->display_irqs_enabled)
		3472	return;
		3473
		3474	dev_priv->display_irqs_enabled = true;
		3475
5354	serge	3476	if (intel_irqs_enabled(dev_priv))
5060	serge	3477	valleyview_display_irqs_install(dev_priv);
		3478	}
		3479
		3480	void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
		3481	{
		3482	assert_spin_locked(&dev_priv->irq_lock);
		3483
		3484	if (!dev_priv->display_irqs_enabled)
		3485	return;
		3486
		3487	dev_priv->display_irqs_enabled = false;
		3488
5354	serge	3489	if (intel_irqs_enabled(dev_priv))
5060	serge	3490	valleyview_display_irqs_uninstall(dev_priv);
		3491	}
		3492
5354	serge	3493	static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3031	serge	3494	{
5060	serge	3495	dev_priv->irq_mask = ~0;
3031	serge	3496
6084	serge	3497	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	3498	POSTING_READ(PORT_HOTPLUG_EN);
		3499
5354	serge	3500	I915_WRITE(VLV_IIR, 0xffffffff);
		3501	I915_WRITE(VLV_IIR, 0xffffffff);
		3502	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3031	serge	3503	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
5354	serge	3504	POSTING_READ(VLV_IMR);
3031	serge	3505
4104	Serge	3506	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3507	* just to make the assert_spin_locked check happy. */
5354	serge	3508	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3509	if (dev_priv->display_irqs_enabled)
		3510	valleyview_display_irqs_install(dev_priv);
5354	serge	3511	spin_unlock_irq(&dev_priv->irq_lock);
		3512	}
3031	serge	3513
5354	serge	3514	static int valleyview_irq_postinstall(struct drm_device *dev)
		3515	{
		3516	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3517
5354	serge	3518	vlv_display_irq_postinstall(dev_priv);
		3519
4104	Serge	3520	gen5_gt_irq_postinstall(dev);
3243	Serge	3521
3031	serge	3522	/* ack & enable invalid PTE error interrupts */
		3523	#if 0 /* FIXME: add support to irq handler for checking these bits */
		3524	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
		3525	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
		3526	#endif
		3527
		3528	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3480	Serge	3529
		3530	return 0;
		3531	}
		3532
4560	Serge	3533	static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
		3534	{
		3535	/* These are interrupts we'll toggle with the ring mask register */
		3536	uint32_t gt_interrupts[] = {
		3537	GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT \|
5354	serge	3538	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT \|
4560	Serge	3539	GT_RENDER_L3_PARITY_ERROR_INTERRUPT \|
5354	serge	3540	GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT \|
		3541	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
4560	Serge	3542	GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT \|
5354	serge	3543	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT \|
		3544	GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT \|
		3545	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
4560	Serge	3546	0,
5354	serge	3547	GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT \|
		3548	GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
4560	Serge	3549	};
		3550
5060	serge	3551	dev_priv->pm_irq_mask = 0xffffffff;
5354	serge	3552	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
		3553	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
		3554	/*
		3555	* RPS interrupts will get enabled/disabled on demand when RPS itself
		3556	* is enabled/disabled.
		3557	*/
		3558	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
		3559	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
4560	Serge	3560	}
		3561
		3562	static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
		3563	{
5354	serge	3564	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
		3565	uint32_t de_pipe_enables;
6084	serge	3566	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
		3567	u32 de_port_enables;
		3568	enum pipe pipe;
5354	serge	3569
6084	serge	3570	if (INTEL_INFO(dev_priv)->gen >= 9) {
5354	serge	3571	de_pipe_masked \|= GEN9_PIPE_PLANE1_FLIP_DONE \|
		3572	GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
6084	serge	3573	de_port_masked \|= GEN9_AUX_CHANNEL_B \| GEN9_AUX_CHANNEL_C \|
		3574	GEN9_AUX_CHANNEL_D;
		3575	if (IS_BROXTON(dev_priv))
		3576	de_port_masked \|= BXT_DE_PORT_GMBUS;
		3577	} else {
5354	serge	3578	de_pipe_masked \|= GEN8_PIPE_PRIMARY_FLIP_DONE \|
6084	serge	3579	GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
		3580	}
5354	serge	3581
		3582	de_pipe_enables = de_pipe_masked \| GEN8_PIPE_VBLANK \|
6084	serge	3583	GEN8_PIPE_FIFO_UNDERRUN;
5354	serge	3584
6084	serge	3585	de_port_enables = de_port_masked;
		3586	if (IS_BROXTON(dev_priv))
		3587	de_port_enables \|= BXT_DE_PORT_HOTPLUG_MASK;
		3588	else if (IS_BROADWELL(dev_priv))
		3589	de_port_enables \|= GEN8_PORT_DP_A_HOTPLUG;
		3590
4560	Serge	3591	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
		3592	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
		3593	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
		3594
5354	serge	3595	for_each_pipe(dev_priv, pipe)
		3596	if (intel_display_power_is_enabled(dev_priv,
5060	serge	3597	POWER_DOMAIN_PIPE(pipe)))
		3598	GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
		3599	dev_priv->de_irq_mask[pipe],
6084	serge	3600	de_pipe_enables);
4560	Serge	3601
6084	serge	3602	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
4560	Serge	3603	}
		3604
		3605	static int gen8_irq_postinstall(struct drm_device *dev)
		3606	{
		3607	struct drm_i915_private *dev_priv = dev->dev_private;
		3608
6084	serge	3609	if (HAS_PCH_SPLIT(dev))
		3610	ibx_irq_pre_postinstall(dev);
5060	serge	3611
4560	Serge	3612	gen8_gt_irq_postinstall(dev_priv);
		3613	gen8_de_irq_postinstall(dev_priv);
		3614
6084	serge	3615	if (HAS_PCH_SPLIT(dev))
		3616	ibx_irq_postinstall(dev);
4560	Serge	3617
		3618	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		3619	POSTING_READ(GEN8_MASTER_IRQ);
		3620
		3621	return 0;
		3622	}
		3623
5060	serge	3624	static int cherryview_irq_postinstall(struct drm_device *dev)
4560	Serge	3625	{
		3626	struct drm_i915_private *dev_priv = dev->dev_private;
		3627
5354	serge	3628	vlv_display_irq_postinstall(dev_priv);
4560	Serge	3629
5060	serge	3630	gen8_gt_irq_postinstall(dev_priv);
4560	Serge	3631
5060	serge	3632	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
		3633	POSTING_READ(GEN8_MASTER_IRQ);
4560	Serge	3634
5060	serge	3635	return 0;
		3636	}
4560	Serge	3637
5060	serge	3638	static void gen8_irq_uninstall(struct drm_device *dev)
		3639	{
		3640	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3641
5060	serge	3642	if (!dev_priv)
		3643	return;
		3644
		3645	gen8_irq_reset(dev);
4560	Serge	3646	}
		3647
5354	serge	3648	static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
		3649	{
		3650	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3651	* just to make the assert_spin_locked check happy. */
		3652	spin_lock_irq(&dev_priv->irq_lock);
		3653	if (dev_priv->display_irqs_enabled)
		3654	valleyview_display_irqs_uninstall(dev_priv);
		3655	spin_unlock_irq(&dev_priv->irq_lock);
		3656
		3657	vlv_display_irq_reset(dev_priv);
		3658
		3659	dev_priv->irq_mask = ~0;
		3660	}
		3661
3031	serge	3662	static void valleyview_irq_uninstall(struct drm_device *dev)
		3663	{
5060	serge	3664	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3665
		3666	if (!dev_priv)
		3667	return;
		3668
5060	serge	3669	I915_WRITE(VLV_MASTER_IER, 0);
4293	Serge	3670
5354	serge	3671	gen5_gt_irq_reset(dev);
3031	serge	3672
		3673	I915_WRITE(HWSTAM, 0xffffffff);
5060	serge	3674
5354	serge	3675	vlv_display_irq_uninstall(dev_priv);
3031	serge	3676	}
		3677
5060	serge	3678	static void cherryview_irq_uninstall(struct drm_device *dev)
3031	serge	3679	{
5060	serge	3680	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3681
		3682	if (!dev_priv)
		3683	return;
		3684
5060	serge	3685	I915_WRITE(GEN8_MASTER_IRQ, 0);
		3686	POSTING_READ(GEN8_MASTER_IRQ);
4293	Serge	3687
5354	serge	3688	gen8_gt_irq_reset(dev_priv);
3031	serge	3689
5354	serge	3690	GEN5_IRQ_RESET(GEN8_PCU_);
3031	serge	3691
5354	serge	3692	vlv_display_irq_uninstall(dev_priv);
5060	serge	3693	}
		3694
		3695	static void ironlake_irq_uninstall(struct drm_device *dev)
		3696	{
		3697	struct drm_i915_private *dev_priv = dev->dev_private;
		3698
		3699	if (!dev_priv)
3746	Serge	3700	return;
		3701
5060	serge	3702	ironlake_irq_reset(dev);
3031	serge	3703	}
		3704
		3705	#if 0
		3706	static void i8xx_irq_preinstall(struct drm_device * dev)
		3707	{
5060	serge	3708	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3709	int pipe;
		3710
5354	serge	3711	for_each_pipe(dev_priv, pipe)
3031	serge	3712	I915_WRITE(PIPESTAT(pipe), 0);
		3713	I915_WRITE16(IMR, 0xffff);
		3714	I915_WRITE16(IER, 0x0);
		3715	POSTING_READ16(IER);
		3716	}
		3717
		3718	static int i8xx_irq_postinstall(struct drm_device *dev)
		3719	{
5060	serge	3720	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3721
		3722	I915_WRITE16(EMR,
		3723	~(I915_ERROR_PAGE_TABLE \| I915_ERROR_MEMORY_REFRESH));
		3724
		3725	/* Unmask the interrupts that we always want on. */
		3726	dev_priv->irq_mask =
		3727	~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3728	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3729	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
6084	serge	3730	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	3731	I915_WRITE16(IMR, dev_priv->irq_mask);
		3732
		3733	I915_WRITE16(IER,
		3734	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3735	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3736	I915_USER_INTERRUPT);
		3737	POSTING_READ16(IER);
		3738
4560	Serge	3739	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3740	* just to make the assert_spin_locked check happy. */
5354	serge	3741	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3742	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		3743	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	3744	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	3745
3031	serge	3746	return 0;
		3747	}
		3748
3746	Serge	3749	/*
		3750	* Returns true when a page flip has completed.
		3751	*/
		3752	static bool i8xx_handle_vblank(struct drm_device *dev,
4560	Serge	3753	int plane, int pipe, u32 iir)
3746	Serge	3754	{
5060	serge	3755	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3756	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3746	Serge	3757
6084	serge	3758	if (!intel_pipe_handle_vblank(dev, pipe))
		3759	return false;
3746	Serge	3760
		3761	if ((iir & flip_pending) == 0)
5354	serge	3762	goto check_page_flip;
3746	Serge	3763
		3764	/* We detect FlipDone by looking for the change in PendingFlip from '1'
		3765	* to '0' on the following vblank, i.e. IIR has the Pendingflip
		3766	* asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
		3767	* the flip is completed (no longer pending). Since this doesn't raise
		3768	* an interrupt per se, we watch for the change at vblank.
		3769	*/
		3770	if (I915_READ16(ISR) & flip_pending)
5354	serge	3771	goto check_page_flip;
3746	Serge	3772
6084	serge	3773	// intel_prepare_page_flip(dev, plane);
		3774	// intel_finish_page_flip(dev, pipe);
5354	serge	3775	return true;
3746	Serge	3776
5354	serge	3777	check_page_flip:
6084	serge	3778	// intel_check_page_flip(dev, pipe);
5354	serge	3779	return false;
3746	Serge	3780	}
		3781
3243	Serge	3782	static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3031	serge	3783	{
5060	serge	3784	struct drm_device *dev = arg;
		3785	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3786	u16 iir, new_iir;
		3787	u32 pipe_stats[2];
		3788	int pipe;
		3789	u16 flip_mask =
		3790	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		3791	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
		3792
6084	serge	3793	if (!intel_irqs_enabled(dev_priv))
		3794	return IRQ_NONE;
		3795
3031	serge	3796	iir = I915_READ16(IIR);
		3797	if (iir == 0)
		3798	return IRQ_NONE;
		3799
		3800	while (iir & ~flip_mask) {
		3801	/* Can't rely on pipestat interrupt bit in iir as it might
		3802	* have been cleared after the pipestat interrupt was received.
		3803	* It doesn't set the bit in iir again, but it still produces
		3804	* interrupts (for non-MSI).
		3805	*/
5354	serge	3806	spin_lock(&dev_priv->irq_lock);
4126	Serge	3807	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	3808	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	3809
5354	serge	3810	for_each_pipe(dev_priv, pipe) {
3031	serge	3811	int reg = PIPESTAT(pipe);
		3812	pipe_stats[pipe] = I915_READ(reg);
		3813
		3814	/*
		3815	* Clear the PIPE*STAT regs before the IIR
		3816	*/
5060	serge	3817	if (pipe_stats[pipe] & 0x8000ffff)
3031	serge	3818	I915_WRITE(reg, pipe_stats[pipe]);
6084	serge	3819	}
5354	serge	3820	spin_unlock(&dev_priv->irq_lock);
3031	serge	3821
		3822	I915_WRITE16(IIR, iir & ~flip_mask);
		3823	new_iir = I915_READ16(IIR); /* Flush posted writes */
		3824
		3825	if (iir & I915_USER_INTERRUPT)
6084	serge	3826	notify_ring(&dev_priv->ring[RCS]);
3031	serge	3827
5354	serge	3828	for_each_pipe(dev_priv, pipe) {
4560	Serge	3829	int plane = pipe;
		3830	if (HAS_FBC(dev))
		3831	plane = !plane;
3031	serge	3832
4560	Serge	3833	if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
		3834	i8xx_handle_vblank(dev, plane, pipe, iir))
		3835	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3031	serge	3836
4560	Serge	3837	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		3838	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	3839
5354	serge	3840	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		3841	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		3842	pipe);
4560	Serge	3843	}
		3844
3031	serge	3845	iir = new_iir;
		3846	}
		3847
		3848	return IRQ_HANDLED;
		3849	}
		3850
		3851	static void i8xx_irq_uninstall(struct drm_device * dev)
		3852	{
5060	serge	3853	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3854	int pipe;
		3855
5354	serge	3856	for_each_pipe(dev_priv, pipe) {
3031	serge	3857	/* Clear enable bits; then clear status bits */
		3858	I915_WRITE(PIPESTAT(pipe), 0);
		3859	I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
		3860	}
		3861	I915_WRITE16(IMR, 0xffff);
		3862	I915_WRITE16(IER, 0x0);
		3863	I915_WRITE16(IIR, I915_READ16(IIR));
		3864	}
		3865
		3866	#endif
		3867
		3868	static void i915_irq_preinstall(struct drm_device * dev)
		3869	{
5060	serge	3870	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3871	int pipe;
		3872
		3873	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	3874	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	3875	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		3876	}
		3877
		3878	I915_WRITE16(HWSTAM, 0xeffe);
5354	serge	3879	for_each_pipe(dev_priv, pipe)
3031	serge	3880	I915_WRITE(PIPESTAT(pipe), 0);
		3881	I915_WRITE(IMR, 0xffffffff);
		3882	I915_WRITE(IER, 0x0);
		3883	POSTING_READ(IER);
		3884	}
		3885
		3886	static int i915_irq_postinstall(struct drm_device *dev)
		3887	{
5060	serge	3888	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3889	u32 enable_mask;
		3890
		3891	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE \| I915_ERROR_MEMORY_REFRESH));
		3892
		3893	/* Unmask the interrupts that we always want on. */
		3894	dev_priv->irq_mask =
		3895	~(I915_ASLE_INTERRUPT \|
		3896	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3897	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3898	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
6084	serge	3899	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	3900
		3901	enable_mask =
		3902	I915_ASLE_INTERRUPT \|
		3903	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		3904	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		3905	I915_USER_INTERRUPT;
3480	Serge	3906
3031	serge	3907	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	3908	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	3909	POSTING_READ(PORT_HOTPLUG_EN);
		3910
3031	serge	3911	/* Enable in IER... */
		3912	enable_mask \|= I915_DISPLAY_PORT_INTERRUPT;
		3913	/* and unmask in IMR */
		3914	dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
		3915	}
		3916
		3917	I915_WRITE(IMR, dev_priv->irq_mask);
		3918	I915_WRITE(IER, enable_mask);
		3919	POSTING_READ(IER);
		3920
4126	Serge	3921	i915_enable_asle_pipestat(dev);
3480	Serge	3922
4560	Serge	3923	/* Interrupt setup is already guaranteed to be single-threaded, this is
		3924	* just to make the assert_spin_locked check happy. */
5354	serge	3925	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	3926	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		3927	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	3928	spin_unlock_irq(&dev_priv->irq_lock);
4560	Serge	3929
3480	Serge	3930	return 0;
		3931	}
		3932
3746	Serge	3933	/*
		3934	* Returns true when a page flip has completed.
		3935	*/
		3936	static bool i915_handle_vblank(struct drm_device *dev,
		3937	int plane, int pipe, u32 iir)
3480	Serge	3938	{
5060	serge	3939	struct drm_i915_private *dev_priv = dev->dev_private;
3746	Serge	3940	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3480	Serge	3941
6088	serge	3942	if (!intel_pipe_handle_vblank(dev, pipe))
		3943	return false;
3480	Serge	3944
3746	Serge	3945	if ((iir & flip_pending) == 0)
5354	serge	3946	goto check_page_flip;
3480	Serge	3947
3746	Serge	3948	/* We detect FlipDone by looking for the change in PendingFlip from '1'
		3949	* to '0' on the following vblank, i.e. IIR has the Pendingflip
		3950	* asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
		3951	* the flip is completed (no longer pending). Since this doesn't raise
		3952	* an interrupt per se, we watch for the change at vblank.
		3953	*/
		3954	if (I915_READ(ISR) & flip_pending)
5354	serge	3955	goto check_page_flip;
3746	Serge	3956
5354	serge	3957	return true;
3746	Serge	3958
5354	serge	3959	check_page_flip:
		3960	return false;
3031	serge	3961	}
		3962
3243	Serge	3963	static irqreturn_t i915_irq_handler(int irq, void *arg)
3031	serge	3964	{
5060	serge	3965	struct drm_device *dev = arg;
		3966	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	3967	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
		3968	u32 flip_mask =
		3969	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		3970	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
		3971	int pipe, ret = IRQ_NONE;
		3972
6084	serge	3973	if (!intel_irqs_enabled(dev_priv))
		3974	return IRQ_NONE;
		3975
3031	serge	3976	iir = I915_READ(IIR);
		3977	do {
		3978	bool irq_received = (iir & ~flip_mask) != 0;
		3979	bool blc_event = false;
		3980
		3981	/* Can't rely on pipestat interrupt bit in iir as it might
		3982	* have been cleared after the pipestat interrupt was received.
		3983	* It doesn't set the bit in iir again, but it still produces
		3984	* interrupts (for non-MSI).
		3985	*/
5354	serge	3986	spin_lock(&dev_priv->irq_lock);
4126	Serge	3987	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	3988	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	3989
5354	serge	3990	for_each_pipe(dev_priv, pipe) {
3031	serge	3991	int reg = PIPESTAT(pipe);
		3992	pipe_stats[pipe] = I915_READ(reg);
		3993
		3994	/* Clear the PIPESTAT regs before the IIR /
		3995	if (pipe_stats[pipe] & 0x8000ffff) {
		3996	I915_WRITE(reg, pipe_stats[pipe]);
		3997	irq_received = true;
		3998	}
		3999	}
5354	serge	4000	spin_unlock(&dev_priv->irq_lock);
3031	serge	4001
		4002	if (!irq_received)
		4003	break;
		4004
		4005	/* Consume port. Then clear IIR or we'll miss events */
5060	serge	4006	if (I915_HAS_HOTPLUG(dev) &&
		4007	iir & I915_DISPLAY_PORT_INTERRUPT)
		4008	i9xx_hpd_irq_handler(dev);
3031	serge	4009
		4010	I915_WRITE(IIR, iir & ~flip_mask);
		4011	new_iir = I915_READ(IIR); /* Flush posted writes */
		4012
		4013	if (iir & I915_USER_INTERRUPT)
6084	serge	4014	notify_ring(&dev_priv->ring[RCS]);
3031	serge	4015
5354	serge	4016	for_each_pipe(dev_priv, pipe) {
3031	serge	4017	int plane = pipe;
4560	Serge	4018	if (HAS_FBC(dev))
3031	serge	4019	plane = !plane;
		4020
3746	Serge	4021	if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
		4022	i915_handle_vblank(dev, plane, pipe, iir))
		4023	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
		4024
3031	serge	4025	if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
		4026	blc_event = true;
4560	Serge	4027
		4028	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		4029	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	4030
5354	serge	4031	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		4032	intel_cpu_fifo_underrun_irq_handler(dev_priv,
		4033	pipe);
3031	serge	4034	}
		4035
4126	Serge	4036	if (blc_event \|\| (iir & I915_ASLE_INTERRUPT))
		4037	intel_opregion_asle_intr(dev);
3031	serge	4038
		4039	/* With MSI, interrupts are only generated when iir
		4040	* transitions from zero to nonzero. If another bit got
		4041	* set while we were handling the existing iir bits, then
		4042	* we would never get another interrupt.
		4043	*
		4044	* This is fine on non-MSI as well, as if we hit this path
		4045	* we avoid exiting the interrupt handler only to generate
		4046	* another one.
		4047	*
		4048	* Note that for MSI this could cause a stray interrupt report
		4049	* if an interrupt landed in the time between writing IIR and
		4050	* the posting read. This should be rare enough to never
		4051	* trigger the 99% of 100,000 interrupts test for disabling
		4052	* stray interrupts.
		4053	*/
		4054	ret = IRQ_HANDLED;
		4055	iir = new_iir;
		4056	} while (iir & ~flip_mask);
		4057
		4058	return ret;
		4059	}
		4060
		4061	static void i915_irq_uninstall(struct drm_device * dev)
		4062	{
5060	serge	4063	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4064	int pipe;
		4065
		4066	if (I915_HAS_HOTPLUG(dev)) {
6084	serge	4067	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4068	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4069	}
		4070
		4071	I915_WRITE16(HWSTAM, 0xffff);
5354	serge	4072	for_each_pipe(dev_priv, pipe) {
3031	serge	4073	/* Clear enable bits; then clear status bits */
		4074	I915_WRITE(PIPESTAT(pipe), 0);
		4075	I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
		4076	}
		4077	I915_WRITE(IMR, 0xffffffff);
		4078	I915_WRITE(IER, 0x0);
		4079
		4080	I915_WRITE(IIR, I915_READ(IIR));
		4081	}
		4082
		4083	static void i965_irq_preinstall(struct drm_device * dev)
		4084	{
5060	serge	4085	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4086	int pipe;
		4087
6084	serge	4088	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4089	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4090
		4091	I915_WRITE(HWSTAM, 0xeffe);
5354	serge	4092	for_each_pipe(dev_priv, pipe)
3031	serge	4093	I915_WRITE(PIPESTAT(pipe), 0);
		4094	I915_WRITE(IMR, 0xffffffff);
		4095	I915_WRITE(IER, 0x0);
		4096	POSTING_READ(IER);
		4097	}
		4098
		4099	static int i965_irq_postinstall(struct drm_device *dev)
		4100	{
5060	serge	4101	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4102	u32 enable_mask;
		4103	u32 error_mask;
		4104
		4105	/* Unmask the interrupts that we always want on. */
		4106	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT \|
		4107	I915_DISPLAY_PORT_INTERRUPT \|
		4108	I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \|
		4109	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT \|
		4110	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4111	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT \|
		4112	I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
		4113
		4114	enable_mask = ~dev_priv->irq_mask;
3746	Serge	4115	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4116	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3031	serge	4117	enable_mask \|= I915_USER_INTERRUPT;
		4118
		4119	if (IS_G4X(dev))
		4120	enable_mask \|= I915_BSD_USER_INTERRUPT;
		4121
4104	Serge	4122	/* Interrupt setup is already guaranteed to be single-threaded, this is
		4123	* just to make the assert_spin_locked check happy. */
5354	serge	4124	spin_lock_irq(&dev_priv->irq_lock);
5060	serge	4125	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
		4126	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
		4127	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
5354	serge	4128	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	4129
		4130	/*
		4131	* Enable some error detection, note the instruction error mask
		4132	* bit is reserved, so we leave it masked.
		4133	*/
		4134	if (IS_G4X(dev)) {
		4135	error_mask = ~(GM45_ERROR_PAGE_TABLE \|
		4136	GM45_ERROR_MEM_PRIV \|
		4137	GM45_ERROR_CP_PRIV \|
		4138	I915_ERROR_MEMORY_REFRESH);
		4139	} else {
		4140	error_mask = ~(I915_ERROR_PAGE_TABLE \|
		4141	I915_ERROR_MEMORY_REFRESH);
		4142	}
		4143	I915_WRITE(EMR, error_mask);
		4144
		4145	I915_WRITE(IMR, dev_priv->irq_mask);
		4146	I915_WRITE(IER, enable_mask);
		4147	POSTING_READ(IER);
		4148
6084	serge	4149	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3480	Serge	4150	POSTING_READ(PORT_HOTPLUG_EN);
		4151
4126	Serge	4152	i915_enable_asle_pipestat(dev);
3480	Serge	4153
		4154	return 0;
		4155	}
		4156
3746	Serge	4157	static void i915_hpd_irq_setup(struct drm_device *dev)
3480	Serge	4158	{
5060	serge	4159	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	4160	u32 hotplug_en;
		4161
4104	Serge	4162	assert_spin_locked(&dev_priv->irq_lock);
		4163
3031	serge	4164	/* Note HDMI and DP share hotplug bits */
6084	serge	4165	/* enable bits are the same for all generations */
		4166	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
		4167	/* Programming the CRT detection parameters tends
		4168	to generate a spurious hotplug event about three
		4169	seconds later. So just do it once.
		4170	*/
		4171	if (IS_G4X(dev))
		4172	hotplug_en \|= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
		4173	hotplug_en \|= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3480	Serge	4174
3031	serge	4175	/* Ignore TV since it's buggy */
6084	serge	4176	i915_hotplug_interrupt_update_locked(dev_priv,
		4177	HOTPLUG_INT_EN_MASK \|
		4178	CRT_HOTPLUG_VOLTAGE_COMPARE_MASK \|
		4179	CRT_HOTPLUG_ACTIVATION_PERIOD_64,
		4180	hotplug_en);
3031	serge	4181	}
		4182
3243	Serge	4183	static irqreturn_t i965_irq_handler(int irq, void *arg)
3031	serge	4184	{
5060	serge	4185	struct drm_device *dev = arg;
		4186	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4187	u32 iir, new_iir;
		4188	u32 pipe_stats[I915_MAX_PIPES];
		4189	int ret = IRQ_NONE, pipe;
3746	Serge	4190	u32 flip_mask =
		4191	I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT \|
		4192	I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3031	serge	4193
6084	serge	4194	if (!intel_irqs_enabled(dev_priv))
		4195	return IRQ_NONE;
		4196
3031	serge	4197	iir = I915_READ(IIR);
		4198
		4199	for (;;) {
5060	serge	4200	bool irq_received = (iir & ~flip_mask) != 0;
3031	serge	4201	bool blc_event = false;
		4202
		4203	/* Can't rely on pipestat interrupt bit in iir as it might
		4204	* have been cleared after the pipestat interrupt was received.
		4205	* It doesn't set the bit in iir again, but it still produces
		4206	* interrupts (for non-MSI).
		4207	*/
5354	serge	4208	spin_lock(&dev_priv->irq_lock);
4126	Serge	4209	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
5354	serge	4210	DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3031	serge	4211
5354	serge	4212	for_each_pipe(dev_priv, pipe) {
3031	serge	4213	int reg = PIPESTAT(pipe);
		4214	pipe_stats[pipe] = I915_READ(reg);
		4215
		4216	/*
		4217	* Clear the PIPE*STAT regs before the IIR
		4218	*/
		4219	if (pipe_stats[pipe] & 0x8000ffff) {
		4220	I915_WRITE(reg, pipe_stats[pipe]);
5060	serge	4221	irq_received = true;
3031	serge	4222	}
		4223	}
5354	serge	4224	spin_unlock(&dev_priv->irq_lock);
3031	serge	4225
		4226	if (!irq_received)
		4227	break;
		4228
		4229	ret = IRQ_HANDLED;
		4230
		4231	/* Consume port. Then clear IIR or we'll miss events */
5060	serge	4232	if (iir & I915_DISPLAY_PORT_INTERRUPT)
		4233	i9xx_hpd_irq_handler(dev);
3031	serge	4234
3746	Serge	4235	I915_WRITE(IIR, iir & ~flip_mask);
3031	serge	4236	new_iir = I915_READ(IIR); /* Flush posted writes */
		4237
		4238	if (iir & I915_USER_INTERRUPT)
6084	serge	4239	notify_ring(&dev_priv->ring[RCS]);
3031	serge	4240	if (iir & I915_BSD_USER_INTERRUPT)
6084	serge	4241	notify_ring(&dev_priv->ring[VCS]);
3031	serge	4242
5354	serge	4243	for_each_pipe(dev_priv, pipe) {
3746	Serge	4244	if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		4245	i915_handle_vblank(dev, pipe, pipe, iir))
		4246	flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
3031	serge	4247
		4248	if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
		4249	blc_event = true;
4560	Serge	4250
		4251	if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
		4252	i9xx_pipe_crc_irq_handler(dev, pipe);
5060	serge	4253
5354	serge	4254	if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
		4255	intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
3031	serge	4256	}
		4257
4126	Serge	4258	if (blc_event \|\| (iir & I915_ASLE_INTERRUPT))
		4259	intel_opregion_asle_intr(dev);
3031	serge	4260
3480	Serge	4261	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		4262	gmbus_irq_handler(dev);
		4263
3031	serge	4264	/* With MSI, interrupts are only generated when iir
		4265	* transitions from zero to nonzero. If another bit got
		4266	* set while we were handling the existing iir bits, then
		4267	* we would never get another interrupt.
		4268	*
		4269	* This is fine on non-MSI as well, as if we hit this path
		4270	* we avoid exiting the interrupt handler only to generate
		4271	* another one.
		4272	*
		4273	* Note that for MSI this could cause a stray interrupt report
		4274	* if an interrupt landed in the time between writing IIR and
		4275	* the posting read. This should be rare enough to never
		4276	* trigger the 99% of 100,000 interrupts test for disabling
		4277	* stray interrupts.
		4278	*/
		4279	iir = new_iir;
		4280	}
		4281
		4282	return ret;
		4283	}
		4284
		4285	static void i965_irq_uninstall(struct drm_device * dev)
		4286	{
5060	serge	4287	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	4288	int pipe;
		4289
		4290	if (!dev_priv)
		4291	return;
		4292
6084	serge	4293	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3031	serge	4294	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
		4295
		4296	I915_WRITE(HWSTAM, 0xffffffff);
5354	serge	4297	for_each_pipe(dev_priv, pipe)
3031	serge	4298	I915_WRITE(PIPESTAT(pipe), 0);
		4299	I915_WRITE(IMR, 0xffffffff);
		4300	I915_WRITE(IER, 0x0);
		4301
5354	serge	4302	for_each_pipe(dev_priv, pipe)
3031	serge	4303	I915_WRITE(PIPESTAT(pipe),
		4304	I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
		4305	I915_WRITE(IIR, I915_READ(IIR));
		4306	}
		4307
5354	serge	4308	/**
		4309	* intel_irq_init - initializes irq support
		4310	* @dev_priv: i915 device instance
		4311	*
		4312	* This function initializes all the irq support including work items, timers
		4313	* and all the vtables. It does not setup the interrupt itself though.
		4314	*/
		4315	void intel_irq_init(struct drm_i915_private *dev_priv)
2351	Serge	4316	{
5354	serge	4317	struct drm_device *dev = dev_priv->dev;
3031	serge	4318
6084	serge	4319	// intel_hpd_init_work(dev_priv);
		4320
4126	Serge	4321	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
		4322	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
3480	Serge	4323
5060	serge	4324	/* Let's track the enabled rps events */
5354	serge	4325	if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
		4326	/* WaGsvRC0ResidencyMethod:vlv */
6084	serge	4327	dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED;
5060	serge	4328	else
6084	serge	4329	dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
3480	Serge	4330
6084	serge	4331	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
		4332	i915_hangcheck_elapsed);
4560	Serge	4333
5354	serge	4334
		4335	if (IS_GEN2(dev_priv)) {
4560	Serge	4336	dev->max_vblank_count = 0;
		4337	dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
5354	serge	4338	} else if (IS_G4X(dev_priv) \|\| INTEL_INFO(dev_priv)->gen >= 5) {
4560	Serge	4339	dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
6084	serge	4340	dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4560	Serge	4341	} else {
6084	serge	4342	dev->driver->get_vblank_counter = i915_get_vblank_counter;
		4343	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4293	Serge	4344	}
3480	Serge	4345
5354	serge	4346	/*
		4347	* Opt out of the vblank disable timer on everything except gen2.
		4348	* Gen2 doesn't have a hardware frame counter and so depends on
		4349	* vblank interrupts to produce sane vblank seuquence numbers.
		4350	*/
		4351	if (!IS_GEN2(dev_priv))
		4352	dev->vblank_disable_immediate = true;
		4353
6084	serge	4354	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4293	Serge	4355	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
3480	Serge	4356
5354	serge	4357	if (IS_CHERRYVIEW(dev_priv)) {
5060	serge	4358	dev->driver->irq_handler = cherryview_irq_handler;
		4359	dev->driver->irq_preinstall = cherryview_irq_preinstall;
		4360	dev->driver->irq_postinstall = cherryview_irq_postinstall;
		4361	dev->driver->irq_uninstall = cherryview_irq_uninstall;
		4362	dev->driver->enable_vblank = valleyview_enable_vblank;
		4363	dev->driver->disable_vblank = valleyview_disable_vblank;
		4364	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
5354	serge	4365	} else if (IS_VALLEYVIEW(dev_priv)) {
3243	Serge	4366	dev->driver->irq_handler = valleyview_irq_handler;
		4367	dev->driver->irq_preinstall = valleyview_irq_preinstall;
		4368	dev->driver->irq_postinstall = valleyview_irq_postinstall;
4293	Serge	4369	dev->driver->irq_uninstall = valleyview_irq_uninstall;
		4370	dev->driver->enable_vblank = valleyview_enable_vblank;
		4371	dev->driver->disable_vblank = valleyview_disable_vblank;
3746	Serge	4372	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
5354	serge	4373	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4560	Serge	4374	dev->driver->irq_handler = gen8_irq_handler;
5060	serge	4375	dev->driver->irq_preinstall = gen8_irq_reset;
4560	Serge	4376	dev->driver->irq_postinstall = gen8_irq_postinstall;
		4377	dev->driver->irq_uninstall = gen8_irq_uninstall;
		4378	dev->driver->enable_vblank = gen8_enable_vblank;
		4379	dev->driver->disable_vblank = gen8_disable_vblank;
6084	serge	4380	if (IS_BROXTON(dev))
		4381	dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
		4382	else if (HAS_PCH_SPT(dev))
		4383	dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		4384	else
		4385	dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
2351	Serge	4386	} else if (HAS_PCH_SPLIT(dev)) {
3243	Serge	4387	dev->driver->irq_handler = ironlake_irq_handler;
5060	serge	4388	dev->driver->irq_preinstall = ironlake_irq_reset;
3243	Serge	4389	dev->driver->irq_postinstall = ironlake_irq_postinstall;
4293	Serge	4390	dev->driver->irq_uninstall = ironlake_irq_uninstall;
		4391	dev->driver->enable_vblank = ironlake_enable_vblank;
		4392	dev->driver->disable_vblank = ironlake_disable_vblank;
6084	serge	4393	dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
2351	Serge	4394	} else {
5354	serge	4395	if (INTEL_INFO(dev_priv)->gen == 2) {
		4396	} else if (INTEL_INFO(dev_priv)->gen == 3) {
3243	Serge	4397	dev->driver->irq_preinstall = i915_irq_preinstall;
		4398	dev->driver->irq_postinstall = i915_irq_postinstall;
4293	Serge	4399	dev->driver->irq_uninstall = i915_irq_uninstall;
3243	Serge	4400	dev->driver->irq_handler = i915_irq_handler;
3031	serge	4401	} else {
3243	Serge	4402	dev->driver->irq_preinstall = i965_irq_preinstall;
		4403	dev->driver->irq_postinstall = i965_irq_postinstall;
4293	Serge	4404	dev->driver->irq_uninstall = i965_irq_uninstall;
3243	Serge	4405	dev->driver->irq_handler = i965_irq_handler;
6084	serge	4406	}
		4407	if (I915_HAS_HOTPLUG(dev_priv))
3746	Serge	4408	dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4293	Serge	4409	dev->driver->enable_vblank = i915_enable_vblank;
		4410	dev->driver->disable_vblank = i915_disable_vblank;
2351	Serge	4411	}
3480	Serge	4412	}
3243	Serge	4413
5354	serge	4414	/**
		4415	* intel_irq_install - enables the hardware interrupt
		4416	* @dev_priv: i915 device instance
		4417	*
		4418	* This function enables the hardware interrupt handling, but leaves the hotplug
		4419	* handling still disabled. It is called after intel_irq_init().
		4420	*
		4421	* In the driver load and resume code we need working interrupts in a few places
		4422	* but don't want to deal with the hassle of concurrent probe and hotplug
		4423	* workers. Hence the split into this two-stage approach.
		4424	*/
		4425	int intel_irq_install(struct drm_i915_private *dev_priv)
3243	Serge	4426	{
5354	serge	4427	/*
		4428	* We enable some interrupt sources in our postinstall hooks, so mark
		4429	* interrupts as enabled _before_ actually enabling them to avoid
		4430	* special cases in our ordering checks.
		4431	*/
		4432	dev_priv->pm.irqs_enabled = true;
2351	Serge	4433
5354	serge	4434	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
3243	Serge	4435	}
		4436
5354	serge	4437	/**
		4438	* intel_irq_uninstall - finilizes all irq handling
		4439	* @dev_priv: i915 device instance
		4440	*
		4441	* This stops interrupt and hotplug handling and unregisters and frees all
		4442	* resources acquired in the init functions.
		4443	*/
		4444	void intel_irq_uninstall(struct drm_i915_private *dev_priv)
2351	Serge	4445	{
5354	serge	4446	// drm_irq_uninstall(dev_priv->dev);
		4447	// intel_hpd_cancel_work(dev_priv);
		4448	dev_priv->pm.irqs_enabled = false;
		4449	}
2351	Serge	4450
5354	serge	4451	/**
		4452	* intel_runtime_pm_disable_interrupts - runtime interrupt disabling
		4453	* @dev_priv: i915 device instance
		4454	*
		4455	* This function is used to disable interrupts at runtime, both in the runtime
		4456	* pm and the system suspend/resume code.
		4457	*/
		4458	void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
		4459	{
		4460	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
		4461	dev_priv->pm.irqs_enabled = false;
4104	Serge	4462	}
2351	Serge	4463
5354	serge	4464	/**
		4465	* intel_runtime_pm_enable_interrupts - runtime interrupt enabling
		4466	* @dev_priv: i915 device instance
		4467	*
		4468	* This function is used to enable interrupts at runtime, both in the runtime
		4469	* pm and the system suspend/resume code.
		4470	*/
		4471	void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
		4472	{
		4473	dev_priv->pm.irqs_enabled = true;
		4474	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
		4475	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
		4476	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/i915_irq.c – Rev 6131