WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	//#include "../../../platform/x86/intel_ips.h"
		32	#include
		33
4560	Serge	34
3031	serge	35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	void getrawmonotonic(struct timespec *ts);
		38
4560	Serge	39	/**
		40	* RC6 is a special power stage which allows the GPU to enter an very
		41	* low-voltage mode when idle, using down to 0V while at this stage. This
		42	* stage is entered automatically when the GPU is idle when RC6 support is
		43	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		44	*
		45	* There are different RC6 modes available in Intel GPU, which differentiate
		46	* among each other with the latency required to enter and leave RC6 and
		47	* voltage consumed by the GPU in different states.
		48	*
		49	* The combination of the following flags define which states GPU is allowed
		50	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		51	* RC6pp is deepest RC6. Their support by hardware varies according to the
		52	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		53	* which brings the most power savings; deeper states save more power, but
		54	* require higher latency to switch to and wake up.
		55	*/
		56	#define INTEL_RC6_ENABLE (1<<0)
		57	#define INTEL_RC6p_ENABLE (1<<1)
		58	#define INTEL_RC6pp_ENABLE (1<<2)
		59
6084	serge	60	static void bxt_init_clock_gating(struct drm_device *dev)
5354	serge	61	{
		62	struct drm_i915_private *dev_priv = dev->dev_private;
		63
6084	serge	64	/* WaDisableSDEUnitClockGating:bxt */
5354	serge	65	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		66	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		67
		68	/*
6084	serge	69	* FIXME:
		70	* GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
5354	serge	71	*/
6084	serge	72	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		73	GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
6937	serge	74
		75	/*
		76	* Wa: Backlight PWM may stop in the asserted state, causing backlight
		77	* to stay fully on.
		78	*/
		79	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
		80	I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) \|
		81	PWM1_GATING_DIS \| PWM2_GATING_DIS);
5354	serge	82	}
		83
3031	serge	84	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		85	{
5060	serge	86	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	87	u32 tmp;
		88
		89	tmp = I915_READ(CLKCFG);
		90
		91	switch (tmp & CLKCFG_FSB_MASK) {
		92	case CLKCFG_FSB_533:
		93	dev_priv->fsb_freq = 533; /* 1334 /
		94	break;
		95	case CLKCFG_FSB_800:
		96	dev_priv->fsb_freq = 800; /* 2004 /
		97	break;
		98	case CLKCFG_FSB_667:
		99	dev_priv->fsb_freq = 667; /* 1674 /
		100	break;
		101	case CLKCFG_FSB_400:
		102	dev_priv->fsb_freq = 400; /* 1004 /
		103	break;
		104	}
		105
		106	switch (tmp & CLKCFG_MEM_MASK) {
		107	case CLKCFG_MEM_533:
		108	dev_priv->mem_freq = 533;
		109	break;
		110	case CLKCFG_MEM_667:
		111	dev_priv->mem_freq = 667;
		112	break;
		113	case CLKCFG_MEM_800:
		114	dev_priv->mem_freq = 800;
		115	break;
		116	}
		117
		118	/* detect pineview DDR3 setting */
		119	tmp = I915_READ(CSHRDDR3CTL);
		120	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		121	}
		122
		123	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		124	{
5060	serge	125	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	126	u16 ddrpll, csipll;
		127
		128	ddrpll = I915_READ16(DDRMPLL1);
		129	csipll = I915_READ16(CSIPLL0);
		130
		131	switch (ddrpll & 0xff) {
		132	case 0xc:
		133	dev_priv->mem_freq = 800;
		134	break;
		135	case 0x10:
		136	dev_priv->mem_freq = 1066;
		137	break;
		138	case 0x14:
		139	dev_priv->mem_freq = 1333;
		140	break;
		141	case 0x18:
		142	dev_priv->mem_freq = 1600;
		143	break;
		144	default:
		145	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		146	ddrpll & 0xff);
		147	dev_priv->mem_freq = 0;
		148	break;
		149	}
		150
		151	dev_priv->ips.r_t = dev_priv->mem_freq;
		152
		153	switch (csipll & 0x3ff) {
		154	case 0x00c:
		155	dev_priv->fsb_freq = 3200;
		156	break;
		157	case 0x00e:
		158	dev_priv->fsb_freq = 3733;
		159	break;
		160	case 0x010:
		161	dev_priv->fsb_freq = 4266;
		162	break;
		163	case 0x012:
		164	dev_priv->fsb_freq = 4800;
		165	break;
		166	case 0x014:
		167	dev_priv->fsb_freq = 5333;
		168	break;
		169	case 0x016:
		170	dev_priv->fsb_freq = 5866;
		171	break;
		172	case 0x018:
		173	dev_priv->fsb_freq = 6400;
		174	break;
		175	default:
		176	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		177	csipll & 0x3ff);
		178	dev_priv->fsb_freq = 0;
		179	break;
		180	}
		181
		182	if (dev_priv->fsb_freq == 3200) {
		183	dev_priv->ips.c_m = 0;
		184	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		185	dev_priv->ips.c_m = 1;
		186	} else {
		187	dev_priv->ips.c_m = 2;
		188	}
		189	}
		190
		191	static const struct cxsr_latency cxsr_latency_table[] = {
		192	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		193	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		194	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		195	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		196	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		197
		198	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		199	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		200	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		201	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		202	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		203
		204	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		205	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		206	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		207	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		208	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		209
		210	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		211	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		212	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		213	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		214	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		215
		216	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		217	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		218	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		219	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		220	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		221
		222	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		223	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		224	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		225	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		226	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		227	};
		228
		229	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		230	int is_ddr3,
		231	int fsb,
		232	int mem)
		233	{
		234	const struct cxsr_latency *latency;
		235	int i;
		236
		237	if (fsb == 0 \|\| mem == 0)
		238	return NULL;
		239
		240	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		241	latency = &cxsr_latency_table[i];
		242	if (is_desktop == latency->is_desktop &&
		243	is_ddr3 == latency->is_ddr3 &&
		244	fsb == latency->fsb_freq && mem == latency->mem_freq)
		245	return latency;
		246	}
		247
		248	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		249
		250	return NULL;
		251	}
		252
6084	serge	253	static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
		254	{
		255	u32 val;
		256
		257	mutex_lock(&dev_priv->rps.hw_lock);
		258
		259	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		260	if (enable)
		261	val &= ~FORCE_DDR_HIGH_FREQ;
		262	else
		263	val \|= FORCE_DDR_HIGH_FREQ;
		264	val &= ~FORCE_DDR_LOW_FREQ;
		265	val \|= FORCE_DDR_FREQ_REQ_ACK;
		266	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		267
		268	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		269	FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
		270	DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
		271
		272	mutex_unlock(&dev_priv->rps.hw_lock);
		273	}
		274
		275	static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
		276	{
		277	u32 val;
		278
		279	mutex_lock(&dev_priv->rps.hw_lock);
		280
		281	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		282	if (enable)
		283	val \|= DSP_MAXFIFO_PM5_ENABLE;
		284	else
		285	val &= ~DSP_MAXFIFO_PM5_ENABLE;
		286	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
		287
		288	mutex_unlock(&dev_priv->rps.hw_lock);
		289	}
		290
		291	#define FW_WM(value, plane) \
		292	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
		293
5060	serge	294	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	295	{
5060	serge	296	struct drm_device *dev = dev_priv->dev;
		297	u32 val;
3031	serge	298
6937	serge	299	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev)) {
5060	serge	300	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
6084	serge	301	POSTING_READ(FW_BLC_SELF_VLV);
		302	dev_priv->wm.vlv.cxsr = enable;
5060	serge	303	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		304	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
6084	serge	305	POSTING_READ(FW_BLC_SELF);
5060	serge	306	} else if (IS_PINEVIEW(dev)) {
		307	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		308	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		309	I915_WRITE(DSPFW3, val);
6084	serge	310	POSTING_READ(DSPFW3);
5060	serge	311	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		312	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		313	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		314	I915_WRITE(FW_BLC_SELF, val);
6084	serge	315	POSTING_READ(FW_BLC_SELF);
5060	serge	316	} else if (IS_I915GM(dev)) {
		317	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		318	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		319	I915_WRITE(INSTPM, val);
6084	serge	320	POSTING_READ(INSTPM);
5060	serge	321	} else {
		322	return;
		323	}
		324
		325	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		326	enable ? "enabled" : "disabled");
3031	serge	327	}
		328
6084	serge	329
3031	serge	330	/*
		331	* Latency for FIFO fetches is dependent on several factors:
		332	* - memory configuration (speed, channels)
		333	* - chipset
		334	* - current MCH state
		335	* It can be fairly high in some situations, so here we assume a fairly
		336	* pessimal value. It's a tradeoff between extra memory fetches (if we
		337	* set this value too high, the FIFO will fetch frequently to stay full)
		338	* and power consumption (set it too low to save power and we might see
		339	* FIFO underruns and display "flicker").
		340	*
		341	* A value of 5us seems to be a good balance; safe for very low end
		342	* platforms but not overly aggressive on lower latency configs.
		343	*/
5354	serge	344	static const int pessimal_latency_ns = 5000;
3031	serge	345
6084	serge	346	#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
		347	((((dsparb) >> (lo_shift)) & 0xff) \| ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
		348
		349	static int vlv_get_fifo_size(struct drm_device *dev,
		350	enum pipe pipe, int plane)
		351	{
		352	struct drm_i915_private *dev_priv = dev->dev_private;
		353	int sprite0_start, sprite1_start, size;
		354
		355	switch (pipe) {
		356	uint32_t dsparb, dsparb2, dsparb3;
		357	case PIPE_A:
		358	dsparb = I915_READ(DSPARB);
		359	dsparb2 = I915_READ(DSPARB2);
		360	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		361	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		362	break;
		363	case PIPE_B:
		364	dsparb = I915_READ(DSPARB);
		365	dsparb2 = I915_READ(DSPARB2);
		366	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		367	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		368	break;
		369	case PIPE_C:
		370	dsparb2 = I915_READ(DSPARB2);
		371	dsparb3 = I915_READ(DSPARB3);
		372	sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		373	sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		374	break;
		375	default:
		376	return 0;
		377	}
		378
		379	switch (plane) {
		380	case 0:
		381	size = sprite0_start;
		382	break;
		383	case 1:
		384	size = sprite1_start - sprite0_start;
		385	break;
		386	case 2:
		387	size = 512 - 1 - sprite1_start;
		388	break;
		389	default:
		390	return 0;
		391	}
		392
		393	DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
		394	pipe_name(pipe), plane == 0 ? "primary" : "sprite",
		395	plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
		396	size);
		397
		398	return size;
		399	}
		400
3031	serge	401	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		402	{
		403	struct drm_i915_private *dev_priv = dev->dev_private;
		404	uint32_t dsparb = I915_READ(DSPARB);
		405	int size;
		406
		407	size = dsparb & 0x7f;
		408	if (plane)
		409	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		410
		411	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		412	plane ? "B" : "A", size);
		413
		414	return size;
		415	}
		416
4560	Serge	417	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	418	{
		419	struct drm_i915_private *dev_priv = dev->dev_private;
		420	uint32_t dsparb = I915_READ(DSPARB);
		421	int size;
		422
		423	size = dsparb & 0x1ff;
		424	if (plane)
		425	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		426	size >>= 1; /* Convert to cachelines */
		427
		428	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		429	plane ? "B" : "A", size);
		430
		431	return size;
		432	}
		433
		434	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		435	{
		436	struct drm_i915_private *dev_priv = dev->dev_private;
		437	uint32_t dsparb = I915_READ(DSPARB);
		438	int size;
		439
		440	size = dsparb & 0x7f;
		441	size >>= 2; /* Convert to cachelines */
		442
		443	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		444	plane ? "B" : "A",
		445	size);
		446
		447	return size;
		448	}
		449
		450	/* Pineview has different values for various configs */
		451	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	452	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		453	.max_wm = PINEVIEW_MAX_WM,
		454	.default_wm = PINEVIEW_DFT_WM,
		455	.guard_size = PINEVIEW_GUARD_WM,
		456	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	457	};
		458	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	459	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		460	.max_wm = PINEVIEW_MAX_WM,
		461	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		462	.guard_size = PINEVIEW_GUARD_WM,
		463	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	464	};
		465	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	466	.fifo_size = PINEVIEW_CURSOR_FIFO,
		467	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		468	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		469	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		470	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	471	};
		472	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	473	.fifo_size = PINEVIEW_CURSOR_FIFO,
		474	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		475	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		476	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		477	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	478	};
		479	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	480	.fifo_size = G4X_FIFO_SIZE,
		481	.max_wm = G4X_MAX_WM,
		482	.default_wm = G4X_MAX_WM,
		483	.guard_size = 2,
		484	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	485	};
		486	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	487	.fifo_size = I965_CURSOR_FIFO,
		488	.max_wm = I965_CURSOR_MAX_WM,
		489	.default_wm = I965_CURSOR_DFT_WM,
		490	.guard_size = 2,
		491	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	492	};
		493	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	494	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		495	.max_wm = VALLEYVIEW_MAX_WM,
		496	.default_wm = VALLEYVIEW_MAX_WM,
		497	.guard_size = 2,
		498	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	499	};
		500	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	501	.fifo_size = I965_CURSOR_FIFO,
		502	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		503	.default_wm = I965_CURSOR_DFT_WM,
		504	.guard_size = 2,
		505	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	506	};
		507	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	508	.fifo_size = I965_CURSOR_FIFO,
		509	.max_wm = I965_CURSOR_MAX_WM,
		510	.default_wm = I965_CURSOR_DFT_WM,
		511	.guard_size = 2,
		512	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	513	};
		514	static const struct intel_watermark_params i945_wm_info = {
5060	serge	515	.fifo_size = I945_FIFO_SIZE,
		516	.max_wm = I915_MAX_WM,
		517	.default_wm = 1,
		518	.guard_size = 2,
		519	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	520	};
		521	static const struct intel_watermark_params i915_wm_info = {
5060	serge	522	.fifo_size = I915_FIFO_SIZE,
		523	.max_wm = I915_MAX_WM,
		524	.default_wm = 1,
		525	.guard_size = 2,
		526	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	527	};
5354	serge	528	static const struct intel_watermark_params i830_a_wm_info = {
5060	serge	529	.fifo_size = I855GM_FIFO_SIZE,
		530	.max_wm = I915_MAX_WM,
		531	.default_wm = 1,
		532	.guard_size = 2,
		533	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	534	};
5354	serge	535	static const struct intel_watermark_params i830_bc_wm_info = {
		536	.fifo_size = I855GM_FIFO_SIZE,
		537	.max_wm = I915_MAX_WM/2,
		538	.default_wm = 1,
		539	.guard_size = 2,
		540	.cacheline_size = I830_FIFO_LINE_SIZE,
		541	};
4560	Serge	542	static const struct intel_watermark_params i845_wm_info = {
5060	serge	543	.fifo_size = I830_FIFO_SIZE,
		544	.max_wm = I915_MAX_WM,
		545	.default_wm = 1,
		546	.guard_size = 2,
		547	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	548	};
		549
		550	/**
		551	* intel_calculate_wm - calculate watermark level
		552	* @clock_in_khz: pixel clock
		553	* @wm: chip FIFO params
		554	* @pixel_size: display pixel size
		555	* @latency_ns: memory latency for the platform
		556	*
		557	* Calculate the watermark level (the level at which the display plane will
		558	* start fetching from memory again). Each chip has a different display
		559	* FIFO size and allocation, so the caller needs to figure that out and pass
		560	* in the correct intel_watermark_params structure.
		561	*
		562	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		563	* on the pixel size. When it reaches the watermark level, it'll start
		564	* fetching FIFO line sized based chunks from memory until the FIFO fills
		565	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		566	* will occur, and a display engine hang could result.
		567	*/
		568	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		569	const struct intel_watermark_params *wm,
		570	int fifo_size,
		571	int pixel_size,
		572	unsigned long latency_ns)
		573	{
		574	long entries_required, wm_size;
		575
		576	/*
		577	* Note: we need to make sure we don't overflow for various clock &
		578	* latency values.
		579	* clocks go from a few thousand to several hundred thousand.
		580	* latency is usually a few thousand
		581	*/
		582	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		583	1000;
		584	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		585
		586	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		587
		588	wm_size = fifo_size - (entries_required + wm->guard_size);
		589
		590	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		591
		592	/* Don't promote wm_size to unsigned... */
		593	if (wm_size > (long)wm->max_wm)
		594	wm_size = wm->max_wm;
		595	if (wm_size <= 0)
		596	wm_size = wm->default_wm;
5354	serge	597
		598	/*
		599	* Bspec seems to indicate that the value shouldn't be lower than
		600	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
		601	* Lets go for 8 which is the burst size since certain platforms
		602	* already use a hardcoded 8 (which is what the spec says should be
		603	* done).
		604	*/
		605	if (wm_size <= 8)
		606	wm_size = 8;
		607
3031	serge	608	return wm_size;
		609	}
		610
		611	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		612	{
		613	struct drm_crtc crtc, enabled = NULL;
		614
5060	serge	615	for_each_crtc(dev, crtc) {
3243	Serge	616	if (intel_crtc_active(crtc)) {
3031	serge	617	if (enabled)
		618	return NULL;
		619	enabled = crtc;
		620	}
		621	}
		622
		623	return enabled;
		624	}
		625
4560	Serge	626	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	627	{
4560	Serge	628	struct drm_device *dev = unused_crtc->dev;
3031	serge	629	struct drm_i915_private *dev_priv = dev->dev_private;
		630	struct drm_crtc *crtc;
		631	const struct cxsr_latency *latency;
		632	u32 reg;
		633	unsigned long wm;
		634
		635	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		636	dev_priv->fsb_freq, dev_priv->mem_freq);
		637	if (!latency) {
		638	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	639	intel_set_memory_cxsr(dev_priv, false);
3031	serge	640	return;
		641	}
		642
		643	crtc = single_enabled_crtc(dev);
		644	if (crtc) {
6084	serge	645	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
		646	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
		647	int clock = adjusted_mode->crtc_clock;
3031	serge	648
		649	/* Display SR */
		650	wm = intel_calculate_wm(clock, &pineview_display_wm,
		651	pineview_display_wm.fifo_size,
		652	pixel_size, latency->display_sr);
		653	reg = I915_READ(DSPFW1);
		654	reg &= ~DSPFW_SR_MASK;
6084	serge	655	reg \|= FW_WM(wm, SR);
3031	serge	656	I915_WRITE(DSPFW1, reg);
		657	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		658
		659	/* cursor SR */
		660	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		661	pineview_display_wm.fifo_size,
		662	pixel_size, latency->cursor_sr);
		663	reg = I915_READ(DSPFW3);
		664	reg &= ~DSPFW_CURSOR_SR_MASK;
6084	serge	665	reg \|= FW_WM(wm, CURSOR_SR);
3031	serge	666	I915_WRITE(DSPFW3, reg);
		667
		668	/* Display HPLL off SR */
		669	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		670	pineview_display_hplloff_wm.fifo_size,
		671	pixel_size, latency->display_hpll_disable);
		672	reg = I915_READ(DSPFW3);
		673	reg &= ~DSPFW_HPLL_SR_MASK;
6084	serge	674	reg \|= FW_WM(wm, HPLL_SR);
3031	serge	675	I915_WRITE(DSPFW3, reg);
		676
		677	/* cursor HPLL off SR */
		678	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		679	pineview_display_hplloff_wm.fifo_size,
		680	pixel_size, latency->cursor_hpll_disable);
		681	reg = I915_READ(DSPFW3);
		682	reg &= ~DSPFW_HPLL_CURSOR_MASK;
6084	serge	683	reg \|= FW_WM(wm, HPLL_CURSOR);
3031	serge	684	I915_WRITE(DSPFW3, reg);
		685	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		686
5060	serge	687	intel_set_memory_cxsr(dev_priv, true);
3031	serge	688	} else {
5060	serge	689	intel_set_memory_cxsr(dev_priv, false);
3031	serge	690	}
		691	}
		692
		693	static bool g4x_compute_wm0(struct drm_device *dev,
		694	int plane,
		695	const struct intel_watermark_params *display,
		696	int display_latency_ns,
		697	const struct intel_watermark_params *cursor,
		698	int cursor_latency_ns,
		699	int *plane_wm,
		700	int *cursor_wm)
		701	{
		702	struct drm_crtc *crtc;
4560	Serge	703	const struct drm_display_mode *adjusted_mode;
3031	serge	704	int htotal, hdisplay, clock, pixel_size;
		705	int line_time_us, line_count;
		706	int entries, tlb_miss;
		707
		708	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	709	if (!intel_crtc_active(crtc)) {
3031	serge	710	*cursor_wm = cursor->guard_size;
		711	*plane_wm = display->guard_size;
6084	serge	712	return false;
3031	serge	713	}
		714
6084	serge	715	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	716	clock = adjusted_mode->crtc_clock;
		717	htotal = adjusted_mode->crtc_htotal;
6084	serge	718	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		719	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	720
		721	/* Use the small buffer method to calculate plane watermark */
		722	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		723	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		724	if (tlb_miss > 0)
		725	entries += tlb_miss;
		726	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		727	*plane_wm = entries + display->guard_size;
		728	if (*plane_wm > (int)display->max_wm)
		729	*plane_wm = display->max_wm;
		730
		731	/* Use the large buffer method to calculate cursor watermark */
5060	serge	732	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	733	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
6084	serge	734	entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
3031	serge	735	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		736	if (tlb_miss > 0)
		737	entries += tlb_miss;
		738	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		739	*cursor_wm = entries + cursor->guard_size;
		740	if (*cursor_wm > (int)cursor->max_wm)
		741	*cursor_wm = (int)cursor->max_wm;
		742
		743	return true;
		744	}
		745
		746	/*
		747	* Check the wm result.
		748	*
		749	* If any calculated watermark values is larger than the maximum value that
		750	* can be programmed into the associated watermark register, that watermark
		751	* must be disabled.
		752	*/
		753	static bool g4x_check_srwm(struct drm_device *dev,
		754	int display_wm, int cursor_wm,
		755	const struct intel_watermark_params *display,
		756	const struct intel_watermark_params *cursor)
		757	{
		758	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		759	display_wm, cursor_wm);
		760
		761	if (display_wm > display->max_wm) {
		762	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		763	display_wm, display->max_wm);
		764	return false;
		765	}
		766
		767	if (cursor_wm > cursor->max_wm) {
		768	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		769	cursor_wm, cursor->max_wm);
		770	return false;
		771	}
		772
		773	if (!(display_wm \|\| cursor_wm)) {
		774	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		775	return false;
		776	}
		777
		778	return true;
		779	}
		780
		781	static bool g4x_compute_srwm(struct drm_device *dev,
		782	int plane,
		783	int latency_ns,
		784	const struct intel_watermark_params *display,
		785	const struct intel_watermark_params *cursor,
		786	int display_wm, int cursor_wm)
		787	{
		788	struct drm_crtc *crtc;
4560	Serge	789	const struct drm_display_mode *adjusted_mode;
3031	serge	790	int hdisplay, htotal, pixel_size, clock;
		791	unsigned long line_time_us;
		792	int line_count, line_size;
		793	int small, large;
		794	int entries;
		795
		796	if (!latency_ns) {
		797	display_wm = cursor_wm = 0;
		798	return false;
		799	}
		800
		801	crtc = intel_get_crtc_for_plane(dev, plane);
6084	serge	802	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	803	clock = adjusted_mode->crtc_clock;
		804	htotal = adjusted_mode->crtc_htotal;
6084	serge	805	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		806	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	807
5060	serge	808	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	809	line_count = (latency_ns / line_time_us + 1000) / 1000;
		810	line_size = hdisplay * pixel_size;
		811
		812	/* Use the minimum of the small and large buffer method for primary */
		813	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		814	large = line_count * line_size;
		815
		816	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		817	*display_wm = entries + display->guard_size;
		818
		819	/* calculate the self-refresh watermark for display cursor */
6084	serge	820	entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
3031	serge	821	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		822	*cursor_wm = entries + cursor->guard_size;
		823
		824	return g4x_check_srwm(dev,
		825	display_wm, cursor_wm,
		826	display, cursor);
		827	}
		828
6084	serge	829	#define FW_WM_VLV(value, plane) \
		830	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
		831
		832	static void vlv_write_wm_values(struct intel_crtc *crtc,
		833	const struct vlv_wm_values *wm)
3031	serge	834	{
6084	serge	835	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
		836	enum pipe pipe = crtc->pipe;
3031	serge	837
6084	serge	838	I915_WRITE(VLV_DDL(pipe),
		839	(wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) \|
		840	(wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) \|
		841	(wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) \|
		842	(wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
3031	serge	843
6084	serge	844	I915_WRITE(DSPFW1,
		845	FW_WM(wm->sr.plane, SR) \|
		846	FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) \|
		847	FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) \|
		848	FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
		849	I915_WRITE(DSPFW2,
		850	FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) \|
		851	FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) \|
		852	FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
		853	I915_WRITE(DSPFW3,
		854	FW_WM(wm->sr.cursor, CURSOR_SR));
3031	serge	855
6084	serge	856	if (IS_CHERRYVIEW(dev_priv)) {
		857	I915_WRITE(DSPFW7_CHV,
		858	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		859	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		860	I915_WRITE(DSPFW8_CHV,
		861	FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) \|
		862	FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
		863	I915_WRITE(DSPFW9_CHV,
		864	FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) \|
		865	FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
		866	I915_WRITE(DSPHOWM,
		867	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		868	FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) \|
		869	FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) \|
		870	FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) \|
		871	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		872	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		873	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		874	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		875	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		876	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		877	} else {
		878	I915_WRITE(DSPFW7,
		879	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		880	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		881	I915_WRITE(DSPHOWM,
		882	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		883	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		884	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		885	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		886	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		887	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		888	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		889	}
3031	serge	890
6084	serge	891	/* zero (unused) WM1 watermarks */
		892	I915_WRITE(DSPFW4, 0);
		893	I915_WRITE(DSPFW5, 0);
		894	I915_WRITE(DSPFW6, 0);
		895	I915_WRITE(DSPHOWM1, 0);
3031	serge	896
6084	serge	897	POSTING_READ(DSPFW1);
3031	serge	898	}
		899
6084	serge	900	#undef FW_WM_VLV
3031	serge	901
6084	serge	902	enum vlv_wm_level {
		903	VLV_WM_LEVEL_PM2,
		904	VLV_WM_LEVEL_PM5,
		905	VLV_WM_LEVEL_DDR_DVFS,
		906	};
		907
		908	/* latency must be in 0.1us units. */
		909	static unsigned int vlv_wm_method2(unsigned int pixel_rate,
		910	unsigned int pipe_htotal,
		911	unsigned int horiz_pixels,
		912	unsigned int bytes_per_pixel,
		913	unsigned int latency)
3031	serge	914	{
6084	serge	915	unsigned int ret;
		916
		917	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		918	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		919	ret = DIV_ROUND_UP(ret, 64);
		920
		921	return ret;
		922	}
		923
		924	static void vlv_setup_wm_latency(struct drm_device *dev)
		925	{
3031	serge	926	struct drm_i915_private *dev_priv = dev->dev_private;
		927
6084	serge	928	/* all latencies in usec */
		929	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
3031	serge	930
6084	serge	931	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
		932
		933	if (IS_CHERRYVIEW(dev_priv)) {
		934	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		935	dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
		936
		937	dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
3031	serge	938	}
6084	serge	939	}
3031	serge	940
6084	serge	941	static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
		942	struct intel_crtc *crtc,
		943	const struct intel_plane_state *state,
		944	int level)
		945	{
		946	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
		947	int clock, htotal, pixel_size, width, wm;
		948
		949	if (dev_priv->wm.pri_latency[level] == 0)
		950	return USHRT_MAX;
		951
		952	if (!state->visible)
		953	return 0;
		954
		955	pixel_size = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		956	clock = crtc->config->base.adjusted_mode.crtc_clock;
		957	htotal = crtc->config->base.adjusted_mode.crtc_htotal;
		958	width = crtc->config->pipe_src_w;
		959	if (WARN_ON(htotal == 0))
		960	htotal = 1;
		961
		962	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		963	/*
		964	* FIXME the formula gives values that are
		965	* too big for the cursor FIFO, and hence we
		966	* would never be able to use cursors. For
		967	* now just hardcode the watermark.
		968	*/
		969	wm = 63;
		970	} else {
		971	wm = vlv_wm_method2(clock, htotal, width, pixel_size,
		972	dev_priv->wm.pri_latency[level] * 10);
5354	serge	973	}
3031	serge	974
6084	serge	975	return min_t(int, wm, USHRT_MAX);
		976	}
5354	serge	977
6084	serge	978	static void vlv_compute_fifo(struct intel_crtc *crtc)
		979	{
		980	struct drm_device *dev = crtc->base.dev;
		981	struct vlv_wm_state *wm_state = &crtc->wm_state;
		982	struct intel_plane *plane;
		983	unsigned int total_rate = 0;
		984	const int fifo_size = 512 - 1;
		985	int fifo_extra, fifo_left = fifo_size;
		986
		987	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		988	struct intel_plane_state *state =
		989	to_intel_plane_state(plane->base.state);
		990
		991	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		992	continue;
		993
		994	if (state->visible) {
		995	wm_state->num_active_planes++;
		996	total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		997	}
3031	serge	998	}
5354	serge	999
6084	serge	1000	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1001	struct intel_plane_state *state =
		1002	to_intel_plane_state(plane->base.state);
		1003	unsigned int rate;
		1004
		1005	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1006	plane->wm.fifo_size = 63;
		1007	continue;
		1008	}
		1009
		1010	if (!state->visible) {
		1011	plane->wm.fifo_size = 0;
		1012	continue;
		1013	}
		1014
		1015	rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		1016	plane->wm.fifo_size = fifo_size * rate / total_rate;
		1017	fifo_left -= plane->wm.fifo_size;
		1018	}
		1019
		1020	fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
		1021
		1022	/* spread the remainder evenly */
		1023	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1024	int plane_extra;
		1025
		1026	if (fifo_left == 0)
		1027	break;
		1028
		1029	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		1030	continue;
		1031
		1032	/* give it all to the first plane if none are active */
		1033	if (plane->wm.fifo_size == 0 &&
		1034	wm_state->num_active_planes)
		1035	continue;
		1036
		1037	plane_extra = min(fifo_extra, fifo_left);
		1038	plane->wm.fifo_size += plane_extra;
		1039	fifo_left -= plane_extra;
		1040	}
		1041
		1042	WARN_ON(fifo_left != 0);
3031	serge	1043	}
		1044
6084	serge	1045	static void vlv_invert_wms(struct intel_crtc *crtc)
		1046	{
		1047	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1048	int level;
3031	serge	1049
6084	serge	1050	for (level = 0; level < wm_state->num_levels; level++) {
		1051	struct drm_device *dev = crtc->base.dev;
		1052	const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1053	struct intel_plane *plane;
		1054
		1055	wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
		1056	wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
		1057
		1058	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1059	switch (plane->base.type) {
		1060	int sprite;
		1061	case DRM_PLANE_TYPE_CURSOR:
		1062	wm_state->wm[level].cursor = plane->wm.fifo_size -
		1063	wm_state->wm[level].cursor;
		1064	break;
		1065	case DRM_PLANE_TYPE_PRIMARY:
		1066	wm_state->wm[level].primary = plane->wm.fifo_size -
		1067	wm_state->wm[level].primary;
		1068	break;
		1069	case DRM_PLANE_TYPE_OVERLAY:
		1070	sprite = plane->plane;
		1071	wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
		1072	wm_state->wm[level].sprite[sprite];
		1073	break;
		1074	}
		1075	}
		1076	}
		1077	}
		1078
		1079	static void vlv_compute_wm(struct intel_crtc *crtc)
3031	serge	1080	{
6084	serge	1081	struct drm_device *dev = crtc->base.dev;
		1082	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1083	struct intel_plane *plane;
		1084	int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1085	int level;
3031	serge	1086
6084	serge	1087	memset(wm_state, 0, sizeof(*wm_state));
3031	serge	1088
6084	serge	1089	wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
		1090	wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
3031	serge	1091
6084	serge	1092	wm_state->num_active_planes = 0;
3031	serge	1093
6084	serge	1094	vlv_compute_fifo(crtc);
		1095
		1096	if (wm_state->num_active_planes != 1)
		1097	wm_state->cxsr = false;
		1098
		1099	if (wm_state->cxsr) {
		1100	for (level = 0; level < wm_state->num_levels; level++) {
		1101	wm_state->sr[level].plane = sr_fifo_size;
		1102	wm_state->sr[level].cursor = 63;
		1103	}
3243	Serge	1104	}
3031	serge	1105
6084	serge	1106	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1107	struct intel_plane_state *state =
		1108	to_intel_plane_state(plane->base.state);
3031	serge	1109
6084	serge	1110	if (!state->visible)
		1111	continue;
5060	serge	1112
6084	serge	1113	/* normal watermarks */
		1114	for (level = 0; level < wm_state->num_levels; level++) {
		1115	int wm = vlv_compute_wm_level(plane, crtc, state, level);
		1116	int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
		1117
		1118	/* hack */
		1119	if (WARN_ON(level == 0 && wm > max_wm))
		1120	wm = max_wm;
		1121
		1122	if (wm > plane->wm.fifo_size)
		1123	break;
		1124
		1125	switch (plane->base.type) {
		1126	int sprite;
		1127	case DRM_PLANE_TYPE_CURSOR:
		1128	wm_state->wm[level].cursor = wm;
		1129	break;
		1130	case DRM_PLANE_TYPE_PRIMARY:
		1131	wm_state->wm[level].primary = wm;
		1132	break;
		1133	case DRM_PLANE_TYPE_OVERLAY:
		1134	sprite = plane->plane;
		1135	wm_state->wm[level].sprite[sprite] = wm;
		1136	break;
		1137	}
		1138	}
		1139
		1140	wm_state->num_levels = level;
		1141
		1142	if (!wm_state->cxsr)
		1143	continue;
		1144
		1145	/* maxfifo watermarks */
		1146	switch (plane->base.type) {
		1147	int sprite, level;
		1148	case DRM_PLANE_TYPE_CURSOR:
		1149	for (level = 0; level < wm_state->num_levels; level++)
		1150	wm_state->sr[level].cursor =
		1151	wm_state->wm[level].cursor;
		1152	break;
		1153	case DRM_PLANE_TYPE_PRIMARY:
		1154	for (level = 0; level < wm_state->num_levels; level++)
		1155	wm_state->sr[level].plane =
		1156	min(wm_state->sr[level].plane,
		1157	wm_state->wm[level].primary);
		1158	break;
		1159	case DRM_PLANE_TYPE_OVERLAY:
		1160	sprite = plane->plane;
		1161	for (level = 0; level < wm_state->num_levels; level++)
		1162	wm_state->sr[level].plane =
		1163	min(wm_state->sr[level].plane,
		1164	wm_state->wm[level].sprite[sprite]);
		1165	break;
		1166	}
		1167	}
		1168
		1169	/* clear any (partially) filled invalid levels */
		1170	for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
		1171	memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
		1172	memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
		1173	}
		1174
		1175	vlv_invert_wms(crtc);
3031	serge	1176	}
		1177
6084	serge	1178	#define VLV_FIFO(plane, value) \
		1179	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
		1180
		1181	static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
5354	serge	1182	{
6084	serge	1183	struct drm_device *dev = crtc->base.dev;
		1184	struct drm_i915_private *dev_priv = to_i915(dev);
		1185	struct intel_plane *plane;
		1186	int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
5354	serge	1187
6084	serge	1188	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1189	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1190	WARN_ON(plane->wm.fifo_size != 63);
		1191	continue;
		1192	}
5354	serge	1193
6084	serge	1194	if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		1195	sprite0_start = plane->wm.fifo_size;
		1196	else if (plane->plane == 0)
		1197	sprite1_start = sprite0_start + plane->wm.fifo_size;
		1198	else
		1199	fifo_size = sprite1_start + plane->wm.fifo_size;
		1200	}
5354	serge	1201
6084	serge	1202	WARN_ON(fifo_size != 512 - 1);
5354	serge	1203
6084	serge	1204	DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
		1205	pipe_name(crtc->pipe), sprite0_start,
		1206	sprite1_start, fifo_size);
5354	serge	1207
6084	serge	1208	switch (crtc->pipe) {
		1209	uint32_t dsparb, dsparb2, dsparb3;
		1210	case PIPE_A:
		1211	dsparb = I915_READ(DSPARB);
		1212	dsparb2 = I915_READ(DSPARB2);
		1213
		1214	dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) \|
		1215	VLV_FIFO(SPRITEB, 0xff));
		1216	dsparb \|= (VLV_FIFO(SPRITEA, sprite0_start) \|
		1217	VLV_FIFO(SPRITEB, sprite1_start));
		1218
		1219	dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) \|
		1220	VLV_FIFO(SPRITEB_HI, 0x1));
		1221	dsparb2 \|= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) \|
		1222	VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
		1223
		1224	I915_WRITE(DSPARB, dsparb);
		1225	I915_WRITE(DSPARB2, dsparb2);
		1226	break;
		1227	case PIPE_B:
		1228	dsparb = I915_READ(DSPARB);
		1229	dsparb2 = I915_READ(DSPARB2);
		1230
		1231	dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) \|
		1232	VLV_FIFO(SPRITED, 0xff));
		1233	dsparb \|= (VLV_FIFO(SPRITEC, sprite0_start) \|
		1234	VLV_FIFO(SPRITED, sprite1_start));
		1235
		1236	dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) \|
		1237	VLV_FIFO(SPRITED_HI, 0xff));
		1238	dsparb2 \|= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) \|
		1239	VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
		1240
		1241	I915_WRITE(DSPARB, dsparb);
		1242	I915_WRITE(DSPARB2, dsparb2);
		1243	break;
		1244	case PIPE_C:
		1245	dsparb3 = I915_READ(DSPARB3);
		1246	dsparb2 = I915_READ(DSPARB2);
		1247
		1248	dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) \|
		1249	VLV_FIFO(SPRITEF, 0xff));
		1250	dsparb3 \|= (VLV_FIFO(SPRITEE, sprite0_start) \|
		1251	VLV_FIFO(SPRITEF, sprite1_start));
		1252
		1253	dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) \|
		1254	VLV_FIFO(SPRITEF_HI, 0xff));
		1255	dsparb2 \|= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) \|
		1256	VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
		1257
		1258	I915_WRITE(DSPARB3, dsparb3);
		1259	I915_WRITE(DSPARB2, dsparb2);
		1260	break;
		1261	default:
		1262	break;
5354	serge	1263	}
6084	serge	1264	}
5354	serge	1265
6084	serge	1266	#undef VLV_FIFO
5354	serge	1267
6084	serge	1268	static void vlv_merge_wm(struct drm_device *dev,
		1269	struct vlv_wm_values *wm)
		1270	{
		1271	struct intel_crtc *crtc;
		1272	int num_active_crtcs = 0;
5354	serge	1273
6084	serge	1274	wm->level = to_i915(dev)->wm.max_level;
		1275	wm->cxsr = true;
		1276
		1277	for_each_intel_crtc(dev, crtc) {
		1278	const struct vlv_wm_state *wm_state = &crtc->wm_state;
		1279
		1280	if (!crtc->active)
		1281	continue;
		1282
		1283	if (!wm_state->cxsr)
		1284	wm->cxsr = false;
		1285
		1286	num_active_crtcs++;
		1287	wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
		1288	}
		1289
		1290	if (num_active_crtcs != 1)
		1291	wm->cxsr = false;
		1292
		1293	if (num_active_crtcs > 1)
		1294	wm->level = VLV_WM_LEVEL_PM2;
		1295
		1296	for_each_intel_crtc(dev, crtc) {
		1297	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1298	enum pipe pipe = crtc->pipe;
		1299
		1300	if (!crtc->active)
		1301	continue;
		1302
		1303	wm->pipe[pipe] = wm_state->wm[wm->level];
		1304	if (wm->cxsr)
		1305	wm->sr = wm_state->sr[wm->level];
		1306
		1307	wm->ddl[pipe].primary = DDL_PRECISION_HIGH \| 2;
		1308	wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH \| 2;
		1309	wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH \| 2;
		1310	wm->ddl[pipe].cursor = DDL_PRECISION_HIGH \| 2;
		1311	}
5354	serge	1312	}
		1313
6084	serge	1314	static void vlv_update_wm(struct drm_crtc *crtc)
5354	serge	1315	{
		1316	struct drm_device *dev = crtc->dev;
		1317	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1318	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1319	enum pipe pipe = intel_crtc->pipe;
		1320	struct vlv_wm_values wm = {};
5354	serge	1321
6084	serge	1322	vlv_compute_wm(intel_crtc);
		1323	vlv_merge_wm(dev, &wm);
5354	serge	1324
6084	serge	1325	if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
		1326	/* FIXME should be part of crtc atomic commit */
		1327	vlv_pipe_set_fifo_size(intel_crtc);
		1328	return;
5354	serge	1329	}
		1330
6084	serge	1331	if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
		1332	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
		1333	chv_set_memory_dvfs(dev_priv, false);
		1334
		1335	if (wm.level < VLV_WM_LEVEL_PM5 &&
		1336	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
		1337	chv_set_memory_pm5(dev_priv, false);
		1338
		1339	if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
		1340	intel_set_memory_cxsr(dev_priv, false);
		1341
		1342	/* FIXME should be part of crtc atomic commit */
		1343	vlv_pipe_set_fifo_size(intel_crtc);
		1344
		1345	vlv_write_wm_values(intel_crtc, &wm);
		1346
		1347	DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
		1348	"sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
		1349	pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
		1350	wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
		1351	wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
		1352
		1353	if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
		1354	intel_set_memory_cxsr(dev_priv, true);
		1355
		1356	if (wm.level >= VLV_WM_LEVEL_PM5 &&
		1357	dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
		1358	chv_set_memory_pm5(dev_priv, true);
		1359
		1360	if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
		1361	dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
		1362	chv_set_memory_dvfs(dev_priv, true);
		1363
		1364	dev_priv->wm.vlv = wm;
5354	serge	1365	}
		1366
6084	serge	1367	#define single_plane_enabled(mask) is_power_of_2(mask)
		1368
4560	Serge	1369	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1370	{
4560	Serge	1371	struct drm_device *dev = crtc->dev;
3031	serge	1372	static const int sr_latency_ns = 12000;
		1373	struct drm_i915_private *dev_priv = dev->dev_private;
		1374	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1375	int plane_sr, cursor_sr;
		1376	unsigned int enabled = 0;
5060	serge	1377	bool cxsr_enabled;
3031	serge	1378
3746	Serge	1379	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1380	&g4x_wm_info, pessimal_latency_ns,
		1381	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1382	&planea_wm, &cursora_wm))
3746	Serge	1383	enabled \|= 1 << PIPE_A;
3031	serge	1384
3746	Serge	1385	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1386	&g4x_wm_info, pessimal_latency_ns,
		1387	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1388	&planeb_wm, &cursorb_wm))
3746	Serge	1389	enabled \|= 1 << PIPE_B;
3031	serge	1390
		1391	if (single_plane_enabled(enabled) &&
		1392	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1393	sr_latency_ns,
		1394	&g4x_wm_info,
		1395	&g4x_cursor_wm_info,
3243	Serge	1396	&plane_sr, &cursor_sr)) {
5060	serge	1397	cxsr_enabled = true;
3243	Serge	1398	} else {
5060	serge	1399	cxsr_enabled = false;
		1400	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1401	plane_sr = cursor_sr = 0;
		1402	}
3031	serge	1403
5354	serge	1404	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1405	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1406	planea_wm, cursora_wm,
		1407	planeb_wm, cursorb_wm,
		1408	plane_sr, cursor_sr);
		1409
		1410	I915_WRITE(DSPFW1,
6084	serge	1411	FW_WM(plane_sr, SR) \|
		1412	FW_WM(cursorb_wm, CURSORB) \|
		1413	FW_WM(planeb_wm, PLANEB) \|
		1414	FW_WM(planea_wm, PLANEA));
3031	serge	1415	I915_WRITE(DSPFW2,
3243	Serge	1416	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
6084	serge	1417	FW_WM(cursora_wm, CURSORA));
3031	serge	1418	/* HPLL off in SR has some issues on G4x... disable it */
		1419	I915_WRITE(DSPFW3,
3243	Serge	1420	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
6084	serge	1421	FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1422
		1423	if (cxsr_enabled)
		1424	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1425	}
		1426
4560	Serge	1427	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1428	{
4560	Serge	1429	struct drm_device *dev = unused_crtc->dev;
3031	serge	1430	struct drm_i915_private *dev_priv = dev->dev_private;
		1431	struct drm_crtc *crtc;
		1432	int srwm = 1;
		1433	int cursor_sr = 16;
5060	serge	1434	bool cxsr_enabled;
3031	serge	1435
		1436	/* Calc sr entries for one plane configs */
		1437	crtc = single_enabled_crtc(dev);
		1438	if (crtc) {
		1439	/* self-refresh has much higher latency */
		1440	static const int sr_latency_ns = 12000;
6084	serge	1441	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1442	int clock = adjusted_mode->crtc_clock;
		1443	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1444	int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		1445	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	1446	unsigned long line_time_us;
		1447	int entries;
		1448
5060	serge	1449	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1450
		1451	/* Use ns/us then divide to preserve precision */
		1452	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1453	pixel_size * hdisplay;
		1454	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1455	srwm = I965_FIFO_SIZE - entries;
		1456	if (srwm < 0)
		1457	srwm = 1;
		1458	srwm &= 0x1ff;
		1459	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1460	entries, srwm);
		1461
		1462	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
6084	serge	1463	pixel_size * crtc->cursor->state->crtc_w;
3031	serge	1464	entries = DIV_ROUND_UP(entries,
		1465	i965_cursor_wm_info.cacheline_size);
		1466	cursor_sr = i965_cursor_wm_info.fifo_size -
		1467	(entries + i965_cursor_wm_info.guard_size);
		1468
		1469	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1470	cursor_sr = i965_cursor_wm_info.max_wm;
		1471
		1472	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1473	"cursor %d\n", srwm, cursor_sr);
		1474
5060	serge	1475	cxsr_enabled = true;
3031	serge	1476	} else {
5060	serge	1477	cxsr_enabled = false;
3031	serge	1478	/* Turn off self refresh if both pipes are enabled */
5060	serge	1479	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1480	}
		1481
		1482	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1483	srwm);
		1484
		1485	/* 965 has limitations... */
6084	serge	1486	I915_WRITE(DSPFW1, FW_WM(srwm, SR) \|
		1487	FW_WM(8, CURSORB) \|
		1488	FW_WM(8, PLANEB) \|
		1489	FW_WM(8, PLANEA));
		1490	I915_WRITE(DSPFW2, FW_WM(8, CURSORA) \|
		1491	FW_WM(8, PLANEC_OLD));
3031	serge	1492	/* update cursor SR watermark */
6084	serge	1493	I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1494
		1495	if (cxsr_enabled)
		1496	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1497	}
		1498
6084	serge	1499	#undef FW_WM
		1500
4560	Serge	1501	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1502	{
4560	Serge	1503	struct drm_device *dev = unused_crtc->dev;
3031	serge	1504	struct drm_i915_private *dev_priv = dev->dev_private;
		1505	const struct intel_watermark_params *wm_info;
		1506	uint32_t fwater_lo;
		1507	uint32_t fwater_hi;
		1508	int cwm, srwm = 1;
		1509	int fifo_size;
		1510	int planea_wm, planeb_wm;
		1511	struct drm_crtc crtc, enabled = NULL;
		1512
		1513	if (IS_I945GM(dev))
		1514	wm_info = &i945_wm_info;
		1515	else if (!IS_GEN2(dev))
		1516	wm_info = &i915_wm_info;
		1517	else
5354	serge	1518	wm_info = &i830_a_wm_info;
3031	serge	1519
		1520	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1521	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1522	if (intel_crtc_active(crtc)) {
4560	Serge	1523	const struct drm_display_mode *adjusted_mode;
6084	serge	1524	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1525	if (IS_GEN2(dev))
		1526	cpp = 4;
		1527
6084	serge	1528	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1529	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1530	wm_info, fifo_size, cpp,
5354	serge	1531	pessimal_latency_ns);
3031	serge	1532	enabled = crtc;
5354	serge	1533	} else {
3031	serge	1534	planea_wm = fifo_size - wm_info->guard_size;
5354	serge	1535	if (planea_wm > (long)wm_info->max_wm)
		1536	planea_wm = wm_info->max_wm;
		1537	}
3031	serge	1538
5354	serge	1539	if (IS_GEN2(dev))
		1540	wm_info = &i830_bc_wm_info;
		1541
3031	serge	1542	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1543	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1544	if (intel_crtc_active(crtc)) {
4560	Serge	1545	const struct drm_display_mode *adjusted_mode;
6084	serge	1546	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1547	if (IS_GEN2(dev))
		1548	cpp = 4;
		1549
6084	serge	1550	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1551	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1552	wm_info, fifo_size, cpp,
5354	serge	1553	pessimal_latency_ns);
3031	serge	1554	if (enabled == NULL)
		1555	enabled = crtc;
		1556	else
		1557	enabled = NULL;
5354	serge	1558	} else {
3031	serge	1559	planeb_wm = fifo_size - wm_info->guard_size;
5354	serge	1560	if (planeb_wm > (long)wm_info->max_wm)
		1561	planeb_wm = wm_info->max_wm;
		1562	}
3031	serge	1563
		1564	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1565
5060	serge	1566	if (IS_I915GM(dev) && enabled) {
		1567	struct drm_i915_gem_object *obj;
		1568
6084	serge	1569	obj = intel_fb_obj(enabled->primary->state->fb);
5060	serge	1570
		1571	/* self-refresh seems busted with untiled */
		1572	if (obj->tiling_mode == I915_TILING_NONE)
		1573	enabled = NULL;
		1574	}
		1575
3031	serge	1576	/*
		1577	* Overlay gets an aggressive default since video jitter is bad.
		1578	*/
		1579	cwm = 2;
		1580
		1581	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1582	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1583
		1584	/* Calc sr entries for one plane configs */
		1585	if (HAS_FW_BLC(dev) && enabled) {
		1586	/* self-refresh has much higher latency */
		1587	static const int sr_latency_ns = 6000;
6084	serge	1588	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
4560	Serge	1589	int clock = adjusted_mode->crtc_clock;
		1590	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1591	int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
		1592	int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
3031	serge	1593	unsigned long line_time_us;
		1594	int entries;
		1595
5060	serge	1596	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1597
		1598	/* Use ns/us then divide to preserve precision */
		1599	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1600	pixel_size * hdisplay;
		1601	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1602	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1603	srwm = wm_info->fifo_size - entries;
		1604	if (srwm < 0)
		1605	srwm = 1;
		1606
		1607	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1608	I915_WRITE(FW_BLC_SELF,
		1609	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1610	else if (IS_I915GM(dev))
		1611	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1612	}
		1613
		1614	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1615	planea_wm, planeb_wm, cwm, srwm);
		1616
		1617	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1618	fwater_hi = (cwm & 0x1f);
		1619
		1620	/* Set request length to 8 cachelines per fetch */
		1621	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1622	fwater_hi = fwater_hi \| (1 << 8);
		1623
		1624	I915_WRITE(FW_BLC, fwater_lo);
		1625	I915_WRITE(FW_BLC2, fwater_hi);
		1626
5060	serge	1627	if (enabled)
		1628	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1629	}
		1630
4560	Serge	1631	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1632	{
4560	Serge	1633	struct drm_device *dev = unused_crtc->dev;
3031	serge	1634	struct drm_i915_private *dev_priv = dev->dev_private;
		1635	struct drm_crtc *crtc;
4560	Serge	1636	const struct drm_display_mode *adjusted_mode;
3031	serge	1637	uint32_t fwater_lo;
		1638	int planea_wm;
		1639
		1640	crtc = single_enabled_crtc(dev);
		1641	if (crtc == NULL)
		1642	return;
		1643
6084	serge	1644	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1645	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1646	&i845_wm_info,
3031	serge	1647	dev_priv->display.get_fifo_size(dev, 0),
5354	serge	1648	4, pessimal_latency_ns);
3031	serge	1649	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1650	fwater_lo \|= (3<<8) \| planea_wm;
		1651
		1652	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1653
		1654	I915_WRITE(FW_BLC, fwater_lo);
		1655	}
		1656
6084	serge	1657	uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
3031	serge	1658	{
4104	Serge	1659	uint32_t pixel_rate;
		1660
6084	serge	1661	pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
4104	Serge	1662
		1663	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1664	* adjust the pixel_rate here. */
		1665
6084	serge	1666	if (pipe_config->pch_pfit.enabled) {
4104	Serge	1667	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6084	serge	1668	uint32_t pfit_size = pipe_config->pch_pfit.size;
4104	Serge	1669
6084	serge	1670	pipe_w = pipe_config->pipe_src_w;
		1671	pipe_h = pipe_config->pipe_src_h;
		1672
4104	Serge	1673	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1674	pfit_h = pfit_size & 0xFFFF;
		1675	if (pipe_w < pfit_w)
		1676	pipe_w = pfit_w;
		1677	if (pipe_h < pfit_h)
		1678	pipe_h = pfit_h;
		1679
		1680	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1681	pfit_w * pfit_h);
		1682	}
		1683
		1684	return pixel_rate;
		1685	}
		1686
		1687	/* latency must be in 0.1us units. */
		1688	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1689	uint32_t latency)
		1690	{
		1691	uint64_t ret;
		1692
		1693	if (WARN(latency == 0, "Latency value missing\n"))
		1694	return UINT_MAX;
		1695
		1696	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1697	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1698
		1699	return ret;
		1700	}
		1701
		1702	/* latency must be in 0.1us units. */
		1703	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1704	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1705	uint32_t latency)
		1706	{
		1707	uint32_t ret;
		1708
		1709	if (WARN(latency == 0, "Latency value missing\n"))
		1710	return UINT_MAX;
		1711
		1712	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1713	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1714	ret = DIV_ROUND_UP(ret, 64) + 2;
		1715	return ret;
		1716	}
		1717
		1718	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1719	uint8_t bytes_per_pixel)
		1720	{
		1721	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1722	}
		1723
4560	Serge	1724	struct ilk_wm_maximums {
4104	Serge	1725	uint16_t pri;
		1726	uint16_t spr;
		1727	uint16_t cur;
		1728	uint16_t fbc;
		1729	};
		1730
		1731	/*
		1732	* For both WM_PIPE and WM_LP.
		1733	* mem_value must be in 0.1us units.
		1734	*/
6084	serge	1735	static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
		1736	const struct intel_plane_state *pstate,
4104	Serge	1737	uint32_t mem_value,
		1738	bool is_lp)
		1739	{
6084	serge	1740	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1741	uint32_t method1, method2;
		1742
6084	serge	1743	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1744	return 0;
		1745
6084	serge	1746	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
4104	Serge	1747
		1748	if (!is_lp)
		1749	return method1;
		1750
6084	serge	1751	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1752	cstate->base.adjusted_mode.crtc_htotal,
		1753	drm_rect_width(&pstate->dst),
		1754	bpp,
4104	Serge	1755	mem_value);
		1756
		1757	return min(method1, method2);
		1758	}
		1759
		1760	/*
		1761	* For both WM_PIPE and WM_LP.
		1762	* mem_value must be in 0.1us units.
		1763	*/
6084	serge	1764	static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
		1765	const struct intel_plane_state *pstate,
4104	Serge	1766	uint32_t mem_value)
		1767	{
6084	serge	1768	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1769	uint32_t method1, method2;
		1770
6084	serge	1771	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1772	return 0;
		1773
6084	serge	1774	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
		1775	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1776	cstate->base.adjusted_mode.crtc_htotal,
		1777	drm_rect_width(&pstate->dst),
		1778	bpp,
4104	Serge	1779	mem_value);
		1780	return min(method1, method2);
		1781	}
		1782
		1783	/*
		1784	* For both WM_PIPE and WM_LP.
		1785	* mem_value must be in 0.1us units.
		1786	*/
6084	serge	1787	static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
		1788	const struct intel_plane_state *pstate,
4104	Serge	1789	uint32_t mem_value)
		1790	{
6660	serge	1791	/*
		1792	* We treat the cursor plane as always-on for the purposes of watermark
		1793	* calculation. Until we have two-stage watermark programming merged,
		1794	* this is necessary to avoid flickering.
		1795	*/
		1796	int cpp = 4;
		1797	int width = pstate->visible ? pstate->base.crtc_w : 64;
6084	serge	1798
6660	serge	1799	if (!cstate->base.active)
4104	Serge	1800	return 0;
		1801
6084	serge	1802	return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1803	cstate->base.adjusted_mode.crtc_htotal,
6660	serge	1804	width, cpp, mem_value);
4104	Serge	1805	}
		1806
		1807	/* Only for WM_LP. */
6084	serge	1808	static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
		1809	const struct intel_plane_state *pstate,
4104	Serge	1810	uint32_t pri_val)
		1811	{
6084	serge	1812	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
		1813
		1814	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1815	return 0;
		1816
6084	serge	1817	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), bpp);
4104	Serge	1818	}
		1819
		1820	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1821	{
4560	Serge	1822	if (INTEL_INFO(dev)->gen >= 8)
		1823	return 3072;
		1824	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1825	return 768;
		1826	else
		1827	return 512;
		1828	}
		1829
5060	serge	1830	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		1831	int level, bool is_sprite)
		1832	{
		1833	if (INTEL_INFO(dev)->gen >= 8)
		1834	/* BDW primary/sprite plane watermarks */
		1835	return level == 0 ? 255 : 2047;
		1836	else if (INTEL_INFO(dev)->gen >= 7)
		1837	/* IVB/HSW primary/sprite plane watermarks */
		1838	return level == 0 ? 127 : 1023;
		1839	else if (!is_sprite)
		1840	/* ILK/SNB primary plane watermarks */
		1841	return level == 0 ? 127 : 511;
		1842	else
		1843	/* ILK/SNB sprite plane watermarks */
		1844	return level == 0 ? 63 : 255;
		1845	}
		1846
		1847	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		1848	int level)
		1849	{
		1850	if (INTEL_INFO(dev)->gen >= 7)
		1851	return level == 0 ? 63 : 255;
		1852	else
		1853	return level == 0 ? 31 : 63;
		1854	}
		1855
		1856	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		1857	{
		1858	if (INTEL_INFO(dev)->gen >= 8)
		1859	return 31;
		1860	else
		1861	return 15;
		1862	}
		1863
4104	Serge	1864	/* Calculate the maximum primary/sprite plane watermark */
		1865	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1866	int level,
		1867	const struct intel_wm_config *config,
		1868	enum intel_ddb_partitioning ddb_partitioning,
		1869	bool is_sprite)
		1870	{
		1871	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1872
		1873	/* if sprites aren't enabled, sprites get nothing */
		1874	if (is_sprite && !config->sprites_enabled)
		1875	return 0;
		1876
		1877	/* HSW allows LP1+ watermarks even with multiple pipes */
		1878	if (level == 0 \|\| config->num_pipes_active > 1) {
		1879	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1880
		1881	/*
		1882	* For some reason the non self refresh
		1883	* FIFO size is only half of the self
		1884	* refresh FIFO size on ILK/SNB.
		1885	*/
		1886	if (INTEL_INFO(dev)->gen <= 6)
		1887	fifo_size /= 2;
		1888	}
		1889
		1890	if (config->sprites_enabled) {
		1891	/* level 0 is always calculated with 1:1 split */
		1892	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1893	if (is_sprite)
		1894	fifo_size *= 5;
		1895	fifo_size /= 6;
		1896	} else {
		1897	fifo_size /= 2;
		1898	}
		1899	}
		1900
		1901	/* clamp to max that the registers can hold */
5060	serge	1902	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	1903	}
		1904
		1905	/* Calculate the maximum cursor plane watermark */
		1906	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1907	int level,
		1908	const struct intel_wm_config *config)
		1909	{
		1910	/* HSW LP1+ watermarks w/ multiple pipes */
		1911	if (level > 0 && config->num_pipes_active > 1)
		1912	return 64;
		1913
		1914	/* otherwise just report max that registers can hold */
5060	serge	1915	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	1916	}
4104	Serge	1917
5060	serge	1918	static void ilk_compute_wm_maximums(const struct drm_device *dev,
6084	serge	1919	int level,
		1920	const struct intel_wm_config *config,
		1921	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1922	struct ilk_wm_maximums *max)
4104	Serge	1923	{
		1924	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1925	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1926	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	1927	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	1928	}
4104	Serge	1929
5060	serge	1930	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		1931	int level,
		1932	struct ilk_wm_maximums *max)
		1933	{
		1934	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		1935	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		1936	max->cur = ilk_cursor_wm_reg_max(dev, level);
		1937	max->fbc = ilk_fbc_wm_reg_max(dev);
		1938	}
		1939
4560	Serge	1940	static bool ilk_validate_wm_level(int level,
		1941	const struct ilk_wm_maximums *max,
6084	serge	1942	struct intel_wm_level *result)
4104	Serge	1943	{
		1944	bool ret;
		1945
		1946	/* already determined to be invalid? */
		1947	if (!result->enable)
		1948	return false;
		1949
		1950	result->enable = result->pri_val <= max->pri &&
		1951	result->spr_val <= max->spr &&
		1952	result->cur_val <= max->cur;
		1953
		1954	ret = result->enable;
		1955
		1956	/*
		1957	* HACK until we can pre-compute everything,
		1958	* and thus fail gracefully if LP0 watermarks
		1959	* are exceeded...
		1960	*/
		1961	if (level == 0 && !result->enable) {
		1962	if (result->pri_val > max->pri)
		1963	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		1964	level, result->pri_val, max->pri);
		1965	if (result->spr_val > max->spr)
		1966	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		1967	level, result->spr_val, max->spr);
		1968	if (result->cur_val > max->cur)
		1969	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		1970	level, result->cur_val, max->cur);
		1971
		1972	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		1973	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		1974	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		1975	result->enable = true;
		1976	}
		1977
		1978	return ret;
		1979	}
		1980
5060	serge	1981	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6084	serge	1982	const struct intel_crtc *intel_crtc,
4104	Serge	1983	int level,
6084	serge	1984	struct intel_crtc_state *cstate,
6937	serge	1985	struct intel_plane_state *pristate,
		1986	struct intel_plane_state *sprstate,
		1987	struct intel_plane_state *curstate,
4104	Serge	1988	struct intel_wm_level *result)
		1989	{
		1990	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		1991	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		1992	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		1993
		1994	/* WM1+ latency values stored in 0.5us units */
		1995	if (level > 0) {
		1996	pri_latency *= 5;
		1997	spr_latency *= 5;
		1998	cur_latency *= 5;
		1999	}
		2000
6937	serge	2001	result->pri_val = ilk_compute_pri_wm(cstate, pristate,
		2002	pri_latency, level);
		2003	result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
		2004	result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
		2005	result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
4104	Serge	2006	result->enable = true;
		2007	}
		2008
		2009	static uint32_t
		2010	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2011	{
3031	serge	2012	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2013	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	2014	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
4104	Serge	2015	u32 linetime, ips_linetime;
3031	serge	2016
6084	serge	2017	if (!intel_crtc->active)
4104	Serge	2018	return 0;
3031	serge	2019
		2020	/* The WM are computed with base on how long it takes to fill a single
		2021	* row at the given clock rate, multiplied by 8.
		2022	* */
6084	serge	2023	linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2024	adjusted_mode->crtc_clock);
		2025	ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2026	dev_priv->cdclk_freq);
3031	serge	2027
4104	Serge	2028	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2029	PIPE_WM_LINETIME_TIME(linetime);
		2030	}
		2031
5354	serge	2032	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
4104	Serge	2033	{
		2034	struct drm_i915_private *dev_priv = dev->dev_private;
		2035
5354	serge	2036	if (IS_GEN9(dev)) {
		2037	uint32_t val;
		2038	int ret, i;
		2039	int level, max_level = ilk_wm_max_level(dev);
		2040
		2041	/* read the first set of memory latencies[0:3] */
		2042	val = 0; /* data0 to be programmed to 0 for first set */
		2043	mutex_lock(&dev_priv->rps.hw_lock);
		2044	ret = sandybridge_pcode_read(dev_priv,
		2045	GEN9_PCODE_READ_MEM_LATENCY,
		2046	&val);
		2047	mutex_unlock(&dev_priv->rps.hw_lock);
		2048
		2049	if (ret) {
		2050	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2051	return;
		2052	}
		2053
		2054	wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2055	wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2056	GEN9_MEM_LATENCY_LEVEL_MASK;
		2057	wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2058	GEN9_MEM_LATENCY_LEVEL_MASK;
		2059	wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2060	GEN9_MEM_LATENCY_LEVEL_MASK;
		2061
		2062	/* read the second set of memory latencies[4:7] */
		2063	val = 1; /* data0 to be programmed to 1 for second set */
		2064	mutex_lock(&dev_priv->rps.hw_lock);
		2065	ret = sandybridge_pcode_read(dev_priv,
		2066	GEN9_PCODE_READ_MEM_LATENCY,
		2067	&val);
		2068	mutex_unlock(&dev_priv->rps.hw_lock);
		2069	if (ret) {
		2070	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2071	return;
		2072	}
		2073
		2074	wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2075	wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2076	GEN9_MEM_LATENCY_LEVEL_MASK;
		2077	wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2078	GEN9_MEM_LATENCY_LEVEL_MASK;
		2079	wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2080	GEN9_MEM_LATENCY_LEVEL_MASK;
		2081
		2082	/*
6084	serge	2083	* WaWmMemoryReadLatency:skl
		2084	*
5354	serge	2085	* punit doesn't take into account the read latency so we need
6937	serge	2086	* to add 2us to the various latency levels we retrieve from
		2087	* the punit.
		2088	* - W0 is a bit special in that it's the only level that
		2089	* can't be disabled if we want to have display working, so
		2090	* we always add 2us there.
		2091	* - For levels >=1, punit returns 0us latency when they are
		2092	* disabled, so we respect that and don't add 2us then
		2093	*
		2094	* Additionally, if a level n (n > 1) has a 0us latency, all
		2095	* levels m (m >= n) need to be disabled. We make sure to
		2096	* sanitize the values out of the punit to satisfy this
		2097	* requirement.
5354	serge	2098	*/
		2099	wm[0] += 2;
6937	serge	2100	for (level = 1; level <= max_level; level++)
		2101	if (wm[level] != 0)
		2102	wm[level] += 2;
		2103	else {
		2104	for (i = level + 1; i <= max_level; i++)
		2105	wm[i] = 0;
		2106
6935	serge	2107	break;
		2108	}
5354	serge	2109	} else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2110	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2111
		2112	wm[0] = (sskpd >> 56) & 0xFF;
		2113	if (wm[0] == 0)
		2114	wm[0] = sskpd & 0xF;
		2115	wm[1] = (sskpd >> 4) & 0xFF;
		2116	wm[2] = (sskpd >> 12) & 0xFF;
		2117	wm[3] = (sskpd >> 20) & 0x1FF;
		2118	wm[4] = (sskpd >> 32) & 0x1FF;
		2119	} else if (INTEL_INFO(dev)->gen >= 6) {
		2120	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2121
		2122	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2123	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2124	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2125	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2126	} else if (INTEL_INFO(dev)->gen >= 5) {
		2127	uint32_t mltr = I915_READ(MLTR_ILK);
		2128
		2129	/* ILK primary LP0 latency is 700 ns */
		2130	wm[0] = 7;
		2131	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2132	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2133	}
		2134	}
		2135
		2136	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2137	{
		2138	/* ILK sprite LP0 latency is 1300 ns */
		2139	if (INTEL_INFO(dev)->gen == 5)
		2140	wm[0] = 13;
		2141	}
		2142
		2143	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2144	{
		2145	/* ILK cursor LP0 latency is 1300 ns */
		2146	if (INTEL_INFO(dev)->gen == 5)
		2147	wm[0] = 13;
		2148
		2149	/* WaDoubleCursorLP3Latency:ivb */
		2150	if (IS_IVYBRIDGE(dev))
		2151	wm[3] *= 2;
		2152	}
		2153
5060	serge	2154	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2155	{
		2156	/* how many WM levels are we expecting */
6084	serge	2157	if (INTEL_INFO(dev)->gen >= 9)
5354	serge	2158	return 7;
		2159	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
4560	Serge	2160	return 4;
		2161	else if (INTEL_INFO(dev)->gen >= 6)
		2162	return 3;
		2163	else
		2164	return 2;
		2165	}
		2166
4104	Serge	2167	static void intel_print_wm_latency(struct drm_device *dev,
		2168	const char *name,
5354	serge	2169	const uint16_t wm[8])
4104	Serge	2170	{
4560	Serge	2171	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2172
		2173	for (level = 0; level <= max_level; level++) {
		2174	unsigned int latency = wm[level];
		2175
		2176	if (latency == 0) {
		2177	DRM_ERROR("%s WM%d latency not provided\n",
		2178	name, level);
		2179	continue;
		2180	}
		2181
5354	serge	2182	/*
		2183	* - latencies are in us on gen9.
		2184	* - before then, WM1+ latency values are in 0.5us units
		2185	*/
		2186	if (IS_GEN9(dev))
		2187	latency *= 10;
		2188	else if (level > 0)
4104	Serge	2189	latency *= 5;
		2190
		2191	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2192	name, level, wm[level],
		2193	latency / 10, latency % 10);
		2194	}
		2195	}
		2196
5060	serge	2197	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2198	uint16_t wm[5], uint16_t min)
4104	Serge	2199	{
5060	serge	2200	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2201
		2202	if (wm[0] >= min)
		2203	return false;
		2204
		2205	wm[0] = max(wm[0], min);
		2206	for (level = 1; level <= max_level; level++)
		2207	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2208
		2209	return true;
		2210	}
		2211
		2212	static void snb_wm_latency_quirk(struct drm_device *dev)
		2213	{
4104	Serge	2214	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2215	bool changed;
4104	Serge	2216
5060	serge	2217	/*
		2218	* The BIOS provided WM memory latency values are often
		2219	* inadequate for high resolution displays. Adjust them.
		2220	*/
		2221	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2222	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2223	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2224
		2225	if (!changed)
		2226	return;
		2227
		2228	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2229	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2230	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2231	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2232	}
		2233
		2234	static void ilk_setup_wm_latency(struct drm_device *dev)
		2235	{
		2236	struct drm_i915_private *dev_priv = dev->dev_private;
		2237
4104	Serge	2238	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2239
		2240	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2241	sizeof(dev_priv->wm.pri_latency));
		2242	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2243	sizeof(dev_priv->wm.pri_latency));
		2244
		2245	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2246	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2247
		2248	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2249	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2250	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2251
		2252	if (IS_GEN6(dev))
		2253	snb_wm_latency_quirk(dev);
4104	Serge	2254	}
		2255
5354	serge	2256	static void skl_setup_wm_latency(struct drm_device *dev)
		2257	{
		2258	struct drm_i915_private *dev_priv = dev->dev_private;
		2259
		2260	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
		2261	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
		2262	}
		2263
4560	Serge	2264	/* Compute new watermarks for the pipe */
6937	serge	2265	static int ilk_compute_pipe_wm(struct intel_crtc *intel_crtc,
		2266	struct drm_atomic_state *state)
4560	Serge	2267	{
6937	serge	2268	struct intel_pipe_wm *pipe_wm;
		2269	struct drm_device *dev = intel_crtc->base.dev;
5060	serge	2270	const struct drm_i915_private *dev_priv = dev->dev_private;
6937	serge	2271	struct intel_crtc_state *cstate = NULL;
6084	serge	2272	struct intel_plane *intel_plane;
6937	serge	2273	struct drm_plane_state *ps;
		2274	struct intel_plane_state *pristate = NULL;
6084	serge	2275	struct intel_plane_state *sprstate = NULL;
6937	serge	2276	struct intel_plane_state *curstate = NULL;
4560	Serge	2277	int level, max_level = ilk_wm_max_level(dev);
		2278	/* LP0 watermark maximums depend on this pipe alone */
		2279	struct intel_wm_config config = {
		2280	.num_pipes_active = 1,
		2281	};
		2282	struct ilk_wm_maximums max;
4104	Serge	2283
6937	serge	2284	cstate = intel_atomic_get_crtc_state(state, intel_crtc);
		2285	if (IS_ERR(cstate))
		2286	return PTR_ERR(cstate);
		2287
		2288	pipe_wm = &cstate->wm.optimal.ilk;
		2289	memset(pipe_wm, 0, sizeof(*pipe_wm));
		2290
6084	serge	2291	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
6937	serge	2292	ps = drm_atomic_get_plane_state(state,
		2293	&intel_plane->base);
		2294	if (IS_ERR(ps))
		2295	return PTR_ERR(ps);
		2296
		2297	if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		2298	pristate = to_intel_plane_state(ps);
		2299	else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY)
		2300	sprstate = to_intel_plane_state(ps);
		2301	else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
		2302	curstate = to_intel_plane_state(ps);
6084	serge	2303	}
4560	Serge	2304
6084	serge	2305	config.sprites_enabled = sprstate->visible;
		2306	config.sprites_scaled = sprstate->visible &&
		2307	(drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 \|\|
		2308	drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
		2309
		2310	pipe_wm->pipe_enabled = cstate->base.active;
6937	serge	2311	pipe_wm->sprites_enabled = config.sprites_enabled;
6084	serge	2312	pipe_wm->sprites_scaled = config.sprites_scaled;
		2313
4560	Serge	2314	/* ILK/SNB: LP2+ watermarks only w/o sprites */
6084	serge	2315	if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
4560	Serge	2316	max_level = 1;
		2317
		2318	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
6084	serge	2319	if (config.sprites_scaled)
4560	Serge	2320	max_level = 0;
		2321
6937	serge	2322	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
		2323	pristate, sprstate, curstate, &pipe_wm->wm[0]);
4560	Serge	2324
		2325	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
6937	serge	2326	pipe_wm->linetime = hsw_compute_linetime_wm(dev,
		2327	&intel_crtc->base);
4560	Serge	2328
5060	serge	2329	/* LP0 watermarks always use 1/2 DDB partitioning */
		2330	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2331
4560	Serge	2332	/* At least LP0 must be valid */
5060	serge	2333	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
6937	serge	2334	return -EINVAL;
5060	serge	2335
		2336	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2337
		2338	for (level = 1; level <= max_level; level++) {
		2339	struct intel_wm_level wm = {};
		2340
6937	serge	2341	ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
		2342	pristate, sprstate, curstate, &wm);
5060	serge	2343
		2344	/*
		2345	* Disable any watermark level that exceeds the
		2346	* register maximums since such watermarks are
		2347	* always invalid.
		2348	*/
		2349	if (!ilk_validate_wm_level(level, &max, &wm))
		2350	break;
		2351
		2352	pipe_wm->wm[level] = wm;
		2353	}
		2354
6937	serge	2355	return 0;
4104	Serge	2356	}
		2357
4560	Serge	2358	/*
		2359	* Merge the watermarks from all active pipes for a specific level.
		2360	*/
		2361	static void ilk_merge_wm_level(struct drm_device *dev,
		2362	int level,
		2363	struct intel_wm_level *ret_wm)
4104	Serge	2364	{
4560	Serge	2365	const struct intel_crtc *intel_crtc;
4104	Serge	2366
5060	serge	2367	ret_wm->enable = true;
4104	Serge	2368
5060	serge	2369	for_each_intel_crtc(dev, intel_crtc) {
6937	serge	2370	const struct intel_crtc_state *cstate =
		2371	to_intel_crtc_state(intel_crtc->base.state);
		2372	const struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
5060	serge	2373	const struct intel_wm_level *wm = &active->wm[level];
		2374
		2375	if (!active->pipe_enabled)
		2376	continue;
		2377
		2378	/*
		2379	* The watermark values may have been used in the past,
		2380	* so we must maintain them in the registers for some
		2381	* time even if the level is now disabled.
		2382	*/
4560	Serge	2383	if (!wm->enable)
5060	serge	2384	ret_wm->enable = false;
4104	Serge	2385
4560	Serge	2386	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2387	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2388	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2389	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2390	}
		2391	}
		2392
		2393	/*
		2394	* Merge all low power watermarks for all active pipes.
		2395	*/
		2396	static void ilk_wm_merge(struct drm_device *dev,
		2397	const struct intel_wm_config *config,
		2398	const struct ilk_wm_maximums *max,
		2399	struct intel_pipe_wm *merged)
		2400	{
6084	serge	2401	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2402	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2403	int last_enabled_level = max_level;
4560	Serge	2404
		2405	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2406	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2407	config->num_pipes_active > 1)
		2408	return;
		2409
		2410	/* ILK: FBC WM must be disabled always */
		2411	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2412
		2413	/* merge each WM1+ level */
4104	Serge	2414	for (level = 1; level <= max_level; level++) {
4560	Serge	2415	struct intel_wm_level *wm = &merged->wm[level];
		2416
		2417	ilk_merge_wm_level(dev, level, wm);
		2418
5060	serge	2419	if (level > last_enabled_level)
		2420	wm->enable = false;
		2421	else if (!ilk_validate_wm_level(level, max, wm))
		2422	/* make sure all following levels get disabled */
		2423	last_enabled_level = level - 1;
4560	Serge	2424
		2425	/*
		2426	* The spec says it is preferred to disable
		2427	* FBC WMs instead of disabling a WM level.
		2428	*/
		2429	if (wm->fbc_val > max->fbc) {
5060	serge	2430	if (wm->enable)
6084	serge	2431	merged->fbc_wm_enabled = false;
4560	Serge	2432	wm->fbc_val = 0;
4104	Serge	2433	}
		2434	}
		2435
4560	Serge	2436	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2437	/*
		2438	* FIXME this is racy. FBC might get enabled later.
		2439	* What we should check here is whether FBC can be
		2440	* enabled sometime later.
		2441	*/
6084	serge	2442	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
6937	serge	2443	intel_fbc_is_active(dev_priv)) {
4560	Serge	2444	for (level = 2; level <= max_level; level++) {
		2445	struct intel_wm_level *wm = &merged->wm[level];
		2446
		2447	wm->enable = false;
		2448	}
		2449	}
		2450	}
		2451
		2452	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2453	{
		2454	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2455	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2456	}
		2457
		2458	/* The value we need to program into the WM_LPx latency field */
		2459	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2460	{
		2461	struct drm_i915_private *dev_priv = dev->dev_private;
		2462
		2463	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2464	return 2 * level;
		2465	else
		2466	return dev_priv->wm.pri_latency[level];
		2467	}
		2468
		2469	static void ilk_compute_wm_results(struct drm_device *dev,
		2470	const struct intel_pipe_wm *merged,
		2471	enum intel_ddb_partitioning partitioning,
		2472	struct ilk_wm_values *results)
		2473	{
		2474	struct intel_crtc *intel_crtc;
		2475	int level, wm_lp;
		2476
		2477	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2478	results->partitioning = partitioning;
		2479
		2480	/* LP1+ register values */
4104	Serge	2481	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2482	const struct intel_wm_level *r;
		2483
4560	Serge	2484	level = ilk_wm_lp_to_level(wm_lp, merged);
		2485
		2486	r = &merged->wm[level];
4104	Serge	2487
5060	serge	2488	/*
		2489	* Maintain the watermark values even if the level is
		2490	* disabled. Doing otherwise could cause underruns.
		2491	*/
		2492	results->wm_lp[wm_lp - 1] =
4560	Serge	2493	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2494	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2495	r->cur_val;
		2496
5060	serge	2497	if (r->enable)
		2498	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2499
4560	Serge	2500	if (INTEL_INFO(dev)->gen >= 8)
		2501	results->wm_lp[wm_lp - 1] \|=
		2502	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2503	else
		2504	results->wm_lp[wm_lp - 1] \|=
		2505	r->fbc_val << WM1_LP_FBC_SHIFT;
		2506
5060	serge	2507	/*
		2508	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2509	* level is disabled. Doing otherwise could cause underruns.
		2510	*/
4560	Serge	2511	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2512	WARN_ON(wm_lp != 1);
		2513	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2514	} else
6084	serge	2515	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
4104	Serge	2516	}
		2517
4560	Serge	2518	/* LP0 register values */
5060	serge	2519	for_each_intel_crtc(dev, intel_crtc) {
6937	serge	2520	const struct intel_crtc_state *cstate =
		2521	to_intel_crtc_state(intel_crtc->base.state);
4560	Serge	2522	enum pipe pipe = intel_crtc->pipe;
6937	serge	2523	const struct intel_wm_level *r = &cstate->wm.optimal.ilk.wm[0];
4104	Serge	2524
4560	Serge	2525	if (WARN_ON(!r->enable))
		2526	continue;
		2527
6937	serge	2528	results->wm_linetime[pipe] = cstate->wm.optimal.ilk.linetime;
4560	Serge	2529
		2530	results->wm_pipe[pipe] =
		2531	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2532	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2533	r->cur_val;
4104	Serge	2534	}
		2535	}
		2536
		2537	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2538	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2539	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2540	struct intel_pipe_wm *r1,
		2541	struct intel_pipe_wm *r2)
4104	Serge	2542	{
4560	Serge	2543	int level, max_level = ilk_wm_max_level(dev);
		2544	int level1 = 0, level2 = 0;
4104	Serge	2545
4560	Serge	2546	for (level = 1; level <= max_level; level++) {
		2547	if (r1->wm[level].enable)
		2548	level1 = level;
		2549	if (r2->wm[level].enable)
		2550	level2 = level;
4104	Serge	2551	}
		2552
4560	Serge	2553	if (level1 == level2) {
		2554	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2555	return r2;
		2556	else
		2557	return r1;
4560	Serge	2558	} else if (level1 > level2) {
4104	Serge	2559	return r1;
		2560	} else {
		2561	return r2;
		2562	}
		2563	}
		2564
4560	Serge	2565	/* dirty bits used to track which watermarks need changes */
		2566	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2567	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2568	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2569	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2570	#define WM_DIRTY_FBC (1 << 24)
		2571	#define WM_DIRTY_DDB (1 << 25)
		2572
5354	serge	2573	static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
4560	Serge	2574	const struct ilk_wm_values *old,
		2575	const struct ilk_wm_values *new)
		2576	{
		2577	unsigned int dirty = 0;
		2578	enum pipe pipe;
		2579	int wm_lp;
		2580
5354	serge	2581	for_each_pipe(dev_priv, pipe) {
4560	Serge	2582	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2583	dirty \|= WM_DIRTY_LINETIME(pipe);
		2584	/* Must disable LP1+ watermarks too */
		2585	dirty \|= WM_DIRTY_LP_ALL;
		2586	}
		2587
		2588	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2589	dirty \|= WM_DIRTY_PIPE(pipe);
		2590	/* Must disable LP1+ watermarks too */
		2591	dirty \|= WM_DIRTY_LP_ALL;
		2592	}
		2593	}
		2594
		2595	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2596	dirty \|= WM_DIRTY_FBC;
		2597	/* Must disable LP1+ watermarks too */
		2598	dirty \|= WM_DIRTY_LP_ALL;
		2599	}
		2600
		2601	if (old->partitioning != new->partitioning) {
		2602	dirty \|= WM_DIRTY_DDB;
		2603	/* Must disable LP1+ watermarks too */
		2604	dirty \|= WM_DIRTY_LP_ALL;
		2605	}
		2606
		2607	/* LP1+ watermarks already deemed dirty, no need to continue */
		2608	if (dirty & WM_DIRTY_LP_ALL)
		2609	return dirty;
		2610
		2611	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2612	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2613	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2614	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2615	break;
		2616	}
		2617
		2618	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2619	for (; wm_lp <= 3; wm_lp++)
		2620	dirty \|= WM_DIRTY_LP(wm_lp);
		2621
		2622	return dirty;
		2623	}
		2624
		2625	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2626	unsigned int dirty)
		2627	{
		2628	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2629	bool changed = false;
		2630
		2631	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2632	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2633	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2634	changed = true;
		2635	}
		2636	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2637	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2638	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2639	changed = true;
		2640	}
		2641	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2642	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2643	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2644	changed = true;
		2645	}
		2646
		2647	/*
		2648	* Don't touch WM1S_LP_EN here.
		2649	* Doing so could cause underruns.
		2650	*/
		2651
		2652	return changed;
		2653	}
		2654
4104	Serge	2655	/*
		2656	* The spec says we shouldn't write when we don't need, because every write
		2657	* causes WMs to be re-evaluated, expending some power.
6084	serge	2658	*/
4560	Serge	2659	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2660	struct ilk_wm_values *results)
4104	Serge	2661	{
4560	Serge	2662	struct drm_device *dev = dev_priv->dev;
		2663	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2664	unsigned int dirty;
4104	Serge	2665	uint32_t val;
3031	serge	2666
5354	serge	2667	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
4560	Serge	2668	if (!dirty)
4104	Serge	2669	return;
		2670
4560	Serge	2671	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2672
4560	Serge	2673	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2674	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2675	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2676	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2677	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2678	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2679
4560	Serge	2680	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2681	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2682	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2683	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2684	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2685	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2686
4560	Serge	2687	if (dirty & WM_DIRTY_DDB) {
		2688	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
6084	serge	2689	val = I915_READ(WM_MISC);
4560	Serge	2690	if (results->partitioning == INTEL_DDB_PART_1_2)
6084	serge	2691	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2692	else
		2693	val \|= WM_MISC_DATA_PARTITION_5_6;
		2694	I915_WRITE(WM_MISC, val);
4560	Serge	2695	} else {
		2696	val = I915_READ(DISP_ARB_CTL2);
		2697	if (results->partitioning == INTEL_DDB_PART_1_2)
		2698	val &= ~DISP_DATA_PARTITION_5_6;
		2699	else
		2700	val \|= DISP_DATA_PARTITION_5_6;
		2701	I915_WRITE(DISP_ARB_CTL2, val);
		2702	}
4104	Serge	2703	}
		2704
4560	Serge	2705	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2706	val = I915_READ(DISP_ARB_CTL);
		2707	if (results->enable_fbc_wm)
		2708	val &= ~DISP_FBC_WM_DIS;
		2709	else
		2710	val \|= DISP_FBC_WM_DIS;
		2711	I915_WRITE(DISP_ARB_CTL, val);
		2712	}
		2713
4560	Serge	2714	if (dirty & WM_DIRTY_LP(1) &&
		2715	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2716	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2717
		2718	if (INTEL_INFO(dev)->gen >= 7) {
		2719	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
6084	serge	2720	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2721	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
6084	serge	2722	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2723	}
4104	Serge	2724
4560	Serge	2725	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2726	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2727	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2728	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2729	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2730	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2731
		2732	dev_priv->wm.hw = *results;
3031	serge	2733	}
		2734
4560	Serge	2735	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2736	{
		2737	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2738
		2739	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2740	}
		2741
5354	serge	2742	/*
		2743	* On gen9, we need to allocate Display Data Buffer (DDB) portions to the
		2744	* different active planes.
		2745	*/
		2746
		2747	#define SKL_DDB_SIZE 896 /* in blocks */
6084	serge	2748	#define BXT_DDB_SIZE 512
5354	serge	2749
6937	serge	2750	/*
		2751	* Return the index of a plane in the SKL DDB and wm result arrays. Primary
		2752	* plane is always in slot 0, cursor is always in slot I915_MAX_PLANES-1, and
		2753	* other universal planes are in indices 1..n. Note that this may leave unused
		2754	* indices between the top "sprite" plane and the cursor.
		2755	*/
		2756	static int
		2757	skl_wm_plane_id(const struct intel_plane *plane)
		2758	{
		2759	switch (plane->base.type) {
		2760	case DRM_PLANE_TYPE_PRIMARY:
		2761	return 0;
		2762	case DRM_PLANE_TYPE_CURSOR:
		2763	return PLANE_CURSOR;
		2764	case DRM_PLANE_TYPE_OVERLAY:
		2765	return plane->plane + 1;
		2766	default:
		2767	MISSING_CASE(plane->base.type);
		2768	return plane->plane;
		2769	}
		2770	}
		2771
5354	serge	2772	static void
		2773	skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
6937	serge	2774	const struct intel_crtc_state *cstate,
5354	serge	2775	const struct intel_wm_config *config,
		2776	struct skl_ddb_entry alloc / out */)
		2777	{
6937	serge	2778	struct drm_crtc *for_crtc = cstate->base.crtc;
5354	serge	2779	struct drm_crtc *crtc;
		2780	unsigned int pipe_size, ddb_size;
		2781	int nth_active_pipe;
		2782
6937	serge	2783	if (!cstate->base.active) {
5354	serge	2784	alloc->start = 0;
		2785	alloc->end = 0;
		2786	return;
		2787	}
		2788
6084	serge	2789	if (IS_BROXTON(dev))
		2790	ddb_size = BXT_DDB_SIZE;
		2791	else
		2792	ddb_size = SKL_DDB_SIZE;
5354	serge	2793
		2794	ddb_size -= 4; /* 4 blocks for bypass path allocation */
		2795
		2796	nth_active_pipe = 0;
		2797	for_each_crtc(dev, crtc) {
6084	serge	2798	if (!to_intel_crtc(crtc)->active)
5354	serge	2799	continue;
		2800
		2801	if (crtc == for_crtc)
		2802	break;
		2803
		2804	nth_active_pipe++;
		2805	}
		2806
		2807	pipe_size = ddb_size / config->num_pipes_active;
		2808	alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
		2809	alloc->end = alloc->start + pipe_size;
		2810	}
		2811
		2812	static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
		2813	{
		2814	if (config->num_pipes_active == 1)
		2815	return 32;
		2816
		2817	return 8;
		2818	}
		2819
		2820	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
		2821	{
		2822	entry->start = reg & 0x3ff;
		2823	entry->end = (reg >> 16) & 0x3ff;
		2824	if (entry->end)
		2825	entry->end += 1;
		2826	}
		2827
		2828	void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
		2829	struct skl_ddb_allocation ddb / out */)
		2830	{
		2831	enum pipe pipe;
		2832	int plane;
		2833	u32 val;
		2834
6084	serge	2835	memset(ddb, 0, sizeof(*ddb));
		2836
5354	serge	2837	for_each_pipe(dev_priv, pipe) {
6937	serge	2838	enum intel_display_power_domain power_domain;
		2839
		2840	power_domain = POWER_DOMAIN_PIPE(pipe);
		2841	if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
6084	serge	2842	continue;
		2843
		2844	for_each_plane(dev_priv, pipe, plane) {
5354	serge	2845	val = I915_READ(PLANE_BUF_CFG(pipe, plane));
		2846	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
		2847	val);
		2848	}
		2849
		2850	val = I915_READ(CUR_BUF_CFG(pipe));
6084	serge	2851	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
		2852	val);
6937	serge	2853
		2854	intel_display_power_put(dev_priv, power_domain);
5354	serge	2855	}
		2856	}
		2857
		2858	static unsigned int
6937	serge	2859	skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
		2860	const struct drm_plane_state *pstate,
		2861	int y)
5354	serge	2862	{
6937	serge	2863	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
		2864	struct drm_framebuffer *fb = pstate->fb;
6084	serge	2865
		2866	/* for planar format */
6937	serge	2867	if (fb->pixel_format == DRM_FORMAT_NV12) {
6084	serge	2868	if (y) /* y-plane data rate */
6937	serge	2869	return intel_crtc->config->pipe_src_w *
		2870	intel_crtc->config->pipe_src_h *
		2871	drm_format_plane_cpp(fb->pixel_format, 0);
6084	serge	2872	else /* uv-plane data rate */
6937	serge	2873	return (intel_crtc->config->pipe_src_w/2) *
		2874	(intel_crtc->config->pipe_src_h/2) *
		2875	drm_format_plane_cpp(fb->pixel_format, 1);
6084	serge	2876	}
		2877
		2878	/* for packed formats */
6937	serge	2879	return intel_crtc->config->pipe_src_w *
		2880	intel_crtc->config->pipe_src_h *
		2881	drm_format_plane_cpp(fb->pixel_format, 0);
5354	serge	2882	}
		2883
		2884	/*
		2885	* We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
		2886	* a 8192x4096@32bpp framebuffer:
		2887	* 3 * 4096 * 8192 * 4 < 2^32
		2888	*/
		2889	static unsigned int
6937	serge	2890	skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate)
5354	serge	2891	{
6937	serge	2892	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
		2893	struct drm_device *dev = intel_crtc->base.dev;
		2894	const struct intel_plane *intel_plane;
5354	serge	2895	unsigned int total_data_rate = 0;
		2896
6937	serge	2897	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2898	const struct drm_plane_state *pstate = intel_plane->base.state;
5354	serge	2899
6937	serge	2900	if (pstate->fb == NULL)
5354	serge	2901	continue;
		2902
6937	serge	2903	if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
		2904	continue;
		2905
		2906	/* packed/uv */
		2907	total_data_rate += skl_plane_relative_data_rate(cstate,
		2908	pstate,
		2909	0);
		2910
		2911	if (pstate->fb->pixel_format == DRM_FORMAT_NV12)
		2912	/* y-plane */
		2913	total_data_rate += skl_plane_relative_data_rate(cstate,
		2914	pstate,
		2915	1);
5354	serge	2916	}
		2917
		2918	return total_data_rate;
		2919	}
		2920
		2921	static void
6937	serge	2922	skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
5354	serge	2923	struct skl_ddb_allocation ddb / out */)
		2924	{
6937	serge	2925	struct drm_crtc *crtc = cstate->base.crtc;
5354	serge	2926	struct drm_device *dev = crtc->dev;
6937	serge	2927	struct drm_i915_private *dev_priv = to_i915(dev);
		2928	struct intel_wm_config *config = &dev_priv->wm.config;
5354	serge	2929	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	2930	struct intel_plane *intel_plane;
5354	serge	2931	enum pipe pipe = intel_crtc->pipe;
		2932	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
		2933	uint16_t alloc_size, start, cursor_blocks;
6084	serge	2934	uint16_t minimum[I915_MAX_PLANES];
		2935	uint16_t y_minimum[I915_MAX_PLANES];
5354	serge	2936	unsigned int total_data_rate;
		2937
6937	serge	2938	skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc);
5354	serge	2939	alloc_size = skl_ddb_entry_size(alloc);
		2940	if (alloc_size == 0) {
		2941	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
6084	serge	2942	memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
		2943	sizeof(ddb->plane[pipe][PLANE_CURSOR]));
5354	serge	2944	return;
		2945	}
		2946
		2947	cursor_blocks = skl_cursor_allocation(config);
6084	serge	2948	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
		2949	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
5354	serge	2950
		2951	alloc_size -= cursor_blocks;
		2952	alloc->end -= cursor_blocks;
		2953
6084	serge	2954	/* 1. Allocate the mininum required blocks for each active plane */
6937	serge	2955	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2956	struct drm_plane *plane = &intel_plane->base;
		2957	struct drm_framebuffer *fb = plane->state->fb;
		2958	int id = skl_wm_plane_id(intel_plane);
6084	serge	2959
6937	serge	2960	if (fb == NULL)
6084	serge	2961	continue;
6937	serge	2962	if (plane->type == DRM_PLANE_TYPE_CURSOR)
		2963	continue;
6084	serge	2964
6937	serge	2965	minimum[id] = 8;
		2966	alloc_size -= minimum[id];
		2967	y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0;
		2968	alloc_size -= y_minimum[id];
6084	serge	2969	}
		2970
		2971	/*
		2972	* 2. Distribute the remaining space in proportion to the amount of
		2973	* data each plane needs to fetch from memory.
5354	serge	2974	*
		2975	* FIXME: we may not allocate every single block here.
6084	serge	2976	*/
6937	serge	2977	total_data_rate = skl_get_total_relative_data_rate(cstate);
5354	serge	2978
		2979	start = alloc->start;
6937	serge	2980	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2981	struct drm_plane *plane = &intel_plane->base;
		2982	struct drm_plane_state *pstate = intel_plane->base.state;
6084	serge	2983	unsigned int data_rate, y_data_rate;
		2984	uint16_t plane_blocks, y_plane_blocks = 0;
6937	serge	2985	int id = skl_wm_plane_id(intel_plane);
5354	serge	2986
6937	serge	2987	if (pstate->fb == NULL)
5354	serge	2988	continue;
6937	serge	2989	if (plane->type == DRM_PLANE_TYPE_CURSOR)
		2990	continue;
5354	serge	2991
6937	serge	2992	data_rate = skl_plane_relative_data_rate(cstate, pstate, 0);
5354	serge	2993
		2994	/*
6084	serge	2995	* allocation for (packed formats) or (uv-plane part of planar format):
5354	serge	2996	* promote the expression to 64 bits to avoid overflowing, the
		2997	* result is < available as data_rate / total_data_rate < 1
		2998	*/
6937	serge	2999	plane_blocks = minimum[id];
6084	serge	3000	plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
		3001	total_data_rate);
5354	serge	3002
6937	serge	3003	ddb->plane[pipe][id].start = start;
		3004	ddb->plane[pipe][id].end = start + plane_blocks;
5354	serge	3005
		3006	start += plane_blocks;
6084	serge	3007
		3008	/*
		3009	* allocation for y_plane part of planar format:
		3010	*/
6937	serge	3011	if (pstate->fb->pixel_format == DRM_FORMAT_NV12) {
		3012	y_data_rate = skl_plane_relative_data_rate(cstate,
		3013	pstate,
		3014	1);
		3015	y_plane_blocks = y_minimum[id];
6084	serge	3016	y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
		3017	total_data_rate);
		3018
6937	serge	3019	ddb->y_plane[pipe][id].start = start;
		3020	ddb->y_plane[pipe][id].end = start + y_plane_blocks;
6084	serge	3021
		3022	start += y_plane_blocks;
		3023	}
		3024
5354	serge	3025	}
		3026
		3027	}
		3028
6084	serge	3029	static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
5354	serge	3030	{
		3031	/* TODO: Take into account the scalers once we support them */
6084	serge	3032	return config->base.adjusted_mode.crtc_clock;
5354	serge	3033	}
		3034
		3035	/*
		3036	* The max latency should be 257 (max the punit can code is 255 and we add 2us
		3037	* for the read latency) and bytes_per_pixel should always be <= 8, so that
		3038	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
		3039	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
		3040	*/
		3041	static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		3042	uint32_t latency)
		3043	{
		3044	uint32_t wm_intermediate_val, ret;
		3045
		3046	if (latency == 0)
		3047	return UINT_MAX;
		3048
6084	serge	3049	wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
5354	serge	3050	ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
		3051
		3052	return ret;
		3053	}
		3054
		3055	static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		3056	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
6084	serge	3057	uint64_t tiling, uint32_t latency)
5354	serge	3058	{
6084	serge	3059	uint32_t ret;
		3060	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3061	uint32_t wm_intermediate_val;
5354	serge	3062
		3063	if (latency == 0)
		3064	return UINT_MAX;
		3065
		3066	plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
6084	serge	3067
		3068	if (tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3069	tiling == I915_FORMAT_MOD_Yf_TILED) {
		3070	plane_bytes_per_line *= 4;
		3071	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3072	plane_blocks_per_line /= 4;
		3073	} else {
		3074	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3075	}
		3076
5354	serge	3077	wm_intermediate_val = latency * pixel_rate;
		3078	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
6084	serge	3079	plane_blocks_per_line;
5354	serge	3080
		3081	return ret;
		3082	}
		3083
		3084	static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
		3085	const struct intel_crtc *intel_crtc)
		3086	{
		3087	struct drm_device *dev = intel_crtc->base.dev;
		3088	struct drm_i915_private *dev_priv = dev->dev_private;
		3089	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3090
6937	serge	3091	/*
		3092	* If ddb allocation of pipes changed, it may require recalculation of
		3093	* watermarks
		3094	*/
		3095	if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe)))
5354	serge	3096	return true;
		3097
		3098	return false;
		3099	}
		3100
6084	serge	3101	static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
6937	serge	3102	struct intel_crtc_state *cstate,
		3103	struct intel_plane *intel_plane,
5354	serge	3104	uint16_t ddb_allocation,
6084	serge	3105	int level,
5354	serge	3106	uint16_t out_blocks, / out */
		3107	uint8_t out_lines / out */)
		3108	{
6937	serge	3109	struct drm_plane *plane = &intel_plane->base;
		3110	struct drm_framebuffer *fb = plane->state->fb;
6084	serge	3111	uint32_t latency = dev_priv->wm.skl_latency[level];
		3112	uint32_t method1, method2;
		3113	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3114	uint32_t res_blocks, res_lines;
		3115	uint32_t selected_result;
		3116	uint8_t bytes_per_pixel;
5354	serge	3117
6937	serge	3118	if (latency == 0 \|\| !cstate->base.active \|\| !fb)
5354	serge	3119	return false;
		3120
6937	serge	3121	bytes_per_pixel = drm_format_plane_cpp(fb->pixel_format, 0);
		3122	method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate),
6084	serge	3123	bytes_per_pixel,
		3124	latency);
6937	serge	3125	method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate),
		3126	cstate->base.adjusted_mode.crtc_htotal,
		3127	cstate->pipe_src_w,
6084	serge	3128	bytes_per_pixel,
6937	serge	3129	fb->modifier[0],
6084	serge	3130	latency);
5354	serge	3131
6937	serge	3132	plane_bytes_per_line = cstate->pipe_src_w * bytes_per_pixel;
6084	serge	3133	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
5354	serge	3134
6937	serge	3135	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED \|\|
		3136	fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
6084	serge	3137	uint32_t min_scanlines = 4;
		3138	uint32_t y_tile_minimum;
6937	serge	3139	if (intel_rotation_90_or_270(plane->state->rotation)) {
		3140	int bpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
		3141	drm_format_plane_cpp(fb->pixel_format, 1) :
		3142	drm_format_plane_cpp(fb->pixel_format, 0);
		3143
		3144	switch (bpp) {
6084	serge	3145	case 1:
		3146	min_scanlines = 16;
		3147	break;
		3148	case 2:
		3149	min_scanlines = 8;
		3150	break;
		3151	case 8:
		3152	WARN(1, "Unsupported pixel depth for rotation");
		3153	}
		3154	}
		3155	y_tile_minimum = plane_blocks_per_line * min_scanlines;
		3156	selected_result = max(method2, y_tile_minimum);
		3157	} else {
		3158	if ((ddb_allocation / plane_blocks_per_line) >= 1)
		3159	selected_result = min(method1, method2);
		3160	else
		3161	selected_result = method1;
		3162	}
5354	serge	3163
6084	serge	3164	res_blocks = selected_result + 1;
		3165	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
5354	serge	3166
6084	serge	3167	if (level >= 1 && level <= 7) {
6937	serge	3168	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED \|\|
		3169	fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)
6084	serge	3170	res_lines += 4;
		3171	else
		3172	res_blocks++;
		3173	}
		3174
		3175	if (res_blocks >= ddb_allocation \|\| res_lines > 31)
5354	serge	3176	return false;
		3177
		3178	*out_blocks = res_blocks;
		3179	*out_lines = res_lines;
		3180
		3181	return true;
		3182	}
		3183
		3184	static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
		3185	struct skl_ddb_allocation *ddb,
6937	serge	3186	struct intel_crtc_state *cstate,
5354	serge	3187	int level,
		3188	struct skl_wm_level *result)
		3189	{
6937	serge	3190	struct drm_device *dev = dev_priv->dev;
		3191	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
		3192	struct intel_plane *intel_plane;
5354	serge	3193	uint16_t ddb_blocks;
6937	serge	3194	enum pipe pipe = intel_crtc->pipe;
5354	serge	3195
6937	serge	3196	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		3197	int i = skl_wm_plane_id(intel_plane);
		3198
5354	serge	3199	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
		3200
6084	serge	3201	result->plane_en[i] = skl_compute_plane_wm(dev_priv,
6937	serge	3202	cstate,
		3203	intel_plane,
5354	serge	3204	ddb_blocks,
6084	serge	3205	level,
5354	serge	3206	&result->plane_res_b[i],
		3207	&result->plane_res_l[i]);
		3208	}
		3209	}
		3210
		3211	static uint32_t
6937	serge	3212	skl_compute_linetime_wm(struct intel_crtc_state *cstate)
5354	serge	3213	{
6937	serge	3214	if (!cstate->base.active)
5354	serge	3215	return 0;
		3216
6937	serge	3217	if (WARN_ON(skl_pipe_pixel_rate(cstate) == 0))
6084	serge	3218	return 0;
		3219
6937	serge	3220	return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000,
		3221	skl_pipe_pixel_rate(cstate));
5354	serge	3222	}
		3223
6937	serge	3224	static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
5354	serge	3225	struct skl_wm_level trans_wm / out */)
		3226	{
6937	serge	3227	struct drm_crtc *crtc = cstate->base.crtc;
5354	serge	3228	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3229	struct intel_plane *intel_plane;
5354	serge	3230
6937	serge	3231	if (!cstate->base.active)
5354	serge	3232	return;
		3233
		3234	/* Until we know more, just disable transition WMs */
6937	serge	3235	for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) {
		3236	int i = skl_wm_plane_id(intel_plane);
		3237
5354	serge	3238	trans_wm->plane_en[i] = false;
6937	serge	3239	}
5354	serge	3240	}
		3241
6937	serge	3242	static void skl_compute_pipe_wm(struct intel_crtc_state *cstate,
5354	serge	3243	struct skl_ddb_allocation *ddb,
		3244	struct skl_pipe_wm *pipe_wm)
		3245	{
6937	serge	3246	struct drm_device *dev = cstate->base.crtc->dev;
5354	serge	3247	const struct drm_i915_private *dev_priv = dev->dev_private;
		3248	int level, max_level = ilk_wm_max_level(dev);
		3249
		3250	for (level = 0; level <= max_level; level++) {
6937	serge	3251	skl_compute_wm_level(dev_priv, ddb, cstate,
		3252	level, &pipe_wm->wm[level]);
5354	serge	3253	}
6937	serge	3254	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
5354	serge	3255
6937	serge	3256	skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
5354	serge	3257	}
		3258
		3259	static void skl_compute_wm_results(struct drm_device *dev,
		3260	struct skl_pipe_wm *p_wm,
		3261	struct skl_wm_values *r,
		3262	struct intel_crtc *intel_crtc)
		3263	{
		3264	int level, max_level = ilk_wm_max_level(dev);
		3265	enum pipe pipe = intel_crtc->pipe;
		3266	uint32_t temp;
		3267	int i;
		3268
		3269	for (level = 0; level <= max_level; level++) {
		3270	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3271	temp = 0;
		3272
		3273	temp \|= p_wm->wm[level].plane_res_l[i] <<
		3274	PLANE_WM_LINES_SHIFT;
		3275	temp \|= p_wm->wm[level].plane_res_b[i];
		3276	if (p_wm->wm[level].plane_en[i])
		3277	temp \|= PLANE_WM_EN;
		3278
		3279	r->plane[pipe][i][level] = temp;
		3280	}
		3281
		3282	temp = 0;
		3283
6084	serge	3284	temp \|= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3285	temp \|= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
5354	serge	3286
6084	serge	3287	if (p_wm->wm[level].plane_en[PLANE_CURSOR])
5354	serge	3288	temp \|= PLANE_WM_EN;
		3289
6084	serge	3290	r->plane[pipe][PLANE_CURSOR][level] = temp;
5354	serge	3291
		3292	}
		3293
		3294	/* transition WMs */
		3295	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3296	temp = 0;
		3297	temp \|= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
		3298	temp \|= p_wm->trans_wm.plane_res_b[i];
		3299	if (p_wm->trans_wm.plane_en[i])
		3300	temp \|= PLANE_WM_EN;
		3301
		3302	r->plane_trans[pipe][i] = temp;
		3303	}
		3304
		3305	temp = 0;
6084	serge	3306	temp \|= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3307	temp \|= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
		3308	if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
5354	serge	3309	temp \|= PLANE_WM_EN;
		3310
6084	serge	3311	r->plane_trans[pipe][PLANE_CURSOR] = temp;
5354	serge	3312
		3313	r->wm_linetime[pipe] = p_wm->linetime;
		3314	}
		3315
6937	serge	3316	static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
		3317	i915_reg_t reg,
5354	serge	3318	const struct skl_ddb_entry *entry)
		3319	{
		3320	if (entry->end)
		3321	I915_WRITE(reg, (entry->end - 1) << 16 \| entry->start);
		3322	else
		3323	I915_WRITE(reg, 0);
		3324	}
		3325
		3326	static void skl_write_wm_values(struct drm_i915_private *dev_priv,
		3327	const struct skl_wm_values *new)
		3328	{
		3329	struct drm_device *dev = dev_priv->dev;
		3330	struct intel_crtc *crtc;
		3331
6937	serge	3332	for_each_intel_crtc(dev, crtc) {
5354	serge	3333	int i, level, max_level = ilk_wm_max_level(dev);
		3334	enum pipe pipe = crtc->pipe;
		3335
		3336	if (!new->dirty[pipe])
		3337	continue;
		3338
		3339	I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
		3340
		3341	for (level = 0; level <= max_level; level++) {
		3342	for (i = 0; i < intel_num_planes(crtc); i++)
		3343	I915_WRITE(PLANE_WM(pipe, i, level),
		3344	new->plane[pipe][i][level]);
		3345	I915_WRITE(CUR_WM(pipe, level),
6084	serge	3346	new->plane[pipe][PLANE_CURSOR][level]);
5354	serge	3347	}
		3348	for (i = 0; i < intel_num_planes(crtc); i++)
		3349	I915_WRITE(PLANE_WM_TRANS(pipe, i),
		3350	new->plane_trans[pipe][i]);
6084	serge	3351	I915_WRITE(CUR_WM_TRANS(pipe),
		3352	new->plane_trans[pipe][PLANE_CURSOR]);
5354	serge	3353
6084	serge	3354	for (i = 0; i < intel_num_planes(crtc); i++) {
5354	serge	3355	skl_ddb_entry_write(dev_priv,
		3356	PLANE_BUF_CFG(pipe, i),
		3357	&new->ddb.plane[pipe][i]);
6084	serge	3358	skl_ddb_entry_write(dev_priv,
		3359	PLANE_NV12_BUF_CFG(pipe, i),
		3360	&new->ddb.y_plane[pipe][i]);
		3361	}
5354	serge	3362
		3363	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
6084	serge	3364	&new->ddb.plane[pipe][PLANE_CURSOR]);
5354	serge	3365	}
		3366	}
		3367
		3368	/*
		3369	* When setting up a new DDB allocation arrangement, we need to correctly
		3370	* sequence the times at which the new allocations for the pipes are taken into
		3371	* account or we'll have pipes fetching from space previously allocated to
		3372	* another pipe.
		3373	*
		3374	* Roughly the sequence looks like:
		3375	* 1. re-allocate the pipe(s) with the allocation being reduced and not
		3376	* overlapping with a previous light-up pipe (another way to put it is:
		3377	* pipes with their new allocation strickly included into their old ones).
		3378	* 2. re-allocate the other pipes that get their allocation reduced
		3379	* 3. allocate the pipes having their allocation increased
		3380	*
		3381	* Steps 1. and 2. are here to take care of the following case:
		3382	* - Initially DDB looks like this:
		3383	* \| B \| C \|
		3384	* - enable pipe A.
		3385	* - pipe B has a reduced DDB allocation that overlaps with the old pipe C
		3386	* allocation
		3387	* \| A \| B \| C \|
		3388	*
		3389	* We need to sequence the re-allocation: C, B, A (and not B, C, A).
		3390	*/
		3391
		3392	static void
		3393	skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
		3394	{
		3395	int plane;
		3396
		3397	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
		3398
6084	serge	3399	for_each_plane(dev_priv, pipe, plane) {
5354	serge	3400	I915_WRITE(PLANE_SURF(pipe, plane),
		3401	I915_READ(PLANE_SURF(pipe, plane)));
		3402	}
		3403	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
		3404	}
		3405
		3406	static bool
		3407	skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
		3408	const struct skl_ddb_allocation *new,
		3409	enum pipe pipe)
		3410	{
		3411	uint16_t old_size, new_size;
		3412
		3413	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
		3414	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
		3415
		3416	return old_size != new_size &&
		3417	new->pipe[pipe].start >= old->pipe[pipe].start &&
		3418	new->pipe[pipe].end <= old->pipe[pipe].end;
		3419	}
		3420
		3421	static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
		3422	struct skl_wm_values *new_values)
		3423	{
		3424	struct drm_device *dev = dev_priv->dev;
		3425	struct skl_ddb_allocation cur_ddb, new_ddb;
6084	serge	3426	bool reallocated[I915_MAX_PIPES] = {};
5354	serge	3427	struct intel_crtc *crtc;
		3428	enum pipe pipe;
		3429
		3430	new_ddb = &new_values->ddb;
		3431	cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3432
		3433	/*
		3434	* First pass: flush the pipes with the new allocation contained into
		3435	* the old space.
		3436	*
		3437	* We'll wait for the vblank on those pipes to ensure we can safely
		3438	* re-allocate the freed space without this pipe fetching from it.
		3439	*/
		3440	for_each_intel_crtc(dev, crtc) {
		3441	if (!crtc->active)
		3442	continue;
		3443
		3444	pipe = crtc->pipe;
		3445
		3446	if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
		3447	continue;
		3448
		3449	skl_wm_flush_pipe(dev_priv, pipe, 1);
		3450	intel_wait_for_vblank(dev, pipe);
		3451
		3452	reallocated[pipe] = true;
		3453	}
		3454
		3455
		3456	/*
		3457	* Second pass: flush the pipes that are having their allocation
		3458	* reduced, but overlapping with a previous allocation.
		3459	*
		3460	* Here as well we need to wait for the vblank to make sure the freed
		3461	* space is not used anymore.
		3462	*/
		3463	for_each_intel_crtc(dev, crtc) {
		3464	if (!crtc->active)
		3465	continue;
		3466
		3467	pipe = crtc->pipe;
		3468
		3469	if (reallocated[pipe])
		3470	continue;
		3471
		3472	if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
		3473	skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
		3474	skl_wm_flush_pipe(dev_priv, pipe, 2);
		3475	intel_wait_for_vblank(dev, pipe);
6084	serge	3476	reallocated[pipe] = true;
5354	serge	3477	}
		3478	}
		3479
		3480	/*
		3481	* Third pass: flush the pipes that got more space allocated.
		3482	*
		3483	* We don't need to actively wait for the update here, next vblank
		3484	* will just get more DDB space with the correct WM values.
		3485	*/
		3486	for_each_intel_crtc(dev, crtc) {
		3487	if (!crtc->active)
		3488	continue;
		3489
		3490	pipe = crtc->pipe;
		3491
		3492	/*
		3493	* At this point, only the pipes more space than before are
		3494	* left to re-allocate.
		3495	*/
		3496	if (reallocated[pipe])
		3497	continue;
		3498
		3499	skl_wm_flush_pipe(dev_priv, pipe, 3);
		3500	}
		3501	}
		3502
		3503	static bool skl_update_pipe_wm(struct drm_crtc *crtc,
		3504	struct skl_ddb_allocation ddb, / out */
		3505	struct skl_pipe_wm pipe_wm / out */)
		3506	{
		3507	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3508	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5354	serge	3509
6937	serge	3510	skl_allocate_pipe_ddb(cstate, ddb);
		3511	skl_compute_pipe_wm(cstate, ddb, pipe_wm);
5354	serge	3512
6937	serge	3513	if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
5354	serge	3514	return false;
		3515
6937	serge	3516	intel_crtc->wm.active.skl = *pipe_wm;
6084	serge	3517
5354	serge	3518	return true;
		3519	}
		3520
		3521	static void skl_update_other_pipe_wm(struct drm_device *dev,
		3522	struct drm_crtc *crtc,
		3523	struct skl_wm_values *r)
		3524	{
		3525	struct intel_crtc *intel_crtc;
		3526	struct intel_crtc *this_crtc = to_intel_crtc(crtc);
		3527
		3528	/*
		3529	* If the WM update hasn't changed the allocation for this_crtc (the
		3530	* crtc we are currently computing the new WM values for), other
		3531	* enabled crtcs will keep the same allocation and we don't need to
		3532	* recompute anything for them.
		3533	*/
		3534	if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
		3535	return;
		3536
		3537	/*
		3538	* Otherwise, because of this_crtc being freshly enabled/disabled, the
		3539	* other active pipes need new DDB allocation and WM values.
		3540	*/
6937	serge	3541	for_each_intel_crtc(dev, intel_crtc) {
5354	serge	3542	struct skl_pipe_wm pipe_wm = {};
		3543	bool wm_changed;
		3544
		3545	if (this_crtc->pipe == intel_crtc->pipe)
		3546	continue;
		3547
		3548	if (!intel_crtc->active)
		3549	continue;
		3550
		3551	wm_changed = skl_update_pipe_wm(&intel_crtc->base,
		3552	&r->ddb, &pipe_wm);
		3553
		3554	/*
		3555	* If we end up re-computing the other pipe WM values, it's
		3556	* because it was really needed, so we expect the WM values to
		3557	* be different.
6084	serge	3558	*/
5354	serge	3559	WARN_ON(!wm_changed);
		3560
6937	serge	3561	skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc);
5354	serge	3562	r->dirty[intel_crtc->pipe] = true;
		3563	}
		3564	}
		3565
6084	serge	3566	static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
		3567	{
		3568	watermarks->wm_linetime[pipe] = 0;
		3569	memset(watermarks->plane[pipe], 0,
		3570	sizeof(uint32_t) * 8 * I915_MAX_PLANES);
		3571	memset(watermarks->plane_trans[pipe],
		3572	0, sizeof(uint32_t) * I915_MAX_PLANES);
		3573	watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
		3574
		3575	/* Clear ddb entries for pipe */
		3576	memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
		3577	memset(&watermarks->ddb.plane[pipe], 0,
		3578	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3579	memset(&watermarks->ddb.y_plane[pipe], 0,
		3580	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3581	memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
		3582	sizeof(struct skl_ddb_entry));
		3583
		3584	}
		3585
5354	serge	3586	static void skl_update_wm(struct drm_crtc *crtc)
		3587	{
		3588	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3589	struct drm_device *dev = crtc->dev;
		3590	struct drm_i915_private *dev_priv = dev->dev_private;
		3591	struct skl_wm_values *results = &dev_priv->wm.skl_results;
6937	serge	3592	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
		3593	struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl;
5354	serge	3594
		3595
6084	serge	3596	/* Clear all dirty flags */
		3597	memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
		3598
		3599	skl_clear_wm(results, intel_crtc->pipe);
		3600
6937	serge	3601	if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm))
5354	serge	3602	return;
		3603
6937	serge	3604	skl_compute_wm_results(dev, pipe_wm, results, intel_crtc);
5354	serge	3605	results->dirty[intel_crtc->pipe] = true;
		3606
6937	serge	3607	skl_update_other_pipe_wm(dev, crtc, results);
5354	serge	3608	skl_write_wm_values(dev_priv, results);
		3609	skl_flush_wm_values(dev_priv, results);
		3610
		3611	/* store the new configuration */
		3612	dev_priv->wm.skl_hw = *results;
		3613	}
		3614
6937	serge	3615	static void ilk_compute_wm_config(struct drm_device *dev,
		3616	struct intel_wm_config *config)
5354	serge	3617	{
6937	serge	3618	struct intel_crtc *crtc;
5354	serge	3619
6937	serge	3620	/* Compute the currently _active_ config */
		3621	for_each_intel_crtc(dev, crtc) {
		3622	const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5354	serge	3623
6937	serge	3624	if (!wm->pipe_enabled)
		3625	continue;
6084	serge	3626
6937	serge	3627	config->sprites_enabled \|= wm->sprites_enabled;
		3628	config->sprites_scaled \|= wm->sprites_scaled;
		3629	config->num_pipes_active++;
		3630	}
5354	serge	3631	}
		3632
6937	serge	3633	static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
4560	Serge	3634	{
6937	serge	3635	struct drm_device *dev = dev_priv->dev;
		3636	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
4560	Serge	3637	struct ilk_wm_maximums max;
6937	serge	3638	struct intel_wm_config config = {};
4560	Serge	3639	struct ilk_wm_values results = {};
4104	Serge	3640	enum intel_ddb_partitioning partitioning;
		3641
5060	serge	3642	ilk_compute_wm_config(dev, &config);
		3643
4560	Serge	3644	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		3645	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		3646
		3647	/* 5/6 split only in single pipe config on IVB+ */
		3648	if (INTEL_INFO(dev)->gen >= 7 &&
		3649	config.num_pipes_active == 1 && config.sprites_enabled) {
		3650	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		3651	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		3652
		3653	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	3654	} else {
4560	Serge	3655	best_lp_wm = &lp_wm_1_2;
4104	Serge	3656	}
		3657
4560	Serge	3658	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	3659	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		3660
4560	Serge	3661	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		3662
		3663	ilk_write_wm_values(dev_priv, &results);
4104	Serge	3664	}
		3665
6937	serge	3666	static void ilk_update_wm(struct drm_crtc *crtc)
4104	Serge	3667	{
6937	serge	3668	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
		3669	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3670	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
4104	Serge	3671
6937	serge	3672	WARN_ON(cstate->base.active != intel_crtc->active);
		3673
4560	Serge	3674	/*
		3675	* IVB workaround: must disable low power watermarks for at least
		3676	* one frame before enabling scaling. LP watermarks can be re-enabled
		3677	* when scaling is disabled.
		3678	*
		3679	* WaCxSRDisabledForSpriteScaling:ivb
		3680	*/
6937	serge	3681	if (cstate->disable_lp_wm) {
		3682	ilk_disable_lp_wm(crtc->dev);
		3683	intel_wait_for_vblank(crtc->dev, intel_crtc->pipe);
		3684	}
4560	Serge	3685
6937	serge	3686	intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk;
		3687
		3688	ilk_program_watermarks(dev_priv);
4104	Serge	3689	}
		3690
5354	serge	3691	static void skl_pipe_wm_active_state(uint32_t val,
		3692	struct skl_pipe_wm *active,
		3693	bool is_transwm,
		3694	bool is_cursor,
		3695	int i,
		3696	int level)
		3697	{
		3698	bool is_enabled = (val & PLANE_WM_EN) != 0;
		3699
		3700	if (!is_transwm) {
		3701	if (!is_cursor) {
		3702	active->wm[level].plane_en[i] = is_enabled;
		3703	active->wm[level].plane_res_b[i] =
		3704	val & PLANE_WM_BLOCKS_MASK;
		3705	active->wm[level].plane_res_l[i] =
		3706	(val >> PLANE_WM_LINES_SHIFT) &
		3707	PLANE_WM_LINES_MASK;
		3708	} else {
6084	serge	3709	active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
		3710	active->wm[level].plane_res_b[PLANE_CURSOR] =
5354	serge	3711	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3712	active->wm[level].plane_res_l[PLANE_CURSOR] =
5354	serge	3713	(val >> PLANE_WM_LINES_SHIFT) &
		3714	PLANE_WM_LINES_MASK;
		3715	}
		3716	} else {
		3717	if (!is_cursor) {
		3718	active->trans_wm.plane_en[i] = is_enabled;
		3719	active->trans_wm.plane_res_b[i] =
		3720	val & PLANE_WM_BLOCKS_MASK;
		3721	active->trans_wm.plane_res_l[i] =
		3722	(val >> PLANE_WM_LINES_SHIFT) &
		3723	PLANE_WM_LINES_MASK;
		3724	} else {
6084	serge	3725	active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
		3726	active->trans_wm.plane_res_b[PLANE_CURSOR] =
5354	serge	3727	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3728	active->trans_wm.plane_res_l[PLANE_CURSOR] =
5354	serge	3729	(val >> PLANE_WM_LINES_SHIFT) &
		3730	PLANE_WM_LINES_MASK;
		3731	}
		3732	}
		3733	}
		3734
		3735	static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
		3736	{
		3737	struct drm_device *dev = crtc->dev;
		3738	struct drm_i915_private *dev_priv = dev->dev_private;
		3739	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
		3740	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3741	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
		3742	struct skl_pipe_wm *active = &cstate->wm.optimal.skl;
5354	serge	3743	enum pipe pipe = intel_crtc->pipe;
		3744	int level, i, max_level;
		3745	uint32_t temp;
		3746
		3747	max_level = ilk_wm_max_level(dev);
		3748
		3749	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
		3750
		3751	for (level = 0; level <= max_level; level++) {
		3752	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3753	hw->plane[pipe][i][level] =
		3754	I915_READ(PLANE_WM(pipe, i, level));
6084	serge	3755	hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
5354	serge	3756	}
		3757
		3758	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3759	hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
6084	serge	3760	hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
5354	serge	3761
6084	serge	3762	if (!intel_crtc->active)
5354	serge	3763	return;
		3764
		3765	hw->dirty[pipe] = true;
		3766
		3767	active->linetime = hw->wm_linetime[pipe];
		3768
		3769	for (level = 0; level <= max_level; level++) {
		3770	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3771	temp = hw->plane[pipe][i][level];
		3772	skl_pipe_wm_active_state(temp, active, false,
		3773	false, i, level);
		3774	}
6084	serge	3775	temp = hw->plane[pipe][PLANE_CURSOR][level];
5354	serge	3776	skl_pipe_wm_active_state(temp, active, false, true, i, level);
		3777	}
		3778
		3779	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3780	temp = hw->plane_trans[pipe][i];
		3781	skl_pipe_wm_active_state(temp, active, true, false, i, 0);
		3782	}
		3783
6084	serge	3784	temp = hw->plane_trans[pipe][PLANE_CURSOR];
5354	serge	3785	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
6937	serge	3786
		3787	intel_crtc->wm.active.skl = *active;
5354	serge	3788	}
		3789
		3790	void skl_wm_get_hw_state(struct drm_device *dev)
		3791	{
		3792	struct drm_i915_private *dev_priv = dev->dev_private;
		3793	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
		3794	struct drm_crtc *crtc;
		3795
		3796	skl_ddb_get_hw_state(dev_priv, ddb);
		3797	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3798	skl_pipe_wm_get_hw_state(crtc);
		3799	}
		3800
4560	Serge	3801	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	3802	{
4560	Serge	3803	struct drm_device *dev = crtc->dev;
		3804	struct drm_i915_private *dev_priv = dev->dev_private;
		3805	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		3806	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3807	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
		3808	struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
4560	Serge	3809	enum pipe pipe = intel_crtc->pipe;
6937	serge	3810	static const i915_reg_t wm0_pipe_reg[] = {
4560	Serge	3811	[PIPE_A] = WM0_PIPEA_ILK,
		3812	[PIPE_B] = WM0_PIPEB_ILK,
		3813	[PIPE_C] = WM0_PIPEC_IVB,
		3814	};
3031	serge	3815
4560	Serge	3816	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		3817	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		3818	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	3819
6660	serge	3820	memset(active, 0, sizeof(*active));
		3821
6084	serge	3822	active->pipe_enabled = intel_crtc->active;
5060	serge	3823
		3824	if (active->pipe_enabled) {
4560	Serge	3825	u32 tmp = hw->wm_pipe[pipe];
3031	serge	3826
4560	Serge	3827	/*
		3828	* For active pipes LP0 watermark is marked as
		3829	* enabled, and LP1+ watermaks as disabled since
		3830	* we can't really reverse compute them in case
		3831	* multiple pipes are active.
		3832	*/
		3833	active->wm[0].enable = true;
		3834	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		3835	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		3836	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		3837	active->linetime = hw->wm_linetime[pipe];
		3838	} else {
		3839	int level, max_level = ilk_wm_max_level(dev);
3031	serge	3840
4560	Serge	3841	/*
		3842	* For inactive pipes, all watermark levels
		3843	* should be marked as enabled but zeroed,
		3844	* which is what we'd compute them to.
		3845	*/
		3846	for (level = 0; level <= max_level; level++)
		3847	active->wm[level].enable = true;
3031	serge	3848	}
6937	serge	3849
		3850	intel_crtc->wm.active.ilk = *active;
3031	serge	3851	}
		3852
6084	serge	3853	#define _FW_WM(value, plane) \
		3854	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
		3855	#define _FW_WM_VLV(value, plane) \
		3856	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
		3857
		3858	static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
		3859	struct vlv_wm_values *wm)
		3860	{
		3861	enum pipe pipe;
		3862	uint32_t tmp;
		3863
		3864	for_each_pipe(dev_priv, pipe) {
		3865	tmp = I915_READ(VLV_DDL(pipe));
		3866
		3867	wm->ddl[pipe].primary =
		3868	(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3869	wm->ddl[pipe].cursor =
		3870	(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3871	wm->ddl[pipe].sprite[0] =
		3872	(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3873	wm->ddl[pipe].sprite[1] =
		3874	(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3875	}
		3876
		3877	tmp = I915_READ(DSPFW1);
		3878	wm->sr.plane = _FW_WM(tmp, SR);
		3879	wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
		3880	wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
		3881	wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
		3882
		3883	tmp = I915_READ(DSPFW2);
		3884	wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
		3885	wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
		3886	wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
		3887
		3888	tmp = I915_READ(DSPFW3);
		3889	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
		3890
		3891	if (IS_CHERRYVIEW(dev_priv)) {
		3892	tmp = I915_READ(DSPFW7_CHV);
		3893	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3894	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3895
		3896	tmp = I915_READ(DSPFW8_CHV);
		3897	wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
		3898	wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
		3899
		3900	tmp = I915_READ(DSPFW9_CHV);
		3901	wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
		3902	wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
		3903
		3904	tmp = I915_READ(DSPHOWM);
		3905	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3906	wm->pipe[PIPE_C].sprite[1] \|= _FW_WM(tmp, SPRITEF_HI) << 8;
		3907	wm->pipe[PIPE_C].sprite[0] \|= _FW_WM(tmp, SPRITEE_HI) << 8;
		3908	wm->pipe[PIPE_C].primary \|= _FW_WM(tmp, PLANEC_HI) << 8;
		3909	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3910	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3911	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3912	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3913	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3914	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3915	} else {
		3916	tmp = I915_READ(DSPFW7);
		3917	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3918	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3919
		3920	tmp = I915_READ(DSPHOWM);
		3921	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3922	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3923	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3924	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3925	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3926	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3927	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3928	}
		3929	}
		3930
		3931	#undef _FW_WM
		3932	#undef _FW_WM_VLV
		3933
		3934	void vlv_wm_get_hw_state(struct drm_device *dev)
		3935	{
		3936	struct drm_i915_private *dev_priv = to_i915(dev);
		3937	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
		3938	struct intel_plane *plane;
		3939	enum pipe pipe;
		3940	u32 val;
		3941
		3942	vlv_read_wm_values(dev_priv, wm);
		3943
		3944	for_each_intel_plane(dev, plane) {
		3945	switch (plane->base.type) {
		3946	int sprite;
		3947	case DRM_PLANE_TYPE_CURSOR:
		3948	plane->wm.fifo_size = 63;
		3949	break;
		3950	case DRM_PLANE_TYPE_PRIMARY:
		3951	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
		3952	break;
		3953	case DRM_PLANE_TYPE_OVERLAY:
		3954	sprite = plane->plane;
		3955	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
		3956	break;
		3957	}
		3958	}
		3959
		3960	wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
		3961	wm->level = VLV_WM_LEVEL_PM2;
		3962
		3963	if (IS_CHERRYVIEW(dev_priv)) {
		3964	mutex_lock(&dev_priv->rps.hw_lock);
		3965
		3966	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		3967	if (val & DSP_MAXFIFO_PM5_ENABLE)
		3968	wm->level = VLV_WM_LEVEL_PM5;
		3969
		3970	/*
		3971	* If DDR DVFS is disabled in the BIOS, Punit
		3972	* will never ack the request. So if that happens
		3973	* assume we don't have to enable/disable DDR DVFS
		3974	* dynamically. To test that just set the REQ_ACK
		3975	* bit to poke the Punit, but don't change the
		3976	* HIGH/LOW bits so that we don't actually change
		3977	* the current state.
		3978	*/
		3979	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		3980	val \|= FORCE_DDR_FREQ_REQ_ACK;
		3981	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		3982
		3983	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		3984	FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
		3985	DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
		3986	"assuming DDR DVFS is disabled\n");
		3987	dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		3988	} else {
		3989	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		3990	if ((val & FORCE_DDR_HIGH_FREQ) == 0)
		3991	wm->level = VLV_WM_LEVEL_DDR_DVFS;
		3992	}
		3993
		3994	mutex_unlock(&dev_priv->rps.hw_lock);
		3995	}
		3996
		3997	for_each_pipe(dev_priv, pipe)
		3998	DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
		3999	pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
		4000	wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
		4001
		4002	DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		4003	wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
		4004	}
		4005
4560	Serge	4006	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	4007	{
		4008	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4009	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		4010	struct drm_crtc *crtc;
3031	serge	4011
5060	serge	4012	for_each_crtc(dev, crtc)
4560	Serge	4013	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	4014
4560	Serge	4015	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		4016	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		4017	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	4018
4560	Serge	4019	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	4020	if (INTEL_INFO(dev)->gen >= 7) {
5354	serge	4021	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		4022	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	4023	}
3031	serge	4024
4560	Serge	4025	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4026	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		4027	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		4028	else if (IS_IVYBRIDGE(dev))
		4029	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		4030	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	4031
4560	Serge	4032	hw->enable_fbc_wm =
		4033	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	4034	}
		4035
		4036	/**
		4037	* intel_update_watermarks - update FIFO watermark values based on current modes
		4038	*
		4039	* Calculate watermark values for the various WM regs based on current mode
		4040	* and plane configuration.
		4041	*
		4042	* There are several cases to deal with here:
		4043	* - normal (i.e. non-self-refresh)
		4044	* - self-refresh (SR) mode
		4045	* - lines are large relative to FIFO size (buffer can hold up to 2)
		4046	* - lines are small relative to FIFO size (buffer can hold more than 2
		4047	* lines), so need to account for TLB latency
		4048	*
		4049	* The normal calculation is:
		4050	* watermark = dotclock * bytes per pixel * latency
		4051	* where latency is platform & configuration dependent (we assume pessimal
		4052	* values here).
		4053	*
		4054	* The SR calculation is:
		4055	* watermark = (trunc(latency/line time)+1) * surface width *
		4056	* bytes per pixel
		4057	* where
		4058	* line time = htotal / dotclock
		4059	* surface width = hdisplay for normal plane and 64 for cursor
		4060	* and latency is assumed to be high, as above.
		4061	*
		4062	* The final value programmed to the register should always be rounded up,
		4063	* and include an extra 2 entries to account for clock crossings.
		4064	*
		4065	* We don't use the sprite, so we can ignore that. And on Crestline we have
		4066	* to set the non-SR watermarks to 8.
		4067	*/
4560	Serge	4068	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	4069	{
4560	Serge	4070	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	4071
		4072	if (dev_priv->display.update_wm)
4560	Serge	4073	dev_priv->display.update_wm(crtc);
3031	serge	4074	}
		4075
		4076	/**
		4077	* Lock protecting IPS related data structures
		4078	*/
		4079	DEFINE_SPINLOCK(mchdev_lock);
		4080
		4081	/* Global for IPS driver to get at the current i915 device. Protected by
		4082	* mchdev_lock. */
		4083	static struct drm_i915_private *i915_mch_dev;
		4084
		4085	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		4086	{
		4087	struct drm_i915_private *dev_priv = dev->dev_private;
		4088	u16 rgvswctl;
		4089
		4090	assert_spin_locked(&mchdev_lock);
		4091
		4092	rgvswctl = I915_READ16(MEMSWCTL);
		4093	if (rgvswctl & MEMCTL_CMD_STS) {
		4094	DRM_DEBUG("gpu busy, RCS change rejected\n");
		4095	return false; /* still busy with another command */
		4096	}
		4097
		4098	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		4099	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		4100	I915_WRITE16(MEMSWCTL, rgvswctl);
		4101	POSTING_READ16(MEMSWCTL);
		4102
		4103	rgvswctl \|= MEMCTL_CMD_STS;
		4104	I915_WRITE16(MEMSWCTL, rgvswctl);
		4105
		4106	return true;
		4107	}
		4108
		4109	static void ironlake_enable_drps(struct drm_device *dev)
		4110	{
		4111	struct drm_i915_private *dev_priv = dev->dev_private;
		4112	u32 rgvmodectl = I915_READ(MEMMODECTL);
		4113	u8 fmax, fmin, fstart, vstart;
		4114
		4115	spin_lock_irq(&mchdev_lock);
		4116
		4117	/* Enable temp reporting */
		4118	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		4119	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		4120
		4121	/* 100ms RC evaluation intervals */
		4122	I915_WRITE(RCUPEI, 100000);
		4123	I915_WRITE(RCDNEI, 100000);
		4124
		4125	/* Set max/min thresholds to 90ms and 80ms respectively */
		4126	I915_WRITE(RCBMAXAVG, 90000);
		4127	I915_WRITE(RCBMINAVG, 80000);
		4128
		4129	I915_WRITE(MEMIHYST, 1);
		4130
		4131	/* Set up min, max, and cur for interrupt handling */
		4132	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		4133	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		4134	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		4135	MEMMODE_FSTART_SHIFT;
		4136
6084	serge	4137	vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
3031	serge	4138	PXVFREQ_PX_SHIFT;
		4139
		4140	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		4141	dev_priv->ips.fstart = fstart;
		4142
		4143	dev_priv->ips.max_delay = fstart;
		4144	dev_priv->ips.min_delay = fmin;
		4145	dev_priv->ips.cur_delay = fstart;
		4146
		4147	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		4148	fmax, fmin, fstart);
		4149
		4150	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		4151
		4152	/*
		4153	* Interrupts will be enabled in ironlake_irq_postinstall
		4154	*/
		4155
		4156	I915_WRITE(VIDSTART, vstart);
		4157	POSTING_READ(VIDSTART);
		4158
		4159	rgvmodectl \|= MEMMODE_SWMODE_EN;
		4160	I915_WRITE(MEMMODECTL, rgvmodectl);
		4161
		4162	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		4163	DRM_ERROR("stuck trying to change perf mode\n");
		4164	mdelay(1);
		4165
		4166	ironlake_set_drps(dev, fstart);
		4167
6084	serge	4168	dev_priv->ips.last_count1 = I915_READ(DMIEC) +
		4169	I915_READ(DDREC) + I915_READ(CSIEC);
5060	serge	4170	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6084	serge	4171	dev_priv->ips.last_count2 = I915_READ(GFXEC);
5060	serge	4172	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	4173
		4174	spin_unlock_irq(&mchdev_lock);
		4175	}
		4176
		4177	static void ironlake_disable_drps(struct drm_device *dev)
		4178	{
		4179	struct drm_i915_private *dev_priv = dev->dev_private;
		4180	u16 rgvswctl;
		4181
		4182	spin_lock_irq(&mchdev_lock);
		4183
		4184	rgvswctl = I915_READ16(MEMSWCTL);
		4185
		4186	/* Ack interrupts, disable EFC interrupt */
		4187	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		4188	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		4189	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		4190	I915_WRITE(DEIIR, DE_PCU_EVENT);
		4191	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		4192
		4193	/* Go back to the starting frequency */
		4194	ironlake_set_drps(dev, dev_priv->ips.fstart);
		4195	mdelay(1);
		4196	rgvswctl \|= MEMCTL_CMD_STS;
		4197	I915_WRITE(MEMSWCTL, rgvswctl);
		4198	mdelay(1);
		4199
		4200	spin_unlock_irq(&mchdev_lock);
		4201	}
		4202
		4203	/* There's a funny hw issue where the hw returns all 0 when reading from
		4204	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		4205	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		4206	* all limits and the gpu stuck at whatever frequency it is at atm).
		4207	*/
6084	serge	4208	static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	4209	{
		4210	u32 limits;
		4211
		4212	/* Only set the down limit when we've reached the lowest level to avoid
		4213	* getting more interrupts, otherwise leave this clear. This prevents a
		4214	* race in the hw when coming out of rc6: There's a tiny window where
		4215	* the hw runs at the minimal clock before selecting the desired
		4216	* frequency, if the down threshold expires in that window we will not
		4217	* receive a down interrupt. */
6084	serge	4218	if (IS_GEN9(dev_priv->dev)) {
		4219	limits = (dev_priv->rps.max_freq_softlimit) << 23;
		4220	if (val <= dev_priv->rps.min_freq_softlimit)
		4221	limits \|= (dev_priv->rps.min_freq_softlimit) << 14;
		4222	} else {
		4223	limits = dev_priv->rps.max_freq_softlimit << 24;
		4224	if (val <= dev_priv->rps.min_freq_softlimit)
		4225	limits \|= dev_priv->rps.min_freq_softlimit << 16;
		4226	}
3031	serge	4227
		4228	return limits;
		4229	}
		4230
4560	Serge	4231	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		4232	{
		4233	int new_power;
6084	serge	4234	u32 threshold_up = 0, threshold_down = 0; /* in % */
		4235	u32 ei_up = 0, ei_down = 0;
4560	Serge	4236
		4237	new_power = dev_priv->rps.power;
		4238	switch (dev_priv->rps.power) {
		4239	case LOW_POWER:
5060	serge	4240	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	4241	new_power = BETWEEN;
		4242	break;
		4243
		4244	case BETWEEN:
5060	serge	4245	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	4246	new_power = LOW_POWER;
5060	serge	4247	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	4248	new_power = HIGH_POWER;
		4249	break;
		4250
		4251	case HIGH_POWER:
5060	serge	4252	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	4253	new_power = BETWEEN;
		4254	break;
		4255	}
		4256	/* Max/min bins are special */
6084	serge	4257	if (val <= dev_priv->rps.min_freq_softlimit)
4560	Serge	4258	new_power = LOW_POWER;
6084	serge	4259	if (val >= dev_priv->rps.max_freq_softlimit)
4560	Serge	4260	new_power = HIGH_POWER;
		4261	if (new_power == dev_priv->rps.power)
		4262	return;
		4263
		4264	/* Note the units here are not exactly 1us, but 1280ns. */
		4265	switch (new_power) {
		4266	case LOW_POWER:
		4267	/* Upclock if more than 95% busy over 16ms */
6084	serge	4268	ei_up = 16000;
		4269	threshold_up = 95;
4560	Serge	4270
		4271	/* Downclock if less than 85% busy over 32ms */
6084	serge	4272	ei_down = 32000;
		4273	threshold_down = 85;
4560	Serge	4274	break;
		4275
		4276	case BETWEEN:
		4277	/* Upclock if more than 90% busy over 13ms */
6084	serge	4278	ei_up = 13000;
		4279	threshold_up = 90;
4560	Serge	4280
		4281	/* Downclock if less than 75% busy over 32ms */
6084	serge	4282	ei_down = 32000;
		4283	threshold_down = 75;
4560	Serge	4284	break;
		4285
		4286	case HIGH_POWER:
		4287	/* Upclock if more than 85% busy over 10ms */
6084	serge	4288	ei_up = 10000;
		4289	threshold_up = 85;
4560	Serge	4290
		4291	/* Downclock if less than 60% busy over 32ms */
6084	serge	4292	ei_down = 32000;
		4293	threshold_down = 60;
4560	Serge	4294	break;
		4295	}
		4296
6084	serge	4297	I915_WRITE(GEN6_RP_UP_EI,
		4298	GT_INTERVAL_FROM_US(dev_priv, ei_up));
		4299	I915_WRITE(GEN6_RP_UP_THRESHOLD,
		4300	GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
		4301
		4302	I915_WRITE(GEN6_RP_DOWN_EI,
		4303	GT_INTERVAL_FROM_US(dev_priv, ei_down));
		4304	I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
		4305	GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
		4306
		4307	I915_WRITE(GEN6_RP_CONTROL,
		4308	GEN6_RP_MEDIA_TURBO \|
		4309	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4310	GEN6_RP_MEDIA_IS_GFX \|
		4311	GEN6_RP_ENABLE \|
		4312	GEN6_RP_UP_BUSY_AVG \|
		4313	GEN6_RP_DOWN_IDLE_AVG);
		4314
4560	Serge	4315	dev_priv->rps.power = new_power;
6084	serge	4316	dev_priv->rps.up_threshold = threshold_up;
		4317	dev_priv->rps.down_threshold = threshold_down;
4560	Serge	4318	dev_priv->rps.last_adj = 0;
		4319	}
		4320
5060	serge	4321	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		4322	{
		4323	u32 mask = 0;
		4324
		4325	if (val > dev_priv->rps.min_freq_softlimit)
6937	serge	4326	mask \|= GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
5060	serge	4327	if (val < dev_priv->rps.max_freq_softlimit)
6084	serge	4328	mask \|= GEN6_PM_RP_UP_EI_EXPIRED \| GEN6_PM_RP_UP_THRESHOLD;
5060	serge	4329
		4330	mask &= dev_priv->pm_rps_events;
		4331
6084	serge	4332	return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
5060	serge	4333	}
		4334
		4335	/* gen6_set_rps is called to update the frequency request, but should also be
		4336	* called when the range (min_delay and max_delay) is modified so that we can
		4337	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6084	serge	4338	static void gen6_set_rps(struct drm_device *dev, u8 val)
3031	serge	4339	{
		4340	struct drm_i915_private *dev_priv = dev->dev_private;
		4341
6084	serge	4342	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
6937	serge	4343	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
6084	serge	4344	return;
		4345
3243	Serge	4346	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6084	serge	4347	WARN_ON(val > dev_priv->rps.max_freq);
		4348	WARN_ON(val < dev_priv->rps.min_freq);
3031	serge	4349
5060	serge	4350	/* min/max delay may still have been modified so be sure to
		4351	* write the limits value.
		4352	*/
		4353	if (val != dev_priv->rps.cur_freq) {
5354	serge	4354	gen6_set_rps_thresholds(dev_priv, val);
4560	Serge	4355
6084	serge	4356	if (IS_GEN9(dev))
5354	serge	4357	I915_WRITE(GEN6_RPNSWREQ,
6084	serge	4358	GEN9_FREQUENCY(val));
		4359	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4360	I915_WRITE(GEN6_RPNSWREQ,
5354	serge	4361	HSW_FREQUENCY(val));
		4362	else
		4363	I915_WRITE(GEN6_RPNSWREQ,
		4364	GEN6_FREQUENCY(val) \|
		4365	GEN6_OFFSET(0) \|
		4366	GEN6_AGGRESSIVE_TURBO);
5060	serge	4367	}
3031	serge	4368
		4369	/* Make sure we continue to get interrupts
		4370	* until we hit the minimum or maximum frequencies.
		4371	*/
6084	serge	4372	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
5060	serge	4373	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	4374
		4375	POSTING_READ(GEN6_RPNSWREQ);
		4376
5060	serge	4377	dev_priv->rps.cur_freq = val;
6084	serge	4378	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3031	serge	4379	}
		4380
6084	serge	4381	static void valleyview_set_rps(struct drm_device *dev, u8 val)
5060	serge	4382	{
6084	serge	4383	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4384
6084	serge	4385	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4386	WARN_ON(val > dev_priv->rps.max_freq);
		4387	WARN_ON(val < dev_priv->rps.min_freq);
5060	serge	4388
6084	serge	4389	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
		4390	"Odd GPU freq value\n"))
		4391	val &= ~1;
5060	serge	4392
6084	serge	4393	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5060	serge	4394
6084	serge	4395	if (val != dev_priv->rps.cur_freq) {
		4396	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		4397	if (!IS_CHERRYVIEW(dev_priv))
		4398	gen6_set_rps_thresholds(dev_priv, val);
		4399	}
5060	serge	4400
6084	serge	4401	dev_priv->rps.cur_freq = val;
		4402	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
		4403	}
5060	serge	4404
6084	serge	4405	/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
		4406	*
		4407	* * If Gfx is Idle, then
		4408	* 1. Forcewake Media well.
		4409	* 2. Request idle freq.
		4410	* 3. Release Forcewake of Media well.
		4411	*/
		4412	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		4413	{
		4414	u32 val = dev_priv->rps.idle_freq;
5060	serge	4415
6084	serge	4416	if (dev_priv->rps.cur_freq <= val)
		4417	return;
5060	serge	4418
6084	serge	4419	/* Wake up the media well, as that takes a lot less
		4420	* power than the Render well. */
		4421	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
		4422	valleyview_set_rps(dev_priv->dev, val);
		4423	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
5060	serge	4424	}
		4425
6084	serge	4426	void gen6_rps_busy(struct drm_i915_private *dev_priv)
3031	serge	4427	{
4560	Serge	4428	mutex_lock(&dev_priv->rps.hw_lock);
		4429	if (dev_priv->rps.enabled) {
6937	serge	4430	if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED))
6084	serge	4431	gen6_rps_reset_ei(dev_priv);
		4432	I915_WRITE(GEN6_PMINTRMSK,
		4433	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4560	Serge	4434	}
		4435	mutex_unlock(&dev_priv->rps.hw_lock);
		4436	}
4104	Serge	4437
6084	serge	4438	void gen6_rps_idle(struct drm_i915_private *dev_priv)
4560	Serge	4439	{
		4440	struct drm_device *dev = dev_priv->dev;
4104	Serge	4441
4560	Serge	4442	mutex_lock(&dev_priv->rps.hw_lock);
		4443	if (dev_priv->rps.enabled) {
6937	serge	4444	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev))
6084	serge	4445	vlv_set_rps_idle(dev_priv);
4560	Serge	4446	else
6084	serge	4447	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4448	dev_priv->rps.last_adj = 0;
6937	serge	4449	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4560	Serge	4450	}
		4451	mutex_unlock(&dev_priv->rps.hw_lock);
6084	serge	4452
		4453	spin_lock(&dev_priv->rps.client_lock);
		4454	while (!list_empty(&dev_priv->rps.clients))
		4455	list_del_init(dev_priv->rps.clients.next);
		4456	spin_unlock(&dev_priv->rps.client_lock);
4104	Serge	4457	}
		4458
6084	serge	4459	void gen6_rps_boost(struct drm_i915_private *dev_priv,
		4460	struct intel_rps_client *rps,
		4461	unsigned long submitted)
4104	Serge	4462	{
6084	serge	4463	/* This is intentionally racy! We peek at the state here, then
		4464	* validate inside the RPS worker.
		4465	*/
		4466	if (!(dev_priv->mm.busy &&
		4467	dev_priv->rps.enabled &&
		4468	dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
		4469	return;
3031	serge	4470
6084	serge	4471	/* Force a RPS boost (and don't count it against the client) if
		4472	* the GPU is severely congested.
		4473	*/
		4474	if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
		4475	rps = NULL;
4104	Serge	4476
6084	serge	4477	spin_lock(&dev_priv->rps.client_lock);
		4478	if (rps == NULL \|\| list_empty(&rps->link)) {
		4479	spin_lock_irq(&dev_priv->irq_lock);
		4480	if (dev_priv->rps.interrupts_enabled) {
		4481	dev_priv->rps.client_boost = true;
		4482	queue_work(dev_priv->wq, &dev_priv->rps.work);
		4483	}
		4484	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	4485
6084	serge	4486	if (rps != NULL) {
		4487	list_add(&rps->link, &dev_priv->rps.clients);
		4488	rps->boosts++;
		4489	} else
		4490	dev_priv->rps.boosts++;
		4491	}
		4492	spin_unlock(&dev_priv->rps.client_lock);
		4493	}
4104	Serge	4494
6084	serge	4495	void intel_set_rps(struct drm_device *dev, u8 val)
		4496	{
6937	serge	4497	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev))
6084	serge	4498	valleyview_set_rps(dev, val);
		4499	else
		4500	gen6_set_rps(dev, val);
4104	Serge	4501	}
		4502
5354	serge	4503	static void gen9_disable_rps(struct drm_device *dev)
5060	serge	4504	{
		4505	struct drm_i915_private *dev_priv = dev->dev_private;
		4506
5354	serge	4507	I915_WRITE(GEN6_RC_CONTROL, 0);
6084	serge	4508	I915_WRITE(GEN9_PG_ENABLE, 0);
5060	serge	4509	}
		4510
4104	Serge	4511	static void gen6_disable_rps(struct drm_device *dev)
		4512	{
		4513	struct drm_i915_private *dev_priv = dev->dev_private;
		4514
		4515	I915_WRITE(GEN6_RC_CONTROL, 0);
		4516	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		4517	}
		4518
5060	serge	4519	static void cherryview_disable_rps(struct drm_device *dev)
		4520	{
		4521	struct drm_i915_private *dev_priv = dev->dev_private;
		4522
		4523	I915_WRITE(GEN6_RC_CONTROL, 0);
		4524	}
		4525
4104	Serge	4526	static void valleyview_disable_rps(struct drm_device *dev)
		4527	{
		4528	struct drm_i915_private *dev_priv = dev->dev_private;
		4529
5354	serge	4530	/* we're doing forcewake before Disabling RC6,
		4531	* This what the BIOS expects when going into suspend */
6084	serge	4532	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4533
4104	Serge	4534	I915_WRITE(GEN6_RC_CONTROL, 0);
		4535
6084	serge	4536	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4537	}
		4538
4560	Serge	4539	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		4540	{
6937	serge	4541	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev)) {
5060	serge	4542	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		4543	mode = GEN6_RC_CTL_RC6_ENABLE;
		4544	else
		4545	mode = 0;
		4546	}
5354	serge	4547	if (HAS_RC6p(dev))
		4548	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
6084	serge	4549	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		4550	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		4551	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
5354	serge	4552
		4553	else
		4554	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
		4555	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4560	Serge	4556	}
		4557
5060	serge	4558	static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3031	serge	4559	{
6084	serge	4560	/* No RC6 before Ironlake and code is gone for ilk. */
		4561	if (INTEL_INFO(dev)->gen < 6)
4104	Serge	4562	return 0;
		4563
3031	serge	4564	/* Respect the kernel parameter if it is set */
5060	serge	4565	if (enable_rc6 >= 0) {
		4566	int mask;
3031	serge	4567
5354	serge	4568	if (HAS_RC6p(dev))
5060	serge	4569	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		4570	INTEL_RC6pp_ENABLE;
		4571	else
		4572	mask = INTEL_RC6_ENABLE;
		4573
		4574	if ((enable_rc6 & mask) != enable_rc6)
		4575	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
6084	serge	4576	enable_rc6 & mask, enable_rc6, mask);
5060	serge	4577
		4578	return enable_rc6 & mask;
		4579	}
		4580
		4581	if (IS_IVYBRIDGE(dev))
		4582	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	4583
6084	serge	4584	return INTEL_RC6_ENABLE;
5060	serge	4585	}
3031	serge	4586
5060	serge	4587	int intel_enable_rc6(const struct drm_device *dev)
		4588	{
		4589	return i915.enable_rc6;
3031	serge	4590	}
		4591
5354	serge	4592	static void gen6_init_rps_frequencies(struct drm_device *dev)
5060	serge	4593	{
		4594	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4595	uint32_t rp_state_cap;
		4596	u32 ddcc_status = 0;
		4597	int ret;
5060	serge	4598
		4599	/* All of these values are in units of 50MHz */
		4600	dev_priv->rps.cur_freq = 0;
5354	serge	4601	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
6084	serge	4602	if (IS_BROXTON(dev)) {
		4603	rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
		4604	dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
		4605	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4606	dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
		4607	} else {
		4608	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		4609	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
		4610	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4611	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		4612	}
		4613
5060	serge	4614	/* hw_max = RP0 until we check for overclocking */
		4615	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	4616
5354	serge	4617	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
6937	serge	4618	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev) \|\|
		4619	IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
5354	serge	4620	ret = sandybridge_pcode_read(dev_priv,
		4621	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
		4622	&ddcc_status);
		4623	if (0 == ret)
		4624	dev_priv->rps.efficient_freq =
6084	serge	4625	clamp_t(u8,
		4626	((ddcc_status >> 8) & 0xff),
		4627	dev_priv->rps.min_freq,
		4628	dev_priv->rps.max_freq);
5354	serge	4629	}
		4630
6937	serge	4631	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
6084	serge	4632	/* Store the frequency values in 16.66 MHZ units, which is
		4633	the natural hardware unit for SKL */
		4634	dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
		4635	dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
		4636	dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
		4637	dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
		4638	dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
		4639	}
		4640
		4641	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		4642
5060	serge	4643	/* Preserve min/max settings in case of re-init */
		4644	if (dev_priv->rps.max_freq_softlimit == 0)
		4645	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	4646
5354	serge	4647	if (dev_priv->rps.min_freq_softlimit == 0) {
		4648	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4649	dev_priv->rps.min_freq_softlimit =
6084	serge	4650	max_t(int, dev_priv->rps.efficient_freq,
		4651	intel_freq_opcode(dev_priv, 450));
5354	serge	4652	else
		4653	dev_priv->rps.min_freq_softlimit =
		4654	dev_priv->rps.min_freq;
		4655	}
4104	Serge	4656	}
		4657
6084	serge	4658	/* See the Gen9_GT_PM_Programming_Guide doc for the below */
5354	serge	4659	static void gen9_enable_rps(struct drm_device *dev)
		4660	{
		4661	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4662
		4663	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4664
		4665	gen6_init_rps_frequencies(dev);
		4666
		4667	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
6937	serge	4668	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
6084	serge	4669	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4670	return;
		4671	}
		4672
		4673	/* Program defaults and thresholds for RPS*/
		4674	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4675	GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
		4676
		4677	/* 1 second timeout*/
		4678	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
		4679	GT_INTERVAL_FROM_US(dev_priv, 1000000));
		4680
		4681	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
		4682
		4683	/* Leaning on the below call to gen6_set_rps to program/setup the
		4684	* Up/Down EI & threshold registers, as well as the RP_CONTROL,
		4685	* RP_INTERRUPT_LIMITS & RPNSWREQ registers */
		4686	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		4687	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4688
		4689	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4690	}
		4691
		4692	static void gen9_enable_rc6(struct drm_device *dev)
		4693	{
		4694	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4695	struct intel_engine_cs *ring;
		4696	uint32_t rc6_mask = 0;
		4697	int unused;
		4698
		4699	/* 1a: Software RC state - RC0 */
		4700	I915_WRITE(GEN6_RC_STATE, 0);
		4701
		4702	/* 1b: Get forcewake during program sequence. Although the driver
		4703	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4704	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4705
		4706	/* 2a: Disable RC states. */
		4707	I915_WRITE(GEN6_RC_CONTROL, 0);
		4708
		4709	/* 2b: Program RC6 thresholds.*/
6084	serge	4710
		4711	/* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
		4712	if (IS_SKYLAKE(dev))
		4713	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
		4714	else
		4715	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
5354	serge	4716	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4717	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4718	for_each_ring(ring, dev_priv, unused)
		4719	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
6084	serge	4720
		4721	if (HAS_GUC_UCODE(dev))
		4722	I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
		4723
5354	serge	4724	I915_WRITE(GEN6_RC_SLEEP, 0);
		4725
6084	serge	4726	/* 2c: Program Coarse Power Gating Policies. */
		4727	I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
		4728	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
		4729
5354	serge	4730	/* 3a: Enable RC6 */
		4731	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4732	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
		4733	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4734	"on" : "off");
6084	serge	4735	/* WaRsUseTimeoutMode */
6937	serge	4736	if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) \|\|
		4737	IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
6084	serge	4738	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
		4739	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4740	GEN7_RC_CTL_TO_MODE \|
		4741	rc6_mask);
		4742	} else {
		4743	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
		4744	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4745	GEN6_RC_CTL_EI_MODE(1) \|
		4746	rc6_mask);
		4747	}
5354	serge	4748
6084	serge	4749	/*
		4750	* 3b: Enable Coarse Power Gating only when RC6 is enabled.
		4751	* WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
		4752	*/
		4753	if ((IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) \|\|
		4754	((IS_SKL_GT3(dev) \|\| IS_SKL_GT4(dev)) && (INTEL_REVID(dev) <= SKL_REVID_F0)))
		4755	I915_WRITE(GEN9_PG_ENABLE, 0);
		4756	else
		4757	I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4758	(GEN9_RENDER_PG_ENABLE \| GEN9_MEDIA_PG_ENABLE) : 0);
5354	serge	4759
6084	serge	4760	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4761
5354	serge	4762	}
		4763
4560	Serge	4764	static void gen8_enable_rps(struct drm_device *dev)
		4765	{
		4766	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4767	struct intel_engine_cs *ring;
5354	serge	4768	uint32_t rc6_mask = 0;
4560	Serge	4769	int unused;
		4770
		4771	/* 1a: Software RC state - RC0 */
		4772	I915_WRITE(GEN6_RC_STATE, 0);
		4773
		4774	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		4775	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4776	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4560	Serge	4777
		4778	/* 2a: Disable RC states. */
		4779	I915_WRITE(GEN6_RC_CONTROL, 0);
		4780
5354	serge	4781	/* Initialize rps frequencies */
		4782	gen6_init_rps_frequencies(dev);
4560	Serge	4783
		4784	/* 2b: Program RC6 thresholds.*/
		4785	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4786	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4787	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4788	for_each_ring(ring, dev_priv, unused)
		4789	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4790	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	4791	if (IS_BROADWELL(dev))
		4792	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		4793	else
6084	serge	4794	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4560	Serge	4795
		4796	/* 3: Enable RC6 */
		4797	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4798	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	4799	intel_print_rc6_info(dev, rc6_mask);
		4800	if (IS_BROADWELL(dev))
		4801	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4802	GEN7_RC_CTL_TO_MODE \|
		4803	rc6_mask);
		4804	else
6084	serge	4805	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4806	GEN6_RC_CTL_EI_MODE(1) \|
		4807	rc6_mask);
4560	Serge	4808
		4809	/* 4 Program defaults and thresholds for RPS*/
5060	serge	4810	I915_WRITE(GEN6_RPNSWREQ,
		4811	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		4812	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4813	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	4814	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		4815	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		4816
		4817	/* Docs recommend 900MHz, and 300 MHz respectively */
		4818	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	4819	dev_priv->rps.max_freq_softlimit << 24 \|
		4820	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	4821
		4822	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		4823	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		4824	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		4825	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		4826
		4827	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4828
		4829	/* 5: Enable RPS */
		4830	I915_WRITE(GEN6_RP_CONTROL,
		4831	GEN6_RP_MEDIA_TURBO \|
		4832	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4833	GEN6_RP_MEDIA_IS_GFX \|
		4834	GEN6_RP_ENABLE \|
		4835	GEN6_RP_UP_BUSY_AVG \|
		4836	GEN6_RP_DOWN_IDLE_AVG);
		4837
		4838	/* 6: Ring frequency + overclocking (our driver does this later */
		4839
5354	serge	4840	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	4841	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4842
6084	serge	4843	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4560	Serge	4844	}
		4845
3031	serge	4846	static void gen6_enable_rps(struct drm_device *dev)
		4847	{
		4848	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4849	struct intel_engine_cs *ring;
		4850	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	4851	u32 gtfifodbg;
		4852	int rc6_mode;
3243	Serge	4853	int i, ret;
3031	serge	4854
3243	Serge	4855	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4856
		4857	/* Here begins a magic sequence of register writes to enable
		4858	* auto-downclocking.
		4859	*
		4860	* Perhaps there might be some value in exposing these to
		4861	* userspace...
		4862	*/
		4863	I915_WRITE(GEN6_RC_STATE, 0);
		4864
		4865	/* Clear the DBG now so we don't confuse earlier errors */
		4866	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		4867	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		4868	I915_WRITE(GTFIFODBG, gtfifodbg);
		4869	}
		4870
6084	serge	4871	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	4872
5354	serge	4873	/* Initialize rps frequencies */
		4874	gen6_init_rps_frequencies(dev);
3031	serge	4875
		4876	/* disable the counters and set deterministic thresholds */
		4877	I915_WRITE(GEN6_RC_CONTROL, 0);
		4878
		4879	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		4880	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		4881	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		4882	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4883	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4884
		4885	for_each_ring(ring, dev_priv, i)
		4886	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4887
		4888	I915_WRITE(GEN6_RC_SLEEP, 0);
		4889	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	4890	if (IS_IVYBRIDGE(dev))
4104	Serge	4891	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		4892	else
6084	serge	4893	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	4894	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	4895	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		4896
		4897	/* Check if we are enabling RC6 */
		4898	rc6_mode = intel_enable_rc6(dev_priv->dev);
		4899	if (rc6_mode & INTEL_RC6_ENABLE)
		4900	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		4901
		4902	/* We don't use those on Haswell */
		4903	if (!IS_HASWELL(dev)) {
		4904	if (rc6_mode & INTEL_RC6p_ENABLE)
		4905	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		4906
		4907	if (rc6_mode & INTEL_RC6pp_ENABLE)
		4908	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		4909	}
		4910
4560	Serge	4911	intel_print_rc6_info(dev, rc6_mask);
3031	serge	4912
		4913	I915_WRITE(GEN6_RC_CONTROL,
		4914	rc6_mask \|
		4915	GEN6_RC_CTL_EI_MODE(1) \|
		4916	GEN6_RC_CTL_HW_ENABLE);
		4917
4560	Serge	4918	/* Power down if completely idle for over 50ms */
		4919	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	4920	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4921
3243	Serge	4922	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	4923	if (ret)
		4924	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		4925
6084	serge	4926	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		4927	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		4928	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	4929	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
6084	serge	4930	(pcu_mbox & 0xff) * 50);
5060	serge	4931	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	4932	}
		4933
4560	Serge	4934	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	4935	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
3031	serge	4936
3243	Serge	4937	rc6vids = 0;
		4938	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		4939	if (IS_GEN6(dev) && ret) {
		4940	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		4941	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		4942	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		4943	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		4944	rc6vids &= 0xffff00;
		4945	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		4946	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		4947	if (ret)
		4948	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		4949	}
		4950
6084	serge	4951	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	4952	}
		4953
5060	serge	4954	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	4955	{
		4956	struct drm_i915_private *dev_priv = dev->dev_private;
		4957	int min_freq = 15;
3746	Serge	4958	unsigned int gpu_freq;
		4959	unsigned int max_ia_freq, min_ring_freq;
6084	serge	4960	unsigned int max_gpu_freq, min_gpu_freq;
3031	serge	4961	int scaling_factor = 180;
4560	Serge	4962	struct cpufreq_policy *policy;
3031	serge	4963
3243	Serge	4964	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4965
		4966	max_ia_freq = cpufreq_quick_get_max(0);
		4967	/*
5060	serge	4968	* Default to measured freq if none found, PCU will ensure we
		4969	* don't go over
3031	serge	4970	*/
		4971	max_ia_freq = tsc_khz;
		4972
		4973	/* Convert from kHz to MHz */
		4974	max_ia_freq /= 1000;
		4975
4560	Serge	4976	min_ring_freq = I915_READ(DCLK) & 0xf;
		4977	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		4978	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	4979
6937	serge	4980	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
6084	serge	4981	/* Convert GT frequency to 50 HZ units */
		4982	min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
		4983	max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
		4984	} else {
		4985	min_gpu_freq = dev_priv->rps.min_freq;
		4986	max_gpu_freq = dev_priv->rps.max_freq;
		4987	}
		4988
3031	serge	4989	/*
		4990	* For each potential GPU frequency, load a ring frequency we'd like
		4991	* to use for memory access. We do this by specifying the IA frequency
		4992	* the PCU should use as a reference to determine the ring frequency.
		4993	*/
6084	serge	4994	for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
		4995	int diff = max_gpu_freq - gpu_freq;
3746	Serge	4996	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	4997
6937	serge	4998	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
6084	serge	4999	/*
		5000	* ring_freq = 2 * GT. ring_freq is in 100MHz units
		5001	* No floor required for ring frequency on SKL.
		5002	*/
		5003	ring_freq = gpu_freq;
		5004	} else if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	5005	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		5006	ring_freq = max(min_ring_freq, gpu_freq);
		5007	} else if (IS_HASWELL(dev)) {
		5008	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	5009	ring_freq = max(min_ring_freq, ring_freq);
		5010	/* leave ia_freq as the default, chosen by cpufreq */
		5011	} else {
		5012	/* On older processors, there is no separate ring
		5013	* clock domain, so in order to boost the bandwidth
		5014	* of the ring, we need to upclock the CPU (ia_freq).
		5015	*
		5016	* For GPU frequencies less than 750MHz,
		5017	* just use the lowest ring freq.
6084	serge	5018	*/
		5019	if (gpu_freq < min_freq)
		5020	ia_freq = 800;
		5021	else
		5022	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		5023	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	5024	}
3031	serge	5025
3243	Serge	5026	sandybridge_pcode_write(dev_priv,
		5027	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	5028	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		5029	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		5030	gpu_freq);
3031	serge	5031	}
		5032	}
		5033
5060	serge	5034	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	5035	{
5060	serge	5036	struct drm_i915_private *dev_priv = dev->dev_private;
		5037
6084	serge	5038	if (!HAS_CORE_RING_FREQ(dev))
5060	serge	5039	return;
		5040
		5041	mutex_lock(&dev_priv->rps.hw_lock);
		5042	__gen6_update_ring_freq(dev);
		5043	mutex_unlock(&dev_priv->rps.hw_lock);
		5044	}
		5045
		5046	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		5047	{
6084	serge	5048	struct drm_device *dev = dev_priv->dev;
4104	Serge	5049	u32 val, rp0;
		5050
6084	serge	5051	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5060	serge	5052
6084	serge	5053	switch (INTEL_INFO(dev)->eu_total) {
		5054	case 8:
		5055	/* (2 * 4) config */
		5056	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
		5057	break;
		5058	case 12:
		5059	/* (2 * 6) config */
		5060	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
		5061	break;
		5062	case 16:
		5063	/* (2 * 8) config */
		5064	default:
		5065	/* Setting (2 * 8) Min RP0 for any other combination */
		5066	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
		5067	break;
		5068	}
		5069
		5070	rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
		5071
5060	serge	5072	return rp0;
		5073	}
		5074
		5075	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5076	{
		5077	u32 val, rpe;
		5078
		5079	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		5080	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		5081
		5082	return rpe;
		5083	}
		5084
		5085	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5086	{
		5087	u32 val, rp1;
		5088
6084	serge	5089	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
		5090	rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5060	serge	5091
		5092	return rp1;
		5093	}
		5094
		5095	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5096	{
		5097	u32 val, rp1;
		5098
4104	Serge	5099	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5100
5060	serge	5101	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		5102
		5103	return rp1;
		5104	}
		5105
		5106	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		5107	{
		5108	u32 val, rp0;
		5109
		5110	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5111
4104	Serge	5112	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		5113	/* Clamp to max */
		5114	rp0 = min_t(u32, rp0, 0xea);
		5115
		5116	return rp0;
		5117	}
		5118
		5119	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5120	{
		5121	u32 val, rpe;
		5122
		5123	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		5124	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		5125	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		5126	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		5127
		5128	return rpe;
		5129	}
		5130
5060	serge	5131	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	5132	{
6937	serge	5133	u32 val;
		5134
		5135	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		5136	/*
		5137	* According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
		5138	* for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
		5139	* a BYT-M B0 the above register contains 0xbf. Moreover when setting
		5140	* a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
		5141	* to make sure it matches what Punit accepts.
		5142	*/
		5143	return max_t(u32, val, 0xc0);
4104	Serge	5144	}
		5145
5060	serge	5146	/* Check that the pctx buffer wasn't move under us. */
		5147	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		5148	{
		5149	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5150
		5151	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		5152	dev_priv->vlv_pctx->stolen->start);
		5153	}
		5154
		5155
		5156	/* Check that the pcbr address is not empty. */
		5157	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		5158	{
		5159	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5160
		5161	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		5162	}
		5163
		5164	static void cherryview_setup_pctx(struct drm_device *dev)
		5165	{
		5166	struct drm_i915_private *dev_priv = dev->dev_private;
		5167	unsigned long pctx_paddr, paddr;
		5168	struct i915_gtt *gtt = &dev_priv->gtt;
		5169	u32 pcbr;
		5170	int pctx_size = 32*1024;
		5171
		5172	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5173
		5174	pcbr = I915_READ(VLV_PCBR);
		5175	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5354	serge	5176	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5060	serge	5177	paddr = (dev_priv->mm.stolen_base +
		5178	(gtt->stolen_size - pctx_size));
		5179
		5180	pctx_paddr = (paddr & (~4095));
		5181	I915_WRITE(VLV_PCBR, pctx_paddr);
		5182	}
5354	serge	5183
		5184	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5060	serge	5185	}
		5186
4104	Serge	5187	static void valleyview_setup_pctx(struct drm_device *dev)
		5188	{
		5189	struct drm_i915_private *dev_priv = dev->dev_private;
		5190	struct drm_i915_gem_object *pctx;
		5191	unsigned long pctx_paddr;
		5192	u32 pcbr;
		5193	int pctx_size = 24*1024;
		5194
5060	serge	5195	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5196
4104	Serge	5197	pcbr = I915_READ(VLV_PCBR);
		5198	if (pcbr) {
		5199	/* BIOS set it up already, grab the pre-alloc'd space */
		5200	int pcbr_offset;
		5201
		5202	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		5203	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		5204	pcbr_offset,
		5205	I915_GTT_OFFSET_NONE,
		5206	pctx_size);
		5207	goto out;
		5208	}
		5209
5354	serge	5210	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
		5211
4104	Serge	5212	/*
		5213	* From the Gunit register HAS:
		5214	* The Gfx driver is expected to program this register and ensure
		5215	* proper allocation within Gfx stolen memory. For example, this
		5216	* register should be programmed such than the PCBR range does not
		5217	* overlap with other ranges, such as the frame buffer, protected
		5218	* memory, or any other relevant ranges.
		5219	*/
		5220	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		5221	if (!pctx) {
		5222	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		5223	return;
		5224	}
		5225
		5226	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		5227	I915_WRITE(VLV_PCBR, pctx_paddr);
		5228
		5229	out:
5354	serge	5230	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4104	Serge	5231	dev_priv->vlv_pctx = pctx;
		5232	}
		5233
5060	serge	5234	static void valleyview_cleanup_pctx(struct drm_device *dev)
		5235	{
		5236	struct drm_i915_private *dev_priv = dev->dev_private;
		5237
		5238	if (WARN_ON(!dev_priv->vlv_pctx))
		5239	return;
		5240
		5241	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		5242	dev_priv->vlv_pctx = NULL;
		5243	}
		5244
		5245	static void valleyview_init_gt_powersave(struct drm_device *dev)
		5246	{
		5247	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5248	u32 val;
5060	serge	5249
		5250	valleyview_setup_pctx(dev);
		5251
		5252	mutex_lock(&dev_priv->rps.hw_lock);
		5253
5354	serge	5254	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5255	switch ((val >> 6) & 3) {
		5256	case 0:
		5257	case 1:
		5258	dev_priv->mem_freq = 800;
		5259	break;
		5260	case 2:
		5261	dev_priv->mem_freq = 1066;
		5262	break;
		5263	case 3:
		5264	dev_priv->mem_freq = 1333;
		5265	break;
		5266	}
		5267	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5268
5060	serge	5269	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		5270	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5271	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5272	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5273	dev_priv->rps.max_freq);
		5274
		5275	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		5276	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5277	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5278	dev_priv->rps.efficient_freq);
		5279
		5280	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		5281	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
6084	serge	5282	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5283	dev_priv->rps.rp1_freq);
		5284
		5285	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		5286	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5287	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5288	dev_priv->rps.min_freq);
		5289
6084	serge	5290	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5291
5060	serge	5292	/* Preserve min/max settings in case of re-init */
		5293	if (dev_priv->rps.max_freq_softlimit == 0)
		5294	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5295
		5296	if (dev_priv->rps.min_freq_softlimit == 0)
		5297	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5298
		5299	mutex_unlock(&dev_priv->rps.hw_lock);
		5300	}
		5301
		5302	static void cherryview_init_gt_powersave(struct drm_device *dev)
		5303	{
		5304	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5305	u32 val;
5060	serge	5306
		5307	cherryview_setup_pctx(dev);
		5308
		5309	mutex_lock(&dev_priv->rps.hw_lock);
		5310
6084	serge	5311	mutex_lock(&dev_priv->sb_lock);
5354	serge	5312	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
6084	serge	5313	mutex_unlock(&dev_priv->sb_lock);
5354	serge	5314
		5315	switch ((val >> 2) & 0x7) {
		5316	case 3:
		5317	dev_priv->mem_freq = 2000;
		5318	break;
6084	serge	5319	default:
5354	serge	5320	dev_priv->mem_freq = 1600;
		5321	break;
		5322	}
		5323	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5324
5060	serge	5325	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		5326	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5327	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5328	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5329	dev_priv->rps.max_freq);
		5330
		5331	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		5332	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5333	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5334	dev_priv->rps.efficient_freq);
		5335
		5336	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		5337	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
6084	serge	5338	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5339	dev_priv->rps.rp1_freq);
		5340
6084	serge	5341	/* PUnit validated range is only [RPe, RP0] */
		5342	dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5060	serge	5343	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5344	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5345	dev_priv->rps.min_freq);
		5346
5354	serge	5347	WARN_ONCE((dev_priv->rps.max_freq \|
		5348	dev_priv->rps.efficient_freq \|
		5349	dev_priv->rps.rp1_freq \|
		5350	dev_priv->rps.min_freq) & 1,
		5351	"Odd GPU freq values\n");
		5352
6084	serge	5353	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5354
5060	serge	5355	/* Preserve min/max settings in case of re-init */
		5356	if (dev_priv->rps.max_freq_softlimit == 0)
		5357	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5358
		5359	if (dev_priv->rps.min_freq_softlimit == 0)
		5360	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5361
		5362	mutex_unlock(&dev_priv->rps.hw_lock);
		5363	}
		5364
		5365	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		5366	{
		5367	valleyview_cleanup_pctx(dev);
		5368	}
		5369
		5370	static void cherryview_enable_rps(struct drm_device *dev)
		5371	{
		5372	struct drm_i915_private *dev_priv = dev->dev_private;
		5373	struct intel_engine_cs *ring;
		5374	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		5375	int i;
		5376
		5377	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5378
		5379	gtfifodbg = I915_READ(GTFIFODBG);
		5380	if (gtfifodbg) {
		5381	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5382	gtfifodbg);
		5383	I915_WRITE(GTFIFODBG, gtfifodbg);
		5384	}
		5385
		5386	cherryview_check_pctx(dev_priv);
		5387
		5388	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		5389	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	5390	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	5391
6084	serge	5392	/* Disable RC states. */
		5393	I915_WRITE(GEN6_RC_CONTROL, 0);
		5394
5060	serge	5395	/* 2a: Program RC6 thresholds.*/
		5396	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		5397	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		5398	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		5399
		5400	for_each_ring(ring, dev_priv, i)
		5401	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5402	I915_WRITE(GEN6_RC_SLEEP, 0);
		5403
6084	serge	5404	/* TO threshold set to 500 us ( 0x186 * 1.28 us) */
		5405	I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5060	serge	5406
		5407	/* allows RC6 residency counter to work */
		5408	I915_WRITE(VLV_COUNTER_CONTROL,
		5409	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		5410	VLV_MEDIA_RC6_COUNT_EN \|
		5411	VLV_RENDER_RC6_COUNT_EN));
		5412
		5413	/* For now we assume BIOS is allocating and populating the PCBR */
		5414	pcbr = I915_READ(VLV_PCBR);
		5415
		5416	/* 3: Enable RC6 */
		5417	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		5418	(pcbr >> VLV_PCBR_ADDR_SHIFT))
6084	serge	5419	rc6_mode = GEN7_RC_CTL_TO_MODE;
5060	serge	5420
		5421	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5422
		5423	/* 4 Program defaults and thresholds for RPS*/
6084	serge	5424	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5060	serge	5425	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5426	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5427	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5428	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5429
		5430	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5431
		5432	/* 5: Enable RPS */
		5433	I915_WRITE(GEN6_RP_CONTROL,
		5434	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
6084	serge	5435	GEN6_RP_MEDIA_IS_GFX \|
5060	serge	5436	GEN6_RP_ENABLE \|
		5437	GEN6_RP_UP_BUSY_AVG \|
		5438	GEN6_RP_DOWN_IDLE_AVG);
		5439
6084	serge	5440	/* Setting Fixed Bias */
		5441	val = VLV_OVERRIDE_EN \|
		5442	VLV_SOC_TDP_EN \|
		5443	CHV_BIAS_CPU_50_SOC_50;
		5444	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5445
5060	serge	5446	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5447
5354	serge	5448	/* RPS code assumes GPLL is used */
		5449	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5450
6084	serge	5451	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5060	serge	5452	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5453
		5454	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		5455	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5456	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5457	dev_priv->rps.cur_freq);
		5458
		5459	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5460	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5461	dev_priv->rps.efficient_freq);
		5462
		5463	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		5464
6084	serge	5465	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	5466	}
		5467
4104	Serge	5468	static void valleyview_enable_rps(struct drm_device *dev)
		5469	{
		5470	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5471	struct intel_engine_cs *ring;
4560	Serge	5472	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	5473	int i;
		5474
		5475	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5476
5060	serge	5477	valleyview_check_pctx(dev_priv);
		5478
4104	Serge	5479	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	5480	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5481	gtfifodbg);
4104	Serge	5482	I915_WRITE(GTFIFODBG, gtfifodbg);
		5483	}
		5484
4560	Serge	5485	/* If VLV, Forcewake all wells, else re-direct to regular path */
6084	serge	5486	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	5487
6084	serge	5488	/* Disable RC states. */
		5489	I915_WRITE(GEN6_RC_CONTROL, 0);
		5490
		5491	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4104	Serge	5492	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5493	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5494	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5495	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5496
		5497	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5498
		5499	I915_WRITE(GEN6_RP_CONTROL,
		5500	GEN6_RP_MEDIA_TURBO \|
		5501	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5502	GEN6_RP_MEDIA_IS_GFX \|
		5503	GEN6_RP_ENABLE \|
		5504	GEN6_RP_UP_BUSY_AVG \|
		5505	GEN6_RP_DOWN_IDLE_CONT);
		5506
		5507	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		5508	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		5509	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		5510
		5511	for_each_ring(ring, dev_priv, i)
		5512	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5513
4560	Serge	5514	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	5515
		5516	/* allows RC6 residency counter to work */
4560	Serge	5517	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	5518	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		5519	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	5520	VLV_MEDIA_RC6_COUNT_EN \|
		5521	VLV_RENDER_RC6_COUNT_EN));
5060	serge	5522
4560	Serge	5523	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		5524	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	5525
4560	Serge	5526	intel_print_rc6_info(dev, rc6_mode);
		5527
		5528	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5529
6084	serge	5530	/* Setting Fixed Bias */
		5531	val = VLV_OVERRIDE_EN \|
		5532	VLV_SOC_TDP_EN \|
		5533	VLV_BIAS_CPU_125_SOC_875;
		5534	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5535
4104	Serge	5536	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5537
5354	serge	5538	/* RPS code assumes GPLL is used */
		5539	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5540
6084	serge	5541	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
4104	Serge	5542	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5543
5060	serge	5544	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	5545	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5546	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5547	dev_priv->rps.cur_freq);
4104	Serge	5548
		5549	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5550	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5551	dev_priv->rps.efficient_freq);
4104	Serge	5552
5060	serge	5553	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	5554
6084	serge	5555	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	5556	}
		5557
3031	serge	5558	static unsigned long intel_pxfreq(u32 vidfreq)
		5559	{
		5560	unsigned long freq;
		5561	int div = (vidfreq & 0x3f0000) >> 16;
		5562	int post = (vidfreq & 0x3000) >> 12;
		5563	int pre = (vidfreq & 0x7);
		5564
		5565	if (!pre)
		5566	return 0;
		5567
		5568	freq = ((div * 133333) / ((1<
		5569
		5570	return freq;
		5571	}
		5572
		5573	static const struct cparams {
		5574	u16 i;
		5575	u16 t;
		5576	u16 m;
		5577	u16 c;
		5578	} cparams[] = {
		5579	{ 1, 1333, 301, 28664 },
		5580	{ 1, 1066, 294, 24460 },
		5581	{ 1, 800, 294, 25192 },
		5582	{ 0, 1333, 276, 27605 },
		5583	{ 0, 1066, 276, 27605 },
		5584	{ 0, 800, 231, 23784 },
		5585	};
		5586
		5587	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		5588	{
		5589	u64 total_count, diff, ret;
		5590	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	5591	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	5592	int i;
		5593
		5594	assert_spin_locked(&mchdev_lock);
		5595
		5596	diff1 = now - dev_priv->ips.last_time1;
		5597
		5598	/* Prevent division-by-zero if we are asking too fast.
		5599	* Also, we don't get interesting results if we are polling
		5600	* faster than once in 10ms, so just return the saved value
		5601	* in such cases.
		5602	*/
		5603	if (diff1 <= 10)
		5604	return dev_priv->ips.chipset_power;
		5605
		5606	count1 = I915_READ(DMIEC);
		5607	count2 = I915_READ(DDREC);
		5608	count3 = I915_READ(CSIEC);
		5609
		5610	total_count = count1 + count2 + count3;
		5611
		5612	/* FIXME: handle per-counter overflow */
		5613	if (total_count < dev_priv->ips.last_count1) {
		5614	diff = ~0UL - dev_priv->ips.last_count1;
		5615	diff += total_count;
		5616	} else {
		5617	diff = total_count - dev_priv->ips.last_count1;
		5618	}
		5619
		5620	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		5621	if (cparams[i].i == dev_priv->ips.c_m &&
		5622	cparams[i].t == dev_priv->ips.r_t) {
		5623	m = cparams[i].m;
		5624	c = cparams[i].c;
		5625	break;
		5626	}
		5627	}
		5628
		5629	diff = div_u64(diff, diff1);
		5630	ret = ((m * diff) + c);
		5631	ret = div_u64(ret, 10);
		5632
		5633	dev_priv->ips.last_count1 = total_count;
		5634	dev_priv->ips.last_time1 = now;
		5635
		5636	dev_priv->ips.chipset_power = ret;
		5637
		5638	return ret;
		5639	}
		5640
		5641	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		5642	{
5060	serge	5643	struct drm_device *dev = dev_priv->dev;
3031	serge	5644	unsigned long val;
		5645
5060	serge	5646	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5647	return 0;
		5648
		5649	spin_lock_irq(&mchdev_lock);
		5650
		5651	val = __i915_chipset_val(dev_priv);
		5652
		5653	spin_unlock_irq(&mchdev_lock);
		5654
		5655	return val;
		5656	}
		5657
		5658	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		5659	{
		5660	unsigned long m, x, b;
		5661	u32 tsfs;
		5662
		5663	tsfs = I915_READ(TSFS);
		5664
		5665	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		5666	x = I915_READ8(TR1);
		5667
		5668	b = tsfs & TSFS_INTR_MASK;
		5669
		5670	return ((m * x) / 127) - b;
		5671	}
		5672
6084	serge	5673	static int _pxvid_to_vd(u8 pxvid)
3031	serge	5674	{
6084	serge	5675	if (pxvid == 0)
		5676	return 0;
		5677
		5678	if (pxvid >= 8 && pxvid < 31)
		5679	pxvid = 31;
		5680
		5681	return (pxvid + 2) * 125;
		5682	}
		5683
		5684	static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		5685	{
5060	serge	5686	struct drm_device *dev = dev_priv->dev;
6084	serge	5687	const int vd = _pxvid_to_vd(pxvid);
		5688	const int vm = vd - 1125;
		5689
5060	serge	5690	if (INTEL_INFO(dev)->is_mobile)
6084	serge	5691	return vm > 0 ? vm : 0;
		5692
		5693	return vd;
3031	serge	5694	}
		5695
		5696	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5697	{
5060	serge	5698	u64 now, diff, diffms;
3031	serge	5699	u32 count;
		5700
		5701	assert_spin_locked(&mchdev_lock);
		5702
5060	serge	5703	now = ktime_get_raw_ns();
		5704	diffms = now - dev_priv->ips.last_time2;
		5705	do_div(diffms, NSEC_PER_MSEC);
3031	serge	5706
		5707	/* Don't divide by 0 */
		5708	if (!diffms)
		5709	return;
		5710
		5711	count = I915_READ(GFXEC);
		5712
		5713	if (count < dev_priv->ips.last_count2) {
		5714	diff = ~0UL - dev_priv->ips.last_count2;
		5715	diff += count;
		5716	} else {
		5717	diff = count - dev_priv->ips.last_count2;
		5718	}
		5719
		5720	dev_priv->ips.last_count2 = count;
		5721	dev_priv->ips.last_time2 = now;
		5722
		5723	/* More magic constants... */
		5724	diff = diff * 1181;
		5725	diff = div_u64(diff, diffms * 10);
		5726	dev_priv->ips.gfx_power = diff;
		5727	}
		5728
		5729	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5730	{
5060	serge	5731	struct drm_device *dev = dev_priv->dev;
		5732
		5733	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5734	return;
		5735
		5736	spin_lock_irq(&mchdev_lock);
		5737
		5738	__i915_update_gfx_val(dev_priv);
		5739
		5740	spin_unlock_irq(&mchdev_lock);
		5741	}
		5742
		5743	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		5744	{
		5745	unsigned long t, corr, state1, corr2, state2;
		5746	u32 pxvid, ext_v;
		5747
		5748	assert_spin_locked(&mchdev_lock);
		5749
6084	serge	5750	pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
3031	serge	5751	pxvid = (pxvid >> 24) & 0x7f;
		5752	ext_v = pvid_to_extvid(dev_priv, pxvid);
		5753
		5754	state1 = ext_v;
		5755
		5756	t = i915_mch_val(dev_priv);
		5757
		5758	/* Revel in the empirically derived constants */
		5759
		5760	/* Correction factor in 1/100000 units */
		5761	if (t > 80)
		5762	corr = ((t * 2349) + 135940);
		5763	else if (t >= 50)
		5764	corr = ((t * 964) + 29317);
		5765	else /* < 50 */
		5766	corr = ((t * 301) + 1004);
		5767
		5768	corr = corr * ((150142 * state1) / 10000 - 78642);
		5769	corr /= 100000;
		5770	corr2 = (corr * dev_priv->ips.corr);
		5771
		5772	state2 = (corr2 * state1) / 10000;
		5773	state2 /= 100; /* convert to mW */
		5774
		5775	__i915_update_gfx_val(dev_priv);
		5776
		5777	return dev_priv->ips.gfx_power + state2;
		5778	}
		5779
		5780	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		5781	{
5060	serge	5782	struct drm_device *dev = dev_priv->dev;
3031	serge	5783	unsigned long val;
		5784
5060	serge	5785	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5786	return 0;
		5787
		5788	spin_lock_irq(&mchdev_lock);
		5789
		5790	val = __i915_gfx_val(dev_priv);
		5791
		5792	spin_unlock_irq(&mchdev_lock);
		5793
		5794	return val;
		5795	}
		5796
		5797	/**
		5798	* i915_read_mch_val - return value for IPS use
		5799	*
		5800	* Calculate and return a value for the IPS driver to use when deciding whether
		5801	* we have thermal and power headroom to increase CPU or GPU power budget.
		5802	*/
		5803	unsigned long i915_read_mch_val(void)
		5804	{
		5805	struct drm_i915_private *dev_priv;
		5806	unsigned long chipset_val, graphics_val, ret = 0;
		5807
		5808	spin_lock_irq(&mchdev_lock);
		5809	if (!i915_mch_dev)
		5810	goto out_unlock;
		5811	dev_priv = i915_mch_dev;
		5812
		5813	chipset_val = __i915_chipset_val(dev_priv);
		5814	graphics_val = __i915_gfx_val(dev_priv);
		5815
		5816	ret = chipset_val + graphics_val;
		5817
		5818	out_unlock:
		5819	spin_unlock_irq(&mchdev_lock);
		5820
		5821	return ret;
		5822	}
		5823	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		5824
		5825	/**
		5826	* i915_gpu_raise - raise GPU frequency limit
		5827	*
		5828	* Raise the limit; IPS indicates we have thermal headroom.
		5829	*/
		5830	bool i915_gpu_raise(void)
		5831	{
		5832	struct drm_i915_private *dev_priv;
		5833	bool ret = true;
		5834
		5835	spin_lock_irq(&mchdev_lock);
		5836	if (!i915_mch_dev) {
		5837	ret = false;
		5838	goto out_unlock;
		5839	}
		5840	dev_priv = i915_mch_dev;
		5841
		5842	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		5843	dev_priv->ips.max_delay--;
		5844
		5845	out_unlock:
		5846	spin_unlock_irq(&mchdev_lock);
		5847
		5848	return ret;
		5849	}
		5850	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		5851
		5852	/**
		5853	* i915_gpu_lower - lower GPU frequency limit
		5854	*
		5855	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		5856	* frequency maximum.
		5857	*/
		5858	bool i915_gpu_lower(void)
		5859	{
		5860	struct drm_i915_private *dev_priv;
		5861	bool ret = true;
		5862
		5863	spin_lock_irq(&mchdev_lock);
		5864	if (!i915_mch_dev) {
		5865	ret = false;
		5866	goto out_unlock;
		5867	}
		5868	dev_priv = i915_mch_dev;
		5869
		5870	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		5871	dev_priv->ips.max_delay++;
		5872
		5873	out_unlock:
		5874	spin_unlock_irq(&mchdev_lock);
		5875
		5876	return ret;
		5877	}
		5878	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		5879
		5880	/**
		5881	* i915_gpu_busy - indicate GPU business to IPS
		5882	*
		5883	* Tell the IPS driver whether or not the GPU is busy.
		5884	*/
		5885	bool i915_gpu_busy(void)
		5886	{
		5887	struct drm_i915_private *dev_priv;
5060	serge	5888	struct intel_engine_cs *ring;
3031	serge	5889	bool ret = false;
		5890	int i;
		5891
		5892	spin_lock_irq(&mchdev_lock);
		5893	if (!i915_mch_dev)
		5894	goto out_unlock;
		5895	dev_priv = i915_mch_dev;
		5896
		5897	for_each_ring(ring, dev_priv, i)
		5898	ret \|= !list_empty(&ring->request_list);
		5899
		5900	out_unlock:
		5901	spin_unlock_irq(&mchdev_lock);
		5902
		5903	return ret;
		5904	}
		5905	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		5906
		5907	/**
		5908	* i915_gpu_turbo_disable - disable graphics turbo
		5909	*
		5910	* Disable graphics turbo by resetting the max frequency and setting the
		5911	* current frequency to the default.
		5912	*/
		5913	bool i915_gpu_turbo_disable(void)
		5914	{
		5915	struct drm_i915_private *dev_priv;
		5916	bool ret = true;
		5917
		5918	spin_lock_irq(&mchdev_lock);
		5919	if (!i915_mch_dev) {
		5920	ret = false;
		5921	goto out_unlock;
		5922	}
		5923	dev_priv = i915_mch_dev;
		5924
		5925	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		5926
		5927	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		5928	ret = false;
		5929
		5930	out_unlock:
		5931	spin_unlock_irq(&mchdev_lock);
		5932
		5933	return ret;
		5934	}
		5935	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		5936
		5937	/**
		5938	* Tells the intel_ips driver that the i915 driver is now loaded, if
		5939	* IPS got loaded first.
		5940	*
		5941	* This awkward dance is so that neither module has to depend on the
		5942	* other in order for IPS to do the appropriate communication of
		5943	* GPU turbo limits to i915.
		5944	*/
		5945	static void
		5946	ips_ping_for_i915_load(void)
		5947	{
		5948	void (*link)(void);
		5949
		5950	// link = symbol_get(ips_link_to_i915_driver);
		5951	// if (link) {
		5952	// link();
		5953	// symbol_put(ips_link_to_i915_driver);
		5954	// }
		5955	}
		5956
		5957	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		5958	{
		5959	/* We only register the i915 ips part with intel-ips once everything is
		5960	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		5961	spin_lock_irq(&mchdev_lock);
		5962	i915_mch_dev = dev_priv;
		5963	spin_unlock_irq(&mchdev_lock);
		5964
		5965	ips_ping_for_i915_load();
		5966	}
		5967
		5968	void intel_gpu_ips_teardown(void)
		5969	{
		5970	spin_lock_irq(&mchdev_lock);
		5971	i915_mch_dev = NULL;
		5972	spin_unlock_irq(&mchdev_lock);
		5973	}
5060	serge	5974
3031	serge	5975	static void intel_init_emon(struct drm_device *dev)
		5976	{
		5977	struct drm_i915_private *dev_priv = dev->dev_private;
		5978	u32 lcfuse;
		5979	u8 pxw[16];
		5980	int i;
		5981
		5982	/* Disable to program */
		5983	I915_WRITE(ECR, 0);
		5984	POSTING_READ(ECR);
		5985
		5986	/* Program energy weights for various events */
		5987	I915_WRITE(SDEW, 0x15040d00);
		5988	I915_WRITE(CSIEW0, 0x007f0000);
		5989	I915_WRITE(CSIEW1, 0x1e220004);
		5990	I915_WRITE(CSIEW2, 0x04000004);
		5991
		5992	for (i = 0; i < 5; i++)
6084	serge	5993	I915_WRITE(PEW(i), 0);
3031	serge	5994	for (i = 0; i < 3; i++)
6084	serge	5995	I915_WRITE(DEW(i), 0);
3031	serge	5996
		5997	/* Program P-state weights to account for frequency power adjustment */
		5998	for (i = 0; i < 16; i++) {
6084	serge	5999	u32 pxvidfreq = I915_READ(PXVFREQ(i));
3031	serge	6000	unsigned long freq = intel_pxfreq(pxvidfreq);
		6001	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		6002	PXVFREQ_PX_SHIFT;
		6003	unsigned long val;
		6004
		6005	val = vid * vid;
		6006	val *= (freq / 1000);
		6007	val *= 255;
		6008	val /= (127127900);
		6009	if (val > 0xff)
		6010	DRM_ERROR("bad pxval: %ld\n", val);
		6011	pxw[i] = val;
		6012	}
		6013	/* Render standby states get 0 weight */
		6014	pxw[14] = 0;
		6015	pxw[15] = 0;
		6016
		6017	for (i = 0; i < 4; i++) {
		6018	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		6019	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
6084	serge	6020	I915_WRITE(PXW(i), val);
3031	serge	6021	}
		6022
		6023	/* Adjust magic regs to magic values (more experimental results) */
		6024	I915_WRITE(OGW0, 0);
		6025	I915_WRITE(OGW1, 0);
		6026	I915_WRITE(EG0, 0x00007f00);
		6027	I915_WRITE(EG1, 0x0000000e);
		6028	I915_WRITE(EG2, 0x000e0000);
		6029	I915_WRITE(EG3, 0x68000300);
		6030	I915_WRITE(EG4, 0x42000000);
		6031	I915_WRITE(EG5, 0x00140031);
		6032	I915_WRITE(EG6, 0);
		6033	I915_WRITE(EG7, 0);
		6034
		6035	for (i = 0; i < 8; i++)
6084	serge	6036	I915_WRITE(PXWL(i), 0);
3031	serge	6037
		6038	/* Enable PMON + select events */
		6039	I915_WRITE(ECR, 0x80000019);
		6040
		6041	lcfuse = I915_READ(LCFUSE02);
		6042
		6043	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		6044	}
		6045
5060	serge	6046	void intel_init_gt_powersave(struct drm_device *dev)
		6047	{
6937	serge	6048	struct drm_i915_private *dev_priv = dev->dev_private;
		6049
5060	serge	6050	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
6937	serge	6051	/*
		6052	* RPM depends on RC6 to save restore the GT HW context, so make RC6 a
		6053	* requirement.
		6054	*/
		6055	if (!i915.enable_rc6) {
		6056	DRM_INFO("RC6 disabled, disabling runtime PM support\n");
		6057	intel_runtime_pm_get(dev_priv);
		6058	}
5060	serge	6059
		6060	if (IS_CHERRYVIEW(dev))
		6061	cherryview_init_gt_powersave(dev);
		6062	else if (IS_VALLEYVIEW(dev))
		6063	valleyview_init_gt_powersave(dev);
		6064	}
		6065
		6066	void intel_cleanup_gt_powersave(struct drm_device *dev)
		6067	{
6937	serge	6068	struct drm_i915_private *dev_priv = dev->dev_private;
		6069
5060	serge	6070	if (IS_CHERRYVIEW(dev))
		6071	return;
		6072	else if (IS_VALLEYVIEW(dev))
		6073	valleyview_cleanup_gt_powersave(dev);
6937	serge	6074
		6075	if (!i915.enable_rc6)
		6076	intel_runtime_pm_put(dev_priv);
5060	serge	6077	}
		6078
5354	serge	6079	static void gen6_suspend_rps(struct drm_device *dev)
		6080	{
		6081	struct drm_i915_private *dev_priv = dev->dev_private;
		6082
		6083	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		6084
6084	serge	6085	gen6_disable_rps_interrupts(dev);
5354	serge	6086	}
		6087
5060	serge	6088	/**
		6089	* intel_suspend_gt_powersave - suspend PM work and helper threads
		6090	* @dev: drm device
		6091	*
		6092	* We don't want to disable RC6 or other features here, we just want
		6093	* to make sure any work we've queued has finished and won't bother
		6094	* us while we're suspended.
		6095	*/
		6096	void intel_suspend_gt_powersave(struct drm_device *dev)
		6097	{
		6098	struct drm_i915_private *dev_priv = dev->dev_private;
		6099
5354	serge	6100	if (INTEL_INFO(dev)->gen < 6)
		6101	return;
5060	serge	6102
5354	serge	6103	gen6_suspend_rps(dev);
5060	serge	6104
		6105	/* Force GPU to min freq during suspend */
		6106	gen6_rps_idle(dev_priv);
		6107	}
		6108
3031	serge	6109	void intel_disable_gt_powersave(struct drm_device *dev)
		6110	{
3243	Serge	6111	struct drm_i915_private *dev_priv = dev->dev_private;
		6112
3031	serge	6113	if (IS_IRONLAKE_M(dev)) {
		6114	ironlake_disable_drps(dev);
4293	Serge	6115	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	6116	intel_suspend_gt_powersave(dev);
		6117
3482	Serge	6118	mutex_lock(&dev_priv->rps.hw_lock);
5354	serge	6119	if (INTEL_INFO(dev)->gen >= 9)
		6120	gen9_disable_rps(dev);
		6121	else if (IS_CHERRYVIEW(dev))
5060	serge	6122	cherryview_disable_rps(dev);
		6123	else if (IS_VALLEYVIEW(dev))
4104	Serge	6124	valleyview_disable_rps(dev);
		6125	else
6084	serge	6126	gen6_disable_rps(dev);
5354	serge	6127
4560	Serge	6128	dev_priv->rps.enabled = false;
3480	Serge	6129	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	6130	}
		6131	}
		6132
3482	Serge	6133	static void intel_gen6_powersave_work(struct work_struct *work)
		6134	{
		6135	struct drm_i915_private *dev_priv =
		6136	container_of(work, struct drm_i915_private,
		6137	rps.delayed_resume_work.work);
		6138	struct drm_device *dev = dev_priv->dev;
		6139
		6140	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	6141
6084	serge	6142	gen6_reset_rps_interrupts(dev);
5354	serge	6143
5060	serge	6144	if (IS_CHERRYVIEW(dev)) {
		6145	cherryview_enable_rps(dev);
		6146	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	6147	valleyview_enable_rps(dev);
5354	serge	6148	} else if (INTEL_INFO(dev)->gen >= 9) {
6084	serge	6149	gen9_enable_rc6(dev);
5354	serge	6150	gen9_enable_rps(dev);
6937	serge	6151	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev))
6084	serge	6152	__gen6_update_ring_freq(dev);
4560	Serge	6153	} else if (IS_BROADWELL(dev)) {
		6154	gen8_enable_rps(dev);
5060	serge	6155	__gen6_update_ring_freq(dev);
4104	Serge	6156	} else {
6084	serge	6157	gen6_enable_rps(dev);
5060	serge	6158	__gen6_update_ring_freq(dev);
4104	Serge	6159	}
6084	serge	6160
		6161	WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
		6162	WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
		6163
		6164	WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
		6165	WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
		6166
4560	Serge	6167	dev_priv->rps.enabled = true;
5354	serge	6168
6084	serge	6169	gen6_enable_rps_interrupts(dev);
5354	serge	6170
3482	Serge	6171	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	6172
		6173	intel_runtime_pm_put(dev_priv);
3482	Serge	6174	}
		6175
3031	serge	6176	void intel_enable_gt_powersave(struct drm_device *dev)
		6177	{
3243	Serge	6178	struct drm_i915_private *dev_priv = dev->dev_private;
		6179
6084	serge	6180	/* Powersaving is controlled by the host when inside a VM */
		6181	if (intel_vgpu_active(dev))
		6182	return;
		6183
3031	serge	6184	if (IS_IRONLAKE_M(dev)) {
5060	serge	6185	mutex_lock(&dev->struct_mutex);
3031	serge	6186	ironlake_enable_drps(dev);
		6187	intel_init_emon(dev);
5060	serge	6188	mutex_unlock(&dev->struct_mutex);
		6189	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	6190	/*
		6191	* PCU communication is slow and this doesn't need to be
		6192	* done at any specific time, so do this out of our fast path
		6193	* to make resume and init faster.
5060	serge	6194	*
		6195	* We depend on the HW RC6 power context save/restore
		6196	* mechanism when entering D3 through runtime PM suspend. So
		6197	* disable RPM until RPS/RC6 is properly setup. We can only
		6198	* get here via the driver load/system resume/runtime resume
		6199	* paths, so the _noresume version is enough (and in case of
		6200	* runtime resume it's necessary).
3243	Serge	6201	*/
5060	serge	6202	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		6203	round_jiffies_up_relative(HZ)))
		6204	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	6205	}
		6206	}
		6207
5060	serge	6208	void intel_reset_gt_powersave(struct drm_device *dev)
		6209	{
		6210	struct drm_i915_private *dev_priv = dev->dev_private;
		6211
5354	serge	6212	if (INTEL_INFO(dev)->gen < 6)
		6213	return;
		6214
		6215	gen6_suspend_rps(dev);
5060	serge	6216	dev_priv->rps.enabled = false;
		6217	}
		6218
3243	Serge	6219	static void ibx_init_clock_gating(struct drm_device *dev)
		6220	{
		6221	struct drm_i915_private *dev_priv = dev->dev_private;
		6222
		6223	/*
		6224	* On Ibex Peak and Cougar Point, we need to disable clock
		6225	* gating for the panel power sequencer or it will fail to
		6226	* start up when no ports are active.
		6227	*/
		6228	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		6229	}
		6230
4104	Serge	6231	static void g4x_disable_trickle_feed(struct drm_device *dev)
		6232	{
		6233	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	6234	enum pipe pipe;
4104	Serge	6235
5354	serge	6236	for_each_pipe(dev_priv, pipe) {
4104	Serge	6237	I915_WRITE(DSPCNTR(pipe),
		6238	I915_READ(DSPCNTR(pipe)) \|
		6239	DISPPLANE_TRICKLE_FEED_DISABLE);
6084	serge	6240
		6241	I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
		6242	POSTING_READ(DSPSURF(pipe));
4104	Serge	6243	}
		6244	}
		6245
4560	Serge	6246	static void ilk_init_lp_watermarks(struct drm_device *dev)
		6247	{
		6248	struct drm_i915_private *dev_priv = dev->dev_private;
		6249
		6250	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		6251	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		6252	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		6253
		6254	/*
		6255	* Don't touch WM1S_LP_EN here.
		6256	* Doing so could cause underruns.
		6257	*/
		6258	}
		6259
3031	serge	6260	static void ironlake_init_clock_gating(struct drm_device *dev)
		6261	{
		6262	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6263	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6264
4104	Serge	6265	/*
		6266	* Required for FBC
		6267	* WaFbcDisableDpfcClockGating:ilk
		6268	*/
3243	Serge	6269	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		6270	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		6271	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	6272
		6273	I915_WRITE(PCH_3DCGDIS0,
		6274	MARIUNIT_CLOCK_GATE_DISABLE \|
		6275	SVSMUNIT_CLOCK_GATE_DISABLE);
		6276	I915_WRITE(PCH_3DCGDIS1,
		6277	VFMUNIT_CLOCK_GATE_DISABLE);
		6278
		6279	/*
		6280	* According to the spec the following bits should be set in
		6281	* order to enable memory self-refresh
		6282	* The bit 22/21 of 0x42004
		6283	* The bit 5 of 0x42020
		6284	* The bit 15 of 0x45000
		6285	*/
		6286	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6287	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6288	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	6289	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	6290	I915_WRITE(DISP_ARB_CTL,
		6291	(I915_READ(DISP_ARB_CTL) \|
		6292	DISP_FBC_WM_DIS));
		6293
4560	Serge	6294	ilk_init_lp_watermarks(dev);
		6295
3031	serge	6296	/*
		6297	* Based on the document from hardware guys the following bits
		6298	* should be set unconditionally in order to enable FBC.
		6299	* The bit 22 of 0x42000
		6300	* The bit 22 of 0x42004
		6301	* The bit 7,8,9 of 0x42020.
		6302	*/
		6303	if (IS_IRONLAKE_M(dev)) {
4104	Serge	6304	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	6305	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6306	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6307	ILK_FBCQ_DIS);
		6308	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6309	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6310	ILK_DPARB_GATE);
		6311	}
		6312
3243	Serge	6313	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		6314
3031	serge	6315	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6316	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6317	ILK_ELPIN_409_SELECT);
		6318	I915_WRITE(_3D_CHICKEN2,
		6319	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		6320	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	6321
4104	Serge	6322	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	6323	I915_WRITE(CACHE_MODE_0,
		6324	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		6325
5060	serge	6326	/* WaDisable_RenderCache_OperationalFlush:ilk */
		6327	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6328
4104	Serge	6329	g4x_disable_trickle_feed(dev);
		6330
3243	Serge	6331	ibx_init_clock_gating(dev);
3031	serge	6332	}
		6333
3243	Serge	6334	static void cpt_init_clock_gating(struct drm_device *dev)
		6335	{
		6336	struct drm_i915_private *dev_priv = dev->dev_private;
		6337	int pipe;
3746	Serge	6338	uint32_t val;
3243	Serge	6339
		6340	/*
		6341	* On Ibex Peak and Cougar Point, we need to disable clock
		6342	* gating for the panel power sequencer or it will fail to
		6343	* start up when no ports are active.
		6344	*/
4280	Serge	6345	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		6346	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		6347	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	6348	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		6349	DPLS_EDP_PPS_FIX_DIS);
		6350	/* The below fixes the weird display corruption, a few pixels shifted
		6351	* downward, on (only) LVDS of some HP laptops with IVY.
		6352	*/
5354	serge	6353	for_each_pipe(dev_priv, pipe) {
3746	Serge	6354	val = I915_READ(TRANS_CHICKEN2(pipe));
		6355	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		6356	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	6357	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	6358	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		6359	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		6360	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		6361	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		6362	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		6363	}
3243	Serge	6364	/* WADP0ClockGatingDisable */
5354	serge	6365	for_each_pipe(dev_priv, pipe) {
3243	Serge	6366	I915_WRITE(TRANS_CHICKEN1(pipe),
		6367	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		6368	}
		6369	}
		6370
3480	Serge	6371	static void gen6_check_mch_setup(struct drm_device *dev)
		6372	{
		6373	struct drm_i915_private *dev_priv = dev->dev_private;
		6374	uint32_t tmp;
		6375
		6376	tmp = I915_READ(MCH_SSKPD);
5060	serge	6377	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		6378	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		6379	tmp);
3480	Serge	6380	}
		6381
3031	serge	6382	static void gen6_init_clock_gating(struct drm_device *dev)
		6383	{
		6384	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6385	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6386
3243	Serge	6387	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	6388
		6389	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6390	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6391	ILK_ELPIN_409_SELECT);
		6392
4104	Serge	6393	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	6394	I915_WRITE(_3D_CHICKEN,
		6395	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		6396
5060	serge	6397	/* WaDisable_RenderCache_OperationalFlush:snb */
		6398	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6399
		6400	/*
		6401	* BSpec recoomends 8x4 when MSAA is used,
		6402	* however in practice 16x4 seems fastest.
		6403	*
		6404	* Note that PS/WM thread counts depend on the WIZ hashing
		6405	* disable bit, which we don't touch here, but it's good
		6406	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6407	*/
		6408	I915_WRITE(GEN6_GT_MODE,
5354	serge	6409	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6410
4560	Serge	6411	ilk_init_lp_watermarks(dev);
3031	serge	6412
		6413	I915_WRITE(CACHE_MODE_0,
		6414	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		6415
		6416	I915_WRITE(GEN6_UCGCTL1,
		6417	I915_READ(GEN6_UCGCTL1) \|
		6418	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		6419	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6420
		6421	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		6422	* gating disable must be set. Failure to set it results in
		6423	* flickering pixels due to Z write ordering failures after
		6424	* some amount of runtime in the Mesa "fire" demo, and Unigine
		6425	* Sanctuary and Tropics, and apparently anything else with
		6426	* alpha test or pixel discard.
		6427	*
		6428	* According to the spec, bit 11 (RCCUNIT) must also be set,
		6429	* but we didn't debug actual testcases to find it out.
		6430	*
5060	serge	6431	* WaDisableRCCUnitClockGating:snb
		6432	* WaDisableRCPBUnitClockGating:snb
3031	serge	6433	*/
		6434	I915_WRITE(GEN6_UCGCTL2,
		6435	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		6436	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		6437
5060	serge	6438	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		6439	I915_WRITE(_3D_CHICKEN3,
		6440	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	6441
		6442	/*
5060	serge	6443	* Bspec says:
		6444	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		6445	* 3DSTATE_SF number of SF output attributes is more than 16."
		6446	*/
		6447	I915_WRITE(_3D_CHICKEN3,
		6448	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		6449
		6450	/*
3031	serge	6451	* According to the spec the following bits should be
		6452	* set in order to enable memory self-refresh and fbc:
		6453	* The bit21 and bit22 of 0x42000
		6454	* The bit21 and bit22 of 0x42004
		6455	* The bit5 and bit7 of 0x42020
		6456	* The bit14 of 0x70180
		6457	* The bit14 of 0x71180
4104	Serge	6458	*
		6459	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	6460	*/
		6461	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6462	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6463	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		6464	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6465	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6466	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	6467	I915_WRITE(ILK_DSPCLK_GATE_D,
		6468	I915_READ(ILK_DSPCLK_GATE_D) \|
		6469	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		6470	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	6471
4104	Serge	6472	g4x_disable_trickle_feed(dev);
3031	serge	6473
3243	Serge	6474	cpt_init_clock_gating(dev);
3480	Serge	6475
		6476	gen6_check_mch_setup(dev);
3031	serge	6477	}
		6478
		6479	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		6480	{
		6481	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		6482
5060	serge	6483	/*
		6484	* WaVSThreadDispatchOverride:ivb,vlv
		6485	*
		6486	* This actually overrides the dispatch
		6487	* mode for all thread types.
		6488	*/
3031	serge	6489	reg &= ~GEN7_FF_SCHED_MASK;
		6490	reg \|= GEN7_FF_TS_SCHED_HW;
		6491	reg \|= GEN7_FF_VS_SCHED_HW;
		6492	reg \|= GEN7_FF_DS_SCHED_HW;
		6493
		6494	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		6495	}
		6496
3243	Serge	6497	static void lpt_init_clock_gating(struct drm_device *dev)
		6498	{
		6499	struct drm_i915_private *dev_priv = dev->dev_private;
		6500
		6501	/*
		6502	* TODO: this bit should only be enabled when really needed, then
		6503	* disabled when not needed anymore in order to save power.
		6504	*/
6084	serge	6505	if (HAS_PCH_LPT_LP(dev))
3243	Serge	6506	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		6507	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		6508	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	6509
		6510	/* WADPOClockGatingDisable:hsw */
6084	serge	6511	I915_WRITE(TRANS_CHICKEN1(PIPE_A),
		6512	I915_READ(TRANS_CHICKEN1(PIPE_A)) \|
4104	Serge	6513	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	6514	}
		6515
4104	Serge	6516	static void lpt_suspend_hw(struct drm_device *dev)
		6517	{
		6518	struct drm_i915_private *dev_priv = dev->dev_private;
		6519
6084	serge	6520	if (HAS_PCH_LPT_LP(dev)) {
4104	Serge	6521	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		6522
		6523	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		6524	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		6525	}
		6526	}
		6527
5354	serge	6528	static void broadwell_init_clock_gating(struct drm_device *dev)
3031	serge	6529	{
		6530	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	6531	enum pipe pipe;
6084	serge	6532	uint32_t misccpctl;
3031	serge	6533
6084	serge	6534	ilk_init_lp_watermarks(dev);
3031	serge	6535
4560	Serge	6536	/* WaSwitchSolVfFArbitrationPriority:bdw */
		6537	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6538
		6539	/* WaPsrDPAMaskVBlankInSRD:bdw */
		6540	I915_WRITE(CHICKEN_PAR1_1,
		6541	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		6542
		6543	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5354	serge	6544	for_each_pipe(dev_priv, pipe) {
5060	serge	6545	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		6546	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		6547	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	6548	}
		6549
		6550	/* WaVSRefCountFullforceMissDisable:bdw */
		6551	/* WaDSRefCountFullforceMissDisable:bdw */
		6552	I915_WRITE(GEN7_FF_THREAD_MODE,
		6553	I915_READ(GEN7_FF_THREAD_MODE) &
		6554	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	6555
		6556	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6557	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6558
		6559	/* WaDisableSDEUnitClockGating:bdw */
		6560	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6561	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		6562
6084	serge	6563	/*
		6564	* WaProgramL3SqcReg1Default:bdw
		6565	* WaTempDisableDOPClkGating:bdw
		6566	*/
		6567	misccpctl = I915_READ(GEN7_MISCCPCTL);
		6568	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		6569	I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6660	serge	6570	/*
		6571	* Wait at least 100 clocks before re-enabling clock gating. See
		6572	* the definition of L3SQCREG1 in BSpec.
		6573	*/
		6574	POSTING_READ(GEN8_L3SQCREG1);
		6575	udelay(1);
6084	serge	6576	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		6577
		6578	/*
		6579	* WaGttCachingOffByDefault:bdw
		6580	* GTT cache may not work with big pages, so if those
		6581	* are ever enabled GTT cache may need to be disabled.
		6582	*/
		6583	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
		6584
5354	serge	6585	lpt_init_clock_gating(dev);
4560	Serge	6586	}
		6587
		6588	static void haswell_init_clock_gating(struct drm_device *dev)
		6589	{
		6590	struct drm_i915_private *dev_priv = dev->dev_private;
		6591
		6592	ilk_init_lp_watermarks(dev);
		6593
4104	Serge	6594	/* L3 caching of data atomics doesn't work -- disable it. */
		6595	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		6596	I915_WRITE(HSW_ROW_CHICKEN3,
		6597	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		6598
		6599	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	6600	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6601	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6602	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6603
4104	Serge	6604	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	6605	I915_WRITE(GEN7_FF_THREAD_MODE,
		6606	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	6607
5060	serge	6608	/* WaDisable_RenderCache_OperationalFlush:hsw */
		6609	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6610
		6611	/* enable HiZ Raw Stall Optimization */
		6612	I915_WRITE(CACHE_MODE_0_GEN7,
		6613	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6614
4104	Serge	6615	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	6616	I915_WRITE(CACHE_MODE_1,
		6617	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6618
5060	serge	6619	/*
		6620	* BSpec recommends 8x4 when MSAA is used,
		6621	* however in practice 16x4 seems fastest.
		6622	*
		6623	* Note that PS/WM thread counts depend on the WIZ hashing
		6624	* disable bit, which we don't touch here, but it's good
		6625	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6626	*/
		6627	I915_WRITE(GEN7_GT_MODE,
5354	serge	6628	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6629
6084	serge	6630	/* WaSampleCChickenBitEnable:hsw */
		6631	I915_WRITE(HALF_SLICE_CHICKEN3,
		6632	_MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
		6633
4104	Serge	6634	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	6635	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6636
4104	Serge	6637	/* WaRsPkgCStateDisplayPMReq:hsw */
		6638	I915_WRITE(CHICKEN_PAR1_1,
		6639	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	6640
3243	Serge	6641	lpt_init_clock_gating(dev);
3031	serge	6642	}
		6643
		6644	static void ivybridge_init_clock_gating(struct drm_device *dev)
		6645	{
		6646	struct drm_i915_private *dev_priv = dev->dev_private;
		6647	uint32_t snpcr;
		6648
4560	Serge	6649	ilk_init_lp_watermarks(dev);
3031	serge	6650
3243	Serge	6651	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6652
4104	Serge	6653	/* WaDisableEarlyCull:ivb */
3243	Serge	6654	I915_WRITE(_3D_CHICKEN3,
		6655	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6656
4104	Serge	6657	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	6658	I915_WRITE(IVB_CHICKEN3,
		6659	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6660	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6661
4104	Serge	6662	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	6663	if (IS_IVB_GT1(dev))
		6664	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		6665	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		6666
5060	serge	6667	/* WaDisable_RenderCache_OperationalFlush:ivb */
		6668	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6669
4104	Serge	6670	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	6671	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		6672	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		6673
4104	Serge	6674	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	6675	I915_WRITE(GEN7_L3CNTLREG1,
		6676	GEN7_WA_FOR_GEN7_L3_CONTROL);
		6677	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6084	serge	6678	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	6679	if (IS_IVB_GT1(dev))
		6680	I915_WRITE(GEN7_ROW_CHICKEN2,
		6681	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6682	else {
		6683	/* must write both registers */
		6684	I915_WRITE(GEN7_ROW_CHICKEN2,
		6685	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	6686	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		6687	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6688	}
3031	serge	6689
4104	Serge	6690	/* WaForceL3Serialization:ivb */
3243	Serge	6691	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6692	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6693
5060	serge	6694	/*
3031	serge	6695	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6696	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	6697	*/
		6698	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6699	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6700
4104	Serge	6701	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	6702	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6703	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6704	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6705
4104	Serge	6706	g4x_disable_trickle_feed(dev);
3031	serge	6707
		6708	gen7_setup_fixed_func_scheduler(dev_priv);
		6709
5060	serge	6710	if (0) { /* causes HiZ corruption on ivb:gt1 */
		6711	/* enable HiZ Raw Stall Optimization */
		6712	I915_WRITE(CACHE_MODE_0_GEN7,
		6713	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6714	}
		6715
4104	Serge	6716	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	6717	I915_WRITE(CACHE_MODE_1,
		6718	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6719
5060	serge	6720	/*
		6721	* BSpec recommends 8x4 when MSAA is used,
		6722	* however in practice 16x4 seems fastest.
		6723	*
		6724	* Note that PS/WM thread counts depend on the WIZ hashing
		6725	* disable bit, which we don't touch here, but it's good
		6726	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6727	*/
		6728	I915_WRITE(GEN7_GT_MODE,
5354	serge	6729	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6730
3031	serge	6731	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		6732	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		6733	snpcr \|= GEN6_MBC_SNPCR_MED;
		6734	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	6735
3746	Serge	6736	if (!HAS_PCH_NOP(dev))
6084	serge	6737	cpt_init_clock_gating(dev);
3480	Serge	6738
		6739	gen6_check_mch_setup(dev);
3031	serge	6740	}
		6741
6084	serge	6742	static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
		6743	{
6937	serge	6744	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6084	serge	6745
		6746	/*
		6747	* Disable trickle feed and enable pnd deadline calculation
		6748	*/
		6749	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		6750	I915_WRITE(CBR1_VLV, 0);
		6751	}
		6752
3031	serge	6753	static void valleyview_init_clock_gating(struct drm_device *dev)
		6754	{
		6755	struct drm_i915_private *dev_priv = dev->dev_private;
		6756
6084	serge	6757	vlv_init_display_clock_gating(dev_priv);
3031	serge	6758
4104	Serge	6759	/* WaDisableEarlyCull:vlv */
3243	Serge	6760	I915_WRITE(_3D_CHICKEN3,
		6761	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6762
4104	Serge	6763	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	6764	I915_WRITE(IVB_CHICKEN3,
		6765	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6766	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6767
5060	serge	6768	/* WaPsdDispatchEnable:vlv */
4104	Serge	6769	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	6770	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	6771	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		6772	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	6773
5060	serge	6774	/* WaDisable_RenderCache_OperationalFlush:vlv */
		6775	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6776
4104	Serge	6777	/* WaForceL3Serialization:vlv */
3243	Serge	6778	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6779	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6780
4104	Serge	6781	/* WaDisableDopClockGating:vlv */
3243	Serge	6782	I915_WRITE(GEN7_ROW_CHICKEN2,
		6783	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		6784
4104	Serge	6785	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	6786	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6787	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6788	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6789
5060	serge	6790	gen7_setup_fixed_func_scheduler(dev_priv);
		6791
		6792	/*
3031	serge	6793	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6794	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	6795	*/
		6796	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6797	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6798
5060	serge	6799	/* WaDisableL3Bank2xClockGate:vlv
		6800	* Disabling L3 clock gating- MMIO 940c[25] = 1
		6801	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		6802	I915_WRITE(GEN7_UCGCTL4,
		6803	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	6804
5060	serge	6805	/*
		6806	* BSpec says this must be set, even though
		6807	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		6808	*/
3031	serge	6809	I915_WRITE(CACHE_MODE_1,
		6810	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6811
		6812	/*
6084	serge	6813	* BSpec recommends 8x4 when MSAA is used,
		6814	* however in practice 16x4 seems fastest.
		6815	*
		6816	* Note that PS/WM thread counts depend on the WIZ hashing
		6817	* disable bit, which we don't touch here, but it's good
		6818	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6819	*/
		6820	I915_WRITE(GEN7_GT_MODE,
		6821	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
		6822
		6823	/*
5060	serge	6824	* WaIncreaseL3CreditsForVLVB0:vlv
		6825	* This is the hardware default actually.
		6826	*/
		6827	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		6828
		6829	/*
4104	Serge	6830	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	6831	* Disable clock gating on th GCFG unit to prevent a delay
		6832	* in the reporting of vblank events.
		6833	*/
5060	serge	6834	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		6835	}
3746	Serge	6836
5060	serge	6837	static void cherryview_init_clock_gating(struct drm_device *dev)
		6838	{
		6839	struct drm_i915_private *dev_priv = dev->dev_private;
		6840
6084	serge	6841	vlv_init_display_clock_gating(dev_priv);
5060	serge	6842
		6843	/* WaVSRefCountFullforceMissDisable:chv */
		6844	/* WaDSRefCountFullforceMissDisable:chv */
		6845	I915_WRITE(GEN7_FF_THREAD_MODE,
		6846	I915_READ(GEN7_FF_THREAD_MODE) &
		6847	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		6848
		6849	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		6850	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6851	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6852
		6853	/* WaDisableCSUnitClockGating:chv */
		6854	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		6855	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6856
		6857	/* WaDisableSDEUnitClockGating:chv */
		6858	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6859	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6084	serge	6860
		6861	/*
		6862	* GTT cache may not work with big pages, so if those
		6863	* are ever enabled GTT cache may need to be disabled.
		6864	*/
		6865	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
3031	serge	6866	}
		6867
		6868	static void g4x_init_clock_gating(struct drm_device *dev)
		6869	{
		6870	struct drm_i915_private *dev_priv = dev->dev_private;
		6871	uint32_t dspclk_gate;
		6872
		6873	I915_WRITE(RENCLK_GATE_D1, 0);
		6874	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		6875	GS_UNIT_CLOCK_GATE_DISABLE \|
		6876	CL_UNIT_CLOCK_GATE_DISABLE);
		6877	I915_WRITE(RAMCLK_GATE_D, 0);
		6878	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		6879	OVRUNIT_CLOCK_GATE_DISABLE \|
		6880	OVCUNIT_CLOCK_GATE_DISABLE;
		6881	if (IS_GM45(dev))
		6882	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		6883	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	6884
		6885	/* WaDisableRenderCachePipelinedFlush */
		6886	I915_WRITE(CACHE_MODE_0,
		6887	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	6888
5060	serge	6889	/* WaDisable_RenderCache_OperationalFlush:g4x */
		6890	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6891
4104	Serge	6892	g4x_disable_trickle_feed(dev);
3031	serge	6893	}
		6894
		6895	static void crestline_init_clock_gating(struct drm_device *dev)
		6896	{
		6897	struct drm_i915_private *dev_priv = dev->dev_private;
		6898
		6899	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		6900	I915_WRITE(RENCLK_GATE_D2, 0);
		6901	I915_WRITE(DSPCLK_GATE_D, 0);
		6902	I915_WRITE(RAMCLK_GATE_D, 0);
		6903	I915_WRITE16(DEUC, 0);
4104	Serge	6904	I915_WRITE(MI_ARB_STATE,
		6905	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6906
		6907	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6908	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6909	}
		6910
		6911	static void broadwater_init_clock_gating(struct drm_device *dev)
		6912	{
		6913	struct drm_i915_private *dev_priv = dev->dev_private;
		6914
		6915	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		6916	I965_RCC_CLOCK_GATE_DISABLE \|
		6917	I965_RCPB_CLOCK_GATE_DISABLE \|
		6918	I965_ISC_CLOCK_GATE_DISABLE \|
		6919	I965_FBC_CLOCK_GATE_DISABLE);
		6920	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	6921	I915_WRITE(MI_ARB_STATE,
		6922	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6923
		6924	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6925	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6926	}
		6927
		6928	static void gen3_init_clock_gating(struct drm_device *dev)
		6929	{
		6930	struct drm_i915_private *dev_priv = dev->dev_private;
		6931	u32 dstate = I915_READ(D_STATE);
		6932
		6933	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		6934	DSTATE_DOT_CLOCK_GATING;
		6935	I915_WRITE(D_STATE, dstate);
		6936
		6937	if (IS_PINEVIEW(dev))
		6938	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		6939
		6940	/* IIR "flip pending" means done if this bit is set */
		6941	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	6942
		6943	/* interrupts should cause a wake up from C3 */
		6944	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		6945
		6946	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		6947	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5354	serge	6948
		6949	I915_WRITE(MI_ARB_STATE,
		6950	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	6951	}
		6952
		6953	static void i85x_init_clock_gating(struct drm_device *dev)
		6954	{
		6955	struct drm_i915_private *dev_priv = dev->dev_private;
		6956
		6957	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	6958
		6959	/* interrupts should cause a wake up from C3 */
		6960	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		6961	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
5354	serge	6962
		6963	I915_WRITE(MEM_MODE,
		6964	_MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	6965	}
		6966
		6967	static void i830_init_clock_gating(struct drm_device *dev)
		6968	{
		6969	struct drm_i915_private *dev_priv = dev->dev_private;
		6970
		6971	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5354	serge	6972
		6973	I915_WRITE(MEM_MODE,
		6974	_MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) \|
		6975	_MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
3031	serge	6976	}
		6977
		6978	void intel_init_clock_gating(struct drm_device *dev)
		6979	{
		6980	struct drm_i915_private *dev_priv = dev->dev_private;
		6981
6084	serge	6982	if (dev_priv->display.init_clock_gating)
		6983	dev_priv->display.init_clock_gating(dev);
3031	serge	6984	}
		6985
4104	Serge	6986	void intel_suspend_hw(struct drm_device *dev)
		6987	{
		6988	if (HAS_PCH_LPT(dev))
		6989	lpt_suspend_hw(dev);
		6990	}
		6991
5354	serge	6992	/* Set up chip specific power management-related functions */
		6993	void intel_init_pm(struct drm_device *dev)
		6994	{
		6995	struct drm_i915_private *dev_priv = dev->dev_private;
		6996
6084	serge	6997	intel_fbc_init(dev_priv);
5354	serge	6998
3031	serge	6999	/* For cxsr */
		7000	if (IS_PINEVIEW(dev))
		7001	i915_pineview_get_mem_freq(dev);
		7002	else if (IS_GEN5(dev))
		7003	i915_ironlake_get_mem_freq(dev);
		7004
		7005	/* For FIFO watermark updates */
5354	serge	7006	if (INTEL_INFO(dev)->gen >= 9) {
		7007	skl_setup_wm_latency(dev);
		7008
6084	serge	7009	if (IS_BROXTON(dev))
		7010	dev_priv->display.init_clock_gating =
		7011	bxt_init_clock_gating;
5354	serge	7012	dev_priv->display.update_wm = skl_update_wm;
		7013	} else if (HAS_PCH_SPLIT(dev)) {
5060	serge	7014	ilk_setup_wm_latency(dev);
4104	Serge	7015
4560	Serge	7016	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		7017	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		7018	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		7019	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		7020	dev_priv->display.update_wm = ilk_update_wm;
6937	serge	7021	dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
6084	serge	7022	} else {
		7023	DRM_DEBUG_KMS("Failed to read display plane latency. "
		7024	"Disable CxSR\n");
		7025	}
4560	Serge	7026
		7027	if (IS_GEN5(dev))
		7028	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		7029	else if (IS_GEN6(dev))
3031	serge	7030	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	7031	else if (IS_IVYBRIDGE(dev))
3031	serge	7032	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	7033	else if (IS_HASWELL(dev))
3031	serge	7034	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	7035	else if (INTEL_INFO(dev)->gen == 8)
5354	serge	7036	dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
5060	serge	7037	} else if (IS_CHERRYVIEW(dev)) {
6084	serge	7038	vlv_setup_wm_latency(dev);
		7039
		7040	dev_priv->display.update_wm = vlv_update_wm;
5060	serge	7041	dev_priv->display.init_clock_gating =
		7042	cherryview_init_clock_gating;
3031	serge	7043	} else if (IS_VALLEYVIEW(dev)) {
6084	serge	7044	vlv_setup_wm_latency(dev);
		7045
		7046	dev_priv->display.update_wm = vlv_update_wm;
3031	serge	7047	dev_priv->display.init_clock_gating =
		7048	valleyview_init_clock_gating;
		7049	} else if (IS_PINEVIEW(dev)) {
		7050	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		7051	dev_priv->is_ddr3,
		7052	dev_priv->fsb_freq,
		7053	dev_priv->mem_freq)) {
		7054	DRM_INFO("failed to find known CxSR latency "
		7055	"(found ddr%s fsb freq %d, mem freq %d), "
		7056	"disabling CxSR\n",
		7057	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		7058	dev_priv->fsb_freq, dev_priv->mem_freq);
		7059	/* Disable CxSR and never update its watermark again */
5060	serge	7060	intel_set_memory_cxsr(dev_priv, false);
3031	serge	7061	dev_priv->display.update_wm = NULL;
		7062	} else
		7063	dev_priv->display.update_wm = pineview_update_wm;
		7064	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		7065	} else if (IS_G4X(dev)) {
		7066	dev_priv->display.update_wm = g4x_update_wm;
		7067	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		7068	} else if (IS_GEN4(dev)) {
		7069	dev_priv->display.update_wm = i965_update_wm;
		7070	if (IS_CRESTLINE(dev))
		7071	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		7072	else if (IS_BROADWATER(dev))
		7073	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		7074	} else if (IS_GEN3(dev)) {
		7075	dev_priv->display.update_wm = i9xx_update_wm;
		7076	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		7077	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	7078	} else if (IS_GEN2(dev)) {
		7079	if (INTEL_INFO(dev)->num_pipes == 1) {
		7080	dev_priv->display.update_wm = i845_update_wm;
		7081	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		7082	} else {
		7083	dev_priv->display.update_wm = i9xx_update_wm;
6084	serge	7084	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	7085	}
		7086
		7087	if (IS_I85X(dev) \|\| IS_I865G(dev))
6084	serge	7088	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	7089	else
		7090	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	7091	} else {
4560	Serge	7092	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	7093	}
		7094	}
		7095
5354	serge	7096	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u32 mbox, u32 val)
3243	Serge	7097	{
		7098	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	7099
3243	Serge	7100	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7101	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		7102	return -EAGAIN;
		7103	}
3031	serge	7104
3243	Serge	7105	I915_WRITE(GEN6_PCODE_DATA, *val);
5354	serge	7106	I915_WRITE(GEN6_PCODE_DATA1, 0);
3243	Serge	7107	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7108
		7109	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7110	500)) {
		7111	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		7112	return -ETIMEDOUT;
6084	serge	7113	}
3243	Serge	7114
		7115	*val = I915_READ(GEN6_PCODE_DATA);
		7116	I915_WRITE(GEN6_PCODE_DATA, 0);
		7117
		7118	return 0;
		7119	}
		7120
5354	serge	7121	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
3243	Serge	7122	{
		7123	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		7124
		7125	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7126	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		7127	return -EAGAIN;
6084	serge	7128	}
3243	Serge	7129
		7130	I915_WRITE(GEN6_PCODE_DATA, val);
		7131	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7132
		7133	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7134	500)) {
		7135	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		7136	return -ETIMEDOUT;
3031	serge	7137	}
3243	Serge	7138
		7139	I915_WRITE(GEN6_PCODE_DATA, 0);
		7140
		7141	return 0;
3031	serge	7142	}
3746	Serge	7143
5354	serge	7144	static int vlv_gpu_freq_div(unsigned int czclk_freq)
3746	Serge	7145	{
5354	serge	7146	switch (czclk_freq) {
		7147	case 200:
		7148	return 10;
		7149	case 267:
		7150	return 12;
		7151	case 320:
		7152	case 333:
		7153	return 16;
		7154	case 400:
		7155	return 20;
4104	Serge	7156	default:
		7157	return -1;
		7158	}
5354	serge	7159	}
3746	Serge	7160
5354	serge	7161	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7162	{
6084	serge	7163	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5354	serge	7164
		7165	div = vlv_gpu_freq_div(czclk_freq);
		7166	if (div < 0)
		7167	return div;
		7168
		7169	return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
4104	Serge	7170	}
3746	Serge	7171
5060	serge	7172	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	7173	{
6084	serge	7174	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
3746	Serge	7175
5354	serge	7176	mul = vlv_gpu_freq_div(czclk_freq);
		7177	if (mul < 0)
		7178	return mul;
3746	Serge	7179
5354	serge	7180	return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
3746	Serge	7181	}
		7182
5060	serge	7183	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7184	{
6084	serge	7185	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7186
5354	serge	7187	div = vlv_gpu_freq_div(czclk_freq) / 2;
		7188	if (div < 0)
		7189	return div;
5060	serge	7190
5354	serge	7191	return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
5060	serge	7192	}
		7193
		7194	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7195	{
6084	serge	7196	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7197
5354	serge	7198	mul = vlv_gpu_freq_div(czclk_freq) / 2;
		7199	if (mul < 0)
		7200	return mul;
5060	serge	7201
5354	serge	7202	/* CHV needs even values */
		7203	return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
5060	serge	7204	}
		7205
6084	serge	7206	int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
5060	serge	7207	{
6084	serge	7208	if (IS_GEN9(dev_priv->dev))
		7209	return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
		7210	GEN9_FREQ_SCALER);
		7211	else if (IS_CHERRYVIEW(dev_priv->dev))
		7212	return chv_gpu_freq(dev_priv, val);
		7213	else if (IS_VALLEYVIEW(dev_priv->dev))
		7214	return byt_gpu_freq(dev_priv, val);
		7215	else
		7216	return val * GT_FREQUENCY_MULTIPLIER;
		7217	}
5060	serge	7218
6084	serge	7219	int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7220	{
		7221	if (IS_GEN9(dev_priv->dev))
		7222	return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
		7223	GT_FREQUENCY_MULTIPLIER);
		7224	else if (IS_CHERRYVIEW(dev_priv->dev))
		7225	return chv_freq_opcode(dev_priv, val);
5060	serge	7226	else if (IS_VALLEYVIEW(dev_priv->dev))
6084	serge	7227	return byt_freq_opcode(dev_priv, val);
		7228	else
		7229	return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
		7230	}
5060	serge	7231
6084	serge	7232	struct request_boost {
		7233	struct work_struct work;
		7234	struct drm_i915_gem_request *req;
		7235	};
		7236
		7237	static void __intel_rps_boost_work(struct work_struct *work)
		7238	{
		7239	struct request_boost *boost = container_of(work, struct request_boost, work);
		7240	struct drm_i915_gem_request *req = boost->req;
		7241
		7242	if (!i915_gem_request_completed(req, true))
		7243	gen6_rps_boost(to_i915(req->ring->dev), NULL,
		7244	req->emitted_jiffies);
		7245
		7246	i915_gem_request_unreference__unlocked(req);
		7247	kfree(boost);
5060	serge	7248	}
		7249
6084	serge	7250	void intel_queue_rps_boost_for_request(struct drm_device *dev,
		7251	struct drm_i915_gem_request *req)
5060	serge	7252	{
6084	serge	7253	struct request_boost *boost;
5060	serge	7254
6084	serge	7255	if (req == NULL \|\| INTEL_INFO(dev)->gen < 6)
		7256	return;
5060	serge	7257
6084	serge	7258	if (i915_gem_request_completed(req, true))
		7259	return;
		7260
		7261	boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
		7262	if (boost == NULL)
		7263	return;
		7264
		7265	i915_gem_request_reference(req);
		7266	boost->req = req;
		7267
		7268	INIT_WORK(&boost->work, __intel_rps_boost_work);
		7269	queue_work(to_i915(dev)->wq, &boost->work);
5060	serge	7270	}
		7271
4560	Serge	7272	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7273	{
4104	Serge	7274	struct drm_i915_private *dev_priv = dev->dev_private;
		7275
4560	Serge	7276	mutex_init(&dev_priv->rps.hw_lock);
6084	serge	7277	spin_lock_init(&dev_priv->rps.client_lock);
4560	Serge	7278
4104	Serge	7279	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7280	intel_gen6_powersave_work);
6084	serge	7281	INIT_LIST_HEAD(&dev_priv->rps.clients);
		7282	INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
		7283	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
5060	serge	7284
		7285	dev_priv->pm.suspended = false;
6937	serge	7286	atomic_set(&dev_priv->pm.wakeref_count, 0);
		7287	atomic_set(&dev_priv->pm.atomic_seq, 0);
3746	Serge	7288	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 6937