WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	//#include "../../../platform/x86/intel_ips.h"
		32	#include
		33
4560	Serge	34
3031	serge	35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	void getrawmonotonic(struct timespec *ts);
		38
4560	Serge	39	/**
7144	serge	40	* DOC: RC6
		41	*
4560	Serge	42	* RC6 is a special power stage which allows the GPU to enter an very
		43	* low-voltage mode when idle, using down to 0V while at this stage. This
		44	* stage is entered automatically when the GPU is idle when RC6 support is
		45	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		46	*
		47	* There are different RC6 modes available in Intel GPU, which differentiate
		48	* among each other with the latency required to enter and leave RC6 and
		49	* voltage consumed by the GPU in different states.
		50	*
		51	* The combination of the following flags define which states GPU is allowed
		52	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		53	* RC6pp is deepest RC6. Their support by hardware varies according to the
		54	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		55	* which brings the most power savings; deeper states save more power, but
		56	* require higher latency to switch to and wake up.
		57	*/
		58	#define INTEL_RC6_ENABLE (1<<0)
		59	#define INTEL_RC6p_ENABLE (1<<1)
		60	#define INTEL_RC6pp_ENABLE (1<<2)
		61
6084	serge	62	static void bxt_init_clock_gating(struct drm_device *dev)
5354	serge	63	{
		64	struct drm_i915_private *dev_priv = dev->dev_private;
		65
6084	serge	66	/* WaDisableSDEUnitClockGating:bxt */
5354	serge	67	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		68	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		69
		70	/*
6084	serge	71	* FIXME:
		72	* GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
5354	serge	73	*/
6084	serge	74	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		75	GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
6937	serge	76
		77	/*
		78	* Wa: Backlight PWM may stop in the asserted state, causing backlight
		79	* to stay fully on.
		80	*/
		81	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
		82	I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) \|
		83	PWM1_GATING_DIS \| PWM2_GATING_DIS);
5354	serge	84	}
		85
3031	serge	86	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		87	{
5060	serge	88	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	89	u32 tmp;
		90
		91	tmp = I915_READ(CLKCFG);
		92
		93	switch (tmp & CLKCFG_FSB_MASK) {
		94	case CLKCFG_FSB_533:
		95	dev_priv->fsb_freq = 533; /* 1334 /
		96	break;
		97	case CLKCFG_FSB_800:
		98	dev_priv->fsb_freq = 800; /* 2004 /
		99	break;
		100	case CLKCFG_FSB_667:
		101	dev_priv->fsb_freq = 667; /* 1674 /
		102	break;
		103	case CLKCFG_FSB_400:
		104	dev_priv->fsb_freq = 400; /* 1004 /
		105	break;
		106	}
		107
		108	switch (tmp & CLKCFG_MEM_MASK) {
		109	case CLKCFG_MEM_533:
		110	dev_priv->mem_freq = 533;
		111	break;
		112	case CLKCFG_MEM_667:
		113	dev_priv->mem_freq = 667;
		114	break;
		115	case CLKCFG_MEM_800:
		116	dev_priv->mem_freq = 800;
		117	break;
		118	}
		119
		120	/* detect pineview DDR3 setting */
		121	tmp = I915_READ(CSHRDDR3CTL);
		122	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		123	}
		124
		125	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		126	{
5060	serge	127	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	128	u16 ddrpll, csipll;
		129
		130	ddrpll = I915_READ16(DDRMPLL1);
		131	csipll = I915_READ16(CSIPLL0);
		132
		133	switch (ddrpll & 0xff) {
		134	case 0xc:
		135	dev_priv->mem_freq = 800;
		136	break;
		137	case 0x10:
		138	dev_priv->mem_freq = 1066;
		139	break;
		140	case 0x14:
		141	dev_priv->mem_freq = 1333;
		142	break;
		143	case 0x18:
		144	dev_priv->mem_freq = 1600;
		145	break;
		146	default:
		147	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		148	ddrpll & 0xff);
		149	dev_priv->mem_freq = 0;
		150	break;
		151	}
		152
		153	dev_priv->ips.r_t = dev_priv->mem_freq;
		154
		155	switch (csipll & 0x3ff) {
		156	case 0x00c:
		157	dev_priv->fsb_freq = 3200;
		158	break;
		159	case 0x00e:
		160	dev_priv->fsb_freq = 3733;
		161	break;
		162	case 0x010:
		163	dev_priv->fsb_freq = 4266;
		164	break;
		165	case 0x012:
		166	dev_priv->fsb_freq = 4800;
		167	break;
		168	case 0x014:
		169	dev_priv->fsb_freq = 5333;
		170	break;
		171	case 0x016:
		172	dev_priv->fsb_freq = 5866;
		173	break;
		174	case 0x018:
		175	dev_priv->fsb_freq = 6400;
		176	break;
		177	default:
		178	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		179	csipll & 0x3ff);
		180	dev_priv->fsb_freq = 0;
		181	break;
		182	}
		183
		184	if (dev_priv->fsb_freq == 3200) {
		185	dev_priv->ips.c_m = 0;
		186	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		187	dev_priv->ips.c_m = 1;
		188	} else {
		189	dev_priv->ips.c_m = 2;
		190	}
		191	}
		192
		193	static const struct cxsr_latency cxsr_latency_table[] = {
		194	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		195	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		196	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		197	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		198	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		199
		200	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		201	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		202	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		203	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		204	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		205
		206	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		207	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		208	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		209	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		210	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		211
		212	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		213	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		214	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		215	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		216	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		217
		218	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		219	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		220	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		221	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		222	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		223
		224	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		225	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		226	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		227	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		228	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		229	};
		230
		231	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		232	int is_ddr3,
		233	int fsb,
		234	int mem)
		235	{
		236	const struct cxsr_latency *latency;
		237	int i;
		238
		239	if (fsb == 0 \|\| mem == 0)
		240	return NULL;
		241
		242	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		243	latency = &cxsr_latency_table[i];
		244	if (is_desktop == latency->is_desktop &&
		245	is_ddr3 == latency->is_ddr3 &&
		246	fsb == latency->fsb_freq && mem == latency->mem_freq)
		247	return latency;
		248	}
		249
		250	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		251
		252	return NULL;
		253	}
		254
6084	serge	255	static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
		256	{
		257	u32 val;
		258
		259	mutex_lock(&dev_priv->rps.hw_lock);
		260
		261	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		262	if (enable)
		263	val &= ~FORCE_DDR_HIGH_FREQ;
		264	else
		265	val \|= FORCE_DDR_HIGH_FREQ;
		266	val &= ~FORCE_DDR_LOW_FREQ;
		267	val \|= FORCE_DDR_FREQ_REQ_ACK;
		268	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		269
		270	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		271	FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
		272	DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
		273
		274	mutex_unlock(&dev_priv->rps.hw_lock);
		275	}
		276
		277	static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
		278	{
		279	u32 val;
		280
		281	mutex_lock(&dev_priv->rps.hw_lock);
		282
		283	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		284	if (enable)
		285	val \|= DSP_MAXFIFO_PM5_ENABLE;
		286	else
		287	val &= ~DSP_MAXFIFO_PM5_ENABLE;
		288	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
		289
		290	mutex_unlock(&dev_priv->rps.hw_lock);
		291	}
		292
		293	#define FW_WM(value, plane) \
		294	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
		295
5060	serge	296	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	297	{
5060	serge	298	struct drm_device *dev = dev_priv->dev;
		299	u32 val;
3031	serge	300
6937	serge	301	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev)) {
5060	serge	302	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
6084	serge	303	POSTING_READ(FW_BLC_SELF_VLV);
		304	dev_priv->wm.vlv.cxsr = enable;
5060	serge	305	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		306	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
6084	serge	307	POSTING_READ(FW_BLC_SELF);
5060	serge	308	} else if (IS_PINEVIEW(dev)) {
		309	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		310	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		311	I915_WRITE(DSPFW3, val);
6084	serge	312	POSTING_READ(DSPFW3);
5060	serge	313	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		314	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		315	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		316	I915_WRITE(FW_BLC_SELF, val);
6084	serge	317	POSTING_READ(FW_BLC_SELF);
5060	serge	318	} else if (IS_I915GM(dev)) {
		319	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		320	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		321	I915_WRITE(INSTPM, val);
6084	serge	322	POSTING_READ(INSTPM);
5060	serge	323	} else {
		324	return;
		325	}
		326
		327	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		328	enable ? "enabled" : "disabled");
3031	serge	329	}
		330
6084	serge	331
3031	serge	332	/*
		333	* Latency for FIFO fetches is dependent on several factors:
		334	* - memory configuration (speed, channels)
		335	* - chipset
		336	* - current MCH state
		337	* It can be fairly high in some situations, so here we assume a fairly
		338	* pessimal value. It's a tradeoff between extra memory fetches (if we
		339	* set this value too high, the FIFO will fetch frequently to stay full)
		340	* and power consumption (set it too low to save power and we might see
		341	* FIFO underruns and display "flicker").
		342	*
		343	* A value of 5us seems to be a good balance; safe for very low end
		344	* platforms but not overly aggressive on lower latency configs.
		345	*/
5354	serge	346	static const int pessimal_latency_ns = 5000;
3031	serge	347
6084	serge	348	#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
		349	((((dsparb) >> (lo_shift)) & 0xff) \| ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
		350
		351	static int vlv_get_fifo_size(struct drm_device *dev,
		352	enum pipe pipe, int plane)
		353	{
		354	struct drm_i915_private *dev_priv = dev->dev_private;
		355	int sprite0_start, sprite1_start, size;
		356
		357	switch (pipe) {
		358	uint32_t dsparb, dsparb2, dsparb3;
		359	case PIPE_A:
		360	dsparb = I915_READ(DSPARB);
		361	dsparb2 = I915_READ(DSPARB2);
		362	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		363	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		364	break;
		365	case PIPE_B:
		366	dsparb = I915_READ(DSPARB);
		367	dsparb2 = I915_READ(DSPARB2);
		368	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		369	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		370	break;
		371	case PIPE_C:
		372	dsparb2 = I915_READ(DSPARB2);
		373	dsparb3 = I915_READ(DSPARB3);
		374	sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		375	sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		376	break;
		377	default:
		378	return 0;
		379	}
		380
		381	switch (plane) {
		382	case 0:
		383	size = sprite0_start;
		384	break;
		385	case 1:
		386	size = sprite1_start - sprite0_start;
		387	break;
		388	case 2:
		389	size = 512 - 1 - sprite1_start;
		390	break;
		391	default:
		392	return 0;
		393	}
		394
		395	DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
		396	pipe_name(pipe), plane == 0 ? "primary" : "sprite",
		397	plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
		398	size);
		399
		400	return size;
		401	}
		402
3031	serge	403	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		404	{
		405	struct drm_i915_private *dev_priv = dev->dev_private;
		406	uint32_t dsparb = I915_READ(DSPARB);
		407	int size;
		408
		409	size = dsparb & 0x7f;
		410	if (plane)
		411	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		412
		413	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		414	plane ? "B" : "A", size);
		415
		416	return size;
		417	}
		418
4560	Serge	419	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	420	{
		421	struct drm_i915_private *dev_priv = dev->dev_private;
		422	uint32_t dsparb = I915_READ(DSPARB);
		423	int size;
		424
		425	size = dsparb & 0x1ff;
		426	if (plane)
		427	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		428	size >>= 1; /* Convert to cachelines */
		429
		430	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		431	plane ? "B" : "A", size);
		432
		433	return size;
		434	}
		435
		436	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		437	{
		438	struct drm_i915_private *dev_priv = dev->dev_private;
		439	uint32_t dsparb = I915_READ(DSPARB);
		440	int size;
		441
		442	size = dsparb & 0x7f;
		443	size >>= 2; /* Convert to cachelines */
		444
		445	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		446	plane ? "B" : "A",
		447	size);
		448
		449	return size;
		450	}
		451
		452	/* Pineview has different values for various configs */
		453	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	454	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		455	.max_wm = PINEVIEW_MAX_WM,
		456	.default_wm = PINEVIEW_DFT_WM,
		457	.guard_size = PINEVIEW_GUARD_WM,
		458	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	459	};
		460	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	461	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		462	.max_wm = PINEVIEW_MAX_WM,
		463	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		464	.guard_size = PINEVIEW_GUARD_WM,
		465	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	466	};
		467	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	468	.fifo_size = PINEVIEW_CURSOR_FIFO,
		469	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		470	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		471	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		472	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	473	};
		474	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	475	.fifo_size = PINEVIEW_CURSOR_FIFO,
		476	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		477	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		478	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		479	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	480	};
		481	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	482	.fifo_size = G4X_FIFO_SIZE,
		483	.max_wm = G4X_MAX_WM,
		484	.default_wm = G4X_MAX_WM,
		485	.guard_size = 2,
		486	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	487	};
		488	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	489	.fifo_size = I965_CURSOR_FIFO,
		490	.max_wm = I965_CURSOR_MAX_WM,
		491	.default_wm = I965_CURSOR_DFT_WM,
		492	.guard_size = 2,
		493	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	494	};
		495	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	496	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		497	.max_wm = VALLEYVIEW_MAX_WM,
		498	.default_wm = VALLEYVIEW_MAX_WM,
		499	.guard_size = 2,
		500	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	501	};
		502	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	503	.fifo_size = I965_CURSOR_FIFO,
		504	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		505	.default_wm = I965_CURSOR_DFT_WM,
		506	.guard_size = 2,
		507	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	508	};
		509	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	510	.fifo_size = I965_CURSOR_FIFO,
		511	.max_wm = I965_CURSOR_MAX_WM,
		512	.default_wm = I965_CURSOR_DFT_WM,
		513	.guard_size = 2,
		514	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	515	};
		516	static const struct intel_watermark_params i945_wm_info = {
5060	serge	517	.fifo_size = I945_FIFO_SIZE,
		518	.max_wm = I915_MAX_WM,
		519	.default_wm = 1,
		520	.guard_size = 2,
		521	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	522	};
		523	static const struct intel_watermark_params i915_wm_info = {
5060	serge	524	.fifo_size = I915_FIFO_SIZE,
		525	.max_wm = I915_MAX_WM,
		526	.default_wm = 1,
		527	.guard_size = 2,
		528	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	529	};
5354	serge	530	static const struct intel_watermark_params i830_a_wm_info = {
5060	serge	531	.fifo_size = I855GM_FIFO_SIZE,
		532	.max_wm = I915_MAX_WM,
		533	.default_wm = 1,
		534	.guard_size = 2,
		535	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	536	};
5354	serge	537	static const struct intel_watermark_params i830_bc_wm_info = {
		538	.fifo_size = I855GM_FIFO_SIZE,
		539	.max_wm = I915_MAX_WM/2,
		540	.default_wm = 1,
		541	.guard_size = 2,
		542	.cacheline_size = I830_FIFO_LINE_SIZE,
		543	};
4560	Serge	544	static const struct intel_watermark_params i845_wm_info = {
5060	serge	545	.fifo_size = I830_FIFO_SIZE,
		546	.max_wm = I915_MAX_WM,
		547	.default_wm = 1,
		548	.guard_size = 2,
		549	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	550	};
		551
		552	/**
		553	* intel_calculate_wm - calculate watermark level
		554	* @clock_in_khz: pixel clock
		555	* @wm: chip FIFO params
7144	serge	556	* @cpp: bytes per pixel
3031	serge	557	* @latency_ns: memory latency for the platform
		558	*
		559	* Calculate the watermark level (the level at which the display plane will
		560	* start fetching from memory again). Each chip has a different display
		561	* FIFO size and allocation, so the caller needs to figure that out and pass
		562	* in the correct intel_watermark_params structure.
		563	*
		564	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		565	* on the pixel size. When it reaches the watermark level, it'll start
		566	* fetching FIFO line sized based chunks from memory until the FIFO fills
		567	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		568	* will occur, and a display engine hang could result.
		569	*/
		570	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		571	const struct intel_watermark_params *wm,
7144	serge	572	int fifo_size, int cpp,
3031	serge	573	unsigned long latency_ns)
		574	{
		575	long entries_required, wm_size;
		576
		577	/*
		578	* Note: we need to make sure we don't overflow for various clock &
		579	* latency values.
		580	* clocks go from a few thousand to several hundred thousand.
		581	* latency is usually a few thousand
		582	*/
7144	serge	583	entries_required = ((clock_in_khz / 1000) * cpp * latency_ns) /
3031	serge	584	1000;
		585	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		586
		587	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		588
		589	wm_size = fifo_size - (entries_required + wm->guard_size);
		590
		591	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		592
		593	/* Don't promote wm_size to unsigned... */
		594	if (wm_size > (long)wm->max_wm)
		595	wm_size = wm->max_wm;
		596	if (wm_size <= 0)
		597	wm_size = wm->default_wm;
5354	serge	598
		599	/*
		600	* Bspec seems to indicate that the value shouldn't be lower than
		601	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
		602	* Lets go for 8 which is the burst size since certain platforms
		603	* already use a hardcoded 8 (which is what the spec says should be
		604	* done).
		605	*/
		606	if (wm_size <= 8)
		607	wm_size = 8;
		608
3031	serge	609	return wm_size;
		610	}
		611
		612	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		613	{
		614	struct drm_crtc crtc, enabled = NULL;
		615
5060	serge	616	for_each_crtc(dev, crtc) {
3243	Serge	617	if (intel_crtc_active(crtc)) {
3031	serge	618	if (enabled)
		619	return NULL;
		620	enabled = crtc;
		621	}
		622	}
		623
		624	return enabled;
		625	}
		626
4560	Serge	627	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	628	{
4560	Serge	629	struct drm_device *dev = unused_crtc->dev;
3031	serge	630	struct drm_i915_private *dev_priv = dev->dev_private;
		631	struct drm_crtc *crtc;
		632	const struct cxsr_latency *latency;
		633	u32 reg;
		634	unsigned long wm;
		635
		636	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		637	dev_priv->fsb_freq, dev_priv->mem_freq);
		638	if (!latency) {
		639	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	640	intel_set_memory_cxsr(dev_priv, false);
3031	serge	641	return;
		642	}
		643
		644	crtc = single_enabled_crtc(dev);
		645	if (crtc) {
6084	serge	646	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
7144	serge	647	int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
6084	serge	648	int clock = adjusted_mode->crtc_clock;
3031	serge	649
		650	/* Display SR */
		651	wm = intel_calculate_wm(clock, &pineview_display_wm,
		652	pineview_display_wm.fifo_size,
7144	serge	653	cpp, latency->display_sr);
3031	serge	654	reg = I915_READ(DSPFW1);
		655	reg &= ~DSPFW_SR_MASK;
6084	serge	656	reg \|= FW_WM(wm, SR);
3031	serge	657	I915_WRITE(DSPFW1, reg);
		658	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		659
		660	/* cursor SR */
		661	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		662	pineview_display_wm.fifo_size,
7144	serge	663	cpp, latency->cursor_sr);
3031	serge	664	reg = I915_READ(DSPFW3);
		665	reg &= ~DSPFW_CURSOR_SR_MASK;
6084	serge	666	reg \|= FW_WM(wm, CURSOR_SR);
3031	serge	667	I915_WRITE(DSPFW3, reg);
		668
		669	/* Display HPLL off SR */
		670	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		671	pineview_display_hplloff_wm.fifo_size,
7144	serge	672	cpp, latency->display_hpll_disable);
3031	serge	673	reg = I915_READ(DSPFW3);
		674	reg &= ~DSPFW_HPLL_SR_MASK;
6084	serge	675	reg \|= FW_WM(wm, HPLL_SR);
3031	serge	676	I915_WRITE(DSPFW3, reg);
		677
		678	/* cursor HPLL off SR */
		679	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		680	pineview_display_hplloff_wm.fifo_size,
7144	serge	681	cpp, latency->cursor_hpll_disable);
3031	serge	682	reg = I915_READ(DSPFW3);
		683	reg &= ~DSPFW_HPLL_CURSOR_MASK;
6084	serge	684	reg \|= FW_WM(wm, HPLL_CURSOR);
3031	serge	685	I915_WRITE(DSPFW3, reg);
		686	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		687
5060	serge	688	intel_set_memory_cxsr(dev_priv, true);
3031	serge	689	} else {
5060	serge	690	intel_set_memory_cxsr(dev_priv, false);
3031	serge	691	}
		692	}
		693
		694	static bool g4x_compute_wm0(struct drm_device *dev,
		695	int plane,
		696	const struct intel_watermark_params *display,
		697	int display_latency_ns,
		698	const struct intel_watermark_params *cursor,
		699	int cursor_latency_ns,
		700	int *plane_wm,
		701	int *cursor_wm)
		702	{
		703	struct drm_crtc *crtc;
4560	Serge	704	const struct drm_display_mode *adjusted_mode;
7144	serge	705	int htotal, hdisplay, clock, cpp;
3031	serge	706	int line_time_us, line_count;
		707	int entries, tlb_miss;
		708
		709	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	710	if (!intel_crtc_active(crtc)) {
3031	serge	711	*cursor_wm = cursor->guard_size;
		712	*plane_wm = display->guard_size;
6084	serge	713	return false;
3031	serge	714	}
		715
6084	serge	716	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	717	clock = adjusted_mode->crtc_clock;
		718	htotal = adjusted_mode->crtc_htotal;
6084	serge	719	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
7144	serge	720	cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
3031	serge	721
		722	/* Use the small buffer method to calculate plane watermark */
7144	serge	723	entries = ((clock * cpp / 1000) * display_latency_ns) / 1000;
3031	serge	724	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		725	if (tlb_miss > 0)
		726	entries += tlb_miss;
		727	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		728	*plane_wm = entries + display->guard_size;
		729	if (*plane_wm > (int)display->max_wm)
		730	*plane_wm = display->max_wm;
		731
		732	/* Use the large buffer method to calculate cursor watermark */
5060	serge	733	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	734	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
7144	serge	735	entries = line_count * crtc->cursor->state->crtc_w * cpp;
3031	serge	736	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		737	if (tlb_miss > 0)
		738	entries += tlb_miss;
		739	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		740	*cursor_wm = entries + cursor->guard_size;
		741	if (*cursor_wm > (int)cursor->max_wm)
		742	*cursor_wm = (int)cursor->max_wm;
		743
		744	return true;
		745	}
		746
		747	/*
		748	* Check the wm result.
		749	*
		750	* If any calculated watermark values is larger than the maximum value that
		751	* can be programmed into the associated watermark register, that watermark
		752	* must be disabled.
		753	*/
		754	static bool g4x_check_srwm(struct drm_device *dev,
		755	int display_wm, int cursor_wm,
		756	const struct intel_watermark_params *display,
		757	const struct intel_watermark_params *cursor)
		758	{
		759	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		760	display_wm, cursor_wm);
		761
		762	if (display_wm > display->max_wm) {
		763	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		764	display_wm, display->max_wm);
		765	return false;
		766	}
		767
		768	if (cursor_wm > cursor->max_wm) {
		769	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		770	cursor_wm, cursor->max_wm);
		771	return false;
		772	}
		773
		774	if (!(display_wm \|\| cursor_wm)) {
		775	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		776	return false;
		777	}
		778
		779	return true;
		780	}
		781
		782	static bool g4x_compute_srwm(struct drm_device *dev,
		783	int plane,
		784	int latency_ns,
		785	const struct intel_watermark_params *display,
		786	const struct intel_watermark_params *cursor,
		787	int display_wm, int cursor_wm)
		788	{
		789	struct drm_crtc *crtc;
4560	Serge	790	const struct drm_display_mode *adjusted_mode;
7144	serge	791	int hdisplay, htotal, cpp, clock;
3031	serge	792	unsigned long line_time_us;
		793	int line_count, line_size;
		794	int small, large;
		795	int entries;
		796
		797	if (!latency_ns) {
		798	display_wm = cursor_wm = 0;
		799	return false;
		800	}
		801
		802	crtc = intel_get_crtc_for_plane(dev, plane);
6084	serge	803	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	804	clock = adjusted_mode->crtc_clock;
		805	htotal = adjusted_mode->crtc_htotal;
6084	serge	806	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
7144	serge	807	cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
3031	serge	808
5060	serge	809	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	810	line_count = (latency_ns / line_time_us + 1000) / 1000;
7144	serge	811	line_size = hdisplay * cpp;
3031	serge	812
		813	/* Use the minimum of the small and large buffer method for primary */
7144	serge	814	small = ((clock * cpp / 1000) * latency_ns) / 1000;
3031	serge	815	large = line_count * line_size;
		816
		817	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		818	*display_wm = entries + display->guard_size;
		819
		820	/* calculate the self-refresh watermark for display cursor */
7144	serge	821	entries = line_count * cpp * crtc->cursor->state->crtc_w;
3031	serge	822	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		823	*cursor_wm = entries + cursor->guard_size;
		824
		825	return g4x_check_srwm(dev,
		826	display_wm, cursor_wm,
		827	display, cursor);
		828	}
		829
6084	serge	830	#define FW_WM_VLV(value, plane) \
		831	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
		832
		833	static void vlv_write_wm_values(struct intel_crtc *crtc,
		834	const struct vlv_wm_values *wm)
3031	serge	835	{
6084	serge	836	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
		837	enum pipe pipe = crtc->pipe;
3031	serge	838
6084	serge	839	I915_WRITE(VLV_DDL(pipe),
		840	(wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) \|
		841	(wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) \|
		842	(wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) \|
		843	(wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
3031	serge	844
6084	serge	845	I915_WRITE(DSPFW1,
		846	FW_WM(wm->sr.plane, SR) \|
		847	FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) \|
		848	FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) \|
		849	FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
		850	I915_WRITE(DSPFW2,
		851	FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) \|
		852	FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) \|
		853	FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
		854	I915_WRITE(DSPFW3,
		855	FW_WM(wm->sr.cursor, CURSOR_SR));
3031	serge	856
6084	serge	857	if (IS_CHERRYVIEW(dev_priv)) {
		858	I915_WRITE(DSPFW7_CHV,
		859	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		860	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		861	I915_WRITE(DSPFW8_CHV,
		862	FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) \|
		863	FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
		864	I915_WRITE(DSPFW9_CHV,
		865	FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) \|
		866	FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
		867	I915_WRITE(DSPHOWM,
		868	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		869	FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) \|
		870	FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) \|
		871	FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) \|
		872	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		873	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		874	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		875	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		876	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		877	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		878	} else {
		879	I915_WRITE(DSPFW7,
		880	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		881	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		882	I915_WRITE(DSPHOWM,
		883	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		884	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		885	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		886	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		887	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		888	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		889	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		890	}
3031	serge	891
6084	serge	892	/* zero (unused) WM1 watermarks */
		893	I915_WRITE(DSPFW4, 0);
		894	I915_WRITE(DSPFW5, 0);
		895	I915_WRITE(DSPFW6, 0);
		896	I915_WRITE(DSPHOWM1, 0);
3031	serge	897
6084	serge	898	POSTING_READ(DSPFW1);
3031	serge	899	}
		900
6084	serge	901	#undef FW_WM_VLV
3031	serge	902
6084	serge	903	enum vlv_wm_level {
		904	VLV_WM_LEVEL_PM2,
		905	VLV_WM_LEVEL_PM5,
		906	VLV_WM_LEVEL_DDR_DVFS,
		907	};
		908
		909	/* latency must be in 0.1us units. */
		910	static unsigned int vlv_wm_method2(unsigned int pixel_rate,
		911	unsigned int pipe_htotal,
		912	unsigned int horiz_pixels,
7144	serge	913	unsigned int cpp,
6084	serge	914	unsigned int latency)
3031	serge	915	{
6084	serge	916	unsigned int ret;
		917
		918	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
7144	serge	919	ret = (ret + 1) * horiz_pixels * cpp;
6084	serge	920	ret = DIV_ROUND_UP(ret, 64);
		921
		922	return ret;
		923	}
		924
		925	static void vlv_setup_wm_latency(struct drm_device *dev)
		926	{
3031	serge	927	struct drm_i915_private *dev_priv = dev->dev_private;
		928
6084	serge	929	/* all latencies in usec */
		930	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
3031	serge	931
6084	serge	932	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
		933
		934	if (IS_CHERRYVIEW(dev_priv)) {
		935	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		936	dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
		937
		938	dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
3031	serge	939	}
6084	serge	940	}
3031	serge	941
6084	serge	942	static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
		943	struct intel_crtc *crtc,
		944	const struct intel_plane_state *state,
		945	int level)
		946	{
		947	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
7144	serge	948	int clock, htotal, cpp, width, wm;
6084	serge	949
		950	if (dev_priv->wm.pri_latency[level] == 0)
		951	return USHRT_MAX;
		952
		953	if (!state->visible)
		954	return 0;
		955
7144	serge	956	cpp = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
6084	serge	957	clock = crtc->config->base.adjusted_mode.crtc_clock;
		958	htotal = crtc->config->base.adjusted_mode.crtc_htotal;
		959	width = crtc->config->pipe_src_w;
		960	if (WARN_ON(htotal == 0))
		961	htotal = 1;
		962
		963	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		964	/*
		965	* FIXME the formula gives values that are
		966	* too big for the cursor FIFO, and hence we
		967	* would never be able to use cursors. For
		968	* now just hardcode the watermark.
		969	*/
		970	wm = 63;
		971	} else {
7144	serge	972	wm = vlv_wm_method2(clock, htotal, width, cpp,
6084	serge	973	dev_priv->wm.pri_latency[level] * 10);
5354	serge	974	}
3031	serge	975
6084	serge	976	return min_t(int, wm, USHRT_MAX);
		977	}
5354	serge	978
6084	serge	979	static void vlv_compute_fifo(struct intel_crtc *crtc)
		980	{
		981	struct drm_device *dev = crtc->base.dev;
		982	struct vlv_wm_state *wm_state = &crtc->wm_state;
		983	struct intel_plane *plane;
		984	unsigned int total_rate = 0;
		985	const int fifo_size = 512 - 1;
		986	int fifo_extra, fifo_left = fifo_size;
		987
		988	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		989	struct intel_plane_state *state =
		990	to_intel_plane_state(plane->base.state);
		991
		992	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		993	continue;
		994
		995	if (state->visible) {
		996	wm_state->num_active_planes++;
		997	total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		998	}
3031	serge	999	}
5354	serge	1000
6084	serge	1001	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1002	struct intel_plane_state *state =
		1003	to_intel_plane_state(plane->base.state);
		1004	unsigned int rate;
		1005
		1006	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1007	plane->wm.fifo_size = 63;
		1008	continue;
		1009	}
		1010
		1011	if (!state->visible) {
		1012	plane->wm.fifo_size = 0;
		1013	continue;
		1014	}
		1015
		1016	rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		1017	plane->wm.fifo_size = fifo_size * rate / total_rate;
		1018	fifo_left -= plane->wm.fifo_size;
		1019	}
		1020
		1021	fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
		1022
		1023	/* spread the remainder evenly */
		1024	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1025	int plane_extra;
		1026
		1027	if (fifo_left == 0)
		1028	break;
		1029
		1030	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		1031	continue;
		1032
		1033	/* give it all to the first plane if none are active */
		1034	if (plane->wm.fifo_size == 0 &&
		1035	wm_state->num_active_planes)
		1036	continue;
		1037
		1038	plane_extra = min(fifo_extra, fifo_left);
		1039	plane->wm.fifo_size += plane_extra;
		1040	fifo_left -= plane_extra;
		1041	}
		1042
		1043	WARN_ON(fifo_left != 0);
3031	serge	1044	}
		1045
6084	serge	1046	static void vlv_invert_wms(struct intel_crtc *crtc)
		1047	{
		1048	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1049	int level;
3031	serge	1050
6084	serge	1051	for (level = 0; level < wm_state->num_levels; level++) {
		1052	struct drm_device *dev = crtc->base.dev;
		1053	const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1054	struct intel_plane *plane;
		1055
		1056	wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
		1057	wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
		1058
		1059	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1060	switch (plane->base.type) {
		1061	int sprite;
		1062	case DRM_PLANE_TYPE_CURSOR:
		1063	wm_state->wm[level].cursor = plane->wm.fifo_size -
		1064	wm_state->wm[level].cursor;
		1065	break;
		1066	case DRM_PLANE_TYPE_PRIMARY:
		1067	wm_state->wm[level].primary = plane->wm.fifo_size -
		1068	wm_state->wm[level].primary;
		1069	break;
		1070	case DRM_PLANE_TYPE_OVERLAY:
		1071	sprite = plane->plane;
		1072	wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
		1073	wm_state->wm[level].sprite[sprite];
		1074	break;
		1075	}
		1076	}
		1077	}
		1078	}
		1079
		1080	static void vlv_compute_wm(struct intel_crtc *crtc)
3031	serge	1081	{
6084	serge	1082	struct drm_device *dev = crtc->base.dev;
		1083	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1084	struct intel_plane *plane;
		1085	int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1086	int level;
3031	serge	1087
6084	serge	1088	memset(wm_state, 0, sizeof(*wm_state));
3031	serge	1089
6084	serge	1090	wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
		1091	wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
3031	serge	1092
6084	serge	1093	wm_state->num_active_planes = 0;
3031	serge	1094
6084	serge	1095	vlv_compute_fifo(crtc);
		1096
		1097	if (wm_state->num_active_planes != 1)
		1098	wm_state->cxsr = false;
		1099
		1100	if (wm_state->cxsr) {
		1101	for (level = 0; level < wm_state->num_levels; level++) {
		1102	wm_state->sr[level].plane = sr_fifo_size;
		1103	wm_state->sr[level].cursor = 63;
		1104	}
3243	Serge	1105	}
3031	serge	1106
6084	serge	1107	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1108	struct intel_plane_state *state =
		1109	to_intel_plane_state(plane->base.state);
3031	serge	1110
6084	serge	1111	if (!state->visible)
		1112	continue;
5060	serge	1113
6084	serge	1114	/* normal watermarks */
		1115	for (level = 0; level < wm_state->num_levels; level++) {
		1116	int wm = vlv_compute_wm_level(plane, crtc, state, level);
		1117	int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
		1118
		1119	/* hack */
		1120	if (WARN_ON(level == 0 && wm > max_wm))
		1121	wm = max_wm;
		1122
		1123	if (wm > plane->wm.fifo_size)
		1124	break;
		1125
		1126	switch (plane->base.type) {
		1127	int sprite;
		1128	case DRM_PLANE_TYPE_CURSOR:
		1129	wm_state->wm[level].cursor = wm;
		1130	break;
		1131	case DRM_PLANE_TYPE_PRIMARY:
		1132	wm_state->wm[level].primary = wm;
		1133	break;
		1134	case DRM_PLANE_TYPE_OVERLAY:
		1135	sprite = plane->plane;
		1136	wm_state->wm[level].sprite[sprite] = wm;
		1137	break;
		1138	}
		1139	}
		1140
		1141	wm_state->num_levels = level;
		1142
		1143	if (!wm_state->cxsr)
		1144	continue;
		1145
		1146	/* maxfifo watermarks */
		1147	switch (plane->base.type) {
		1148	int sprite, level;
		1149	case DRM_PLANE_TYPE_CURSOR:
		1150	for (level = 0; level < wm_state->num_levels; level++)
		1151	wm_state->sr[level].cursor =
		1152	wm_state->wm[level].cursor;
		1153	break;
		1154	case DRM_PLANE_TYPE_PRIMARY:
		1155	for (level = 0; level < wm_state->num_levels; level++)
		1156	wm_state->sr[level].plane =
		1157	min(wm_state->sr[level].plane,
		1158	wm_state->wm[level].primary);
		1159	break;
		1160	case DRM_PLANE_TYPE_OVERLAY:
		1161	sprite = plane->plane;
		1162	for (level = 0; level < wm_state->num_levels; level++)
		1163	wm_state->sr[level].plane =
		1164	min(wm_state->sr[level].plane,
		1165	wm_state->wm[level].sprite[sprite]);
		1166	break;
		1167	}
		1168	}
		1169
		1170	/* clear any (partially) filled invalid levels */
		1171	for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
		1172	memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
		1173	memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
		1174	}
		1175
		1176	vlv_invert_wms(crtc);
3031	serge	1177	}
		1178
6084	serge	1179	#define VLV_FIFO(plane, value) \
		1180	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
		1181
		1182	static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
5354	serge	1183	{
6084	serge	1184	struct drm_device *dev = crtc->base.dev;
		1185	struct drm_i915_private *dev_priv = to_i915(dev);
		1186	struct intel_plane *plane;
		1187	int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
5354	serge	1188
6084	serge	1189	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1190	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1191	WARN_ON(plane->wm.fifo_size != 63);
		1192	continue;
		1193	}
5354	serge	1194
6084	serge	1195	if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		1196	sprite0_start = plane->wm.fifo_size;
		1197	else if (plane->plane == 0)
		1198	sprite1_start = sprite0_start + plane->wm.fifo_size;
		1199	else
		1200	fifo_size = sprite1_start + plane->wm.fifo_size;
		1201	}
5354	serge	1202
6084	serge	1203	WARN_ON(fifo_size != 512 - 1);
5354	serge	1204
6084	serge	1205	DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
		1206	pipe_name(crtc->pipe), sprite0_start,
		1207	sprite1_start, fifo_size);
5354	serge	1208
6084	serge	1209	switch (crtc->pipe) {
		1210	uint32_t dsparb, dsparb2, dsparb3;
		1211	case PIPE_A:
		1212	dsparb = I915_READ(DSPARB);
		1213	dsparb2 = I915_READ(DSPARB2);
		1214
		1215	dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) \|
		1216	VLV_FIFO(SPRITEB, 0xff));
		1217	dsparb \|= (VLV_FIFO(SPRITEA, sprite0_start) \|
		1218	VLV_FIFO(SPRITEB, sprite1_start));
		1219
		1220	dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) \|
		1221	VLV_FIFO(SPRITEB_HI, 0x1));
		1222	dsparb2 \|= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) \|
		1223	VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
		1224
		1225	I915_WRITE(DSPARB, dsparb);
		1226	I915_WRITE(DSPARB2, dsparb2);
		1227	break;
		1228	case PIPE_B:
		1229	dsparb = I915_READ(DSPARB);
		1230	dsparb2 = I915_READ(DSPARB2);
		1231
		1232	dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) \|
		1233	VLV_FIFO(SPRITED, 0xff));
		1234	dsparb \|= (VLV_FIFO(SPRITEC, sprite0_start) \|
		1235	VLV_FIFO(SPRITED, sprite1_start));
		1236
		1237	dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) \|
		1238	VLV_FIFO(SPRITED_HI, 0xff));
		1239	dsparb2 \|= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) \|
		1240	VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
		1241
		1242	I915_WRITE(DSPARB, dsparb);
		1243	I915_WRITE(DSPARB2, dsparb2);
		1244	break;
		1245	case PIPE_C:
		1246	dsparb3 = I915_READ(DSPARB3);
		1247	dsparb2 = I915_READ(DSPARB2);
		1248
		1249	dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) \|
		1250	VLV_FIFO(SPRITEF, 0xff));
		1251	dsparb3 \|= (VLV_FIFO(SPRITEE, sprite0_start) \|
		1252	VLV_FIFO(SPRITEF, sprite1_start));
		1253
		1254	dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) \|
		1255	VLV_FIFO(SPRITEF_HI, 0xff));
		1256	dsparb2 \|= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) \|
		1257	VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
		1258
		1259	I915_WRITE(DSPARB3, dsparb3);
		1260	I915_WRITE(DSPARB2, dsparb2);
		1261	break;
		1262	default:
		1263	break;
5354	serge	1264	}
6084	serge	1265	}
5354	serge	1266
6084	serge	1267	#undef VLV_FIFO
5354	serge	1268
6084	serge	1269	static void vlv_merge_wm(struct drm_device *dev,
		1270	struct vlv_wm_values *wm)
		1271	{
		1272	struct intel_crtc *crtc;
		1273	int num_active_crtcs = 0;
5354	serge	1274
6084	serge	1275	wm->level = to_i915(dev)->wm.max_level;
		1276	wm->cxsr = true;
		1277
		1278	for_each_intel_crtc(dev, crtc) {
		1279	const struct vlv_wm_state *wm_state = &crtc->wm_state;
		1280
		1281	if (!crtc->active)
		1282	continue;
		1283
		1284	if (!wm_state->cxsr)
		1285	wm->cxsr = false;
		1286
		1287	num_active_crtcs++;
		1288	wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
		1289	}
		1290
		1291	if (num_active_crtcs != 1)
		1292	wm->cxsr = false;
		1293
		1294	if (num_active_crtcs > 1)
		1295	wm->level = VLV_WM_LEVEL_PM2;
		1296
		1297	for_each_intel_crtc(dev, crtc) {
		1298	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1299	enum pipe pipe = crtc->pipe;
		1300
		1301	if (!crtc->active)
		1302	continue;
		1303
		1304	wm->pipe[pipe] = wm_state->wm[wm->level];
		1305	if (wm->cxsr)
		1306	wm->sr = wm_state->sr[wm->level];
		1307
		1308	wm->ddl[pipe].primary = DDL_PRECISION_HIGH \| 2;
		1309	wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH \| 2;
		1310	wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH \| 2;
		1311	wm->ddl[pipe].cursor = DDL_PRECISION_HIGH \| 2;
		1312	}
5354	serge	1313	}
		1314
6084	serge	1315	static void vlv_update_wm(struct drm_crtc *crtc)
5354	serge	1316	{
		1317	struct drm_device *dev = crtc->dev;
		1318	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1319	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1320	enum pipe pipe = intel_crtc->pipe;
		1321	struct vlv_wm_values wm = {};
5354	serge	1322
6084	serge	1323	vlv_compute_wm(intel_crtc);
		1324	vlv_merge_wm(dev, &wm);
5354	serge	1325
6084	serge	1326	if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
		1327	/* FIXME should be part of crtc atomic commit */
		1328	vlv_pipe_set_fifo_size(intel_crtc);
		1329	return;
5354	serge	1330	}
		1331
6084	serge	1332	if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
		1333	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
		1334	chv_set_memory_dvfs(dev_priv, false);
		1335
		1336	if (wm.level < VLV_WM_LEVEL_PM5 &&
		1337	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
		1338	chv_set_memory_pm5(dev_priv, false);
		1339
		1340	if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
		1341	intel_set_memory_cxsr(dev_priv, false);
		1342
		1343	/* FIXME should be part of crtc atomic commit */
		1344	vlv_pipe_set_fifo_size(intel_crtc);
		1345
		1346	vlv_write_wm_values(intel_crtc, &wm);
		1347
		1348	DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
		1349	"sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
		1350	pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
		1351	wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
		1352	wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
		1353
		1354	if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
		1355	intel_set_memory_cxsr(dev_priv, true);
		1356
		1357	if (wm.level >= VLV_WM_LEVEL_PM5 &&
		1358	dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
		1359	chv_set_memory_pm5(dev_priv, true);
		1360
		1361	if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
		1362	dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
		1363	chv_set_memory_dvfs(dev_priv, true);
		1364
		1365	dev_priv->wm.vlv = wm;
5354	serge	1366	}
		1367
6084	serge	1368	#define single_plane_enabled(mask) is_power_of_2(mask)
		1369
4560	Serge	1370	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1371	{
4560	Serge	1372	struct drm_device *dev = crtc->dev;
3031	serge	1373	static const int sr_latency_ns = 12000;
		1374	struct drm_i915_private *dev_priv = dev->dev_private;
		1375	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1376	int plane_sr, cursor_sr;
		1377	unsigned int enabled = 0;
5060	serge	1378	bool cxsr_enabled;
3031	serge	1379
3746	Serge	1380	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1381	&g4x_wm_info, pessimal_latency_ns,
		1382	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1383	&planea_wm, &cursora_wm))
3746	Serge	1384	enabled \|= 1 << PIPE_A;
3031	serge	1385
3746	Serge	1386	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1387	&g4x_wm_info, pessimal_latency_ns,
		1388	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1389	&planeb_wm, &cursorb_wm))
3746	Serge	1390	enabled \|= 1 << PIPE_B;
3031	serge	1391
		1392	if (single_plane_enabled(enabled) &&
		1393	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1394	sr_latency_ns,
		1395	&g4x_wm_info,
		1396	&g4x_cursor_wm_info,
3243	Serge	1397	&plane_sr, &cursor_sr)) {
5060	serge	1398	cxsr_enabled = true;
3243	Serge	1399	} else {
5060	serge	1400	cxsr_enabled = false;
		1401	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1402	plane_sr = cursor_sr = 0;
		1403	}
3031	serge	1404
5354	serge	1405	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1406	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1407	planea_wm, cursora_wm,
		1408	planeb_wm, cursorb_wm,
		1409	plane_sr, cursor_sr);
		1410
		1411	I915_WRITE(DSPFW1,
6084	serge	1412	FW_WM(plane_sr, SR) \|
		1413	FW_WM(cursorb_wm, CURSORB) \|
		1414	FW_WM(planeb_wm, PLANEB) \|
		1415	FW_WM(planea_wm, PLANEA));
3031	serge	1416	I915_WRITE(DSPFW2,
3243	Serge	1417	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
6084	serge	1418	FW_WM(cursora_wm, CURSORA));
3031	serge	1419	/* HPLL off in SR has some issues on G4x... disable it */
		1420	I915_WRITE(DSPFW3,
3243	Serge	1421	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
6084	serge	1422	FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1423
		1424	if (cxsr_enabled)
		1425	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1426	}
		1427
4560	Serge	1428	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1429	{
4560	Serge	1430	struct drm_device *dev = unused_crtc->dev;
3031	serge	1431	struct drm_i915_private *dev_priv = dev->dev_private;
		1432	struct drm_crtc *crtc;
		1433	int srwm = 1;
		1434	int cursor_sr = 16;
5060	serge	1435	bool cxsr_enabled;
3031	serge	1436
		1437	/* Calc sr entries for one plane configs */
		1438	crtc = single_enabled_crtc(dev);
		1439	if (crtc) {
		1440	/* self-refresh has much higher latency */
		1441	static const int sr_latency_ns = 12000;
6084	serge	1442	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1443	int clock = adjusted_mode->crtc_clock;
		1444	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1445	int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
7144	serge	1446	int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
3031	serge	1447	unsigned long line_time_us;
		1448	int entries;
		1449
5060	serge	1450	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1451
		1452	/* Use ns/us then divide to preserve precision */
		1453	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
7144	serge	1454	cpp * hdisplay;
3031	serge	1455	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1456	srwm = I965_FIFO_SIZE - entries;
		1457	if (srwm < 0)
		1458	srwm = 1;
		1459	srwm &= 0x1ff;
		1460	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1461	entries, srwm);
		1462
		1463	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
7144	serge	1464	cpp * crtc->cursor->state->crtc_w;
3031	serge	1465	entries = DIV_ROUND_UP(entries,
		1466	i965_cursor_wm_info.cacheline_size);
		1467	cursor_sr = i965_cursor_wm_info.fifo_size -
		1468	(entries + i965_cursor_wm_info.guard_size);
		1469
		1470	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1471	cursor_sr = i965_cursor_wm_info.max_wm;
		1472
		1473	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1474	"cursor %d\n", srwm, cursor_sr);
		1475
5060	serge	1476	cxsr_enabled = true;
3031	serge	1477	} else {
5060	serge	1478	cxsr_enabled = false;
3031	serge	1479	/* Turn off self refresh if both pipes are enabled */
5060	serge	1480	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1481	}
		1482
		1483	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1484	srwm);
		1485
		1486	/* 965 has limitations... */
6084	serge	1487	I915_WRITE(DSPFW1, FW_WM(srwm, SR) \|
		1488	FW_WM(8, CURSORB) \|
		1489	FW_WM(8, PLANEB) \|
		1490	FW_WM(8, PLANEA));
		1491	I915_WRITE(DSPFW2, FW_WM(8, CURSORA) \|
		1492	FW_WM(8, PLANEC_OLD));
3031	serge	1493	/* update cursor SR watermark */
6084	serge	1494	I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1495
		1496	if (cxsr_enabled)
		1497	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1498	}
		1499
6084	serge	1500	#undef FW_WM
		1501
4560	Serge	1502	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1503	{
4560	Serge	1504	struct drm_device *dev = unused_crtc->dev;
3031	serge	1505	struct drm_i915_private *dev_priv = dev->dev_private;
		1506	const struct intel_watermark_params *wm_info;
		1507	uint32_t fwater_lo;
		1508	uint32_t fwater_hi;
		1509	int cwm, srwm = 1;
		1510	int fifo_size;
		1511	int planea_wm, planeb_wm;
		1512	struct drm_crtc crtc, enabled = NULL;
		1513
		1514	if (IS_I945GM(dev))
		1515	wm_info = &i945_wm_info;
		1516	else if (!IS_GEN2(dev))
		1517	wm_info = &i915_wm_info;
		1518	else
5354	serge	1519	wm_info = &i830_a_wm_info;
3031	serge	1520
		1521	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1522	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1523	if (intel_crtc_active(crtc)) {
4560	Serge	1524	const struct drm_display_mode *adjusted_mode;
7144	serge	1525	int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
3243	Serge	1526	if (IS_GEN2(dev))
		1527	cpp = 4;
		1528
6084	serge	1529	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1530	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1531	wm_info, fifo_size, cpp,
5354	serge	1532	pessimal_latency_ns);
3031	serge	1533	enabled = crtc;
5354	serge	1534	} else {
3031	serge	1535	planea_wm = fifo_size - wm_info->guard_size;
5354	serge	1536	if (planea_wm > (long)wm_info->max_wm)
		1537	planea_wm = wm_info->max_wm;
		1538	}
3031	serge	1539
5354	serge	1540	if (IS_GEN2(dev))
		1541	wm_info = &i830_bc_wm_info;
		1542
3031	serge	1543	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1544	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1545	if (intel_crtc_active(crtc)) {
4560	Serge	1546	const struct drm_display_mode *adjusted_mode;
7144	serge	1547	int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
3243	Serge	1548	if (IS_GEN2(dev))
		1549	cpp = 4;
		1550
6084	serge	1551	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1552	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1553	wm_info, fifo_size, cpp,
5354	serge	1554	pessimal_latency_ns);
3031	serge	1555	if (enabled == NULL)
		1556	enabled = crtc;
		1557	else
		1558	enabled = NULL;
5354	serge	1559	} else {
3031	serge	1560	planeb_wm = fifo_size - wm_info->guard_size;
5354	serge	1561	if (planeb_wm > (long)wm_info->max_wm)
		1562	planeb_wm = wm_info->max_wm;
		1563	}
3031	serge	1564
		1565	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1566
5060	serge	1567	if (IS_I915GM(dev) && enabled) {
		1568	struct drm_i915_gem_object *obj;
		1569
6084	serge	1570	obj = intel_fb_obj(enabled->primary->state->fb);
5060	serge	1571
		1572	/* self-refresh seems busted with untiled */
		1573	if (obj->tiling_mode == I915_TILING_NONE)
		1574	enabled = NULL;
		1575	}
		1576
3031	serge	1577	/*
		1578	* Overlay gets an aggressive default since video jitter is bad.
		1579	*/
		1580	cwm = 2;
		1581
		1582	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1583	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1584
		1585	/* Calc sr entries for one plane configs */
		1586	if (HAS_FW_BLC(dev) && enabled) {
		1587	/* self-refresh has much higher latency */
		1588	static const int sr_latency_ns = 6000;
6084	serge	1589	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
4560	Serge	1590	int clock = adjusted_mode->crtc_clock;
		1591	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1592	int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
7144	serge	1593	int cpp = drm_format_plane_cpp(enabled->primary->state->fb->pixel_format, 0);
3031	serge	1594	unsigned long line_time_us;
		1595	int entries;
		1596
5060	serge	1597	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1598
		1599	/* Use ns/us then divide to preserve precision */
		1600	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
7144	serge	1601	cpp * hdisplay;
3031	serge	1602	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1603	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1604	srwm = wm_info->fifo_size - entries;
		1605	if (srwm < 0)
		1606	srwm = 1;
		1607
		1608	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1609	I915_WRITE(FW_BLC_SELF,
		1610	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1611	else if (IS_I915GM(dev))
		1612	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1613	}
		1614
		1615	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1616	planea_wm, planeb_wm, cwm, srwm);
		1617
		1618	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1619	fwater_hi = (cwm & 0x1f);
		1620
		1621	/* Set request length to 8 cachelines per fetch */
		1622	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1623	fwater_hi = fwater_hi \| (1 << 8);
		1624
		1625	I915_WRITE(FW_BLC, fwater_lo);
		1626	I915_WRITE(FW_BLC2, fwater_hi);
		1627
5060	serge	1628	if (enabled)
		1629	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1630	}
		1631
4560	Serge	1632	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1633	{
4560	Serge	1634	struct drm_device *dev = unused_crtc->dev;
3031	serge	1635	struct drm_i915_private *dev_priv = dev->dev_private;
		1636	struct drm_crtc *crtc;
4560	Serge	1637	const struct drm_display_mode *adjusted_mode;
3031	serge	1638	uint32_t fwater_lo;
		1639	int planea_wm;
		1640
		1641	crtc = single_enabled_crtc(dev);
		1642	if (crtc == NULL)
		1643	return;
		1644
6084	serge	1645	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1646	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1647	&i845_wm_info,
3031	serge	1648	dev_priv->display.get_fifo_size(dev, 0),
5354	serge	1649	4, pessimal_latency_ns);
3031	serge	1650	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1651	fwater_lo \|= (3<<8) \| planea_wm;
		1652
		1653	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1654
		1655	I915_WRITE(FW_BLC, fwater_lo);
		1656	}
		1657
6084	serge	1658	uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
3031	serge	1659	{
4104	Serge	1660	uint32_t pixel_rate;
		1661
6084	serge	1662	pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
4104	Serge	1663
		1664	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1665	* adjust the pixel_rate here. */
		1666
6084	serge	1667	if (pipe_config->pch_pfit.enabled) {
4104	Serge	1668	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6084	serge	1669	uint32_t pfit_size = pipe_config->pch_pfit.size;
4104	Serge	1670
6084	serge	1671	pipe_w = pipe_config->pipe_src_w;
		1672	pipe_h = pipe_config->pipe_src_h;
		1673
4104	Serge	1674	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1675	pfit_h = pfit_size & 0xFFFF;
		1676	if (pipe_w < pfit_w)
		1677	pipe_w = pfit_w;
		1678	if (pipe_h < pfit_h)
		1679	pipe_h = pfit_h;
		1680
7144	serge	1681	if (WARN_ON(!pfit_w \|\| !pfit_h))
		1682	return pixel_rate;
		1683
4104	Serge	1684	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1685	pfit_w * pfit_h);
		1686	}
		1687
		1688	return pixel_rate;
		1689	}
		1690
		1691	/* latency must be in 0.1us units. */
7144	serge	1692	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latency)
4104	Serge	1693	{
		1694	uint64_t ret;
		1695
		1696	if (WARN(latency == 0, "Latency value missing\n"))
		1697	return UINT_MAX;
		1698
7144	serge	1699	ret = (uint64_t) pixel_rate * cpp * latency;
4104	Serge	1700	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1701
		1702	return ret;
		1703	}
		1704
		1705	/* latency must be in 0.1us units. */
		1706	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
7144	serge	1707	uint32_t horiz_pixels, uint8_t cpp,
4104	Serge	1708	uint32_t latency)
		1709	{
		1710	uint32_t ret;
		1711
		1712	if (WARN(latency == 0, "Latency value missing\n"))
		1713	return UINT_MAX;
7144	serge	1714	if (WARN_ON(!pipe_htotal))
		1715	return UINT_MAX;
4104	Serge	1716
		1717	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
7144	serge	1718	ret = (ret + 1) * horiz_pixels * cpp;
4104	Serge	1719	ret = DIV_ROUND_UP(ret, 64) + 2;
		1720	return ret;
		1721	}
		1722
		1723	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
7144	serge	1724	uint8_t cpp)
4104	Serge	1725	{
7144	serge	1726	/*
		1727	* Neither of these should be possible since this function shouldn't be
		1728	* called if the CRTC is off or the plane is invisible. But let's be
		1729	* extra paranoid to avoid a potential divide-by-zero if we screw up
		1730	* elsewhere in the driver.
		1731	*/
		1732	if (WARN_ON(!cpp))
		1733	return 0;
		1734	if (WARN_ON(!horiz_pixels))
		1735	return 0;
		1736
		1737	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
4104	Serge	1738	}
		1739
4560	Serge	1740	struct ilk_wm_maximums {
4104	Serge	1741	uint16_t pri;
		1742	uint16_t spr;
		1743	uint16_t cur;
		1744	uint16_t fbc;
		1745	};
		1746
		1747	/*
		1748	* For both WM_PIPE and WM_LP.
		1749	* mem_value must be in 0.1us units.
		1750	*/
6084	serge	1751	static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
		1752	const struct intel_plane_state *pstate,
4104	Serge	1753	uint32_t mem_value,
		1754	bool is_lp)
		1755	{
7144	serge	1756	int cpp = pstate->base.fb ?
		1757	drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
4104	Serge	1758	uint32_t method1, method2;
		1759
6084	serge	1760	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1761	return 0;
		1762
7144	serge	1763	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), cpp, mem_value);
4104	Serge	1764
		1765	if (!is_lp)
		1766	return method1;
		1767
6084	serge	1768	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1769	cstate->base.adjusted_mode.crtc_htotal,
		1770	drm_rect_width(&pstate->dst),
7144	serge	1771	cpp, mem_value);
4104	Serge	1772
		1773	return min(method1, method2);
		1774	}
		1775
		1776	/*
		1777	* For both WM_PIPE and WM_LP.
		1778	* mem_value must be in 0.1us units.
		1779	*/
6084	serge	1780	static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
		1781	const struct intel_plane_state *pstate,
4104	Serge	1782	uint32_t mem_value)
		1783	{
7144	serge	1784	int cpp = pstate->base.fb ?
		1785	drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
4104	Serge	1786	uint32_t method1, method2;
		1787
6084	serge	1788	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1789	return 0;
		1790
7144	serge	1791	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), cpp, mem_value);
6084	serge	1792	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1793	cstate->base.adjusted_mode.crtc_htotal,
		1794	drm_rect_width(&pstate->dst),
7144	serge	1795	cpp, mem_value);
4104	Serge	1796	return min(method1, method2);
		1797	}
		1798
		1799	/*
		1800	* For both WM_PIPE and WM_LP.
		1801	* mem_value must be in 0.1us units.
		1802	*/
6084	serge	1803	static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
		1804	const struct intel_plane_state *pstate,
4104	Serge	1805	uint32_t mem_value)
		1806	{
6660	serge	1807	/*
		1808	* We treat the cursor plane as always-on for the purposes of watermark
		1809	* calculation. Until we have two-stage watermark programming merged,
		1810	* this is necessary to avoid flickering.
		1811	*/
		1812	int cpp = 4;
		1813	int width = pstate->visible ? pstate->base.crtc_w : 64;
6084	serge	1814
6660	serge	1815	if (!cstate->base.active)
4104	Serge	1816	return 0;
		1817
6084	serge	1818	return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1819	cstate->base.adjusted_mode.crtc_htotal,
6660	serge	1820	width, cpp, mem_value);
4104	Serge	1821	}
		1822
		1823	/* Only for WM_LP. */
6084	serge	1824	static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
		1825	const struct intel_plane_state *pstate,
4104	Serge	1826	uint32_t pri_val)
		1827	{
7144	serge	1828	int cpp = pstate->base.fb ?
		1829	drm_format_plane_cpp(pstate->base.fb->pixel_format, 0) : 0;
6084	serge	1830
		1831	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1832	return 0;
		1833
7144	serge	1834	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), cpp);
4104	Serge	1835	}
		1836
		1837	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1838	{
4560	Serge	1839	if (INTEL_INFO(dev)->gen >= 8)
		1840	return 3072;
		1841	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1842	return 768;
		1843	else
		1844	return 512;
		1845	}
		1846
5060	serge	1847	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		1848	int level, bool is_sprite)
		1849	{
		1850	if (INTEL_INFO(dev)->gen >= 8)
		1851	/* BDW primary/sprite plane watermarks */
		1852	return level == 0 ? 255 : 2047;
		1853	else if (INTEL_INFO(dev)->gen >= 7)
		1854	/* IVB/HSW primary/sprite plane watermarks */
		1855	return level == 0 ? 127 : 1023;
		1856	else if (!is_sprite)
		1857	/* ILK/SNB primary plane watermarks */
		1858	return level == 0 ? 127 : 511;
		1859	else
		1860	/* ILK/SNB sprite plane watermarks */
		1861	return level == 0 ? 63 : 255;
		1862	}
		1863
		1864	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		1865	int level)
		1866	{
		1867	if (INTEL_INFO(dev)->gen >= 7)
		1868	return level == 0 ? 63 : 255;
		1869	else
		1870	return level == 0 ? 31 : 63;
		1871	}
		1872
		1873	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		1874	{
		1875	if (INTEL_INFO(dev)->gen >= 8)
		1876	return 31;
		1877	else
		1878	return 15;
		1879	}
		1880
4104	Serge	1881	/* Calculate the maximum primary/sprite plane watermark */
		1882	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1883	int level,
		1884	const struct intel_wm_config *config,
		1885	enum intel_ddb_partitioning ddb_partitioning,
		1886	bool is_sprite)
		1887	{
		1888	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1889
		1890	/* if sprites aren't enabled, sprites get nothing */
		1891	if (is_sprite && !config->sprites_enabled)
		1892	return 0;
		1893
		1894	/* HSW allows LP1+ watermarks even with multiple pipes */
		1895	if (level == 0 \|\| config->num_pipes_active > 1) {
		1896	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1897
		1898	/*
		1899	* For some reason the non self refresh
		1900	* FIFO size is only half of the self
		1901	* refresh FIFO size on ILK/SNB.
		1902	*/
		1903	if (INTEL_INFO(dev)->gen <= 6)
		1904	fifo_size /= 2;
		1905	}
		1906
		1907	if (config->sprites_enabled) {
		1908	/* level 0 is always calculated with 1:1 split */
		1909	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1910	if (is_sprite)
		1911	fifo_size *= 5;
		1912	fifo_size /= 6;
		1913	} else {
		1914	fifo_size /= 2;
		1915	}
		1916	}
		1917
		1918	/* clamp to max that the registers can hold */
5060	serge	1919	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	1920	}
		1921
		1922	/* Calculate the maximum cursor plane watermark */
		1923	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1924	int level,
		1925	const struct intel_wm_config *config)
		1926	{
		1927	/* HSW LP1+ watermarks w/ multiple pipes */
		1928	if (level > 0 && config->num_pipes_active > 1)
		1929	return 64;
		1930
		1931	/* otherwise just report max that registers can hold */
5060	serge	1932	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	1933	}
4104	Serge	1934
5060	serge	1935	static void ilk_compute_wm_maximums(const struct drm_device *dev,
6084	serge	1936	int level,
		1937	const struct intel_wm_config *config,
		1938	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1939	struct ilk_wm_maximums *max)
4104	Serge	1940	{
		1941	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1942	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1943	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	1944	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	1945	}
4104	Serge	1946
5060	serge	1947	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		1948	int level,
		1949	struct ilk_wm_maximums *max)
		1950	{
		1951	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		1952	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		1953	max->cur = ilk_cursor_wm_reg_max(dev, level);
		1954	max->fbc = ilk_fbc_wm_reg_max(dev);
		1955	}
		1956
4560	Serge	1957	static bool ilk_validate_wm_level(int level,
		1958	const struct ilk_wm_maximums *max,
6084	serge	1959	struct intel_wm_level *result)
4104	Serge	1960	{
		1961	bool ret;
		1962
		1963	/* already determined to be invalid? */
		1964	if (!result->enable)
		1965	return false;
		1966
		1967	result->enable = result->pri_val <= max->pri &&
		1968	result->spr_val <= max->spr &&
		1969	result->cur_val <= max->cur;
		1970
		1971	ret = result->enable;
		1972
		1973	/*
		1974	* HACK until we can pre-compute everything,
		1975	* and thus fail gracefully if LP0 watermarks
		1976	* are exceeded...
		1977	*/
		1978	if (level == 0 && !result->enable) {
		1979	if (result->pri_val > max->pri)
		1980	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		1981	level, result->pri_val, max->pri);
		1982	if (result->spr_val > max->spr)
		1983	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		1984	level, result->spr_val, max->spr);
		1985	if (result->cur_val > max->cur)
		1986	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		1987	level, result->cur_val, max->cur);
		1988
		1989	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		1990	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		1991	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		1992	result->enable = true;
		1993	}
		1994
		1995	return ret;
		1996	}
		1997
5060	serge	1998	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6084	serge	1999	const struct intel_crtc *intel_crtc,
4104	Serge	2000	int level,
6084	serge	2001	struct intel_crtc_state *cstate,
6937	serge	2002	struct intel_plane_state *pristate,
		2003	struct intel_plane_state *sprstate,
		2004	struct intel_plane_state *curstate,
4104	Serge	2005	struct intel_wm_level *result)
		2006	{
		2007	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		2008	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		2009	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		2010
		2011	/* WM1+ latency values stored in 0.5us units */
		2012	if (level > 0) {
		2013	pri_latency *= 5;
		2014	spr_latency *= 5;
		2015	cur_latency *= 5;
		2016	}
		2017
6937	serge	2018	result->pri_val = ilk_compute_pri_wm(cstate, pristate,
		2019	pri_latency, level);
		2020	result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
		2021	result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
		2022	result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
4104	Serge	2023	result->enable = true;
		2024	}
		2025
		2026	static uint32_t
7144	serge	2027	hsw_compute_linetime_wm(struct drm_device *dev,
		2028	struct intel_crtc_state *cstate)
4104	Serge	2029	{
3031	serge	2030	struct drm_i915_private *dev_priv = dev->dev_private;
7144	serge	2031	const struct drm_display_mode *adjusted_mode =
		2032	&cstate->base.adjusted_mode;
4104	Serge	2033	u32 linetime, ips_linetime;
3031	serge	2034
7144	serge	2035	if (!cstate->base.active)
4104	Serge	2036	return 0;
7144	serge	2037	if (WARN_ON(adjusted_mode->crtc_clock == 0))
		2038	return 0;
		2039	if (WARN_ON(dev_priv->cdclk_freq == 0))
		2040	return 0;
3031	serge	2041
		2042	/* The WM are computed with base on how long it takes to fill a single
		2043	* row at the given clock rate, multiplied by 8.
		2044	* */
6084	serge	2045	linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2046	adjusted_mode->crtc_clock);
		2047	ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2048	dev_priv->cdclk_freq);
3031	serge	2049
4104	Serge	2050	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2051	PIPE_WM_LINETIME_TIME(linetime);
		2052	}
		2053
5354	serge	2054	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
4104	Serge	2055	{
		2056	struct drm_i915_private *dev_priv = dev->dev_private;
		2057
5354	serge	2058	if (IS_GEN9(dev)) {
		2059	uint32_t val;
		2060	int ret, i;
		2061	int level, max_level = ilk_wm_max_level(dev);
		2062
		2063	/* read the first set of memory latencies[0:3] */
		2064	val = 0; /* data0 to be programmed to 0 for first set */
		2065	mutex_lock(&dev_priv->rps.hw_lock);
		2066	ret = sandybridge_pcode_read(dev_priv,
		2067	GEN9_PCODE_READ_MEM_LATENCY,
		2068	&val);
		2069	mutex_unlock(&dev_priv->rps.hw_lock);
		2070
		2071	if (ret) {
		2072	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2073	return;
		2074	}
		2075
		2076	wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2077	wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2078	GEN9_MEM_LATENCY_LEVEL_MASK;
		2079	wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2080	GEN9_MEM_LATENCY_LEVEL_MASK;
		2081	wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2082	GEN9_MEM_LATENCY_LEVEL_MASK;
		2083
		2084	/* read the second set of memory latencies[4:7] */
		2085	val = 1; /* data0 to be programmed to 1 for second set */
		2086	mutex_lock(&dev_priv->rps.hw_lock);
		2087	ret = sandybridge_pcode_read(dev_priv,
		2088	GEN9_PCODE_READ_MEM_LATENCY,
		2089	&val);
		2090	mutex_unlock(&dev_priv->rps.hw_lock);
		2091	if (ret) {
		2092	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2093	return;
		2094	}
		2095
		2096	wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2097	wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2098	GEN9_MEM_LATENCY_LEVEL_MASK;
		2099	wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2100	GEN9_MEM_LATENCY_LEVEL_MASK;
		2101	wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2102	GEN9_MEM_LATENCY_LEVEL_MASK;
		2103
		2104	/*
6084	serge	2105	* WaWmMemoryReadLatency:skl
		2106	*
5354	serge	2107	* punit doesn't take into account the read latency so we need
6937	serge	2108	* to add 2us to the various latency levels we retrieve from
		2109	* the punit.
		2110	* - W0 is a bit special in that it's the only level that
		2111	* can't be disabled if we want to have display working, so
		2112	* we always add 2us there.
		2113	* - For levels >=1, punit returns 0us latency when they are
		2114	* disabled, so we respect that and don't add 2us then
		2115	*
		2116	* Additionally, if a level n (n > 1) has a 0us latency, all
		2117	* levels m (m >= n) need to be disabled. We make sure to
		2118	* sanitize the values out of the punit to satisfy this
		2119	* requirement.
5354	serge	2120	*/
		2121	wm[0] += 2;
6937	serge	2122	for (level = 1; level <= max_level; level++)
		2123	if (wm[level] != 0)
		2124	wm[level] += 2;
		2125	else {
		2126	for (i = level + 1; i <= max_level; i++)
		2127	wm[i] = 0;
		2128
7144	serge	2129	break;
6935	serge	2130	}
5354	serge	2131	} else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2132	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2133
		2134	wm[0] = (sskpd >> 56) & 0xFF;
		2135	if (wm[0] == 0)
		2136	wm[0] = sskpd & 0xF;
		2137	wm[1] = (sskpd >> 4) & 0xFF;
		2138	wm[2] = (sskpd >> 12) & 0xFF;
		2139	wm[3] = (sskpd >> 20) & 0x1FF;
		2140	wm[4] = (sskpd >> 32) & 0x1FF;
		2141	} else if (INTEL_INFO(dev)->gen >= 6) {
		2142	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2143
		2144	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2145	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2146	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2147	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2148	} else if (INTEL_INFO(dev)->gen >= 5) {
		2149	uint32_t mltr = I915_READ(MLTR_ILK);
		2150
		2151	/* ILK primary LP0 latency is 700 ns */
		2152	wm[0] = 7;
		2153	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2154	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2155	}
		2156	}
		2157
		2158	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2159	{
		2160	/* ILK sprite LP0 latency is 1300 ns */
		2161	if (INTEL_INFO(dev)->gen == 5)
		2162	wm[0] = 13;
		2163	}
		2164
		2165	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2166	{
		2167	/* ILK cursor LP0 latency is 1300 ns */
		2168	if (INTEL_INFO(dev)->gen == 5)
		2169	wm[0] = 13;
		2170
		2171	/* WaDoubleCursorLP3Latency:ivb */
		2172	if (IS_IVYBRIDGE(dev))
		2173	wm[3] *= 2;
		2174	}
		2175
5060	serge	2176	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2177	{
		2178	/* how many WM levels are we expecting */
6084	serge	2179	if (INTEL_INFO(dev)->gen >= 9)
5354	serge	2180	return 7;
		2181	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
4560	Serge	2182	return 4;
		2183	else if (INTEL_INFO(dev)->gen >= 6)
		2184	return 3;
		2185	else
		2186	return 2;
		2187	}
		2188
4104	Serge	2189	static void intel_print_wm_latency(struct drm_device *dev,
		2190	const char *name,
5354	serge	2191	const uint16_t wm[8])
4104	Serge	2192	{
4560	Serge	2193	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2194
		2195	for (level = 0; level <= max_level; level++) {
		2196	unsigned int latency = wm[level];
		2197
		2198	if (latency == 0) {
		2199	DRM_ERROR("%s WM%d latency not provided\n",
		2200	name, level);
		2201	continue;
		2202	}
		2203
5354	serge	2204	/*
		2205	* - latencies are in us on gen9.
		2206	* - before then, WM1+ latency values are in 0.5us units
		2207	*/
		2208	if (IS_GEN9(dev))
		2209	latency *= 10;
		2210	else if (level > 0)
4104	Serge	2211	latency *= 5;
		2212
		2213	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2214	name, level, wm[level],
		2215	latency / 10, latency % 10);
		2216	}
		2217	}
		2218
5060	serge	2219	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2220	uint16_t wm[5], uint16_t min)
4104	Serge	2221	{
5060	serge	2222	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2223
		2224	if (wm[0] >= min)
		2225	return false;
		2226
		2227	wm[0] = max(wm[0], min);
		2228	for (level = 1; level <= max_level; level++)
		2229	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2230
		2231	return true;
		2232	}
		2233
		2234	static void snb_wm_latency_quirk(struct drm_device *dev)
		2235	{
4104	Serge	2236	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2237	bool changed;
4104	Serge	2238
5060	serge	2239	/*
		2240	* The BIOS provided WM memory latency values are often
		2241	* inadequate for high resolution displays. Adjust them.
		2242	*/
		2243	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2244	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2245	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2246
		2247	if (!changed)
		2248	return;
		2249
		2250	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2251	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2252	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2253	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2254	}
		2255
		2256	static void ilk_setup_wm_latency(struct drm_device *dev)
		2257	{
		2258	struct drm_i915_private *dev_priv = dev->dev_private;
		2259
4104	Serge	2260	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2261
		2262	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2263	sizeof(dev_priv->wm.pri_latency));
		2264	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2265	sizeof(dev_priv->wm.pri_latency));
		2266
		2267	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2268	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2269
		2270	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2271	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2272	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2273
		2274	if (IS_GEN6(dev))
		2275	snb_wm_latency_quirk(dev);
4104	Serge	2276	}
		2277
5354	serge	2278	static void skl_setup_wm_latency(struct drm_device *dev)
		2279	{
		2280	struct drm_i915_private *dev_priv = dev->dev_private;
		2281
		2282	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
		2283	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
		2284	}
		2285
4560	Serge	2286	/* Compute new watermarks for the pipe */
6937	serge	2287	static int ilk_compute_pipe_wm(struct intel_crtc *intel_crtc,
		2288	struct drm_atomic_state *state)
4560	Serge	2289	{
6937	serge	2290	struct intel_pipe_wm *pipe_wm;
		2291	struct drm_device *dev = intel_crtc->base.dev;
5060	serge	2292	const struct drm_i915_private *dev_priv = dev->dev_private;
6937	serge	2293	struct intel_crtc_state *cstate = NULL;
6084	serge	2294	struct intel_plane *intel_plane;
6937	serge	2295	struct drm_plane_state *ps;
		2296	struct intel_plane_state *pristate = NULL;
6084	serge	2297	struct intel_plane_state *sprstate = NULL;
6937	serge	2298	struct intel_plane_state *curstate = NULL;
4560	Serge	2299	int level, max_level = ilk_wm_max_level(dev);
		2300	/* LP0 watermark maximums depend on this pipe alone */
		2301	struct intel_wm_config config = {
		2302	.num_pipes_active = 1,
		2303	};
		2304	struct ilk_wm_maximums max;
4104	Serge	2305
6937	serge	2306	cstate = intel_atomic_get_crtc_state(state, intel_crtc);
		2307	if (IS_ERR(cstate))
		2308	return PTR_ERR(cstate);
		2309
		2310	pipe_wm = &cstate->wm.optimal.ilk;
		2311	memset(pipe_wm, 0, sizeof(*pipe_wm));
		2312
6084	serge	2313	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
6937	serge	2314	ps = drm_atomic_get_plane_state(state,
		2315	&intel_plane->base);
		2316	if (IS_ERR(ps))
		2317	return PTR_ERR(ps);
		2318
		2319	if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		2320	pristate = to_intel_plane_state(ps);
		2321	else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY)
		2322	sprstate = to_intel_plane_state(ps);
		2323	else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
		2324	curstate = to_intel_plane_state(ps);
6084	serge	2325	}
4560	Serge	2326
6084	serge	2327	config.sprites_enabled = sprstate->visible;
		2328	config.sprites_scaled = sprstate->visible &&
		2329	(drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 \|\|
		2330	drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
		2331
		2332	pipe_wm->pipe_enabled = cstate->base.active;
6937	serge	2333	pipe_wm->sprites_enabled = config.sprites_enabled;
6084	serge	2334	pipe_wm->sprites_scaled = config.sprites_scaled;
		2335
4560	Serge	2336	/* ILK/SNB: LP2+ watermarks only w/o sprites */
6084	serge	2337	if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
4560	Serge	2338	max_level = 1;
		2339
		2340	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
6084	serge	2341	if (config.sprites_scaled)
4560	Serge	2342	max_level = 0;
		2343
6937	serge	2344	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
		2345	pristate, sprstate, curstate, &pipe_wm->wm[0]);
4560	Serge	2346
		2347	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
7144	serge	2348	pipe_wm->linetime = hsw_compute_linetime_wm(dev, cstate);
4560	Serge	2349
5060	serge	2350	/* LP0 watermarks always use 1/2 DDB partitioning */
		2351	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2352
4560	Serge	2353	/* At least LP0 must be valid */
5060	serge	2354	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
6937	serge	2355	return -EINVAL;
5060	serge	2356
		2357	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2358
		2359	for (level = 1; level <= max_level; level++) {
		2360	struct intel_wm_level wm = {};
		2361
6937	serge	2362	ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
		2363	pristate, sprstate, curstate, &wm);
5060	serge	2364
		2365	/*
		2366	* Disable any watermark level that exceeds the
		2367	* register maximums since such watermarks are
		2368	* always invalid.
		2369	*/
		2370	if (!ilk_validate_wm_level(level, &max, &wm))
		2371	break;
		2372
		2373	pipe_wm->wm[level] = wm;
		2374	}
		2375
6937	serge	2376	return 0;
4104	Serge	2377	}
		2378
4560	Serge	2379	/*
		2380	* Merge the watermarks from all active pipes for a specific level.
		2381	*/
		2382	static void ilk_merge_wm_level(struct drm_device *dev,
		2383	int level,
		2384	struct intel_wm_level *ret_wm)
4104	Serge	2385	{
4560	Serge	2386	const struct intel_crtc *intel_crtc;
4104	Serge	2387
5060	serge	2388	ret_wm->enable = true;
4104	Serge	2389
5060	serge	2390	for_each_intel_crtc(dev, intel_crtc) {
6937	serge	2391	const struct intel_crtc_state *cstate =
		2392	to_intel_crtc_state(intel_crtc->base.state);
		2393	const struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
5060	serge	2394	const struct intel_wm_level *wm = &active->wm[level];
		2395
		2396	if (!active->pipe_enabled)
		2397	continue;
		2398
		2399	/*
		2400	* The watermark values may have been used in the past,
		2401	* so we must maintain them in the registers for some
		2402	* time even if the level is now disabled.
		2403	*/
4560	Serge	2404	if (!wm->enable)
5060	serge	2405	ret_wm->enable = false;
4104	Serge	2406
4560	Serge	2407	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2408	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2409	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2410	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2411	}
		2412	}
		2413
		2414	/*
		2415	* Merge all low power watermarks for all active pipes.
		2416	*/
		2417	static void ilk_wm_merge(struct drm_device *dev,
		2418	const struct intel_wm_config *config,
		2419	const struct ilk_wm_maximums *max,
		2420	struct intel_pipe_wm *merged)
		2421	{
6084	serge	2422	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2423	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2424	int last_enabled_level = max_level;
4560	Serge	2425
		2426	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2427	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2428	config->num_pipes_active > 1)
		2429	return;
		2430
		2431	/* ILK: FBC WM must be disabled always */
		2432	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2433
		2434	/* merge each WM1+ level */
4104	Serge	2435	for (level = 1; level <= max_level; level++) {
4560	Serge	2436	struct intel_wm_level *wm = &merged->wm[level];
		2437
		2438	ilk_merge_wm_level(dev, level, wm);
		2439
5060	serge	2440	if (level > last_enabled_level)
		2441	wm->enable = false;
		2442	else if (!ilk_validate_wm_level(level, max, wm))
		2443	/* make sure all following levels get disabled */
		2444	last_enabled_level = level - 1;
4560	Serge	2445
		2446	/*
		2447	* The spec says it is preferred to disable
		2448	* FBC WMs instead of disabling a WM level.
		2449	*/
		2450	if (wm->fbc_val > max->fbc) {
5060	serge	2451	if (wm->enable)
6084	serge	2452	merged->fbc_wm_enabled = false;
4560	Serge	2453	wm->fbc_val = 0;
4104	Serge	2454	}
		2455	}
		2456
4560	Serge	2457	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2458	/*
		2459	* FIXME this is racy. FBC might get enabled later.
		2460	* What we should check here is whether FBC can be
		2461	* enabled sometime later.
		2462	*/
6084	serge	2463	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
6937	serge	2464	intel_fbc_is_active(dev_priv)) {
4560	Serge	2465	for (level = 2; level <= max_level; level++) {
		2466	struct intel_wm_level *wm = &merged->wm[level];
		2467
		2468	wm->enable = false;
		2469	}
		2470	}
		2471	}
		2472
		2473	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2474	{
		2475	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2476	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2477	}
		2478
		2479	/* The value we need to program into the WM_LPx latency field */
		2480	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2481	{
		2482	struct drm_i915_private *dev_priv = dev->dev_private;
		2483
		2484	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2485	return 2 * level;
		2486	else
		2487	return dev_priv->wm.pri_latency[level];
		2488	}
		2489
		2490	static void ilk_compute_wm_results(struct drm_device *dev,
		2491	const struct intel_pipe_wm *merged,
		2492	enum intel_ddb_partitioning partitioning,
		2493	struct ilk_wm_values *results)
		2494	{
		2495	struct intel_crtc *intel_crtc;
		2496	int level, wm_lp;
		2497
		2498	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2499	results->partitioning = partitioning;
		2500
		2501	/* LP1+ register values */
4104	Serge	2502	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2503	const struct intel_wm_level *r;
		2504
4560	Serge	2505	level = ilk_wm_lp_to_level(wm_lp, merged);
		2506
		2507	r = &merged->wm[level];
4104	Serge	2508
5060	serge	2509	/*
		2510	* Maintain the watermark values even if the level is
		2511	* disabled. Doing otherwise could cause underruns.
		2512	*/
		2513	results->wm_lp[wm_lp - 1] =
4560	Serge	2514	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2515	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2516	r->cur_val;
		2517
5060	serge	2518	if (r->enable)
		2519	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2520
4560	Serge	2521	if (INTEL_INFO(dev)->gen >= 8)
		2522	results->wm_lp[wm_lp - 1] \|=
		2523	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2524	else
		2525	results->wm_lp[wm_lp - 1] \|=
		2526	r->fbc_val << WM1_LP_FBC_SHIFT;
		2527
5060	serge	2528	/*
		2529	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2530	* level is disabled. Doing otherwise could cause underruns.
		2531	*/
4560	Serge	2532	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2533	WARN_ON(wm_lp != 1);
		2534	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2535	} else
6084	serge	2536	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
4104	Serge	2537	}
		2538
4560	Serge	2539	/* LP0 register values */
5060	serge	2540	for_each_intel_crtc(dev, intel_crtc) {
6937	serge	2541	const struct intel_crtc_state *cstate =
		2542	to_intel_crtc_state(intel_crtc->base.state);
4560	Serge	2543	enum pipe pipe = intel_crtc->pipe;
6937	serge	2544	const struct intel_wm_level *r = &cstate->wm.optimal.ilk.wm[0];
4104	Serge	2545
4560	Serge	2546	if (WARN_ON(!r->enable))
		2547	continue;
		2548
6937	serge	2549	results->wm_linetime[pipe] = cstate->wm.optimal.ilk.linetime;
4560	Serge	2550
		2551	results->wm_pipe[pipe] =
		2552	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2553	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2554	r->cur_val;
4104	Serge	2555	}
		2556	}
		2557
		2558	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2559	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2560	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2561	struct intel_pipe_wm *r1,
		2562	struct intel_pipe_wm *r2)
4104	Serge	2563	{
4560	Serge	2564	int level, max_level = ilk_wm_max_level(dev);
		2565	int level1 = 0, level2 = 0;
4104	Serge	2566
4560	Serge	2567	for (level = 1; level <= max_level; level++) {
		2568	if (r1->wm[level].enable)
		2569	level1 = level;
		2570	if (r2->wm[level].enable)
		2571	level2 = level;
4104	Serge	2572	}
		2573
4560	Serge	2574	if (level1 == level2) {
		2575	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2576	return r2;
		2577	else
		2578	return r1;
4560	Serge	2579	} else if (level1 > level2) {
4104	Serge	2580	return r1;
		2581	} else {
		2582	return r2;
		2583	}
		2584	}
		2585
4560	Serge	2586	/* dirty bits used to track which watermarks need changes */
		2587	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2588	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2589	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2590	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2591	#define WM_DIRTY_FBC (1 << 24)
		2592	#define WM_DIRTY_DDB (1 << 25)
		2593
5354	serge	2594	static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
4560	Serge	2595	const struct ilk_wm_values *old,
		2596	const struct ilk_wm_values *new)
		2597	{
		2598	unsigned int dirty = 0;
		2599	enum pipe pipe;
		2600	int wm_lp;
		2601
5354	serge	2602	for_each_pipe(dev_priv, pipe) {
4560	Serge	2603	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2604	dirty \|= WM_DIRTY_LINETIME(pipe);
		2605	/* Must disable LP1+ watermarks too */
		2606	dirty \|= WM_DIRTY_LP_ALL;
		2607	}
		2608
		2609	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2610	dirty \|= WM_DIRTY_PIPE(pipe);
		2611	/* Must disable LP1+ watermarks too */
		2612	dirty \|= WM_DIRTY_LP_ALL;
		2613	}
		2614	}
		2615
		2616	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2617	dirty \|= WM_DIRTY_FBC;
		2618	/* Must disable LP1+ watermarks too */
		2619	dirty \|= WM_DIRTY_LP_ALL;
		2620	}
		2621
		2622	if (old->partitioning != new->partitioning) {
		2623	dirty \|= WM_DIRTY_DDB;
		2624	/* Must disable LP1+ watermarks too */
		2625	dirty \|= WM_DIRTY_LP_ALL;
		2626	}
		2627
		2628	/* LP1+ watermarks already deemed dirty, no need to continue */
		2629	if (dirty & WM_DIRTY_LP_ALL)
		2630	return dirty;
		2631
		2632	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2633	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2634	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2635	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2636	break;
		2637	}
		2638
		2639	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2640	for (; wm_lp <= 3; wm_lp++)
		2641	dirty \|= WM_DIRTY_LP(wm_lp);
		2642
		2643	return dirty;
		2644	}
		2645
		2646	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2647	unsigned int dirty)
		2648	{
		2649	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2650	bool changed = false;
		2651
		2652	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2653	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2654	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2655	changed = true;
		2656	}
		2657	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2658	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2659	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2660	changed = true;
		2661	}
		2662	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2663	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2664	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2665	changed = true;
		2666	}
		2667
		2668	/*
		2669	* Don't touch WM1S_LP_EN here.
		2670	* Doing so could cause underruns.
		2671	*/
		2672
		2673	return changed;
		2674	}
		2675
4104	Serge	2676	/*
		2677	* The spec says we shouldn't write when we don't need, because every write
		2678	* causes WMs to be re-evaluated, expending some power.
6084	serge	2679	*/
4560	Serge	2680	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2681	struct ilk_wm_values *results)
4104	Serge	2682	{
4560	Serge	2683	struct drm_device *dev = dev_priv->dev;
		2684	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2685	unsigned int dirty;
4104	Serge	2686	uint32_t val;
3031	serge	2687
5354	serge	2688	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
4560	Serge	2689	if (!dirty)
4104	Serge	2690	return;
		2691
4560	Serge	2692	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2693
4560	Serge	2694	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2695	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2696	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2697	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2698	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2699	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2700
4560	Serge	2701	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2702	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2703	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2704	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2705	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2706	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2707
4560	Serge	2708	if (dirty & WM_DIRTY_DDB) {
		2709	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
6084	serge	2710	val = I915_READ(WM_MISC);
4560	Serge	2711	if (results->partitioning == INTEL_DDB_PART_1_2)
6084	serge	2712	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2713	else
		2714	val \|= WM_MISC_DATA_PARTITION_5_6;
		2715	I915_WRITE(WM_MISC, val);
4560	Serge	2716	} else {
		2717	val = I915_READ(DISP_ARB_CTL2);
		2718	if (results->partitioning == INTEL_DDB_PART_1_2)
		2719	val &= ~DISP_DATA_PARTITION_5_6;
		2720	else
		2721	val \|= DISP_DATA_PARTITION_5_6;
		2722	I915_WRITE(DISP_ARB_CTL2, val);
		2723	}
4104	Serge	2724	}
		2725
4560	Serge	2726	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2727	val = I915_READ(DISP_ARB_CTL);
		2728	if (results->enable_fbc_wm)
		2729	val &= ~DISP_FBC_WM_DIS;
		2730	else
		2731	val \|= DISP_FBC_WM_DIS;
		2732	I915_WRITE(DISP_ARB_CTL, val);
		2733	}
		2734
4560	Serge	2735	if (dirty & WM_DIRTY_LP(1) &&
		2736	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2737	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2738
		2739	if (INTEL_INFO(dev)->gen >= 7) {
		2740	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
6084	serge	2741	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2742	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
6084	serge	2743	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2744	}
4104	Serge	2745
4560	Serge	2746	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2747	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2748	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2749	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2750	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2751	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2752
		2753	dev_priv->wm.hw = *results;
3031	serge	2754	}
		2755
4560	Serge	2756	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2757	{
		2758	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2759
		2760	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2761	}
		2762
5354	serge	2763	/*
		2764	* On gen9, we need to allocate Display Data Buffer (DDB) portions to the
		2765	* different active planes.
		2766	*/
		2767
		2768	#define SKL_DDB_SIZE 896 /* in blocks */
6084	serge	2769	#define BXT_DDB_SIZE 512
5354	serge	2770
6937	serge	2771	/*
		2772	* Return the index of a plane in the SKL DDB and wm result arrays. Primary
		2773	* plane is always in slot 0, cursor is always in slot I915_MAX_PLANES-1, and
		2774	* other universal planes are in indices 1..n. Note that this may leave unused
		2775	* indices between the top "sprite" plane and the cursor.
		2776	*/
		2777	static int
		2778	skl_wm_plane_id(const struct intel_plane *plane)
		2779	{
		2780	switch (plane->base.type) {
		2781	case DRM_PLANE_TYPE_PRIMARY:
		2782	return 0;
		2783	case DRM_PLANE_TYPE_CURSOR:
		2784	return PLANE_CURSOR;
		2785	case DRM_PLANE_TYPE_OVERLAY:
		2786	return plane->plane + 1;
		2787	default:
		2788	MISSING_CASE(plane->base.type);
		2789	return plane->plane;
		2790	}
		2791	}
		2792
5354	serge	2793	static void
		2794	skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
6937	serge	2795	const struct intel_crtc_state *cstate,
5354	serge	2796	const struct intel_wm_config *config,
		2797	struct skl_ddb_entry alloc / out */)
		2798	{
6937	serge	2799	struct drm_crtc *for_crtc = cstate->base.crtc;
5354	serge	2800	struct drm_crtc *crtc;
		2801	unsigned int pipe_size, ddb_size;
		2802	int nth_active_pipe;
		2803
6937	serge	2804	if (!cstate->base.active) {
5354	serge	2805	alloc->start = 0;
		2806	alloc->end = 0;
		2807	return;
		2808	}
		2809
6084	serge	2810	if (IS_BROXTON(dev))
		2811	ddb_size = BXT_DDB_SIZE;
		2812	else
		2813	ddb_size = SKL_DDB_SIZE;
5354	serge	2814
		2815	ddb_size -= 4; /* 4 blocks for bypass path allocation */
		2816
		2817	nth_active_pipe = 0;
		2818	for_each_crtc(dev, crtc) {
6084	serge	2819	if (!to_intel_crtc(crtc)->active)
5354	serge	2820	continue;
		2821
		2822	if (crtc == for_crtc)
		2823	break;
		2824
		2825	nth_active_pipe++;
		2826	}
		2827
		2828	pipe_size = ddb_size / config->num_pipes_active;
		2829	alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
		2830	alloc->end = alloc->start + pipe_size;
		2831	}
		2832
		2833	static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
		2834	{
		2835	if (config->num_pipes_active == 1)
		2836	return 32;
		2837
		2838	return 8;
		2839	}
		2840
		2841	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
		2842	{
		2843	entry->start = reg & 0x3ff;
		2844	entry->end = (reg >> 16) & 0x3ff;
		2845	if (entry->end)
		2846	entry->end += 1;
		2847	}
		2848
		2849	void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
		2850	struct skl_ddb_allocation ddb / out */)
		2851	{
		2852	enum pipe pipe;
		2853	int plane;
		2854	u32 val;
		2855
6084	serge	2856	memset(ddb, 0, sizeof(*ddb));
		2857
5354	serge	2858	for_each_pipe(dev_priv, pipe) {
6937	serge	2859	enum intel_display_power_domain power_domain;
		2860
		2861	power_domain = POWER_DOMAIN_PIPE(pipe);
		2862	if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
6084	serge	2863	continue;
		2864
		2865	for_each_plane(dev_priv, pipe, plane) {
5354	serge	2866	val = I915_READ(PLANE_BUF_CFG(pipe, plane));
		2867	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
		2868	val);
		2869	}
		2870
		2871	val = I915_READ(CUR_BUF_CFG(pipe));
6084	serge	2872	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
		2873	val);
6937	serge	2874
		2875	intel_display_power_put(dev_priv, power_domain);
5354	serge	2876	}
		2877	}
		2878
		2879	static unsigned int
6937	serge	2880	skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
		2881	const struct drm_plane_state *pstate,
		2882	int y)
5354	serge	2883	{
7144	serge	2884	struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
6937	serge	2885	struct drm_framebuffer *fb = pstate->fb;
7144	serge	2886	uint32_t width = 0, height = 0;
6084	serge	2887
7144	serge	2888	width = drm_rect_width(&intel_pstate->src) >> 16;
		2889	height = drm_rect_height(&intel_pstate->src) >> 16;
		2890
		2891	if (intel_rotation_90_or_270(pstate->rotation))
		2892	swap(width, height);
		2893
6084	serge	2894	/* for planar format */
6937	serge	2895	if (fb->pixel_format == DRM_FORMAT_NV12) {
6084	serge	2896	if (y) /* y-plane data rate */
7144	serge	2897	return width * height *
6937	serge	2898	drm_format_plane_cpp(fb->pixel_format, 0);
6084	serge	2899	else /* uv-plane data rate */
7144	serge	2900	return (width / 2) * (height / 2) *
6937	serge	2901	drm_format_plane_cpp(fb->pixel_format, 1);
6084	serge	2902	}
		2903
		2904	/* for packed formats */
7144	serge	2905	return width * height * drm_format_plane_cpp(fb->pixel_format, 0);
5354	serge	2906	}
		2907
		2908	/*
		2909	* We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
		2910	* a 8192x4096@32bpp framebuffer:
		2911	* 3 * 4096 * 8192 * 4 < 2^32
		2912	*/
		2913	static unsigned int
6937	serge	2914	skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate)
5354	serge	2915	{
6937	serge	2916	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
		2917	struct drm_device *dev = intel_crtc->base.dev;
		2918	const struct intel_plane *intel_plane;
5354	serge	2919	unsigned int total_data_rate = 0;
		2920
6937	serge	2921	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2922	const struct drm_plane_state *pstate = intel_plane->base.state;
5354	serge	2923
6937	serge	2924	if (pstate->fb == NULL)
5354	serge	2925	continue;
		2926
6937	serge	2927	if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
		2928	continue;
		2929
		2930	/* packed/uv */
		2931	total_data_rate += skl_plane_relative_data_rate(cstate,
		2932	pstate,
		2933	0);
		2934
		2935	if (pstate->fb->pixel_format == DRM_FORMAT_NV12)
		2936	/* y-plane */
		2937	total_data_rate += skl_plane_relative_data_rate(cstate,
		2938	pstate,
		2939	1);
5354	serge	2940	}
		2941
		2942	return total_data_rate;
		2943	}
		2944
		2945	static void
6937	serge	2946	skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
5354	serge	2947	struct skl_ddb_allocation ddb / out */)
		2948	{
6937	serge	2949	struct drm_crtc *crtc = cstate->base.crtc;
5354	serge	2950	struct drm_device *dev = crtc->dev;
6937	serge	2951	struct drm_i915_private *dev_priv = to_i915(dev);
		2952	struct intel_wm_config *config = &dev_priv->wm.config;
5354	serge	2953	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	2954	struct intel_plane *intel_plane;
5354	serge	2955	enum pipe pipe = intel_crtc->pipe;
		2956	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
		2957	uint16_t alloc_size, start, cursor_blocks;
6084	serge	2958	uint16_t minimum[I915_MAX_PLANES];
		2959	uint16_t y_minimum[I915_MAX_PLANES];
5354	serge	2960	unsigned int total_data_rate;
		2961
6937	serge	2962	skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc);
5354	serge	2963	alloc_size = skl_ddb_entry_size(alloc);
		2964	if (alloc_size == 0) {
		2965	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
6084	serge	2966	memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
		2967	sizeof(ddb->plane[pipe][PLANE_CURSOR]));
5354	serge	2968	return;
		2969	}
		2970
		2971	cursor_blocks = skl_cursor_allocation(config);
6084	serge	2972	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
		2973	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
5354	serge	2974
		2975	alloc_size -= cursor_blocks;
		2976	alloc->end -= cursor_blocks;
		2977
6084	serge	2978	/* 1. Allocate the mininum required blocks for each active plane */
6937	serge	2979	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2980	struct drm_plane *plane = &intel_plane->base;
		2981	struct drm_framebuffer *fb = plane->state->fb;
		2982	int id = skl_wm_plane_id(intel_plane);
6084	serge	2983
7144	serge	2984	if (!to_intel_plane_state(plane->state)->visible)
6084	serge	2985	continue;
7144	serge	2986
6937	serge	2987	if (plane->type == DRM_PLANE_TYPE_CURSOR)
		2988	continue;
6084	serge	2989
6937	serge	2990	minimum[id] = 8;
		2991	alloc_size -= minimum[id];
		2992	y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0;
		2993	alloc_size -= y_minimum[id];
6084	serge	2994	}
		2995
		2996	/*
		2997	* 2. Distribute the remaining space in proportion to the amount of
		2998	* data each plane needs to fetch from memory.
5354	serge	2999	*
		3000	* FIXME: we may not allocate every single block here.
6084	serge	3001	*/
6937	serge	3002	total_data_rate = skl_get_total_relative_data_rate(cstate);
5354	serge	3003
		3004	start = alloc->start;
6937	serge	3005	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		3006	struct drm_plane *plane = &intel_plane->base;
		3007	struct drm_plane_state *pstate = intel_plane->base.state;
6084	serge	3008	unsigned int data_rate, y_data_rate;
		3009	uint16_t plane_blocks, y_plane_blocks = 0;
6937	serge	3010	int id = skl_wm_plane_id(intel_plane);
5354	serge	3011
7144	serge	3012	if (!to_intel_plane_state(pstate)->visible)
5354	serge	3013	continue;
6937	serge	3014	if (plane->type == DRM_PLANE_TYPE_CURSOR)
		3015	continue;
5354	serge	3016
6937	serge	3017	data_rate = skl_plane_relative_data_rate(cstate, pstate, 0);
5354	serge	3018
		3019	/*
6084	serge	3020	* allocation for (packed formats) or (uv-plane part of planar format):
5354	serge	3021	* promote the expression to 64 bits to avoid overflowing, the
		3022	* result is < available as data_rate / total_data_rate < 1
		3023	*/
6937	serge	3024	plane_blocks = minimum[id];
6084	serge	3025	plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
		3026	total_data_rate);
5354	serge	3027
6937	serge	3028	ddb->plane[pipe][id].start = start;
		3029	ddb->plane[pipe][id].end = start + plane_blocks;
5354	serge	3030
		3031	start += plane_blocks;
6084	serge	3032
		3033	/*
		3034	* allocation for y_plane part of planar format:
		3035	*/
6937	serge	3036	if (pstate->fb->pixel_format == DRM_FORMAT_NV12) {
		3037	y_data_rate = skl_plane_relative_data_rate(cstate,
		3038	pstate,
		3039	1);
		3040	y_plane_blocks = y_minimum[id];
6084	serge	3041	y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
		3042	total_data_rate);
		3043
6937	serge	3044	ddb->y_plane[pipe][id].start = start;
		3045	ddb->y_plane[pipe][id].end = start + y_plane_blocks;
6084	serge	3046
		3047	start += y_plane_blocks;
		3048	}
		3049
5354	serge	3050	}
		3051
		3052	}
		3053
6084	serge	3054	static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
5354	serge	3055	{
		3056	/* TODO: Take into account the scalers once we support them */
6084	serge	3057	return config->base.adjusted_mode.crtc_clock;
5354	serge	3058	}
		3059
		3060	/*
		3061	* The max latency should be 257 (max the punit can code is 255 and we add 2us
7144	serge	3062	* for the read latency) and cpp should always be <= 8, so that
5354	serge	3063	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
		3064	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
		3065	*/
7144	serge	3066	static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp, uint32_t latency)
5354	serge	3067	{
		3068	uint32_t wm_intermediate_val, ret;
		3069
		3070	if (latency == 0)
		3071	return UINT_MAX;
		3072
7144	serge	3073	wm_intermediate_val = latency * pixel_rate * cpp / 512;
5354	serge	3074	ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
		3075
		3076	return ret;
		3077	}
		3078
		3079	static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
7144	serge	3080	uint32_t horiz_pixels, uint8_t cpp,
6084	serge	3081	uint64_t tiling, uint32_t latency)
5354	serge	3082	{
6084	serge	3083	uint32_t ret;
		3084	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3085	uint32_t wm_intermediate_val;
5354	serge	3086
		3087	if (latency == 0)
		3088	return UINT_MAX;
		3089
7144	serge	3090	plane_bytes_per_line = horiz_pixels * cpp;
6084	serge	3091
		3092	if (tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3093	tiling == I915_FORMAT_MOD_Yf_TILED) {
		3094	plane_bytes_per_line *= 4;
		3095	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3096	plane_blocks_per_line /= 4;
		3097	} else {
		3098	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3099	}
		3100
5354	serge	3101	wm_intermediate_val = latency * pixel_rate;
		3102	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
6084	serge	3103	plane_blocks_per_line;
5354	serge	3104
		3105	return ret;
		3106	}
		3107
		3108	static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
		3109	const struct intel_crtc *intel_crtc)
		3110	{
		3111	struct drm_device *dev = intel_crtc->base.dev;
		3112	struct drm_i915_private *dev_priv = dev->dev_private;
		3113	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3114
6937	serge	3115	/*
		3116	* If ddb allocation of pipes changed, it may require recalculation of
		3117	* watermarks
		3118	*/
		3119	if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe)))
5354	serge	3120	return true;
		3121
		3122	return false;
		3123	}
		3124
6084	serge	3125	static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
6937	serge	3126	struct intel_crtc_state *cstate,
		3127	struct intel_plane *intel_plane,
5354	serge	3128	uint16_t ddb_allocation,
6084	serge	3129	int level,
5354	serge	3130	uint16_t out_blocks, / out */
		3131	uint8_t out_lines / out */)
		3132	{
6937	serge	3133	struct drm_plane *plane = &intel_plane->base;
		3134	struct drm_framebuffer *fb = plane->state->fb;
7144	serge	3135	struct intel_plane_state *intel_pstate =
		3136	to_intel_plane_state(plane->state);
6084	serge	3137	uint32_t latency = dev_priv->wm.skl_latency[level];
		3138	uint32_t method1, method2;
		3139	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3140	uint32_t res_blocks, res_lines;
		3141	uint32_t selected_result;
7144	serge	3142	uint8_t cpp;
		3143	uint32_t width = 0, height = 0;
5354	serge	3144
7144	serge	3145	if (latency == 0 \|\| !cstate->base.active \|\| !intel_pstate->visible)
5354	serge	3146	return false;
		3147
7144	serge	3148	width = drm_rect_width(&intel_pstate->src) >> 16;
		3149	height = drm_rect_height(&intel_pstate->src) >> 16;
		3150
		3151	if (intel_rotation_90_or_270(plane->state->rotation))
		3152	swap(width, height);
		3153
		3154	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
6937	serge	3155	method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate),
7144	serge	3156	cpp, latency);
6937	serge	3157	method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate),
		3158	cstate->base.adjusted_mode.crtc_htotal,
7144	serge	3159	width,
		3160	cpp,
6937	serge	3161	fb->modifier[0],
6084	serge	3162	latency);
5354	serge	3163
7144	serge	3164	plane_bytes_per_line = width * cpp;
6084	serge	3165	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
5354	serge	3166
6937	serge	3167	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED \|\|
		3168	fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
6084	serge	3169	uint32_t min_scanlines = 4;
		3170	uint32_t y_tile_minimum;
6937	serge	3171	if (intel_rotation_90_or_270(plane->state->rotation)) {
7144	serge	3172	int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
6937	serge	3173	drm_format_plane_cpp(fb->pixel_format, 1) :
		3174	drm_format_plane_cpp(fb->pixel_format, 0);
		3175
7144	serge	3176	switch (cpp) {
6084	serge	3177	case 1:
		3178	min_scanlines = 16;
		3179	break;
		3180	case 2:
		3181	min_scanlines = 8;
		3182	break;
		3183	case 8:
		3184	WARN(1, "Unsupported pixel depth for rotation");
		3185	}
		3186	}
		3187	y_tile_minimum = plane_blocks_per_line * min_scanlines;
		3188	selected_result = max(method2, y_tile_minimum);
		3189	} else {
		3190	if ((ddb_allocation / plane_blocks_per_line) >= 1)
		3191	selected_result = min(method1, method2);
		3192	else
		3193	selected_result = method1;
		3194	}
5354	serge	3195
6084	serge	3196	res_blocks = selected_result + 1;
		3197	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
5354	serge	3198
6084	serge	3199	if (level >= 1 && level <= 7) {
6937	serge	3200	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED \|\|
		3201	fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)
6084	serge	3202	res_lines += 4;
		3203	else
		3204	res_blocks++;
		3205	}
		3206
		3207	if (res_blocks >= ddb_allocation \|\| res_lines > 31)
5354	serge	3208	return false;
		3209
		3210	*out_blocks = res_blocks;
		3211	*out_lines = res_lines;
		3212
		3213	return true;
		3214	}
		3215
		3216	static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
		3217	struct skl_ddb_allocation *ddb,
6937	serge	3218	struct intel_crtc_state *cstate,
5354	serge	3219	int level,
		3220	struct skl_wm_level *result)
		3221	{
6937	serge	3222	struct drm_device *dev = dev_priv->dev;
		3223	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
		3224	struct intel_plane *intel_plane;
5354	serge	3225	uint16_t ddb_blocks;
6937	serge	3226	enum pipe pipe = intel_crtc->pipe;
5354	serge	3227
6937	serge	3228	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		3229	int i = skl_wm_plane_id(intel_plane);
		3230
5354	serge	3231	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
		3232
6084	serge	3233	result->plane_en[i] = skl_compute_plane_wm(dev_priv,
6937	serge	3234	cstate,
		3235	intel_plane,
5354	serge	3236	ddb_blocks,
6084	serge	3237	level,
5354	serge	3238	&result->plane_res_b[i],
		3239	&result->plane_res_l[i]);
		3240	}
		3241	}
		3242
		3243	static uint32_t
6937	serge	3244	skl_compute_linetime_wm(struct intel_crtc_state *cstate)
5354	serge	3245	{
6937	serge	3246	if (!cstate->base.active)
5354	serge	3247	return 0;
		3248
6937	serge	3249	if (WARN_ON(skl_pipe_pixel_rate(cstate) == 0))
6084	serge	3250	return 0;
		3251
6937	serge	3252	return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000,
		3253	skl_pipe_pixel_rate(cstate));
5354	serge	3254	}
		3255
6937	serge	3256	static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
5354	serge	3257	struct skl_wm_level trans_wm / out */)
		3258	{
6937	serge	3259	struct drm_crtc *crtc = cstate->base.crtc;
5354	serge	3260	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3261	struct intel_plane *intel_plane;
5354	serge	3262
6937	serge	3263	if (!cstate->base.active)
5354	serge	3264	return;
		3265
		3266	/* Until we know more, just disable transition WMs */
6937	serge	3267	for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) {
		3268	int i = skl_wm_plane_id(intel_plane);
		3269
5354	serge	3270	trans_wm->plane_en[i] = false;
6937	serge	3271	}
5354	serge	3272	}
		3273
6937	serge	3274	static void skl_compute_pipe_wm(struct intel_crtc_state *cstate,
5354	serge	3275	struct skl_ddb_allocation *ddb,
		3276	struct skl_pipe_wm *pipe_wm)
		3277	{
6937	serge	3278	struct drm_device *dev = cstate->base.crtc->dev;
5354	serge	3279	const struct drm_i915_private *dev_priv = dev->dev_private;
		3280	int level, max_level = ilk_wm_max_level(dev);
		3281
		3282	for (level = 0; level <= max_level; level++) {
6937	serge	3283	skl_compute_wm_level(dev_priv, ddb, cstate,
		3284	level, &pipe_wm->wm[level]);
5354	serge	3285	}
6937	serge	3286	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
5354	serge	3287
6937	serge	3288	skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
5354	serge	3289	}
		3290
		3291	static void skl_compute_wm_results(struct drm_device *dev,
		3292	struct skl_pipe_wm *p_wm,
		3293	struct skl_wm_values *r,
		3294	struct intel_crtc *intel_crtc)
		3295	{
		3296	int level, max_level = ilk_wm_max_level(dev);
		3297	enum pipe pipe = intel_crtc->pipe;
		3298	uint32_t temp;
		3299	int i;
		3300
		3301	for (level = 0; level <= max_level; level++) {
		3302	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3303	temp = 0;
		3304
		3305	temp \|= p_wm->wm[level].plane_res_l[i] <<
		3306	PLANE_WM_LINES_SHIFT;
		3307	temp \|= p_wm->wm[level].plane_res_b[i];
		3308	if (p_wm->wm[level].plane_en[i])
		3309	temp \|= PLANE_WM_EN;
		3310
		3311	r->plane[pipe][i][level] = temp;
		3312	}
		3313
		3314	temp = 0;
		3315
6084	serge	3316	temp \|= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3317	temp \|= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
5354	serge	3318
6084	serge	3319	if (p_wm->wm[level].plane_en[PLANE_CURSOR])
5354	serge	3320	temp \|= PLANE_WM_EN;
		3321
6084	serge	3322	r->plane[pipe][PLANE_CURSOR][level] = temp;
5354	serge	3323
		3324	}
		3325
		3326	/* transition WMs */
		3327	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3328	temp = 0;
		3329	temp \|= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
		3330	temp \|= p_wm->trans_wm.plane_res_b[i];
		3331	if (p_wm->trans_wm.plane_en[i])
		3332	temp \|= PLANE_WM_EN;
		3333
		3334	r->plane_trans[pipe][i] = temp;
		3335	}
		3336
		3337	temp = 0;
6084	serge	3338	temp \|= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3339	temp \|= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
		3340	if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
5354	serge	3341	temp \|= PLANE_WM_EN;
		3342
6084	serge	3343	r->plane_trans[pipe][PLANE_CURSOR] = temp;
5354	serge	3344
		3345	r->wm_linetime[pipe] = p_wm->linetime;
		3346	}
		3347
6937	serge	3348	static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
		3349	i915_reg_t reg,
5354	serge	3350	const struct skl_ddb_entry *entry)
		3351	{
		3352	if (entry->end)
		3353	I915_WRITE(reg, (entry->end - 1) << 16 \| entry->start);
		3354	else
		3355	I915_WRITE(reg, 0);
		3356	}
		3357
		3358	static void skl_write_wm_values(struct drm_i915_private *dev_priv,
		3359	const struct skl_wm_values *new)
		3360	{
		3361	struct drm_device *dev = dev_priv->dev;
		3362	struct intel_crtc *crtc;
		3363
6937	serge	3364	for_each_intel_crtc(dev, crtc) {
5354	serge	3365	int i, level, max_level = ilk_wm_max_level(dev);
		3366	enum pipe pipe = crtc->pipe;
		3367
		3368	if (!new->dirty[pipe])
		3369	continue;
		3370
		3371	I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
		3372
		3373	for (level = 0; level <= max_level; level++) {
		3374	for (i = 0; i < intel_num_planes(crtc); i++)
		3375	I915_WRITE(PLANE_WM(pipe, i, level),
		3376	new->plane[pipe][i][level]);
		3377	I915_WRITE(CUR_WM(pipe, level),
6084	serge	3378	new->plane[pipe][PLANE_CURSOR][level]);
5354	serge	3379	}
		3380	for (i = 0; i < intel_num_planes(crtc); i++)
		3381	I915_WRITE(PLANE_WM_TRANS(pipe, i),
		3382	new->plane_trans[pipe][i]);
6084	serge	3383	I915_WRITE(CUR_WM_TRANS(pipe),
		3384	new->plane_trans[pipe][PLANE_CURSOR]);
5354	serge	3385
6084	serge	3386	for (i = 0; i < intel_num_planes(crtc); i++) {
5354	serge	3387	skl_ddb_entry_write(dev_priv,
		3388	PLANE_BUF_CFG(pipe, i),
		3389	&new->ddb.plane[pipe][i]);
6084	serge	3390	skl_ddb_entry_write(dev_priv,
		3391	PLANE_NV12_BUF_CFG(pipe, i),
		3392	&new->ddb.y_plane[pipe][i]);
		3393	}
5354	serge	3394
		3395	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
6084	serge	3396	&new->ddb.plane[pipe][PLANE_CURSOR]);
5354	serge	3397	}
		3398	}
		3399
		3400	/*
		3401	* When setting up a new DDB allocation arrangement, we need to correctly
		3402	* sequence the times at which the new allocations for the pipes are taken into
		3403	* account or we'll have pipes fetching from space previously allocated to
		3404	* another pipe.
		3405	*
		3406	* Roughly the sequence looks like:
		3407	* 1. re-allocate the pipe(s) with the allocation being reduced and not
		3408	* overlapping with a previous light-up pipe (another way to put it is:
		3409	* pipes with their new allocation strickly included into their old ones).
		3410	* 2. re-allocate the other pipes that get their allocation reduced
		3411	* 3. allocate the pipes having their allocation increased
		3412	*
		3413	* Steps 1. and 2. are here to take care of the following case:
		3414	* - Initially DDB looks like this:
		3415	* \| B \| C \|
		3416	* - enable pipe A.
		3417	* - pipe B has a reduced DDB allocation that overlaps with the old pipe C
		3418	* allocation
		3419	* \| A \| B \| C \|
		3420	*
		3421	* We need to sequence the re-allocation: C, B, A (and not B, C, A).
		3422	*/
		3423
		3424	static void
		3425	skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
		3426	{
		3427	int plane;
		3428
		3429	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
		3430
6084	serge	3431	for_each_plane(dev_priv, pipe, plane) {
5354	serge	3432	I915_WRITE(PLANE_SURF(pipe, plane),
		3433	I915_READ(PLANE_SURF(pipe, plane)));
		3434	}
		3435	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
		3436	}
		3437
		3438	static bool
		3439	skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
		3440	const struct skl_ddb_allocation *new,
		3441	enum pipe pipe)
		3442	{
		3443	uint16_t old_size, new_size;
		3444
		3445	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
		3446	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
		3447
		3448	return old_size != new_size &&
		3449	new->pipe[pipe].start >= old->pipe[pipe].start &&
		3450	new->pipe[pipe].end <= old->pipe[pipe].end;
		3451	}
		3452
		3453	static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
		3454	struct skl_wm_values *new_values)
		3455	{
		3456	struct drm_device *dev = dev_priv->dev;
		3457	struct skl_ddb_allocation cur_ddb, new_ddb;
6084	serge	3458	bool reallocated[I915_MAX_PIPES] = {};
5354	serge	3459	struct intel_crtc *crtc;
		3460	enum pipe pipe;
		3461
		3462	new_ddb = &new_values->ddb;
		3463	cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3464
		3465	/*
		3466	* First pass: flush the pipes with the new allocation contained into
		3467	* the old space.
		3468	*
		3469	* We'll wait for the vblank on those pipes to ensure we can safely
		3470	* re-allocate the freed space without this pipe fetching from it.
		3471	*/
		3472	for_each_intel_crtc(dev, crtc) {
		3473	if (!crtc->active)
		3474	continue;
		3475
		3476	pipe = crtc->pipe;
		3477
		3478	if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
		3479	continue;
		3480
		3481	skl_wm_flush_pipe(dev_priv, pipe, 1);
		3482	intel_wait_for_vblank(dev, pipe);
		3483
		3484	reallocated[pipe] = true;
		3485	}
		3486
		3487
		3488	/*
		3489	* Second pass: flush the pipes that are having their allocation
		3490	* reduced, but overlapping with a previous allocation.
		3491	*
		3492	* Here as well we need to wait for the vblank to make sure the freed
		3493	* space is not used anymore.
		3494	*/
		3495	for_each_intel_crtc(dev, crtc) {
		3496	if (!crtc->active)
		3497	continue;
		3498
		3499	pipe = crtc->pipe;
		3500
		3501	if (reallocated[pipe])
		3502	continue;
		3503
		3504	if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
		3505	skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
		3506	skl_wm_flush_pipe(dev_priv, pipe, 2);
		3507	intel_wait_for_vblank(dev, pipe);
6084	serge	3508	reallocated[pipe] = true;
5354	serge	3509	}
		3510	}
		3511
		3512	/*
		3513	* Third pass: flush the pipes that got more space allocated.
		3514	*
		3515	* We don't need to actively wait for the update here, next vblank
		3516	* will just get more DDB space with the correct WM values.
		3517	*/
		3518	for_each_intel_crtc(dev, crtc) {
		3519	if (!crtc->active)
		3520	continue;
		3521
		3522	pipe = crtc->pipe;
		3523
		3524	/*
		3525	* At this point, only the pipes more space than before are
		3526	* left to re-allocate.
		3527	*/
		3528	if (reallocated[pipe])
		3529	continue;
		3530
		3531	skl_wm_flush_pipe(dev_priv, pipe, 3);
		3532	}
		3533	}
		3534
		3535	static bool skl_update_pipe_wm(struct drm_crtc *crtc,
		3536	struct skl_ddb_allocation ddb, / out */
		3537	struct skl_pipe_wm pipe_wm / out */)
		3538	{
		3539	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3540	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5354	serge	3541
6937	serge	3542	skl_allocate_pipe_ddb(cstate, ddb);
		3543	skl_compute_pipe_wm(cstate, ddb, pipe_wm);
5354	serge	3544
6937	serge	3545	if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
5354	serge	3546	return false;
		3547
6937	serge	3548	intel_crtc->wm.active.skl = *pipe_wm;
6084	serge	3549
5354	serge	3550	return true;
		3551	}
		3552
		3553	static void skl_update_other_pipe_wm(struct drm_device *dev,
		3554	struct drm_crtc *crtc,
		3555	struct skl_wm_values *r)
		3556	{
		3557	struct intel_crtc *intel_crtc;
		3558	struct intel_crtc *this_crtc = to_intel_crtc(crtc);
		3559
		3560	/*
		3561	* If the WM update hasn't changed the allocation for this_crtc (the
		3562	* crtc we are currently computing the new WM values for), other
		3563	* enabled crtcs will keep the same allocation and we don't need to
		3564	* recompute anything for them.
		3565	*/
		3566	if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
		3567	return;
		3568
		3569	/*
		3570	* Otherwise, because of this_crtc being freshly enabled/disabled, the
		3571	* other active pipes need new DDB allocation and WM values.
		3572	*/
6937	serge	3573	for_each_intel_crtc(dev, intel_crtc) {
5354	serge	3574	struct skl_pipe_wm pipe_wm = {};
		3575	bool wm_changed;
		3576
		3577	if (this_crtc->pipe == intel_crtc->pipe)
		3578	continue;
		3579
		3580	if (!intel_crtc->active)
		3581	continue;
		3582
		3583	wm_changed = skl_update_pipe_wm(&intel_crtc->base,
		3584	&r->ddb, &pipe_wm);
		3585
		3586	/*
		3587	* If we end up re-computing the other pipe WM values, it's
		3588	* because it was really needed, so we expect the WM values to
		3589	* be different.
6084	serge	3590	*/
5354	serge	3591	WARN_ON(!wm_changed);
		3592
6937	serge	3593	skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc);
5354	serge	3594	r->dirty[intel_crtc->pipe] = true;
		3595	}
		3596	}
		3597
6084	serge	3598	static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
		3599	{
		3600	watermarks->wm_linetime[pipe] = 0;
		3601	memset(watermarks->plane[pipe], 0,
		3602	sizeof(uint32_t) * 8 * I915_MAX_PLANES);
		3603	memset(watermarks->plane_trans[pipe],
		3604	0, sizeof(uint32_t) * I915_MAX_PLANES);
		3605	watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
		3606
		3607	/* Clear ddb entries for pipe */
		3608	memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
		3609	memset(&watermarks->ddb.plane[pipe], 0,
		3610	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3611	memset(&watermarks->ddb.y_plane[pipe], 0,
		3612	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3613	memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
		3614	sizeof(struct skl_ddb_entry));
		3615
		3616	}
		3617
5354	serge	3618	static void skl_update_wm(struct drm_crtc *crtc)
		3619	{
		3620	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3621	struct drm_device *dev = crtc->dev;
		3622	struct drm_i915_private *dev_priv = dev->dev_private;
		3623	struct skl_wm_values *results = &dev_priv->wm.skl_results;
6937	serge	3624	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
		3625	struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl;
5354	serge	3626
		3627
6084	serge	3628	/* Clear all dirty flags */
		3629	memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
		3630
		3631	skl_clear_wm(results, intel_crtc->pipe);
		3632
6937	serge	3633	if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm))
5354	serge	3634	return;
		3635
6937	serge	3636	skl_compute_wm_results(dev, pipe_wm, results, intel_crtc);
5354	serge	3637	results->dirty[intel_crtc->pipe] = true;
		3638
6937	serge	3639	skl_update_other_pipe_wm(dev, crtc, results);
5354	serge	3640	skl_write_wm_values(dev_priv, results);
		3641	skl_flush_wm_values(dev_priv, results);
		3642
		3643	/* store the new configuration */
		3644	dev_priv->wm.skl_hw = *results;
		3645	}
		3646
6937	serge	3647	static void ilk_compute_wm_config(struct drm_device *dev,
		3648	struct intel_wm_config *config)
5354	serge	3649	{
6937	serge	3650	struct intel_crtc *crtc;
5354	serge	3651
6937	serge	3652	/* Compute the currently _active_ config */
		3653	for_each_intel_crtc(dev, crtc) {
		3654	const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5354	serge	3655
6937	serge	3656	if (!wm->pipe_enabled)
		3657	continue;
6084	serge	3658
6937	serge	3659	config->sprites_enabled \|= wm->sprites_enabled;
		3660	config->sprites_scaled \|= wm->sprites_scaled;
		3661	config->num_pipes_active++;
		3662	}
5354	serge	3663	}
		3664
7144	serge	3665	static void ilk_program_watermarks(struct intel_crtc_state *cstate)
4560	Serge	3666	{
7144	serge	3667	struct drm_crtc *crtc = cstate->base.crtc;
		3668	struct drm_device *dev = crtc->dev;
		3669	struct drm_i915_private *dev_priv = to_i915(dev);
6937	serge	3670	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
4560	Serge	3671	struct ilk_wm_maximums max;
6937	serge	3672	struct intel_wm_config config = {};
4560	Serge	3673	struct ilk_wm_values results = {};
4104	Serge	3674	enum intel_ddb_partitioning partitioning;
		3675
5060	serge	3676	ilk_compute_wm_config(dev, &config);
		3677
4560	Serge	3678	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		3679	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		3680
		3681	/* 5/6 split only in single pipe config on IVB+ */
		3682	if (INTEL_INFO(dev)->gen >= 7 &&
		3683	config.num_pipes_active == 1 && config.sprites_enabled) {
		3684	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		3685	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		3686
		3687	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	3688	} else {
4560	Serge	3689	best_lp_wm = &lp_wm_1_2;
4104	Serge	3690	}
		3691
4560	Serge	3692	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	3693	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		3694
4560	Serge	3695	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		3696
		3697	ilk_write_wm_values(dev_priv, &results);
4104	Serge	3698	}
		3699
6937	serge	3700	static void ilk_update_wm(struct drm_crtc *crtc)
4104	Serge	3701	{
6937	serge	3702	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3703	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
4104	Serge	3704
6937	serge	3705	WARN_ON(cstate->base.active != intel_crtc->active);
		3706
4560	Serge	3707	/*
		3708	* IVB workaround: must disable low power watermarks for at least
		3709	* one frame before enabling scaling. LP watermarks can be re-enabled
		3710	* when scaling is disabled.
		3711	*
		3712	* WaCxSRDisabledForSpriteScaling:ivb
		3713	*/
6937	serge	3714	if (cstate->disable_lp_wm) {
		3715	ilk_disable_lp_wm(crtc->dev);
		3716	intel_wait_for_vblank(crtc->dev, intel_crtc->pipe);
		3717	}
4560	Serge	3718
6937	serge	3719	intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk;
		3720
7144	serge	3721	ilk_program_watermarks(cstate);
4104	Serge	3722	}
		3723
5354	serge	3724	static void skl_pipe_wm_active_state(uint32_t val,
		3725	struct skl_pipe_wm *active,
		3726	bool is_transwm,
		3727	bool is_cursor,
		3728	int i,
		3729	int level)
		3730	{
		3731	bool is_enabled = (val & PLANE_WM_EN) != 0;
		3732
		3733	if (!is_transwm) {
		3734	if (!is_cursor) {
		3735	active->wm[level].plane_en[i] = is_enabled;
		3736	active->wm[level].plane_res_b[i] =
		3737	val & PLANE_WM_BLOCKS_MASK;
		3738	active->wm[level].plane_res_l[i] =
		3739	(val >> PLANE_WM_LINES_SHIFT) &
		3740	PLANE_WM_LINES_MASK;
		3741	} else {
6084	serge	3742	active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
		3743	active->wm[level].plane_res_b[PLANE_CURSOR] =
5354	serge	3744	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3745	active->wm[level].plane_res_l[PLANE_CURSOR] =
5354	serge	3746	(val >> PLANE_WM_LINES_SHIFT) &
		3747	PLANE_WM_LINES_MASK;
		3748	}
		3749	} else {
		3750	if (!is_cursor) {
		3751	active->trans_wm.plane_en[i] = is_enabled;
		3752	active->trans_wm.plane_res_b[i] =
		3753	val & PLANE_WM_BLOCKS_MASK;
		3754	active->trans_wm.plane_res_l[i] =
		3755	(val >> PLANE_WM_LINES_SHIFT) &
		3756	PLANE_WM_LINES_MASK;
		3757	} else {
6084	serge	3758	active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
		3759	active->trans_wm.plane_res_b[PLANE_CURSOR] =
5354	serge	3760	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3761	active->trans_wm.plane_res_l[PLANE_CURSOR] =
5354	serge	3762	(val >> PLANE_WM_LINES_SHIFT) &
		3763	PLANE_WM_LINES_MASK;
		3764	}
		3765	}
		3766	}
		3767
		3768	static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
		3769	{
		3770	struct drm_device *dev = crtc->dev;
		3771	struct drm_i915_private *dev_priv = dev->dev_private;
		3772	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
		3773	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3774	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
		3775	struct skl_pipe_wm *active = &cstate->wm.optimal.skl;
5354	serge	3776	enum pipe pipe = intel_crtc->pipe;
		3777	int level, i, max_level;
		3778	uint32_t temp;
		3779
		3780	max_level = ilk_wm_max_level(dev);
		3781
		3782	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
		3783
		3784	for (level = 0; level <= max_level; level++) {
		3785	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3786	hw->plane[pipe][i][level] =
		3787	I915_READ(PLANE_WM(pipe, i, level));
6084	serge	3788	hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
5354	serge	3789	}
		3790
		3791	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3792	hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
6084	serge	3793	hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
5354	serge	3794
6084	serge	3795	if (!intel_crtc->active)
5354	serge	3796	return;
		3797
		3798	hw->dirty[pipe] = true;
		3799
		3800	active->linetime = hw->wm_linetime[pipe];
		3801
		3802	for (level = 0; level <= max_level; level++) {
		3803	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3804	temp = hw->plane[pipe][i][level];
		3805	skl_pipe_wm_active_state(temp, active, false,
		3806	false, i, level);
		3807	}
6084	serge	3808	temp = hw->plane[pipe][PLANE_CURSOR][level];
5354	serge	3809	skl_pipe_wm_active_state(temp, active, false, true, i, level);
		3810	}
		3811
		3812	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3813	temp = hw->plane_trans[pipe][i];
		3814	skl_pipe_wm_active_state(temp, active, true, false, i, 0);
		3815	}
		3816
6084	serge	3817	temp = hw->plane_trans[pipe][PLANE_CURSOR];
5354	serge	3818	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
6937	serge	3819
		3820	intel_crtc->wm.active.skl = *active;
5354	serge	3821	}
		3822
		3823	void skl_wm_get_hw_state(struct drm_device *dev)
		3824	{
		3825	struct drm_i915_private *dev_priv = dev->dev_private;
		3826	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
		3827	struct drm_crtc *crtc;
		3828
		3829	skl_ddb_get_hw_state(dev_priv, ddb);
		3830	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3831	skl_pipe_wm_get_hw_state(crtc);
		3832	}
		3833
4560	Serge	3834	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	3835	{
4560	Serge	3836	struct drm_device *dev = crtc->dev;
		3837	struct drm_i915_private *dev_priv = dev->dev_private;
		3838	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		3839	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6937	serge	3840	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
		3841	struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
4560	Serge	3842	enum pipe pipe = intel_crtc->pipe;
6937	serge	3843	static const i915_reg_t wm0_pipe_reg[] = {
4560	Serge	3844	[PIPE_A] = WM0_PIPEA_ILK,
		3845	[PIPE_B] = WM0_PIPEB_ILK,
		3846	[PIPE_C] = WM0_PIPEC_IVB,
		3847	};
3031	serge	3848
4560	Serge	3849	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		3850	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		3851	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	3852
6660	serge	3853	memset(active, 0, sizeof(*active));
		3854
6084	serge	3855	active->pipe_enabled = intel_crtc->active;
5060	serge	3856
		3857	if (active->pipe_enabled) {
4560	Serge	3858	u32 tmp = hw->wm_pipe[pipe];
3031	serge	3859
4560	Serge	3860	/*
		3861	* For active pipes LP0 watermark is marked as
		3862	* enabled, and LP1+ watermaks as disabled since
		3863	* we can't really reverse compute them in case
		3864	* multiple pipes are active.
		3865	*/
		3866	active->wm[0].enable = true;
		3867	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		3868	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		3869	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		3870	active->linetime = hw->wm_linetime[pipe];
		3871	} else {
		3872	int level, max_level = ilk_wm_max_level(dev);
3031	serge	3873
4560	Serge	3874	/*
		3875	* For inactive pipes, all watermark levels
		3876	* should be marked as enabled but zeroed,
		3877	* which is what we'd compute them to.
		3878	*/
		3879	for (level = 0; level <= max_level; level++)
		3880	active->wm[level].enable = true;
3031	serge	3881	}
6937	serge	3882
		3883	intel_crtc->wm.active.ilk = *active;
3031	serge	3884	}
		3885
6084	serge	3886	#define _FW_WM(value, plane) \
		3887	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
		3888	#define _FW_WM_VLV(value, plane) \
		3889	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
		3890
		3891	static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
		3892	struct vlv_wm_values *wm)
		3893	{
		3894	enum pipe pipe;
		3895	uint32_t tmp;
		3896
		3897	for_each_pipe(dev_priv, pipe) {
		3898	tmp = I915_READ(VLV_DDL(pipe));
		3899
		3900	wm->ddl[pipe].primary =
		3901	(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3902	wm->ddl[pipe].cursor =
		3903	(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3904	wm->ddl[pipe].sprite[0] =
		3905	(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3906	wm->ddl[pipe].sprite[1] =
		3907	(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3908	}
		3909
		3910	tmp = I915_READ(DSPFW1);
		3911	wm->sr.plane = _FW_WM(tmp, SR);
		3912	wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
		3913	wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
		3914	wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
		3915
		3916	tmp = I915_READ(DSPFW2);
		3917	wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
		3918	wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
		3919	wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
		3920
		3921	tmp = I915_READ(DSPFW3);
		3922	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
		3923
		3924	if (IS_CHERRYVIEW(dev_priv)) {
		3925	tmp = I915_READ(DSPFW7_CHV);
		3926	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3927	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3928
		3929	tmp = I915_READ(DSPFW8_CHV);
		3930	wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
		3931	wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
		3932
		3933	tmp = I915_READ(DSPFW9_CHV);
		3934	wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
		3935	wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
		3936
		3937	tmp = I915_READ(DSPHOWM);
		3938	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3939	wm->pipe[PIPE_C].sprite[1] \|= _FW_WM(tmp, SPRITEF_HI) << 8;
		3940	wm->pipe[PIPE_C].sprite[0] \|= _FW_WM(tmp, SPRITEE_HI) << 8;
		3941	wm->pipe[PIPE_C].primary \|= _FW_WM(tmp, PLANEC_HI) << 8;
		3942	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3943	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3944	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3945	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3946	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3947	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3948	} else {
		3949	tmp = I915_READ(DSPFW7);
		3950	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3951	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3952
		3953	tmp = I915_READ(DSPHOWM);
		3954	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3955	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3956	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3957	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3958	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3959	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3960	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3961	}
		3962	}
		3963
		3964	#undef _FW_WM
		3965	#undef _FW_WM_VLV
		3966
		3967	void vlv_wm_get_hw_state(struct drm_device *dev)
		3968	{
		3969	struct drm_i915_private *dev_priv = to_i915(dev);
		3970	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
		3971	struct intel_plane *plane;
		3972	enum pipe pipe;
		3973	u32 val;
		3974
		3975	vlv_read_wm_values(dev_priv, wm);
		3976
		3977	for_each_intel_plane(dev, plane) {
		3978	switch (plane->base.type) {
		3979	int sprite;
		3980	case DRM_PLANE_TYPE_CURSOR:
		3981	plane->wm.fifo_size = 63;
		3982	break;
		3983	case DRM_PLANE_TYPE_PRIMARY:
		3984	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
		3985	break;
		3986	case DRM_PLANE_TYPE_OVERLAY:
		3987	sprite = plane->plane;
		3988	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
		3989	break;
		3990	}
		3991	}
		3992
		3993	wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
		3994	wm->level = VLV_WM_LEVEL_PM2;
		3995
		3996	if (IS_CHERRYVIEW(dev_priv)) {
		3997	mutex_lock(&dev_priv->rps.hw_lock);
		3998
		3999	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		4000	if (val & DSP_MAXFIFO_PM5_ENABLE)
		4001	wm->level = VLV_WM_LEVEL_PM5;
		4002
		4003	/*
		4004	* If DDR DVFS is disabled in the BIOS, Punit
		4005	* will never ack the request. So if that happens
		4006	* assume we don't have to enable/disable DDR DVFS
		4007	* dynamically. To test that just set the REQ_ACK
		4008	* bit to poke the Punit, but don't change the
		4009	* HIGH/LOW bits so that we don't actually change
		4010	* the current state.
		4011	*/
		4012	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4013	val \|= FORCE_DDR_FREQ_REQ_ACK;
		4014	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		4015
		4016	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		4017	FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
		4018	DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
		4019	"assuming DDR DVFS is disabled\n");
		4020	dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		4021	} else {
		4022	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4023	if ((val & FORCE_DDR_HIGH_FREQ) == 0)
		4024	wm->level = VLV_WM_LEVEL_DDR_DVFS;
		4025	}
		4026
		4027	mutex_unlock(&dev_priv->rps.hw_lock);
		4028	}
		4029
		4030	for_each_pipe(dev_priv, pipe)
		4031	DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
		4032	pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
		4033	wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
		4034
		4035	DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		4036	wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
		4037	}
		4038
4560	Serge	4039	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	4040	{
		4041	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4042	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		4043	struct drm_crtc *crtc;
3031	serge	4044
5060	serge	4045	for_each_crtc(dev, crtc)
4560	Serge	4046	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	4047
4560	Serge	4048	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		4049	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		4050	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	4051
4560	Serge	4052	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	4053	if (INTEL_INFO(dev)->gen >= 7) {
5354	serge	4054	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		4055	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	4056	}
3031	serge	4057
4560	Serge	4058	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4059	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		4060	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		4061	else if (IS_IVYBRIDGE(dev))
		4062	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		4063	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	4064
4560	Serge	4065	hw->enable_fbc_wm =
		4066	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	4067	}
		4068
		4069	/**
		4070	* intel_update_watermarks - update FIFO watermark values based on current modes
		4071	*
		4072	* Calculate watermark values for the various WM regs based on current mode
		4073	* and plane configuration.
		4074	*
		4075	* There are several cases to deal with here:
		4076	* - normal (i.e. non-self-refresh)
		4077	* - self-refresh (SR) mode
		4078	* - lines are large relative to FIFO size (buffer can hold up to 2)
		4079	* - lines are small relative to FIFO size (buffer can hold more than 2
		4080	* lines), so need to account for TLB latency
		4081	*
		4082	* The normal calculation is:
		4083	* watermark = dotclock * bytes per pixel * latency
		4084	* where latency is platform & configuration dependent (we assume pessimal
		4085	* values here).
		4086	*
		4087	* The SR calculation is:
		4088	* watermark = (trunc(latency/line time)+1) * surface width *
		4089	* bytes per pixel
		4090	* where
		4091	* line time = htotal / dotclock
		4092	* surface width = hdisplay for normal plane and 64 for cursor
		4093	* and latency is assumed to be high, as above.
		4094	*
		4095	* The final value programmed to the register should always be rounded up,
		4096	* and include an extra 2 entries to account for clock crossings.
		4097	*
		4098	* We don't use the sprite, so we can ignore that. And on Crestline we have
		4099	* to set the non-SR watermarks to 8.
		4100	*/
4560	Serge	4101	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	4102	{
4560	Serge	4103	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	4104
		4105	if (dev_priv->display.update_wm)
4560	Serge	4106	dev_priv->display.update_wm(crtc);
3031	serge	4107	}
		4108
7144	serge	4109	/*
3031	serge	4110	* Lock protecting IPS related data structures
		4111	*/
		4112	DEFINE_SPINLOCK(mchdev_lock);
		4113
		4114	/* Global for IPS driver to get at the current i915 device. Protected by
		4115	* mchdev_lock. */
		4116	static struct drm_i915_private *i915_mch_dev;
		4117
		4118	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		4119	{
		4120	struct drm_i915_private *dev_priv = dev->dev_private;
		4121	u16 rgvswctl;
		4122
		4123	assert_spin_locked(&mchdev_lock);
		4124
		4125	rgvswctl = I915_READ16(MEMSWCTL);
		4126	if (rgvswctl & MEMCTL_CMD_STS) {
		4127	DRM_DEBUG("gpu busy, RCS change rejected\n");
		4128	return false; /* still busy with another command */
		4129	}
		4130
		4131	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		4132	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		4133	I915_WRITE16(MEMSWCTL, rgvswctl);
		4134	POSTING_READ16(MEMSWCTL);
		4135
		4136	rgvswctl \|= MEMCTL_CMD_STS;
		4137	I915_WRITE16(MEMSWCTL, rgvswctl);
		4138
		4139	return true;
		4140	}
		4141
		4142	static void ironlake_enable_drps(struct drm_device *dev)
		4143	{
		4144	struct drm_i915_private *dev_priv = dev->dev_private;
7144	serge	4145	u32 rgvmodectl;
3031	serge	4146	u8 fmax, fmin, fstart, vstart;
		4147
		4148	spin_lock_irq(&mchdev_lock);
		4149
7144	serge	4150	rgvmodectl = I915_READ(MEMMODECTL);
		4151
3031	serge	4152	/* Enable temp reporting */
		4153	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		4154	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		4155
		4156	/* 100ms RC evaluation intervals */
		4157	I915_WRITE(RCUPEI, 100000);
		4158	I915_WRITE(RCDNEI, 100000);
		4159
		4160	/* Set max/min thresholds to 90ms and 80ms respectively */
		4161	I915_WRITE(RCBMAXAVG, 90000);
		4162	I915_WRITE(RCBMINAVG, 80000);
		4163
		4164	I915_WRITE(MEMIHYST, 1);
		4165
		4166	/* Set up min, max, and cur for interrupt handling */
		4167	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		4168	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		4169	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		4170	MEMMODE_FSTART_SHIFT;
		4171
6084	serge	4172	vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
3031	serge	4173	PXVFREQ_PX_SHIFT;
		4174
		4175	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		4176	dev_priv->ips.fstart = fstart;
		4177
		4178	dev_priv->ips.max_delay = fstart;
		4179	dev_priv->ips.min_delay = fmin;
		4180	dev_priv->ips.cur_delay = fstart;
		4181
		4182	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		4183	fmax, fmin, fstart);
		4184
		4185	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		4186
		4187	/*
		4188	* Interrupts will be enabled in ironlake_irq_postinstall
		4189	*/
		4190
		4191	I915_WRITE(VIDSTART, vstart);
		4192	POSTING_READ(VIDSTART);
		4193
		4194	rgvmodectl \|= MEMMODE_SWMODE_EN;
		4195	I915_WRITE(MEMMODECTL, rgvmodectl);
		4196
		4197	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		4198	DRM_ERROR("stuck trying to change perf mode\n");
		4199	mdelay(1);
		4200
		4201	ironlake_set_drps(dev, fstart);
		4202
6084	serge	4203	dev_priv->ips.last_count1 = I915_READ(DMIEC) +
		4204	I915_READ(DDREC) + I915_READ(CSIEC);
5060	serge	4205	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6084	serge	4206	dev_priv->ips.last_count2 = I915_READ(GFXEC);
5060	serge	4207	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	4208
		4209	spin_unlock_irq(&mchdev_lock);
		4210	}
		4211
		4212	static void ironlake_disable_drps(struct drm_device *dev)
		4213	{
		4214	struct drm_i915_private *dev_priv = dev->dev_private;
		4215	u16 rgvswctl;
		4216
		4217	spin_lock_irq(&mchdev_lock);
		4218
		4219	rgvswctl = I915_READ16(MEMSWCTL);
		4220
		4221	/* Ack interrupts, disable EFC interrupt */
		4222	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		4223	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		4224	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		4225	I915_WRITE(DEIIR, DE_PCU_EVENT);
		4226	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		4227
		4228	/* Go back to the starting frequency */
		4229	ironlake_set_drps(dev, dev_priv->ips.fstart);
		4230	mdelay(1);
		4231	rgvswctl \|= MEMCTL_CMD_STS;
		4232	I915_WRITE(MEMSWCTL, rgvswctl);
		4233	mdelay(1);
		4234
		4235	spin_unlock_irq(&mchdev_lock);
		4236	}
		4237
		4238	/* There's a funny hw issue where the hw returns all 0 when reading from
		4239	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		4240	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		4241	* all limits and the gpu stuck at whatever frequency it is at atm).
		4242	*/
6084	serge	4243	static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	4244	{
		4245	u32 limits;
		4246
		4247	/* Only set the down limit when we've reached the lowest level to avoid
		4248	* getting more interrupts, otherwise leave this clear. This prevents a
		4249	* race in the hw when coming out of rc6: There's a tiny window where
		4250	* the hw runs at the minimal clock before selecting the desired
		4251	* frequency, if the down threshold expires in that window we will not
		4252	* receive a down interrupt. */
6084	serge	4253	if (IS_GEN9(dev_priv->dev)) {
		4254	limits = (dev_priv->rps.max_freq_softlimit) << 23;
		4255	if (val <= dev_priv->rps.min_freq_softlimit)
		4256	limits \|= (dev_priv->rps.min_freq_softlimit) << 14;
		4257	} else {
		4258	limits = dev_priv->rps.max_freq_softlimit << 24;
		4259	if (val <= dev_priv->rps.min_freq_softlimit)
		4260	limits \|= dev_priv->rps.min_freq_softlimit << 16;
		4261	}
3031	serge	4262
		4263	return limits;
		4264	}
		4265
4560	Serge	4266	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		4267	{
		4268	int new_power;
6084	serge	4269	u32 threshold_up = 0, threshold_down = 0; /* in % */
		4270	u32 ei_up = 0, ei_down = 0;
4560	Serge	4271
		4272	new_power = dev_priv->rps.power;
		4273	switch (dev_priv->rps.power) {
		4274	case LOW_POWER:
5060	serge	4275	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	4276	new_power = BETWEEN;
		4277	break;
		4278
		4279	case BETWEEN:
5060	serge	4280	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	4281	new_power = LOW_POWER;
5060	serge	4282	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	4283	new_power = HIGH_POWER;
		4284	break;
		4285
		4286	case HIGH_POWER:
5060	serge	4287	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	4288	new_power = BETWEEN;
		4289	break;
		4290	}
		4291	/* Max/min bins are special */
6084	serge	4292	if (val <= dev_priv->rps.min_freq_softlimit)
4560	Serge	4293	new_power = LOW_POWER;
6084	serge	4294	if (val >= dev_priv->rps.max_freq_softlimit)
4560	Serge	4295	new_power = HIGH_POWER;
		4296	if (new_power == dev_priv->rps.power)
		4297	return;
		4298
		4299	/* Note the units here are not exactly 1us, but 1280ns. */
		4300	switch (new_power) {
		4301	case LOW_POWER:
		4302	/* Upclock if more than 95% busy over 16ms */
6084	serge	4303	ei_up = 16000;
		4304	threshold_up = 95;
4560	Serge	4305
		4306	/* Downclock if less than 85% busy over 32ms */
6084	serge	4307	ei_down = 32000;
		4308	threshold_down = 85;
4560	Serge	4309	break;
		4310
		4311	case BETWEEN:
		4312	/* Upclock if more than 90% busy over 13ms */
6084	serge	4313	ei_up = 13000;
		4314	threshold_up = 90;
4560	Serge	4315
		4316	/* Downclock if less than 75% busy over 32ms */
6084	serge	4317	ei_down = 32000;
		4318	threshold_down = 75;
4560	Serge	4319	break;
		4320
		4321	case HIGH_POWER:
		4322	/* Upclock if more than 85% busy over 10ms */
6084	serge	4323	ei_up = 10000;
		4324	threshold_up = 85;
4560	Serge	4325
		4326	/* Downclock if less than 60% busy over 32ms */
6084	serge	4327	ei_down = 32000;
		4328	threshold_down = 60;
4560	Serge	4329	break;
		4330	}
		4331
6084	serge	4332	I915_WRITE(GEN6_RP_UP_EI,
		4333	GT_INTERVAL_FROM_US(dev_priv, ei_up));
		4334	I915_WRITE(GEN6_RP_UP_THRESHOLD,
		4335	GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
		4336
		4337	I915_WRITE(GEN6_RP_DOWN_EI,
		4338	GT_INTERVAL_FROM_US(dev_priv, ei_down));
		4339	I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
		4340	GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
		4341
		4342	I915_WRITE(GEN6_RP_CONTROL,
		4343	GEN6_RP_MEDIA_TURBO \|
		4344	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4345	GEN6_RP_MEDIA_IS_GFX \|
		4346	GEN6_RP_ENABLE \|
		4347	GEN6_RP_UP_BUSY_AVG \|
		4348	GEN6_RP_DOWN_IDLE_AVG);
		4349
4560	Serge	4350	dev_priv->rps.power = new_power;
6084	serge	4351	dev_priv->rps.up_threshold = threshold_up;
		4352	dev_priv->rps.down_threshold = threshold_down;
4560	Serge	4353	dev_priv->rps.last_adj = 0;
		4354	}
		4355
5060	serge	4356	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		4357	{
		4358	u32 mask = 0;
		4359
		4360	if (val > dev_priv->rps.min_freq_softlimit)
6937	serge	4361	mask \|= GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
5060	serge	4362	if (val < dev_priv->rps.max_freq_softlimit)
6084	serge	4363	mask \|= GEN6_PM_RP_UP_EI_EXPIRED \| GEN6_PM_RP_UP_THRESHOLD;
5060	serge	4364
		4365	mask &= dev_priv->pm_rps_events;
		4366
6084	serge	4367	return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
5060	serge	4368	}
		4369
		4370	/* gen6_set_rps is called to update the frequency request, but should also be
		4371	* called when the range (min_delay and max_delay) is modified so that we can
		4372	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6084	serge	4373	static void gen6_set_rps(struct drm_device *dev, u8 val)
3031	serge	4374	{
		4375	struct drm_i915_private *dev_priv = dev->dev_private;
		4376
6084	serge	4377	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
6937	serge	4378	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
6084	serge	4379	return;
		4380
3243	Serge	4381	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6084	serge	4382	WARN_ON(val > dev_priv->rps.max_freq);
		4383	WARN_ON(val < dev_priv->rps.min_freq);
3031	serge	4384
5060	serge	4385	/* min/max delay may still have been modified so be sure to
		4386	* write the limits value.
		4387	*/
		4388	if (val != dev_priv->rps.cur_freq) {
5354	serge	4389	gen6_set_rps_thresholds(dev_priv, val);
4560	Serge	4390
6084	serge	4391	if (IS_GEN9(dev))
5354	serge	4392	I915_WRITE(GEN6_RPNSWREQ,
6084	serge	4393	GEN9_FREQUENCY(val));
		4394	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4395	I915_WRITE(GEN6_RPNSWREQ,
5354	serge	4396	HSW_FREQUENCY(val));
		4397	else
		4398	I915_WRITE(GEN6_RPNSWREQ,
		4399	GEN6_FREQUENCY(val) \|
		4400	GEN6_OFFSET(0) \|
		4401	GEN6_AGGRESSIVE_TURBO);
5060	serge	4402	}
3031	serge	4403
		4404	/* Make sure we continue to get interrupts
		4405	* until we hit the minimum or maximum frequencies.
		4406	*/
6084	serge	4407	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
5060	serge	4408	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	4409
		4410	POSTING_READ(GEN6_RPNSWREQ);
		4411
5060	serge	4412	dev_priv->rps.cur_freq = val;
6084	serge	4413	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3031	serge	4414	}
		4415
6084	serge	4416	static void valleyview_set_rps(struct drm_device *dev, u8 val)
5060	serge	4417	{
6084	serge	4418	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4419
6084	serge	4420	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4421	WARN_ON(val > dev_priv->rps.max_freq);
		4422	WARN_ON(val < dev_priv->rps.min_freq);
5060	serge	4423
6084	serge	4424	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
		4425	"Odd GPU freq value\n"))
		4426	val &= ~1;
5060	serge	4427
6084	serge	4428	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5060	serge	4429
6084	serge	4430	if (val != dev_priv->rps.cur_freq) {
		4431	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		4432	if (!IS_CHERRYVIEW(dev_priv))
		4433	gen6_set_rps_thresholds(dev_priv, val);
		4434	}
5060	serge	4435
6084	serge	4436	dev_priv->rps.cur_freq = val;
		4437	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
		4438	}
5060	serge	4439
6084	serge	4440	/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
		4441	*
		4442	* * If Gfx is Idle, then
		4443	* 1. Forcewake Media well.
		4444	* 2. Request idle freq.
		4445	* 3. Release Forcewake of Media well.
		4446	*/
		4447	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		4448	{
		4449	u32 val = dev_priv->rps.idle_freq;
5060	serge	4450
6084	serge	4451	if (dev_priv->rps.cur_freq <= val)
		4452	return;
5060	serge	4453
6084	serge	4454	/* Wake up the media well, as that takes a lot less
		4455	* power than the Render well. */
		4456	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
		4457	valleyview_set_rps(dev_priv->dev, val);
		4458	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
5060	serge	4459	}
		4460
6084	serge	4461	void gen6_rps_busy(struct drm_i915_private *dev_priv)
3031	serge	4462	{
4560	Serge	4463	mutex_lock(&dev_priv->rps.hw_lock);
		4464	if (dev_priv->rps.enabled) {
6937	serge	4465	if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED))
6084	serge	4466	gen6_rps_reset_ei(dev_priv);
		4467	I915_WRITE(GEN6_PMINTRMSK,
		4468	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4560	Serge	4469	}
		4470	mutex_unlock(&dev_priv->rps.hw_lock);
		4471	}
4104	Serge	4472
6084	serge	4473	void gen6_rps_idle(struct drm_i915_private *dev_priv)
4560	Serge	4474	{
		4475	struct drm_device *dev = dev_priv->dev;
4104	Serge	4476
4560	Serge	4477	mutex_lock(&dev_priv->rps.hw_lock);
		4478	if (dev_priv->rps.enabled) {
6937	serge	4479	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev))
6084	serge	4480	vlv_set_rps_idle(dev_priv);
4560	Serge	4481	else
6084	serge	4482	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4483	dev_priv->rps.last_adj = 0;
6937	serge	4484	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4560	Serge	4485	}
		4486	mutex_unlock(&dev_priv->rps.hw_lock);
6084	serge	4487
		4488	spin_lock(&dev_priv->rps.client_lock);
		4489	while (!list_empty(&dev_priv->rps.clients))
		4490	list_del_init(dev_priv->rps.clients.next);
		4491	spin_unlock(&dev_priv->rps.client_lock);
4104	Serge	4492	}
		4493
6084	serge	4494	void gen6_rps_boost(struct drm_i915_private *dev_priv,
		4495	struct intel_rps_client *rps,
		4496	unsigned long submitted)
4104	Serge	4497	{
6084	serge	4498	/* This is intentionally racy! We peek at the state here, then
		4499	* validate inside the RPS worker.
		4500	*/
		4501	if (!(dev_priv->mm.busy &&
		4502	dev_priv->rps.enabled &&
		4503	dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
		4504	return;
3031	serge	4505
6084	serge	4506	/* Force a RPS boost (and don't count it against the client) if
		4507	* the GPU is severely congested.
		4508	*/
		4509	if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
		4510	rps = NULL;
4104	Serge	4511
6084	serge	4512	spin_lock(&dev_priv->rps.client_lock);
		4513	if (rps == NULL \|\| list_empty(&rps->link)) {
		4514	spin_lock_irq(&dev_priv->irq_lock);
		4515	if (dev_priv->rps.interrupts_enabled) {
		4516	dev_priv->rps.client_boost = true;
		4517	queue_work(dev_priv->wq, &dev_priv->rps.work);
		4518	}
		4519	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	4520
6084	serge	4521	if (rps != NULL) {
		4522	list_add(&rps->link, &dev_priv->rps.clients);
		4523	rps->boosts++;
		4524	} else
		4525	dev_priv->rps.boosts++;
		4526	}
		4527	spin_unlock(&dev_priv->rps.client_lock);
		4528	}
4104	Serge	4529
6084	serge	4530	void intel_set_rps(struct drm_device *dev, u8 val)
		4531	{
6937	serge	4532	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev))
6084	serge	4533	valleyview_set_rps(dev, val);
		4534	else
		4535	gen6_set_rps(dev, val);
4104	Serge	4536	}
		4537
5354	serge	4538	static void gen9_disable_rps(struct drm_device *dev)
5060	serge	4539	{
		4540	struct drm_i915_private *dev_priv = dev->dev_private;
		4541
5354	serge	4542	I915_WRITE(GEN6_RC_CONTROL, 0);
6084	serge	4543	I915_WRITE(GEN9_PG_ENABLE, 0);
5060	serge	4544	}
		4545
4104	Serge	4546	static void gen6_disable_rps(struct drm_device *dev)
		4547	{
		4548	struct drm_i915_private *dev_priv = dev->dev_private;
		4549
		4550	I915_WRITE(GEN6_RC_CONTROL, 0);
		4551	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		4552	}
		4553
5060	serge	4554	static void cherryview_disable_rps(struct drm_device *dev)
		4555	{
		4556	struct drm_i915_private *dev_priv = dev->dev_private;
		4557
		4558	I915_WRITE(GEN6_RC_CONTROL, 0);
		4559	}
		4560
4104	Serge	4561	static void valleyview_disable_rps(struct drm_device *dev)
		4562	{
		4563	struct drm_i915_private *dev_priv = dev->dev_private;
		4564
5354	serge	4565	/* we're doing forcewake before Disabling RC6,
		4566	* This what the BIOS expects when going into suspend */
6084	serge	4567	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4568
4104	Serge	4569	I915_WRITE(GEN6_RC_CONTROL, 0);
		4570
6084	serge	4571	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4572	}
		4573
4560	Serge	4574	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		4575	{
6937	serge	4576	if (IS_VALLEYVIEW(dev) \|\| IS_CHERRYVIEW(dev)) {
5060	serge	4577	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		4578	mode = GEN6_RC_CTL_RC6_ENABLE;
		4579	else
		4580	mode = 0;
		4581	}
5354	serge	4582	if (HAS_RC6p(dev))
		4583	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
7144	serge	4584	onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
		4585	onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
		4586	onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
5354	serge	4587
		4588	else
		4589	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
7144	serge	4590	onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
4560	Serge	4591	}
		4592
7144	serge	4593	static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
3031	serge	4594	{
7144	serge	4595	struct drm_i915_private *dev_priv = dev->dev_private;
		4596	bool enable_rc6 = true;
		4597	unsigned long rc6_ctx_base;
		4598
		4599	if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
		4600	DRM_DEBUG_KMS("RC6 Base location not set properly.\n");
		4601	enable_rc6 = false;
		4602	}
		4603
		4604	/*
		4605	* The exact context size is not known for BXT, so assume a page size
		4606	* for this check.
		4607	*/
		4608	rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
		4609	if (!((rc6_ctx_base >= dev_priv->gtt.stolen_reserved_base) &&
		4610	(rc6_ctx_base + PAGE_SIZE <= dev_priv->gtt.stolen_reserved_base +
		4611	dev_priv->gtt.stolen_reserved_size))) {
		4612	DRM_DEBUG_KMS("RC6 Base address not as expected.\n");
		4613	enable_rc6 = false;
		4614	}
		4615
		4616	if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
		4617	((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
		4618	((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
		4619	((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
		4620	DRM_DEBUG_KMS("Engine Idle wait time not set properly.\n");
		4621	enable_rc6 = false;
		4622	}
		4623
		4624	if (!(I915_READ(GEN6_RC_CONTROL) & (GEN6_RC_CTL_RC6_ENABLE \|
		4625	GEN6_RC_CTL_HW_ENABLE)) &&
		4626	((I915_READ(GEN6_RC_CONTROL) & GEN6_RC_CTL_HW_ENABLE) \|\|
		4627	!(I915_READ(GEN6_RC_STATE) & RC6_STATE))) {
		4628	DRM_DEBUG_KMS("HW/SW RC6 is not enabled by BIOS.\n");
		4629	enable_rc6 = false;
		4630	}
		4631
		4632	return enable_rc6;
		4633	}
		4634
		4635	int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
		4636	{
6084	serge	4637	/* No RC6 before Ironlake and code is gone for ilk. */
		4638	if (INTEL_INFO(dev)->gen < 6)
4104	Serge	4639	return 0;
		4640
7144	serge	4641	if (!enable_rc6)
		4642	return 0;
		4643
		4644	if (IS_BROXTON(dev) && !bxt_check_bios_rc6_setup(dev)) {
		4645	DRM_INFO("RC6 disabled by BIOS\n");
		4646	return 0;
		4647	}
		4648
3031	serge	4649	/* Respect the kernel parameter if it is set */
5060	serge	4650	if (enable_rc6 >= 0) {
		4651	int mask;
3031	serge	4652
5354	serge	4653	if (HAS_RC6p(dev))
5060	serge	4654	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		4655	INTEL_RC6pp_ENABLE;
		4656	else
		4657	mask = INTEL_RC6_ENABLE;
		4658
		4659	if ((enable_rc6 & mask) != enable_rc6)
		4660	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
6084	serge	4661	enable_rc6 & mask, enable_rc6, mask);
5060	serge	4662
		4663	return enable_rc6 & mask;
		4664	}
		4665
		4666	if (IS_IVYBRIDGE(dev))
		4667	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	4668
6084	serge	4669	return INTEL_RC6_ENABLE;
5060	serge	4670	}
3031	serge	4671
5060	serge	4672	int intel_enable_rc6(const struct drm_device *dev)
		4673	{
		4674	return i915.enable_rc6;
3031	serge	4675	}
		4676
5354	serge	4677	static void gen6_init_rps_frequencies(struct drm_device *dev)
5060	serge	4678	{
		4679	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4680	uint32_t rp_state_cap;
		4681	u32 ddcc_status = 0;
		4682	int ret;
5060	serge	4683
		4684	/* All of these values are in units of 50MHz */
		4685	dev_priv->rps.cur_freq = 0;
5354	serge	4686	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
6084	serge	4687	if (IS_BROXTON(dev)) {
		4688	rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
		4689	dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
		4690	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4691	dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
		4692	} else {
		4693	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		4694	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
		4695	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4696	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		4697	}
		4698
5060	serge	4699	/* hw_max = RP0 until we check for overclocking */
		4700	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	4701
5354	serge	4702	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
6937	serge	4703	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev) \|\|
		4704	IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
5354	serge	4705	ret = sandybridge_pcode_read(dev_priv,
		4706	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
		4707	&ddcc_status);
		4708	if (0 == ret)
		4709	dev_priv->rps.efficient_freq =
6084	serge	4710	clamp_t(u8,
		4711	((ddcc_status >> 8) & 0xff),
		4712	dev_priv->rps.min_freq,
		4713	dev_priv->rps.max_freq);
5354	serge	4714	}
		4715
6937	serge	4716	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
6084	serge	4717	/* Store the frequency values in 16.66 MHZ units, which is
		4718	the natural hardware unit for SKL */
		4719	dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
		4720	dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
		4721	dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
		4722	dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
		4723	dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
		4724	}
		4725
		4726	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		4727
5060	serge	4728	/* Preserve min/max settings in case of re-init */
		4729	if (dev_priv->rps.max_freq_softlimit == 0)
		4730	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	4731
5354	serge	4732	if (dev_priv->rps.min_freq_softlimit == 0) {
		4733	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4734	dev_priv->rps.min_freq_softlimit =
6084	serge	4735	max_t(int, dev_priv->rps.efficient_freq,
		4736	intel_freq_opcode(dev_priv, 450));
5354	serge	4737	else
		4738	dev_priv->rps.min_freq_softlimit =
		4739	dev_priv->rps.min_freq;
		4740	}
4104	Serge	4741	}
		4742
6084	serge	4743	/* See the Gen9_GT_PM_Programming_Guide doc for the below */
5354	serge	4744	static void gen9_enable_rps(struct drm_device *dev)
		4745	{
		4746	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4747
		4748	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4749
		4750	gen6_init_rps_frequencies(dev);
		4751
		4752	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
6937	serge	4753	if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
6084	serge	4754	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4755	return;
		4756	}
		4757
		4758	/* Program defaults and thresholds for RPS*/
		4759	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4760	GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
		4761
		4762	/* 1 second timeout*/
		4763	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
		4764	GT_INTERVAL_FROM_US(dev_priv, 1000000));
		4765
		4766	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
		4767
		4768	/* Leaning on the below call to gen6_set_rps to program/setup the
		4769	* Up/Down EI & threshold registers, as well as the RP_CONTROL,
		4770	* RP_INTERRUPT_LIMITS & RPNSWREQ registers */
		4771	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		4772	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4773
		4774	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4775	}
		4776
		4777	static void gen9_enable_rc6(struct drm_device *dev)
		4778	{
		4779	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4780	struct intel_engine_cs *ring;
		4781	uint32_t rc6_mask = 0;
		4782	int unused;
		4783
		4784	/* 1a: Software RC state - RC0 */
		4785	I915_WRITE(GEN6_RC_STATE, 0);
		4786
		4787	/* 1b: Get forcewake during program sequence. Although the driver
		4788	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4789	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4790
		4791	/* 2a: Disable RC states. */
		4792	I915_WRITE(GEN6_RC_CONTROL, 0);
		4793
		4794	/* 2b: Program RC6 thresholds.*/
6084	serge	4795
		4796	/* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
		4797	if (IS_SKYLAKE(dev))
		4798	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
		4799	else
		4800	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
5354	serge	4801	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4802	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4803	for_each_ring(ring, dev_priv, unused)
		4804	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
6084	serge	4805
		4806	if (HAS_GUC_UCODE(dev))
		4807	I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
		4808
5354	serge	4809	I915_WRITE(GEN6_RC_SLEEP, 0);
		4810
6084	serge	4811	/* 2c: Program Coarse Power Gating Policies. */
		4812	I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
		4813	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
		4814
5354	serge	4815	/* 3a: Enable RC6 */
		4816	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4817	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7144	serge	4818	DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
6084	serge	4819	/* WaRsUseTimeoutMode */
6937	serge	4820	if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) \|\|
		4821	IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
6084	serge	4822	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
		4823	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4824	GEN7_RC_CTL_TO_MODE \|
		4825	rc6_mask);
		4826	} else {
		4827	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
		4828	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4829	GEN6_RC_CTL_EI_MODE(1) \|
		4830	rc6_mask);
		4831	}
5354	serge	4832
6084	serge	4833	/*
		4834	* 3b: Enable Coarse Power Gating only when RC6 is enabled.
		4835	* WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
		4836	*/
7144	serge	4837	if (NEEDS_WaRsDisableCoarsePowerGating(dev))
6084	serge	4838	I915_WRITE(GEN9_PG_ENABLE, 0);
		4839	else
		4840	I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4841	(GEN9_RENDER_PG_ENABLE \| GEN9_MEDIA_PG_ENABLE) : 0);
5354	serge	4842
6084	serge	4843	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4844
5354	serge	4845	}
		4846
4560	Serge	4847	static void gen8_enable_rps(struct drm_device *dev)
		4848	{
		4849	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4850	struct intel_engine_cs *ring;
5354	serge	4851	uint32_t rc6_mask = 0;
4560	Serge	4852	int unused;
		4853
		4854	/* 1a: Software RC state - RC0 */
		4855	I915_WRITE(GEN6_RC_STATE, 0);
		4856
		4857	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		4858	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4859	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4560	Serge	4860
		4861	/* 2a: Disable RC states. */
		4862	I915_WRITE(GEN6_RC_CONTROL, 0);
		4863
5354	serge	4864	/* Initialize rps frequencies */
		4865	gen6_init_rps_frequencies(dev);
4560	Serge	4866
		4867	/* 2b: Program RC6 thresholds.*/
		4868	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4869	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4870	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4871	for_each_ring(ring, dev_priv, unused)
		4872	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4873	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	4874	if (IS_BROADWELL(dev))
		4875	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		4876	else
6084	serge	4877	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4560	Serge	4878
		4879	/* 3: Enable RC6 */
		4880	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4881	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	4882	intel_print_rc6_info(dev, rc6_mask);
		4883	if (IS_BROADWELL(dev))
		4884	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4885	GEN7_RC_CTL_TO_MODE \|
		4886	rc6_mask);
		4887	else
6084	serge	4888	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4889	GEN6_RC_CTL_EI_MODE(1) \|
		4890	rc6_mask);
4560	Serge	4891
		4892	/* 4 Program defaults and thresholds for RPS*/
5060	serge	4893	I915_WRITE(GEN6_RPNSWREQ,
		4894	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		4895	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4896	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	4897	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		4898	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		4899
		4900	/* Docs recommend 900MHz, and 300 MHz respectively */
		4901	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	4902	dev_priv->rps.max_freq_softlimit << 24 \|
		4903	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	4904
		4905	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		4906	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		4907	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		4908	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		4909
		4910	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4911
		4912	/* 5: Enable RPS */
		4913	I915_WRITE(GEN6_RP_CONTROL,
		4914	GEN6_RP_MEDIA_TURBO \|
		4915	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4916	GEN6_RP_MEDIA_IS_GFX \|
		4917	GEN6_RP_ENABLE \|
		4918	GEN6_RP_UP_BUSY_AVG \|
		4919	GEN6_RP_DOWN_IDLE_AVG);
		4920
		4921	/* 6: Ring frequency + overclocking (our driver does this later */
		4922
5354	serge	4923	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	4924	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4925
6084	serge	4926	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4560	Serge	4927	}
		4928
3031	serge	4929	static void gen6_enable_rps(struct drm_device *dev)
		4930	{
		4931	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4932	struct intel_engine_cs *ring;
		4933	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	4934	u32 gtfifodbg;
		4935	int rc6_mode;
3243	Serge	4936	int i, ret;
3031	serge	4937
3243	Serge	4938	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4939
		4940	/* Here begins a magic sequence of register writes to enable
		4941	* auto-downclocking.
		4942	*
		4943	* Perhaps there might be some value in exposing these to
		4944	* userspace...
		4945	*/
		4946	I915_WRITE(GEN6_RC_STATE, 0);
		4947
		4948	/* Clear the DBG now so we don't confuse earlier errors */
		4949	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		4950	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		4951	I915_WRITE(GTFIFODBG, gtfifodbg);
		4952	}
		4953
6084	serge	4954	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	4955
5354	serge	4956	/* Initialize rps frequencies */
		4957	gen6_init_rps_frequencies(dev);
3031	serge	4958
		4959	/* disable the counters and set deterministic thresholds */
		4960	I915_WRITE(GEN6_RC_CONTROL, 0);
		4961
		4962	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		4963	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		4964	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		4965	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4966	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4967
		4968	for_each_ring(ring, dev_priv, i)
		4969	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4970
		4971	I915_WRITE(GEN6_RC_SLEEP, 0);
		4972	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	4973	if (IS_IVYBRIDGE(dev))
4104	Serge	4974	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		4975	else
6084	serge	4976	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	4977	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	4978	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		4979
		4980	/* Check if we are enabling RC6 */
		4981	rc6_mode = intel_enable_rc6(dev_priv->dev);
		4982	if (rc6_mode & INTEL_RC6_ENABLE)
		4983	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		4984
		4985	/* We don't use those on Haswell */
		4986	if (!IS_HASWELL(dev)) {
		4987	if (rc6_mode & INTEL_RC6p_ENABLE)
		4988	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		4989
		4990	if (rc6_mode & INTEL_RC6pp_ENABLE)
		4991	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		4992	}
		4993
4560	Serge	4994	intel_print_rc6_info(dev, rc6_mask);
3031	serge	4995
		4996	I915_WRITE(GEN6_RC_CONTROL,
		4997	rc6_mask \|
		4998	GEN6_RC_CTL_EI_MODE(1) \|
		4999	GEN6_RC_CTL_HW_ENABLE);
		5000
4560	Serge	5001	/* Power down if completely idle for over 50ms */
		5002	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	5003	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5004
3243	Serge	5005	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	5006	if (ret)
		5007	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		5008
6084	serge	5009	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		5010	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		5011	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	5012	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
6084	serge	5013	(pcu_mbox & 0xff) * 50);
5060	serge	5014	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	5015	}
		5016
4560	Serge	5017	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	5018	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
3031	serge	5019
3243	Serge	5020	rc6vids = 0;
		5021	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		5022	if (IS_GEN6(dev) && ret) {
		5023	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		5024	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		5025	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		5026	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		5027	rc6vids &= 0xffff00;
		5028	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		5029	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		5030	if (ret)
		5031	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		5032	}
		5033
6084	serge	5034	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	5035	}
		5036
5060	serge	5037	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	5038	{
		5039	struct drm_i915_private *dev_priv = dev->dev_private;
		5040	int min_freq = 15;
3746	Serge	5041	unsigned int gpu_freq;
		5042	unsigned int max_ia_freq, min_ring_freq;
6084	serge	5043	unsigned int max_gpu_freq, min_gpu_freq;
3031	serge	5044	int scaling_factor = 180;
4560	Serge	5045	struct cpufreq_policy *policy;
3031	serge	5046
3243	Serge	5047	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5048
		5049	max_ia_freq = cpufreq_quick_get_max(0);
		5050	/*
5060	serge	5051	* Default to measured freq if none found, PCU will ensure we
		5052	* don't go over
3031	serge	5053	*/
		5054	max_ia_freq = tsc_khz;
		5055
		5056	/* Convert from kHz to MHz */
		5057	max_ia_freq /= 1000;
		5058
4560	Serge	5059	min_ring_freq = I915_READ(DCLK) & 0xf;
		5060	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		5061	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	5062
6937	serge	5063	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
6084	serge	5064	/* Convert GT frequency to 50 HZ units */
		5065	min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
		5066	max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
		5067	} else {
		5068	min_gpu_freq = dev_priv->rps.min_freq;
		5069	max_gpu_freq = dev_priv->rps.max_freq;
		5070	}
		5071
3031	serge	5072	/*
		5073	* For each potential GPU frequency, load a ring frequency we'd like
		5074	* to use for memory access. We do this by specifying the IA frequency
		5075	* the PCU should use as a reference to determine the ring frequency.
		5076	*/
6084	serge	5077	for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
		5078	int diff = max_gpu_freq - gpu_freq;
3746	Serge	5079	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	5080
6937	serge	5081	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev)) {
6084	serge	5082	/*
		5083	* ring_freq = 2 * GT. ring_freq is in 100MHz units
		5084	* No floor required for ring frequency on SKL.
		5085	*/
		5086	ring_freq = gpu_freq;
		5087	} else if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	5088	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		5089	ring_freq = max(min_ring_freq, gpu_freq);
		5090	} else if (IS_HASWELL(dev)) {
		5091	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	5092	ring_freq = max(min_ring_freq, ring_freq);
		5093	/* leave ia_freq as the default, chosen by cpufreq */
		5094	} else {
		5095	/* On older processors, there is no separate ring
		5096	* clock domain, so in order to boost the bandwidth
		5097	* of the ring, we need to upclock the CPU (ia_freq).
		5098	*
		5099	* For GPU frequencies less than 750MHz,
		5100	* just use the lowest ring freq.
6084	serge	5101	*/
		5102	if (gpu_freq < min_freq)
		5103	ia_freq = 800;
		5104	else
		5105	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		5106	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	5107	}
3031	serge	5108
3243	Serge	5109	sandybridge_pcode_write(dev_priv,
		5110	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	5111	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		5112	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		5113	gpu_freq);
3031	serge	5114	}
		5115	}
		5116
5060	serge	5117	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	5118	{
5060	serge	5119	struct drm_i915_private *dev_priv = dev->dev_private;
		5120
6084	serge	5121	if (!HAS_CORE_RING_FREQ(dev))
5060	serge	5122	return;
		5123
		5124	mutex_lock(&dev_priv->rps.hw_lock);
		5125	__gen6_update_ring_freq(dev);
		5126	mutex_unlock(&dev_priv->rps.hw_lock);
		5127	}
		5128
		5129	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		5130	{
6084	serge	5131	struct drm_device *dev = dev_priv->dev;
4104	Serge	5132	u32 val, rp0;
		5133
6084	serge	5134	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5060	serge	5135
6084	serge	5136	switch (INTEL_INFO(dev)->eu_total) {
		5137	case 8:
		5138	/* (2 * 4) config */
		5139	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
		5140	break;
		5141	case 12:
		5142	/* (2 * 6) config */
		5143	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
		5144	break;
		5145	case 16:
		5146	/* (2 * 8) config */
		5147	default:
		5148	/* Setting (2 * 8) Min RP0 for any other combination */
		5149	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
		5150	break;
		5151	}
		5152
		5153	rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
		5154
5060	serge	5155	return rp0;
		5156	}
		5157
		5158	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5159	{
		5160	u32 val, rpe;
		5161
		5162	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		5163	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		5164
		5165	return rpe;
		5166	}
		5167
		5168	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5169	{
		5170	u32 val, rp1;
		5171
6084	serge	5172	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
		5173	rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5060	serge	5174
		5175	return rp1;
		5176	}
		5177
		5178	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5179	{
		5180	u32 val, rp1;
		5181
4104	Serge	5182	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5183
5060	serge	5184	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		5185
		5186	return rp1;
		5187	}
		5188
		5189	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		5190	{
		5191	u32 val, rp0;
		5192
		5193	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5194
4104	Serge	5195	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		5196	/* Clamp to max */
		5197	rp0 = min_t(u32, rp0, 0xea);
		5198
		5199	return rp0;
		5200	}
		5201
		5202	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5203	{
		5204	u32 val, rpe;
		5205
		5206	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		5207	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		5208	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		5209	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		5210
		5211	return rpe;
		5212	}
		5213
5060	serge	5214	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	5215	{
6937	serge	5216	u32 val;
		5217
		5218	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		5219	/*
		5220	* According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
		5221	* for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
		5222	* a BYT-M B0 the above register contains 0xbf. Moreover when setting
		5223	* a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
		5224	* to make sure it matches what Punit accepts.
		5225	*/
		5226	return max_t(u32, val, 0xc0);
4104	Serge	5227	}
		5228
5060	serge	5229	/* Check that the pctx buffer wasn't move under us. */
		5230	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		5231	{
		5232	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5233
		5234	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		5235	dev_priv->vlv_pctx->stolen->start);
		5236	}
		5237
		5238
		5239	/* Check that the pcbr address is not empty. */
		5240	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		5241	{
		5242	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5243
		5244	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		5245	}
		5246
		5247	static void cherryview_setup_pctx(struct drm_device *dev)
		5248	{
		5249	struct drm_i915_private *dev_priv = dev->dev_private;
		5250	unsigned long pctx_paddr, paddr;
		5251	struct i915_gtt *gtt = &dev_priv->gtt;
		5252	u32 pcbr;
		5253	int pctx_size = 32*1024;
		5254
		5255	pcbr = I915_READ(VLV_PCBR);
		5256	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5354	serge	5257	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5060	serge	5258	paddr = (dev_priv->mm.stolen_base +
		5259	(gtt->stolen_size - pctx_size));
		5260
		5261	pctx_paddr = (paddr & (~4095));
		5262	I915_WRITE(VLV_PCBR, pctx_paddr);
		5263	}
5354	serge	5264
		5265	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5060	serge	5266	}
		5267
4104	Serge	5268	static void valleyview_setup_pctx(struct drm_device *dev)
		5269	{
		5270	struct drm_i915_private *dev_priv = dev->dev_private;
		5271	struct drm_i915_gem_object *pctx;
		5272	unsigned long pctx_paddr;
		5273	u32 pcbr;
		5274	int pctx_size = 24*1024;
		5275
7144	serge	5276	mutex_lock(&dev->struct_mutex);
5060	serge	5277
4104	Serge	5278	pcbr = I915_READ(VLV_PCBR);
		5279	if (pcbr) {
		5280	/* BIOS set it up already, grab the pre-alloc'd space */
		5281	int pcbr_offset;
		5282
		5283	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		5284	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		5285	pcbr_offset,
		5286	I915_GTT_OFFSET_NONE,
		5287	pctx_size);
		5288	goto out;
		5289	}
		5290
5354	serge	5291	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
		5292
4104	Serge	5293	/*
		5294	* From the Gunit register HAS:
		5295	* The Gfx driver is expected to program this register and ensure
		5296	* proper allocation within Gfx stolen memory. For example, this
		5297	* register should be programmed such than the PCBR range does not
		5298	* overlap with other ranges, such as the frame buffer, protected
		5299	* memory, or any other relevant ranges.
		5300	*/
		5301	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		5302	if (!pctx) {
		5303	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7144	serge	5304	goto out;
4104	Serge	5305	}
		5306
		5307	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		5308	I915_WRITE(VLV_PCBR, pctx_paddr);
		5309
		5310	out:
5354	serge	5311	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4104	Serge	5312	dev_priv->vlv_pctx = pctx;
7144	serge	5313	mutex_unlock(&dev->struct_mutex);
4104	Serge	5314	}
		5315
5060	serge	5316	static void valleyview_cleanup_pctx(struct drm_device *dev)
		5317	{
		5318	struct drm_i915_private *dev_priv = dev->dev_private;
		5319
		5320	if (WARN_ON(!dev_priv->vlv_pctx))
		5321	return;
		5322
7144	serge	5323	drm_gem_object_unreference_unlocked(&dev_priv->vlv_pctx->base);
5060	serge	5324	dev_priv->vlv_pctx = NULL;
		5325	}
		5326
		5327	static void valleyview_init_gt_powersave(struct drm_device *dev)
		5328	{
		5329	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5330	u32 val;
5060	serge	5331
		5332	valleyview_setup_pctx(dev);
		5333
		5334	mutex_lock(&dev_priv->rps.hw_lock);
		5335
5354	serge	5336	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5337	switch ((val >> 6) & 3) {
		5338	case 0:
		5339	case 1:
		5340	dev_priv->mem_freq = 800;
		5341	break;
		5342	case 2:
		5343	dev_priv->mem_freq = 1066;
		5344	break;
		5345	case 3:
		5346	dev_priv->mem_freq = 1333;
		5347	break;
		5348	}
		5349	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5350
5060	serge	5351	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		5352	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5353	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5354	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5355	dev_priv->rps.max_freq);
		5356
		5357	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		5358	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5359	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5360	dev_priv->rps.efficient_freq);
		5361
		5362	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		5363	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
6084	serge	5364	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5365	dev_priv->rps.rp1_freq);
		5366
		5367	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		5368	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5369	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5370	dev_priv->rps.min_freq);
		5371
6084	serge	5372	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5373
5060	serge	5374	/* Preserve min/max settings in case of re-init */
		5375	if (dev_priv->rps.max_freq_softlimit == 0)
		5376	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5377
		5378	if (dev_priv->rps.min_freq_softlimit == 0)
		5379	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5380
		5381	mutex_unlock(&dev_priv->rps.hw_lock);
		5382	}
		5383
		5384	static void cherryview_init_gt_powersave(struct drm_device *dev)
		5385	{
		5386	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5387	u32 val;
5060	serge	5388
		5389	cherryview_setup_pctx(dev);
		5390
		5391	mutex_lock(&dev_priv->rps.hw_lock);
		5392
6084	serge	5393	mutex_lock(&dev_priv->sb_lock);
5354	serge	5394	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
6084	serge	5395	mutex_unlock(&dev_priv->sb_lock);
5354	serge	5396
		5397	switch ((val >> 2) & 0x7) {
		5398	case 3:
		5399	dev_priv->mem_freq = 2000;
		5400	break;
6084	serge	5401	default:
5354	serge	5402	dev_priv->mem_freq = 1600;
		5403	break;
		5404	}
		5405	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5406
5060	serge	5407	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		5408	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5409	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5410	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5411	dev_priv->rps.max_freq);
		5412
		5413	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		5414	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5415	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5416	dev_priv->rps.efficient_freq);
		5417
		5418	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		5419	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
6084	serge	5420	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5421	dev_priv->rps.rp1_freq);
		5422
6084	serge	5423	/* PUnit validated range is only [RPe, RP0] */
		5424	dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5060	serge	5425	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5426	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5427	dev_priv->rps.min_freq);
		5428
5354	serge	5429	WARN_ONCE((dev_priv->rps.max_freq \|
		5430	dev_priv->rps.efficient_freq \|
		5431	dev_priv->rps.rp1_freq \|
		5432	dev_priv->rps.min_freq) & 1,
		5433	"Odd GPU freq values\n");
		5434
6084	serge	5435	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5436
5060	serge	5437	/* Preserve min/max settings in case of re-init */
		5438	if (dev_priv->rps.max_freq_softlimit == 0)
		5439	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5440
		5441	if (dev_priv->rps.min_freq_softlimit == 0)
		5442	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5443
		5444	mutex_unlock(&dev_priv->rps.hw_lock);
		5445	}
		5446
		5447	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		5448	{
		5449	valleyview_cleanup_pctx(dev);
		5450	}
		5451
		5452	static void cherryview_enable_rps(struct drm_device *dev)
		5453	{
		5454	struct drm_i915_private *dev_priv = dev->dev_private;
		5455	struct intel_engine_cs *ring;
		5456	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		5457	int i;
		5458
		5459	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5460
		5461	gtfifodbg = I915_READ(GTFIFODBG);
		5462	if (gtfifodbg) {
		5463	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5464	gtfifodbg);
		5465	I915_WRITE(GTFIFODBG, gtfifodbg);
		5466	}
		5467
		5468	cherryview_check_pctx(dev_priv);
		5469
		5470	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		5471	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	5472	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	5473
6084	serge	5474	/* Disable RC states. */
		5475	I915_WRITE(GEN6_RC_CONTROL, 0);
		5476
5060	serge	5477	/* 2a: Program RC6 thresholds.*/
		5478	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		5479	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		5480	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		5481
		5482	for_each_ring(ring, dev_priv, i)
		5483	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5484	I915_WRITE(GEN6_RC_SLEEP, 0);
		5485
6084	serge	5486	/* TO threshold set to 500 us ( 0x186 * 1.28 us) */
		5487	I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5060	serge	5488
		5489	/* allows RC6 residency counter to work */
		5490	I915_WRITE(VLV_COUNTER_CONTROL,
		5491	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		5492	VLV_MEDIA_RC6_COUNT_EN \|
		5493	VLV_RENDER_RC6_COUNT_EN));
		5494
		5495	/* For now we assume BIOS is allocating and populating the PCBR */
		5496	pcbr = I915_READ(VLV_PCBR);
		5497
		5498	/* 3: Enable RC6 */
		5499	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		5500	(pcbr >> VLV_PCBR_ADDR_SHIFT))
6084	serge	5501	rc6_mode = GEN7_RC_CTL_TO_MODE;
5060	serge	5502
		5503	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5504
		5505	/* 4 Program defaults and thresholds for RPS*/
6084	serge	5506	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5060	serge	5507	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5508	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5509	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5510	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5511
		5512	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5513
		5514	/* 5: Enable RPS */
		5515	I915_WRITE(GEN6_RP_CONTROL,
		5516	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
6084	serge	5517	GEN6_RP_MEDIA_IS_GFX \|
5060	serge	5518	GEN6_RP_ENABLE \|
		5519	GEN6_RP_UP_BUSY_AVG \|
		5520	GEN6_RP_DOWN_IDLE_AVG);
		5521
6084	serge	5522	/* Setting Fixed Bias */
		5523	val = VLV_OVERRIDE_EN \|
		5524	VLV_SOC_TDP_EN \|
		5525	CHV_BIAS_CPU_50_SOC_50;
		5526	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5527
5060	serge	5528	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5529
5354	serge	5530	/* RPS code assumes GPLL is used */
		5531	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5532
6084	serge	5533	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5060	serge	5534	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5535
		5536	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		5537	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5538	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5539	dev_priv->rps.cur_freq);
		5540
		5541	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5542	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5543	dev_priv->rps.efficient_freq);
		5544
		5545	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		5546
6084	serge	5547	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	5548	}
		5549
4104	Serge	5550	static void valleyview_enable_rps(struct drm_device *dev)
		5551	{
		5552	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5553	struct intel_engine_cs *ring;
4560	Serge	5554	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	5555	int i;
		5556
		5557	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5558
5060	serge	5559	valleyview_check_pctx(dev_priv);
		5560
4104	Serge	5561	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	5562	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5563	gtfifodbg);
4104	Serge	5564	I915_WRITE(GTFIFODBG, gtfifodbg);
		5565	}
		5566
4560	Serge	5567	/* If VLV, Forcewake all wells, else re-direct to regular path */
6084	serge	5568	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	5569
6084	serge	5570	/* Disable RC states. */
		5571	I915_WRITE(GEN6_RC_CONTROL, 0);
		5572
		5573	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4104	Serge	5574	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5575	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5576	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5577	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5578
		5579	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5580
		5581	I915_WRITE(GEN6_RP_CONTROL,
		5582	GEN6_RP_MEDIA_TURBO \|
		5583	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5584	GEN6_RP_MEDIA_IS_GFX \|
		5585	GEN6_RP_ENABLE \|
		5586	GEN6_RP_UP_BUSY_AVG \|
		5587	GEN6_RP_DOWN_IDLE_CONT);
		5588
		5589	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		5590	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		5591	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		5592
		5593	for_each_ring(ring, dev_priv, i)
		5594	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5595
4560	Serge	5596	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	5597
		5598	/* allows RC6 residency counter to work */
4560	Serge	5599	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	5600	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		5601	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	5602	VLV_MEDIA_RC6_COUNT_EN \|
		5603	VLV_RENDER_RC6_COUNT_EN));
5060	serge	5604
4560	Serge	5605	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		5606	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	5607
4560	Serge	5608	intel_print_rc6_info(dev, rc6_mode);
		5609
		5610	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5611
6084	serge	5612	/* Setting Fixed Bias */
		5613	val = VLV_OVERRIDE_EN \|
		5614	VLV_SOC_TDP_EN \|
		5615	VLV_BIAS_CPU_125_SOC_875;
		5616	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5617
4104	Serge	5618	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5619
5354	serge	5620	/* RPS code assumes GPLL is used */
		5621	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5622
6084	serge	5623	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
4104	Serge	5624	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5625
5060	serge	5626	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	5627	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5628	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5629	dev_priv->rps.cur_freq);
4104	Serge	5630
		5631	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5632	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5633	dev_priv->rps.efficient_freq);
4104	Serge	5634
5060	serge	5635	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	5636
6084	serge	5637	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	5638	}
		5639
3031	serge	5640	static unsigned long intel_pxfreq(u32 vidfreq)
		5641	{
		5642	unsigned long freq;
		5643	int div = (vidfreq & 0x3f0000) >> 16;
		5644	int post = (vidfreq & 0x3000) >> 12;
		5645	int pre = (vidfreq & 0x7);
		5646
		5647	if (!pre)
		5648	return 0;
		5649
		5650	freq = ((div * 133333) / ((1<
		5651
		5652	return freq;
		5653	}
		5654
		5655	static const struct cparams {
		5656	u16 i;
		5657	u16 t;
		5658	u16 m;
		5659	u16 c;
		5660	} cparams[] = {
		5661	{ 1, 1333, 301, 28664 },
		5662	{ 1, 1066, 294, 24460 },
		5663	{ 1, 800, 294, 25192 },
		5664	{ 0, 1333, 276, 27605 },
		5665	{ 0, 1066, 276, 27605 },
		5666	{ 0, 800, 231, 23784 },
		5667	};
		5668
		5669	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		5670	{
		5671	u64 total_count, diff, ret;
		5672	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	5673	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	5674	int i;
		5675
		5676	assert_spin_locked(&mchdev_lock);
		5677
		5678	diff1 = now - dev_priv->ips.last_time1;
		5679
		5680	/* Prevent division-by-zero if we are asking too fast.
		5681	* Also, we don't get interesting results if we are polling
		5682	* faster than once in 10ms, so just return the saved value
		5683	* in such cases.
		5684	*/
		5685	if (diff1 <= 10)
		5686	return dev_priv->ips.chipset_power;
		5687
		5688	count1 = I915_READ(DMIEC);
		5689	count2 = I915_READ(DDREC);
		5690	count3 = I915_READ(CSIEC);
		5691
		5692	total_count = count1 + count2 + count3;
		5693
		5694	/* FIXME: handle per-counter overflow */
		5695	if (total_count < dev_priv->ips.last_count1) {
		5696	diff = ~0UL - dev_priv->ips.last_count1;
		5697	diff += total_count;
		5698	} else {
		5699	diff = total_count - dev_priv->ips.last_count1;
		5700	}
		5701
		5702	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		5703	if (cparams[i].i == dev_priv->ips.c_m &&
		5704	cparams[i].t == dev_priv->ips.r_t) {
		5705	m = cparams[i].m;
		5706	c = cparams[i].c;
		5707	break;
		5708	}
		5709	}
		5710
		5711	diff = div_u64(diff, diff1);
		5712	ret = ((m * diff) + c);
		5713	ret = div_u64(ret, 10);
		5714
		5715	dev_priv->ips.last_count1 = total_count;
		5716	dev_priv->ips.last_time1 = now;
		5717
		5718	dev_priv->ips.chipset_power = ret;
		5719
		5720	return ret;
		5721	}
		5722
		5723	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		5724	{
5060	serge	5725	struct drm_device *dev = dev_priv->dev;
3031	serge	5726	unsigned long val;
		5727
5060	serge	5728	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5729	return 0;
		5730
		5731	spin_lock_irq(&mchdev_lock);
		5732
		5733	val = __i915_chipset_val(dev_priv);
		5734
		5735	spin_unlock_irq(&mchdev_lock);
		5736
		5737	return val;
		5738	}
		5739
		5740	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		5741	{
		5742	unsigned long m, x, b;
		5743	u32 tsfs;
		5744
		5745	tsfs = I915_READ(TSFS);
		5746
		5747	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		5748	x = I915_READ8(TR1);
		5749
		5750	b = tsfs & TSFS_INTR_MASK;
		5751
		5752	return ((m * x) / 127) - b;
		5753	}
		5754
6084	serge	5755	static int _pxvid_to_vd(u8 pxvid)
3031	serge	5756	{
6084	serge	5757	if (pxvid == 0)
		5758	return 0;
		5759
		5760	if (pxvid >= 8 && pxvid < 31)
		5761	pxvid = 31;
		5762
		5763	return (pxvid + 2) * 125;
		5764	}
		5765
		5766	static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		5767	{
5060	serge	5768	struct drm_device *dev = dev_priv->dev;
6084	serge	5769	const int vd = _pxvid_to_vd(pxvid);
		5770	const int vm = vd - 1125;
		5771
5060	serge	5772	if (INTEL_INFO(dev)->is_mobile)
6084	serge	5773	return vm > 0 ? vm : 0;
		5774
		5775	return vd;
3031	serge	5776	}
		5777
		5778	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5779	{
5060	serge	5780	u64 now, diff, diffms;
3031	serge	5781	u32 count;
		5782
		5783	assert_spin_locked(&mchdev_lock);
		5784
5060	serge	5785	now = ktime_get_raw_ns();
		5786	diffms = now - dev_priv->ips.last_time2;
		5787	do_div(diffms, NSEC_PER_MSEC);
3031	serge	5788
		5789	/* Don't divide by 0 */
		5790	if (!diffms)
		5791	return;
		5792
		5793	count = I915_READ(GFXEC);
		5794
		5795	if (count < dev_priv->ips.last_count2) {
		5796	diff = ~0UL - dev_priv->ips.last_count2;
		5797	diff += count;
		5798	} else {
		5799	diff = count - dev_priv->ips.last_count2;
		5800	}
		5801
		5802	dev_priv->ips.last_count2 = count;
		5803	dev_priv->ips.last_time2 = now;
		5804
		5805	/* More magic constants... */
		5806	diff = diff * 1181;
		5807	diff = div_u64(diff, diffms * 10);
		5808	dev_priv->ips.gfx_power = diff;
		5809	}
		5810
		5811	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5812	{
5060	serge	5813	struct drm_device *dev = dev_priv->dev;
		5814
		5815	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5816	return;
		5817
		5818	spin_lock_irq(&mchdev_lock);
		5819
		5820	__i915_update_gfx_val(dev_priv);
		5821
		5822	spin_unlock_irq(&mchdev_lock);
		5823	}
		5824
		5825	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		5826	{
		5827	unsigned long t, corr, state1, corr2, state2;
		5828	u32 pxvid, ext_v;
		5829
		5830	assert_spin_locked(&mchdev_lock);
		5831
6084	serge	5832	pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
3031	serge	5833	pxvid = (pxvid >> 24) & 0x7f;
		5834	ext_v = pvid_to_extvid(dev_priv, pxvid);
		5835
		5836	state1 = ext_v;
		5837
		5838	t = i915_mch_val(dev_priv);
		5839
		5840	/* Revel in the empirically derived constants */
		5841
		5842	/* Correction factor in 1/100000 units */
		5843	if (t > 80)
		5844	corr = ((t * 2349) + 135940);
		5845	else if (t >= 50)
		5846	corr = ((t * 964) + 29317);
		5847	else /* < 50 */
		5848	corr = ((t * 301) + 1004);
		5849
		5850	corr = corr * ((150142 * state1) / 10000 - 78642);
		5851	corr /= 100000;
		5852	corr2 = (corr * dev_priv->ips.corr);
		5853
		5854	state2 = (corr2 * state1) / 10000;
		5855	state2 /= 100; /* convert to mW */
		5856
		5857	__i915_update_gfx_val(dev_priv);
		5858
		5859	return dev_priv->ips.gfx_power + state2;
		5860	}
		5861
		5862	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		5863	{
5060	serge	5864	struct drm_device *dev = dev_priv->dev;
3031	serge	5865	unsigned long val;
		5866
5060	serge	5867	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5868	return 0;
		5869
		5870	spin_lock_irq(&mchdev_lock);
		5871
		5872	val = __i915_gfx_val(dev_priv);
		5873
		5874	spin_unlock_irq(&mchdev_lock);
		5875
		5876	return val;
		5877	}
		5878
		5879	/**
		5880	* i915_read_mch_val - return value for IPS use
		5881	*
		5882	* Calculate and return a value for the IPS driver to use when deciding whether
		5883	* we have thermal and power headroom to increase CPU or GPU power budget.
		5884	*/
		5885	unsigned long i915_read_mch_val(void)
		5886	{
		5887	struct drm_i915_private *dev_priv;
		5888	unsigned long chipset_val, graphics_val, ret = 0;
		5889
		5890	spin_lock_irq(&mchdev_lock);
		5891	if (!i915_mch_dev)
		5892	goto out_unlock;
		5893	dev_priv = i915_mch_dev;
		5894
		5895	chipset_val = __i915_chipset_val(dev_priv);
		5896	graphics_val = __i915_gfx_val(dev_priv);
		5897
		5898	ret = chipset_val + graphics_val;
		5899
		5900	out_unlock:
		5901	spin_unlock_irq(&mchdev_lock);
		5902
		5903	return ret;
		5904	}
		5905	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		5906
		5907	/**
		5908	* i915_gpu_raise - raise GPU frequency limit
		5909	*
		5910	* Raise the limit; IPS indicates we have thermal headroom.
		5911	*/
		5912	bool i915_gpu_raise(void)
		5913	{
		5914	struct drm_i915_private *dev_priv;
		5915	bool ret = true;
		5916
		5917	spin_lock_irq(&mchdev_lock);
		5918	if (!i915_mch_dev) {
		5919	ret = false;
		5920	goto out_unlock;
		5921	}
		5922	dev_priv = i915_mch_dev;
		5923
		5924	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		5925	dev_priv->ips.max_delay--;
		5926
		5927	out_unlock:
		5928	spin_unlock_irq(&mchdev_lock);
		5929
		5930	return ret;
		5931	}
		5932	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		5933
		5934	/**
		5935	* i915_gpu_lower - lower GPU frequency limit
		5936	*
		5937	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		5938	* frequency maximum.
		5939	*/
		5940	bool i915_gpu_lower(void)
		5941	{
		5942	struct drm_i915_private *dev_priv;
		5943	bool ret = true;
		5944
		5945	spin_lock_irq(&mchdev_lock);
		5946	if (!i915_mch_dev) {
		5947	ret = false;
		5948	goto out_unlock;
		5949	}
		5950	dev_priv = i915_mch_dev;
		5951
		5952	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		5953	dev_priv->ips.max_delay++;
		5954
		5955	out_unlock:
		5956	spin_unlock_irq(&mchdev_lock);
		5957
		5958	return ret;
		5959	}
		5960	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		5961
		5962	/**
		5963	* i915_gpu_busy - indicate GPU business to IPS
		5964	*
		5965	* Tell the IPS driver whether or not the GPU is busy.
		5966	*/
		5967	bool i915_gpu_busy(void)
		5968	{
		5969	struct drm_i915_private *dev_priv;
5060	serge	5970	struct intel_engine_cs *ring;
3031	serge	5971	bool ret = false;
		5972	int i;
		5973
		5974	spin_lock_irq(&mchdev_lock);
		5975	if (!i915_mch_dev)
		5976	goto out_unlock;
		5977	dev_priv = i915_mch_dev;
		5978
		5979	for_each_ring(ring, dev_priv, i)
		5980	ret \|= !list_empty(&ring->request_list);
		5981
		5982	out_unlock:
		5983	spin_unlock_irq(&mchdev_lock);
		5984
		5985	return ret;
		5986	}
		5987	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		5988
		5989	/**
		5990	* i915_gpu_turbo_disable - disable graphics turbo
		5991	*
		5992	* Disable graphics turbo by resetting the max frequency and setting the
		5993	* current frequency to the default.
		5994	*/
		5995	bool i915_gpu_turbo_disable(void)
		5996	{
		5997	struct drm_i915_private *dev_priv;
		5998	bool ret = true;
		5999
		6000	spin_lock_irq(&mchdev_lock);
		6001	if (!i915_mch_dev) {
		6002	ret = false;
		6003	goto out_unlock;
		6004	}
		6005	dev_priv = i915_mch_dev;
		6006
		6007	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		6008
		6009	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		6010	ret = false;
		6011
		6012	out_unlock:
		6013	spin_unlock_irq(&mchdev_lock);
		6014
		6015	return ret;
		6016	}
		6017	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		6018
		6019	/**
		6020	* Tells the intel_ips driver that the i915 driver is now loaded, if
		6021	* IPS got loaded first.
		6022	*
		6023	* This awkward dance is so that neither module has to depend on the
		6024	* other in order for IPS to do the appropriate communication of
		6025	* GPU turbo limits to i915.
		6026	*/
		6027	static void
		6028	ips_ping_for_i915_load(void)
		6029	{
		6030	void (*link)(void);
		6031
		6032	// link = symbol_get(ips_link_to_i915_driver);
		6033	// if (link) {
		6034	// link();
		6035	// symbol_put(ips_link_to_i915_driver);
		6036	// }
		6037	}
		6038
		6039	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		6040	{
		6041	/* We only register the i915 ips part with intel-ips once everything is
		6042	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		6043	spin_lock_irq(&mchdev_lock);
		6044	i915_mch_dev = dev_priv;
		6045	spin_unlock_irq(&mchdev_lock);
		6046
		6047	ips_ping_for_i915_load();
		6048	}
		6049
		6050	void intel_gpu_ips_teardown(void)
		6051	{
		6052	spin_lock_irq(&mchdev_lock);
		6053	i915_mch_dev = NULL;
		6054	spin_unlock_irq(&mchdev_lock);
		6055	}
5060	serge	6056
3031	serge	6057	static void intel_init_emon(struct drm_device *dev)
		6058	{
		6059	struct drm_i915_private *dev_priv = dev->dev_private;
		6060	u32 lcfuse;
		6061	u8 pxw[16];
		6062	int i;
		6063
		6064	/* Disable to program */
		6065	I915_WRITE(ECR, 0);
		6066	POSTING_READ(ECR);
		6067
		6068	/* Program energy weights for various events */
		6069	I915_WRITE(SDEW, 0x15040d00);
		6070	I915_WRITE(CSIEW0, 0x007f0000);
		6071	I915_WRITE(CSIEW1, 0x1e220004);
		6072	I915_WRITE(CSIEW2, 0x04000004);
		6073
		6074	for (i = 0; i < 5; i++)
6084	serge	6075	I915_WRITE(PEW(i), 0);
3031	serge	6076	for (i = 0; i < 3; i++)
6084	serge	6077	I915_WRITE(DEW(i), 0);
3031	serge	6078
		6079	/* Program P-state weights to account for frequency power adjustment */
		6080	for (i = 0; i < 16; i++) {
6084	serge	6081	u32 pxvidfreq = I915_READ(PXVFREQ(i));
3031	serge	6082	unsigned long freq = intel_pxfreq(pxvidfreq);
		6083	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		6084	PXVFREQ_PX_SHIFT;
		6085	unsigned long val;
		6086
		6087	val = vid * vid;
		6088	val *= (freq / 1000);
		6089	val *= 255;
		6090	val /= (127127900);
		6091	if (val > 0xff)
		6092	DRM_ERROR("bad pxval: %ld\n", val);
		6093	pxw[i] = val;
		6094	}
		6095	/* Render standby states get 0 weight */
		6096	pxw[14] = 0;
		6097	pxw[15] = 0;
		6098
		6099	for (i = 0; i < 4; i++) {
		6100	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		6101	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
6084	serge	6102	I915_WRITE(PXW(i), val);
3031	serge	6103	}
		6104
		6105	/* Adjust magic regs to magic values (more experimental results) */
		6106	I915_WRITE(OGW0, 0);
		6107	I915_WRITE(OGW1, 0);
		6108	I915_WRITE(EG0, 0x00007f00);
		6109	I915_WRITE(EG1, 0x0000000e);
		6110	I915_WRITE(EG2, 0x000e0000);
		6111	I915_WRITE(EG3, 0x68000300);
		6112	I915_WRITE(EG4, 0x42000000);
		6113	I915_WRITE(EG5, 0x00140031);
		6114	I915_WRITE(EG6, 0);
		6115	I915_WRITE(EG7, 0);
		6116
		6117	for (i = 0; i < 8; i++)
6084	serge	6118	I915_WRITE(PXWL(i), 0);
3031	serge	6119
		6120	/* Enable PMON + select events */
		6121	I915_WRITE(ECR, 0x80000019);
		6122
		6123	lcfuse = I915_READ(LCFUSE02);
		6124
		6125	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		6126	}
		6127
5060	serge	6128	void intel_init_gt_powersave(struct drm_device *dev)
		6129	{
6937	serge	6130	struct drm_i915_private *dev_priv = dev->dev_private;
		6131
		6132	/*
		6133	* RPM depends on RC6 to save restore the GT HW context, so make RC6 a
		6134	* requirement.
		6135	*/
		6136	if (!i915.enable_rc6) {
		6137	DRM_INFO("RC6 disabled, disabling runtime PM support\n");
		6138	intel_runtime_pm_get(dev_priv);
		6139	}
5060	serge	6140
		6141	if (IS_CHERRYVIEW(dev))
		6142	cherryview_init_gt_powersave(dev);
		6143	else if (IS_VALLEYVIEW(dev))
		6144	valleyview_init_gt_powersave(dev);
		6145	}
		6146
		6147	void intel_cleanup_gt_powersave(struct drm_device *dev)
		6148	{
6937	serge	6149	struct drm_i915_private *dev_priv = dev->dev_private;
		6150
5060	serge	6151	if (IS_CHERRYVIEW(dev))
		6152	return;
		6153	else if (IS_VALLEYVIEW(dev))
		6154	valleyview_cleanup_gt_powersave(dev);
6937	serge	6155
		6156	if (!i915.enable_rc6)
		6157	intel_runtime_pm_put(dev_priv);
5060	serge	6158	}
		6159
5354	serge	6160	static void gen6_suspend_rps(struct drm_device *dev)
		6161	{
		6162	struct drm_i915_private *dev_priv = dev->dev_private;
		6163
		6164	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		6165
6084	serge	6166	gen6_disable_rps_interrupts(dev);
5354	serge	6167	}
		6168
5060	serge	6169	/**
		6170	* intel_suspend_gt_powersave - suspend PM work and helper threads
		6171	* @dev: drm device
		6172	*
		6173	* We don't want to disable RC6 or other features here, we just want
		6174	* to make sure any work we've queued has finished and won't bother
		6175	* us while we're suspended.
		6176	*/
		6177	void intel_suspend_gt_powersave(struct drm_device *dev)
		6178	{
		6179	struct drm_i915_private *dev_priv = dev->dev_private;
		6180
5354	serge	6181	if (INTEL_INFO(dev)->gen < 6)
		6182	return;
5060	serge	6183
5354	serge	6184	gen6_suspend_rps(dev);
5060	serge	6185
		6186	/* Force GPU to min freq during suspend */
		6187	gen6_rps_idle(dev_priv);
		6188	}
		6189
3031	serge	6190	void intel_disable_gt_powersave(struct drm_device *dev)
		6191	{
3243	Serge	6192	struct drm_i915_private *dev_priv = dev->dev_private;
		6193
3031	serge	6194	if (IS_IRONLAKE_M(dev)) {
		6195	ironlake_disable_drps(dev);
4293	Serge	6196	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	6197	intel_suspend_gt_powersave(dev);
		6198
3482	Serge	6199	mutex_lock(&dev_priv->rps.hw_lock);
5354	serge	6200	if (INTEL_INFO(dev)->gen >= 9)
		6201	gen9_disable_rps(dev);
		6202	else if (IS_CHERRYVIEW(dev))
5060	serge	6203	cherryview_disable_rps(dev);
		6204	else if (IS_VALLEYVIEW(dev))
4104	Serge	6205	valleyview_disable_rps(dev);
		6206	else
6084	serge	6207	gen6_disable_rps(dev);
5354	serge	6208
4560	Serge	6209	dev_priv->rps.enabled = false;
3480	Serge	6210	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	6211	}
		6212	}
		6213
3482	Serge	6214	static void intel_gen6_powersave_work(struct work_struct *work)
		6215	{
		6216	struct drm_i915_private *dev_priv =
		6217	container_of(work, struct drm_i915_private,
		6218	rps.delayed_resume_work.work);
		6219	struct drm_device *dev = dev_priv->dev;
		6220
		6221	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	6222
6084	serge	6223	gen6_reset_rps_interrupts(dev);
5354	serge	6224
5060	serge	6225	if (IS_CHERRYVIEW(dev)) {
		6226	cherryview_enable_rps(dev);
		6227	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	6228	valleyview_enable_rps(dev);
5354	serge	6229	} else if (INTEL_INFO(dev)->gen >= 9) {
6084	serge	6230	gen9_enable_rc6(dev);
5354	serge	6231	gen9_enable_rps(dev);
6937	serge	6232	if (IS_SKYLAKE(dev) \|\| IS_KABYLAKE(dev))
6084	serge	6233	__gen6_update_ring_freq(dev);
4560	Serge	6234	} else if (IS_BROADWELL(dev)) {
		6235	gen8_enable_rps(dev);
5060	serge	6236	__gen6_update_ring_freq(dev);
4104	Serge	6237	} else {
6084	serge	6238	gen6_enable_rps(dev);
5060	serge	6239	__gen6_update_ring_freq(dev);
4104	Serge	6240	}
6084	serge	6241
		6242	WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
		6243	WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
		6244
		6245	WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
		6246	WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
		6247
4560	Serge	6248	dev_priv->rps.enabled = true;
5354	serge	6249
6084	serge	6250	gen6_enable_rps_interrupts(dev);
5354	serge	6251
3482	Serge	6252	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	6253
		6254	intel_runtime_pm_put(dev_priv);
3482	Serge	6255	}
		6256
3031	serge	6257	void intel_enable_gt_powersave(struct drm_device *dev)
		6258	{
3243	Serge	6259	struct drm_i915_private *dev_priv = dev->dev_private;
		6260
6084	serge	6261	/* Powersaving is controlled by the host when inside a VM */
		6262	if (intel_vgpu_active(dev))
		6263	return;
		6264
3031	serge	6265	if (IS_IRONLAKE_M(dev)) {
7144	serge	6266	ironlake_enable_drps(dev);
5060	serge	6267	mutex_lock(&dev->struct_mutex);
3031	serge	6268	intel_init_emon(dev);
5060	serge	6269	mutex_unlock(&dev->struct_mutex);
		6270	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	6271	/*
		6272	* PCU communication is slow and this doesn't need to be
		6273	* done at any specific time, so do this out of our fast path
		6274	* to make resume and init faster.
5060	serge	6275	*
		6276	* We depend on the HW RC6 power context save/restore
		6277	* mechanism when entering D3 through runtime PM suspend. So
		6278	* disable RPM until RPS/RC6 is properly setup. We can only
		6279	* get here via the driver load/system resume/runtime resume
		6280	* paths, so the _noresume version is enough (and in case of
		6281	* runtime resume it's necessary).
3243	Serge	6282	*/
5060	serge	6283	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		6284	round_jiffies_up_relative(HZ)))
		6285	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	6286	}
		6287	}
		6288
5060	serge	6289	void intel_reset_gt_powersave(struct drm_device *dev)
		6290	{
		6291	struct drm_i915_private *dev_priv = dev->dev_private;
		6292
5354	serge	6293	if (INTEL_INFO(dev)->gen < 6)
		6294	return;
		6295
		6296	gen6_suspend_rps(dev);
5060	serge	6297	dev_priv->rps.enabled = false;
		6298	}
		6299
3243	Serge	6300	static void ibx_init_clock_gating(struct drm_device *dev)
		6301	{
		6302	struct drm_i915_private *dev_priv = dev->dev_private;
		6303
		6304	/*
		6305	* On Ibex Peak and Cougar Point, we need to disable clock
		6306	* gating for the panel power sequencer or it will fail to
		6307	* start up when no ports are active.
		6308	*/
		6309	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		6310	}
		6311
4104	Serge	6312	static void g4x_disable_trickle_feed(struct drm_device *dev)
		6313	{
		6314	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	6315	enum pipe pipe;
4104	Serge	6316
5354	serge	6317	for_each_pipe(dev_priv, pipe) {
4104	Serge	6318	I915_WRITE(DSPCNTR(pipe),
		6319	I915_READ(DSPCNTR(pipe)) \|
		6320	DISPPLANE_TRICKLE_FEED_DISABLE);
6084	serge	6321
		6322	I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
		6323	POSTING_READ(DSPSURF(pipe));
4104	Serge	6324	}
		6325	}
		6326
4560	Serge	6327	static void ilk_init_lp_watermarks(struct drm_device *dev)
		6328	{
		6329	struct drm_i915_private *dev_priv = dev->dev_private;
		6330
		6331	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		6332	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		6333	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		6334
		6335	/*
		6336	* Don't touch WM1S_LP_EN here.
		6337	* Doing so could cause underruns.
		6338	*/
		6339	}
		6340
3031	serge	6341	static void ironlake_init_clock_gating(struct drm_device *dev)
		6342	{
		6343	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6344	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6345
4104	Serge	6346	/*
		6347	* Required for FBC
		6348	* WaFbcDisableDpfcClockGating:ilk
		6349	*/
3243	Serge	6350	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		6351	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		6352	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	6353
		6354	I915_WRITE(PCH_3DCGDIS0,
		6355	MARIUNIT_CLOCK_GATE_DISABLE \|
		6356	SVSMUNIT_CLOCK_GATE_DISABLE);
		6357	I915_WRITE(PCH_3DCGDIS1,
		6358	VFMUNIT_CLOCK_GATE_DISABLE);
		6359
		6360	/*
		6361	* According to the spec the following bits should be set in
		6362	* order to enable memory self-refresh
		6363	* The bit 22/21 of 0x42004
		6364	* The bit 5 of 0x42020
		6365	* The bit 15 of 0x45000
		6366	*/
		6367	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6368	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6369	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	6370	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	6371	I915_WRITE(DISP_ARB_CTL,
		6372	(I915_READ(DISP_ARB_CTL) \|
		6373	DISP_FBC_WM_DIS));
		6374
4560	Serge	6375	ilk_init_lp_watermarks(dev);
		6376
3031	serge	6377	/*
		6378	* Based on the document from hardware guys the following bits
		6379	* should be set unconditionally in order to enable FBC.
		6380	* The bit 22 of 0x42000
		6381	* The bit 22 of 0x42004
		6382	* The bit 7,8,9 of 0x42020.
		6383	*/
		6384	if (IS_IRONLAKE_M(dev)) {
4104	Serge	6385	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	6386	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6387	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6388	ILK_FBCQ_DIS);
		6389	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6390	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6391	ILK_DPARB_GATE);
		6392	}
		6393
3243	Serge	6394	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		6395
3031	serge	6396	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6397	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6398	ILK_ELPIN_409_SELECT);
		6399	I915_WRITE(_3D_CHICKEN2,
		6400	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		6401	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	6402
4104	Serge	6403	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	6404	I915_WRITE(CACHE_MODE_0,
		6405	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		6406
5060	serge	6407	/* WaDisable_RenderCache_OperationalFlush:ilk */
		6408	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6409
4104	Serge	6410	g4x_disable_trickle_feed(dev);
		6411
3243	Serge	6412	ibx_init_clock_gating(dev);
3031	serge	6413	}
		6414
3243	Serge	6415	static void cpt_init_clock_gating(struct drm_device *dev)
		6416	{
		6417	struct drm_i915_private *dev_priv = dev->dev_private;
		6418	int pipe;
3746	Serge	6419	uint32_t val;
3243	Serge	6420
		6421	/*
		6422	* On Ibex Peak and Cougar Point, we need to disable clock
		6423	* gating for the panel power sequencer or it will fail to
		6424	* start up when no ports are active.
		6425	*/
4280	Serge	6426	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		6427	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		6428	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	6429	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		6430	DPLS_EDP_PPS_FIX_DIS);
		6431	/* The below fixes the weird display corruption, a few pixels shifted
		6432	* downward, on (only) LVDS of some HP laptops with IVY.
		6433	*/
5354	serge	6434	for_each_pipe(dev_priv, pipe) {
3746	Serge	6435	val = I915_READ(TRANS_CHICKEN2(pipe));
		6436	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		6437	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	6438	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	6439	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		6440	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		6441	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		6442	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		6443	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		6444	}
3243	Serge	6445	/* WADP0ClockGatingDisable */
5354	serge	6446	for_each_pipe(dev_priv, pipe) {
3243	Serge	6447	I915_WRITE(TRANS_CHICKEN1(pipe),
		6448	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		6449	}
		6450	}
		6451
3480	Serge	6452	static void gen6_check_mch_setup(struct drm_device *dev)
		6453	{
		6454	struct drm_i915_private *dev_priv = dev->dev_private;
		6455	uint32_t tmp;
		6456
		6457	tmp = I915_READ(MCH_SSKPD);
5060	serge	6458	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		6459	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		6460	tmp);
3480	Serge	6461	}
		6462
3031	serge	6463	static void gen6_init_clock_gating(struct drm_device *dev)
		6464	{
		6465	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6466	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6467
3243	Serge	6468	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	6469
		6470	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6471	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6472	ILK_ELPIN_409_SELECT);
		6473
4104	Serge	6474	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	6475	I915_WRITE(_3D_CHICKEN,
		6476	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		6477
5060	serge	6478	/* WaDisable_RenderCache_OperationalFlush:snb */
		6479	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6480
		6481	/*
		6482	* BSpec recoomends 8x4 when MSAA is used,
		6483	* however in practice 16x4 seems fastest.
		6484	*
		6485	* Note that PS/WM thread counts depend on the WIZ hashing
		6486	* disable bit, which we don't touch here, but it's good
		6487	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6488	*/
		6489	I915_WRITE(GEN6_GT_MODE,
5354	serge	6490	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6491
4560	Serge	6492	ilk_init_lp_watermarks(dev);
3031	serge	6493
		6494	I915_WRITE(CACHE_MODE_0,
		6495	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		6496
		6497	I915_WRITE(GEN6_UCGCTL1,
		6498	I915_READ(GEN6_UCGCTL1) \|
		6499	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		6500	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6501
		6502	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		6503	* gating disable must be set. Failure to set it results in
		6504	* flickering pixels due to Z write ordering failures after
		6505	* some amount of runtime in the Mesa "fire" demo, and Unigine
		6506	* Sanctuary and Tropics, and apparently anything else with
		6507	* alpha test or pixel discard.
		6508	*
		6509	* According to the spec, bit 11 (RCCUNIT) must also be set,
		6510	* but we didn't debug actual testcases to find it out.
		6511	*
5060	serge	6512	* WaDisableRCCUnitClockGating:snb
		6513	* WaDisableRCPBUnitClockGating:snb
3031	serge	6514	*/
		6515	I915_WRITE(GEN6_UCGCTL2,
		6516	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		6517	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		6518
5060	serge	6519	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		6520	I915_WRITE(_3D_CHICKEN3,
		6521	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	6522
		6523	/*
5060	serge	6524	* Bspec says:
		6525	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		6526	* 3DSTATE_SF number of SF output attributes is more than 16."
		6527	*/
		6528	I915_WRITE(_3D_CHICKEN3,
		6529	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		6530
		6531	/*
3031	serge	6532	* According to the spec the following bits should be
		6533	* set in order to enable memory self-refresh and fbc:
		6534	* The bit21 and bit22 of 0x42000
		6535	* The bit21 and bit22 of 0x42004
		6536	* The bit5 and bit7 of 0x42020
		6537	* The bit14 of 0x70180
		6538	* The bit14 of 0x71180
4104	Serge	6539	*
		6540	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	6541	*/
		6542	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6543	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6544	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		6545	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6546	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6547	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	6548	I915_WRITE(ILK_DSPCLK_GATE_D,
		6549	I915_READ(ILK_DSPCLK_GATE_D) \|
		6550	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		6551	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	6552
4104	Serge	6553	g4x_disable_trickle_feed(dev);
3031	serge	6554
3243	Serge	6555	cpt_init_clock_gating(dev);
3480	Serge	6556
		6557	gen6_check_mch_setup(dev);
3031	serge	6558	}
		6559
		6560	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		6561	{
		6562	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		6563
5060	serge	6564	/*
		6565	* WaVSThreadDispatchOverride:ivb,vlv
		6566	*
		6567	* This actually overrides the dispatch
		6568	* mode for all thread types.
		6569	*/
3031	serge	6570	reg &= ~GEN7_FF_SCHED_MASK;
		6571	reg \|= GEN7_FF_TS_SCHED_HW;
		6572	reg \|= GEN7_FF_VS_SCHED_HW;
		6573	reg \|= GEN7_FF_DS_SCHED_HW;
		6574
		6575	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		6576	}
		6577
3243	Serge	6578	static void lpt_init_clock_gating(struct drm_device *dev)
		6579	{
		6580	struct drm_i915_private *dev_priv = dev->dev_private;
		6581
		6582	/*
		6583	* TODO: this bit should only be enabled when really needed, then
		6584	* disabled when not needed anymore in order to save power.
		6585	*/
6084	serge	6586	if (HAS_PCH_LPT_LP(dev))
3243	Serge	6587	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		6588	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		6589	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	6590
		6591	/* WADPOClockGatingDisable:hsw */
6084	serge	6592	I915_WRITE(TRANS_CHICKEN1(PIPE_A),
		6593	I915_READ(TRANS_CHICKEN1(PIPE_A)) \|
4104	Serge	6594	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	6595	}
		6596
4104	Serge	6597	static void lpt_suspend_hw(struct drm_device *dev)
		6598	{
		6599	struct drm_i915_private *dev_priv = dev->dev_private;
		6600
6084	serge	6601	if (HAS_PCH_LPT_LP(dev)) {
4104	Serge	6602	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		6603
		6604	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		6605	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		6606	}
		6607	}
		6608
5354	serge	6609	static void broadwell_init_clock_gating(struct drm_device *dev)
3031	serge	6610	{
		6611	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	6612	enum pipe pipe;
6084	serge	6613	uint32_t misccpctl;
3031	serge	6614
6084	serge	6615	ilk_init_lp_watermarks(dev);
3031	serge	6616
4560	Serge	6617	/* WaSwitchSolVfFArbitrationPriority:bdw */
		6618	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6619
		6620	/* WaPsrDPAMaskVBlankInSRD:bdw */
		6621	I915_WRITE(CHICKEN_PAR1_1,
		6622	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		6623
		6624	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5354	serge	6625	for_each_pipe(dev_priv, pipe) {
5060	serge	6626	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		6627	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		6628	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	6629	}
		6630
		6631	/* WaVSRefCountFullforceMissDisable:bdw */
		6632	/* WaDSRefCountFullforceMissDisable:bdw */
		6633	I915_WRITE(GEN7_FF_THREAD_MODE,
		6634	I915_READ(GEN7_FF_THREAD_MODE) &
		6635	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	6636
		6637	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6638	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6639
		6640	/* WaDisableSDEUnitClockGating:bdw */
		6641	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6642	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		6643
6084	serge	6644	/*
		6645	* WaProgramL3SqcReg1Default:bdw
		6646	* WaTempDisableDOPClkGating:bdw
		6647	*/
		6648	misccpctl = I915_READ(GEN7_MISCCPCTL);
		6649	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		6650	I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6660	serge	6651	/*
		6652	* Wait at least 100 clocks before re-enabling clock gating. See
		6653	* the definition of L3SQCREG1 in BSpec.
		6654	*/
		6655	POSTING_READ(GEN8_L3SQCREG1);
		6656	udelay(1);
6084	serge	6657	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		6658
		6659	/*
		6660	* WaGttCachingOffByDefault:bdw
		6661	* GTT cache may not work with big pages, so if those
		6662	* are ever enabled GTT cache may need to be disabled.
		6663	*/
		6664	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
		6665
5354	serge	6666	lpt_init_clock_gating(dev);
4560	Serge	6667	}
		6668
		6669	static void haswell_init_clock_gating(struct drm_device *dev)
		6670	{
		6671	struct drm_i915_private *dev_priv = dev->dev_private;
		6672
		6673	ilk_init_lp_watermarks(dev);
		6674
4104	Serge	6675	/* L3 caching of data atomics doesn't work -- disable it. */
		6676	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		6677	I915_WRITE(HSW_ROW_CHICKEN3,
		6678	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		6679
		6680	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	6681	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6682	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6683	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6684
4104	Serge	6685	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	6686	I915_WRITE(GEN7_FF_THREAD_MODE,
		6687	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	6688
5060	serge	6689	/* WaDisable_RenderCache_OperationalFlush:hsw */
		6690	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6691
		6692	/* enable HiZ Raw Stall Optimization */
		6693	I915_WRITE(CACHE_MODE_0_GEN7,
		6694	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6695
4104	Serge	6696	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	6697	I915_WRITE(CACHE_MODE_1,
		6698	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6699
5060	serge	6700	/*
		6701	* BSpec recommends 8x4 when MSAA is used,
		6702	* however in practice 16x4 seems fastest.
		6703	*
		6704	* Note that PS/WM thread counts depend on the WIZ hashing
		6705	* disable bit, which we don't touch here, but it's good
		6706	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6707	*/
		6708	I915_WRITE(GEN7_GT_MODE,
5354	serge	6709	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6710
6084	serge	6711	/* WaSampleCChickenBitEnable:hsw */
		6712	I915_WRITE(HALF_SLICE_CHICKEN3,
		6713	_MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
		6714
4104	Serge	6715	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	6716	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6717
4104	Serge	6718	/* WaRsPkgCStateDisplayPMReq:hsw */
		6719	I915_WRITE(CHICKEN_PAR1_1,
		6720	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	6721
3243	Serge	6722	lpt_init_clock_gating(dev);
3031	serge	6723	}
		6724
		6725	static void ivybridge_init_clock_gating(struct drm_device *dev)
		6726	{
		6727	struct drm_i915_private *dev_priv = dev->dev_private;
		6728	uint32_t snpcr;
		6729
4560	Serge	6730	ilk_init_lp_watermarks(dev);
3031	serge	6731
3243	Serge	6732	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6733
4104	Serge	6734	/* WaDisableEarlyCull:ivb */
3243	Serge	6735	I915_WRITE(_3D_CHICKEN3,
		6736	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6737
4104	Serge	6738	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	6739	I915_WRITE(IVB_CHICKEN3,
		6740	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6741	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6742
4104	Serge	6743	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	6744	if (IS_IVB_GT1(dev))
		6745	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		6746	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		6747
5060	serge	6748	/* WaDisable_RenderCache_OperationalFlush:ivb */
		6749	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6750
4104	Serge	6751	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	6752	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		6753	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		6754
4104	Serge	6755	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	6756	I915_WRITE(GEN7_L3CNTLREG1,
		6757	GEN7_WA_FOR_GEN7_L3_CONTROL);
		6758	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6084	serge	6759	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	6760	if (IS_IVB_GT1(dev))
		6761	I915_WRITE(GEN7_ROW_CHICKEN2,
		6762	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6763	else {
		6764	/* must write both registers */
		6765	I915_WRITE(GEN7_ROW_CHICKEN2,
		6766	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	6767	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		6768	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6769	}
3031	serge	6770
4104	Serge	6771	/* WaForceL3Serialization:ivb */
3243	Serge	6772	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6773	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6774
5060	serge	6775	/*
3031	serge	6776	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6777	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	6778	*/
		6779	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6780	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6781
4104	Serge	6782	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	6783	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6784	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6785	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6786
4104	Serge	6787	g4x_disable_trickle_feed(dev);
3031	serge	6788
		6789	gen7_setup_fixed_func_scheduler(dev_priv);
		6790
5060	serge	6791	if (0) { /* causes HiZ corruption on ivb:gt1 */
		6792	/* enable HiZ Raw Stall Optimization */
		6793	I915_WRITE(CACHE_MODE_0_GEN7,
		6794	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6795	}
		6796
4104	Serge	6797	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	6798	I915_WRITE(CACHE_MODE_1,
		6799	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6800
5060	serge	6801	/*
		6802	* BSpec recommends 8x4 when MSAA is used,
		6803	* however in practice 16x4 seems fastest.
		6804	*
		6805	* Note that PS/WM thread counts depend on the WIZ hashing
		6806	* disable bit, which we don't touch here, but it's good
		6807	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6808	*/
		6809	I915_WRITE(GEN7_GT_MODE,
5354	serge	6810	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6811
3031	serge	6812	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		6813	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		6814	snpcr \|= GEN6_MBC_SNPCR_MED;
		6815	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	6816
3746	Serge	6817	if (!HAS_PCH_NOP(dev))
6084	serge	6818	cpt_init_clock_gating(dev);
3480	Serge	6819
		6820	gen6_check_mch_setup(dev);
3031	serge	6821	}
		6822
6084	serge	6823	static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
		6824	{
6937	serge	6825	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6084	serge	6826
		6827	/*
		6828	* Disable trickle feed and enable pnd deadline calculation
		6829	*/
		6830	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		6831	I915_WRITE(CBR1_VLV, 0);
		6832	}
		6833
3031	serge	6834	static void valleyview_init_clock_gating(struct drm_device *dev)
		6835	{
		6836	struct drm_i915_private *dev_priv = dev->dev_private;
		6837
6084	serge	6838	vlv_init_display_clock_gating(dev_priv);
3031	serge	6839
4104	Serge	6840	/* WaDisableEarlyCull:vlv */
3243	Serge	6841	I915_WRITE(_3D_CHICKEN3,
		6842	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6843
4104	Serge	6844	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	6845	I915_WRITE(IVB_CHICKEN3,
		6846	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6847	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6848
5060	serge	6849	/* WaPsdDispatchEnable:vlv */
4104	Serge	6850	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	6851	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	6852	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		6853	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	6854
5060	serge	6855	/* WaDisable_RenderCache_OperationalFlush:vlv */
		6856	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6857
4104	Serge	6858	/* WaForceL3Serialization:vlv */
3243	Serge	6859	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6860	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6861
4104	Serge	6862	/* WaDisableDopClockGating:vlv */
3243	Serge	6863	I915_WRITE(GEN7_ROW_CHICKEN2,
		6864	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		6865
4104	Serge	6866	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	6867	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6868	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6869	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6870
5060	serge	6871	gen7_setup_fixed_func_scheduler(dev_priv);
		6872
		6873	/*
3031	serge	6874	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6875	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	6876	*/
		6877	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6878	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6879
5060	serge	6880	/* WaDisableL3Bank2xClockGate:vlv
		6881	* Disabling L3 clock gating- MMIO 940c[25] = 1
		6882	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		6883	I915_WRITE(GEN7_UCGCTL4,
		6884	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	6885
5060	serge	6886	/*
		6887	* BSpec says this must be set, even though
		6888	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		6889	*/
3031	serge	6890	I915_WRITE(CACHE_MODE_1,
		6891	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6892
		6893	/*
6084	serge	6894	* BSpec recommends 8x4 when MSAA is used,
		6895	* however in practice 16x4 seems fastest.
		6896	*
		6897	* Note that PS/WM thread counts depend on the WIZ hashing
		6898	* disable bit, which we don't touch here, but it's good
		6899	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6900	*/
		6901	I915_WRITE(GEN7_GT_MODE,
		6902	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
		6903
		6904	/*
5060	serge	6905	* WaIncreaseL3CreditsForVLVB0:vlv
		6906	* This is the hardware default actually.
		6907	*/
		6908	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		6909
		6910	/*
4104	Serge	6911	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	6912	* Disable clock gating on th GCFG unit to prevent a delay
		6913	* in the reporting of vblank events.
		6914	*/
5060	serge	6915	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		6916	}
3746	Serge	6917
5060	serge	6918	static void cherryview_init_clock_gating(struct drm_device *dev)
		6919	{
		6920	struct drm_i915_private *dev_priv = dev->dev_private;
		6921
6084	serge	6922	vlv_init_display_clock_gating(dev_priv);
5060	serge	6923
		6924	/* WaVSRefCountFullforceMissDisable:chv */
		6925	/* WaDSRefCountFullforceMissDisable:chv */
		6926	I915_WRITE(GEN7_FF_THREAD_MODE,
		6927	I915_READ(GEN7_FF_THREAD_MODE) &
		6928	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		6929
		6930	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		6931	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6932	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6933
		6934	/* WaDisableCSUnitClockGating:chv */
		6935	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		6936	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6937
		6938	/* WaDisableSDEUnitClockGating:chv */
		6939	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6940	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6084	serge	6941
		6942	/*
		6943	* GTT cache may not work with big pages, so if those
		6944	* are ever enabled GTT cache may need to be disabled.
		6945	*/
		6946	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
3031	serge	6947	}
		6948
		6949	static void g4x_init_clock_gating(struct drm_device *dev)
		6950	{
		6951	struct drm_i915_private *dev_priv = dev->dev_private;
		6952	uint32_t dspclk_gate;
		6953
		6954	I915_WRITE(RENCLK_GATE_D1, 0);
		6955	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		6956	GS_UNIT_CLOCK_GATE_DISABLE \|
		6957	CL_UNIT_CLOCK_GATE_DISABLE);
		6958	I915_WRITE(RAMCLK_GATE_D, 0);
		6959	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		6960	OVRUNIT_CLOCK_GATE_DISABLE \|
		6961	OVCUNIT_CLOCK_GATE_DISABLE;
		6962	if (IS_GM45(dev))
		6963	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		6964	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	6965
		6966	/* WaDisableRenderCachePipelinedFlush */
		6967	I915_WRITE(CACHE_MODE_0,
		6968	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	6969
5060	serge	6970	/* WaDisable_RenderCache_OperationalFlush:g4x */
		6971	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6972
4104	Serge	6973	g4x_disable_trickle_feed(dev);
3031	serge	6974	}
		6975
		6976	static void crestline_init_clock_gating(struct drm_device *dev)
		6977	{
		6978	struct drm_i915_private *dev_priv = dev->dev_private;
		6979
		6980	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		6981	I915_WRITE(RENCLK_GATE_D2, 0);
		6982	I915_WRITE(DSPCLK_GATE_D, 0);
		6983	I915_WRITE(RAMCLK_GATE_D, 0);
		6984	I915_WRITE16(DEUC, 0);
4104	Serge	6985	I915_WRITE(MI_ARB_STATE,
		6986	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6987
		6988	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6989	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6990	}
		6991
		6992	static void broadwater_init_clock_gating(struct drm_device *dev)
		6993	{
		6994	struct drm_i915_private *dev_priv = dev->dev_private;
		6995
		6996	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		6997	I965_RCC_CLOCK_GATE_DISABLE \|
		6998	I965_RCPB_CLOCK_GATE_DISABLE \|
		6999	I965_ISC_CLOCK_GATE_DISABLE \|
		7000	I965_FBC_CLOCK_GATE_DISABLE);
		7001	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	7002	I915_WRITE(MI_ARB_STATE,
		7003	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	7004
		7005	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		7006	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	7007	}
		7008
		7009	static void gen3_init_clock_gating(struct drm_device *dev)
		7010	{
		7011	struct drm_i915_private *dev_priv = dev->dev_private;
		7012	u32 dstate = I915_READ(D_STATE);
		7013
		7014	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		7015	DSTATE_DOT_CLOCK_GATING;
		7016	I915_WRITE(D_STATE, dstate);
		7017
		7018	if (IS_PINEVIEW(dev))
		7019	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		7020
		7021	/* IIR "flip pending" means done if this bit is set */
		7022	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	7023
		7024	/* interrupts should cause a wake up from C3 */
		7025	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		7026
		7027	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		7028	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5354	serge	7029
		7030	I915_WRITE(MI_ARB_STATE,
		7031	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7032	}
		7033
		7034	static void i85x_init_clock_gating(struct drm_device *dev)
		7035	{
		7036	struct drm_i915_private *dev_priv = dev->dev_private;
		7037
		7038	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	7039
		7040	/* interrupts should cause a wake up from C3 */
		7041	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		7042	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
5354	serge	7043
		7044	I915_WRITE(MEM_MODE,
		7045	_MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7046	}
		7047
		7048	static void i830_init_clock_gating(struct drm_device *dev)
		7049	{
		7050	struct drm_i915_private *dev_priv = dev->dev_private;
		7051
		7052	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5354	serge	7053
		7054	I915_WRITE(MEM_MODE,
		7055	_MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) \|
		7056	_MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
3031	serge	7057	}
		7058
		7059	void intel_init_clock_gating(struct drm_device *dev)
		7060	{
		7061	struct drm_i915_private *dev_priv = dev->dev_private;
		7062
6084	serge	7063	if (dev_priv->display.init_clock_gating)
		7064	dev_priv->display.init_clock_gating(dev);
3031	serge	7065	}
		7066
4104	Serge	7067	void intel_suspend_hw(struct drm_device *dev)
		7068	{
		7069	if (HAS_PCH_LPT(dev))
		7070	lpt_suspend_hw(dev);
		7071	}
		7072
5354	serge	7073	/* Set up chip specific power management-related functions */
		7074	void intel_init_pm(struct drm_device *dev)
		7075	{
		7076	struct drm_i915_private *dev_priv = dev->dev_private;
		7077
6084	serge	7078	intel_fbc_init(dev_priv);
5354	serge	7079
3031	serge	7080	/* For cxsr */
		7081	if (IS_PINEVIEW(dev))
		7082	i915_pineview_get_mem_freq(dev);
		7083	else if (IS_GEN5(dev))
		7084	i915_ironlake_get_mem_freq(dev);
		7085
		7086	/* For FIFO watermark updates */
5354	serge	7087	if (INTEL_INFO(dev)->gen >= 9) {
		7088	skl_setup_wm_latency(dev);
		7089
6084	serge	7090	if (IS_BROXTON(dev))
		7091	dev_priv->display.init_clock_gating =
		7092	bxt_init_clock_gating;
5354	serge	7093	dev_priv->display.update_wm = skl_update_wm;
		7094	} else if (HAS_PCH_SPLIT(dev)) {
5060	serge	7095	ilk_setup_wm_latency(dev);
4104	Serge	7096
4560	Serge	7097	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		7098	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		7099	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		7100	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		7101	dev_priv->display.update_wm = ilk_update_wm;
6937	serge	7102	dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7144	serge	7103	dev_priv->display.program_watermarks = ilk_program_watermarks;
6084	serge	7104	} else {
		7105	DRM_DEBUG_KMS("Failed to read display plane latency. "
		7106	"Disable CxSR\n");
		7107	}
4560	Serge	7108
		7109	if (IS_GEN5(dev))
		7110	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		7111	else if (IS_GEN6(dev))
3031	serge	7112	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	7113	else if (IS_IVYBRIDGE(dev))
3031	serge	7114	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	7115	else if (IS_HASWELL(dev))
3031	serge	7116	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	7117	else if (INTEL_INFO(dev)->gen == 8)
5354	serge	7118	dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
5060	serge	7119	} else if (IS_CHERRYVIEW(dev)) {
6084	serge	7120	vlv_setup_wm_latency(dev);
		7121
		7122	dev_priv->display.update_wm = vlv_update_wm;
5060	serge	7123	dev_priv->display.init_clock_gating =
		7124	cherryview_init_clock_gating;
3031	serge	7125	} else if (IS_VALLEYVIEW(dev)) {
6084	serge	7126	vlv_setup_wm_latency(dev);
		7127
		7128	dev_priv->display.update_wm = vlv_update_wm;
3031	serge	7129	dev_priv->display.init_clock_gating =
		7130	valleyview_init_clock_gating;
		7131	} else if (IS_PINEVIEW(dev)) {
		7132	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		7133	dev_priv->is_ddr3,
		7134	dev_priv->fsb_freq,
		7135	dev_priv->mem_freq)) {
		7136	DRM_INFO("failed to find known CxSR latency "
		7137	"(found ddr%s fsb freq %d, mem freq %d), "
		7138	"disabling CxSR\n",
		7139	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		7140	dev_priv->fsb_freq, dev_priv->mem_freq);
		7141	/* Disable CxSR and never update its watermark again */
5060	serge	7142	intel_set_memory_cxsr(dev_priv, false);
3031	serge	7143	dev_priv->display.update_wm = NULL;
		7144	} else
		7145	dev_priv->display.update_wm = pineview_update_wm;
		7146	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		7147	} else if (IS_G4X(dev)) {
		7148	dev_priv->display.update_wm = g4x_update_wm;
		7149	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		7150	} else if (IS_GEN4(dev)) {
		7151	dev_priv->display.update_wm = i965_update_wm;
		7152	if (IS_CRESTLINE(dev))
		7153	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		7154	else if (IS_BROADWATER(dev))
		7155	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		7156	} else if (IS_GEN3(dev)) {
		7157	dev_priv->display.update_wm = i9xx_update_wm;
		7158	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		7159	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	7160	} else if (IS_GEN2(dev)) {
		7161	if (INTEL_INFO(dev)->num_pipes == 1) {
		7162	dev_priv->display.update_wm = i845_update_wm;
		7163	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		7164	} else {
		7165	dev_priv->display.update_wm = i9xx_update_wm;
6084	serge	7166	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	7167	}
		7168
		7169	if (IS_I85X(dev) \|\| IS_I865G(dev))
6084	serge	7170	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	7171	else
		7172	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	7173	} else {
4560	Serge	7174	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	7175	}
		7176	}
		7177
5354	serge	7178	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u32 mbox, u32 val)
3243	Serge	7179	{
		7180	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	7181
3243	Serge	7182	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7183	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		7184	return -EAGAIN;
		7185	}
3031	serge	7186
3243	Serge	7187	I915_WRITE(GEN6_PCODE_DATA, *val);
5354	serge	7188	I915_WRITE(GEN6_PCODE_DATA1, 0);
3243	Serge	7189	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7190
		7191	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7192	500)) {
		7193	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		7194	return -ETIMEDOUT;
6084	serge	7195	}
3243	Serge	7196
		7197	*val = I915_READ(GEN6_PCODE_DATA);
		7198	I915_WRITE(GEN6_PCODE_DATA, 0);
		7199
		7200	return 0;
		7201	}
		7202
5354	serge	7203	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
3243	Serge	7204	{
		7205	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		7206
		7207	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7208	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		7209	return -EAGAIN;
6084	serge	7210	}
3243	Serge	7211
		7212	I915_WRITE(GEN6_PCODE_DATA, val);
		7213	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7214
		7215	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7216	500)) {
		7217	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		7218	return -ETIMEDOUT;
3031	serge	7219	}
3243	Serge	7220
		7221	I915_WRITE(GEN6_PCODE_DATA, 0);
		7222
		7223	return 0;
3031	serge	7224	}
3746	Serge	7225
5354	serge	7226	static int vlv_gpu_freq_div(unsigned int czclk_freq)
3746	Serge	7227	{
5354	serge	7228	switch (czclk_freq) {
		7229	case 200:
		7230	return 10;
		7231	case 267:
		7232	return 12;
		7233	case 320:
		7234	case 333:
		7235	return 16;
		7236	case 400:
		7237	return 20;
4104	Serge	7238	default:
		7239	return -1;
		7240	}
5354	serge	7241	}
3746	Serge	7242
5354	serge	7243	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7244	{
6084	serge	7245	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5354	serge	7246
		7247	div = vlv_gpu_freq_div(czclk_freq);
		7248	if (div < 0)
		7249	return div;
		7250
		7251	return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
4104	Serge	7252	}
3746	Serge	7253
5060	serge	7254	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	7255	{
6084	serge	7256	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
3746	Serge	7257
5354	serge	7258	mul = vlv_gpu_freq_div(czclk_freq);
		7259	if (mul < 0)
		7260	return mul;
3746	Serge	7261
5354	serge	7262	return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
3746	Serge	7263	}
		7264
5060	serge	7265	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7266	{
6084	serge	7267	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7268
7144	serge	7269	div = vlv_gpu_freq_div(czclk_freq);
5354	serge	7270	if (div < 0)
		7271	return div;
7144	serge	7272	div /= 2;
5060	serge	7273
5354	serge	7274	return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
5060	serge	7275	}
		7276
		7277	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7278	{
6084	serge	7279	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7280
7144	serge	7281	mul = vlv_gpu_freq_div(czclk_freq);
5354	serge	7282	if (mul < 0)
		7283	return mul;
7144	serge	7284	mul /= 2;
5060	serge	7285
5354	serge	7286	/* CHV needs even values */
		7287	return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
5060	serge	7288	}
		7289
6084	serge	7290	int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
5060	serge	7291	{
6084	serge	7292	if (IS_GEN9(dev_priv->dev))
		7293	return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
		7294	GEN9_FREQ_SCALER);
		7295	else if (IS_CHERRYVIEW(dev_priv->dev))
		7296	return chv_gpu_freq(dev_priv, val);
		7297	else if (IS_VALLEYVIEW(dev_priv->dev))
		7298	return byt_gpu_freq(dev_priv, val);
		7299	else
		7300	return val * GT_FREQUENCY_MULTIPLIER;
		7301	}
5060	serge	7302
6084	serge	7303	int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7304	{
		7305	if (IS_GEN9(dev_priv->dev))
		7306	return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
		7307	GT_FREQUENCY_MULTIPLIER);
		7308	else if (IS_CHERRYVIEW(dev_priv->dev))
		7309	return chv_freq_opcode(dev_priv, val);
5060	serge	7310	else if (IS_VALLEYVIEW(dev_priv->dev))
6084	serge	7311	return byt_freq_opcode(dev_priv, val);
		7312	else
		7313	return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
		7314	}
5060	serge	7315
6084	serge	7316	struct request_boost {
		7317	struct work_struct work;
		7318	struct drm_i915_gem_request *req;
		7319	};
		7320
		7321	static void __intel_rps_boost_work(struct work_struct *work)
		7322	{
		7323	struct request_boost *boost = container_of(work, struct request_boost, work);
		7324	struct drm_i915_gem_request *req = boost->req;
		7325
		7326	if (!i915_gem_request_completed(req, true))
		7327	gen6_rps_boost(to_i915(req->ring->dev), NULL,
		7328	req->emitted_jiffies);
		7329
		7330	i915_gem_request_unreference__unlocked(req);
		7331	kfree(boost);
5060	serge	7332	}
		7333
6084	serge	7334	void intel_queue_rps_boost_for_request(struct drm_device *dev,
		7335	struct drm_i915_gem_request *req)
5060	serge	7336	{
6084	serge	7337	struct request_boost *boost;
5060	serge	7338
6084	serge	7339	if (req == NULL \|\| INTEL_INFO(dev)->gen < 6)
		7340	return;
5060	serge	7341
6084	serge	7342	if (i915_gem_request_completed(req, true))
		7343	return;
		7344
		7345	boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
		7346	if (boost == NULL)
		7347	return;
		7348
		7349	i915_gem_request_reference(req);
		7350	boost->req = req;
		7351
		7352	INIT_WORK(&boost->work, __intel_rps_boost_work);
		7353	queue_work(to_i915(dev)->wq, &boost->work);
5060	serge	7354	}
		7355
4560	Serge	7356	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7357	{
4104	Serge	7358	struct drm_i915_private *dev_priv = dev->dev_private;
		7359
4560	Serge	7360	mutex_init(&dev_priv->rps.hw_lock);
6084	serge	7361	spin_lock_init(&dev_priv->rps.client_lock);
4560	Serge	7362
4104	Serge	7363	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7364	intel_gen6_powersave_work);
6084	serge	7365	INIT_LIST_HEAD(&dev_priv->rps.clients);
		7366	INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
		7367	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
5060	serge	7368
		7369	dev_priv->pm.suspended = false;
6937	serge	7370	atomic_set(&dev_priv->pm.wakeref_count, 0);
		7371	atomic_set(&dev_priv->pm.atomic_seq, 0);
3746	Serge	7372	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 7144