WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	//#include "../../../platform/x86/intel_ips.h"
		32	#include
		33
4560	Serge	34
3031	serge	35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	void getrawmonotonic(struct timespec *ts);
		38
4560	Serge	39	/**
		40	* RC6 is a special power stage which allows the GPU to enter an very
		41	* low-voltage mode when idle, using down to 0V while at this stage. This
		42	* stage is entered automatically when the GPU is idle when RC6 support is
		43	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		44	*
		45	* There are different RC6 modes available in Intel GPU, which differentiate
		46	* among each other with the latency required to enter and leave RC6 and
		47	* voltage consumed by the GPU in different states.
		48	*
		49	* The combination of the following flags define which states GPU is allowed
		50	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		51	* RC6pp is deepest RC6. Their support by hardware varies according to the
		52	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		53	* which brings the most power savings; deeper states save more power, but
		54	* require higher latency to switch to and wake up.
		55	*/
		56	#define INTEL_RC6_ENABLE (1<<0)
		57	#define INTEL_RC6p_ENABLE (1<<1)
		58	#define INTEL_RC6pp_ENABLE (1<<2)
		59
6084	serge	60	static void bxt_init_clock_gating(struct drm_device *dev)
5354	serge	61	{
		62	struct drm_i915_private *dev_priv = dev->dev_private;
		63
6084	serge	64	/* WaDisableSDEUnitClockGating:bxt */
5354	serge	65	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		66	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		67
		68	/*
6084	serge	69	* FIXME:
		70	* GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
5354	serge	71	*/
6084	serge	72	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		73	GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
5354	serge	74	}
		75
3031	serge	76	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		77	{
5060	serge	78	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	79	u32 tmp;
		80
		81	tmp = I915_READ(CLKCFG);
		82
		83	switch (tmp & CLKCFG_FSB_MASK) {
		84	case CLKCFG_FSB_533:
		85	dev_priv->fsb_freq = 533; /* 1334 /
		86	break;
		87	case CLKCFG_FSB_800:
		88	dev_priv->fsb_freq = 800; /* 2004 /
		89	break;
		90	case CLKCFG_FSB_667:
		91	dev_priv->fsb_freq = 667; /* 1674 /
		92	break;
		93	case CLKCFG_FSB_400:
		94	dev_priv->fsb_freq = 400; /* 1004 /
		95	break;
		96	}
		97
		98	switch (tmp & CLKCFG_MEM_MASK) {
		99	case CLKCFG_MEM_533:
		100	dev_priv->mem_freq = 533;
		101	break;
		102	case CLKCFG_MEM_667:
		103	dev_priv->mem_freq = 667;
		104	break;
		105	case CLKCFG_MEM_800:
		106	dev_priv->mem_freq = 800;
		107	break;
		108	}
		109
		110	/* detect pineview DDR3 setting */
		111	tmp = I915_READ(CSHRDDR3CTL);
		112	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		113	}
		114
		115	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		116	{
5060	serge	117	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	118	u16 ddrpll, csipll;
		119
		120	ddrpll = I915_READ16(DDRMPLL1);
		121	csipll = I915_READ16(CSIPLL0);
		122
		123	switch (ddrpll & 0xff) {
		124	case 0xc:
		125	dev_priv->mem_freq = 800;
		126	break;
		127	case 0x10:
		128	dev_priv->mem_freq = 1066;
		129	break;
		130	case 0x14:
		131	dev_priv->mem_freq = 1333;
		132	break;
		133	case 0x18:
		134	dev_priv->mem_freq = 1600;
		135	break;
		136	default:
		137	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		138	ddrpll & 0xff);
		139	dev_priv->mem_freq = 0;
		140	break;
		141	}
		142
		143	dev_priv->ips.r_t = dev_priv->mem_freq;
		144
		145	switch (csipll & 0x3ff) {
		146	case 0x00c:
		147	dev_priv->fsb_freq = 3200;
		148	break;
		149	case 0x00e:
		150	dev_priv->fsb_freq = 3733;
		151	break;
		152	case 0x010:
		153	dev_priv->fsb_freq = 4266;
		154	break;
		155	case 0x012:
		156	dev_priv->fsb_freq = 4800;
		157	break;
		158	case 0x014:
		159	dev_priv->fsb_freq = 5333;
		160	break;
		161	case 0x016:
		162	dev_priv->fsb_freq = 5866;
		163	break;
		164	case 0x018:
		165	dev_priv->fsb_freq = 6400;
		166	break;
		167	default:
		168	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		169	csipll & 0x3ff);
		170	dev_priv->fsb_freq = 0;
		171	break;
		172	}
		173
		174	if (dev_priv->fsb_freq == 3200) {
		175	dev_priv->ips.c_m = 0;
		176	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		177	dev_priv->ips.c_m = 1;
		178	} else {
		179	dev_priv->ips.c_m = 2;
		180	}
		181	}
		182
		183	static const struct cxsr_latency cxsr_latency_table[] = {
		184	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		185	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		186	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		187	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		188	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		189
		190	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		191	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		192	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		193	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		194	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		195
		196	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		197	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		198	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		199	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		200	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		201
		202	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		203	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		204	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		205	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		206	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		207
		208	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		209	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		210	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		211	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		212	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		213
		214	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		215	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		216	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		217	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		218	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		219	};
		220
		221	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		222	int is_ddr3,
		223	int fsb,
		224	int mem)
		225	{
		226	const struct cxsr_latency *latency;
		227	int i;
		228
		229	if (fsb == 0 \|\| mem == 0)
		230	return NULL;
		231
		232	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		233	latency = &cxsr_latency_table[i];
		234	if (is_desktop == latency->is_desktop &&
		235	is_ddr3 == latency->is_ddr3 &&
		236	fsb == latency->fsb_freq && mem == latency->mem_freq)
		237	return latency;
		238	}
		239
		240	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		241
		242	return NULL;
		243	}
		244
6084	serge	245	static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
		246	{
		247	u32 val;
		248
		249	mutex_lock(&dev_priv->rps.hw_lock);
		250
		251	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		252	if (enable)
		253	val &= ~FORCE_DDR_HIGH_FREQ;
		254	else
		255	val \|= FORCE_DDR_HIGH_FREQ;
		256	val &= ~FORCE_DDR_LOW_FREQ;
		257	val \|= FORCE_DDR_FREQ_REQ_ACK;
		258	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		259
		260	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		261	FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
		262	DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
		263
		264	mutex_unlock(&dev_priv->rps.hw_lock);
		265	}
		266
		267	static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
		268	{
		269	u32 val;
		270
		271	mutex_lock(&dev_priv->rps.hw_lock);
		272
		273	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		274	if (enable)
		275	val \|= DSP_MAXFIFO_PM5_ENABLE;
		276	else
		277	val &= ~DSP_MAXFIFO_PM5_ENABLE;
		278	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
		279
		280	mutex_unlock(&dev_priv->rps.hw_lock);
		281	}
		282
		283	#define FW_WM(value, plane) \
		284	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
		285
5060	serge	286	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	287	{
5060	serge	288	struct drm_device *dev = dev_priv->dev;
		289	u32 val;
3031	serge	290
5060	serge	291	if (IS_VALLEYVIEW(dev)) {
		292	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
6084	serge	293	POSTING_READ(FW_BLC_SELF_VLV);
		294	dev_priv->wm.vlv.cxsr = enable;
5060	serge	295	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		296	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
6084	serge	297	POSTING_READ(FW_BLC_SELF);
5060	serge	298	} else if (IS_PINEVIEW(dev)) {
		299	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		300	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		301	I915_WRITE(DSPFW3, val);
6084	serge	302	POSTING_READ(DSPFW3);
5060	serge	303	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		304	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		305	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		306	I915_WRITE(FW_BLC_SELF, val);
6084	serge	307	POSTING_READ(FW_BLC_SELF);
5060	serge	308	} else if (IS_I915GM(dev)) {
		309	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		310	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		311	I915_WRITE(INSTPM, val);
6084	serge	312	POSTING_READ(INSTPM);
5060	serge	313	} else {
		314	return;
		315	}
		316
		317	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		318	enable ? "enabled" : "disabled");
3031	serge	319	}
		320
6084	serge	321
3031	serge	322	/*
		323	* Latency for FIFO fetches is dependent on several factors:
		324	* - memory configuration (speed, channels)
		325	* - chipset
		326	* - current MCH state
		327	* It can be fairly high in some situations, so here we assume a fairly
		328	* pessimal value. It's a tradeoff between extra memory fetches (if we
		329	* set this value too high, the FIFO will fetch frequently to stay full)
		330	* and power consumption (set it too low to save power and we might see
		331	* FIFO underruns and display "flicker").
		332	*
		333	* A value of 5us seems to be a good balance; safe for very low end
		334	* platforms but not overly aggressive on lower latency configs.
		335	*/
5354	serge	336	static const int pessimal_latency_ns = 5000;
3031	serge	337
6084	serge	338	#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
		339	((((dsparb) >> (lo_shift)) & 0xff) \| ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
		340
		341	static int vlv_get_fifo_size(struct drm_device *dev,
		342	enum pipe pipe, int plane)
		343	{
		344	struct drm_i915_private *dev_priv = dev->dev_private;
		345	int sprite0_start, sprite1_start, size;
		346
		347	switch (pipe) {
		348	uint32_t dsparb, dsparb2, dsparb3;
		349	case PIPE_A:
		350	dsparb = I915_READ(DSPARB);
		351	dsparb2 = I915_READ(DSPARB2);
		352	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		353	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		354	break;
		355	case PIPE_B:
		356	dsparb = I915_READ(DSPARB);
		357	dsparb2 = I915_READ(DSPARB2);
		358	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		359	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		360	break;
		361	case PIPE_C:
		362	dsparb2 = I915_READ(DSPARB2);
		363	dsparb3 = I915_READ(DSPARB3);
		364	sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		365	sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		366	break;
		367	default:
		368	return 0;
		369	}
		370
		371	switch (plane) {
		372	case 0:
		373	size = sprite0_start;
		374	break;
		375	case 1:
		376	size = sprite1_start - sprite0_start;
		377	break;
		378	case 2:
		379	size = 512 - 1 - sprite1_start;
		380	break;
		381	default:
		382	return 0;
		383	}
		384
		385	DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
		386	pipe_name(pipe), plane == 0 ? "primary" : "sprite",
		387	plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
		388	size);
		389
		390	return size;
		391	}
		392
3031	serge	393	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		394	{
		395	struct drm_i915_private *dev_priv = dev->dev_private;
		396	uint32_t dsparb = I915_READ(DSPARB);
		397	int size;
		398
		399	size = dsparb & 0x7f;
		400	if (plane)
		401	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		402
		403	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		404	plane ? "B" : "A", size);
		405
		406	return size;
		407	}
		408
4560	Serge	409	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	410	{
		411	struct drm_i915_private *dev_priv = dev->dev_private;
		412	uint32_t dsparb = I915_READ(DSPARB);
		413	int size;
		414
		415	size = dsparb & 0x1ff;
		416	if (plane)
		417	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		418	size >>= 1; /* Convert to cachelines */
		419
		420	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		421	plane ? "B" : "A", size);
		422
		423	return size;
		424	}
		425
		426	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		427	{
		428	struct drm_i915_private *dev_priv = dev->dev_private;
		429	uint32_t dsparb = I915_READ(DSPARB);
		430	int size;
		431
		432	size = dsparb & 0x7f;
		433	size >>= 2; /* Convert to cachelines */
		434
		435	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		436	plane ? "B" : "A",
		437	size);
		438
		439	return size;
		440	}
		441
		442	/* Pineview has different values for various configs */
		443	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	444	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		445	.max_wm = PINEVIEW_MAX_WM,
		446	.default_wm = PINEVIEW_DFT_WM,
		447	.guard_size = PINEVIEW_GUARD_WM,
		448	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	449	};
		450	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	451	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		452	.max_wm = PINEVIEW_MAX_WM,
		453	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		454	.guard_size = PINEVIEW_GUARD_WM,
		455	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	456	};
		457	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	458	.fifo_size = PINEVIEW_CURSOR_FIFO,
		459	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		460	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		461	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		462	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	463	};
		464	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	465	.fifo_size = PINEVIEW_CURSOR_FIFO,
		466	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		467	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		468	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		469	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	470	};
		471	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	472	.fifo_size = G4X_FIFO_SIZE,
		473	.max_wm = G4X_MAX_WM,
		474	.default_wm = G4X_MAX_WM,
		475	.guard_size = 2,
		476	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	477	};
		478	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	479	.fifo_size = I965_CURSOR_FIFO,
		480	.max_wm = I965_CURSOR_MAX_WM,
		481	.default_wm = I965_CURSOR_DFT_WM,
		482	.guard_size = 2,
		483	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	484	};
		485	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	486	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		487	.max_wm = VALLEYVIEW_MAX_WM,
		488	.default_wm = VALLEYVIEW_MAX_WM,
		489	.guard_size = 2,
		490	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	491	};
		492	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	493	.fifo_size = I965_CURSOR_FIFO,
		494	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		495	.default_wm = I965_CURSOR_DFT_WM,
		496	.guard_size = 2,
		497	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	498	};
		499	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	500	.fifo_size = I965_CURSOR_FIFO,
		501	.max_wm = I965_CURSOR_MAX_WM,
		502	.default_wm = I965_CURSOR_DFT_WM,
		503	.guard_size = 2,
		504	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	505	};
		506	static const struct intel_watermark_params i945_wm_info = {
5060	serge	507	.fifo_size = I945_FIFO_SIZE,
		508	.max_wm = I915_MAX_WM,
		509	.default_wm = 1,
		510	.guard_size = 2,
		511	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	512	};
		513	static const struct intel_watermark_params i915_wm_info = {
5060	serge	514	.fifo_size = I915_FIFO_SIZE,
		515	.max_wm = I915_MAX_WM,
		516	.default_wm = 1,
		517	.guard_size = 2,
		518	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	519	};
5354	serge	520	static const struct intel_watermark_params i830_a_wm_info = {
5060	serge	521	.fifo_size = I855GM_FIFO_SIZE,
		522	.max_wm = I915_MAX_WM,
		523	.default_wm = 1,
		524	.guard_size = 2,
		525	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	526	};
5354	serge	527	static const struct intel_watermark_params i830_bc_wm_info = {
		528	.fifo_size = I855GM_FIFO_SIZE,
		529	.max_wm = I915_MAX_WM/2,
		530	.default_wm = 1,
		531	.guard_size = 2,
		532	.cacheline_size = I830_FIFO_LINE_SIZE,
		533	};
4560	Serge	534	static const struct intel_watermark_params i845_wm_info = {
5060	serge	535	.fifo_size = I830_FIFO_SIZE,
		536	.max_wm = I915_MAX_WM,
		537	.default_wm = 1,
		538	.guard_size = 2,
		539	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	540	};
		541
		542	/**
		543	* intel_calculate_wm - calculate watermark level
		544	* @clock_in_khz: pixel clock
		545	* @wm: chip FIFO params
		546	* @pixel_size: display pixel size
		547	* @latency_ns: memory latency for the platform
		548	*
		549	* Calculate the watermark level (the level at which the display plane will
		550	* start fetching from memory again). Each chip has a different display
		551	* FIFO size and allocation, so the caller needs to figure that out and pass
		552	* in the correct intel_watermark_params structure.
		553	*
		554	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		555	* on the pixel size. When it reaches the watermark level, it'll start
		556	* fetching FIFO line sized based chunks from memory until the FIFO fills
		557	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		558	* will occur, and a display engine hang could result.
		559	*/
		560	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		561	const struct intel_watermark_params *wm,
		562	int fifo_size,
		563	int pixel_size,
		564	unsigned long latency_ns)
		565	{
		566	long entries_required, wm_size;
		567
		568	/*
		569	* Note: we need to make sure we don't overflow for various clock &
		570	* latency values.
		571	* clocks go from a few thousand to several hundred thousand.
		572	* latency is usually a few thousand
		573	*/
		574	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		575	1000;
		576	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		577
		578	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		579
		580	wm_size = fifo_size - (entries_required + wm->guard_size);
		581
		582	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		583
		584	/* Don't promote wm_size to unsigned... */
		585	if (wm_size > (long)wm->max_wm)
		586	wm_size = wm->max_wm;
		587	if (wm_size <= 0)
		588	wm_size = wm->default_wm;
5354	serge	589
		590	/*
		591	* Bspec seems to indicate that the value shouldn't be lower than
		592	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
		593	* Lets go for 8 which is the burst size since certain platforms
		594	* already use a hardcoded 8 (which is what the spec says should be
		595	* done).
		596	*/
		597	if (wm_size <= 8)
		598	wm_size = 8;
		599
3031	serge	600	return wm_size;
		601	}
		602
		603	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		604	{
		605	struct drm_crtc crtc, enabled = NULL;
		606
5060	serge	607	for_each_crtc(dev, crtc) {
3243	Serge	608	if (intel_crtc_active(crtc)) {
3031	serge	609	if (enabled)
		610	return NULL;
		611	enabled = crtc;
		612	}
		613	}
		614
		615	return enabled;
		616	}
		617
4560	Serge	618	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	619	{
4560	Serge	620	struct drm_device *dev = unused_crtc->dev;
3031	serge	621	struct drm_i915_private *dev_priv = dev->dev_private;
		622	struct drm_crtc *crtc;
		623	const struct cxsr_latency *latency;
		624	u32 reg;
		625	unsigned long wm;
		626
		627	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		628	dev_priv->fsb_freq, dev_priv->mem_freq);
		629	if (!latency) {
		630	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	631	intel_set_memory_cxsr(dev_priv, false);
3031	serge	632	return;
		633	}
		634
		635	crtc = single_enabled_crtc(dev);
		636	if (crtc) {
6084	serge	637	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
		638	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
		639	int clock = adjusted_mode->crtc_clock;
3031	serge	640
		641	/* Display SR */
		642	wm = intel_calculate_wm(clock, &pineview_display_wm,
		643	pineview_display_wm.fifo_size,
		644	pixel_size, latency->display_sr);
		645	reg = I915_READ(DSPFW1);
		646	reg &= ~DSPFW_SR_MASK;
6084	serge	647	reg \|= FW_WM(wm, SR);
3031	serge	648	I915_WRITE(DSPFW1, reg);
		649	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		650
		651	/* cursor SR */
		652	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		653	pineview_display_wm.fifo_size,
		654	pixel_size, latency->cursor_sr);
		655	reg = I915_READ(DSPFW3);
		656	reg &= ~DSPFW_CURSOR_SR_MASK;
6084	serge	657	reg \|= FW_WM(wm, CURSOR_SR);
3031	serge	658	I915_WRITE(DSPFW3, reg);
		659
		660	/* Display HPLL off SR */
		661	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		662	pineview_display_hplloff_wm.fifo_size,
		663	pixel_size, latency->display_hpll_disable);
		664	reg = I915_READ(DSPFW3);
		665	reg &= ~DSPFW_HPLL_SR_MASK;
6084	serge	666	reg \|= FW_WM(wm, HPLL_SR);
3031	serge	667	I915_WRITE(DSPFW3, reg);
		668
		669	/* cursor HPLL off SR */
		670	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		671	pineview_display_hplloff_wm.fifo_size,
		672	pixel_size, latency->cursor_hpll_disable);
		673	reg = I915_READ(DSPFW3);
		674	reg &= ~DSPFW_HPLL_CURSOR_MASK;
6084	serge	675	reg \|= FW_WM(wm, HPLL_CURSOR);
3031	serge	676	I915_WRITE(DSPFW3, reg);
		677	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		678
5060	serge	679	intel_set_memory_cxsr(dev_priv, true);
3031	serge	680	} else {
5060	serge	681	intel_set_memory_cxsr(dev_priv, false);
3031	serge	682	}
		683	}
		684
		685	static bool g4x_compute_wm0(struct drm_device *dev,
		686	int plane,
		687	const struct intel_watermark_params *display,
		688	int display_latency_ns,
		689	const struct intel_watermark_params *cursor,
		690	int cursor_latency_ns,
		691	int *plane_wm,
		692	int *cursor_wm)
		693	{
		694	struct drm_crtc *crtc;
4560	Serge	695	const struct drm_display_mode *adjusted_mode;
3031	serge	696	int htotal, hdisplay, clock, pixel_size;
		697	int line_time_us, line_count;
		698	int entries, tlb_miss;
		699
		700	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	701	if (!intel_crtc_active(crtc)) {
3031	serge	702	*cursor_wm = cursor->guard_size;
		703	*plane_wm = display->guard_size;
6084	serge	704	return false;
3031	serge	705	}
		706
6084	serge	707	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	708	clock = adjusted_mode->crtc_clock;
		709	htotal = adjusted_mode->crtc_htotal;
6084	serge	710	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		711	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	712
		713	/* Use the small buffer method to calculate plane watermark */
		714	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		715	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		716	if (tlb_miss > 0)
		717	entries += tlb_miss;
		718	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		719	*plane_wm = entries + display->guard_size;
		720	if (*plane_wm > (int)display->max_wm)
		721	*plane_wm = display->max_wm;
		722
		723	/* Use the large buffer method to calculate cursor watermark */
5060	serge	724	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	725	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
6084	serge	726	entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
3031	serge	727	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		728	if (tlb_miss > 0)
		729	entries += tlb_miss;
		730	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		731	*cursor_wm = entries + cursor->guard_size;
		732	if (*cursor_wm > (int)cursor->max_wm)
		733	*cursor_wm = (int)cursor->max_wm;
		734
		735	return true;
		736	}
		737
		738	/*
		739	* Check the wm result.
		740	*
		741	* If any calculated watermark values is larger than the maximum value that
		742	* can be programmed into the associated watermark register, that watermark
		743	* must be disabled.
		744	*/
		745	static bool g4x_check_srwm(struct drm_device *dev,
		746	int display_wm, int cursor_wm,
		747	const struct intel_watermark_params *display,
		748	const struct intel_watermark_params *cursor)
		749	{
		750	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		751	display_wm, cursor_wm);
		752
		753	if (display_wm > display->max_wm) {
		754	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		755	display_wm, display->max_wm);
		756	return false;
		757	}
		758
		759	if (cursor_wm > cursor->max_wm) {
		760	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		761	cursor_wm, cursor->max_wm);
		762	return false;
		763	}
		764
		765	if (!(display_wm \|\| cursor_wm)) {
		766	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		767	return false;
		768	}
		769
		770	return true;
		771	}
		772
		773	static bool g4x_compute_srwm(struct drm_device *dev,
		774	int plane,
		775	int latency_ns,
		776	const struct intel_watermark_params *display,
		777	const struct intel_watermark_params *cursor,
		778	int display_wm, int cursor_wm)
		779	{
		780	struct drm_crtc *crtc;
4560	Serge	781	const struct drm_display_mode *adjusted_mode;
3031	serge	782	int hdisplay, htotal, pixel_size, clock;
		783	unsigned long line_time_us;
		784	int line_count, line_size;
		785	int small, large;
		786	int entries;
		787
		788	if (!latency_ns) {
		789	display_wm = cursor_wm = 0;
		790	return false;
		791	}
		792
		793	crtc = intel_get_crtc_for_plane(dev, plane);
6084	serge	794	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	795	clock = adjusted_mode->crtc_clock;
		796	htotal = adjusted_mode->crtc_htotal;
6084	serge	797	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		798	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	799
5060	serge	800	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	801	line_count = (latency_ns / line_time_us + 1000) / 1000;
		802	line_size = hdisplay * pixel_size;
		803
		804	/* Use the minimum of the small and large buffer method for primary */
		805	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		806	large = line_count * line_size;
		807
		808	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		809	*display_wm = entries + display->guard_size;
		810
		811	/* calculate the self-refresh watermark for display cursor */
6084	serge	812	entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
3031	serge	813	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		814	*cursor_wm = entries + cursor->guard_size;
		815
		816	return g4x_check_srwm(dev,
		817	display_wm, cursor_wm,
		818	display, cursor);
		819	}
		820
6084	serge	821	#define FW_WM_VLV(value, plane) \
		822	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
		823
		824	static void vlv_write_wm_values(struct intel_crtc *crtc,
		825	const struct vlv_wm_values *wm)
3031	serge	826	{
6084	serge	827	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
		828	enum pipe pipe = crtc->pipe;
3031	serge	829
6084	serge	830	I915_WRITE(VLV_DDL(pipe),
		831	(wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) \|
		832	(wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) \|
		833	(wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) \|
		834	(wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
3031	serge	835
6084	serge	836	I915_WRITE(DSPFW1,
		837	FW_WM(wm->sr.plane, SR) \|
		838	FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) \|
		839	FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) \|
		840	FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
		841	I915_WRITE(DSPFW2,
		842	FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) \|
		843	FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) \|
		844	FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
		845	I915_WRITE(DSPFW3,
		846	FW_WM(wm->sr.cursor, CURSOR_SR));
3031	serge	847
6084	serge	848	if (IS_CHERRYVIEW(dev_priv)) {
		849	I915_WRITE(DSPFW7_CHV,
		850	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		851	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		852	I915_WRITE(DSPFW8_CHV,
		853	FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) \|
		854	FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
		855	I915_WRITE(DSPFW9_CHV,
		856	FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) \|
		857	FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
		858	I915_WRITE(DSPHOWM,
		859	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		860	FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) \|
		861	FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) \|
		862	FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) \|
		863	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		864	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		865	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		866	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		867	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		868	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		869	} else {
		870	I915_WRITE(DSPFW7,
		871	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		872	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		873	I915_WRITE(DSPHOWM,
		874	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		875	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		876	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		877	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		878	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		879	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		880	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		881	}
3031	serge	882
6084	serge	883	/* zero (unused) WM1 watermarks */
		884	I915_WRITE(DSPFW4, 0);
		885	I915_WRITE(DSPFW5, 0);
		886	I915_WRITE(DSPFW6, 0);
		887	I915_WRITE(DSPHOWM1, 0);
3031	serge	888
6084	serge	889	POSTING_READ(DSPFW1);
3031	serge	890	}
		891
6084	serge	892	#undef FW_WM_VLV
3031	serge	893
6084	serge	894	enum vlv_wm_level {
		895	VLV_WM_LEVEL_PM2,
		896	VLV_WM_LEVEL_PM5,
		897	VLV_WM_LEVEL_DDR_DVFS,
		898	};
		899
		900	/* latency must be in 0.1us units. */
		901	static unsigned int vlv_wm_method2(unsigned int pixel_rate,
		902	unsigned int pipe_htotal,
		903	unsigned int horiz_pixels,
		904	unsigned int bytes_per_pixel,
		905	unsigned int latency)
3031	serge	906	{
6084	serge	907	unsigned int ret;
		908
		909	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		910	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		911	ret = DIV_ROUND_UP(ret, 64);
		912
		913	return ret;
		914	}
		915
		916	static void vlv_setup_wm_latency(struct drm_device *dev)
		917	{
3031	serge	918	struct drm_i915_private *dev_priv = dev->dev_private;
		919
6084	serge	920	/* all latencies in usec */
		921	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
3031	serge	922
6084	serge	923	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
		924
		925	if (IS_CHERRYVIEW(dev_priv)) {
		926	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		927	dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
		928
		929	dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
3031	serge	930	}
6084	serge	931	}
3031	serge	932
6084	serge	933	static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
		934	struct intel_crtc *crtc,
		935	const struct intel_plane_state *state,
		936	int level)
		937	{
		938	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
		939	int clock, htotal, pixel_size, width, wm;
		940
		941	if (dev_priv->wm.pri_latency[level] == 0)
		942	return USHRT_MAX;
		943
		944	if (!state->visible)
		945	return 0;
		946
		947	pixel_size = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		948	clock = crtc->config->base.adjusted_mode.crtc_clock;
		949	htotal = crtc->config->base.adjusted_mode.crtc_htotal;
		950	width = crtc->config->pipe_src_w;
		951	if (WARN_ON(htotal == 0))
		952	htotal = 1;
		953
		954	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		955	/*
		956	* FIXME the formula gives values that are
		957	* too big for the cursor FIFO, and hence we
		958	* would never be able to use cursors. For
		959	* now just hardcode the watermark.
		960	*/
		961	wm = 63;
		962	} else {
		963	wm = vlv_wm_method2(clock, htotal, width, pixel_size,
		964	dev_priv->wm.pri_latency[level] * 10);
5354	serge	965	}
3031	serge	966
6084	serge	967	return min_t(int, wm, USHRT_MAX);
		968	}
5354	serge	969
6084	serge	970	static void vlv_compute_fifo(struct intel_crtc *crtc)
		971	{
		972	struct drm_device *dev = crtc->base.dev;
		973	struct vlv_wm_state *wm_state = &crtc->wm_state;
		974	struct intel_plane *plane;
		975	unsigned int total_rate = 0;
		976	const int fifo_size = 512 - 1;
		977	int fifo_extra, fifo_left = fifo_size;
		978
		979	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		980	struct intel_plane_state *state =
		981	to_intel_plane_state(plane->base.state);
		982
		983	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		984	continue;
		985
		986	if (state->visible) {
		987	wm_state->num_active_planes++;
		988	total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		989	}
3031	serge	990	}
5354	serge	991
6084	serge	992	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		993	struct intel_plane_state *state =
		994	to_intel_plane_state(plane->base.state);
		995	unsigned int rate;
		996
		997	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		998	plane->wm.fifo_size = 63;
		999	continue;
		1000	}
		1001
		1002	if (!state->visible) {
		1003	plane->wm.fifo_size = 0;
		1004	continue;
		1005	}
		1006
		1007	rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		1008	plane->wm.fifo_size = fifo_size * rate / total_rate;
		1009	fifo_left -= plane->wm.fifo_size;
		1010	}
		1011
		1012	fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
		1013
		1014	/* spread the remainder evenly */
		1015	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1016	int plane_extra;
		1017
		1018	if (fifo_left == 0)
		1019	break;
		1020
		1021	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		1022	continue;
		1023
		1024	/* give it all to the first plane if none are active */
		1025	if (plane->wm.fifo_size == 0 &&
		1026	wm_state->num_active_planes)
		1027	continue;
		1028
		1029	plane_extra = min(fifo_extra, fifo_left);
		1030	plane->wm.fifo_size += plane_extra;
		1031	fifo_left -= plane_extra;
		1032	}
		1033
		1034	WARN_ON(fifo_left != 0);
3031	serge	1035	}
		1036
6084	serge	1037	static void vlv_invert_wms(struct intel_crtc *crtc)
		1038	{
		1039	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1040	int level;
3031	serge	1041
6084	serge	1042	for (level = 0; level < wm_state->num_levels; level++) {
		1043	struct drm_device *dev = crtc->base.dev;
		1044	const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1045	struct intel_plane *plane;
		1046
		1047	wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
		1048	wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
		1049
		1050	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1051	switch (plane->base.type) {
		1052	int sprite;
		1053	case DRM_PLANE_TYPE_CURSOR:
		1054	wm_state->wm[level].cursor = plane->wm.fifo_size -
		1055	wm_state->wm[level].cursor;
		1056	break;
		1057	case DRM_PLANE_TYPE_PRIMARY:
		1058	wm_state->wm[level].primary = plane->wm.fifo_size -
		1059	wm_state->wm[level].primary;
		1060	break;
		1061	case DRM_PLANE_TYPE_OVERLAY:
		1062	sprite = plane->plane;
		1063	wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
		1064	wm_state->wm[level].sprite[sprite];
		1065	break;
		1066	}
		1067	}
		1068	}
		1069	}
		1070
		1071	static void vlv_compute_wm(struct intel_crtc *crtc)
3031	serge	1072	{
6084	serge	1073	struct drm_device *dev = crtc->base.dev;
		1074	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1075	struct intel_plane *plane;
		1076	int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1077	int level;
3031	serge	1078
6084	serge	1079	memset(wm_state, 0, sizeof(*wm_state));
3031	serge	1080
6084	serge	1081	wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
		1082	wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
3031	serge	1083
6084	serge	1084	wm_state->num_active_planes = 0;
3031	serge	1085
6084	serge	1086	vlv_compute_fifo(crtc);
		1087
		1088	if (wm_state->num_active_planes != 1)
		1089	wm_state->cxsr = false;
		1090
		1091	if (wm_state->cxsr) {
		1092	for (level = 0; level < wm_state->num_levels; level++) {
		1093	wm_state->sr[level].plane = sr_fifo_size;
		1094	wm_state->sr[level].cursor = 63;
		1095	}
3243	Serge	1096	}
3031	serge	1097
6084	serge	1098	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1099	struct intel_plane_state *state =
		1100	to_intel_plane_state(plane->base.state);
3031	serge	1101
6084	serge	1102	if (!state->visible)
		1103	continue;
5060	serge	1104
6084	serge	1105	/* normal watermarks */
		1106	for (level = 0; level < wm_state->num_levels; level++) {
		1107	int wm = vlv_compute_wm_level(plane, crtc, state, level);
		1108	int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
		1109
		1110	/* hack */
		1111	if (WARN_ON(level == 0 && wm > max_wm))
		1112	wm = max_wm;
		1113
		1114	if (wm > plane->wm.fifo_size)
		1115	break;
		1116
		1117	switch (plane->base.type) {
		1118	int sprite;
		1119	case DRM_PLANE_TYPE_CURSOR:
		1120	wm_state->wm[level].cursor = wm;
		1121	break;
		1122	case DRM_PLANE_TYPE_PRIMARY:
		1123	wm_state->wm[level].primary = wm;
		1124	break;
		1125	case DRM_PLANE_TYPE_OVERLAY:
		1126	sprite = plane->plane;
		1127	wm_state->wm[level].sprite[sprite] = wm;
		1128	break;
		1129	}
		1130	}
		1131
		1132	wm_state->num_levels = level;
		1133
		1134	if (!wm_state->cxsr)
		1135	continue;
		1136
		1137	/* maxfifo watermarks */
		1138	switch (plane->base.type) {
		1139	int sprite, level;
		1140	case DRM_PLANE_TYPE_CURSOR:
		1141	for (level = 0; level < wm_state->num_levels; level++)
		1142	wm_state->sr[level].cursor =
		1143	wm_state->wm[level].cursor;
		1144	break;
		1145	case DRM_PLANE_TYPE_PRIMARY:
		1146	for (level = 0; level < wm_state->num_levels; level++)
		1147	wm_state->sr[level].plane =
		1148	min(wm_state->sr[level].plane,
		1149	wm_state->wm[level].primary);
		1150	break;
		1151	case DRM_PLANE_TYPE_OVERLAY:
		1152	sprite = plane->plane;
		1153	for (level = 0; level < wm_state->num_levels; level++)
		1154	wm_state->sr[level].plane =
		1155	min(wm_state->sr[level].plane,
		1156	wm_state->wm[level].sprite[sprite]);
		1157	break;
		1158	}
		1159	}
		1160
		1161	/* clear any (partially) filled invalid levels */
		1162	for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
		1163	memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
		1164	memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
		1165	}
		1166
		1167	vlv_invert_wms(crtc);
3031	serge	1168	}
		1169
6084	serge	1170	#define VLV_FIFO(plane, value) \
		1171	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
		1172
		1173	static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
5354	serge	1174	{
6084	serge	1175	struct drm_device *dev = crtc->base.dev;
		1176	struct drm_i915_private *dev_priv = to_i915(dev);
		1177	struct intel_plane *plane;
		1178	int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
5354	serge	1179
6084	serge	1180	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1181	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1182	WARN_ON(plane->wm.fifo_size != 63);
		1183	continue;
		1184	}
5354	serge	1185
6084	serge	1186	if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		1187	sprite0_start = plane->wm.fifo_size;
		1188	else if (plane->plane == 0)
		1189	sprite1_start = sprite0_start + plane->wm.fifo_size;
		1190	else
		1191	fifo_size = sprite1_start + plane->wm.fifo_size;
		1192	}
5354	serge	1193
6084	serge	1194	WARN_ON(fifo_size != 512 - 1);
5354	serge	1195
6084	serge	1196	DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
		1197	pipe_name(crtc->pipe), sprite0_start,
		1198	sprite1_start, fifo_size);
5354	serge	1199
6084	serge	1200	switch (crtc->pipe) {
		1201	uint32_t dsparb, dsparb2, dsparb3;
		1202	case PIPE_A:
		1203	dsparb = I915_READ(DSPARB);
		1204	dsparb2 = I915_READ(DSPARB2);
		1205
		1206	dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) \|
		1207	VLV_FIFO(SPRITEB, 0xff));
		1208	dsparb \|= (VLV_FIFO(SPRITEA, sprite0_start) \|
		1209	VLV_FIFO(SPRITEB, sprite1_start));
		1210
		1211	dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) \|
		1212	VLV_FIFO(SPRITEB_HI, 0x1));
		1213	dsparb2 \|= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) \|
		1214	VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
		1215
		1216	I915_WRITE(DSPARB, dsparb);
		1217	I915_WRITE(DSPARB2, dsparb2);
		1218	break;
		1219	case PIPE_B:
		1220	dsparb = I915_READ(DSPARB);
		1221	dsparb2 = I915_READ(DSPARB2);
		1222
		1223	dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) \|
		1224	VLV_FIFO(SPRITED, 0xff));
		1225	dsparb \|= (VLV_FIFO(SPRITEC, sprite0_start) \|
		1226	VLV_FIFO(SPRITED, sprite1_start));
		1227
		1228	dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) \|
		1229	VLV_FIFO(SPRITED_HI, 0xff));
		1230	dsparb2 \|= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) \|
		1231	VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
		1232
		1233	I915_WRITE(DSPARB, dsparb);
		1234	I915_WRITE(DSPARB2, dsparb2);
		1235	break;
		1236	case PIPE_C:
		1237	dsparb3 = I915_READ(DSPARB3);
		1238	dsparb2 = I915_READ(DSPARB2);
		1239
		1240	dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) \|
		1241	VLV_FIFO(SPRITEF, 0xff));
		1242	dsparb3 \|= (VLV_FIFO(SPRITEE, sprite0_start) \|
		1243	VLV_FIFO(SPRITEF, sprite1_start));
		1244
		1245	dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) \|
		1246	VLV_FIFO(SPRITEF_HI, 0xff));
		1247	dsparb2 \|= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) \|
		1248	VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
		1249
		1250	I915_WRITE(DSPARB3, dsparb3);
		1251	I915_WRITE(DSPARB2, dsparb2);
		1252	break;
		1253	default:
		1254	break;
5354	serge	1255	}
6084	serge	1256	}
5354	serge	1257
6084	serge	1258	#undef VLV_FIFO
5354	serge	1259
6084	serge	1260	static void vlv_merge_wm(struct drm_device *dev,
		1261	struct vlv_wm_values *wm)
		1262	{
		1263	struct intel_crtc *crtc;
		1264	int num_active_crtcs = 0;
5354	serge	1265
6084	serge	1266	wm->level = to_i915(dev)->wm.max_level;
		1267	wm->cxsr = true;
		1268
		1269	for_each_intel_crtc(dev, crtc) {
		1270	const struct vlv_wm_state *wm_state = &crtc->wm_state;
		1271
		1272	if (!crtc->active)
		1273	continue;
		1274
		1275	if (!wm_state->cxsr)
		1276	wm->cxsr = false;
		1277
		1278	num_active_crtcs++;
		1279	wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
		1280	}
		1281
		1282	if (num_active_crtcs != 1)
		1283	wm->cxsr = false;
		1284
		1285	if (num_active_crtcs > 1)
		1286	wm->level = VLV_WM_LEVEL_PM2;
		1287
		1288	for_each_intel_crtc(dev, crtc) {
		1289	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1290	enum pipe pipe = crtc->pipe;
		1291
		1292	if (!crtc->active)
		1293	continue;
		1294
		1295	wm->pipe[pipe] = wm_state->wm[wm->level];
		1296	if (wm->cxsr)
		1297	wm->sr = wm_state->sr[wm->level];
		1298
		1299	wm->ddl[pipe].primary = DDL_PRECISION_HIGH \| 2;
		1300	wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH \| 2;
		1301	wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH \| 2;
		1302	wm->ddl[pipe].cursor = DDL_PRECISION_HIGH \| 2;
		1303	}
5354	serge	1304	}
		1305
6084	serge	1306	static void vlv_update_wm(struct drm_crtc *crtc)
5354	serge	1307	{
		1308	struct drm_device *dev = crtc->dev;
		1309	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1310	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1311	enum pipe pipe = intel_crtc->pipe;
		1312	struct vlv_wm_values wm = {};
5354	serge	1313
6084	serge	1314	vlv_compute_wm(intel_crtc);
		1315	vlv_merge_wm(dev, &wm);
5354	serge	1316
6084	serge	1317	if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
		1318	/* FIXME should be part of crtc atomic commit */
		1319	vlv_pipe_set_fifo_size(intel_crtc);
		1320	return;
5354	serge	1321	}
		1322
6084	serge	1323	if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
		1324	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
		1325	chv_set_memory_dvfs(dev_priv, false);
		1326
		1327	if (wm.level < VLV_WM_LEVEL_PM5 &&
		1328	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
		1329	chv_set_memory_pm5(dev_priv, false);
		1330
		1331	if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
		1332	intel_set_memory_cxsr(dev_priv, false);
		1333
		1334	/* FIXME should be part of crtc atomic commit */
		1335	vlv_pipe_set_fifo_size(intel_crtc);
		1336
		1337	vlv_write_wm_values(intel_crtc, &wm);
		1338
		1339	DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
		1340	"sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
		1341	pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
		1342	wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
		1343	wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
		1344
		1345	if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
		1346	intel_set_memory_cxsr(dev_priv, true);
		1347
		1348	if (wm.level >= VLV_WM_LEVEL_PM5 &&
		1349	dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
		1350	chv_set_memory_pm5(dev_priv, true);
		1351
		1352	if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
		1353	dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
		1354	chv_set_memory_dvfs(dev_priv, true);
		1355
		1356	dev_priv->wm.vlv = wm;
5354	serge	1357	}
		1358
6084	serge	1359	#define single_plane_enabled(mask) is_power_of_2(mask)
		1360
4560	Serge	1361	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1362	{
4560	Serge	1363	struct drm_device *dev = crtc->dev;
3031	serge	1364	static const int sr_latency_ns = 12000;
		1365	struct drm_i915_private *dev_priv = dev->dev_private;
		1366	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1367	int plane_sr, cursor_sr;
		1368	unsigned int enabled = 0;
5060	serge	1369	bool cxsr_enabled;
3031	serge	1370
3746	Serge	1371	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1372	&g4x_wm_info, pessimal_latency_ns,
		1373	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1374	&planea_wm, &cursora_wm))
3746	Serge	1375	enabled \|= 1 << PIPE_A;
3031	serge	1376
3746	Serge	1377	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1378	&g4x_wm_info, pessimal_latency_ns,
		1379	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1380	&planeb_wm, &cursorb_wm))
3746	Serge	1381	enabled \|= 1 << PIPE_B;
3031	serge	1382
		1383	if (single_plane_enabled(enabled) &&
		1384	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1385	sr_latency_ns,
		1386	&g4x_wm_info,
		1387	&g4x_cursor_wm_info,
3243	Serge	1388	&plane_sr, &cursor_sr)) {
5060	serge	1389	cxsr_enabled = true;
3243	Serge	1390	} else {
5060	serge	1391	cxsr_enabled = false;
		1392	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1393	plane_sr = cursor_sr = 0;
		1394	}
3031	serge	1395
5354	serge	1396	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1397	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1398	planea_wm, cursora_wm,
		1399	planeb_wm, cursorb_wm,
		1400	plane_sr, cursor_sr);
		1401
		1402	I915_WRITE(DSPFW1,
6084	serge	1403	FW_WM(plane_sr, SR) \|
		1404	FW_WM(cursorb_wm, CURSORB) \|
		1405	FW_WM(planeb_wm, PLANEB) \|
		1406	FW_WM(planea_wm, PLANEA));
3031	serge	1407	I915_WRITE(DSPFW2,
3243	Serge	1408	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
6084	serge	1409	FW_WM(cursora_wm, CURSORA));
3031	serge	1410	/* HPLL off in SR has some issues on G4x... disable it */
		1411	I915_WRITE(DSPFW3,
3243	Serge	1412	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
6084	serge	1413	FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1414
		1415	if (cxsr_enabled)
		1416	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1417	}
		1418
4560	Serge	1419	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1420	{
4560	Serge	1421	struct drm_device *dev = unused_crtc->dev;
3031	serge	1422	struct drm_i915_private *dev_priv = dev->dev_private;
		1423	struct drm_crtc *crtc;
		1424	int srwm = 1;
		1425	int cursor_sr = 16;
5060	serge	1426	bool cxsr_enabled;
3031	serge	1427
		1428	/* Calc sr entries for one plane configs */
		1429	crtc = single_enabled_crtc(dev);
		1430	if (crtc) {
		1431	/* self-refresh has much higher latency */
		1432	static const int sr_latency_ns = 12000;
6084	serge	1433	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1434	int clock = adjusted_mode->crtc_clock;
		1435	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1436	int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		1437	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	1438	unsigned long line_time_us;
		1439	int entries;
		1440
5060	serge	1441	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1442
		1443	/* Use ns/us then divide to preserve precision */
		1444	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1445	pixel_size * hdisplay;
		1446	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1447	srwm = I965_FIFO_SIZE - entries;
		1448	if (srwm < 0)
		1449	srwm = 1;
		1450	srwm &= 0x1ff;
		1451	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1452	entries, srwm);
		1453
		1454	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
6084	serge	1455	pixel_size * crtc->cursor->state->crtc_w;
3031	serge	1456	entries = DIV_ROUND_UP(entries,
		1457	i965_cursor_wm_info.cacheline_size);
		1458	cursor_sr = i965_cursor_wm_info.fifo_size -
		1459	(entries + i965_cursor_wm_info.guard_size);
		1460
		1461	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1462	cursor_sr = i965_cursor_wm_info.max_wm;
		1463
		1464	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1465	"cursor %d\n", srwm, cursor_sr);
		1466
5060	serge	1467	cxsr_enabled = true;
3031	serge	1468	} else {
5060	serge	1469	cxsr_enabled = false;
3031	serge	1470	/* Turn off self refresh if both pipes are enabled */
5060	serge	1471	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1472	}
		1473
		1474	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1475	srwm);
		1476
		1477	/* 965 has limitations... */
6084	serge	1478	I915_WRITE(DSPFW1, FW_WM(srwm, SR) \|
		1479	FW_WM(8, CURSORB) \|
		1480	FW_WM(8, PLANEB) \|
		1481	FW_WM(8, PLANEA));
		1482	I915_WRITE(DSPFW2, FW_WM(8, CURSORA) \|
		1483	FW_WM(8, PLANEC_OLD));
3031	serge	1484	/* update cursor SR watermark */
6084	serge	1485	I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1486
		1487	if (cxsr_enabled)
		1488	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1489	}
		1490
6084	serge	1491	#undef FW_WM
		1492
4560	Serge	1493	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1494	{
4560	Serge	1495	struct drm_device *dev = unused_crtc->dev;
3031	serge	1496	struct drm_i915_private *dev_priv = dev->dev_private;
		1497	const struct intel_watermark_params *wm_info;
		1498	uint32_t fwater_lo;
		1499	uint32_t fwater_hi;
		1500	int cwm, srwm = 1;
		1501	int fifo_size;
		1502	int planea_wm, planeb_wm;
		1503	struct drm_crtc crtc, enabled = NULL;
		1504
		1505	if (IS_I945GM(dev))
		1506	wm_info = &i945_wm_info;
		1507	else if (!IS_GEN2(dev))
		1508	wm_info = &i915_wm_info;
		1509	else
5354	serge	1510	wm_info = &i830_a_wm_info;
3031	serge	1511
		1512	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1513	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1514	if (intel_crtc_active(crtc)) {
4560	Serge	1515	const struct drm_display_mode *adjusted_mode;
6084	serge	1516	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1517	if (IS_GEN2(dev))
		1518	cpp = 4;
		1519
6084	serge	1520	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1521	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1522	wm_info, fifo_size, cpp,
5354	serge	1523	pessimal_latency_ns);
3031	serge	1524	enabled = crtc;
5354	serge	1525	} else {
3031	serge	1526	planea_wm = fifo_size - wm_info->guard_size;
5354	serge	1527	if (planea_wm > (long)wm_info->max_wm)
		1528	planea_wm = wm_info->max_wm;
		1529	}
3031	serge	1530
5354	serge	1531	if (IS_GEN2(dev))
		1532	wm_info = &i830_bc_wm_info;
		1533
3031	serge	1534	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1535	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1536	if (intel_crtc_active(crtc)) {
4560	Serge	1537	const struct drm_display_mode *adjusted_mode;
6084	serge	1538	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1539	if (IS_GEN2(dev))
		1540	cpp = 4;
		1541
6084	serge	1542	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1543	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1544	wm_info, fifo_size, cpp,
5354	serge	1545	pessimal_latency_ns);
3031	serge	1546	if (enabled == NULL)
		1547	enabled = crtc;
		1548	else
		1549	enabled = NULL;
5354	serge	1550	} else {
3031	serge	1551	planeb_wm = fifo_size - wm_info->guard_size;
5354	serge	1552	if (planeb_wm > (long)wm_info->max_wm)
		1553	planeb_wm = wm_info->max_wm;
		1554	}
3031	serge	1555
		1556	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1557
5060	serge	1558	if (IS_I915GM(dev) && enabled) {
		1559	struct drm_i915_gem_object *obj;
		1560
6084	serge	1561	obj = intel_fb_obj(enabled->primary->state->fb);
5060	serge	1562
		1563	/* self-refresh seems busted with untiled */
		1564	if (obj->tiling_mode == I915_TILING_NONE)
		1565	enabled = NULL;
		1566	}
		1567
3031	serge	1568	/*
		1569	* Overlay gets an aggressive default since video jitter is bad.
		1570	*/
		1571	cwm = 2;
		1572
		1573	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1574	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1575
		1576	/* Calc sr entries for one plane configs */
		1577	if (HAS_FW_BLC(dev) && enabled) {
		1578	/* self-refresh has much higher latency */
		1579	static const int sr_latency_ns = 6000;
6084	serge	1580	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
4560	Serge	1581	int clock = adjusted_mode->crtc_clock;
		1582	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1583	int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
		1584	int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
3031	serge	1585	unsigned long line_time_us;
		1586	int entries;
		1587
5060	serge	1588	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1589
		1590	/* Use ns/us then divide to preserve precision */
		1591	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1592	pixel_size * hdisplay;
		1593	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1594	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1595	srwm = wm_info->fifo_size - entries;
		1596	if (srwm < 0)
		1597	srwm = 1;
		1598
		1599	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1600	I915_WRITE(FW_BLC_SELF,
		1601	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1602	else if (IS_I915GM(dev))
		1603	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1604	}
		1605
		1606	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1607	planea_wm, planeb_wm, cwm, srwm);
		1608
		1609	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1610	fwater_hi = (cwm & 0x1f);
		1611
		1612	/* Set request length to 8 cachelines per fetch */
		1613	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1614	fwater_hi = fwater_hi \| (1 << 8);
		1615
		1616	I915_WRITE(FW_BLC, fwater_lo);
		1617	I915_WRITE(FW_BLC2, fwater_hi);
		1618
5060	serge	1619	if (enabled)
		1620	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1621	}
		1622
4560	Serge	1623	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1624	{
4560	Serge	1625	struct drm_device *dev = unused_crtc->dev;
3031	serge	1626	struct drm_i915_private *dev_priv = dev->dev_private;
		1627	struct drm_crtc *crtc;
4560	Serge	1628	const struct drm_display_mode *adjusted_mode;
3031	serge	1629	uint32_t fwater_lo;
		1630	int planea_wm;
		1631
		1632	crtc = single_enabled_crtc(dev);
		1633	if (crtc == NULL)
		1634	return;
		1635
6084	serge	1636	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1637	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1638	&i845_wm_info,
3031	serge	1639	dev_priv->display.get_fifo_size(dev, 0),
5354	serge	1640	4, pessimal_latency_ns);
3031	serge	1641	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1642	fwater_lo \|= (3<<8) \| planea_wm;
		1643
		1644	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1645
		1646	I915_WRITE(FW_BLC, fwater_lo);
		1647	}
		1648
6084	serge	1649	uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
3031	serge	1650	{
4104	Serge	1651	uint32_t pixel_rate;
		1652
6084	serge	1653	pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
4104	Serge	1654
		1655	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1656	* adjust the pixel_rate here. */
		1657
6084	serge	1658	if (pipe_config->pch_pfit.enabled) {
4104	Serge	1659	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6084	serge	1660	uint32_t pfit_size = pipe_config->pch_pfit.size;
4104	Serge	1661
6084	serge	1662	pipe_w = pipe_config->pipe_src_w;
		1663	pipe_h = pipe_config->pipe_src_h;
		1664
4104	Serge	1665	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1666	pfit_h = pfit_size & 0xFFFF;
		1667	if (pipe_w < pfit_w)
		1668	pipe_w = pfit_w;
		1669	if (pipe_h < pfit_h)
		1670	pipe_h = pfit_h;
		1671
		1672	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1673	pfit_w * pfit_h);
		1674	}
		1675
		1676	return pixel_rate;
		1677	}
		1678
		1679	/* latency must be in 0.1us units. */
		1680	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1681	uint32_t latency)
		1682	{
		1683	uint64_t ret;
		1684
		1685	if (WARN(latency == 0, "Latency value missing\n"))
		1686	return UINT_MAX;
		1687
		1688	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1689	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1690
		1691	return ret;
		1692	}
		1693
		1694	/* latency must be in 0.1us units. */
		1695	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1696	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1697	uint32_t latency)
		1698	{
		1699	uint32_t ret;
		1700
		1701	if (WARN(latency == 0, "Latency value missing\n"))
		1702	return UINT_MAX;
		1703
		1704	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1705	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1706	ret = DIV_ROUND_UP(ret, 64) + 2;
		1707	return ret;
		1708	}
		1709
		1710	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1711	uint8_t bytes_per_pixel)
		1712	{
		1713	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1714	}
		1715
5354	serge	1716	struct skl_pipe_wm_parameters {
		1717	bool active;
		1718	uint32_t pipe_htotal;
		1719	uint32_t pixel_rate; /* in KHz */
		1720	struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
		1721	};
		1722
4560	Serge	1723	struct ilk_wm_maximums {
4104	Serge	1724	uint16_t pri;
		1725	uint16_t spr;
		1726	uint16_t cur;
		1727	uint16_t fbc;
		1728	};
		1729
		1730	/* used in computing the new watermarks state */
		1731	struct intel_wm_config {
		1732	unsigned int num_pipes_active;
		1733	bool sprites_enabled;
		1734	bool sprites_scaled;
		1735	};
		1736
		1737	/*
		1738	* For both WM_PIPE and WM_LP.
		1739	* mem_value must be in 0.1us units.
		1740	*/
6084	serge	1741	static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
		1742	const struct intel_plane_state *pstate,
4104	Serge	1743	uint32_t mem_value,
		1744	bool is_lp)
		1745	{
6084	serge	1746	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1747	uint32_t method1, method2;
		1748
6084	serge	1749	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1750	return 0;
		1751
6084	serge	1752	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
4104	Serge	1753
		1754	if (!is_lp)
		1755	return method1;
		1756
6084	serge	1757	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1758	cstate->base.adjusted_mode.crtc_htotal,
		1759	drm_rect_width(&pstate->dst),
		1760	bpp,
4104	Serge	1761	mem_value);
		1762
		1763	return min(method1, method2);
		1764	}
		1765
		1766	/*
		1767	* For both WM_PIPE and WM_LP.
		1768	* mem_value must be in 0.1us units.
		1769	*/
6084	serge	1770	static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
		1771	const struct intel_plane_state *pstate,
4104	Serge	1772	uint32_t mem_value)
		1773	{
6084	serge	1774	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1775	uint32_t method1, method2;
		1776
6084	serge	1777	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1778	return 0;
		1779
6084	serge	1780	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
		1781	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1782	cstate->base.adjusted_mode.crtc_htotal,
		1783	drm_rect_width(&pstate->dst),
		1784	bpp,
4104	Serge	1785	mem_value);
		1786	return min(method1, method2);
		1787	}
		1788
		1789	/*
		1790	* For both WM_PIPE and WM_LP.
		1791	* mem_value must be in 0.1us units.
		1792	*/
6084	serge	1793	static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
		1794	const struct intel_plane_state *pstate,
4104	Serge	1795	uint32_t mem_value)
		1796	{
6084	serge	1797	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
		1798
		1799	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1800	return 0;
		1801
6084	serge	1802	return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1803	cstate->base.adjusted_mode.crtc_htotal,
		1804	drm_rect_width(&pstate->dst),
		1805	bpp,
4104	Serge	1806	mem_value);
		1807	}
		1808
		1809	/* Only for WM_LP. */
6084	serge	1810	static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
		1811	const struct intel_plane_state *pstate,
4104	Serge	1812	uint32_t pri_val)
		1813	{
6084	serge	1814	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
		1815
		1816	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1817	return 0;
		1818
6084	serge	1819	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), bpp);
4104	Serge	1820	}
		1821
		1822	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1823	{
4560	Serge	1824	if (INTEL_INFO(dev)->gen >= 8)
		1825	return 3072;
		1826	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1827	return 768;
		1828	else
		1829	return 512;
		1830	}
		1831
5060	serge	1832	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		1833	int level, bool is_sprite)
		1834	{
		1835	if (INTEL_INFO(dev)->gen >= 8)
		1836	/* BDW primary/sprite plane watermarks */
		1837	return level == 0 ? 255 : 2047;
		1838	else if (INTEL_INFO(dev)->gen >= 7)
		1839	/* IVB/HSW primary/sprite plane watermarks */
		1840	return level == 0 ? 127 : 1023;
		1841	else if (!is_sprite)
		1842	/* ILK/SNB primary plane watermarks */
		1843	return level == 0 ? 127 : 511;
		1844	else
		1845	/* ILK/SNB sprite plane watermarks */
		1846	return level == 0 ? 63 : 255;
		1847	}
		1848
		1849	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		1850	int level)
		1851	{
		1852	if (INTEL_INFO(dev)->gen >= 7)
		1853	return level == 0 ? 63 : 255;
		1854	else
		1855	return level == 0 ? 31 : 63;
		1856	}
		1857
		1858	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		1859	{
		1860	if (INTEL_INFO(dev)->gen >= 8)
		1861	return 31;
		1862	else
		1863	return 15;
		1864	}
		1865
4104	Serge	1866	/* Calculate the maximum primary/sprite plane watermark */
		1867	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1868	int level,
		1869	const struct intel_wm_config *config,
		1870	enum intel_ddb_partitioning ddb_partitioning,
		1871	bool is_sprite)
		1872	{
		1873	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1874
		1875	/* if sprites aren't enabled, sprites get nothing */
		1876	if (is_sprite && !config->sprites_enabled)
		1877	return 0;
		1878
		1879	/* HSW allows LP1+ watermarks even with multiple pipes */
		1880	if (level == 0 \|\| config->num_pipes_active > 1) {
		1881	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1882
		1883	/*
		1884	* For some reason the non self refresh
		1885	* FIFO size is only half of the self
		1886	* refresh FIFO size on ILK/SNB.
		1887	*/
		1888	if (INTEL_INFO(dev)->gen <= 6)
		1889	fifo_size /= 2;
		1890	}
		1891
		1892	if (config->sprites_enabled) {
		1893	/* level 0 is always calculated with 1:1 split */
		1894	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1895	if (is_sprite)
		1896	fifo_size *= 5;
		1897	fifo_size /= 6;
		1898	} else {
		1899	fifo_size /= 2;
		1900	}
		1901	}
		1902
		1903	/* clamp to max that the registers can hold */
5060	serge	1904	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	1905	}
		1906
		1907	/* Calculate the maximum cursor plane watermark */
		1908	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1909	int level,
		1910	const struct intel_wm_config *config)
		1911	{
		1912	/* HSW LP1+ watermarks w/ multiple pipes */
		1913	if (level > 0 && config->num_pipes_active > 1)
		1914	return 64;
		1915
		1916	/* otherwise just report max that registers can hold */
5060	serge	1917	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	1918	}
4104	Serge	1919
5060	serge	1920	static void ilk_compute_wm_maximums(const struct drm_device *dev,
6084	serge	1921	int level,
		1922	const struct intel_wm_config *config,
		1923	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1924	struct ilk_wm_maximums *max)
4104	Serge	1925	{
		1926	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1927	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1928	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	1929	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	1930	}
4104	Serge	1931
5060	serge	1932	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		1933	int level,
		1934	struct ilk_wm_maximums *max)
		1935	{
		1936	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		1937	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		1938	max->cur = ilk_cursor_wm_reg_max(dev, level);
		1939	max->fbc = ilk_fbc_wm_reg_max(dev);
		1940	}
		1941
4560	Serge	1942	static bool ilk_validate_wm_level(int level,
		1943	const struct ilk_wm_maximums *max,
6084	serge	1944	struct intel_wm_level *result)
4104	Serge	1945	{
		1946	bool ret;
		1947
		1948	/* already determined to be invalid? */
		1949	if (!result->enable)
		1950	return false;
		1951
		1952	result->enable = result->pri_val <= max->pri &&
		1953	result->spr_val <= max->spr &&
		1954	result->cur_val <= max->cur;
		1955
		1956	ret = result->enable;
		1957
		1958	/*
		1959	* HACK until we can pre-compute everything,
		1960	* and thus fail gracefully if LP0 watermarks
		1961	* are exceeded...
		1962	*/
		1963	if (level == 0 && !result->enable) {
		1964	if (result->pri_val > max->pri)
		1965	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		1966	level, result->pri_val, max->pri);
		1967	if (result->spr_val > max->spr)
		1968	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		1969	level, result->spr_val, max->spr);
		1970	if (result->cur_val > max->cur)
		1971	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		1972	level, result->cur_val, max->cur);
		1973
		1974	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		1975	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		1976	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		1977	result->enable = true;
		1978	}
		1979
		1980	return ret;
		1981	}
		1982
5060	serge	1983	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6084	serge	1984	const struct intel_crtc *intel_crtc,
4104	Serge	1985	int level,
6084	serge	1986	struct intel_crtc_state *cstate,
4104	Serge	1987	struct intel_wm_level *result)
		1988	{
6084	serge	1989	struct intel_plane *intel_plane;
4104	Serge	1990	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		1991	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		1992	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		1993
		1994	/* WM1+ latency values stored in 0.5us units */
		1995	if (level > 0) {
		1996	pri_latency *= 5;
		1997	spr_latency *= 5;
		1998	cur_latency *= 5;
		1999	}
		2000
6084	serge	2001	for_each_intel_plane_on_crtc(dev_priv->dev, intel_crtc, intel_plane) {
		2002	struct intel_plane_state *pstate =
		2003	to_intel_plane_state(intel_plane->base.state);
		2004
		2005	switch (intel_plane->base.type) {
		2006	case DRM_PLANE_TYPE_PRIMARY:
		2007	result->pri_val = ilk_compute_pri_wm(cstate, pstate,
		2008	pri_latency,
		2009	level);
		2010	result->fbc_val = ilk_compute_fbc_wm(cstate, pstate,
		2011	result->pri_val);
		2012	break;
		2013	case DRM_PLANE_TYPE_OVERLAY:
		2014	result->spr_val = ilk_compute_spr_wm(cstate, pstate,
		2015	spr_latency);
		2016	break;
		2017	case DRM_PLANE_TYPE_CURSOR:
		2018	result->cur_val = ilk_compute_cur_wm(cstate, pstate,
		2019	cur_latency);
		2020	break;
		2021	}
		2022	}
		2023
4104	Serge	2024	result->enable = true;
		2025	}
		2026
		2027	static uint32_t
		2028	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2029	{
3031	serge	2030	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2031	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	2032	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
4104	Serge	2033	u32 linetime, ips_linetime;
3031	serge	2034
6084	serge	2035	if (!intel_crtc->active)
4104	Serge	2036	return 0;
3031	serge	2037
		2038	/* The WM are computed with base on how long it takes to fill a single
		2039	* row at the given clock rate, multiplied by 8.
		2040	* */
6084	serge	2041	linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2042	adjusted_mode->crtc_clock);
		2043	ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2044	dev_priv->cdclk_freq);
3031	serge	2045
4104	Serge	2046	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2047	PIPE_WM_LINETIME_TIME(linetime);
		2048	}
		2049
5354	serge	2050	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
4104	Serge	2051	{
		2052	struct drm_i915_private *dev_priv = dev->dev_private;
		2053
5354	serge	2054	if (IS_GEN9(dev)) {
		2055	uint32_t val;
		2056	int ret, i;
		2057	int level, max_level = ilk_wm_max_level(dev);
		2058
		2059	/* read the first set of memory latencies[0:3] */
		2060	val = 0; /* data0 to be programmed to 0 for first set */
		2061	mutex_lock(&dev_priv->rps.hw_lock);
		2062	ret = sandybridge_pcode_read(dev_priv,
		2063	GEN9_PCODE_READ_MEM_LATENCY,
		2064	&val);
		2065	mutex_unlock(&dev_priv->rps.hw_lock);
		2066
		2067	if (ret) {
		2068	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2069	return;
		2070	}
		2071
		2072	wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2073	wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2074	GEN9_MEM_LATENCY_LEVEL_MASK;
		2075	wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2076	GEN9_MEM_LATENCY_LEVEL_MASK;
		2077	wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2078	GEN9_MEM_LATENCY_LEVEL_MASK;
		2079
		2080	/* read the second set of memory latencies[4:7] */
		2081	val = 1; /* data0 to be programmed to 1 for second set */
		2082	mutex_lock(&dev_priv->rps.hw_lock);
		2083	ret = sandybridge_pcode_read(dev_priv,
		2084	GEN9_PCODE_READ_MEM_LATENCY,
		2085	&val);
		2086	mutex_unlock(&dev_priv->rps.hw_lock);
		2087	if (ret) {
		2088	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2089	return;
		2090	}
		2091
		2092	wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2093	wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2094	GEN9_MEM_LATENCY_LEVEL_MASK;
		2095	wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2096	GEN9_MEM_LATENCY_LEVEL_MASK;
		2097	wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2098	GEN9_MEM_LATENCY_LEVEL_MASK;
		2099
		2100	/*
6084	serge	2101	* WaWmMemoryReadLatency:skl
		2102	*
5354	serge	2103	* punit doesn't take into account the read latency so we need
		2104	* to add 2us to the various latency levels we retrieve from
		2105	* the punit.
		2106	* - W0 is a bit special in that it's the only level that
		2107	* can't be disabled if we want to have display working, so
		2108	* we always add 2us there.
		2109	* - For levels >=1, punit returns 0us latency when they are
		2110	* disabled, so we respect that and don't add 2us then
		2111	*
		2112	* Additionally, if a level n (n > 1) has a 0us latency, all
		2113	* levels m (m >= n) need to be disabled. We make sure to
		2114	* sanitize the values out of the punit to satisfy this
		2115	* requirement.
		2116	*/
		2117	wm[0] += 2;
		2118	for (level = 1; level <= max_level; level++)
		2119	if (wm[level] != 0)
		2120	wm[level] += 2;
		2121	else {
		2122	for (i = level + 1; i <= max_level; i++)
		2123	wm[i] = 0;
		2124
		2125	break;
		2126	}
		2127	} else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2128	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2129
		2130	wm[0] = (sskpd >> 56) & 0xFF;
		2131	if (wm[0] == 0)
		2132	wm[0] = sskpd & 0xF;
		2133	wm[1] = (sskpd >> 4) & 0xFF;
		2134	wm[2] = (sskpd >> 12) & 0xFF;
		2135	wm[3] = (sskpd >> 20) & 0x1FF;
		2136	wm[4] = (sskpd >> 32) & 0x1FF;
		2137	} else if (INTEL_INFO(dev)->gen >= 6) {
		2138	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2139
		2140	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2141	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2142	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2143	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2144	} else if (INTEL_INFO(dev)->gen >= 5) {
		2145	uint32_t mltr = I915_READ(MLTR_ILK);
		2146
		2147	/* ILK primary LP0 latency is 700 ns */
		2148	wm[0] = 7;
		2149	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2150	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2151	}
		2152	}
		2153
		2154	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2155	{
		2156	/* ILK sprite LP0 latency is 1300 ns */
		2157	if (INTEL_INFO(dev)->gen == 5)
		2158	wm[0] = 13;
		2159	}
		2160
		2161	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2162	{
		2163	/* ILK cursor LP0 latency is 1300 ns */
		2164	if (INTEL_INFO(dev)->gen == 5)
		2165	wm[0] = 13;
		2166
		2167	/* WaDoubleCursorLP3Latency:ivb */
		2168	if (IS_IVYBRIDGE(dev))
		2169	wm[3] *= 2;
		2170	}
		2171
5060	serge	2172	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2173	{
		2174	/* how many WM levels are we expecting */
6084	serge	2175	if (INTEL_INFO(dev)->gen >= 9)
5354	serge	2176	return 7;
		2177	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
4560	Serge	2178	return 4;
		2179	else if (INTEL_INFO(dev)->gen >= 6)
		2180	return 3;
		2181	else
		2182	return 2;
		2183	}
		2184
4104	Serge	2185	static void intel_print_wm_latency(struct drm_device *dev,
		2186	const char *name,
5354	serge	2187	const uint16_t wm[8])
4104	Serge	2188	{
4560	Serge	2189	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2190
		2191	for (level = 0; level <= max_level; level++) {
		2192	unsigned int latency = wm[level];
		2193
		2194	if (latency == 0) {
		2195	DRM_ERROR("%s WM%d latency not provided\n",
		2196	name, level);
		2197	continue;
		2198	}
		2199
5354	serge	2200	/*
		2201	* - latencies are in us on gen9.
		2202	* - before then, WM1+ latency values are in 0.5us units
		2203	*/
		2204	if (IS_GEN9(dev))
		2205	latency *= 10;
		2206	else if (level > 0)
4104	Serge	2207	latency *= 5;
		2208
		2209	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2210	name, level, wm[level],
		2211	latency / 10, latency % 10);
		2212	}
		2213	}
		2214
5060	serge	2215	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2216	uint16_t wm[5], uint16_t min)
4104	Serge	2217	{
5060	serge	2218	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2219
		2220	if (wm[0] >= min)
		2221	return false;
		2222
		2223	wm[0] = max(wm[0], min);
		2224	for (level = 1; level <= max_level; level++)
		2225	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2226
		2227	return true;
		2228	}
		2229
		2230	static void snb_wm_latency_quirk(struct drm_device *dev)
		2231	{
4104	Serge	2232	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2233	bool changed;
4104	Serge	2234
5060	serge	2235	/*
		2236	* The BIOS provided WM memory latency values are often
		2237	* inadequate for high resolution displays. Adjust them.
		2238	*/
		2239	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2240	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2241	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2242
		2243	if (!changed)
		2244	return;
		2245
		2246	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2247	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2248	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2249	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2250	}
		2251
		2252	static void ilk_setup_wm_latency(struct drm_device *dev)
		2253	{
		2254	struct drm_i915_private *dev_priv = dev->dev_private;
		2255
4104	Serge	2256	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2257
		2258	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2259	sizeof(dev_priv->wm.pri_latency));
		2260	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2261	sizeof(dev_priv->wm.pri_latency));
		2262
		2263	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2264	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2265
		2266	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2267	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2268	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2269
		2270	if (IS_GEN6(dev))
		2271	snb_wm_latency_quirk(dev);
4104	Serge	2272	}
		2273
5354	serge	2274	static void skl_setup_wm_latency(struct drm_device *dev)
		2275	{
		2276	struct drm_i915_private *dev_priv = dev->dev_private;
		2277
		2278	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
		2279	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
		2280	}
		2281
5060	serge	2282	static void ilk_compute_wm_config(struct drm_device *dev,
		2283	struct intel_wm_config *config)
		2284	{
		2285	struct intel_crtc *intel_crtc;
		2286
		2287	/* Compute the currently _active_ config */
		2288	for_each_intel_crtc(dev, intel_crtc) {
		2289	const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
		2290
		2291	if (!wm->pipe_enabled)
		2292	continue;
		2293
		2294	config->sprites_enabled \|= wm->sprites_enabled;
		2295	config->sprites_scaled \|= wm->sprites_scaled;
		2296	config->num_pipes_active++;
4104	Serge	2297	}
4560	Serge	2298	}
4104	Serge	2299
4560	Serge	2300	/* Compute new watermarks for the pipe */
6084	serge	2301	static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
4560	Serge	2302	struct intel_pipe_wm *pipe_wm)
		2303	{
6084	serge	2304	struct drm_crtc *crtc = cstate->base.crtc;
4560	Serge	2305	struct drm_device *dev = crtc->dev;
5060	serge	2306	const struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	2307	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2308	struct intel_plane *intel_plane;
		2309	struct intel_plane_state *sprstate = NULL;
4560	Serge	2310	int level, max_level = ilk_wm_max_level(dev);
		2311	/* LP0 watermark maximums depend on this pipe alone */
		2312	struct intel_wm_config config = {
		2313	.num_pipes_active = 1,
		2314	};
		2315	struct ilk_wm_maximums max;
4104	Serge	2316
6084	serge	2317	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2318	if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) {
		2319	sprstate = to_intel_plane_state(intel_plane->base.state);
		2320	break;
		2321	}
		2322	}
4560	Serge	2323
6084	serge	2324	config.sprites_enabled = sprstate->visible;
		2325	config.sprites_scaled = sprstate->visible &&
		2326	(drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 \|\|
		2327	drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
		2328
		2329	pipe_wm->pipe_enabled = cstate->base.active;
		2330	pipe_wm->sprites_enabled = sprstate->visible;
		2331	pipe_wm->sprites_scaled = config.sprites_scaled;
		2332
4560	Serge	2333	/* ILK/SNB: LP2+ watermarks only w/o sprites */
6084	serge	2334	if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
4560	Serge	2335	max_level = 1;
		2336
		2337	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
6084	serge	2338	if (config.sprites_scaled)
4560	Serge	2339	max_level = 0;
		2340
6084	serge	2341	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, &pipe_wm->wm[0]);
4560	Serge	2342
		2343	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2344	pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
		2345
5060	serge	2346	/* LP0 watermarks always use 1/2 DDB partitioning */
		2347	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2348
4560	Serge	2349	/* At least LP0 must be valid */
5060	serge	2350	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
		2351	return false;
		2352
		2353	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2354
		2355	for (level = 1; level <= max_level; level++) {
		2356	struct intel_wm_level wm = {};
		2357
6084	serge	2358	ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, &wm);
5060	serge	2359
		2360	/*
		2361	* Disable any watermark level that exceeds the
		2362	* register maximums since such watermarks are
		2363	* always invalid.
		2364	*/
		2365	if (!ilk_validate_wm_level(level, &max, &wm))
		2366	break;
		2367
		2368	pipe_wm->wm[level] = wm;
		2369	}
		2370
		2371	return true;
4104	Serge	2372	}
		2373
4560	Serge	2374	/*
		2375	* Merge the watermarks from all active pipes for a specific level.
		2376	*/
		2377	static void ilk_merge_wm_level(struct drm_device *dev,
		2378	int level,
		2379	struct intel_wm_level *ret_wm)
4104	Serge	2380	{
4560	Serge	2381	const struct intel_crtc *intel_crtc;
4104	Serge	2382
5060	serge	2383	ret_wm->enable = true;
4104	Serge	2384
5060	serge	2385	for_each_intel_crtc(dev, intel_crtc) {
		2386	const struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2387	const struct intel_wm_level *wm = &active->wm[level];
		2388
		2389	if (!active->pipe_enabled)
		2390	continue;
		2391
		2392	/*
		2393	* The watermark values may have been used in the past,
		2394	* so we must maintain them in the registers for some
		2395	* time even if the level is now disabled.
		2396	*/
4560	Serge	2397	if (!wm->enable)
5060	serge	2398	ret_wm->enable = false;
4104	Serge	2399
4560	Serge	2400	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2401	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2402	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2403	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2404	}
		2405	}
		2406
		2407	/*
		2408	* Merge all low power watermarks for all active pipes.
		2409	*/
		2410	static void ilk_wm_merge(struct drm_device *dev,
		2411	const struct intel_wm_config *config,
		2412	const struct ilk_wm_maximums *max,
		2413	struct intel_pipe_wm *merged)
		2414	{
6084	serge	2415	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2416	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2417	int last_enabled_level = max_level;
4560	Serge	2418
		2419	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2420	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2421	config->num_pipes_active > 1)
		2422	return;
		2423
		2424	/* ILK: FBC WM must be disabled always */
		2425	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2426
		2427	/* merge each WM1+ level */
4104	Serge	2428	for (level = 1; level <= max_level; level++) {
4560	Serge	2429	struct intel_wm_level *wm = &merged->wm[level];
		2430
		2431	ilk_merge_wm_level(dev, level, wm);
		2432
5060	serge	2433	if (level > last_enabled_level)
		2434	wm->enable = false;
		2435	else if (!ilk_validate_wm_level(level, max, wm))
		2436	/* make sure all following levels get disabled */
		2437	last_enabled_level = level - 1;
4560	Serge	2438
		2439	/*
		2440	* The spec says it is preferred to disable
		2441	* FBC WMs instead of disabling a WM level.
		2442	*/
		2443	if (wm->fbc_val > max->fbc) {
5060	serge	2444	if (wm->enable)
6084	serge	2445	merged->fbc_wm_enabled = false;
4560	Serge	2446	wm->fbc_val = 0;
4104	Serge	2447	}
		2448	}
		2449
4560	Serge	2450	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2451	/*
		2452	* FIXME this is racy. FBC might get enabled later.
		2453	* What we should check here is whether FBC can be
		2454	* enabled sometime later.
		2455	*/
6084	serge	2456	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
		2457	intel_fbc_enabled(dev_priv)) {
4560	Serge	2458	for (level = 2; level <= max_level; level++) {
		2459	struct intel_wm_level *wm = &merged->wm[level];
		2460
		2461	wm->enable = false;
		2462	}
		2463	}
		2464	}
		2465
		2466	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2467	{
		2468	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2469	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2470	}
		2471
		2472	/* The value we need to program into the WM_LPx latency field */
		2473	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2474	{
		2475	struct drm_i915_private *dev_priv = dev->dev_private;
		2476
		2477	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2478	return 2 * level;
		2479	else
		2480	return dev_priv->wm.pri_latency[level];
		2481	}
		2482
		2483	static void ilk_compute_wm_results(struct drm_device *dev,
		2484	const struct intel_pipe_wm *merged,
		2485	enum intel_ddb_partitioning partitioning,
		2486	struct ilk_wm_values *results)
		2487	{
		2488	struct intel_crtc *intel_crtc;
		2489	int level, wm_lp;
		2490
		2491	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2492	results->partitioning = partitioning;
		2493
		2494	/* LP1+ register values */
4104	Serge	2495	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2496	const struct intel_wm_level *r;
		2497
4560	Serge	2498	level = ilk_wm_lp_to_level(wm_lp, merged);
		2499
		2500	r = &merged->wm[level];
4104	Serge	2501
5060	serge	2502	/*
		2503	* Maintain the watermark values even if the level is
		2504	* disabled. Doing otherwise could cause underruns.
		2505	*/
		2506	results->wm_lp[wm_lp - 1] =
4560	Serge	2507	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2508	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2509	r->cur_val;
		2510
5060	serge	2511	if (r->enable)
		2512	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2513
4560	Serge	2514	if (INTEL_INFO(dev)->gen >= 8)
		2515	results->wm_lp[wm_lp - 1] \|=
		2516	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2517	else
		2518	results->wm_lp[wm_lp - 1] \|=
		2519	r->fbc_val << WM1_LP_FBC_SHIFT;
		2520
5060	serge	2521	/*
		2522	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2523	* level is disabled. Doing otherwise could cause underruns.
		2524	*/
4560	Serge	2525	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2526	WARN_ON(wm_lp != 1);
		2527	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2528	} else
6084	serge	2529	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
4104	Serge	2530	}
		2531
4560	Serge	2532	/* LP0 register values */
5060	serge	2533	for_each_intel_crtc(dev, intel_crtc) {
4560	Serge	2534	enum pipe pipe = intel_crtc->pipe;
		2535	const struct intel_wm_level *r =
		2536	&intel_crtc->wm.active.wm[0];
4104	Serge	2537
4560	Serge	2538	if (WARN_ON(!r->enable))
		2539	continue;
		2540
		2541	results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
		2542
		2543	results->wm_pipe[pipe] =
		2544	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2545	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2546	r->cur_val;
4104	Serge	2547	}
		2548	}
		2549
		2550	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2551	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2552	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2553	struct intel_pipe_wm *r1,
		2554	struct intel_pipe_wm *r2)
4104	Serge	2555	{
4560	Serge	2556	int level, max_level = ilk_wm_max_level(dev);
		2557	int level1 = 0, level2 = 0;
4104	Serge	2558
4560	Serge	2559	for (level = 1; level <= max_level; level++) {
		2560	if (r1->wm[level].enable)
		2561	level1 = level;
		2562	if (r2->wm[level].enable)
		2563	level2 = level;
4104	Serge	2564	}
		2565
4560	Serge	2566	if (level1 == level2) {
		2567	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2568	return r2;
		2569	else
		2570	return r1;
4560	Serge	2571	} else if (level1 > level2) {
4104	Serge	2572	return r1;
		2573	} else {
		2574	return r2;
		2575	}
		2576	}
		2577
4560	Serge	2578	/* dirty bits used to track which watermarks need changes */
		2579	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2580	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2581	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2582	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2583	#define WM_DIRTY_FBC (1 << 24)
		2584	#define WM_DIRTY_DDB (1 << 25)
		2585
5354	serge	2586	static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
4560	Serge	2587	const struct ilk_wm_values *old,
		2588	const struct ilk_wm_values *new)
		2589	{
		2590	unsigned int dirty = 0;
		2591	enum pipe pipe;
		2592	int wm_lp;
		2593
5354	serge	2594	for_each_pipe(dev_priv, pipe) {
4560	Serge	2595	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2596	dirty \|= WM_DIRTY_LINETIME(pipe);
		2597	/* Must disable LP1+ watermarks too */
		2598	dirty \|= WM_DIRTY_LP_ALL;
		2599	}
		2600
		2601	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2602	dirty \|= WM_DIRTY_PIPE(pipe);
		2603	/* Must disable LP1+ watermarks too */
		2604	dirty \|= WM_DIRTY_LP_ALL;
		2605	}
		2606	}
		2607
		2608	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2609	dirty \|= WM_DIRTY_FBC;
		2610	/* Must disable LP1+ watermarks too */
		2611	dirty \|= WM_DIRTY_LP_ALL;
		2612	}
		2613
		2614	if (old->partitioning != new->partitioning) {
		2615	dirty \|= WM_DIRTY_DDB;
		2616	/* Must disable LP1+ watermarks too */
		2617	dirty \|= WM_DIRTY_LP_ALL;
		2618	}
		2619
		2620	/* LP1+ watermarks already deemed dirty, no need to continue */
		2621	if (dirty & WM_DIRTY_LP_ALL)
		2622	return dirty;
		2623
		2624	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2625	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2626	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2627	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2628	break;
		2629	}
		2630
		2631	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2632	for (; wm_lp <= 3; wm_lp++)
		2633	dirty \|= WM_DIRTY_LP(wm_lp);
		2634
		2635	return dirty;
		2636	}
		2637
		2638	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2639	unsigned int dirty)
		2640	{
		2641	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2642	bool changed = false;
		2643
		2644	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2645	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2646	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2647	changed = true;
		2648	}
		2649	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2650	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2651	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2652	changed = true;
		2653	}
		2654	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2655	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2656	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2657	changed = true;
		2658	}
		2659
		2660	/*
		2661	* Don't touch WM1S_LP_EN here.
		2662	* Doing so could cause underruns.
		2663	*/
		2664
		2665	return changed;
		2666	}
		2667
4104	Serge	2668	/*
		2669	* The spec says we shouldn't write when we don't need, because every write
		2670	* causes WMs to be re-evaluated, expending some power.
6084	serge	2671	*/
4560	Serge	2672	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2673	struct ilk_wm_values *results)
4104	Serge	2674	{
4560	Serge	2675	struct drm_device *dev = dev_priv->dev;
		2676	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2677	unsigned int dirty;
4104	Serge	2678	uint32_t val;
3031	serge	2679
5354	serge	2680	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
4560	Serge	2681	if (!dirty)
4104	Serge	2682	return;
		2683
4560	Serge	2684	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2685
4560	Serge	2686	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2687	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2688	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2689	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2690	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2691	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2692
4560	Serge	2693	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2694	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2695	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2696	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2697	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2698	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2699
4560	Serge	2700	if (dirty & WM_DIRTY_DDB) {
		2701	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
6084	serge	2702	val = I915_READ(WM_MISC);
4560	Serge	2703	if (results->partitioning == INTEL_DDB_PART_1_2)
6084	serge	2704	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2705	else
		2706	val \|= WM_MISC_DATA_PARTITION_5_6;
		2707	I915_WRITE(WM_MISC, val);
4560	Serge	2708	} else {
		2709	val = I915_READ(DISP_ARB_CTL2);
		2710	if (results->partitioning == INTEL_DDB_PART_1_2)
		2711	val &= ~DISP_DATA_PARTITION_5_6;
		2712	else
		2713	val \|= DISP_DATA_PARTITION_5_6;
		2714	I915_WRITE(DISP_ARB_CTL2, val);
		2715	}
4104	Serge	2716	}
		2717
4560	Serge	2718	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2719	val = I915_READ(DISP_ARB_CTL);
		2720	if (results->enable_fbc_wm)
		2721	val &= ~DISP_FBC_WM_DIS;
		2722	else
		2723	val \|= DISP_FBC_WM_DIS;
		2724	I915_WRITE(DISP_ARB_CTL, val);
		2725	}
		2726
4560	Serge	2727	if (dirty & WM_DIRTY_LP(1) &&
		2728	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2729	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2730
		2731	if (INTEL_INFO(dev)->gen >= 7) {
		2732	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
6084	serge	2733	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2734	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
6084	serge	2735	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2736	}
4104	Serge	2737
4560	Serge	2738	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2739	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2740	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2741	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2742	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2743	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2744
		2745	dev_priv->wm.hw = *results;
3031	serge	2746	}
		2747
4560	Serge	2748	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2749	{
		2750	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2751
		2752	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2753	}
		2754
5354	serge	2755	/*
		2756	* On gen9, we need to allocate Display Data Buffer (DDB) portions to the
		2757	* different active planes.
		2758	*/
		2759
		2760	#define SKL_DDB_SIZE 896 /* in blocks */
6084	serge	2761	#define BXT_DDB_SIZE 512
5354	serge	2762
		2763	static void
		2764	skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
		2765	struct drm_crtc *for_crtc,
		2766	const struct intel_wm_config *config,
		2767	const struct skl_pipe_wm_parameters *params,
		2768	struct skl_ddb_entry alloc / out */)
		2769	{
		2770	struct drm_crtc *crtc;
		2771	unsigned int pipe_size, ddb_size;
		2772	int nth_active_pipe;
		2773
		2774	if (!params->active) {
		2775	alloc->start = 0;
		2776	alloc->end = 0;
		2777	return;
		2778	}
		2779
6084	serge	2780	if (IS_BROXTON(dev))
		2781	ddb_size = BXT_DDB_SIZE;
		2782	else
		2783	ddb_size = SKL_DDB_SIZE;
5354	serge	2784
		2785	ddb_size -= 4; /* 4 blocks for bypass path allocation */
		2786
		2787	nth_active_pipe = 0;
		2788	for_each_crtc(dev, crtc) {
6084	serge	2789	if (!to_intel_crtc(crtc)->active)
5354	serge	2790	continue;
		2791
		2792	if (crtc == for_crtc)
		2793	break;
		2794
		2795	nth_active_pipe++;
		2796	}
		2797
		2798	pipe_size = ddb_size / config->num_pipes_active;
		2799	alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
		2800	alloc->end = alloc->start + pipe_size;
		2801	}
		2802
		2803	static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
		2804	{
		2805	if (config->num_pipes_active == 1)
		2806	return 32;
		2807
		2808	return 8;
		2809	}
		2810
		2811	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
		2812	{
		2813	entry->start = reg & 0x3ff;
		2814	entry->end = (reg >> 16) & 0x3ff;
		2815	if (entry->end)
		2816	entry->end += 1;
		2817	}
		2818
		2819	void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
		2820	struct skl_ddb_allocation ddb / out */)
		2821	{
		2822	enum pipe pipe;
		2823	int plane;
		2824	u32 val;
		2825
6084	serge	2826	memset(ddb, 0, sizeof(*ddb));
		2827
5354	serge	2828	for_each_pipe(dev_priv, pipe) {
6084	serge	2829	if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe)))
		2830	continue;
		2831
		2832	for_each_plane(dev_priv, pipe, plane) {
5354	serge	2833	val = I915_READ(PLANE_BUF_CFG(pipe, plane));
		2834	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
		2835	val);
		2836	}
		2837
		2838	val = I915_READ(CUR_BUF_CFG(pipe));
6084	serge	2839	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
		2840	val);
5354	serge	2841	}
		2842	}
		2843
		2844	static unsigned int
6084	serge	2845	skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y)
5354	serge	2846	{
6084	serge	2847
		2848	/* for planar format */
		2849	if (p->y_bytes_per_pixel) {
		2850	if (y) /* y-plane data rate */
		2851	return p->horiz_pixels * p->vert_pixels * p->y_bytes_per_pixel;
		2852	else /* uv-plane data rate */
		2853	return (p->horiz_pixels/2) * (p->vert_pixels/2) * p->bytes_per_pixel;
		2854	}
		2855
		2856	/* for packed formats */
5354	serge	2857	return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
		2858	}
		2859
		2860	/*
		2861	* We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
		2862	* a 8192x4096@32bpp framebuffer:
		2863	* 3 * 4096 * 8192 * 4 < 2^32
		2864	*/
		2865	static unsigned int
		2866	skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
		2867	const struct skl_pipe_wm_parameters *params)
		2868	{
		2869	unsigned int total_data_rate = 0;
		2870	int plane;
		2871
		2872	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		2873	const struct intel_plane_wm_parameters *p;
		2874
		2875	p = ¶ms->plane[plane];
		2876	if (!p->enabled)
		2877	continue;
		2878
6084	serge	2879	total_data_rate += skl_plane_relative_data_rate(p, 0); /* packed/uv */
		2880	if (p->y_bytes_per_pixel) {
		2881	total_data_rate += skl_plane_relative_data_rate(p, 1); /* y-plane */
		2882	}
5354	serge	2883	}
		2884
		2885	return total_data_rate;
		2886	}
		2887
		2888	static void
		2889	skl_allocate_pipe_ddb(struct drm_crtc *crtc,
		2890	const struct intel_wm_config *config,
		2891	const struct skl_pipe_wm_parameters *params,
		2892	struct skl_ddb_allocation ddb / out */)
		2893	{
		2894	struct drm_device *dev = crtc->dev;
6084	serge	2895	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	2896	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2897	enum pipe pipe = intel_crtc->pipe;
		2898	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
		2899	uint16_t alloc_size, start, cursor_blocks;
6084	serge	2900	uint16_t minimum[I915_MAX_PLANES];
		2901	uint16_t y_minimum[I915_MAX_PLANES];
5354	serge	2902	unsigned int total_data_rate;
		2903	int plane;
		2904
		2905	skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
		2906	alloc_size = skl_ddb_entry_size(alloc);
		2907	if (alloc_size == 0) {
		2908	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
6084	serge	2909	memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
		2910	sizeof(ddb->plane[pipe][PLANE_CURSOR]));
5354	serge	2911	return;
		2912	}
		2913
		2914	cursor_blocks = skl_cursor_allocation(config);
6084	serge	2915	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
		2916	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
5354	serge	2917
		2918	alloc_size -= cursor_blocks;
		2919	alloc->end -= cursor_blocks;
		2920
6084	serge	2921	/* 1. Allocate the mininum required blocks for each active plane */
		2922	for_each_plane(dev_priv, pipe, plane) {
		2923	const struct intel_plane_wm_parameters *p;
		2924
		2925	p = ¶ms->plane[plane];
		2926	if (!p->enabled)
		2927	continue;
		2928
		2929	minimum[plane] = 8;
		2930	alloc_size -= minimum[plane];
		2931	y_minimum[plane] = p->y_bytes_per_pixel ? 8 : 0;
		2932	alloc_size -= y_minimum[plane];
		2933	}
		2934
		2935	/*
		2936	* 2. Distribute the remaining space in proportion to the amount of
		2937	* data each plane needs to fetch from memory.
5354	serge	2938	*
		2939	* FIXME: we may not allocate every single block here.
6084	serge	2940	*/
5354	serge	2941	total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
		2942
		2943	start = alloc->start;
		2944	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		2945	const struct intel_plane_wm_parameters *p;
6084	serge	2946	unsigned int data_rate, y_data_rate;
		2947	uint16_t plane_blocks, y_plane_blocks = 0;
5354	serge	2948
		2949	p = ¶ms->plane[plane];
		2950	if (!p->enabled)
		2951	continue;
		2952
6084	serge	2953	data_rate = skl_plane_relative_data_rate(p, 0);
5354	serge	2954
		2955	/*
6084	serge	2956	* allocation for (packed formats) or (uv-plane part of planar format):
5354	serge	2957	* promote the expression to 64 bits to avoid overflowing, the
		2958	* result is < available as data_rate / total_data_rate < 1
		2959	*/
6084	serge	2960	plane_blocks = minimum[plane];
		2961	plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
		2962	total_data_rate);
5354	serge	2963
		2964	ddb->plane[pipe][plane].start = start;
		2965	ddb->plane[pipe][plane].end = start + plane_blocks;
		2966
		2967	start += plane_blocks;
6084	serge	2968
		2969	/*
		2970	* allocation for y_plane part of planar format:
		2971	*/
		2972	if (p->y_bytes_per_pixel) {
		2973	y_data_rate = skl_plane_relative_data_rate(p, 1);
		2974	y_plane_blocks = y_minimum[plane];
		2975	y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
		2976	total_data_rate);
		2977
		2978	ddb->y_plane[pipe][plane].start = start;
		2979	ddb->y_plane[pipe][plane].end = start + y_plane_blocks;
		2980
		2981	start += y_plane_blocks;
		2982	}
		2983
5354	serge	2984	}
		2985
		2986	}
		2987
6084	serge	2988	static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
5354	serge	2989	{
		2990	/* TODO: Take into account the scalers once we support them */
6084	serge	2991	return config->base.adjusted_mode.crtc_clock;
5354	serge	2992	}
		2993
		2994	/*
		2995	* The max latency should be 257 (max the punit can code is 255 and we add 2us
		2996	* for the read latency) and bytes_per_pixel should always be <= 8, so that
		2997	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
		2998	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
		2999	*/
		3000	static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		3001	uint32_t latency)
		3002	{
		3003	uint32_t wm_intermediate_val, ret;
		3004
		3005	if (latency == 0)
		3006	return UINT_MAX;
		3007
6084	serge	3008	wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
5354	serge	3009	ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
		3010
		3011	return ret;
		3012	}
		3013
		3014	static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		3015	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
6084	serge	3016	uint64_t tiling, uint32_t latency)
5354	serge	3017	{
6084	serge	3018	uint32_t ret;
		3019	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3020	uint32_t wm_intermediate_val;
5354	serge	3021
		3022	if (latency == 0)
		3023	return UINT_MAX;
		3024
		3025	plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
6084	serge	3026
		3027	if (tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3028	tiling == I915_FORMAT_MOD_Yf_TILED) {
		3029	plane_bytes_per_line *= 4;
		3030	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3031	plane_blocks_per_line /= 4;
		3032	} else {
		3033	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3034	}
		3035
5354	serge	3036	wm_intermediate_val = latency * pixel_rate;
		3037	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
6084	serge	3038	plane_blocks_per_line;
5354	serge	3039
		3040	return ret;
		3041	}
		3042
		3043	static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
		3044	const struct intel_crtc *intel_crtc)
		3045	{
		3046	struct drm_device *dev = intel_crtc->base.dev;
		3047	struct drm_i915_private *dev_priv = dev->dev_private;
		3048	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3049	enum pipe pipe = intel_crtc->pipe;
		3050
		3051	if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
		3052	sizeof(new_ddb->plane[pipe])))
		3053	return true;
		3054
6084	serge	3055	if (memcmp(&new_ddb->plane[pipe][PLANE_CURSOR], &cur_ddb->plane[pipe][PLANE_CURSOR],
		3056	sizeof(new_ddb->plane[pipe][PLANE_CURSOR])))
5354	serge	3057	return true;
		3058
		3059	return false;
		3060	}
		3061
		3062	static void skl_compute_wm_global_parameters(struct drm_device *dev,
		3063	struct intel_wm_config *config)
		3064	{
		3065	struct drm_crtc *crtc;
		3066	struct drm_plane *plane;
		3067
		3068	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
6084	serge	3069	config->num_pipes_active += to_intel_crtc(crtc)->active;
5354	serge	3070
		3071	/* FIXME: I don't think we need those two global parameters on SKL */
		3072	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3073	struct intel_plane *intel_plane = to_intel_plane(plane);
		3074
		3075	config->sprites_enabled \|= intel_plane->wm.enabled;
		3076	config->sprites_scaled \|= intel_plane->wm.scaled;
		3077	}
		3078	}
		3079
		3080	static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
		3081	struct skl_pipe_wm_parameters *p)
		3082	{
		3083	struct drm_device *dev = crtc->dev;
		3084	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3085	enum pipe pipe = intel_crtc->pipe;
		3086	struct drm_plane *plane;
6084	serge	3087	struct drm_framebuffer *fb;
5354	serge	3088	int i = 1; /* Index for sprite planes start */
		3089
6084	serge	3090	p->active = intel_crtc->active;
5354	serge	3091	if (p->active) {
6084	serge	3092	p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
		3093	p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
5354	serge	3094
6084	serge	3095	fb = crtc->primary->state->fb;
		3096	/* For planar: Bpp is for uv plane, y_Bpp is for y plane */
		3097	if (fb) {
		3098	p->plane[0].enabled = true;
		3099	p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
		3100	drm_format_plane_cpp(fb->pixel_format, 1) :
		3101	drm_format_plane_cpp(fb->pixel_format, 0);
		3102	p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
		3103	drm_format_plane_cpp(fb->pixel_format, 0) : 0;
		3104	p->plane[0].tiling = fb->modifier[0];
		3105	} else {
		3106	p->plane[0].enabled = false;
		3107	p->plane[0].bytes_per_pixel = 0;
		3108	p->plane[0].y_bytes_per_pixel = 0;
		3109	p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
		3110	}
		3111	p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
		3112	p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
		3113	p->plane[0].rotation = crtc->primary->state->rotation;
5354	serge	3114
6084	serge	3115	fb = crtc->cursor->state->fb;
		3116	p->plane[PLANE_CURSOR].y_bytes_per_pixel = 0;
		3117	if (fb) {
		3118	p->plane[PLANE_CURSOR].enabled = true;
		3119	p->plane[PLANE_CURSOR].bytes_per_pixel = fb->bits_per_pixel / 8;
		3120	p->plane[PLANE_CURSOR].horiz_pixels = crtc->cursor->state->crtc_w;
		3121	p->plane[PLANE_CURSOR].vert_pixels = crtc->cursor->state->crtc_h;
		3122	} else {
		3123	p->plane[PLANE_CURSOR].enabled = false;
		3124	p->plane[PLANE_CURSOR].bytes_per_pixel = 0;
		3125	p->plane[PLANE_CURSOR].horiz_pixels = 64;
		3126	p->plane[PLANE_CURSOR].vert_pixels = 64;
		3127	}
5354	serge	3128	}
		3129
		3130	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3131	struct intel_plane *intel_plane = to_intel_plane(plane);
		3132
6084	serge	3133	if (intel_plane->pipe == pipe &&
		3134	plane->type == DRM_PLANE_TYPE_OVERLAY)
5354	serge	3135	p->plane[i++] = intel_plane->wm;
		3136	}
		3137	}
		3138
6084	serge	3139	static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
		3140	struct skl_pipe_wm_parameters *p,
5354	serge	3141	struct intel_plane_wm_parameters *p_params,
		3142	uint16_t ddb_allocation,
6084	serge	3143	int level,
5354	serge	3144	uint16_t out_blocks, / out */
		3145	uint8_t out_lines / out */)
		3146	{
6084	serge	3147	uint32_t latency = dev_priv->wm.skl_latency[level];
		3148	uint32_t method1, method2;
		3149	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3150	uint32_t res_blocks, res_lines;
		3151	uint32_t selected_result;
		3152	uint8_t bytes_per_pixel;
5354	serge	3153
6084	serge	3154	if (latency == 0 \|\| !p->active \|\| !p_params->enabled)
5354	serge	3155	return false;
		3156
6084	serge	3157	bytes_per_pixel = p_params->y_bytes_per_pixel ?
		3158	p_params->y_bytes_per_pixel :
		3159	p_params->bytes_per_pixel;
5354	serge	3160	method1 = skl_wm_method1(p->pixel_rate,
6084	serge	3161	bytes_per_pixel,
		3162	latency);
5354	serge	3163	method2 = skl_wm_method2(p->pixel_rate,
		3164	p->pipe_htotal,
		3165	p_params->horiz_pixels,
6084	serge	3166	bytes_per_pixel,
		3167	p_params->tiling,
		3168	latency);
5354	serge	3169
6084	serge	3170	plane_bytes_per_line = p_params->horiz_pixels * bytes_per_pixel;
		3171	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
5354	serge	3172
6084	serge	3173	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3174	p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
		3175	uint32_t min_scanlines = 4;
		3176	uint32_t y_tile_minimum;
		3177	if (intel_rotation_90_or_270(p_params->rotation)) {
		3178	switch (p_params->bytes_per_pixel) {
		3179	case 1:
		3180	min_scanlines = 16;
		3181	break;
		3182	case 2:
		3183	min_scanlines = 8;
		3184	break;
		3185	case 8:
		3186	WARN(1, "Unsupported pixel depth for rotation");
		3187	}
		3188	}
		3189	y_tile_minimum = plane_blocks_per_line * min_scanlines;
		3190	selected_result = max(method2, y_tile_minimum);
		3191	} else {
		3192	if ((ddb_allocation / plane_blocks_per_line) >= 1)
		3193	selected_result = min(method1, method2);
		3194	else
		3195	selected_result = method1;
		3196	}
5354	serge	3197
6084	serge	3198	res_blocks = selected_result + 1;
		3199	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
5354	serge	3200
6084	serge	3201	if (level >= 1 && level <= 7) {
		3202	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3203	p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
		3204	res_lines += 4;
		3205	else
		3206	res_blocks++;
		3207	}
		3208
		3209	if (res_blocks >= ddb_allocation \|\| res_lines > 31)
5354	serge	3210	return false;
		3211
		3212	*out_blocks = res_blocks;
		3213	*out_lines = res_lines;
		3214
		3215	return true;
		3216	}
		3217
		3218	static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
		3219	struct skl_ddb_allocation *ddb,
		3220	struct skl_pipe_wm_parameters *p,
		3221	enum pipe pipe,
		3222	int level,
		3223	int num_planes,
		3224	struct skl_wm_level *result)
		3225	{
		3226	uint16_t ddb_blocks;
		3227	int i;
		3228
		3229	for (i = 0; i < num_planes; i++) {
		3230	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
		3231
6084	serge	3232	result->plane_en[i] = skl_compute_plane_wm(dev_priv,
		3233	p, &p->plane[i],
5354	serge	3234	ddb_blocks,
6084	serge	3235	level,
5354	serge	3236	&result->plane_res_b[i],
		3237	&result->plane_res_l[i]);
		3238	}
		3239
6084	serge	3240	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][PLANE_CURSOR]);
		3241	result->plane_en[PLANE_CURSOR] = skl_compute_plane_wm(dev_priv, p,
		3242	&p->plane[PLANE_CURSOR],
		3243	ddb_blocks, level,
		3244	&result->plane_res_b[PLANE_CURSOR],
		3245	&result->plane_res_l[PLANE_CURSOR]);
5354	serge	3246	}
		3247
		3248	static uint32_t
		3249	skl_compute_linetime_wm(struct drm_crtc crtc, struct skl_pipe_wm_parameters p)
		3250	{
6084	serge	3251	if (!to_intel_crtc(crtc)->active)
5354	serge	3252	return 0;
		3253
6084	serge	3254	if (WARN_ON(p->pixel_rate == 0))
		3255	return 0;
		3256
5354	serge	3257	return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
		3258	}
		3259
		3260	static void skl_compute_transition_wm(struct drm_crtc *crtc,
		3261	struct skl_pipe_wm_parameters *params,
		3262	struct skl_wm_level trans_wm / out */)
		3263	{
		3264	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3265	int i;
		3266
		3267	if (!params->active)
		3268	return;
		3269
		3270	/* Until we know more, just disable transition WMs */
		3271	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3272	trans_wm->plane_en[i] = false;
6084	serge	3273	trans_wm->plane_en[PLANE_CURSOR] = false;
5354	serge	3274	}
		3275
		3276	static void skl_compute_pipe_wm(struct drm_crtc *crtc,
		3277	struct skl_ddb_allocation *ddb,
		3278	struct skl_pipe_wm_parameters *params,
		3279	struct skl_pipe_wm *pipe_wm)
		3280	{
		3281	struct drm_device *dev = crtc->dev;
		3282	const struct drm_i915_private *dev_priv = dev->dev_private;
		3283	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3284	int level, max_level = ilk_wm_max_level(dev);
		3285
		3286	for (level = 0; level <= max_level; level++) {
		3287	skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
		3288	level, intel_num_planes(intel_crtc),
		3289	&pipe_wm->wm[level]);
		3290	}
		3291	pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
		3292
		3293	skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
		3294	}
		3295
		3296	static void skl_compute_wm_results(struct drm_device *dev,
		3297	struct skl_pipe_wm_parameters *p,
		3298	struct skl_pipe_wm *p_wm,
		3299	struct skl_wm_values *r,
		3300	struct intel_crtc *intel_crtc)
		3301	{
		3302	int level, max_level = ilk_wm_max_level(dev);
		3303	enum pipe pipe = intel_crtc->pipe;
		3304	uint32_t temp;
		3305	int i;
		3306
		3307	for (level = 0; level <= max_level; level++) {
		3308	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3309	temp = 0;
		3310
		3311	temp \|= p_wm->wm[level].plane_res_l[i] <<
		3312	PLANE_WM_LINES_SHIFT;
		3313	temp \|= p_wm->wm[level].plane_res_b[i];
		3314	if (p_wm->wm[level].plane_en[i])
		3315	temp \|= PLANE_WM_EN;
		3316
		3317	r->plane[pipe][i][level] = temp;
		3318	}
		3319
		3320	temp = 0;
		3321
6084	serge	3322	temp \|= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3323	temp \|= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
5354	serge	3324
6084	serge	3325	if (p_wm->wm[level].plane_en[PLANE_CURSOR])
5354	serge	3326	temp \|= PLANE_WM_EN;
		3327
6084	serge	3328	r->plane[pipe][PLANE_CURSOR][level] = temp;
5354	serge	3329
		3330	}
		3331
		3332	/* transition WMs */
		3333	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3334	temp = 0;
		3335	temp \|= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
		3336	temp \|= p_wm->trans_wm.plane_res_b[i];
		3337	if (p_wm->trans_wm.plane_en[i])
		3338	temp \|= PLANE_WM_EN;
		3339
		3340	r->plane_trans[pipe][i] = temp;
		3341	}
		3342
		3343	temp = 0;
6084	serge	3344	temp \|= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3345	temp \|= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
		3346	if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
5354	serge	3347	temp \|= PLANE_WM_EN;
		3348
6084	serge	3349	r->plane_trans[pipe][PLANE_CURSOR] = temp;
5354	serge	3350
		3351	r->wm_linetime[pipe] = p_wm->linetime;
		3352	}
		3353
		3354	static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
		3355	const struct skl_ddb_entry *entry)
		3356	{
		3357	if (entry->end)
		3358	I915_WRITE(reg, (entry->end - 1) << 16 \| entry->start);
		3359	else
		3360	I915_WRITE(reg, 0);
		3361	}
		3362
		3363	static void skl_write_wm_values(struct drm_i915_private *dev_priv,
		3364	const struct skl_wm_values *new)
		3365	{
		3366	struct drm_device *dev = dev_priv->dev;
		3367	struct intel_crtc *crtc;
		3368
		3369	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
		3370	int i, level, max_level = ilk_wm_max_level(dev);
		3371	enum pipe pipe = crtc->pipe;
		3372
		3373	if (!new->dirty[pipe])
		3374	continue;
		3375
		3376	I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
		3377
		3378	for (level = 0; level <= max_level; level++) {
		3379	for (i = 0; i < intel_num_planes(crtc); i++)
		3380	I915_WRITE(PLANE_WM(pipe, i, level),
		3381	new->plane[pipe][i][level]);
		3382	I915_WRITE(CUR_WM(pipe, level),
6084	serge	3383	new->plane[pipe][PLANE_CURSOR][level]);
5354	serge	3384	}
		3385	for (i = 0; i < intel_num_planes(crtc); i++)
		3386	I915_WRITE(PLANE_WM_TRANS(pipe, i),
		3387	new->plane_trans[pipe][i]);
6084	serge	3388	I915_WRITE(CUR_WM_TRANS(pipe),
		3389	new->plane_trans[pipe][PLANE_CURSOR]);
5354	serge	3390
6084	serge	3391	for (i = 0; i < intel_num_planes(crtc); i++) {
5354	serge	3392	skl_ddb_entry_write(dev_priv,
		3393	PLANE_BUF_CFG(pipe, i),
		3394	&new->ddb.plane[pipe][i]);
6084	serge	3395	skl_ddb_entry_write(dev_priv,
		3396	PLANE_NV12_BUF_CFG(pipe, i),
		3397	&new->ddb.y_plane[pipe][i]);
		3398	}
5354	serge	3399
		3400	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
6084	serge	3401	&new->ddb.plane[pipe][PLANE_CURSOR]);
5354	serge	3402	}
		3403	}
		3404
		3405	/*
		3406	* When setting up a new DDB allocation arrangement, we need to correctly
		3407	* sequence the times at which the new allocations for the pipes are taken into
		3408	* account or we'll have pipes fetching from space previously allocated to
		3409	* another pipe.
		3410	*
		3411	* Roughly the sequence looks like:
		3412	* 1. re-allocate the pipe(s) with the allocation being reduced and not
		3413	* overlapping with a previous light-up pipe (another way to put it is:
		3414	* pipes with their new allocation strickly included into their old ones).
		3415	* 2. re-allocate the other pipes that get their allocation reduced
		3416	* 3. allocate the pipes having their allocation increased
		3417	*
		3418	* Steps 1. and 2. are here to take care of the following case:
		3419	* - Initially DDB looks like this:
		3420	* \| B \| C \|
		3421	* - enable pipe A.
		3422	* - pipe B has a reduced DDB allocation that overlaps with the old pipe C
		3423	* allocation
		3424	* \| A \| B \| C \|
		3425	*
		3426	* We need to sequence the re-allocation: C, B, A (and not B, C, A).
		3427	*/
		3428
		3429	static void
		3430	skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
		3431	{
		3432	int plane;
		3433
		3434	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
		3435
6084	serge	3436	for_each_plane(dev_priv, pipe, plane) {
5354	serge	3437	I915_WRITE(PLANE_SURF(pipe, plane),
		3438	I915_READ(PLANE_SURF(pipe, plane)));
		3439	}
		3440	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
		3441	}
		3442
		3443	static bool
		3444	skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
		3445	const struct skl_ddb_allocation *new,
		3446	enum pipe pipe)
		3447	{
		3448	uint16_t old_size, new_size;
		3449
		3450	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
		3451	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
		3452
		3453	return old_size != new_size &&
		3454	new->pipe[pipe].start >= old->pipe[pipe].start &&
		3455	new->pipe[pipe].end <= old->pipe[pipe].end;
		3456	}
		3457
		3458	static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
		3459	struct skl_wm_values *new_values)
		3460	{
		3461	struct drm_device *dev = dev_priv->dev;
		3462	struct skl_ddb_allocation cur_ddb, new_ddb;
6084	serge	3463	bool reallocated[I915_MAX_PIPES] = {};
5354	serge	3464	struct intel_crtc *crtc;
		3465	enum pipe pipe;
		3466
		3467	new_ddb = &new_values->ddb;
		3468	cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3469
		3470	/*
		3471	* First pass: flush the pipes with the new allocation contained into
		3472	* the old space.
		3473	*
		3474	* We'll wait for the vblank on those pipes to ensure we can safely
		3475	* re-allocate the freed space without this pipe fetching from it.
		3476	*/
		3477	for_each_intel_crtc(dev, crtc) {
		3478	if (!crtc->active)
		3479	continue;
		3480
		3481	pipe = crtc->pipe;
		3482
		3483	if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
		3484	continue;
		3485
		3486	skl_wm_flush_pipe(dev_priv, pipe, 1);
		3487	intel_wait_for_vblank(dev, pipe);
		3488
		3489	reallocated[pipe] = true;
		3490	}
		3491
		3492
		3493	/*
		3494	* Second pass: flush the pipes that are having their allocation
		3495	* reduced, but overlapping with a previous allocation.
		3496	*
		3497	* Here as well we need to wait for the vblank to make sure the freed
		3498	* space is not used anymore.
		3499	*/
		3500	for_each_intel_crtc(dev, crtc) {
		3501	if (!crtc->active)
		3502	continue;
		3503
		3504	pipe = crtc->pipe;
		3505
		3506	if (reallocated[pipe])
		3507	continue;
		3508
		3509	if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
		3510	skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
		3511	skl_wm_flush_pipe(dev_priv, pipe, 2);
		3512	intel_wait_for_vblank(dev, pipe);
6084	serge	3513	reallocated[pipe] = true;
5354	serge	3514	}
		3515	}
		3516
		3517	/*
		3518	* Third pass: flush the pipes that got more space allocated.
		3519	*
		3520	* We don't need to actively wait for the update here, next vblank
		3521	* will just get more DDB space with the correct WM values.
		3522	*/
		3523	for_each_intel_crtc(dev, crtc) {
		3524	if (!crtc->active)
		3525	continue;
		3526
		3527	pipe = crtc->pipe;
		3528
		3529	/*
		3530	* At this point, only the pipes more space than before are
		3531	* left to re-allocate.
		3532	*/
		3533	if (reallocated[pipe])
		3534	continue;
		3535
		3536	skl_wm_flush_pipe(dev_priv, pipe, 3);
		3537	}
		3538	}
		3539
		3540	static bool skl_update_pipe_wm(struct drm_crtc *crtc,
		3541	struct skl_pipe_wm_parameters *params,
		3542	struct intel_wm_config *config,
		3543	struct skl_ddb_allocation ddb, / out */
		3544	struct skl_pipe_wm pipe_wm / out */)
		3545	{
		3546	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3547
		3548	skl_compute_wm_pipe_parameters(crtc, params);
		3549	skl_allocate_pipe_ddb(crtc, config, params, ddb);
		3550	skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
		3551
		3552	if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
		3553	return false;
		3554
		3555	intel_crtc->wm.skl_active = *pipe_wm;
6084	serge	3556
5354	serge	3557	return true;
		3558	}
		3559
		3560	static void skl_update_other_pipe_wm(struct drm_device *dev,
		3561	struct drm_crtc *crtc,
		3562	struct intel_wm_config *config,
		3563	struct skl_wm_values *r)
		3564	{
		3565	struct intel_crtc *intel_crtc;
		3566	struct intel_crtc *this_crtc = to_intel_crtc(crtc);
		3567
		3568	/*
		3569	* If the WM update hasn't changed the allocation for this_crtc (the
		3570	* crtc we are currently computing the new WM values for), other
		3571	* enabled crtcs will keep the same allocation and we don't need to
		3572	* recompute anything for them.
		3573	*/
		3574	if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
		3575	return;
		3576
		3577	/*
		3578	* Otherwise, because of this_crtc being freshly enabled/disabled, the
		3579	* other active pipes need new DDB allocation and WM values.
		3580	*/
		3581	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
		3582	base.head) {
		3583	struct skl_pipe_wm_parameters params = {};
		3584	struct skl_pipe_wm pipe_wm = {};
		3585	bool wm_changed;
		3586
		3587	if (this_crtc->pipe == intel_crtc->pipe)
		3588	continue;
		3589
		3590	if (!intel_crtc->active)
		3591	continue;
		3592
		3593	wm_changed = skl_update_pipe_wm(&intel_crtc->base,
		3594	¶ms, config,
		3595	&r->ddb, &pipe_wm);
		3596
		3597	/*
		3598	* If we end up re-computing the other pipe WM values, it's
		3599	* because it was really needed, so we expect the WM values to
		3600	* be different.
6084	serge	3601	*/
5354	serge	3602	WARN_ON(!wm_changed);
		3603
		3604	skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc);
		3605	r->dirty[intel_crtc->pipe] = true;
		3606	}
		3607	}
		3608
6084	serge	3609	static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
		3610	{
		3611	watermarks->wm_linetime[pipe] = 0;
		3612	memset(watermarks->plane[pipe], 0,
		3613	sizeof(uint32_t) * 8 * I915_MAX_PLANES);
		3614	memset(watermarks->plane_trans[pipe],
		3615	0, sizeof(uint32_t) * I915_MAX_PLANES);
		3616	watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
		3617
		3618	/* Clear ddb entries for pipe */
		3619	memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
		3620	memset(&watermarks->ddb.plane[pipe], 0,
		3621	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3622	memset(&watermarks->ddb.y_plane[pipe], 0,
		3623	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3624	memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
		3625	sizeof(struct skl_ddb_entry));
		3626
		3627	}
		3628
5354	serge	3629	static void skl_update_wm(struct drm_crtc *crtc)
		3630	{
		3631	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3632	struct drm_device *dev = crtc->dev;
		3633	struct drm_i915_private *dev_priv = dev->dev_private;
		3634	struct skl_pipe_wm_parameters params = {};
		3635	struct skl_wm_values *results = &dev_priv->wm.skl_results;
		3636	struct skl_pipe_wm pipe_wm = {};
		3637	struct intel_wm_config config = {};
		3638
		3639
6084	serge	3640	/* Clear all dirty flags */
		3641	memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
		3642
		3643	skl_clear_wm(results, intel_crtc->pipe);
		3644
5354	serge	3645	skl_compute_wm_global_parameters(dev, &config);
		3646
		3647	if (!skl_update_pipe_wm(crtc, ¶ms, &config,
		3648	&results->ddb, &pipe_wm))
		3649	return;
		3650
		3651	skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc);
		3652	results->dirty[intel_crtc->pipe] = true;
		3653
		3654	skl_update_other_pipe_wm(dev, crtc, &config, results);
		3655	skl_write_wm_values(dev_priv, results);
		3656	skl_flush_wm_values(dev_priv, results);
		3657
		3658	/* store the new configuration */
		3659	dev_priv->wm.skl_hw = *results;
		3660	}
		3661
		3662	static void
		3663	skl_update_sprite_wm(struct drm_plane plane, struct drm_crtc crtc,
		3664	uint32_t sprite_width, uint32_t sprite_height,
		3665	int pixel_size, bool enabled, bool scaled)
		3666	{
		3667	struct intel_plane *intel_plane = to_intel_plane(plane);
6084	serge	3668	struct drm_framebuffer *fb = plane->state->fb;
5354	serge	3669
		3670	intel_plane->wm.enabled = enabled;
		3671	intel_plane->wm.scaled = scaled;
		3672	intel_plane->wm.horiz_pixels = sprite_width;
		3673	intel_plane->wm.vert_pixels = sprite_height;
6084	serge	3674	intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
5354	serge	3675
6084	serge	3676	/* For planar: Bpp is for UV plane, y_Bpp is for Y plane */
		3677	intel_plane->wm.bytes_per_pixel =
		3678	(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
		3679	drm_format_plane_cpp(plane->state->fb->pixel_format, 1) : pixel_size;
		3680	intel_plane->wm.y_bytes_per_pixel =
		3681	(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
		3682	drm_format_plane_cpp(plane->state->fb->pixel_format, 0) : 0;
		3683
		3684	/*
		3685	* Framebuffer can be NULL on plane disable, but it does not
		3686	* matter for watermarks if we assume no tiling in that case.
		3687	*/
		3688	if (fb)
		3689	intel_plane->wm.tiling = fb->modifier[0];
		3690	intel_plane->wm.rotation = plane->state->rotation;
		3691
5354	serge	3692	skl_update_wm(crtc);
		3693	}
		3694
4560	Serge	3695	static void ilk_update_wm(struct drm_crtc *crtc)
		3696	{
		3697	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	3698	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
4560	Serge	3699	struct drm_device *dev = crtc->dev;
		3700	struct drm_i915_private *dev_priv = dev->dev_private;
		3701	struct ilk_wm_maximums max;
		3702	struct ilk_wm_values results = {};
4104	Serge	3703	enum intel_ddb_partitioning partitioning;
4560	Serge	3704	struct intel_pipe_wm pipe_wm = {};
		3705	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
		3706	struct intel_wm_config config = {};
4104	Serge	3707
6084	serge	3708	WARN_ON(cstate->base.active != intel_crtc->active);
4104	Serge	3709
6084	serge	3710	intel_compute_pipe_wm(cstate, &pipe_wm);
4560	Serge	3711
		3712	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
		3713	return;
		3714
		3715	intel_crtc->wm.active = pipe_wm;
		3716
5060	serge	3717	ilk_compute_wm_config(dev, &config);
		3718
4560	Serge	3719	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		3720	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		3721
		3722	/* 5/6 split only in single pipe config on IVB+ */
		3723	if (INTEL_INFO(dev)->gen >= 7 &&
		3724	config.num_pipes_active == 1 && config.sprites_enabled) {
		3725	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		3726	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		3727
		3728	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	3729	} else {
4560	Serge	3730	best_lp_wm = &lp_wm_1_2;
4104	Serge	3731	}
		3732
4560	Serge	3733	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	3734	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		3735
4560	Serge	3736	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		3737
		3738	ilk_write_wm_values(dev_priv, &results);
4104	Serge	3739	}
		3740
5060	serge	3741	static void
		3742	ilk_update_sprite_wm(struct drm_plane *plane,
5354	serge	3743	struct drm_crtc *crtc,
5060	serge	3744	uint32_t sprite_width, uint32_t sprite_height,
		3745	int pixel_size, bool enabled, bool scaled)
4104	Serge	3746	{
4560	Serge	3747	struct drm_device *dev = plane->dev;
5354	serge	3748	struct intel_plane *intel_plane = to_intel_plane(plane);
4104	Serge	3749
4560	Serge	3750	/*
		3751	* IVB workaround: must disable low power watermarks for at least
		3752	* one frame before enabling scaling. LP watermarks can be re-enabled
		3753	* when scaling is disabled.
		3754	*
		3755	* WaCxSRDisabledForSpriteScaling:ivb
		3756	*/
		3757	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
		3758	intel_wait_for_vblank(dev, intel_plane->pipe);
		3759
		3760	ilk_update_wm(crtc);
4104	Serge	3761	}
		3762
5354	serge	3763	static void skl_pipe_wm_active_state(uint32_t val,
		3764	struct skl_pipe_wm *active,
		3765	bool is_transwm,
		3766	bool is_cursor,
		3767	int i,
		3768	int level)
		3769	{
		3770	bool is_enabled = (val & PLANE_WM_EN) != 0;
		3771
		3772	if (!is_transwm) {
		3773	if (!is_cursor) {
		3774	active->wm[level].plane_en[i] = is_enabled;
		3775	active->wm[level].plane_res_b[i] =
		3776	val & PLANE_WM_BLOCKS_MASK;
		3777	active->wm[level].plane_res_l[i] =
		3778	(val >> PLANE_WM_LINES_SHIFT) &
		3779	PLANE_WM_LINES_MASK;
		3780	} else {
6084	serge	3781	active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
		3782	active->wm[level].plane_res_b[PLANE_CURSOR] =
5354	serge	3783	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3784	active->wm[level].plane_res_l[PLANE_CURSOR] =
5354	serge	3785	(val >> PLANE_WM_LINES_SHIFT) &
		3786	PLANE_WM_LINES_MASK;
		3787	}
		3788	} else {
		3789	if (!is_cursor) {
		3790	active->trans_wm.plane_en[i] = is_enabled;
		3791	active->trans_wm.plane_res_b[i] =
		3792	val & PLANE_WM_BLOCKS_MASK;
		3793	active->trans_wm.plane_res_l[i] =
		3794	(val >> PLANE_WM_LINES_SHIFT) &
		3795	PLANE_WM_LINES_MASK;
		3796	} else {
6084	serge	3797	active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
		3798	active->trans_wm.plane_res_b[PLANE_CURSOR] =
5354	serge	3799	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3800	active->trans_wm.plane_res_l[PLANE_CURSOR] =
5354	serge	3801	(val >> PLANE_WM_LINES_SHIFT) &
		3802	PLANE_WM_LINES_MASK;
		3803	}
		3804	}
		3805	}
		3806
		3807	static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
		3808	{
		3809	struct drm_device *dev = crtc->dev;
		3810	struct drm_i915_private *dev_priv = dev->dev_private;
		3811	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
		3812	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3813	struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
		3814	enum pipe pipe = intel_crtc->pipe;
		3815	int level, i, max_level;
		3816	uint32_t temp;
		3817
		3818	max_level = ilk_wm_max_level(dev);
		3819
		3820	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
		3821
		3822	for (level = 0; level <= max_level; level++) {
		3823	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3824	hw->plane[pipe][i][level] =
		3825	I915_READ(PLANE_WM(pipe, i, level));
6084	serge	3826	hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
5354	serge	3827	}
		3828
		3829	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3830	hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
6084	serge	3831	hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
5354	serge	3832
6084	serge	3833	if (!intel_crtc->active)
5354	serge	3834	return;
		3835
		3836	hw->dirty[pipe] = true;
		3837
		3838	active->linetime = hw->wm_linetime[pipe];
		3839
		3840	for (level = 0; level <= max_level; level++) {
		3841	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3842	temp = hw->plane[pipe][i][level];
		3843	skl_pipe_wm_active_state(temp, active, false,
		3844	false, i, level);
		3845	}
6084	serge	3846	temp = hw->plane[pipe][PLANE_CURSOR][level];
5354	serge	3847	skl_pipe_wm_active_state(temp, active, false, true, i, level);
		3848	}
		3849
		3850	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3851	temp = hw->plane_trans[pipe][i];
		3852	skl_pipe_wm_active_state(temp, active, true, false, i, 0);
		3853	}
		3854
6084	serge	3855	temp = hw->plane_trans[pipe][PLANE_CURSOR];
5354	serge	3856	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
		3857	}
		3858
		3859	void skl_wm_get_hw_state(struct drm_device *dev)
		3860	{
		3861	struct drm_i915_private *dev_priv = dev->dev_private;
		3862	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
		3863	struct drm_crtc *crtc;
		3864
		3865	skl_ddb_get_hw_state(dev_priv, ddb);
		3866	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3867	skl_pipe_wm_get_hw_state(crtc);
		3868	}
		3869
4560	Serge	3870	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	3871	{
4560	Serge	3872	struct drm_device *dev = crtc->dev;
		3873	struct drm_i915_private *dev_priv = dev->dev_private;
		3874	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		3875	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3876	struct intel_pipe_wm *active = &intel_crtc->wm.active;
		3877	enum pipe pipe = intel_crtc->pipe;
		3878	static const unsigned int wm0_pipe_reg[] = {
		3879	[PIPE_A] = WM0_PIPEA_ILK,
		3880	[PIPE_B] = WM0_PIPEB_ILK,
		3881	[PIPE_C] = WM0_PIPEC_IVB,
		3882	};
3031	serge	3883
4560	Serge	3884	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		3885	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		3886	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	3887
6084	serge	3888	active->pipe_enabled = intel_crtc->active;
5060	serge	3889
		3890	if (active->pipe_enabled) {
4560	Serge	3891	u32 tmp = hw->wm_pipe[pipe];
3031	serge	3892
4560	Serge	3893	/*
		3894	* For active pipes LP0 watermark is marked as
		3895	* enabled, and LP1+ watermaks as disabled since
		3896	* we can't really reverse compute them in case
		3897	* multiple pipes are active.
		3898	*/
		3899	active->wm[0].enable = true;
		3900	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		3901	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		3902	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		3903	active->linetime = hw->wm_linetime[pipe];
		3904	} else {
		3905	int level, max_level = ilk_wm_max_level(dev);
3031	serge	3906
4560	Serge	3907	/*
		3908	* For inactive pipes, all watermark levels
		3909	* should be marked as enabled but zeroed,
		3910	* which is what we'd compute them to.
		3911	*/
		3912	for (level = 0; level <= max_level; level++)
		3913	active->wm[level].enable = true;
3031	serge	3914	}
		3915	}
		3916
6084	serge	3917	#define _FW_WM(value, plane) \
		3918	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
		3919	#define _FW_WM_VLV(value, plane) \
		3920	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
		3921
		3922	static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
		3923	struct vlv_wm_values *wm)
		3924	{
		3925	enum pipe pipe;
		3926	uint32_t tmp;
		3927
		3928	for_each_pipe(dev_priv, pipe) {
		3929	tmp = I915_READ(VLV_DDL(pipe));
		3930
		3931	wm->ddl[pipe].primary =
		3932	(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3933	wm->ddl[pipe].cursor =
		3934	(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3935	wm->ddl[pipe].sprite[0] =
		3936	(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3937	wm->ddl[pipe].sprite[1] =
		3938	(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3939	}
		3940
		3941	tmp = I915_READ(DSPFW1);
		3942	wm->sr.plane = _FW_WM(tmp, SR);
		3943	wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
		3944	wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
		3945	wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
		3946
		3947	tmp = I915_READ(DSPFW2);
		3948	wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
		3949	wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
		3950	wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
		3951
		3952	tmp = I915_READ(DSPFW3);
		3953	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
		3954
		3955	if (IS_CHERRYVIEW(dev_priv)) {
		3956	tmp = I915_READ(DSPFW7_CHV);
		3957	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3958	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3959
		3960	tmp = I915_READ(DSPFW8_CHV);
		3961	wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
		3962	wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
		3963
		3964	tmp = I915_READ(DSPFW9_CHV);
		3965	wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
		3966	wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
		3967
		3968	tmp = I915_READ(DSPHOWM);
		3969	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3970	wm->pipe[PIPE_C].sprite[1] \|= _FW_WM(tmp, SPRITEF_HI) << 8;
		3971	wm->pipe[PIPE_C].sprite[0] \|= _FW_WM(tmp, SPRITEE_HI) << 8;
		3972	wm->pipe[PIPE_C].primary \|= _FW_WM(tmp, PLANEC_HI) << 8;
		3973	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3974	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3975	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3976	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3977	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3978	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3979	} else {
		3980	tmp = I915_READ(DSPFW7);
		3981	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3982	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3983
		3984	tmp = I915_READ(DSPHOWM);
		3985	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3986	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3987	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3988	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3989	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3990	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3991	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3992	}
		3993	}
		3994
		3995	#undef _FW_WM
		3996	#undef _FW_WM_VLV
		3997
		3998	void vlv_wm_get_hw_state(struct drm_device *dev)
		3999	{
		4000	struct drm_i915_private *dev_priv = to_i915(dev);
		4001	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
		4002	struct intel_plane *plane;
		4003	enum pipe pipe;
		4004	u32 val;
		4005
		4006	vlv_read_wm_values(dev_priv, wm);
		4007
		4008	for_each_intel_plane(dev, plane) {
		4009	switch (plane->base.type) {
		4010	int sprite;
		4011	case DRM_PLANE_TYPE_CURSOR:
		4012	plane->wm.fifo_size = 63;
		4013	break;
		4014	case DRM_PLANE_TYPE_PRIMARY:
		4015	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
		4016	break;
		4017	case DRM_PLANE_TYPE_OVERLAY:
		4018	sprite = plane->plane;
		4019	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
		4020	break;
		4021	}
		4022	}
		4023
		4024	wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
		4025	wm->level = VLV_WM_LEVEL_PM2;
		4026
		4027	if (IS_CHERRYVIEW(dev_priv)) {
		4028	mutex_lock(&dev_priv->rps.hw_lock);
		4029
		4030	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		4031	if (val & DSP_MAXFIFO_PM5_ENABLE)
		4032	wm->level = VLV_WM_LEVEL_PM5;
		4033
		4034	/*
		4035	* If DDR DVFS is disabled in the BIOS, Punit
		4036	* will never ack the request. So if that happens
		4037	* assume we don't have to enable/disable DDR DVFS
		4038	* dynamically. To test that just set the REQ_ACK
		4039	* bit to poke the Punit, but don't change the
		4040	* HIGH/LOW bits so that we don't actually change
		4041	* the current state.
		4042	*/
		4043	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4044	val \|= FORCE_DDR_FREQ_REQ_ACK;
		4045	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		4046
		4047	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		4048	FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
		4049	DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
		4050	"assuming DDR DVFS is disabled\n");
		4051	dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		4052	} else {
		4053	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4054	if ((val & FORCE_DDR_HIGH_FREQ) == 0)
		4055	wm->level = VLV_WM_LEVEL_DDR_DVFS;
		4056	}
		4057
		4058	mutex_unlock(&dev_priv->rps.hw_lock);
		4059	}
		4060
		4061	for_each_pipe(dev_priv, pipe)
		4062	DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
		4063	pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
		4064	wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
		4065
		4066	DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		4067	wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
		4068	}
		4069
4560	Serge	4070	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	4071	{
		4072	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4073	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		4074	struct drm_crtc *crtc;
3031	serge	4075
5060	serge	4076	for_each_crtc(dev, crtc)
4560	Serge	4077	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	4078
4560	Serge	4079	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		4080	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		4081	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	4082
4560	Serge	4083	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	4084	if (INTEL_INFO(dev)->gen >= 7) {
5354	serge	4085	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		4086	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	4087	}
3031	serge	4088
4560	Serge	4089	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4090	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		4091	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		4092	else if (IS_IVYBRIDGE(dev))
		4093	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		4094	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	4095
4560	Serge	4096	hw->enable_fbc_wm =
		4097	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	4098	}
		4099
		4100	/**
		4101	* intel_update_watermarks - update FIFO watermark values based on current modes
		4102	*
		4103	* Calculate watermark values for the various WM regs based on current mode
		4104	* and plane configuration.
		4105	*
		4106	* There are several cases to deal with here:
		4107	* - normal (i.e. non-self-refresh)
		4108	* - self-refresh (SR) mode
		4109	* - lines are large relative to FIFO size (buffer can hold up to 2)
		4110	* - lines are small relative to FIFO size (buffer can hold more than 2
		4111	* lines), so need to account for TLB latency
		4112	*
		4113	* The normal calculation is:
		4114	* watermark = dotclock * bytes per pixel * latency
		4115	* where latency is platform & configuration dependent (we assume pessimal
		4116	* values here).
		4117	*
		4118	* The SR calculation is:
		4119	* watermark = (trunc(latency/line time)+1) * surface width *
		4120	* bytes per pixel
		4121	* where
		4122	* line time = htotal / dotclock
		4123	* surface width = hdisplay for normal plane and 64 for cursor
		4124	* and latency is assumed to be high, as above.
		4125	*
		4126	* The final value programmed to the register should always be rounded up,
		4127	* and include an extra 2 entries to account for clock crossings.
		4128	*
		4129	* We don't use the sprite, so we can ignore that. And on Crestline we have
		4130	* to set the non-SR watermarks to 8.
		4131	*/
4560	Serge	4132	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	4133	{
4560	Serge	4134	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	4135
		4136	if (dev_priv->display.update_wm)
4560	Serge	4137	dev_priv->display.update_wm(crtc);
3031	serge	4138	}
		4139
4104	Serge	4140	void intel_update_sprite_watermarks(struct drm_plane *plane,
		4141	struct drm_crtc *crtc,
5060	serge	4142	uint32_t sprite_width,
		4143	uint32_t sprite_height,
		4144	int pixel_size,
4104	Serge	4145	bool enabled, bool scaled)
3031	serge	4146	{
4104	Serge	4147	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	4148
		4149	if (dev_priv->display.update_sprite_wm)
5060	serge	4150	dev_priv->display.update_sprite_wm(plane, crtc,
		4151	sprite_width, sprite_height,
4104	Serge	4152	pixel_size, enabled, scaled);
3031	serge	4153	}
		4154
		4155	/**
		4156	* Lock protecting IPS related data structures
		4157	*/
		4158	DEFINE_SPINLOCK(mchdev_lock);
		4159
		4160	/* Global for IPS driver to get at the current i915 device. Protected by
		4161	* mchdev_lock. */
		4162	static struct drm_i915_private *i915_mch_dev;
		4163
		4164	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		4165	{
		4166	struct drm_i915_private *dev_priv = dev->dev_private;
		4167	u16 rgvswctl;
		4168
		4169	assert_spin_locked(&mchdev_lock);
		4170
		4171	rgvswctl = I915_READ16(MEMSWCTL);
		4172	if (rgvswctl & MEMCTL_CMD_STS) {
		4173	DRM_DEBUG("gpu busy, RCS change rejected\n");
		4174	return false; /* still busy with another command */
		4175	}
		4176
		4177	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		4178	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		4179	I915_WRITE16(MEMSWCTL, rgvswctl);
		4180	POSTING_READ16(MEMSWCTL);
		4181
		4182	rgvswctl \|= MEMCTL_CMD_STS;
		4183	I915_WRITE16(MEMSWCTL, rgvswctl);
		4184
		4185	return true;
		4186	}
		4187
		4188	static void ironlake_enable_drps(struct drm_device *dev)
		4189	{
		4190	struct drm_i915_private *dev_priv = dev->dev_private;
		4191	u32 rgvmodectl = I915_READ(MEMMODECTL);
		4192	u8 fmax, fmin, fstart, vstart;
		4193
		4194	spin_lock_irq(&mchdev_lock);
		4195
		4196	/* Enable temp reporting */
		4197	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		4198	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		4199
		4200	/* 100ms RC evaluation intervals */
		4201	I915_WRITE(RCUPEI, 100000);
		4202	I915_WRITE(RCDNEI, 100000);
		4203
		4204	/* Set max/min thresholds to 90ms and 80ms respectively */
		4205	I915_WRITE(RCBMAXAVG, 90000);
		4206	I915_WRITE(RCBMINAVG, 80000);
		4207
		4208	I915_WRITE(MEMIHYST, 1);
		4209
		4210	/* Set up min, max, and cur for interrupt handling */
		4211	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		4212	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		4213	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		4214	MEMMODE_FSTART_SHIFT;
		4215
6084	serge	4216	vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
3031	serge	4217	PXVFREQ_PX_SHIFT;
		4218
		4219	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		4220	dev_priv->ips.fstart = fstart;
		4221
		4222	dev_priv->ips.max_delay = fstart;
		4223	dev_priv->ips.min_delay = fmin;
		4224	dev_priv->ips.cur_delay = fstart;
		4225
		4226	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		4227	fmax, fmin, fstart);
		4228
		4229	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		4230
		4231	/*
		4232	* Interrupts will be enabled in ironlake_irq_postinstall
		4233	*/
		4234
		4235	I915_WRITE(VIDSTART, vstart);
		4236	POSTING_READ(VIDSTART);
		4237
		4238	rgvmodectl \|= MEMMODE_SWMODE_EN;
		4239	I915_WRITE(MEMMODECTL, rgvmodectl);
		4240
		4241	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		4242	DRM_ERROR("stuck trying to change perf mode\n");
		4243	mdelay(1);
		4244
		4245	ironlake_set_drps(dev, fstart);
		4246
6084	serge	4247	dev_priv->ips.last_count1 = I915_READ(DMIEC) +
		4248	I915_READ(DDREC) + I915_READ(CSIEC);
5060	serge	4249	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6084	serge	4250	dev_priv->ips.last_count2 = I915_READ(GFXEC);
5060	serge	4251	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	4252
		4253	spin_unlock_irq(&mchdev_lock);
		4254	}
		4255
		4256	static void ironlake_disable_drps(struct drm_device *dev)
		4257	{
		4258	struct drm_i915_private *dev_priv = dev->dev_private;
		4259	u16 rgvswctl;
		4260
		4261	spin_lock_irq(&mchdev_lock);
		4262
		4263	rgvswctl = I915_READ16(MEMSWCTL);
		4264
		4265	/* Ack interrupts, disable EFC interrupt */
		4266	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		4267	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		4268	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		4269	I915_WRITE(DEIIR, DE_PCU_EVENT);
		4270	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		4271
		4272	/* Go back to the starting frequency */
		4273	ironlake_set_drps(dev, dev_priv->ips.fstart);
		4274	mdelay(1);
		4275	rgvswctl \|= MEMCTL_CMD_STS;
		4276	I915_WRITE(MEMSWCTL, rgvswctl);
		4277	mdelay(1);
		4278
		4279	spin_unlock_irq(&mchdev_lock);
		4280	}
		4281
		4282	/* There's a funny hw issue where the hw returns all 0 when reading from
		4283	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		4284	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		4285	* all limits and the gpu stuck at whatever frequency it is at atm).
		4286	*/
6084	serge	4287	static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	4288	{
		4289	u32 limits;
		4290
		4291	/* Only set the down limit when we've reached the lowest level to avoid
		4292	* getting more interrupts, otherwise leave this clear. This prevents a
		4293	* race in the hw when coming out of rc6: There's a tiny window where
		4294	* the hw runs at the minimal clock before selecting the desired
		4295	* frequency, if the down threshold expires in that window we will not
		4296	* receive a down interrupt. */
6084	serge	4297	if (IS_GEN9(dev_priv->dev)) {
		4298	limits = (dev_priv->rps.max_freq_softlimit) << 23;
		4299	if (val <= dev_priv->rps.min_freq_softlimit)
		4300	limits \|= (dev_priv->rps.min_freq_softlimit) << 14;
		4301	} else {
		4302	limits = dev_priv->rps.max_freq_softlimit << 24;
		4303	if (val <= dev_priv->rps.min_freq_softlimit)
		4304	limits \|= dev_priv->rps.min_freq_softlimit << 16;
		4305	}
3031	serge	4306
		4307	return limits;
		4308	}
		4309
4560	Serge	4310	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		4311	{
		4312	int new_power;
6084	serge	4313	u32 threshold_up = 0, threshold_down = 0; /* in % */
		4314	u32 ei_up = 0, ei_down = 0;
4560	Serge	4315
		4316	new_power = dev_priv->rps.power;
		4317	switch (dev_priv->rps.power) {
		4318	case LOW_POWER:
5060	serge	4319	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	4320	new_power = BETWEEN;
		4321	break;
		4322
		4323	case BETWEEN:
5060	serge	4324	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	4325	new_power = LOW_POWER;
5060	serge	4326	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	4327	new_power = HIGH_POWER;
		4328	break;
		4329
		4330	case HIGH_POWER:
5060	serge	4331	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	4332	new_power = BETWEEN;
		4333	break;
		4334	}
		4335	/* Max/min bins are special */
6084	serge	4336	if (val <= dev_priv->rps.min_freq_softlimit)
4560	Serge	4337	new_power = LOW_POWER;
6084	serge	4338	if (val >= dev_priv->rps.max_freq_softlimit)
4560	Serge	4339	new_power = HIGH_POWER;
		4340	if (new_power == dev_priv->rps.power)
		4341	return;
		4342
		4343	/* Note the units here are not exactly 1us, but 1280ns. */
		4344	switch (new_power) {
		4345	case LOW_POWER:
		4346	/* Upclock if more than 95% busy over 16ms */
6084	serge	4347	ei_up = 16000;
		4348	threshold_up = 95;
4560	Serge	4349
		4350	/* Downclock if less than 85% busy over 32ms */
6084	serge	4351	ei_down = 32000;
		4352	threshold_down = 85;
4560	Serge	4353	break;
		4354
		4355	case BETWEEN:
		4356	/* Upclock if more than 90% busy over 13ms */
6084	serge	4357	ei_up = 13000;
		4358	threshold_up = 90;
4560	Serge	4359
		4360	/* Downclock if less than 75% busy over 32ms */
6084	serge	4361	ei_down = 32000;
		4362	threshold_down = 75;
4560	Serge	4363	break;
		4364
		4365	case HIGH_POWER:
		4366	/* Upclock if more than 85% busy over 10ms */
6084	serge	4367	ei_up = 10000;
		4368	threshold_up = 85;
4560	Serge	4369
		4370	/* Downclock if less than 60% busy over 32ms */
6084	serge	4371	ei_down = 32000;
		4372	threshold_down = 60;
4560	Serge	4373	break;
		4374	}
		4375
6084	serge	4376	I915_WRITE(GEN6_RP_UP_EI,
		4377	GT_INTERVAL_FROM_US(dev_priv, ei_up));
		4378	I915_WRITE(GEN6_RP_UP_THRESHOLD,
		4379	GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
		4380
		4381	I915_WRITE(GEN6_RP_DOWN_EI,
		4382	GT_INTERVAL_FROM_US(dev_priv, ei_down));
		4383	I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
		4384	GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
		4385
		4386	I915_WRITE(GEN6_RP_CONTROL,
		4387	GEN6_RP_MEDIA_TURBO \|
		4388	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4389	GEN6_RP_MEDIA_IS_GFX \|
		4390	GEN6_RP_ENABLE \|
		4391	GEN6_RP_UP_BUSY_AVG \|
		4392	GEN6_RP_DOWN_IDLE_AVG);
		4393
4560	Serge	4394	dev_priv->rps.power = new_power;
6084	serge	4395	dev_priv->rps.up_threshold = threshold_up;
		4396	dev_priv->rps.down_threshold = threshold_down;
4560	Serge	4397	dev_priv->rps.last_adj = 0;
		4398	}
		4399
5060	serge	4400	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		4401	{
		4402	u32 mask = 0;
		4403
		4404	if (val > dev_priv->rps.min_freq_softlimit)
6084	serge	4405	mask \|= GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
5060	serge	4406	if (val < dev_priv->rps.max_freq_softlimit)
6084	serge	4407	mask \|= GEN6_PM_RP_UP_EI_EXPIRED \| GEN6_PM_RP_UP_THRESHOLD;
5060	serge	4408
		4409	mask &= dev_priv->pm_rps_events;
		4410
6084	serge	4411	return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
5060	serge	4412	}
		4413
		4414	/* gen6_set_rps is called to update the frequency request, but should also be
		4415	* called when the range (min_delay and max_delay) is modified so that we can
		4416	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6084	serge	4417	static void gen6_set_rps(struct drm_device *dev, u8 val)
3031	serge	4418	{
		4419	struct drm_i915_private *dev_priv = dev->dev_private;
		4420
6084	serge	4421	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
		4422	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0))
		4423	return;
		4424
3243	Serge	4425	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6084	serge	4426	WARN_ON(val > dev_priv->rps.max_freq);
		4427	WARN_ON(val < dev_priv->rps.min_freq);
3031	serge	4428
5060	serge	4429	/* min/max delay may still have been modified so be sure to
		4430	* write the limits value.
		4431	*/
		4432	if (val != dev_priv->rps.cur_freq) {
5354	serge	4433	gen6_set_rps_thresholds(dev_priv, val);
4560	Serge	4434
6084	serge	4435	if (IS_GEN9(dev))
5354	serge	4436	I915_WRITE(GEN6_RPNSWREQ,
6084	serge	4437	GEN9_FREQUENCY(val));
		4438	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4439	I915_WRITE(GEN6_RPNSWREQ,
5354	serge	4440	HSW_FREQUENCY(val));
		4441	else
		4442	I915_WRITE(GEN6_RPNSWREQ,
		4443	GEN6_FREQUENCY(val) \|
		4444	GEN6_OFFSET(0) \|
		4445	GEN6_AGGRESSIVE_TURBO);
5060	serge	4446	}
3031	serge	4447
		4448	/* Make sure we continue to get interrupts
		4449	* until we hit the minimum or maximum frequencies.
		4450	*/
6084	serge	4451	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
5060	serge	4452	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	4453
		4454	POSTING_READ(GEN6_RPNSWREQ);
		4455
5060	serge	4456	dev_priv->rps.cur_freq = val;
6084	serge	4457	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3031	serge	4458	}
		4459
6084	serge	4460	static void valleyview_set_rps(struct drm_device *dev, u8 val)
5060	serge	4461	{
6084	serge	4462	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4463
6084	serge	4464	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4465	WARN_ON(val > dev_priv->rps.max_freq);
		4466	WARN_ON(val < dev_priv->rps.min_freq);
5060	serge	4467
6084	serge	4468	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
		4469	"Odd GPU freq value\n"))
		4470	val &= ~1;
5060	serge	4471
6084	serge	4472	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5060	serge	4473
6084	serge	4474	if (val != dev_priv->rps.cur_freq) {
		4475	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		4476	if (!IS_CHERRYVIEW(dev_priv))
		4477	gen6_set_rps_thresholds(dev_priv, val);
		4478	}
5060	serge	4479
6084	serge	4480	dev_priv->rps.cur_freq = val;
		4481	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
		4482	}
5060	serge	4483
6084	serge	4484	/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
		4485	*
		4486	* * If Gfx is Idle, then
		4487	* 1. Forcewake Media well.
		4488	* 2. Request idle freq.
		4489	* 3. Release Forcewake of Media well.
		4490	*/
		4491	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		4492	{
		4493	u32 val = dev_priv->rps.idle_freq;
5060	serge	4494
6084	serge	4495	if (dev_priv->rps.cur_freq <= val)
		4496	return;
5060	serge	4497
6084	serge	4498	/* Wake up the media well, as that takes a lot less
		4499	* power than the Render well. */
		4500	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
		4501	valleyview_set_rps(dev_priv->dev, val);
		4502	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
5060	serge	4503	}
		4504
6084	serge	4505	void gen6_rps_busy(struct drm_i915_private *dev_priv)
3031	serge	4506	{
4560	Serge	4507	mutex_lock(&dev_priv->rps.hw_lock);
		4508	if (dev_priv->rps.enabled) {
6084	serge	4509	if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED))
		4510	gen6_rps_reset_ei(dev_priv);
		4511	I915_WRITE(GEN6_PMINTRMSK,
		4512	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4560	Serge	4513	}
		4514	mutex_unlock(&dev_priv->rps.hw_lock);
		4515	}
4104	Serge	4516
6084	serge	4517	void gen6_rps_idle(struct drm_i915_private *dev_priv)
4560	Serge	4518	{
		4519	struct drm_device *dev = dev_priv->dev;
4104	Serge	4520
4560	Serge	4521	mutex_lock(&dev_priv->rps.hw_lock);
		4522	if (dev_priv->rps.enabled) {
		4523	if (IS_VALLEYVIEW(dev))
6084	serge	4524	vlv_set_rps_idle(dev_priv);
4560	Serge	4525	else
6084	serge	4526	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4527	dev_priv->rps.last_adj = 0;
6084	serge	4528	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4560	Serge	4529	}
		4530	mutex_unlock(&dev_priv->rps.hw_lock);
6084	serge	4531
		4532	spin_lock(&dev_priv->rps.client_lock);
		4533	while (!list_empty(&dev_priv->rps.clients))
		4534	list_del_init(dev_priv->rps.clients.next);
		4535	spin_unlock(&dev_priv->rps.client_lock);
4104	Serge	4536	}
		4537
6084	serge	4538	void gen6_rps_boost(struct drm_i915_private *dev_priv,
		4539	struct intel_rps_client *rps,
		4540	unsigned long submitted)
4104	Serge	4541	{
6084	serge	4542	/* This is intentionally racy! We peek at the state here, then
		4543	* validate inside the RPS worker.
		4544	*/
		4545	if (!(dev_priv->mm.busy &&
		4546	dev_priv->rps.enabled &&
		4547	dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
		4548	return;
3031	serge	4549
6084	serge	4550	/* Force a RPS boost (and don't count it against the client) if
		4551	* the GPU is severely congested.
		4552	*/
		4553	if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
		4554	rps = NULL;
4104	Serge	4555
6084	serge	4556	spin_lock(&dev_priv->rps.client_lock);
		4557	if (rps == NULL \|\| list_empty(&rps->link)) {
		4558	spin_lock_irq(&dev_priv->irq_lock);
		4559	if (dev_priv->rps.interrupts_enabled) {
		4560	dev_priv->rps.client_boost = true;
		4561	queue_work(dev_priv->wq, &dev_priv->rps.work);
		4562	}
		4563	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	4564
6084	serge	4565	if (rps != NULL) {
		4566	list_add(&rps->link, &dev_priv->rps.clients);
		4567	rps->boosts++;
		4568	} else
		4569	dev_priv->rps.boosts++;
		4570	}
		4571	spin_unlock(&dev_priv->rps.client_lock);
		4572	}
4104	Serge	4573
6084	serge	4574	void intel_set_rps(struct drm_device *dev, u8 val)
		4575	{
		4576	if (IS_VALLEYVIEW(dev))
		4577	valleyview_set_rps(dev, val);
		4578	else
		4579	gen6_set_rps(dev, val);
4104	Serge	4580	}
		4581
5354	serge	4582	static void gen9_disable_rps(struct drm_device *dev)
5060	serge	4583	{
		4584	struct drm_i915_private *dev_priv = dev->dev_private;
		4585
5354	serge	4586	I915_WRITE(GEN6_RC_CONTROL, 0);
6084	serge	4587	I915_WRITE(GEN9_PG_ENABLE, 0);
5060	serge	4588	}
		4589
4104	Serge	4590	static void gen6_disable_rps(struct drm_device *dev)
		4591	{
		4592	struct drm_i915_private *dev_priv = dev->dev_private;
		4593
		4594	I915_WRITE(GEN6_RC_CONTROL, 0);
		4595	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		4596	}
		4597
5060	serge	4598	static void cherryview_disable_rps(struct drm_device *dev)
		4599	{
		4600	struct drm_i915_private *dev_priv = dev->dev_private;
		4601
		4602	I915_WRITE(GEN6_RC_CONTROL, 0);
		4603	}
		4604
4104	Serge	4605	static void valleyview_disable_rps(struct drm_device *dev)
		4606	{
		4607	struct drm_i915_private *dev_priv = dev->dev_private;
		4608
5354	serge	4609	/* we're doing forcewake before Disabling RC6,
		4610	* This what the BIOS expects when going into suspend */
6084	serge	4611	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4612
4104	Serge	4613	I915_WRITE(GEN6_RC_CONTROL, 0);
		4614
6084	serge	4615	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4616	}
		4617
4560	Serge	4618	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		4619	{
5060	serge	4620	if (IS_VALLEYVIEW(dev)) {
		4621	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		4622	mode = GEN6_RC_CTL_RC6_ENABLE;
		4623	else
		4624	mode = 0;
		4625	}
5354	serge	4626	if (HAS_RC6p(dev))
		4627	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
6084	serge	4628	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		4629	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		4630	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
5354	serge	4631
		4632	else
		4633	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
		4634	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4560	Serge	4635	}
		4636
5060	serge	4637	static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3031	serge	4638	{
6084	serge	4639	/* No RC6 before Ironlake and code is gone for ilk. */
		4640	if (INTEL_INFO(dev)->gen < 6)
4104	Serge	4641	return 0;
		4642
3031	serge	4643	/* Respect the kernel parameter if it is set */
5060	serge	4644	if (enable_rc6 >= 0) {
		4645	int mask;
3031	serge	4646
5354	serge	4647	if (HAS_RC6p(dev))
5060	serge	4648	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		4649	INTEL_RC6pp_ENABLE;
		4650	else
		4651	mask = INTEL_RC6_ENABLE;
		4652
		4653	if ((enable_rc6 & mask) != enable_rc6)
		4654	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
6084	serge	4655	enable_rc6 & mask, enable_rc6, mask);
5060	serge	4656
		4657	return enable_rc6 & mask;
		4658	}
		4659
		4660	if (IS_IVYBRIDGE(dev))
		4661	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	4662
6084	serge	4663	return INTEL_RC6_ENABLE;
5060	serge	4664	}
3031	serge	4665
5060	serge	4666	int intel_enable_rc6(const struct drm_device *dev)
		4667	{
		4668	return i915.enable_rc6;
3031	serge	4669	}
		4670
5354	serge	4671	static void gen6_init_rps_frequencies(struct drm_device *dev)
5060	serge	4672	{
		4673	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4674	uint32_t rp_state_cap;
		4675	u32 ddcc_status = 0;
		4676	int ret;
5060	serge	4677
		4678	/* All of these values are in units of 50MHz */
		4679	dev_priv->rps.cur_freq = 0;
5354	serge	4680	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
6084	serge	4681	if (IS_BROXTON(dev)) {
		4682	rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
		4683	dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
		4684	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4685	dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
		4686	} else {
		4687	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		4688	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
		4689	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4690	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		4691	}
		4692
5060	serge	4693	/* hw_max = RP0 until we check for overclocking */
		4694	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	4695
5354	serge	4696	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
6084	serge	4697	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev) \|\| IS_SKYLAKE(dev)) {
5354	serge	4698	ret = sandybridge_pcode_read(dev_priv,
		4699	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
		4700	&ddcc_status);
		4701	if (0 == ret)
		4702	dev_priv->rps.efficient_freq =
6084	serge	4703	clamp_t(u8,
		4704	((ddcc_status >> 8) & 0xff),
		4705	dev_priv->rps.min_freq,
		4706	dev_priv->rps.max_freq);
5354	serge	4707	}
		4708
6084	serge	4709	if (IS_SKYLAKE(dev)) {
		4710	/* Store the frequency values in 16.66 MHZ units, which is
		4711	the natural hardware unit for SKL */
		4712	dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
		4713	dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
		4714	dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
		4715	dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
		4716	dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
		4717	}
		4718
		4719	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		4720
5060	serge	4721	/* Preserve min/max settings in case of re-init */
		4722	if (dev_priv->rps.max_freq_softlimit == 0)
		4723	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	4724
5354	serge	4725	if (dev_priv->rps.min_freq_softlimit == 0) {
		4726	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4727	dev_priv->rps.min_freq_softlimit =
6084	serge	4728	max_t(int, dev_priv->rps.efficient_freq,
		4729	intel_freq_opcode(dev_priv, 450));
5354	serge	4730	else
		4731	dev_priv->rps.min_freq_softlimit =
		4732	dev_priv->rps.min_freq;
		4733	}
4104	Serge	4734	}
		4735
6084	serge	4736	/* See the Gen9_GT_PM_Programming_Guide doc for the below */
5354	serge	4737	static void gen9_enable_rps(struct drm_device *dev)
		4738	{
		4739	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4740
		4741	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4742
		4743	gen6_init_rps_frequencies(dev);
		4744
		4745	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
		4746	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) {
		4747	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4748	return;
		4749	}
		4750
		4751	/* Program defaults and thresholds for RPS*/
		4752	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4753	GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
		4754
		4755	/* 1 second timeout*/
		4756	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
		4757	GT_INTERVAL_FROM_US(dev_priv, 1000000));
		4758
		4759	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
		4760
		4761	/* Leaning on the below call to gen6_set_rps to program/setup the
		4762	* Up/Down EI & threshold registers, as well as the RP_CONTROL,
		4763	* RP_INTERRUPT_LIMITS & RPNSWREQ registers */
		4764	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		4765	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4766
		4767	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4768	}
		4769
		4770	static void gen9_enable_rc6(struct drm_device *dev)
		4771	{
		4772	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4773	struct intel_engine_cs *ring;
		4774	uint32_t rc6_mask = 0;
		4775	int unused;
		4776
		4777	/* 1a: Software RC state - RC0 */
		4778	I915_WRITE(GEN6_RC_STATE, 0);
		4779
		4780	/* 1b: Get forcewake during program sequence. Although the driver
		4781	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4782	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4783
		4784	/* 2a: Disable RC states. */
		4785	I915_WRITE(GEN6_RC_CONTROL, 0);
		4786
		4787	/* 2b: Program RC6 thresholds.*/
6084	serge	4788
		4789	/* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
		4790	if (IS_SKYLAKE(dev))
		4791	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
		4792	else
		4793	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
5354	serge	4794	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4795	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4796	for_each_ring(ring, dev_priv, unused)
		4797	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
6084	serge	4798
		4799	if (HAS_GUC_UCODE(dev))
		4800	I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
		4801
5354	serge	4802	I915_WRITE(GEN6_RC_SLEEP, 0);
		4803
6084	serge	4804	/* 2c: Program Coarse Power Gating Policies. */
		4805	I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
		4806	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
		4807
5354	serge	4808	/* 3a: Enable RC6 */
		4809	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4810	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
		4811	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4812	"on" : "off");
6084	serge	4813	/* WaRsUseTimeoutMode */
		4814	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_D0) \|\|
		4815	(IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_A0)) {
		4816	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
		4817	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4818	GEN7_RC_CTL_TO_MODE \|
		4819	rc6_mask);
		4820	} else {
		4821	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
		4822	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4823	GEN6_RC_CTL_EI_MODE(1) \|
		4824	rc6_mask);
		4825	}
5354	serge	4826
6084	serge	4827	/*
		4828	* 3b: Enable Coarse Power Gating only when RC6 is enabled.
		4829	* WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
		4830	*/
		4831	if ((IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) \|\|
		4832	((IS_SKL_GT3(dev) \|\| IS_SKL_GT4(dev)) && (INTEL_REVID(dev) <= SKL_REVID_F0)))
		4833	I915_WRITE(GEN9_PG_ENABLE, 0);
		4834	else
		4835	I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4836	(GEN9_RENDER_PG_ENABLE \| GEN9_MEDIA_PG_ENABLE) : 0);
5354	serge	4837
6084	serge	4838	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4839
5354	serge	4840	}
		4841
4560	Serge	4842	static void gen8_enable_rps(struct drm_device *dev)
		4843	{
		4844	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4845	struct intel_engine_cs *ring;
5354	serge	4846	uint32_t rc6_mask = 0;
4560	Serge	4847	int unused;
		4848
		4849	/* 1a: Software RC state - RC0 */
		4850	I915_WRITE(GEN6_RC_STATE, 0);
		4851
		4852	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		4853	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4854	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4560	Serge	4855
		4856	/* 2a: Disable RC states. */
		4857	I915_WRITE(GEN6_RC_CONTROL, 0);
		4858
5354	serge	4859	/* Initialize rps frequencies */
		4860	gen6_init_rps_frequencies(dev);
4560	Serge	4861
		4862	/* 2b: Program RC6 thresholds.*/
		4863	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4864	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4865	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4866	for_each_ring(ring, dev_priv, unused)
		4867	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4868	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	4869	if (IS_BROADWELL(dev))
		4870	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		4871	else
6084	serge	4872	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4560	Serge	4873
		4874	/* 3: Enable RC6 */
		4875	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4876	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	4877	intel_print_rc6_info(dev, rc6_mask);
		4878	if (IS_BROADWELL(dev))
		4879	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4880	GEN7_RC_CTL_TO_MODE \|
		4881	rc6_mask);
		4882	else
6084	serge	4883	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4884	GEN6_RC_CTL_EI_MODE(1) \|
		4885	rc6_mask);
4560	Serge	4886
		4887	/* 4 Program defaults and thresholds for RPS*/
5060	serge	4888	I915_WRITE(GEN6_RPNSWREQ,
		4889	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		4890	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4891	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	4892	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		4893	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		4894
		4895	/* Docs recommend 900MHz, and 300 MHz respectively */
		4896	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	4897	dev_priv->rps.max_freq_softlimit << 24 \|
		4898	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	4899
		4900	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		4901	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		4902	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		4903	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		4904
		4905	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4906
		4907	/* 5: Enable RPS */
		4908	I915_WRITE(GEN6_RP_CONTROL,
		4909	GEN6_RP_MEDIA_TURBO \|
		4910	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4911	GEN6_RP_MEDIA_IS_GFX \|
		4912	GEN6_RP_ENABLE \|
		4913	GEN6_RP_UP_BUSY_AVG \|
		4914	GEN6_RP_DOWN_IDLE_AVG);
		4915
		4916	/* 6: Ring frequency + overclocking (our driver does this later */
		4917
5354	serge	4918	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	4919	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4920
6084	serge	4921	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4560	Serge	4922	}
		4923
3031	serge	4924	static void gen6_enable_rps(struct drm_device *dev)
		4925	{
		4926	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4927	struct intel_engine_cs *ring;
		4928	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	4929	u32 gtfifodbg;
		4930	int rc6_mode;
3243	Serge	4931	int i, ret;
3031	serge	4932
3243	Serge	4933	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4934
		4935	/* Here begins a magic sequence of register writes to enable
		4936	* auto-downclocking.
		4937	*
		4938	* Perhaps there might be some value in exposing these to
		4939	* userspace...
		4940	*/
		4941	I915_WRITE(GEN6_RC_STATE, 0);
		4942
		4943	/* Clear the DBG now so we don't confuse earlier errors */
		4944	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		4945	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		4946	I915_WRITE(GTFIFODBG, gtfifodbg);
		4947	}
		4948
6084	serge	4949	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	4950
5354	serge	4951	/* Initialize rps frequencies */
		4952	gen6_init_rps_frequencies(dev);
3031	serge	4953
		4954	/* disable the counters and set deterministic thresholds */
		4955	I915_WRITE(GEN6_RC_CONTROL, 0);
		4956
		4957	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		4958	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		4959	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		4960	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4961	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4962
		4963	for_each_ring(ring, dev_priv, i)
		4964	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4965
		4966	I915_WRITE(GEN6_RC_SLEEP, 0);
		4967	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	4968	if (IS_IVYBRIDGE(dev))
4104	Serge	4969	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		4970	else
6084	serge	4971	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	4972	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	4973	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		4974
		4975	/* Check if we are enabling RC6 */
		4976	rc6_mode = intel_enable_rc6(dev_priv->dev);
		4977	if (rc6_mode & INTEL_RC6_ENABLE)
		4978	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		4979
		4980	/* We don't use those on Haswell */
		4981	if (!IS_HASWELL(dev)) {
		4982	if (rc6_mode & INTEL_RC6p_ENABLE)
		4983	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		4984
		4985	if (rc6_mode & INTEL_RC6pp_ENABLE)
		4986	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		4987	}
		4988
4560	Serge	4989	intel_print_rc6_info(dev, rc6_mask);
3031	serge	4990
		4991	I915_WRITE(GEN6_RC_CONTROL,
		4992	rc6_mask \|
		4993	GEN6_RC_CTL_EI_MODE(1) \|
		4994	GEN6_RC_CTL_HW_ENABLE);
		4995
4560	Serge	4996	/* Power down if completely idle for over 50ms */
		4997	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	4998	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4999
3243	Serge	5000	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	5001	if (ret)
		5002	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		5003
6084	serge	5004	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		5005	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		5006	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	5007	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
6084	serge	5008	(pcu_mbox & 0xff) * 50);
5060	serge	5009	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	5010	}
		5011
4560	Serge	5012	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	5013	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
3031	serge	5014
3243	Serge	5015	rc6vids = 0;
		5016	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		5017	if (IS_GEN6(dev) && ret) {
		5018	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		5019	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		5020	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		5021	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		5022	rc6vids &= 0xffff00;
		5023	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		5024	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		5025	if (ret)
		5026	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		5027	}
		5028
6084	serge	5029	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	5030	}
		5031
5060	serge	5032	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	5033	{
		5034	struct drm_i915_private *dev_priv = dev->dev_private;
		5035	int min_freq = 15;
3746	Serge	5036	unsigned int gpu_freq;
		5037	unsigned int max_ia_freq, min_ring_freq;
6084	serge	5038	unsigned int max_gpu_freq, min_gpu_freq;
3031	serge	5039	int scaling_factor = 180;
4560	Serge	5040	struct cpufreq_policy *policy;
3031	serge	5041
3243	Serge	5042	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5043
		5044	max_ia_freq = cpufreq_quick_get_max(0);
		5045	/*
5060	serge	5046	* Default to measured freq if none found, PCU will ensure we
		5047	* don't go over
3031	serge	5048	*/
		5049	max_ia_freq = tsc_khz;
		5050
		5051	/* Convert from kHz to MHz */
		5052	max_ia_freq /= 1000;
		5053
4560	Serge	5054	min_ring_freq = I915_READ(DCLK) & 0xf;
		5055	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		5056	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	5057
6084	serge	5058	if (IS_SKYLAKE(dev)) {
		5059	/* Convert GT frequency to 50 HZ units */
		5060	min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
		5061	max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
		5062	} else {
		5063	min_gpu_freq = dev_priv->rps.min_freq;
		5064	max_gpu_freq = dev_priv->rps.max_freq;
		5065	}
		5066
3031	serge	5067	/*
		5068	* For each potential GPU frequency, load a ring frequency we'd like
		5069	* to use for memory access. We do this by specifying the IA frequency
		5070	* the PCU should use as a reference to determine the ring frequency.
		5071	*/
6084	serge	5072	for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
		5073	int diff = max_gpu_freq - gpu_freq;
3746	Serge	5074	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	5075
6084	serge	5076	if (IS_SKYLAKE(dev)) {
		5077	/*
		5078	* ring_freq = 2 * GT. ring_freq is in 100MHz units
		5079	* No floor required for ring frequency on SKL.
		5080	*/
		5081	ring_freq = gpu_freq;
		5082	} else if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	5083	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		5084	ring_freq = max(min_ring_freq, gpu_freq);
		5085	} else if (IS_HASWELL(dev)) {
		5086	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	5087	ring_freq = max(min_ring_freq, ring_freq);
		5088	/* leave ia_freq as the default, chosen by cpufreq */
		5089	} else {
		5090	/* On older processors, there is no separate ring
		5091	* clock domain, so in order to boost the bandwidth
		5092	* of the ring, we need to upclock the CPU (ia_freq).
		5093	*
		5094	* For GPU frequencies less than 750MHz,
		5095	* just use the lowest ring freq.
6084	serge	5096	*/
		5097	if (gpu_freq < min_freq)
		5098	ia_freq = 800;
		5099	else
		5100	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		5101	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	5102	}
3031	serge	5103
3243	Serge	5104	sandybridge_pcode_write(dev_priv,
		5105	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	5106	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		5107	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		5108	gpu_freq);
3031	serge	5109	}
		5110	}
		5111
5060	serge	5112	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	5113	{
5060	serge	5114	struct drm_i915_private *dev_priv = dev->dev_private;
		5115
6084	serge	5116	if (!HAS_CORE_RING_FREQ(dev))
5060	serge	5117	return;
		5118
		5119	mutex_lock(&dev_priv->rps.hw_lock);
		5120	__gen6_update_ring_freq(dev);
		5121	mutex_unlock(&dev_priv->rps.hw_lock);
		5122	}
		5123
		5124	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		5125	{
6084	serge	5126	struct drm_device *dev = dev_priv->dev;
4104	Serge	5127	u32 val, rp0;
		5128
6084	serge	5129	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5060	serge	5130
6084	serge	5131	switch (INTEL_INFO(dev)->eu_total) {
		5132	case 8:
		5133	/* (2 * 4) config */
		5134	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
		5135	break;
		5136	case 12:
		5137	/* (2 * 6) config */
		5138	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
		5139	break;
		5140	case 16:
		5141	/* (2 * 8) config */
		5142	default:
		5143	/* Setting (2 * 8) Min RP0 for any other combination */
		5144	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
		5145	break;
		5146	}
		5147
		5148	rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
		5149
5060	serge	5150	return rp0;
		5151	}
		5152
		5153	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5154	{
		5155	u32 val, rpe;
		5156
		5157	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		5158	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		5159
		5160	return rpe;
		5161	}
		5162
		5163	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5164	{
		5165	u32 val, rp1;
		5166
6084	serge	5167	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
		5168	rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5060	serge	5169
		5170	return rp1;
		5171	}
		5172
		5173	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5174	{
		5175	u32 val, rp1;
		5176
4104	Serge	5177	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5178
5060	serge	5179	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		5180
		5181	return rp1;
		5182	}
		5183
		5184	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		5185	{
		5186	u32 val, rp0;
		5187
		5188	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5189
4104	Serge	5190	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		5191	/* Clamp to max */
		5192	rp0 = min_t(u32, rp0, 0xea);
		5193
		5194	return rp0;
		5195	}
		5196
		5197	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5198	{
		5199	u32 val, rpe;
		5200
		5201	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		5202	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		5203	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		5204	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		5205
		5206	return rpe;
		5207	}
		5208
5060	serge	5209	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	5210	{
		5211	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		5212	}
		5213
5060	serge	5214	/* Check that the pctx buffer wasn't move under us. */
		5215	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		5216	{
		5217	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5218
		5219	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		5220	dev_priv->vlv_pctx->stolen->start);
		5221	}
		5222
		5223
		5224	/* Check that the pcbr address is not empty. */
		5225	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		5226	{
		5227	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5228
		5229	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		5230	}
		5231
		5232	static void cherryview_setup_pctx(struct drm_device *dev)
		5233	{
		5234	struct drm_i915_private *dev_priv = dev->dev_private;
		5235	unsigned long pctx_paddr, paddr;
		5236	struct i915_gtt *gtt = &dev_priv->gtt;
		5237	u32 pcbr;
		5238	int pctx_size = 32*1024;
		5239
		5240	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5241
		5242	pcbr = I915_READ(VLV_PCBR);
		5243	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5354	serge	5244	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5060	serge	5245	paddr = (dev_priv->mm.stolen_base +
		5246	(gtt->stolen_size - pctx_size));
		5247
		5248	pctx_paddr = (paddr & (~4095));
		5249	I915_WRITE(VLV_PCBR, pctx_paddr);
		5250	}
5354	serge	5251
		5252	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5060	serge	5253	}
		5254
4104	Serge	5255	static void valleyview_setup_pctx(struct drm_device *dev)
		5256	{
		5257	struct drm_i915_private *dev_priv = dev->dev_private;
		5258	struct drm_i915_gem_object *pctx;
		5259	unsigned long pctx_paddr;
		5260	u32 pcbr;
		5261	int pctx_size = 24*1024;
		5262
5060	serge	5263	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5264
4104	Serge	5265	pcbr = I915_READ(VLV_PCBR);
		5266	if (pcbr) {
		5267	/* BIOS set it up already, grab the pre-alloc'd space */
		5268	int pcbr_offset;
		5269
		5270	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		5271	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		5272	pcbr_offset,
		5273	I915_GTT_OFFSET_NONE,
		5274	pctx_size);
		5275	goto out;
		5276	}
		5277
5354	serge	5278	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
		5279
4104	Serge	5280	/*
		5281	* From the Gunit register HAS:
		5282	* The Gfx driver is expected to program this register and ensure
		5283	* proper allocation within Gfx stolen memory. For example, this
		5284	* register should be programmed such than the PCBR range does not
		5285	* overlap with other ranges, such as the frame buffer, protected
		5286	* memory, or any other relevant ranges.
		5287	*/
		5288	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		5289	if (!pctx) {
		5290	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		5291	return;
		5292	}
		5293
		5294	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		5295	I915_WRITE(VLV_PCBR, pctx_paddr);
		5296
		5297	out:
5354	serge	5298	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4104	Serge	5299	dev_priv->vlv_pctx = pctx;
		5300	}
		5301
5060	serge	5302	static void valleyview_cleanup_pctx(struct drm_device *dev)
		5303	{
		5304	struct drm_i915_private *dev_priv = dev->dev_private;
		5305
		5306	if (WARN_ON(!dev_priv->vlv_pctx))
		5307	return;
		5308
		5309	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		5310	dev_priv->vlv_pctx = NULL;
		5311	}
		5312
		5313	static void valleyview_init_gt_powersave(struct drm_device *dev)
		5314	{
		5315	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5316	u32 val;
5060	serge	5317
		5318	valleyview_setup_pctx(dev);
		5319
		5320	mutex_lock(&dev_priv->rps.hw_lock);
		5321
5354	serge	5322	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5323	switch ((val >> 6) & 3) {
		5324	case 0:
		5325	case 1:
		5326	dev_priv->mem_freq = 800;
		5327	break;
		5328	case 2:
		5329	dev_priv->mem_freq = 1066;
		5330	break;
		5331	case 3:
		5332	dev_priv->mem_freq = 1333;
		5333	break;
		5334	}
		5335	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5336
5060	serge	5337	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		5338	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5339	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5340	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5341	dev_priv->rps.max_freq);
		5342
		5343	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		5344	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5345	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5346	dev_priv->rps.efficient_freq);
		5347
		5348	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		5349	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
6084	serge	5350	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5351	dev_priv->rps.rp1_freq);
		5352
		5353	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		5354	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5355	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5356	dev_priv->rps.min_freq);
		5357
6084	serge	5358	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5359
5060	serge	5360	/* Preserve min/max settings in case of re-init */
		5361	if (dev_priv->rps.max_freq_softlimit == 0)
		5362	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5363
		5364	if (dev_priv->rps.min_freq_softlimit == 0)
		5365	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5366
		5367	mutex_unlock(&dev_priv->rps.hw_lock);
		5368	}
		5369
		5370	static void cherryview_init_gt_powersave(struct drm_device *dev)
		5371	{
		5372	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5373	u32 val;
5060	serge	5374
		5375	cherryview_setup_pctx(dev);
		5376
		5377	mutex_lock(&dev_priv->rps.hw_lock);
		5378
6084	serge	5379	mutex_lock(&dev_priv->sb_lock);
5354	serge	5380	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
6084	serge	5381	mutex_unlock(&dev_priv->sb_lock);
5354	serge	5382
		5383	switch ((val >> 2) & 0x7) {
		5384	case 3:
		5385	dev_priv->mem_freq = 2000;
		5386	break;
6084	serge	5387	default:
5354	serge	5388	dev_priv->mem_freq = 1600;
		5389	break;
		5390	}
		5391	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5392
5060	serge	5393	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		5394	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5395	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5396	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5397	dev_priv->rps.max_freq);
		5398
		5399	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		5400	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5401	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5402	dev_priv->rps.efficient_freq);
		5403
		5404	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		5405	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
6084	serge	5406	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5407	dev_priv->rps.rp1_freq);
		5408
6084	serge	5409	/* PUnit validated range is only [RPe, RP0] */
		5410	dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5060	serge	5411	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5412	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5413	dev_priv->rps.min_freq);
		5414
5354	serge	5415	WARN_ONCE((dev_priv->rps.max_freq \|
		5416	dev_priv->rps.efficient_freq \|
		5417	dev_priv->rps.rp1_freq \|
		5418	dev_priv->rps.min_freq) & 1,
		5419	"Odd GPU freq values\n");
		5420
6084	serge	5421	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5422
5060	serge	5423	/* Preserve min/max settings in case of re-init */
		5424	if (dev_priv->rps.max_freq_softlimit == 0)
		5425	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5426
		5427	if (dev_priv->rps.min_freq_softlimit == 0)
		5428	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5429
		5430	mutex_unlock(&dev_priv->rps.hw_lock);
		5431	}
		5432
		5433	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		5434	{
		5435	valleyview_cleanup_pctx(dev);
		5436	}
		5437
		5438	static void cherryview_enable_rps(struct drm_device *dev)
		5439	{
		5440	struct drm_i915_private *dev_priv = dev->dev_private;
		5441	struct intel_engine_cs *ring;
		5442	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		5443	int i;
		5444
		5445	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5446
		5447	gtfifodbg = I915_READ(GTFIFODBG);
		5448	if (gtfifodbg) {
		5449	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5450	gtfifodbg);
		5451	I915_WRITE(GTFIFODBG, gtfifodbg);
		5452	}
		5453
		5454	cherryview_check_pctx(dev_priv);
		5455
		5456	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		5457	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	5458	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	5459
6084	serge	5460	/* Disable RC states. */
		5461	I915_WRITE(GEN6_RC_CONTROL, 0);
		5462
5060	serge	5463	/* 2a: Program RC6 thresholds.*/
		5464	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		5465	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		5466	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		5467
		5468	for_each_ring(ring, dev_priv, i)
		5469	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5470	I915_WRITE(GEN6_RC_SLEEP, 0);
		5471
6084	serge	5472	/* TO threshold set to 500 us ( 0x186 * 1.28 us) */
		5473	I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5060	serge	5474
		5475	/* allows RC6 residency counter to work */
		5476	I915_WRITE(VLV_COUNTER_CONTROL,
		5477	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		5478	VLV_MEDIA_RC6_COUNT_EN \|
		5479	VLV_RENDER_RC6_COUNT_EN));
		5480
		5481	/* For now we assume BIOS is allocating and populating the PCBR */
		5482	pcbr = I915_READ(VLV_PCBR);
		5483
		5484	/* 3: Enable RC6 */
		5485	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		5486	(pcbr >> VLV_PCBR_ADDR_SHIFT))
6084	serge	5487	rc6_mode = GEN7_RC_CTL_TO_MODE;
5060	serge	5488
		5489	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5490
		5491	/* 4 Program defaults and thresholds for RPS*/
6084	serge	5492	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5060	serge	5493	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5494	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5495	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5496	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5497
		5498	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5499
		5500	/* 5: Enable RPS */
		5501	I915_WRITE(GEN6_RP_CONTROL,
		5502	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
6084	serge	5503	GEN6_RP_MEDIA_IS_GFX \|
5060	serge	5504	GEN6_RP_ENABLE \|
		5505	GEN6_RP_UP_BUSY_AVG \|
		5506	GEN6_RP_DOWN_IDLE_AVG);
		5507
6084	serge	5508	/* Setting Fixed Bias */
		5509	val = VLV_OVERRIDE_EN \|
		5510	VLV_SOC_TDP_EN \|
		5511	CHV_BIAS_CPU_50_SOC_50;
		5512	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5513
5060	serge	5514	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5515
5354	serge	5516	/* RPS code assumes GPLL is used */
		5517	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5518
6084	serge	5519	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5060	serge	5520	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5521
		5522	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		5523	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5524	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5525	dev_priv->rps.cur_freq);
		5526
		5527	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5528	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5529	dev_priv->rps.efficient_freq);
		5530
		5531	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		5532
6084	serge	5533	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	5534	}
		5535
4104	Serge	5536	static void valleyview_enable_rps(struct drm_device *dev)
		5537	{
		5538	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5539	struct intel_engine_cs *ring;
4560	Serge	5540	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	5541	int i;
		5542
		5543	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5544
5060	serge	5545	valleyview_check_pctx(dev_priv);
		5546
4104	Serge	5547	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	5548	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5549	gtfifodbg);
4104	Serge	5550	I915_WRITE(GTFIFODBG, gtfifodbg);
		5551	}
		5552
4560	Serge	5553	/* If VLV, Forcewake all wells, else re-direct to regular path */
6084	serge	5554	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	5555
6084	serge	5556	/* Disable RC states. */
		5557	I915_WRITE(GEN6_RC_CONTROL, 0);
		5558
		5559	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4104	Serge	5560	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5561	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5562	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5563	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5564
		5565	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5566
		5567	I915_WRITE(GEN6_RP_CONTROL,
		5568	GEN6_RP_MEDIA_TURBO \|
		5569	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5570	GEN6_RP_MEDIA_IS_GFX \|
		5571	GEN6_RP_ENABLE \|
		5572	GEN6_RP_UP_BUSY_AVG \|
		5573	GEN6_RP_DOWN_IDLE_CONT);
		5574
		5575	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		5576	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		5577	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		5578
		5579	for_each_ring(ring, dev_priv, i)
		5580	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5581
4560	Serge	5582	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	5583
		5584	/* allows RC6 residency counter to work */
4560	Serge	5585	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	5586	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		5587	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	5588	VLV_MEDIA_RC6_COUNT_EN \|
		5589	VLV_RENDER_RC6_COUNT_EN));
5060	serge	5590
4560	Serge	5591	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		5592	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	5593
4560	Serge	5594	intel_print_rc6_info(dev, rc6_mode);
		5595
		5596	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5597
6084	serge	5598	/* Setting Fixed Bias */
		5599	val = VLV_OVERRIDE_EN \|
		5600	VLV_SOC_TDP_EN \|
		5601	VLV_BIAS_CPU_125_SOC_875;
		5602	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5603
4104	Serge	5604	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5605
5354	serge	5606	/* RPS code assumes GPLL is used */
		5607	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5608
6084	serge	5609	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
4104	Serge	5610	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5611
5060	serge	5612	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	5613	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5614	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5615	dev_priv->rps.cur_freq);
4104	Serge	5616
		5617	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5618	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5619	dev_priv->rps.efficient_freq);
4104	Serge	5620
5060	serge	5621	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	5622
6084	serge	5623	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	5624	}
		5625
3031	serge	5626	static unsigned long intel_pxfreq(u32 vidfreq)
		5627	{
		5628	unsigned long freq;
		5629	int div = (vidfreq & 0x3f0000) >> 16;
		5630	int post = (vidfreq & 0x3000) >> 12;
		5631	int pre = (vidfreq & 0x7);
		5632
		5633	if (!pre)
		5634	return 0;
		5635
		5636	freq = ((div * 133333) / ((1<
		5637
		5638	return freq;
		5639	}
		5640
		5641	static const struct cparams {
		5642	u16 i;
		5643	u16 t;
		5644	u16 m;
		5645	u16 c;
		5646	} cparams[] = {
		5647	{ 1, 1333, 301, 28664 },
		5648	{ 1, 1066, 294, 24460 },
		5649	{ 1, 800, 294, 25192 },
		5650	{ 0, 1333, 276, 27605 },
		5651	{ 0, 1066, 276, 27605 },
		5652	{ 0, 800, 231, 23784 },
		5653	};
		5654
		5655	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		5656	{
		5657	u64 total_count, diff, ret;
		5658	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	5659	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	5660	int i;
		5661
		5662	assert_spin_locked(&mchdev_lock);
		5663
		5664	diff1 = now - dev_priv->ips.last_time1;
		5665
		5666	/* Prevent division-by-zero if we are asking too fast.
		5667	* Also, we don't get interesting results if we are polling
		5668	* faster than once in 10ms, so just return the saved value
		5669	* in such cases.
		5670	*/
		5671	if (diff1 <= 10)
		5672	return dev_priv->ips.chipset_power;
		5673
		5674	count1 = I915_READ(DMIEC);
		5675	count2 = I915_READ(DDREC);
		5676	count3 = I915_READ(CSIEC);
		5677
		5678	total_count = count1 + count2 + count3;
		5679
		5680	/* FIXME: handle per-counter overflow */
		5681	if (total_count < dev_priv->ips.last_count1) {
		5682	diff = ~0UL - dev_priv->ips.last_count1;
		5683	diff += total_count;
		5684	} else {
		5685	diff = total_count - dev_priv->ips.last_count1;
		5686	}
		5687
		5688	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		5689	if (cparams[i].i == dev_priv->ips.c_m &&
		5690	cparams[i].t == dev_priv->ips.r_t) {
		5691	m = cparams[i].m;
		5692	c = cparams[i].c;
		5693	break;
		5694	}
		5695	}
		5696
		5697	diff = div_u64(diff, diff1);
		5698	ret = ((m * diff) + c);
		5699	ret = div_u64(ret, 10);
		5700
		5701	dev_priv->ips.last_count1 = total_count;
		5702	dev_priv->ips.last_time1 = now;
		5703
		5704	dev_priv->ips.chipset_power = ret;
		5705
		5706	return ret;
		5707	}
		5708
		5709	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		5710	{
5060	serge	5711	struct drm_device *dev = dev_priv->dev;
3031	serge	5712	unsigned long val;
		5713
5060	serge	5714	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5715	return 0;
		5716
		5717	spin_lock_irq(&mchdev_lock);
		5718
		5719	val = __i915_chipset_val(dev_priv);
		5720
		5721	spin_unlock_irq(&mchdev_lock);
		5722
		5723	return val;
		5724	}
		5725
		5726	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		5727	{
		5728	unsigned long m, x, b;
		5729	u32 tsfs;
		5730
		5731	tsfs = I915_READ(TSFS);
		5732
		5733	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		5734	x = I915_READ8(TR1);
		5735
		5736	b = tsfs & TSFS_INTR_MASK;
		5737
		5738	return ((m * x) / 127) - b;
		5739	}
		5740
6084	serge	5741	static int _pxvid_to_vd(u8 pxvid)
3031	serge	5742	{
6084	serge	5743	if (pxvid == 0)
		5744	return 0;
		5745
		5746	if (pxvid >= 8 && pxvid < 31)
		5747	pxvid = 31;
		5748
		5749	return (pxvid + 2) * 125;
		5750	}
		5751
		5752	static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		5753	{
5060	serge	5754	struct drm_device *dev = dev_priv->dev;
6084	serge	5755	const int vd = _pxvid_to_vd(pxvid);
		5756	const int vm = vd - 1125;
		5757
5060	serge	5758	if (INTEL_INFO(dev)->is_mobile)
6084	serge	5759	return vm > 0 ? vm : 0;
		5760
		5761	return vd;
3031	serge	5762	}
		5763
		5764	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5765	{
5060	serge	5766	u64 now, diff, diffms;
3031	serge	5767	u32 count;
		5768
		5769	assert_spin_locked(&mchdev_lock);
		5770
5060	serge	5771	now = ktime_get_raw_ns();
		5772	diffms = now - dev_priv->ips.last_time2;
		5773	do_div(diffms, NSEC_PER_MSEC);
3031	serge	5774
		5775	/* Don't divide by 0 */
		5776	if (!diffms)
		5777	return;
		5778
		5779	count = I915_READ(GFXEC);
		5780
		5781	if (count < dev_priv->ips.last_count2) {
		5782	diff = ~0UL - dev_priv->ips.last_count2;
		5783	diff += count;
		5784	} else {
		5785	diff = count - dev_priv->ips.last_count2;
		5786	}
		5787
		5788	dev_priv->ips.last_count2 = count;
		5789	dev_priv->ips.last_time2 = now;
		5790
		5791	/* More magic constants... */
		5792	diff = diff * 1181;
		5793	diff = div_u64(diff, diffms * 10);
		5794	dev_priv->ips.gfx_power = diff;
		5795	}
		5796
		5797	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5798	{
5060	serge	5799	struct drm_device *dev = dev_priv->dev;
		5800
		5801	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5802	return;
		5803
		5804	spin_lock_irq(&mchdev_lock);
		5805
		5806	__i915_update_gfx_val(dev_priv);
		5807
		5808	spin_unlock_irq(&mchdev_lock);
		5809	}
		5810
		5811	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		5812	{
		5813	unsigned long t, corr, state1, corr2, state2;
		5814	u32 pxvid, ext_v;
		5815
		5816	assert_spin_locked(&mchdev_lock);
		5817
6084	serge	5818	pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
3031	serge	5819	pxvid = (pxvid >> 24) & 0x7f;
		5820	ext_v = pvid_to_extvid(dev_priv, pxvid);
		5821
		5822	state1 = ext_v;
		5823
		5824	t = i915_mch_val(dev_priv);
		5825
		5826	/* Revel in the empirically derived constants */
		5827
		5828	/* Correction factor in 1/100000 units */
		5829	if (t > 80)
		5830	corr = ((t * 2349) + 135940);
		5831	else if (t >= 50)
		5832	corr = ((t * 964) + 29317);
		5833	else /* < 50 */
		5834	corr = ((t * 301) + 1004);
		5835
		5836	corr = corr * ((150142 * state1) / 10000 - 78642);
		5837	corr /= 100000;
		5838	corr2 = (corr * dev_priv->ips.corr);
		5839
		5840	state2 = (corr2 * state1) / 10000;
		5841	state2 /= 100; /* convert to mW */
		5842
		5843	__i915_update_gfx_val(dev_priv);
		5844
		5845	return dev_priv->ips.gfx_power + state2;
		5846	}
		5847
		5848	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		5849	{
5060	serge	5850	struct drm_device *dev = dev_priv->dev;
3031	serge	5851	unsigned long val;
		5852
5060	serge	5853	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5854	return 0;
		5855
		5856	spin_lock_irq(&mchdev_lock);
		5857
		5858	val = __i915_gfx_val(dev_priv);
		5859
		5860	spin_unlock_irq(&mchdev_lock);
		5861
		5862	return val;
		5863	}
		5864
		5865	/**
		5866	* i915_read_mch_val - return value for IPS use
		5867	*
		5868	* Calculate and return a value for the IPS driver to use when deciding whether
		5869	* we have thermal and power headroom to increase CPU or GPU power budget.
		5870	*/
		5871	unsigned long i915_read_mch_val(void)
		5872	{
		5873	struct drm_i915_private *dev_priv;
		5874	unsigned long chipset_val, graphics_val, ret = 0;
		5875
		5876	spin_lock_irq(&mchdev_lock);
		5877	if (!i915_mch_dev)
		5878	goto out_unlock;
		5879	dev_priv = i915_mch_dev;
		5880
		5881	chipset_val = __i915_chipset_val(dev_priv);
		5882	graphics_val = __i915_gfx_val(dev_priv);
		5883
		5884	ret = chipset_val + graphics_val;
		5885
		5886	out_unlock:
		5887	spin_unlock_irq(&mchdev_lock);
		5888
		5889	return ret;
		5890	}
		5891	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		5892
		5893	/**
		5894	* i915_gpu_raise - raise GPU frequency limit
		5895	*
		5896	* Raise the limit; IPS indicates we have thermal headroom.
		5897	*/
		5898	bool i915_gpu_raise(void)
		5899	{
		5900	struct drm_i915_private *dev_priv;
		5901	bool ret = true;
		5902
		5903	spin_lock_irq(&mchdev_lock);
		5904	if (!i915_mch_dev) {
		5905	ret = false;
		5906	goto out_unlock;
		5907	}
		5908	dev_priv = i915_mch_dev;
		5909
		5910	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		5911	dev_priv->ips.max_delay--;
		5912
		5913	out_unlock:
		5914	spin_unlock_irq(&mchdev_lock);
		5915
		5916	return ret;
		5917	}
		5918	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		5919
		5920	/**
		5921	* i915_gpu_lower - lower GPU frequency limit
		5922	*
		5923	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		5924	* frequency maximum.
		5925	*/
		5926	bool i915_gpu_lower(void)
		5927	{
		5928	struct drm_i915_private *dev_priv;
		5929	bool ret = true;
		5930
		5931	spin_lock_irq(&mchdev_lock);
		5932	if (!i915_mch_dev) {
		5933	ret = false;
		5934	goto out_unlock;
		5935	}
		5936	dev_priv = i915_mch_dev;
		5937
		5938	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		5939	dev_priv->ips.max_delay++;
		5940
		5941	out_unlock:
		5942	spin_unlock_irq(&mchdev_lock);
		5943
		5944	return ret;
		5945	}
		5946	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		5947
		5948	/**
		5949	* i915_gpu_busy - indicate GPU business to IPS
		5950	*
		5951	* Tell the IPS driver whether or not the GPU is busy.
		5952	*/
		5953	bool i915_gpu_busy(void)
		5954	{
		5955	struct drm_i915_private *dev_priv;
5060	serge	5956	struct intel_engine_cs *ring;
3031	serge	5957	bool ret = false;
		5958	int i;
		5959
		5960	spin_lock_irq(&mchdev_lock);
		5961	if (!i915_mch_dev)
		5962	goto out_unlock;
		5963	dev_priv = i915_mch_dev;
		5964
		5965	for_each_ring(ring, dev_priv, i)
		5966	ret \|= !list_empty(&ring->request_list);
		5967
		5968	out_unlock:
		5969	spin_unlock_irq(&mchdev_lock);
		5970
		5971	return ret;
		5972	}
		5973	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		5974
		5975	/**
		5976	* i915_gpu_turbo_disable - disable graphics turbo
		5977	*
		5978	* Disable graphics turbo by resetting the max frequency and setting the
		5979	* current frequency to the default.
		5980	*/
		5981	bool i915_gpu_turbo_disable(void)
		5982	{
		5983	struct drm_i915_private *dev_priv;
		5984	bool ret = true;
		5985
		5986	spin_lock_irq(&mchdev_lock);
		5987	if (!i915_mch_dev) {
		5988	ret = false;
		5989	goto out_unlock;
		5990	}
		5991	dev_priv = i915_mch_dev;
		5992
		5993	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		5994
		5995	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		5996	ret = false;
		5997
		5998	out_unlock:
		5999	spin_unlock_irq(&mchdev_lock);
		6000
		6001	return ret;
		6002	}
		6003	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		6004
		6005	/**
		6006	* Tells the intel_ips driver that the i915 driver is now loaded, if
		6007	* IPS got loaded first.
		6008	*
		6009	* This awkward dance is so that neither module has to depend on the
		6010	* other in order for IPS to do the appropriate communication of
		6011	* GPU turbo limits to i915.
		6012	*/
		6013	static void
		6014	ips_ping_for_i915_load(void)
		6015	{
		6016	void (*link)(void);
		6017
		6018	// link = symbol_get(ips_link_to_i915_driver);
		6019	// if (link) {
		6020	// link();
		6021	// symbol_put(ips_link_to_i915_driver);
		6022	// }
		6023	}
		6024
		6025	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		6026	{
		6027	/* We only register the i915 ips part with intel-ips once everything is
		6028	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		6029	spin_lock_irq(&mchdev_lock);
		6030	i915_mch_dev = dev_priv;
		6031	spin_unlock_irq(&mchdev_lock);
		6032
		6033	ips_ping_for_i915_load();
		6034	}
		6035
		6036	void intel_gpu_ips_teardown(void)
		6037	{
		6038	spin_lock_irq(&mchdev_lock);
		6039	i915_mch_dev = NULL;
		6040	spin_unlock_irq(&mchdev_lock);
		6041	}
5060	serge	6042
3031	serge	6043	static void intel_init_emon(struct drm_device *dev)
		6044	{
		6045	struct drm_i915_private *dev_priv = dev->dev_private;
		6046	u32 lcfuse;
		6047	u8 pxw[16];
		6048	int i;
		6049
		6050	/* Disable to program */
		6051	I915_WRITE(ECR, 0);
		6052	POSTING_READ(ECR);
		6053
		6054	/* Program energy weights for various events */
		6055	I915_WRITE(SDEW, 0x15040d00);
		6056	I915_WRITE(CSIEW0, 0x007f0000);
		6057	I915_WRITE(CSIEW1, 0x1e220004);
		6058	I915_WRITE(CSIEW2, 0x04000004);
		6059
		6060	for (i = 0; i < 5; i++)
6084	serge	6061	I915_WRITE(PEW(i), 0);
3031	serge	6062	for (i = 0; i < 3; i++)
6084	serge	6063	I915_WRITE(DEW(i), 0);
3031	serge	6064
		6065	/* Program P-state weights to account for frequency power adjustment */
		6066	for (i = 0; i < 16; i++) {
6084	serge	6067	u32 pxvidfreq = I915_READ(PXVFREQ(i));
3031	serge	6068	unsigned long freq = intel_pxfreq(pxvidfreq);
		6069	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		6070	PXVFREQ_PX_SHIFT;
		6071	unsigned long val;
		6072
		6073	val = vid * vid;
		6074	val *= (freq / 1000);
		6075	val *= 255;
		6076	val /= (127127900);
		6077	if (val > 0xff)
		6078	DRM_ERROR("bad pxval: %ld\n", val);
		6079	pxw[i] = val;
		6080	}
		6081	/* Render standby states get 0 weight */
		6082	pxw[14] = 0;
		6083	pxw[15] = 0;
		6084
		6085	for (i = 0; i < 4; i++) {
		6086	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		6087	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
6084	serge	6088	I915_WRITE(PXW(i), val);
3031	serge	6089	}
		6090
		6091	/* Adjust magic regs to magic values (more experimental results) */
		6092	I915_WRITE(OGW0, 0);
		6093	I915_WRITE(OGW1, 0);
		6094	I915_WRITE(EG0, 0x00007f00);
		6095	I915_WRITE(EG1, 0x0000000e);
		6096	I915_WRITE(EG2, 0x000e0000);
		6097	I915_WRITE(EG3, 0x68000300);
		6098	I915_WRITE(EG4, 0x42000000);
		6099	I915_WRITE(EG5, 0x00140031);
		6100	I915_WRITE(EG6, 0);
		6101	I915_WRITE(EG7, 0);
		6102
		6103	for (i = 0; i < 8; i++)
6084	serge	6104	I915_WRITE(PXWL(i), 0);
3031	serge	6105
		6106	/* Enable PMON + select events */
		6107	I915_WRITE(ECR, 0x80000019);
		6108
		6109	lcfuse = I915_READ(LCFUSE02);
		6110
		6111	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		6112	}
		6113
5060	serge	6114	void intel_init_gt_powersave(struct drm_device *dev)
		6115	{
		6116	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
		6117
		6118	if (IS_CHERRYVIEW(dev))
		6119	cherryview_init_gt_powersave(dev);
		6120	else if (IS_VALLEYVIEW(dev))
		6121	valleyview_init_gt_powersave(dev);
		6122	}
		6123
		6124	void intel_cleanup_gt_powersave(struct drm_device *dev)
		6125	{
		6126	if (IS_CHERRYVIEW(dev))
		6127	return;
		6128	else if (IS_VALLEYVIEW(dev))
		6129	valleyview_cleanup_gt_powersave(dev);
		6130	}
		6131
5354	serge	6132	static void gen6_suspend_rps(struct drm_device *dev)
		6133	{
		6134	struct drm_i915_private *dev_priv = dev->dev_private;
		6135
		6136	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		6137
6084	serge	6138	gen6_disable_rps_interrupts(dev);
5354	serge	6139	}
		6140
5060	serge	6141	/**
		6142	* intel_suspend_gt_powersave - suspend PM work and helper threads
		6143	* @dev: drm device
		6144	*
		6145	* We don't want to disable RC6 or other features here, we just want
		6146	* to make sure any work we've queued has finished and won't bother
		6147	* us while we're suspended.
		6148	*/
		6149	void intel_suspend_gt_powersave(struct drm_device *dev)
		6150	{
		6151	struct drm_i915_private *dev_priv = dev->dev_private;
		6152
5354	serge	6153	if (INTEL_INFO(dev)->gen < 6)
		6154	return;
5060	serge	6155
5354	serge	6156	gen6_suspend_rps(dev);
5060	serge	6157
		6158	/* Force GPU to min freq during suspend */
		6159	gen6_rps_idle(dev_priv);
		6160	}
		6161
3031	serge	6162	void intel_disable_gt_powersave(struct drm_device *dev)
		6163	{
3243	Serge	6164	struct drm_i915_private *dev_priv = dev->dev_private;
		6165
3031	serge	6166	if (IS_IRONLAKE_M(dev)) {
		6167	ironlake_disable_drps(dev);
4293	Serge	6168	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	6169	intel_suspend_gt_powersave(dev);
		6170
3482	Serge	6171	mutex_lock(&dev_priv->rps.hw_lock);
5354	serge	6172	if (INTEL_INFO(dev)->gen >= 9)
		6173	gen9_disable_rps(dev);
		6174	else if (IS_CHERRYVIEW(dev))
5060	serge	6175	cherryview_disable_rps(dev);
		6176	else if (IS_VALLEYVIEW(dev))
4104	Serge	6177	valleyview_disable_rps(dev);
		6178	else
6084	serge	6179	gen6_disable_rps(dev);
5354	serge	6180
4560	Serge	6181	dev_priv->rps.enabled = false;
3480	Serge	6182	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	6183	}
		6184	}
		6185
3482	Serge	6186	static void intel_gen6_powersave_work(struct work_struct *work)
		6187	{
		6188	struct drm_i915_private *dev_priv =
		6189	container_of(work, struct drm_i915_private,
		6190	rps.delayed_resume_work.work);
		6191	struct drm_device *dev = dev_priv->dev;
		6192
		6193	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	6194
6084	serge	6195	gen6_reset_rps_interrupts(dev);
5354	serge	6196
5060	serge	6197	if (IS_CHERRYVIEW(dev)) {
		6198	cherryview_enable_rps(dev);
		6199	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	6200	valleyview_enable_rps(dev);
5354	serge	6201	} else if (INTEL_INFO(dev)->gen >= 9) {
6084	serge	6202	gen9_enable_rc6(dev);
5354	serge	6203	gen9_enable_rps(dev);
6084	serge	6204	if (IS_SKYLAKE(dev))
		6205	__gen6_update_ring_freq(dev);
4560	Serge	6206	} else if (IS_BROADWELL(dev)) {
		6207	gen8_enable_rps(dev);
5060	serge	6208	__gen6_update_ring_freq(dev);
4104	Serge	6209	} else {
6084	serge	6210	gen6_enable_rps(dev);
5060	serge	6211	__gen6_update_ring_freq(dev);
4104	Serge	6212	}
6084	serge	6213
		6214	WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
		6215	WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
		6216
		6217	WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
		6218	WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
		6219
4560	Serge	6220	dev_priv->rps.enabled = true;
5354	serge	6221
6084	serge	6222	gen6_enable_rps_interrupts(dev);
5354	serge	6223
3482	Serge	6224	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	6225
		6226	intel_runtime_pm_put(dev_priv);
3482	Serge	6227	}
		6228
3031	serge	6229	void intel_enable_gt_powersave(struct drm_device *dev)
		6230	{
3243	Serge	6231	struct drm_i915_private *dev_priv = dev->dev_private;
		6232
6084	serge	6233	/* Powersaving is controlled by the host when inside a VM */
		6234	if (intel_vgpu_active(dev))
		6235	return;
		6236
3031	serge	6237	if (IS_IRONLAKE_M(dev)) {
5060	serge	6238	mutex_lock(&dev->struct_mutex);
3031	serge	6239	ironlake_enable_drps(dev);
		6240	intel_init_emon(dev);
5060	serge	6241	mutex_unlock(&dev->struct_mutex);
		6242	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	6243	/*
		6244	* PCU communication is slow and this doesn't need to be
		6245	* done at any specific time, so do this out of our fast path
		6246	* to make resume and init faster.
5060	serge	6247	*
		6248	* We depend on the HW RC6 power context save/restore
		6249	* mechanism when entering D3 through runtime PM suspend. So
		6250	* disable RPM until RPS/RC6 is properly setup. We can only
		6251	* get here via the driver load/system resume/runtime resume
		6252	* paths, so the _noresume version is enough (and in case of
		6253	* runtime resume it's necessary).
3243	Serge	6254	*/
5060	serge	6255	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		6256	round_jiffies_up_relative(HZ)))
		6257	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	6258	}
		6259	}
		6260
5060	serge	6261	void intel_reset_gt_powersave(struct drm_device *dev)
		6262	{
		6263	struct drm_i915_private *dev_priv = dev->dev_private;
		6264
5354	serge	6265	if (INTEL_INFO(dev)->gen < 6)
		6266	return;
		6267
		6268	gen6_suspend_rps(dev);
5060	serge	6269	dev_priv->rps.enabled = false;
		6270	}
		6271
3243	Serge	6272	static void ibx_init_clock_gating(struct drm_device *dev)
		6273	{
		6274	struct drm_i915_private *dev_priv = dev->dev_private;
		6275
		6276	/*
		6277	* On Ibex Peak and Cougar Point, we need to disable clock
		6278	* gating for the panel power sequencer or it will fail to
		6279	* start up when no ports are active.
		6280	*/
		6281	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		6282	}
		6283
4104	Serge	6284	static void g4x_disable_trickle_feed(struct drm_device *dev)
		6285	{
		6286	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	6287	enum pipe pipe;
4104	Serge	6288
5354	serge	6289	for_each_pipe(dev_priv, pipe) {
4104	Serge	6290	I915_WRITE(DSPCNTR(pipe),
		6291	I915_READ(DSPCNTR(pipe)) \|
		6292	DISPPLANE_TRICKLE_FEED_DISABLE);
6084	serge	6293
		6294	I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
		6295	POSTING_READ(DSPSURF(pipe));
4104	Serge	6296	}
		6297	}
		6298
4560	Serge	6299	static void ilk_init_lp_watermarks(struct drm_device *dev)
		6300	{
		6301	struct drm_i915_private *dev_priv = dev->dev_private;
		6302
		6303	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		6304	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		6305	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		6306
		6307	/*
		6308	* Don't touch WM1S_LP_EN here.
		6309	* Doing so could cause underruns.
		6310	*/
		6311	}
		6312
3031	serge	6313	static void ironlake_init_clock_gating(struct drm_device *dev)
		6314	{
		6315	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6316	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6317
4104	Serge	6318	/*
		6319	* Required for FBC
		6320	* WaFbcDisableDpfcClockGating:ilk
		6321	*/
3243	Serge	6322	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		6323	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		6324	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	6325
		6326	I915_WRITE(PCH_3DCGDIS0,
		6327	MARIUNIT_CLOCK_GATE_DISABLE \|
		6328	SVSMUNIT_CLOCK_GATE_DISABLE);
		6329	I915_WRITE(PCH_3DCGDIS1,
		6330	VFMUNIT_CLOCK_GATE_DISABLE);
		6331
		6332	/*
		6333	* According to the spec the following bits should be set in
		6334	* order to enable memory self-refresh
		6335	* The bit 22/21 of 0x42004
		6336	* The bit 5 of 0x42020
		6337	* The bit 15 of 0x45000
		6338	*/
		6339	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6340	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6341	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	6342	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	6343	I915_WRITE(DISP_ARB_CTL,
		6344	(I915_READ(DISP_ARB_CTL) \|
		6345	DISP_FBC_WM_DIS));
		6346
4560	Serge	6347	ilk_init_lp_watermarks(dev);
		6348
3031	serge	6349	/*
		6350	* Based on the document from hardware guys the following bits
		6351	* should be set unconditionally in order to enable FBC.
		6352	* The bit 22 of 0x42000
		6353	* The bit 22 of 0x42004
		6354	* The bit 7,8,9 of 0x42020.
		6355	*/
		6356	if (IS_IRONLAKE_M(dev)) {
4104	Serge	6357	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	6358	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6359	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6360	ILK_FBCQ_DIS);
		6361	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6362	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6363	ILK_DPARB_GATE);
		6364	}
		6365
3243	Serge	6366	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		6367
3031	serge	6368	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6369	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6370	ILK_ELPIN_409_SELECT);
		6371	I915_WRITE(_3D_CHICKEN2,
		6372	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		6373	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	6374
4104	Serge	6375	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	6376	I915_WRITE(CACHE_MODE_0,
		6377	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		6378
5060	serge	6379	/* WaDisable_RenderCache_OperationalFlush:ilk */
		6380	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6381
4104	Serge	6382	g4x_disable_trickle_feed(dev);
		6383
3243	Serge	6384	ibx_init_clock_gating(dev);
3031	serge	6385	}
		6386
3243	Serge	6387	static void cpt_init_clock_gating(struct drm_device *dev)
		6388	{
		6389	struct drm_i915_private *dev_priv = dev->dev_private;
		6390	int pipe;
3746	Serge	6391	uint32_t val;
3243	Serge	6392
		6393	/*
		6394	* On Ibex Peak and Cougar Point, we need to disable clock
		6395	* gating for the panel power sequencer or it will fail to
		6396	* start up when no ports are active.
		6397	*/
4280	Serge	6398	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		6399	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		6400	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	6401	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		6402	DPLS_EDP_PPS_FIX_DIS);
		6403	/* The below fixes the weird display corruption, a few pixels shifted
		6404	* downward, on (only) LVDS of some HP laptops with IVY.
		6405	*/
5354	serge	6406	for_each_pipe(dev_priv, pipe) {
3746	Serge	6407	val = I915_READ(TRANS_CHICKEN2(pipe));
		6408	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		6409	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	6410	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	6411	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		6412	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		6413	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		6414	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		6415	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		6416	}
3243	Serge	6417	/* WADP0ClockGatingDisable */
5354	serge	6418	for_each_pipe(dev_priv, pipe) {
3243	Serge	6419	I915_WRITE(TRANS_CHICKEN1(pipe),
		6420	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		6421	}
		6422	}
		6423
3480	Serge	6424	static void gen6_check_mch_setup(struct drm_device *dev)
		6425	{
		6426	struct drm_i915_private *dev_priv = dev->dev_private;
		6427	uint32_t tmp;
		6428
		6429	tmp = I915_READ(MCH_SSKPD);
5060	serge	6430	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		6431	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		6432	tmp);
3480	Serge	6433	}
		6434
3031	serge	6435	static void gen6_init_clock_gating(struct drm_device *dev)
		6436	{
		6437	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6438	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6439
3243	Serge	6440	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	6441
		6442	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6443	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6444	ILK_ELPIN_409_SELECT);
		6445
4104	Serge	6446	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	6447	I915_WRITE(_3D_CHICKEN,
		6448	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		6449
5060	serge	6450	/* WaDisable_RenderCache_OperationalFlush:snb */
		6451	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6452
		6453	/*
		6454	* BSpec recoomends 8x4 when MSAA is used,
		6455	* however in practice 16x4 seems fastest.
		6456	*
		6457	* Note that PS/WM thread counts depend on the WIZ hashing
		6458	* disable bit, which we don't touch here, but it's good
		6459	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6460	*/
		6461	I915_WRITE(GEN6_GT_MODE,
5354	serge	6462	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6463
4560	Serge	6464	ilk_init_lp_watermarks(dev);
3031	serge	6465
		6466	I915_WRITE(CACHE_MODE_0,
		6467	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		6468
		6469	I915_WRITE(GEN6_UCGCTL1,
		6470	I915_READ(GEN6_UCGCTL1) \|
		6471	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		6472	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6473
		6474	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		6475	* gating disable must be set. Failure to set it results in
		6476	* flickering pixels due to Z write ordering failures after
		6477	* some amount of runtime in the Mesa "fire" demo, and Unigine
		6478	* Sanctuary and Tropics, and apparently anything else with
		6479	* alpha test or pixel discard.
		6480	*
		6481	* According to the spec, bit 11 (RCCUNIT) must also be set,
		6482	* but we didn't debug actual testcases to find it out.
		6483	*
5060	serge	6484	* WaDisableRCCUnitClockGating:snb
		6485	* WaDisableRCPBUnitClockGating:snb
3031	serge	6486	*/
		6487	I915_WRITE(GEN6_UCGCTL2,
		6488	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		6489	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		6490
5060	serge	6491	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		6492	I915_WRITE(_3D_CHICKEN3,
		6493	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	6494
		6495	/*
5060	serge	6496	* Bspec says:
		6497	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		6498	* 3DSTATE_SF number of SF output attributes is more than 16."
		6499	*/
		6500	I915_WRITE(_3D_CHICKEN3,
		6501	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		6502
		6503	/*
3031	serge	6504	* According to the spec the following bits should be
		6505	* set in order to enable memory self-refresh and fbc:
		6506	* The bit21 and bit22 of 0x42000
		6507	* The bit21 and bit22 of 0x42004
		6508	* The bit5 and bit7 of 0x42020
		6509	* The bit14 of 0x70180
		6510	* The bit14 of 0x71180
4104	Serge	6511	*
		6512	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	6513	*/
		6514	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6515	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6516	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		6517	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6518	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6519	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	6520	I915_WRITE(ILK_DSPCLK_GATE_D,
		6521	I915_READ(ILK_DSPCLK_GATE_D) \|
		6522	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		6523	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	6524
4104	Serge	6525	g4x_disable_trickle_feed(dev);
3031	serge	6526
3243	Serge	6527	cpt_init_clock_gating(dev);
3480	Serge	6528
		6529	gen6_check_mch_setup(dev);
3031	serge	6530	}
		6531
		6532	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		6533	{
		6534	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		6535
5060	serge	6536	/*
		6537	* WaVSThreadDispatchOverride:ivb,vlv
		6538	*
		6539	* This actually overrides the dispatch
		6540	* mode for all thread types.
		6541	*/
3031	serge	6542	reg &= ~GEN7_FF_SCHED_MASK;
		6543	reg \|= GEN7_FF_TS_SCHED_HW;
		6544	reg \|= GEN7_FF_VS_SCHED_HW;
		6545	reg \|= GEN7_FF_DS_SCHED_HW;
		6546
		6547	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		6548	}
		6549
3243	Serge	6550	static void lpt_init_clock_gating(struct drm_device *dev)
		6551	{
		6552	struct drm_i915_private *dev_priv = dev->dev_private;
		6553
		6554	/*
		6555	* TODO: this bit should only be enabled when really needed, then
		6556	* disabled when not needed anymore in order to save power.
		6557	*/
6084	serge	6558	if (HAS_PCH_LPT_LP(dev))
3243	Serge	6559	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		6560	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		6561	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	6562
		6563	/* WADPOClockGatingDisable:hsw */
6084	serge	6564	I915_WRITE(TRANS_CHICKEN1(PIPE_A),
		6565	I915_READ(TRANS_CHICKEN1(PIPE_A)) \|
4104	Serge	6566	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	6567	}
		6568
4104	Serge	6569	static void lpt_suspend_hw(struct drm_device *dev)
		6570	{
		6571	struct drm_i915_private *dev_priv = dev->dev_private;
		6572
6084	serge	6573	if (HAS_PCH_LPT_LP(dev)) {
4104	Serge	6574	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		6575
		6576	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		6577	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		6578	}
		6579	}
		6580
5354	serge	6581	static void broadwell_init_clock_gating(struct drm_device *dev)
3031	serge	6582	{
		6583	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	6584	enum pipe pipe;
6084	serge	6585	uint32_t misccpctl;
3031	serge	6586
6084	serge	6587	ilk_init_lp_watermarks(dev);
3031	serge	6588
4560	Serge	6589	/* WaSwitchSolVfFArbitrationPriority:bdw */
		6590	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6591
		6592	/* WaPsrDPAMaskVBlankInSRD:bdw */
		6593	I915_WRITE(CHICKEN_PAR1_1,
		6594	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		6595
		6596	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5354	serge	6597	for_each_pipe(dev_priv, pipe) {
5060	serge	6598	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		6599	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		6600	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	6601	}
		6602
		6603	/* WaVSRefCountFullforceMissDisable:bdw */
		6604	/* WaDSRefCountFullforceMissDisable:bdw */
		6605	I915_WRITE(GEN7_FF_THREAD_MODE,
		6606	I915_READ(GEN7_FF_THREAD_MODE) &
		6607	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	6608
		6609	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6610	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6611
		6612	/* WaDisableSDEUnitClockGating:bdw */
		6613	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6614	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		6615
6084	serge	6616	/*
		6617	* WaProgramL3SqcReg1Default:bdw
		6618	* WaTempDisableDOPClkGating:bdw
		6619	*/
		6620	misccpctl = I915_READ(GEN7_MISCCPCTL);
		6621	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		6622	I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
		6623	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		6624
		6625	/*
		6626	* WaGttCachingOffByDefault:bdw
		6627	* GTT cache may not work with big pages, so if those
		6628	* are ever enabled GTT cache may need to be disabled.
		6629	*/
		6630	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
		6631
5354	serge	6632	lpt_init_clock_gating(dev);
4560	Serge	6633	}
		6634
		6635	static void haswell_init_clock_gating(struct drm_device *dev)
		6636	{
		6637	struct drm_i915_private *dev_priv = dev->dev_private;
		6638
		6639	ilk_init_lp_watermarks(dev);
		6640
4104	Serge	6641	/* L3 caching of data atomics doesn't work -- disable it. */
		6642	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		6643	I915_WRITE(HSW_ROW_CHICKEN3,
		6644	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		6645
		6646	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	6647	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6648	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6649	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6650
4104	Serge	6651	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	6652	I915_WRITE(GEN7_FF_THREAD_MODE,
		6653	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	6654
5060	serge	6655	/* WaDisable_RenderCache_OperationalFlush:hsw */
		6656	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6657
		6658	/* enable HiZ Raw Stall Optimization */
		6659	I915_WRITE(CACHE_MODE_0_GEN7,
		6660	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6661
4104	Serge	6662	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	6663	I915_WRITE(CACHE_MODE_1,
		6664	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6665
5060	serge	6666	/*
		6667	* BSpec recommends 8x4 when MSAA is used,
		6668	* however in practice 16x4 seems fastest.
		6669	*
		6670	* Note that PS/WM thread counts depend on the WIZ hashing
		6671	* disable bit, which we don't touch here, but it's good
		6672	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6673	*/
		6674	I915_WRITE(GEN7_GT_MODE,
5354	serge	6675	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6676
6084	serge	6677	/* WaSampleCChickenBitEnable:hsw */
		6678	I915_WRITE(HALF_SLICE_CHICKEN3,
		6679	_MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
		6680
4104	Serge	6681	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	6682	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6683
4104	Serge	6684	/* WaRsPkgCStateDisplayPMReq:hsw */
		6685	I915_WRITE(CHICKEN_PAR1_1,
		6686	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	6687
3243	Serge	6688	lpt_init_clock_gating(dev);
3031	serge	6689	}
		6690
		6691	static void ivybridge_init_clock_gating(struct drm_device *dev)
		6692	{
		6693	struct drm_i915_private *dev_priv = dev->dev_private;
		6694	uint32_t snpcr;
		6695
4560	Serge	6696	ilk_init_lp_watermarks(dev);
3031	serge	6697
3243	Serge	6698	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6699
4104	Serge	6700	/* WaDisableEarlyCull:ivb */
3243	Serge	6701	I915_WRITE(_3D_CHICKEN3,
		6702	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6703
4104	Serge	6704	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	6705	I915_WRITE(IVB_CHICKEN3,
		6706	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6707	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6708
4104	Serge	6709	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	6710	if (IS_IVB_GT1(dev))
		6711	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		6712	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		6713
5060	serge	6714	/* WaDisable_RenderCache_OperationalFlush:ivb */
		6715	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6716
4104	Serge	6717	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	6718	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		6719	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		6720
4104	Serge	6721	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	6722	I915_WRITE(GEN7_L3CNTLREG1,
		6723	GEN7_WA_FOR_GEN7_L3_CONTROL);
		6724	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6084	serge	6725	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	6726	if (IS_IVB_GT1(dev))
		6727	I915_WRITE(GEN7_ROW_CHICKEN2,
		6728	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6729	else {
		6730	/* must write both registers */
		6731	I915_WRITE(GEN7_ROW_CHICKEN2,
		6732	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	6733	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		6734	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6735	}
3031	serge	6736
4104	Serge	6737	/* WaForceL3Serialization:ivb */
3243	Serge	6738	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6739	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6740
5060	serge	6741	/*
3031	serge	6742	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6743	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	6744	*/
		6745	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6746	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6747
4104	Serge	6748	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	6749	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6750	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6751	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6752
4104	Serge	6753	g4x_disable_trickle_feed(dev);
3031	serge	6754
		6755	gen7_setup_fixed_func_scheduler(dev_priv);
		6756
5060	serge	6757	if (0) { /* causes HiZ corruption on ivb:gt1 */
		6758	/* enable HiZ Raw Stall Optimization */
		6759	I915_WRITE(CACHE_MODE_0_GEN7,
		6760	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6761	}
		6762
4104	Serge	6763	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	6764	I915_WRITE(CACHE_MODE_1,
		6765	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6766
5060	serge	6767	/*
		6768	* BSpec recommends 8x4 when MSAA is used,
		6769	* however in practice 16x4 seems fastest.
		6770	*
		6771	* Note that PS/WM thread counts depend on the WIZ hashing
		6772	* disable bit, which we don't touch here, but it's good
		6773	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6774	*/
		6775	I915_WRITE(GEN7_GT_MODE,
5354	serge	6776	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6777
3031	serge	6778	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		6779	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		6780	snpcr \|= GEN6_MBC_SNPCR_MED;
		6781	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	6782
3746	Serge	6783	if (!HAS_PCH_NOP(dev))
6084	serge	6784	cpt_init_clock_gating(dev);
3480	Serge	6785
		6786	gen6_check_mch_setup(dev);
3031	serge	6787	}
		6788
6084	serge	6789	static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
		6790	{
		6791	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
		6792
		6793	/*
		6794	* Disable trickle feed and enable pnd deadline calculation
		6795	*/
		6796	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		6797	I915_WRITE(CBR1_VLV, 0);
		6798	}
		6799
3031	serge	6800	static void valleyview_init_clock_gating(struct drm_device *dev)
		6801	{
		6802	struct drm_i915_private *dev_priv = dev->dev_private;
		6803
6084	serge	6804	vlv_init_display_clock_gating(dev_priv);
3031	serge	6805
4104	Serge	6806	/* WaDisableEarlyCull:vlv */
3243	Serge	6807	I915_WRITE(_3D_CHICKEN3,
		6808	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6809
4104	Serge	6810	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	6811	I915_WRITE(IVB_CHICKEN3,
		6812	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6813	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6814
5060	serge	6815	/* WaPsdDispatchEnable:vlv */
4104	Serge	6816	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	6817	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	6818	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		6819	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	6820
5060	serge	6821	/* WaDisable_RenderCache_OperationalFlush:vlv */
		6822	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6823
4104	Serge	6824	/* WaForceL3Serialization:vlv */
3243	Serge	6825	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6826	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6827
4104	Serge	6828	/* WaDisableDopClockGating:vlv */
3243	Serge	6829	I915_WRITE(GEN7_ROW_CHICKEN2,
		6830	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		6831
4104	Serge	6832	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	6833	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6834	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6835	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6836
5060	serge	6837	gen7_setup_fixed_func_scheduler(dev_priv);
		6838
		6839	/*
3031	serge	6840	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6841	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	6842	*/
		6843	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6844	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6845
5060	serge	6846	/* WaDisableL3Bank2xClockGate:vlv
		6847	* Disabling L3 clock gating- MMIO 940c[25] = 1
		6848	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		6849	I915_WRITE(GEN7_UCGCTL4,
		6850	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	6851
5060	serge	6852	/*
		6853	* BSpec says this must be set, even though
		6854	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		6855	*/
3031	serge	6856	I915_WRITE(CACHE_MODE_1,
		6857	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6858
		6859	/*
6084	serge	6860	* BSpec recommends 8x4 when MSAA is used,
		6861	* however in practice 16x4 seems fastest.
		6862	*
		6863	* Note that PS/WM thread counts depend on the WIZ hashing
		6864	* disable bit, which we don't touch here, but it's good
		6865	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6866	*/
		6867	I915_WRITE(GEN7_GT_MODE,
		6868	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
		6869
		6870	/*
5060	serge	6871	* WaIncreaseL3CreditsForVLVB0:vlv
		6872	* This is the hardware default actually.
		6873	*/
		6874	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		6875
		6876	/*
4104	Serge	6877	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	6878	* Disable clock gating on th GCFG unit to prevent a delay
		6879	* in the reporting of vblank events.
		6880	*/
5060	serge	6881	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		6882	}
3746	Serge	6883
5060	serge	6884	static void cherryview_init_clock_gating(struct drm_device *dev)
		6885	{
		6886	struct drm_i915_private *dev_priv = dev->dev_private;
		6887
6084	serge	6888	vlv_init_display_clock_gating(dev_priv);
5060	serge	6889
		6890	/* WaVSRefCountFullforceMissDisable:chv */
		6891	/* WaDSRefCountFullforceMissDisable:chv */
		6892	I915_WRITE(GEN7_FF_THREAD_MODE,
		6893	I915_READ(GEN7_FF_THREAD_MODE) &
		6894	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		6895
		6896	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		6897	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6898	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6899
		6900	/* WaDisableCSUnitClockGating:chv */
		6901	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		6902	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6903
		6904	/* WaDisableSDEUnitClockGating:chv */
		6905	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6906	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6084	serge	6907
		6908	/*
		6909	* GTT cache may not work with big pages, so if those
		6910	* are ever enabled GTT cache may need to be disabled.
		6911	*/
		6912	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
3031	serge	6913	}
		6914
		6915	static void g4x_init_clock_gating(struct drm_device *dev)
		6916	{
		6917	struct drm_i915_private *dev_priv = dev->dev_private;
		6918	uint32_t dspclk_gate;
		6919
		6920	I915_WRITE(RENCLK_GATE_D1, 0);
		6921	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		6922	GS_UNIT_CLOCK_GATE_DISABLE \|
		6923	CL_UNIT_CLOCK_GATE_DISABLE);
		6924	I915_WRITE(RAMCLK_GATE_D, 0);
		6925	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		6926	OVRUNIT_CLOCK_GATE_DISABLE \|
		6927	OVCUNIT_CLOCK_GATE_DISABLE;
		6928	if (IS_GM45(dev))
		6929	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		6930	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	6931
		6932	/* WaDisableRenderCachePipelinedFlush */
		6933	I915_WRITE(CACHE_MODE_0,
		6934	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	6935
5060	serge	6936	/* WaDisable_RenderCache_OperationalFlush:g4x */
		6937	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6938
4104	Serge	6939	g4x_disable_trickle_feed(dev);
3031	serge	6940	}
		6941
		6942	static void crestline_init_clock_gating(struct drm_device *dev)
		6943	{
		6944	struct drm_i915_private *dev_priv = dev->dev_private;
		6945
		6946	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		6947	I915_WRITE(RENCLK_GATE_D2, 0);
		6948	I915_WRITE(DSPCLK_GATE_D, 0);
		6949	I915_WRITE(RAMCLK_GATE_D, 0);
		6950	I915_WRITE16(DEUC, 0);
4104	Serge	6951	I915_WRITE(MI_ARB_STATE,
		6952	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6953
		6954	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6955	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6956	}
		6957
		6958	static void broadwater_init_clock_gating(struct drm_device *dev)
		6959	{
		6960	struct drm_i915_private *dev_priv = dev->dev_private;
		6961
		6962	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		6963	I965_RCC_CLOCK_GATE_DISABLE \|
		6964	I965_RCPB_CLOCK_GATE_DISABLE \|
		6965	I965_ISC_CLOCK_GATE_DISABLE \|
		6966	I965_FBC_CLOCK_GATE_DISABLE);
		6967	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	6968	I915_WRITE(MI_ARB_STATE,
		6969	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6970
		6971	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6972	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6973	}
		6974
		6975	static void gen3_init_clock_gating(struct drm_device *dev)
		6976	{
		6977	struct drm_i915_private *dev_priv = dev->dev_private;
		6978	u32 dstate = I915_READ(D_STATE);
		6979
		6980	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		6981	DSTATE_DOT_CLOCK_GATING;
		6982	I915_WRITE(D_STATE, dstate);
		6983
		6984	if (IS_PINEVIEW(dev))
		6985	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		6986
		6987	/* IIR "flip pending" means done if this bit is set */
		6988	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	6989
		6990	/* interrupts should cause a wake up from C3 */
		6991	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		6992
		6993	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		6994	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5354	serge	6995
		6996	I915_WRITE(MI_ARB_STATE,
		6997	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	6998	}
		6999
		7000	static void i85x_init_clock_gating(struct drm_device *dev)
		7001	{
		7002	struct drm_i915_private *dev_priv = dev->dev_private;
		7003
		7004	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	7005
		7006	/* interrupts should cause a wake up from C3 */
		7007	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		7008	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
5354	serge	7009
		7010	I915_WRITE(MEM_MODE,
		7011	_MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7012	}
		7013
		7014	static void i830_init_clock_gating(struct drm_device *dev)
		7015	{
		7016	struct drm_i915_private *dev_priv = dev->dev_private;
		7017
		7018	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5354	serge	7019
		7020	I915_WRITE(MEM_MODE,
		7021	_MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) \|
		7022	_MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
3031	serge	7023	}
		7024
		7025	void intel_init_clock_gating(struct drm_device *dev)
		7026	{
		7027	struct drm_i915_private *dev_priv = dev->dev_private;
		7028
6084	serge	7029	if (dev_priv->display.init_clock_gating)
		7030	dev_priv->display.init_clock_gating(dev);
3031	serge	7031	}
		7032
4104	Serge	7033	void intel_suspend_hw(struct drm_device *dev)
		7034	{
		7035	if (HAS_PCH_LPT(dev))
		7036	lpt_suspend_hw(dev);
		7037	}
		7038
5354	serge	7039	/* Set up chip specific power management-related functions */
		7040	void intel_init_pm(struct drm_device *dev)
		7041	{
		7042	struct drm_i915_private *dev_priv = dev->dev_private;
		7043
6084	serge	7044	intel_fbc_init(dev_priv);
5354	serge	7045
3031	serge	7046	/* For cxsr */
		7047	if (IS_PINEVIEW(dev))
		7048	i915_pineview_get_mem_freq(dev);
		7049	else if (IS_GEN5(dev))
		7050	i915_ironlake_get_mem_freq(dev);
		7051
		7052	/* For FIFO watermark updates */
5354	serge	7053	if (INTEL_INFO(dev)->gen >= 9) {
		7054	skl_setup_wm_latency(dev);
		7055
6084	serge	7056	if (IS_BROXTON(dev))
		7057	dev_priv->display.init_clock_gating =
		7058	bxt_init_clock_gating;
5354	serge	7059	dev_priv->display.update_wm = skl_update_wm;
		7060	dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
		7061	} else if (HAS_PCH_SPLIT(dev)) {
5060	serge	7062	ilk_setup_wm_latency(dev);
4104	Serge	7063
4560	Serge	7064	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		7065	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		7066	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		7067	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		7068	dev_priv->display.update_wm = ilk_update_wm;
		7069	dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6084	serge	7070	} else {
		7071	DRM_DEBUG_KMS("Failed to read display plane latency. "
		7072	"Disable CxSR\n");
		7073	}
4560	Serge	7074
		7075	if (IS_GEN5(dev))
		7076	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		7077	else if (IS_GEN6(dev))
3031	serge	7078	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	7079	else if (IS_IVYBRIDGE(dev))
3031	serge	7080	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	7081	else if (IS_HASWELL(dev))
3031	serge	7082	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	7083	else if (INTEL_INFO(dev)->gen == 8)
5354	serge	7084	dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
5060	serge	7085	} else if (IS_CHERRYVIEW(dev)) {
6084	serge	7086	vlv_setup_wm_latency(dev);
		7087
		7088	dev_priv->display.update_wm = vlv_update_wm;
5060	serge	7089	dev_priv->display.init_clock_gating =
		7090	cherryview_init_clock_gating;
3031	serge	7091	} else if (IS_VALLEYVIEW(dev)) {
6084	serge	7092	vlv_setup_wm_latency(dev);
		7093
		7094	dev_priv->display.update_wm = vlv_update_wm;
3031	serge	7095	dev_priv->display.init_clock_gating =
		7096	valleyview_init_clock_gating;
		7097	} else if (IS_PINEVIEW(dev)) {
		7098	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		7099	dev_priv->is_ddr3,
		7100	dev_priv->fsb_freq,
		7101	dev_priv->mem_freq)) {
		7102	DRM_INFO("failed to find known CxSR latency "
		7103	"(found ddr%s fsb freq %d, mem freq %d), "
		7104	"disabling CxSR\n",
		7105	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		7106	dev_priv->fsb_freq, dev_priv->mem_freq);
		7107	/* Disable CxSR and never update its watermark again */
5060	serge	7108	intel_set_memory_cxsr(dev_priv, false);
3031	serge	7109	dev_priv->display.update_wm = NULL;
		7110	} else
		7111	dev_priv->display.update_wm = pineview_update_wm;
		7112	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		7113	} else if (IS_G4X(dev)) {
		7114	dev_priv->display.update_wm = g4x_update_wm;
		7115	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		7116	} else if (IS_GEN4(dev)) {
		7117	dev_priv->display.update_wm = i965_update_wm;
		7118	if (IS_CRESTLINE(dev))
		7119	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		7120	else if (IS_BROADWATER(dev))
		7121	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		7122	} else if (IS_GEN3(dev)) {
		7123	dev_priv->display.update_wm = i9xx_update_wm;
		7124	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		7125	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	7126	} else if (IS_GEN2(dev)) {
		7127	if (INTEL_INFO(dev)->num_pipes == 1) {
		7128	dev_priv->display.update_wm = i845_update_wm;
		7129	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		7130	} else {
		7131	dev_priv->display.update_wm = i9xx_update_wm;
6084	serge	7132	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	7133	}
		7134
		7135	if (IS_I85X(dev) \|\| IS_I865G(dev))
6084	serge	7136	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	7137	else
		7138	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	7139	} else {
4560	Serge	7140	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	7141	}
		7142	}
		7143
5354	serge	7144	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u32 mbox, u32 val)
3243	Serge	7145	{
		7146	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	7147
3243	Serge	7148	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7149	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		7150	return -EAGAIN;
		7151	}
3031	serge	7152
3243	Serge	7153	I915_WRITE(GEN6_PCODE_DATA, *val);
5354	serge	7154	I915_WRITE(GEN6_PCODE_DATA1, 0);
3243	Serge	7155	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7156
		7157	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7158	500)) {
		7159	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		7160	return -ETIMEDOUT;
6084	serge	7161	}
3243	Serge	7162
		7163	*val = I915_READ(GEN6_PCODE_DATA);
		7164	I915_WRITE(GEN6_PCODE_DATA, 0);
		7165
		7166	return 0;
		7167	}
		7168
5354	serge	7169	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
3243	Serge	7170	{
		7171	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		7172
		7173	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7174	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		7175	return -EAGAIN;
6084	serge	7176	}
3243	Serge	7177
		7178	I915_WRITE(GEN6_PCODE_DATA, val);
		7179	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7180
		7181	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7182	500)) {
		7183	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		7184	return -ETIMEDOUT;
3031	serge	7185	}
3243	Serge	7186
		7187	I915_WRITE(GEN6_PCODE_DATA, 0);
		7188
		7189	return 0;
3031	serge	7190	}
3746	Serge	7191
5354	serge	7192	static int vlv_gpu_freq_div(unsigned int czclk_freq)
3746	Serge	7193	{
5354	serge	7194	switch (czclk_freq) {
		7195	case 200:
		7196	return 10;
		7197	case 267:
		7198	return 12;
		7199	case 320:
		7200	case 333:
		7201	return 16;
		7202	case 400:
		7203	return 20;
4104	Serge	7204	default:
		7205	return -1;
		7206	}
5354	serge	7207	}
3746	Serge	7208
5354	serge	7209	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7210	{
6084	serge	7211	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5354	serge	7212
		7213	div = vlv_gpu_freq_div(czclk_freq);
		7214	if (div < 0)
		7215	return div;
		7216
		7217	return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
4104	Serge	7218	}
3746	Serge	7219
5060	serge	7220	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	7221	{
6084	serge	7222	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
3746	Serge	7223
5354	serge	7224	mul = vlv_gpu_freq_div(czclk_freq);
		7225	if (mul < 0)
		7226	return mul;
3746	Serge	7227
5354	serge	7228	return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
3746	Serge	7229	}
		7230
5060	serge	7231	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7232	{
6084	serge	7233	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7234
5354	serge	7235	div = vlv_gpu_freq_div(czclk_freq) / 2;
		7236	if (div < 0)
		7237	return div;
5060	serge	7238
5354	serge	7239	return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
5060	serge	7240	}
		7241
		7242	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7243	{
6084	serge	7244	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7245
5354	serge	7246	mul = vlv_gpu_freq_div(czclk_freq) / 2;
		7247	if (mul < 0)
		7248	return mul;
5060	serge	7249
5354	serge	7250	/* CHV needs even values */
		7251	return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
5060	serge	7252	}
		7253
6084	serge	7254	int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
5060	serge	7255	{
6084	serge	7256	if (IS_GEN9(dev_priv->dev))
		7257	return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
		7258	GEN9_FREQ_SCALER);
		7259	else if (IS_CHERRYVIEW(dev_priv->dev))
		7260	return chv_gpu_freq(dev_priv, val);
		7261	else if (IS_VALLEYVIEW(dev_priv->dev))
		7262	return byt_gpu_freq(dev_priv, val);
		7263	else
		7264	return val * GT_FREQUENCY_MULTIPLIER;
		7265	}
5060	serge	7266
6084	serge	7267	int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7268	{
		7269	if (IS_GEN9(dev_priv->dev))
		7270	return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
		7271	GT_FREQUENCY_MULTIPLIER);
		7272	else if (IS_CHERRYVIEW(dev_priv->dev))
		7273	return chv_freq_opcode(dev_priv, val);
5060	serge	7274	else if (IS_VALLEYVIEW(dev_priv->dev))
6084	serge	7275	return byt_freq_opcode(dev_priv, val);
		7276	else
		7277	return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
		7278	}
5060	serge	7279
6084	serge	7280	struct request_boost {
		7281	struct work_struct work;
		7282	struct drm_i915_gem_request *req;
		7283	};
		7284
		7285	static void __intel_rps_boost_work(struct work_struct *work)
		7286	{
		7287	struct request_boost *boost = container_of(work, struct request_boost, work);
		7288	struct drm_i915_gem_request *req = boost->req;
		7289
		7290	if (!i915_gem_request_completed(req, true))
		7291	gen6_rps_boost(to_i915(req->ring->dev), NULL,
		7292	req->emitted_jiffies);
		7293
		7294	i915_gem_request_unreference__unlocked(req);
		7295	kfree(boost);
5060	serge	7296	}
		7297
6084	serge	7298	void intel_queue_rps_boost_for_request(struct drm_device *dev,
		7299	struct drm_i915_gem_request *req)
5060	serge	7300	{
6084	serge	7301	struct request_boost *boost;
5060	serge	7302
6084	serge	7303	if (req == NULL \|\| INTEL_INFO(dev)->gen < 6)
		7304	return;
5060	serge	7305
6084	serge	7306	if (i915_gem_request_completed(req, true))
		7307	return;
		7308
		7309	boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
		7310	if (boost == NULL)
		7311	return;
		7312
		7313	i915_gem_request_reference(req);
		7314	boost->req = req;
		7315
		7316	INIT_WORK(&boost->work, __intel_rps_boost_work);
		7317	queue_work(to_i915(dev)->wq, &boost->work);
5060	serge	7318	}
		7319
4560	Serge	7320	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7321	{
4104	Serge	7322	struct drm_i915_private *dev_priv = dev->dev_private;
		7323
4560	Serge	7324	mutex_init(&dev_priv->rps.hw_lock);
6084	serge	7325	spin_lock_init(&dev_priv->rps.client_lock);
4560	Serge	7326
4104	Serge	7327	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7328	intel_gen6_powersave_work);
6084	serge	7329	INIT_LIST_HEAD(&dev_priv->rps.clients);
		7330	INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
		7331	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
5060	serge	7332
		7333	dev_priv->pm.suspended = false;
3746	Serge	7334	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 6084